Home | History | Annotate | Line # | Download | only in pci
if_vioif.c revision 1.112
      1  1.112  riastrad /*	$NetBSD: if_vioif.c,v 1.112 2024/06/29 12:11:12 riastradh Exp $	*/
      2    1.1   hannken 
      3    1.1   hannken /*
      4   1.66   reinoud  * Copyright (c) 2020 The NetBSD Foundation, Inc.
      5    1.1   hannken  * Copyright (c) 2010 Minoura Makoto.
      6    1.1   hannken  * All rights reserved.
      7    1.1   hannken  *
      8    1.1   hannken  * Redistribution and use in source and binary forms, with or without
      9    1.1   hannken  * modification, are permitted provided that the following conditions
     10    1.1   hannken  * are met:
     11    1.1   hannken  * 1. Redistributions of source code must retain the above copyright
     12    1.1   hannken  *    notice, this list of conditions and the following disclaimer.
     13    1.1   hannken  * 2. Redistributions in binary form must reproduce the above copyright
     14    1.1   hannken  *    notice, this list of conditions and the following disclaimer in the
     15    1.1   hannken  *    documentation and/or other materials provided with the distribution.
     16    1.1   hannken  *
     17    1.1   hannken  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
     18    1.1   hannken  * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
     19    1.1   hannken  * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
     20    1.1   hannken  * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
     21    1.1   hannken  * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
     22    1.1   hannken  * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
     23    1.1   hannken  * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
     24    1.1   hannken  * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
     25    1.1   hannken  * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
     26    1.1   hannken  * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
     27    1.1   hannken  */
     28    1.1   hannken 
     29    1.1   hannken #include <sys/cdefs.h>
     30  1.112  riastrad __KERNEL_RCSID(0, "$NetBSD: if_vioif.c,v 1.112 2024/06/29 12:11:12 riastradh Exp $");
     31   1.15     ozaki 
     32   1.15     ozaki #ifdef _KERNEL_OPT
     33   1.15     ozaki #include "opt_net_mpsafe.h"
     34   1.15     ozaki #endif
     35    1.1   hannken 
     36    1.1   hannken #include <sys/param.h>
     37    1.1   hannken #include <sys/systm.h>
     38    1.1   hannken #include <sys/kernel.h>
     39   1.55  yamaguch #include <sys/atomic.h>
     40    1.1   hannken #include <sys/bus.h>
     41    1.1   hannken #include <sys/condvar.h>
     42    1.1   hannken #include <sys/device.h>
     43   1.63  yamaguch #include <sys/evcnt.h>
     44    1.1   hannken #include <sys/intr.h>
     45    1.1   hannken #include <sys/kmem.h>
     46    1.1   hannken #include <sys/mbuf.h>
     47    1.1   hannken #include <sys/mutex.h>
     48    1.1   hannken #include <sys/sockio.h>
     49   1.71  yamaguch #include <sys/syslog.h>
     50   1.12     ozaki #include <sys/cpu.h>
     51   1.26  pgoyette #include <sys/module.h>
     52   1.46  yamaguch #include <sys/pcq.h>
     53   1.55  yamaguch #include <sys/workqueue.h>
     54   1.85  yamaguch #include <sys/xcall.h>
     55    1.1   hannken 
     56    1.1   hannken #include <dev/pci/virtioreg.h>
     57    1.1   hannken #include <dev/pci/virtiovar.h>
     58    1.1   hannken 
     59    1.1   hannken #include <net/if.h>
     60   1.74  yamaguch #include <net/if_dl.h>
     61    1.1   hannken #include <net/if_media.h>
     62    1.1   hannken #include <net/if_ether.h>
     63    1.1   hannken 
     64    1.1   hannken #include <net/bpf.h>
     65    1.1   hannken 
     66   1.26  pgoyette #include "ioconf.h"
     67    1.1   hannken 
     68    1.7     ozaki #ifdef NET_MPSAFE
     69    1.7     ozaki #define VIOIF_MPSAFE	1
     70   1.46  yamaguch #define VIOIF_MULTIQ	1
     71    1.7     ozaki #endif
     72    1.7     ozaki 
     73    1.1   hannken /*
     74    1.1   hannken  * if_vioifreg.h:
     75    1.1   hannken  */
     76    1.1   hannken /* Configuration registers */
     77   1.66   reinoud #define VIRTIO_NET_CONFIG_MAC		 0 /* 8bit x 6byte */
     78   1.66   reinoud #define VIRTIO_NET_CONFIG_STATUS	 6 /* 16bit */
     79   1.66   reinoud #define VIRTIO_NET_CONFIG_MAX_VQ_PAIRS	 8 /* 16bit */
     80   1.66   reinoud #define VIRTIO_NET_CONFIG_MTU		10 /* 16bit */
     81    1.1   hannken 
     82    1.1   hannken /* Feature bits */
     83   1.46  yamaguch #define VIRTIO_NET_F_CSUM		__BIT(0)
     84   1.46  yamaguch #define VIRTIO_NET_F_GUEST_CSUM		__BIT(1)
     85   1.46  yamaguch #define VIRTIO_NET_F_MAC		__BIT(5)
     86   1.46  yamaguch #define VIRTIO_NET_F_GSO		__BIT(6)
     87   1.46  yamaguch #define VIRTIO_NET_F_GUEST_TSO4		__BIT(7)
     88   1.46  yamaguch #define VIRTIO_NET_F_GUEST_TSO6		__BIT(8)
     89   1.46  yamaguch #define VIRTIO_NET_F_GUEST_ECN		__BIT(9)
     90   1.46  yamaguch #define VIRTIO_NET_F_GUEST_UFO		__BIT(10)
     91   1.46  yamaguch #define VIRTIO_NET_F_HOST_TSO4		__BIT(11)
     92   1.46  yamaguch #define VIRTIO_NET_F_HOST_TSO6		__BIT(12)
     93   1.46  yamaguch #define VIRTIO_NET_F_HOST_ECN		__BIT(13)
     94   1.46  yamaguch #define VIRTIO_NET_F_HOST_UFO		__BIT(14)
     95   1.46  yamaguch #define VIRTIO_NET_F_MRG_RXBUF		__BIT(15)
     96   1.46  yamaguch #define VIRTIO_NET_F_STATUS		__BIT(16)
     97   1.46  yamaguch #define VIRTIO_NET_F_CTRL_VQ		__BIT(17)
     98   1.46  yamaguch #define VIRTIO_NET_F_CTRL_RX		__BIT(18)
     99   1.46  yamaguch #define VIRTIO_NET_F_CTRL_VLAN		__BIT(19)
    100   1.46  yamaguch #define VIRTIO_NET_F_CTRL_RX_EXTRA	__BIT(20)
    101   1.46  yamaguch #define VIRTIO_NET_F_GUEST_ANNOUNCE	__BIT(21)
    102   1.46  yamaguch #define VIRTIO_NET_F_MQ			__BIT(22)
    103   1.78  yamaguch #define VIRTIO_NET_F_CTRL_MAC_ADDR 	__BIT(23)
    104    1.1   hannken 
    105   1.79       uwe #define VIRTIO_NET_FLAG_BITS			\
    106   1.79       uwe 	VIRTIO_COMMON_FLAG_BITS			\
    107   1.79       uwe 	"b\x17" "CTRL_MAC\0"			\
    108   1.79       uwe 	"b\x16" "MQ\0"				\
    109   1.79       uwe 	"b\x15" "GUEST_ANNOUNCE\0"		\
    110   1.79       uwe 	"b\x14" "CTRL_RX_EXTRA\0"		\
    111   1.79       uwe 	"b\x13" "CTRL_VLAN\0"			\
    112   1.79       uwe 	"b\x12" "CTRL_RX\0"			\
    113   1.79       uwe 	"b\x11" "CTRL_VQ\0"			\
    114   1.79       uwe 	"b\x10" "STATUS\0"			\
    115   1.79       uwe 	"b\x0f" "MRG_RXBUF\0"			\
    116   1.79       uwe 	"b\x0e" "HOST_UFO\0"			\
    117   1.79       uwe 	"b\x0d" "HOST_ECN\0"			\
    118   1.79       uwe 	"b\x0c" "HOST_TSO6\0"			\
    119   1.79       uwe 	"b\x0b" "HOST_TSO4\0"			\
    120   1.79       uwe 	"b\x0a" "GUEST_UFO\0"			\
    121   1.79       uwe 	"b\x09" "GUEST_ECN\0"			\
    122   1.79       uwe 	"b\x08" "GUEST_TSO6\0"			\
    123   1.79       uwe 	"b\x07" "GUEST_TSO4\0"			\
    124   1.79       uwe 	"b\x06" "GSO\0"				\
    125   1.79       uwe 	"b\x05" "MAC\0"				\
    126   1.79       uwe 	"b\x01" "GUEST_CSUM\0"			\
    127   1.79       uwe 	"b\x00" "CSUM\0"
    128   1.18  christos 
    129    1.1   hannken /* Status */
    130    1.1   hannken #define VIRTIO_NET_S_LINK_UP	1
    131    1.1   hannken 
    132    1.1   hannken /* Packet header structure */
    133    1.1   hannken struct virtio_net_hdr {
    134    1.1   hannken 	uint8_t		flags;
    135    1.1   hannken 	uint8_t		gso_type;
    136    1.1   hannken 	uint16_t	hdr_len;
    137    1.1   hannken 	uint16_t	gso_size;
    138    1.1   hannken 	uint16_t	csum_start;
    139    1.1   hannken 	uint16_t	csum_offset;
    140   1.66   reinoud 
    141   1.66   reinoud 	uint16_t	num_buffers; /* VIRTIO_NET_F_MRG_RXBUF enabled or v1 */
    142    1.1   hannken } __packed;
    143    1.1   hannken 
    144    1.1   hannken #define VIRTIO_NET_HDR_F_NEEDS_CSUM	1 /* flags */
    145    1.1   hannken #define VIRTIO_NET_HDR_GSO_NONE		0 /* gso_type */
    146    1.1   hannken #define VIRTIO_NET_HDR_GSO_TCPV4	1 /* gso_type */
    147    1.1   hannken #define VIRTIO_NET_HDR_GSO_UDP		3 /* gso_type */
    148    1.1   hannken #define VIRTIO_NET_HDR_GSO_TCPV6	4 /* gso_type */
    149    1.1   hannken #define VIRTIO_NET_HDR_GSO_ECN		0x80 /* gso_type, |'ed */
    150    1.1   hannken 
    151    1.1   hannken #define VIRTIO_NET_MAX_GSO_LEN		(65536+ETHER_HDR_LEN)
    152    1.1   hannken 
    153    1.1   hannken /* Control virtqueue */
    154    1.1   hannken struct virtio_net_ctrl_cmd {
    155    1.1   hannken 	uint8_t	class;
    156    1.1   hannken 	uint8_t	command;
    157    1.1   hannken } __packed;
    158    1.1   hannken #define VIRTIO_NET_CTRL_RX		0
    159    1.1   hannken # define VIRTIO_NET_CTRL_RX_PROMISC	0
    160    1.1   hannken # define VIRTIO_NET_CTRL_RX_ALLMULTI	1
    161    1.1   hannken 
    162    1.1   hannken #define VIRTIO_NET_CTRL_MAC		1
    163    1.1   hannken # define VIRTIO_NET_CTRL_MAC_TABLE_SET	0
    164   1.74  yamaguch # define  VIRTIO_NET_CTRL_MAC_ADDR_SET	1
    165    1.1   hannken 
    166    1.1   hannken #define VIRTIO_NET_CTRL_VLAN		2
    167    1.1   hannken # define VIRTIO_NET_CTRL_VLAN_ADD	0
    168    1.1   hannken # define VIRTIO_NET_CTRL_VLAN_DEL	1
    169    1.1   hannken 
    170   1.46  yamaguch #define VIRTIO_NET_CTRL_MQ			4
    171   1.46  yamaguch # define VIRTIO_NET_CTRL_MQ_VQ_PAIRS_SET	0
    172   1.46  yamaguch # define VIRTIO_NET_CTRL_MQ_VQ_PAIRS_MIN	1
    173   1.46  yamaguch # define VIRTIO_NET_CTRL_MQ_VQ_PAIRS_MAX	0x8000
    174   1.46  yamaguch 
    175    1.1   hannken struct virtio_net_ctrl_status {
    176    1.1   hannken 	uint8_t	ack;
    177    1.1   hannken } __packed;
    178    1.1   hannken #define VIRTIO_NET_OK			0
    179    1.1   hannken #define VIRTIO_NET_ERR			1
    180    1.1   hannken 
    181    1.1   hannken struct virtio_net_ctrl_rx {
    182    1.1   hannken 	uint8_t	onoff;
    183    1.1   hannken } __packed;
    184    1.1   hannken 
    185    1.1   hannken struct virtio_net_ctrl_mac_tbl {
    186    1.1   hannken 	uint32_t nentries;
    187    1.1   hannken 	uint8_t macs[][ETHER_ADDR_LEN];
    188    1.1   hannken } __packed;
    189    1.1   hannken 
    190   1.74  yamaguch struct virtio_net_ctrl_mac_addr {
    191   1.74  yamaguch 	uint8_t mac[ETHER_ADDR_LEN];
    192   1.74  yamaguch } __packed;
    193   1.74  yamaguch 
    194    1.1   hannken struct virtio_net_ctrl_vlan {
    195    1.1   hannken 	uint16_t id;
    196    1.1   hannken } __packed;
    197    1.1   hannken 
    198   1.46  yamaguch struct virtio_net_ctrl_mq {
    199   1.46  yamaguch 	uint16_t virtqueue_pairs;
    200   1.46  yamaguch } __packed;
    201   1.46  yamaguch 
    202    1.1   hannken /*
    203    1.1   hannken  * if_vioifvar.h:
    204    1.1   hannken  */
    205   1.43  yamaguch 
    206   1.43  yamaguch /*
    207   1.43  yamaguch  * Locking notes:
    208  1.109    andvar  * + a field in vioif_netqueue is protected by netq_lock (a spin mutex)
    209   1.43  yamaguch  *      - more than one lock cannot be held at onece
    210   1.98  yamaguch  * + a field in vioif_tx_context and vioif_rx_context is also protected
    211   1.98  yamaguch  *   by netq_lock.
    212   1.43  yamaguch  * + ctrlq_inuse is protected by ctrlq_wait_lock.
    213   1.43  yamaguch  *      - other fields in vioif_ctrlqueue are protected by ctrlq_inuse
    214   1.98  yamaguch  *      - netq_lock cannot be held along with ctrlq_wait_lock
    215   1.62  yamaguch  * + fields in vioif_softc except queues are protected by
    216   1.62  yamaguch  *   sc->sc_lock(an adaptive mutex)
    217   1.62  yamaguch  *      - the lock is held before acquisition of other locks
    218   1.43  yamaguch  */
    219   1.43  yamaguch 
    220   1.66   reinoud struct vioif_ctrl_cmdspec {
    221   1.66   reinoud 	bus_dmamap_t	dmamap;
    222   1.66   reinoud 	void		*buf;
    223   1.66   reinoud 	bus_size_t	bufsize;
    224   1.66   reinoud };
    225   1.66   reinoud 
    226   1.55  yamaguch struct vioif_work {
    227   1.55  yamaguch 	struct work	 cookie;
    228   1.55  yamaguch 	void		(*func)(void *);
    229   1.55  yamaguch 	void		*arg;
    230   1.55  yamaguch 	unsigned int	 added;
    231   1.55  yamaguch };
    232   1.55  yamaguch 
    233   1.95  yamaguch struct vioif_net_map {
    234   1.95  yamaguch 	struct virtio_net_hdr	*vnm_hdr;
    235   1.95  yamaguch 	bus_dmamap_t		 vnm_hdr_map;
    236   1.95  yamaguch 	struct mbuf		*vnm_mbuf;
    237   1.95  yamaguch 	bus_dmamap_t		 vnm_mbuf_map;
    238   1.95  yamaguch };
    239   1.95  yamaguch 
    240   1.98  yamaguch #define VIOIF_NETQ_RX		0
    241   1.98  yamaguch #define VIOIF_NETQ_TX		1
    242   1.98  yamaguch #define VIOIF_NETQ_IDX		2
    243   1.98  yamaguch #define VIOIF_NETQ_DIR(n)	((n) % VIOIF_NETQ_IDX)
    244   1.98  yamaguch #define VIOIF_NETQ_PAIRIDX(n)	((n) / VIOIF_NETQ_IDX)
    245   1.98  yamaguch #define VIOIF_NETQ_RXQID(n)	((n) * VIOIF_NETQ_IDX + VIOIF_NETQ_RX)
    246   1.98  yamaguch #define VIOIF_NETQ_TXQID(n)	((n) * VIOIF_NETQ_IDX + VIOIF_NETQ_TX)
    247   1.98  yamaguch 
    248   1.98  yamaguch struct vioif_netqueue {
    249   1.98  yamaguch 	kmutex_t		 netq_lock;
    250   1.98  yamaguch 	struct virtqueue	*netq_vq;
    251   1.98  yamaguch 	bool			 netq_stopping;
    252   1.98  yamaguch 	bool			 netq_running_handle;
    253   1.98  yamaguch 	void			*netq_maps_kva;
    254   1.98  yamaguch 	struct vioif_net_map	*netq_maps;
    255   1.98  yamaguch 
    256   1.98  yamaguch 	void			*netq_softint;
    257   1.98  yamaguch 	struct vioif_work	 netq_work;
    258   1.98  yamaguch 	bool			 netq_workqueue;
    259   1.98  yamaguch 
    260   1.98  yamaguch 	char			 netq_evgroup[32];
    261   1.98  yamaguch 	struct evcnt		 netq_mbuf_load_failed;
    262   1.99  yamaguch 	struct evcnt		 netq_enqueue_failed;
    263   1.43  yamaguch 
    264   1.98  yamaguch 	void			*netq_ctx;
    265   1.43  yamaguch };
    266   1.43  yamaguch 
    267   1.98  yamaguch struct vioif_tx_context {
    268   1.98  yamaguch 	bool			 txc_link_active;
    269  1.102  yamaguch 	bool			 txc_no_free_slots;
    270   1.98  yamaguch 	pcq_t			*txc_intrq;
    271   1.98  yamaguch 	void			*txc_deferred_transmit;
    272   1.43  yamaguch 
    273   1.98  yamaguch 	struct evcnt		 txc_defrag_failed;
    274   1.98  yamaguch };
    275   1.43  yamaguch 
    276   1.98  yamaguch struct vioif_rx_context {
    277   1.98  yamaguch 	struct evcnt		 rxc_mbuf_enobufs;
    278   1.43  yamaguch };
    279   1.43  yamaguch struct vioif_ctrlqueue {
    280   1.43  yamaguch 	struct virtqueue		*ctrlq_vq;
    281   1.43  yamaguch 	enum {
    282   1.43  yamaguch 		FREE, INUSE, DONE
    283   1.43  yamaguch 	}				ctrlq_inuse;
    284   1.43  yamaguch 	kcondvar_t			ctrlq_wait;
    285   1.43  yamaguch 	kmutex_t			ctrlq_wait_lock;
    286   1.44  yamaguch 	struct lwp			*ctrlq_owner;
    287   1.43  yamaguch 
    288   1.43  yamaguch 	struct virtio_net_ctrl_cmd	*ctrlq_cmd;
    289   1.43  yamaguch 	struct virtio_net_ctrl_status	*ctrlq_status;
    290   1.43  yamaguch 	struct virtio_net_ctrl_rx	*ctrlq_rx;
    291   1.43  yamaguch 	struct virtio_net_ctrl_mac_tbl	*ctrlq_mac_tbl_uc;
    292   1.43  yamaguch 	struct virtio_net_ctrl_mac_tbl	*ctrlq_mac_tbl_mc;
    293   1.74  yamaguch 	struct virtio_net_ctrl_mac_addr	*ctrlq_mac_addr;
    294   1.46  yamaguch 	struct virtio_net_ctrl_mq	*ctrlq_mq;
    295   1.43  yamaguch 
    296   1.43  yamaguch 	bus_dmamap_t			ctrlq_cmd_dmamap;
    297   1.43  yamaguch 	bus_dmamap_t			ctrlq_status_dmamap;
    298   1.43  yamaguch 	bus_dmamap_t			ctrlq_rx_dmamap;
    299   1.43  yamaguch 	bus_dmamap_t			ctrlq_tbl_uc_dmamap;
    300   1.43  yamaguch 	bus_dmamap_t			ctrlq_tbl_mc_dmamap;
    301   1.74  yamaguch 	bus_dmamap_t			ctrlq_mac_addr_dmamap;
    302   1.46  yamaguch 	bus_dmamap_t			ctrlq_mq_dmamap;
    303   1.63  yamaguch 
    304   1.63  yamaguch 	struct evcnt			ctrlq_cmd_load_failed;
    305   1.63  yamaguch 	struct evcnt			ctrlq_cmd_failed;
    306   1.43  yamaguch };
    307   1.43  yamaguch 
    308    1.1   hannken struct vioif_softc {
    309    1.1   hannken 	device_t		sc_dev;
    310   1.62  yamaguch 	kmutex_t		sc_lock;
    311   1.55  yamaguch 	struct sysctllog	*sc_sysctllog;
    312    1.1   hannken 
    313    1.1   hannken 	struct virtio_softc	*sc_virtio;
    314   1.46  yamaguch 	struct virtqueue	*sc_vqs;
    315   1.66   reinoud 	u_int			 sc_hdr_size;
    316   1.46  yamaguch 
    317   1.46  yamaguch 	int			sc_max_nvq_pairs;
    318   1.46  yamaguch 	int			sc_req_nvq_pairs;
    319   1.46  yamaguch 	int			sc_act_nvq_pairs;
    320    1.1   hannken 
    321    1.1   hannken 	uint8_t			sc_mac[ETHER_ADDR_LEN];
    322    1.1   hannken 	struct ethercom		sc_ethercom;
    323   1.82  knakahar 	int			sc_link_state;
    324    1.1   hannken 
    325   1.98  yamaguch 	struct vioif_netqueue	*sc_netqs;
    326   1.43  yamaguch 
    327   1.43  yamaguch 	bool			sc_has_ctrl;
    328   1.43  yamaguch 	struct vioif_ctrlqueue	sc_ctrlq;
    329   1.43  yamaguch 
    330  1.100  yamaguch 	bus_dma_segment_t	 sc_segs[1];
    331   1.43  yamaguch 	void			*sc_dmamem;
    332   1.43  yamaguch 	void			*sc_kmem;
    333    1.1   hannken 
    334  1.101  yamaguch 	void			*sc_cfg_softint;
    335   1.55  yamaguch 
    336   1.55  yamaguch 	struct workqueue	*sc_txrx_workqueue;
    337   1.55  yamaguch 	bool			 sc_txrx_workqueue_sysctl;
    338   1.55  yamaguch 	u_int			 sc_tx_intr_process_limit;
    339   1.55  yamaguch 	u_int			 sc_tx_process_limit;
    340   1.55  yamaguch 	u_int			 sc_rx_intr_process_limit;
    341   1.55  yamaguch 	u_int			 sc_rx_process_limit;
    342    1.1   hannken };
    343    1.1   hannken #define VIRTIO_NET_TX_MAXNSEGS		(16) /* XXX */
    344    1.1   hannken #define VIRTIO_NET_CTRL_MAC_MAXENTRIES	(64) /* XXX */
    345    1.1   hannken 
    346   1.55  yamaguch #define VIOIF_TX_INTR_PROCESS_LIMIT	256
    347   1.55  yamaguch #define VIOIF_TX_PROCESS_LIMIT		256
    348   1.55  yamaguch #define VIOIF_RX_INTR_PROCESS_LIMIT	0U
    349   1.55  yamaguch #define VIOIF_RX_PROCESS_LIMIT		256
    350   1.55  yamaguch 
    351   1.55  yamaguch #define VIOIF_WORKQUEUE_PRI		PRI_SOFTNET
    352   1.82  knakahar #define VIOIF_IS_LINK_ACTIVE(_sc)	((_sc)->sc_link_state == LINK_STATE_UP ? \
    353   1.82  knakahar 					    true : false)
    354   1.55  yamaguch 
    355    1.1   hannken /* cfattach interface functions */
    356    1.1   hannken static int	vioif_match(device_t, cfdata_t, void *);
    357    1.1   hannken static void	vioif_attach(device_t, device_t, void *);
    358   1.55  yamaguch static int	vioif_finalize_teardown(device_t);
    359    1.1   hannken 
    360    1.1   hannken /* ifnet interface functions */
    361    1.1   hannken static int	vioif_init(struct ifnet *);
    362    1.1   hannken static void	vioif_stop(struct ifnet *, int);
    363    1.1   hannken static void	vioif_start(struct ifnet *);
    364   1.46  yamaguch static int	vioif_transmit(struct ifnet *, struct mbuf *);
    365    1.1   hannken static int	vioif_ioctl(struct ifnet *, u_long, void *);
    366    1.1   hannken static void	vioif_watchdog(struct ifnet *);
    367  1.101  yamaguch static int	vioif_ifflags(struct vioif_softc *);
    368   1.75  yamaguch static int	vioif_ifflags_cb(struct ethercom *);
    369    1.1   hannken 
    370   1.98  yamaguch /* tx & rx */
    371  1.101  yamaguch static int	vioif_netqueue_init(struct vioif_softc *,
    372  1.101  yamaguch 		    struct virtio_softc *, size_t, u_int);
    373  1.101  yamaguch static void	vioif_netqueue_teardown(struct vioif_softc *,
    374  1.101  yamaguch 		    struct virtio_softc *, size_t);
    375   1.99  yamaguch static void	vioif_net_intr_enable(struct vioif_softc *,
    376   1.99  yamaguch 		    struct virtio_softc *);
    377   1.99  yamaguch static void	vioif_net_intr_disable(struct vioif_softc *,
    378   1.99  yamaguch 		    struct virtio_softc *);
    379  1.101  yamaguch static void	vioif_net_sched_handle(struct vioif_softc *,
    380  1.101  yamaguch 		    struct vioif_netqueue *);
    381   1.98  yamaguch 
    382    1.1   hannken /* rx */
    383   1.66   reinoud static void	vioif_populate_rx_mbufs_locked(struct vioif_softc *,
    384   1.98  yamaguch 		    struct vioif_netqueue *);
    385   1.54  yamaguch static int	vioif_rx_intr(void *);
    386   1.55  yamaguch static void	vioif_rx_handle(void *);
    387  1.101  yamaguch static void	vioif_rx_queue_clear(struct vioif_softc *,
    388  1.101  yamaguch 		    struct virtio_softc *, struct vioif_netqueue *);
    389    1.1   hannken 
    390    1.1   hannken /* tx */
    391  1.101  yamaguch static void	vioif_start_locked(struct ifnet *, struct vioif_netqueue *);
    392  1.101  yamaguch static void	vioif_transmit_locked(struct ifnet *, struct vioif_netqueue *);
    393  1.101  yamaguch static void	vioif_deferred_transmit(void *);
    394   1.54  yamaguch static int	vioif_tx_intr(void *);
    395   1.55  yamaguch static void	vioif_tx_handle(void *);
    396   1.93  yamaguch static void	vioif_tx_queue_clear(struct vioif_softc *, struct virtio_softc *,
    397   1.98  yamaguch 		    struct vioif_netqueue *);
    398    1.1   hannken 
    399  1.101  yamaguch /* controls */
    400  1.101  yamaguch static int	vioif_ctrl_intr(void *);
    401    1.1   hannken static int	vioif_ctrl_rx(struct vioif_softc *, int, bool);
    402    1.1   hannken static int	vioif_set_promisc(struct vioif_softc *, bool);
    403    1.1   hannken static int	vioif_set_allmulti(struct vioif_softc *, bool);
    404    1.1   hannken static int	vioif_set_rx_filter(struct vioif_softc *);
    405    1.1   hannken static int	vioif_rx_filter(struct vioif_softc *);
    406   1.74  yamaguch static int	vioif_set_mac_addr(struct vioif_softc *);
    407  1.101  yamaguch static int	vioif_ctrl_mq_vq_pairs_set(struct vioif_softc *, int);
    408  1.101  yamaguch 
    409  1.101  yamaguch /* config interrupt */
    410   1.34     ozaki static int	vioif_config_change(struct virtio_softc *);
    411  1.101  yamaguch static void	vioif_cfg_softint(void *);
    412  1.101  yamaguch static void	vioif_update_link_status(struct vioif_softc *);
    413  1.101  yamaguch 
    414  1.101  yamaguch /* others */
    415  1.101  yamaguch static void	vioif_alloc_queues(struct vioif_softc *);
    416  1.101  yamaguch static void	vioif_free_queues(struct vioif_softc *);
    417  1.101  yamaguch static int	vioif_alloc_mems(struct vioif_softc *);
    418  1.101  yamaguch static struct workqueue*
    419  1.101  yamaguch 		vioif_workq_create(const char *, pri_t, int, int);
    420  1.101  yamaguch static void	vioif_workq_destroy(struct workqueue *);
    421  1.101  yamaguch static void	vioif_work_set(struct vioif_work *, void(*)(void *), void *);
    422  1.101  yamaguch static void	vioif_work_add(struct workqueue *, struct vioif_work *);
    423  1.101  yamaguch static void	vioif_work_wait(struct workqueue *, struct vioif_work *);
    424   1.55  yamaguch static int	vioif_setup_sysctl(struct vioif_softc *);
    425   1.63  yamaguch static void	vioif_setup_stats(struct vioif_softc *);
    426    1.1   hannken 
    427    1.1   hannken CFATTACH_DECL_NEW(vioif, sizeof(struct vioif_softc),
    428    1.1   hannken 		  vioif_match, vioif_attach, NULL, NULL);
    429    1.1   hannken 
    430  1.101  yamaguch static void
    431  1.101  yamaguch vioif_intr_barrier(void)
    432  1.101  yamaguch {
    433  1.101  yamaguch 
    434  1.101  yamaguch 	/* wait for finish all interrupt handler */
    435  1.101  yamaguch 	xc_barrier(0);
    436  1.101  yamaguch }
    437  1.101  yamaguch 
    438  1.101  yamaguch static void
    439  1.101  yamaguch vioif_notify(struct virtio_softc *vsc, struct virtqueue *vq)
    440  1.101  yamaguch {
    441  1.101  yamaguch 
    442  1.101  yamaguch 	virtio_enqueue_commit(vsc, vq, -1, true);
    443  1.101  yamaguch }
    444  1.101  yamaguch 
    445    1.1   hannken static int
    446    1.1   hannken vioif_match(device_t parent, cfdata_t match, void *aux)
    447    1.1   hannken {
    448   1.32  jdolecek 	struct virtio_attach_args *va = aux;
    449    1.1   hannken 
    450   1.66   reinoud 	if (va->sc_childdevid == VIRTIO_DEVICE_ID_NETWORK)
    451    1.1   hannken 		return 1;
    452    1.1   hannken 
    453    1.1   hannken 	return 0;
    454    1.1   hannken }
    455    1.1   hannken 
    456  1.101  yamaguch static void
    457  1.101  yamaguch vioif_attach(device_t parent, device_t self, void *aux)
    458   1.58  yamaguch {
    459  1.101  yamaguch 	struct vioif_softc *sc = device_private(self);
    460  1.101  yamaguch 	struct virtio_softc *vsc = device_private(parent);
    461  1.101  yamaguch 	struct vioif_netqueue *txq0;
    462  1.101  yamaguch 	struct vioif_ctrlqueue *ctrlq = &sc->sc_ctrlq;
    463  1.101  yamaguch 	uint64_t features, req_features;
    464  1.101  yamaguch 	struct ifnet *ifp = &sc->sc_ethercom.ec_if;
    465  1.101  yamaguch 	u_int softint_flags;
    466  1.101  yamaguch 	int r, i, req_flags;
    467  1.101  yamaguch 	char xnamebuf[MAXCOMLEN];
    468  1.103  yamaguch 	size_t nvqs;
    469  1.101  yamaguch 
    470  1.101  yamaguch 	if (virtio_child(vsc) != NULL) {
    471  1.101  yamaguch 		aprint_normal(": child already attached for %s; "
    472  1.101  yamaguch 		    "something wrong...\n", device_xname(parent));
    473  1.101  yamaguch 		return;
    474  1.101  yamaguch 	}
    475  1.101  yamaguch 
    476  1.101  yamaguch 	sc->sc_dev = self;
    477  1.101  yamaguch 	sc->sc_virtio = vsc;
    478  1.101  yamaguch 	sc->sc_link_state = LINK_STATE_UNKNOWN;
    479  1.101  yamaguch 
    480  1.101  yamaguch 	sc->sc_max_nvq_pairs = 1;
    481  1.101  yamaguch 	sc->sc_req_nvq_pairs = 1;
    482  1.101  yamaguch 	sc->sc_act_nvq_pairs = 1;
    483  1.101  yamaguch 	sc->sc_txrx_workqueue_sysctl = true;
    484  1.101  yamaguch 	sc->sc_tx_intr_process_limit = VIOIF_TX_INTR_PROCESS_LIMIT;
    485  1.101  yamaguch 	sc->sc_tx_process_limit = VIOIF_TX_PROCESS_LIMIT;
    486  1.101  yamaguch 	sc->sc_rx_intr_process_limit = VIOIF_RX_INTR_PROCESS_LIMIT;
    487  1.101  yamaguch 	sc->sc_rx_process_limit = VIOIF_RX_PROCESS_LIMIT;
    488  1.101  yamaguch 
    489  1.101  yamaguch 	mutex_init(&sc->sc_lock, MUTEX_DEFAULT, IPL_NONE);
    490  1.101  yamaguch 
    491  1.101  yamaguch 	snprintf(xnamebuf, sizeof(xnamebuf), "%s_txrx", device_xname(self));
    492  1.101  yamaguch 	sc->sc_txrx_workqueue = vioif_workq_create(xnamebuf, VIOIF_WORKQUEUE_PRI,
    493  1.101  yamaguch 	    IPL_NET, WQ_PERCPU | WQ_MPSAFE);
    494  1.101  yamaguch 	if (sc->sc_txrx_workqueue == NULL)
    495  1.101  yamaguch 		goto err;
    496   1.58  yamaguch 
    497  1.101  yamaguch 	req_flags = 0;
    498   1.58  yamaguch 
    499  1.101  yamaguch #ifdef VIOIF_MPSAFE
    500  1.101  yamaguch 	req_flags |= VIRTIO_F_INTR_MPSAFE;
    501  1.101  yamaguch #endif
    502  1.101  yamaguch 	req_flags |= VIRTIO_F_INTR_MSIX;
    503   1.58  yamaguch 
    504  1.101  yamaguch 	req_features =
    505  1.101  yamaguch 	    VIRTIO_NET_F_MAC | VIRTIO_NET_F_STATUS | VIRTIO_NET_F_CTRL_VQ |
    506  1.101  yamaguch 	    VIRTIO_NET_F_CTRL_RX | VIRTIO_F_NOTIFY_ON_EMPTY;
    507  1.101  yamaguch 	req_features |= VIRTIO_F_RING_EVENT_IDX;
    508  1.101  yamaguch 	req_features |= VIRTIO_NET_F_CTRL_MAC_ADDR;
    509  1.101  yamaguch #ifdef VIOIF_MULTIQ
    510  1.101  yamaguch 	req_features |= VIRTIO_NET_F_MQ;
    511  1.101  yamaguch #endif
    512  1.103  yamaguch 
    513  1.103  yamaguch 	virtio_child_attach_start(vsc, self, IPL_NET,
    514  1.101  yamaguch 	    req_features, VIRTIO_NET_FLAG_BITS);
    515  1.103  yamaguch 	features = virtio_features(vsc);
    516   1.58  yamaguch 
    517  1.101  yamaguch 	if (features == 0)
    518  1.101  yamaguch 		goto err;
    519   1.58  yamaguch 
    520  1.101  yamaguch 	if (features & VIRTIO_NET_F_MAC) {
    521  1.101  yamaguch 		for (i = 0; i < __arraycount(sc->sc_mac); i++) {
    522  1.101  yamaguch 			sc->sc_mac[i] = virtio_read_device_config_1(vsc,
    523  1.101  yamaguch 			    VIRTIO_NET_CONFIG_MAC + i);
    524  1.101  yamaguch 		}
    525  1.101  yamaguch 	} else {
    526  1.101  yamaguch 		/* code stolen from sys/net/if_tap.c */
    527  1.101  yamaguch 		struct timeval tv;
    528  1.101  yamaguch 		uint32_t ui;
    529  1.101  yamaguch 		getmicrouptime(&tv);
    530  1.101  yamaguch 		ui = (tv.tv_sec ^ tv.tv_usec) & 0xffffff;
    531  1.101  yamaguch 		memcpy(sc->sc_mac+3, (uint8_t *)&ui, 3);
    532  1.101  yamaguch 		for (i = 0; i < __arraycount(sc->sc_mac); i++) {
    533  1.101  yamaguch 			virtio_write_device_config_1(vsc,
    534  1.101  yamaguch 			    VIRTIO_NET_CONFIG_MAC + i, sc->sc_mac[i]);
    535  1.101  yamaguch 		}
    536   1.58  yamaguch 	}
    537   1.58  yamaguch 
    538  1.101  yamaguch 	/* 'Ethernet' with capital follows other ethernet driver attachment */
    539  1.101  yamaguch 	aprint_normal_dev(self, "Ethernet address %s\n",
    540  1.101  yamaguch 	    ether_sprintf(sc->sc_mac));
    541  1.101  yamaguch 
    542  1.101  yamaguch 	if (features & (VIRTIO_NET_F_MRG_RXBUF | VIRTIO_F_VERSION_1)) {
    543  1.101  yamaguch 		sc->sc_hdr_size = sizeof(struct virtio_net_hdr);
    544  1.101  yamaguch 	} else {
    545  1.101  yamaguch 		sc->sc_hdr_size = offsetof(struct virtio_net_hdr, num_buffers);
    546  1.101  yamaguch 	}
    547   1.58  yamaguch 
    548  1.101  yamaguch 	if ((features & VIRTIO_NET_F_CTRL_VQ) &&
    549  1.101  yamaguch 	    (features & VIRTIO_NET_F_CTRL_RX)) {
    550  1.101  yamaguch 		sc->sc_has_ctrl = true;
    551   1.58  yamaguch 
    552  1.101  yamaguch 		cv_init(&ctrlq->ctrlq_wait, "ctrl_vq");
    553  1.101  yamaguch 		mutex_init(&ctrlq->ctrlq_wait_lock, MUTEX_DEFAULT, IPL_NET);
    554  1.101  yamaguch 		ctrlq->ctrlq_inuse = FREE;
    555  1.101  yamaguch 	} else {
    556  1.101  yamaguch 		sc->sc_has_ctrl = false;
    557   1.58  yamaguch 	}
    558   1.58  yamaguch 
    559  1.101  yamaguch 	if (sc->sc_has_ctrl && (features & VIRTIO_NET_F_MQ)) {
    560  1.101  yamaguch 		sc->sc_max_nvq_pairs = virtio_read_device_config_2(vsc,
    561  1.101  yamaguch 		    VIRTIO_NET_CONFIG_MAX_VQ_PAIRS);
    562  1.101  yamaguch 
    563  1.101  yamaguch 		if (sc->sc_max_nvq_pairs > VIRTIO_NET_CTRL_MQ_VQ_PAIRS_MAX)
    564  1.101  yamaguch 			goto err;
    565   1.58  yamaguch 
    566  1.101  yamaguch 		/* Limit the number of queue pairs to use */
    567  1.101  yamaguch 		sc->sc_req_nvq_pairs = MIN(sc->sc_max_nvq_pairs, ncpu);
    568  1.103  yamaguch 
    569  1.103  yamaguch 		if (sc->sc_max_nvq_pairs > 1)
    570  1.103  yamaguch 			req_flags |= VIRTIO_F_INTR_PERVQ;
    571  1.101  yamaguch 	}
    572   1.58  yamaguch 
    573  1.101  yamaguch 	vioif_alloc_queues(sc);
    574   1.58  yamaguch 
    575  1.101  yamaguch #ifdef VIOIF_MPSAFE
    576  1.101  yamaguch 	softint_flags = SOFTINT_NET | SOFTINT_MPSAFE;
    577  1.101  yamaguch #else
    578  1.101  yamaguch 	softint_flags = SOFTINT_NET;
    579  1.101  yamaguch #endif
    580   1.58  yamaguch 
    581  1.101  yamaguch 	/*
    582  1.101  yamaguch 	 * Initialize network queues
    583  1.101  yamaguch 	 */
    584  1.103  yamaguch 	nvqs = sc->sc_max_nvq_pairs * 2;
    585  1.103  yamaguch 	for (i = 0; i < nvqs; i++) {
    586  1.101  yamaguch 		r = vioif_netqueue_init(sc, vsc, i, softint_flags);
    587  1.101  yamaguch 		if (r != 0)
    588  1.101  yamaguch 			goto err;
    589  1.101  yamaguch 	}
    590   1.46  yamaguch 
    591  1.101  yamaguch 	if (sc->sc_has_ctrl) {
    592  1.103  yamaguch 		int ctrlq_idx = nvqs;
    593  1.103  yamaguch 
    594  1.103  yamaguch 		nvqs++;
    595  1.101  yamaguch 		/*
    596  1.101  yamaguch 		 * Allocating a virtqueue for control channel
    597  1.101  yamaguch 		 */
    598  1.101  yamaguch 		sc->sc_ctrlq.ctrlq_vq = &sc->sc_vqs[ctrlq_idx];
    599  1.104  yamaguch 		virtio_init_vq(vsc, ctrlq->ctrlq_vq, ctrlq_idx,
    600  1.104  yamaguch 		    vioif_ctrl_intr, ctrlq);
    601  1.104  yamaguch 
    602  1.104  yamaguch 		r = virtio_alloc_vq(vsc, ctrlq->ctrlq_vq, NBPG, 1, "control");
    603  1.101  yamaguch 		if (r != 0) {
    604  1.101  yamaguch 			aprint_error_dev(self, "failed to allocate "
    605  1.101  yamaguch 			    "a virtqueue for control channel, error code %d\n",
    606  1.101  yamaguch 			    r);
    607   1.46  yamaguch 
    608  1.101  yamaguch 			sc->sc_has_ctrl = false;
    609  1.101  yamaguch 			cv_destroy(&ctrlq->ctrlq_wait);
    610  1.101  yamaguch 			mutex_destroy(&ctrlq->ctrlq_wait_lock);
    611  1.101  yamaguch 		}
    612  1.101  yamaguch 	}
    613   1.46  yamaguch 
    614  1.101  yamaguch 	sc->sc_cfg_softint = softint_establish(softint_flags,
    615  1.101  yamaguch 	    vioif_cfg_softint, sc);
    616  1.101  yamaguch 	if (sc->sc_cfg_softint == NULL) {
    617  1.101  yamaguch 		aprint_error_dev(self, "cannot establish ctl softint\n");
    618  1.101  yamaguch 		goto err;
    619  1.101  yamaguch 	}
    620   1.46  yamaguch 
    621  1.101  yamaguch 	if (vioif_alloc_mems(sc) < 0)
    622  1.101  yamaguch 		goto err;
    623   1.46  yamaguch 
    624  1.103  yamaguch 	r = virtio_child_attach_finish(vsc, sc->sc_vqs, nvqs,
    625  1.104  yamaguch 	    vioif_config_change, req_flags);
    626  1.103  yamaguch 	if (r != 0)
    627  1.101  yamaguch 		goto err;
    628   1.46  yamaguch 
    629  1.101  yamaguch 	if (vioif_setup_sysctl(sc) != 0) {
    630  1.101  yamaguch 		aprint_error_dev(self, "unable to create sysctl node\n");
    631  1.101  yamaguch 		/* continue */
    632  1.101  yamaguch 	}
    633   1.98  yamaguch 
    634  1.101  yamaguch 	vioif_setup_stats(sc);
    635   1.98  yamaguch 
    636  1.101  yamaguch 	strlcpy(ifp->if_xname, device_xname(self), IFNAMSIZ);
    637  1.101  yamaguch 	ifp->if_softc = sc;
    638  1.101  yamaguch 	ifp->if_flags = IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST;
    639  1.101  yamaguch #ifdef VIOIF_MPSAFE
    640  1.101  yamaguch 	ifp->if_extflags = IFEF_MPSAFE;
    641  1.101  yamaguch #endif
    642  1.101  yamaguch 	ifp->if_start = vioif_start;
    643  1.101  yamaguch 	if (sc->sc_req_nvq_pairs > 1)
    644  1.101  yamaguch 		ifp->if_transmit = vioif_transmit;
    645  1.101  yamaguch 	ifp->if_ioctl = vioif_ioctl;
    646  1.101  yamaguch 	ifp->if_init = vioif_init;
    647  1.101  yamaguch 	ifp->if_stop = vioif_stop;
    648  1.101  yamaguch 	ifp->if_capabilities = 0;
    649  1.101  yamaguch 	ifp->if_watchdog = vioif_watchdog;
    650  1.101  yamaguch 	txq0 = &sc->sc_netqs[VIOIF_NETQ_TXQID(0)];
    651  1.101  yamaguch 	IFQ_SET_MAXLEN(&ifp->if_snd, MAX(txq0->netq_vq->vq_num, IFQ_MAXLEN));
    652  1.101  yamaguch 	IFQ_SET_READY(&ifp->if_snd);
    653   1.98  yamaguch 
    654  1.101  yamaguch 	sc->sc_ethercom.ec_capabilities |= ETHERCAP_VLAN_MTU;
    655   1.98  yamaguch 
    656  1.101  yamaguch 	if_attach(ifp);
    657  1.101  yamaguch 	if_deferred_start_init(ifp, NULL);
    658  1.101  yamaguch 	ether_ifattach(ifp, sc->sc_mac);
    659  1.101  yamaguch 	ether_set_ifflags_cb(&sc->sc_ethercom, vioif_ifflags_cb);
    660   1.98  yamaguch 
    661  1.101  yamaguch 	return;
    662   1.98  yamaguch 
    663   1.98  yamaguch err:
    664  1.103  yamaguch 	nvqs = sc->sc_max_nvq_pairs * 2;
    665  1.103  yamaguch 	for (i = 0; i < nvqs; i++) {
    666  1.101  yamaguch 		vioif_netqueue_teardown(sc, vsc, i);
    667   1.98  yamaguch 	}
    668   1.98  yamaguch 
    669  1.101  yamaguch 	if (sc->sc_has_ctrl) {
    670  1.101  yamaguch 		cv_destroy(&ctrlq->ctrlq_wait);
    671  1.101  yamaguch 		mutex_destroy(&ctrlq->ctrlq_wait_lock);
    672  1.101  yamaguch 		virtio_free_vq(vsc, ctrlq->ctrlq_vq);
    673  1.101  yamaguch 		ctrlq->ctrlq_vq = NULL;
    674   1.98  yamaguch 	}
    675   1.98  yamaguch 
    676  1.101  yamaguch 	vioif_free_queues(sc);
    677  1.101  yamaguch 	mutex_destroy(&sc->sc_lock);
    678  1.101  yamaguch 	virtio_child_attach_failed(vsc);
    679  1.101  yamaguch 	config_finalize_register(self, vioif_finalize_teardown);
    680   1.98  yamaguch 
    681  1.101  yamaguch 	return;
    682   1.98  yamaguch }
    683   1.98  yamaguch 
    684  1.101  yamaguch static int
    685  1.101  yamaguch vioif_finalize_teardown(device_t self)
    686   1.98  yamaguch {
    687  1.101  yamaguch 	struct vioif_softc *sc = device_private(self);
    688   1.98  yamaguch 
    689  1.101  yamaguch 	if (sc->sc_txrx_workqueue != NULL) {
    690  1.101  yamaguch 		vioif_workq_destroy(sc->sc_txrx_workqueue);
    691  1.101  yamaguch 		sc->sc_txrx_workqueue = NULL;
    692   1.46  yamaguch 	}
    693   1.98  yamaguch 
    694  1.101  yamaguch 	return 0;
    695   1.46  yamaguch }
    696   1.46  yamaguch 
    697    1.1   hannken /*
    698  1.101  yamaguch  * Interface functions for ifnet
    699    1.1   hannken  */
    700    1.1   hannken static int
    701  1.101  yamaguch vioif_init(struct ifnet *ifp)
    702    1.1   hannken {
    703  1.101  yamaguch 	struct vioif_softc *sc = ifp->if_softc;
    704    1.1   hannken 	struct virtio_softc *vsc = sc->sc_virtio;
    705   1.98  yamaguch 	struct vioif_netqueue *netq;
    706   1.98  yamaguch 	struct vioif_ctrlqueue *ctrlq = &sc->sc_ctrlq;
    707  1.101  yamaguch 	int r, i;
    708  1.101  yamaguch 
    709  1.101  yamaguch 	vioif_stop(ifp, 0);
    710  1.101  yamaguch 
    711  1.101  yamaguch 	r = virtio_reinit_start(vsc);
    712  1.101  yamaguch 	if (r != 0) {
    713  1.101  yamaguch 		log(LOG_ERR, "%s: reset failed\n", ifp->if_xname);
    714  1.101  yamaguch 		return EIO;
    715  1.101  yamaguch 	}
    716    1.1   hannken 
    717  1.101  yamaguch 	virtio_negotiate_features(vsc, virtio_features(vsc));
    718   1.98  yamaguch 
    719  1.101  yamaguch 	for (i = 0; i < sc->sc_req_nvq_pairs; i++) {
    720  1.101  yamaguch 		netq = &sc->sc_netqs[VIOIF_NETQ_RXQID(i)];
    721   1.46  yamaguch 
    722  1.101  yamaguch 		mutex_enter(&netq->netq_lock);
    723  1.101  yamaguch 		vioif_populate_rx_mbufs_locked(sc, netq);
    724  1.101  yamaguch 		mutex_exit(&netq->netq_lock);
    725   1.46  yamaguch 	}
    726   1.98  yamaguch 
    727  1.101  yamaguch 	virtio_reinit_end(vsc);
    728  1.101  yamaguch 
    729  1.101  yamaguch 	if (sc->sc_has_ctrl)
    730  1.101  yamaguch 		virtio_start_vq_intr(vsc, ctrlq->ctrlq_vq);
    731   1.98  yamaguch 
    732  1.101  yamaguch 	r = vioif_ctrl_mq_vq_pairs_set(sc, sc->sc_req_nvq_pairs);
    733  1.101  yamaguch 	if (r == 0)
    734  1.101  yamaguch 		sc->sc_act_nvq_pairs = sc->sc_req_nvq_pairs;
    735  1.101  yamaguch 	else
    736  1.101  yamaguch 		sc->sc_act_nvq_pairs = 1;
    737  1.101  yamaguch 
    738  1.101  yamaguch 	SET(ifp->if_flags, IFF_RUNNING);
    739   1.42  yamaguch 
    740  1.101  yamaguch 	vioif_net_intr_enable(sc, vsc);
    741   1.42  yamaguch 
    742  1.101  yamaguch 	vioif_update_link_status(sc);
    743  1.101  yamaguch 	r = vioif_rx_filter(sc);
    744   1.98  yamaguch 
    745  1.101  yamaguch 	return r;
    746  1.101  yamaguch }
    747   1.98  yamaguch 
    748  1.101  yamaguch static void
    749  1.101  yamaguch vioif_stop(struct ifnet *ifp, int disable)
    750  1.101  yamaguch {
    751  1.101  yamaguch 	struct vioif_softc *sc = ifp->if_softc;
    752  1.101  yamaguch 	struct virtio_softc *vsc = sc->sc_virtio;
    753  1.101  yamaguch 	struct vioif_netqueue *netq;
    754  1.101  yamaguch 	struct vioif_ctrlqueue *ctrlq = &sc->sc_ctrlq;
    755  1.101  yamaguch 	size_t i, act_qnum;
    756    1.1   hannken 
    757  1.101  yamaguch 	act_qnum = sc->sc_act_nvq_pairs * 2;
    758   1.46  yamaguch 
    759  1.101  yamaguch 	CLR(ifp->if_flags, IFF_RUNNING);
    760  1.101  yamaguch 	for (i = 0; i < act_qnum; i++) {
    761  1.101  yamaguch 		netq = &sc->sc_netqs[i];
    762   1.95  yamaguch 
    763  1.101  yamaguch 		mutex_enter(&netq->netq_lock);
    764  1.101  yamaguch 		netq->netq_stopping = true;
    765  1.101  yamaguch 		mutex_exit(&netq->netq_lock);
    766   1.98  yamaguch 	}
    767   1.46  yamaguch 
    768  1.101  yamaguch 	/* disable interrupts */
    769  1.101  yamaguch 	vioif_net_intr_disable(sc, vsc);
    770  1.101  yamaguch 	if (sc->sc_has_ctrl)
    771  1.101  yamaguch 		virtio_stop_vq_intr(vsc, ctrlq->ctrlq_vq);
    772  1.101  yamaguch 
    773  1.101  yamaguch 	/*
    774  1.101  yamaguch 	 * only way to stop interrupt, I/O and DMA is resetting...
    775  1.101  yamaguch 	 *
    776  1.101  yamaguch 	 * NOTE: Devices based on VirtIO draft specification can not
    777  1.101  yamaguch 	 * stop interrupt completely even if virtio_stop_vq_intr() is called.
    778  1.101  yamaguch 	 */
    779  1.101  yamaguch 	virtio_reset(vsc);
    780  1.101  yamaguch 
    781  1.101  yamaguch 	vioif_intr_barrier();
    782   1.98  yamaguch 
    783  1.101  yamaguch 	for (i = 0; i < act_qnum; i++) {
    784  1.101  yamaguch 		netq = &sc->sc_netqs[i];
    785  1.101  yamaguch 		vioif_work_wait(sc->sc_txrx_workqueue, &netq->netq_work);
    786  1.101  yamaguch 	}
    787   1.42  yamaguch 
    788  1.101  yamaguch 	for (i = 0; i < sc->sc_act_nvq_pairs; i++) {
    789  1.101  yamaguch 		netq = &sc->sc_netqs[VIOIF_NETQ_RXQID(i)];
    790  1.101  yamaguch 		vioif_rx_queue_clear(sc, vsc, netq);
    791   1.98  yamaguch 
    792  1.101  yamaguch 		netq = &sc->sc_netqs[VIOIF_NETQ_TXQID(i)];
    793  1.101  yamaguch 		vioif_tx_queue_clear(sc, vsc, netq);
    794   1.98  yamaguch 	}
    795   1.98  yamaguch 
    796  1.101  yamaguch 	/* all packet processing is stopped */
    797  1.101  yamaguch 	for (i = 0; i < act_qnum; i++) {
    798  1.101  yamaguch 		netq = &sc->sc_netqs[i];
    799    1.1   hannken 
    800  1.101  yamaguch 		mutex_enter(&netq->netq_lock);
    801  1.101  yamaguch 		netq->netq_stopping = false;
    802  1.101  yamaguch 		mutex_exit(&netq->netq_lock);
    803  1.101  yamaguch 	}
    804  1.101  yamaguch }
    805   1.95  yamaguch 
    806  1.101  yamaguch static void
    807  1.101  yamaguch vioif_start(struct ifnet *ifp)
    808  1.101  yamaguch {
    809  1.101  yamaguch 	struct vioif_softc *sc = ifp->if_softc;
    810  1.101  yamaguch 	struct vioif_netqueue *txq0 = &sc->sc_netqs[VIOIF_NETQ_TXQID(0)];
    811   1.58  yamaguch 
    812  1.101  yamaguch #ifdef VIOIF_MPSAFE
    813  1.101  yamaguch 	KASSERT(if_is_mpsafe(ifp));
    814  1.101  yamaguch #endif
    815   1.58  yamaguch 
    816  1.101  yamaguch 	mutex_enter(&txq0->netq_lock);
    817  1.101  yamaguch 	vioif_start_locked(ifp, txq0);
    818  1.101  yamaguch 	mutex_exit(&txq0->netq_lock);
    819  1.101  yamaguch }
    820    1.1   hannken 
    821  1.101  yamaguch static inline int
    822  1.101  yamaguch vioif_select_txqueue(struct ifnet *ifp, struct mbuf *m)
    823  1.101  yamaguch {
    824  1.101  yamaguch 	struct vioif_softc *sc = ifp->if_softc;
    825  1.101  yamaguch 	u_int cpuid = cpu_index(curcpu());
    826   1.58  yamaguch 
    827  1.101  yamaguch 	return VIOIF_NETQ_TXQID(cpuid % sc->sc_act_nvq_pairs);
    828  1.101  yamaguch }
    829    1.1   hannken 
    830  1.101  yamaguch static int
    831  1.101  yamaguch vioif_transmit(struct ifnet *ifp, struct mbuf *m)
    832  1.101  yamaguch {
    833  1.101  yamaguch 	struct vioif_softc *sc = ifp->if_softc;
    834  1.101  yamaguch 	struct vioif_netqueue *netq;
    835  1.101  yamaguch 	struct vioif_tx_context *txc;
    836  1.101  yamaguch 	int qid;
    837    1.1   hannken 
    838  1.101  yamaguch 	qid = vioif_select_txqueue(ifp, m);
    839  1.101  yamaguch 	netq = &sc->sc_netqs[qid];
    840  1.101  yamaguch 	txc = netq->netq_ctx;
    841   1.46  yamaguch 
    842  1.101  yamaguch 	if (__predict_false(!pcq_put(txc->txc_intrq, m))) {
    843  1.101  yamaguch 		m_freem(m);
    844  1.101  yamaguch 		return ENOBUFS;
    845  1.101  yamaguch 	}
    846    1.1   hannken 
    847  1.101  yamaguch 	net_stat_ref_t nsr = IF_STAT_GETREF(ifp);
    848  1.112  riastrad 	if_statadd_ref(ifp, nsr, if_obytes, m->m_pkthdr.len);
    849  1.101  yamaguch 	if (m->m_flags & M_MCAST)
    850  1.112  riastrad 		if_statinc_ref(ifp, nsr, if_omcasts);
    851  1.101  yamaguch 	IF_STAT_PUTREF(ifp);
    852   1.74  yamaguch 
    853  1.101  yamaguch 	if (mutex_tryenter(&netq->netq_lock)) {
    854  1.101  yamaguch 		vioif_transmit_locked(ifp, netq);
    855  1.101  yamaguch 		mutex_exit(&netq->netq_lock);
    856    1.1   hannken 	}
    857    1.1   hannken 
    858    1.1   hannken 	return 0;
    859  1.101  yamaguch }
    860    1.1   hannken 
    861  1.101  yamaguch void
    862  1.101  yamaguch vioif_watchdog(struct ifnet *ifp)
    863  1.101  yamaguch {
    864  1.101  yamaguch 	struct vioif_softc *sc = ifp->if_softc;
    865  1.101  yamaguch 	struct vioif_netqueue *netq;
    866  1.101  yamaguch 	int i;
    867  1.101  yamaguch 
    868  1.102  yamaguch 	if (ISSET(ifp->if_flags, IFF_RUNNING)) {
    869  1.102  yamaguch 		if (ISSET(ifp->if_flags, IFF_DEBUG)) {
    870  1.102  yamaguch 			log(LOG_DEBUG, "%s: watchdog timed out\n",
    871  1.102  yamaguch 			    ifp->if_xname);
    872  1.102  yamaguch 		}
    873  1.102  yamaguch 
    874  1.101  yamaguch 		for (i = 0; i < sc->sc_act_nvq_pairs; i++) {
    875  1.101  yamaguch 			netq = &sc->sc_netqs[VIOIF_NETQ_TXQID(i)];
    876   1.46  yamaguch 
    877  1.101  yamaguch 			mutex_enter(&netq->netq_lock);
    878  1.101  yamaguch 			if (!netq->netq_running_handle) {
    879  1.101  yamaguch 				netq->netq_running_handle = true;
    880  1.101  yamaguch 				vioif_net_sched_handle(sc, netq);
    881  1.101  yamaguch 			}
    882  1.101  yamaguch 			mutex_exit(&netq->netq_lock);
    883   1.46  yamaguch 		}
    884    1.1   hannken 	}
    885  1.101  yamaguch }
    886  1.101  yamaguch 
    887  1.101  yamaguch static int
    888  1.101  yamaguch vioif_ioctl(struct ifnet *ifp, u_long cmd, void *data)
    889  1.101  yamaguch {
    890  1.101  yamaguch 	int s, r;
    891  1.101  yamaguch 
    892  1.101  yamaguch 	s = splnet();
    893  1.101  yamaguch 
    894  1.101  yamaguch 	r = ether_ioctl(ifp, cmd, data);
    895  1.101  yamaguch 	if (r == ENETRESET && (cmd == SIOCADDMULTI || cmd == SIOCDELMULTI)) {
    896  1.101  yamaguch 		if (ifp->if_flags & IFF_RUNNING) {
    897  1.101  yamaguch 			r = vioif_rx_filter(ifp->if_softc);
    898  1.101  yamaguch 		} else {
    899  1.101  yamaguch 			r = 0;
    900  1.101  yamaguch 		}
    901    1.1   hannken 	}
    902  1.101  yamaguch 
    903  1.101  yamaguch 	splx(s);
    904  1.101  yamaguch 
    905  1.101  yamaguch 	return r;
    906    1.1   hannken }
    907    1.1   hannken 
    908  1.101  yamaguch static int
    909  1.101  yamaguch vioif_ifflags(struct vioif_softc *sc)
    910    1.1   hannken {
    911    1.1   hannken 	struct ifnet *ifp = &sc->sc_ethercom.ec_if;
    912  1.101  yamaguch 	bool onoff;
    913  1.101  yamaguch 	int r;
    914    1.1   hannken 
    915  1.101  yamaguch 	if (!sc->sc_has_ctrl) {
    916  1.101  yamaguch 		/* no ctrl vq; always promisc and allmulti */
    917  1.101  yamaguch 		ifp->if_flags |= (IFF_PROMISC | IFF_ALLMULTI);
    918  1.101  yamaguch 		return 0;
    919    1.1   hannken 	}
    920    1.1   hannken 
    921  1.101  yamaguch 	onoff = ifp->if_flags & IFF_ALLMULTI ? true : false;
    922  1.101  yamaguch 	r = vioif_set_allmulti(sc, onoff);
    923  1.101  yamaguch 	if (r != 0) {
    924  1.101  yamaguch 		log(LOG_WARNING,
    925  1.101  yamaguch 		    "%s: couldn't %sable ALLMULTI\n",
    926  1.101  yamaguch 		    ifp->if_xname, onoff ? "en" : "dis");
    927  1.105  yamaguch 		if (onoff) {
    928  1.105  yamaguch 			CLR(ifp->if_flags, IFF_ALLMULTI);
    929  1.105  yamaguch 		} else {
    930  1.105  yamaguch 			SET(ifp->if_flags, IFF_ALLMULTI);
    931  1.101  yamaguch 		}
    932  1.101  yamaguch 	}
    933    1.1   hannken 
    934  1.101  yamaguch 	onoff = ifp->if_flags & IFF_PROMISC ? true : false;
    935  1.101  yamaguch 	r = vioif_set_promisc(sc, onoff);
    936  1.101  yamaguch 	if (r != 0) {
    937  1.101  yamaguch 		log(LOG_WARNING,
    938  1.101  yamaguch 		    "%s: couldn't %sable PROMISC\n",
    939  1.101  yamaguch 		    ifp->if_xname, onoff ? "en" : "dis");
    940  1.105  yamaguch 		if (onoff) {
    941  1.105  yamaguch 			CLR(ifp->if_flags, IFF_PROMISC);
    942  1.105  yamaguch 		} else {
    943  1.105  yamaguch 			SET(ifp->if_flags, IFF_PROMISC);
    944  1.101  yamaguch 		}
    945  1.101  yamaguch 	}
    946   1.55  yamaguch 
    947  1.101  yamaguch 	return 0;
    948  1.101  yamaguch }
    949   1.62  yamaguch 
    950  1.101  yamaguch static int
    951  1.101  yamaguch vioif_ifflags_cb(struct ethercom *ec)
    952  1.101  yamaguch {
    953  1.101  yamaguch 	struct ifnet *ifp = &ec->ec_if;
    954  1.101  yamaguch 	struct vioif_softc *sc = ifp->if_softc;
    955   1.46  yamaguch 
    956  1.101  yamaguch 	return vioif_ifflags(sc);
    957  1.101  yamaguch }
    958    1.1   hannken 
    959  1.101  yamaguch static int
    960  1.101  yamaguch vioif_setup_sysctl(struct vioif_softc *sc)
    961  1.101  yamaguch {
    962  1.101  yamaguch 	const char *devname;
    963  1.101  yamaguch 	struct sysctllog **log;
    964  1.101  yamaguch 	const struct sysctlnode *rnode, *rxnode, *txnode;
    965  1.101  yamaguch 	int error;
    966   1.32  jdolecek 
    967  1.101  yamaguch 	log = &sc->sc_sysctllog;
    968  1.101  yamaguch 	devname = device_xname(sc->sc_dev);
    969   1.32  jdolecek 
    970  1.101  yamaguch 	error = sysctl_createv(log, 0, NULL, &rnode,
    971  1.101  yamaguch 	    0, CTLTYPE_NODE, devname,
    972  1.101  yamaguch 	    SYSCTL_DESCR("virtio-net information and settings"),
    973  1.101  yamaguch 	    NULL, 0, NULL, 0, CTL_HW, CTL_CREATE, CTL_EOL);
    974  1.101  yamaguch 	if (error)
    975  1.101  yamaguch 		goto out;
    976    1.7     ozaki 
    977  1.101  yamaguch 	error = sysctl_createv(log, 0, &rnode, NULL,
    978  1.101  yamaguch 	    CTLFLAG_READWRITE, CTLTYPE_BOOL, "txrx_workqueue",
    979  1.101  yamaguch 	    SYSCTL_DESCR("Use workqueue for packet processing"),
    980  1.101  yamaguch 	    NULL, 0, &sc->sc_txrx_workqueue_sysctl, 0, CTL_CREATE, CTL_EOL);
    981  1.101  yamaguch 	if (error)
    982  1.101  yamaguch 		goto out;
    983   1.32  jdolecek 
    984  1.101  yamaguch 	error = sysctl_createv(log, 0, &rnode, &rxnode,
    985  1.101  yamaguch 	    0, CTLTYPE_NODE, "rx",
    986  1.101  yamaguch 	    SYSCTL_DESCR("virtio-net information and settings for Rx"),
    987  1.101  yamaguch 	    NULL, 0, NULL, 0, CTL_CREATE, CTL_EOL);
    988  1.101  yamaguch 	if (error)
    989  1.101  yamaguch 		goto out;
    990    1.1   hannken 
    991  1.101  yamaguch 	error = sysctl_createv(log, 0, &rxnode, NULL,
    992  1.101  yamaguch 	    CTLFLAG_READWRITE, CTLTYPE_INT, "intr_process_limit",
    993  1.101  yamaguch 	    SYSCTL_DESCR("max number of Rx packets to process for interrupt processing"),
    994  1.101  yamaguch 	    NULL, 0, &sc->sc_rx_intr_process_limit, 0, CTL_CREATE, CTL_EOL);
    995  1.101  yamaguch 	if (error)
    996  1.101  yamaguch 		goto out;
    997   1.66   reinoud 
    998  1.101  yamaguch 	error = sysctl_createv(log, 0, &rxnode, NULL,
    999  1.101  yamaguch 	    CTLFLAG_READWRITE, CTLTYPE_INT, "process_limit",
   1000  1.101  yamaguch 	    SYSCTL_DESCR("max number of Rx packets to process for deferred processing"),
   1001  1.101  yamaguch 	    NULL, 0, &sc->sc_rx_process_limit, 0, CTL_CREATE, CTL_EOL);
   1002  1.101  yamaguch 	if (error)
   1003  1.101  yamaguch 		goto out;
   1004   1.46  yamaguch 
   1005  1.101  yamaguch 	error = sysctl_createv(log, 0, &rnode, &txnode,
   1006  1.101  yamaguch 	    0, CTLTYPE_NODE, "tx",
   1007  1.101  yamaguch 	    SYSCTL_DESCR("virtio-net information and settings for Tx"),
   1008  1.101  yamaguch 	    NULL, 0, NULL, 0, CTL_CREATE, CTL_EOL);
   1009  1.101  yamaguch 	if (error)
   1010  1.101  yamaguch 		goto out;
   1011   1.46  yamaguch 
   1012  1.101  yamaguch 	error = sysctl_createv(log, 0, &txnode, NULL,
   1013  1.101  yamaguch 	    CTLFLAG_READWRITE, CTLTYPE_INT, "intr_process_limit",
   1014  1.101  yamaguch 	    SYSCTL_DESCR("max number of Tx packets to process for interrupt processing"),
   1015  1.101  yamaguch 	    NULL, 0, &sc->sc_tx_intr_process_limit, 0, CTL_CREATE, CTL_EOL);
   1016  1.101  yamaguch 	if (error)
   1017  1.101  yamaguch 		goto out;
   1018   1.46  yamaguch 
   1019  1.101  yamaguch 	error = sysctl_createv(log, 0, &txnode, NULL,
   1020  1.101  yamaguch 	    CTLFLAG_READWRITE, CTLTYPE_INT, "process_limit",
   1021  1.101  yamaguch 	    SYSCTL_DESCR("max number of Tx packets to process for deferred processing"),
   1022  1.101  yamaguch 	    NULL, 0, &sc->sc_tx_process_limit, 0, CTL_CREATE, CTL_EOL);
   1023   1.46  yamaguch 
   1024  1.101  yamaguch out:
   1025  1.101  yamaguch 	if (error)
   1026  1.101  yamaguch 		sysctl_teardown(log);
   1027   1.46  yamaguch 
   1028  1.101  yamaguch 	return error;
   1029  1.101  yamaguch }
   1030   1.46  yamaguch 
   1031  1.101  yamaguch static void
   1032  1.101  yamaguch vioif_setup_stats(struct vioif_softc *sc)
   1033  1.101  yamaguch {
   1034  1.101  yamaguch 	struct vioif_netqueue *netq;
   1035  1.101  yamaguch 	struct vioif_tx_context *txc;
   1036  1.101  yamaguch 	struct vioif_rx_context *rxc;
   1037  1.101  yamaguch 	size_t i, netq_num;
   1038    1.7     ozaki 
   1039   1.98  yamaguch 	netq_num = sc->sc_max_nvq_pairs * 2;
   1040   1.98  yamaguch 	for (i = 0; i < netq_num; i++) {
   1041  1.101  yamaguch 		netq = &sc->sc_netqs[i];
   1042  1.101  yamaguch 		evcnt_attach_dynamic(&netq->netq_mbuf_load_failed, EVCNT_TYPE_MISC,
   1043  1.101  yamaguch 		    NULL, netq->netq_evgroup, "failed to load mbuf to DMA");
   1044  1.101  yamaguch 		evcnt_attach_dynamic(&netq->netq_enqueue_failed,
   1045  1.101  yamaguch 		    EVCNT_TYPE_MISC, NULL, netq->netq_evgroup,
   1046  1.101  yamaguch 		    "virtqueue enqueue failed failed");
   1047   1.17     ozaki 
   1048  1.101  yamaguch 		switch (VIOIF_NETQ_DIR(i)) {
   1049  1.101  yamaguch 		case VIOIF_NETQ_RX:
   1050  1.101  yamaguch 			rxc = netq->netq_ctx;
   1051  1.101  yamaguch 			evcnt_attach_dynamic(&rxc->rxc_mbuf_enobufs,
   1052  1.101  yamaguch 			    EVCNT_TYPE_MISC, NULL, netq->netq_evgroup,
   1053  1.101  yamaguch 			    "no receive buffer");
   1054  1.101  yamaguch 			break;
   1055  1.101  yamaguch 		case VIOIF_NETQ_TX:
   1056  1.101  yamaguch 			txc = netq->netq_ctx;
   1057  1.101  yamaguch 			evcnt_attach_dynamic(&txc->txc_defrag_failed,
   1058  1.101  yamaguch 			    EVCNT_TYPE_MISC, NULL, netq->netq_evgroup,
   1059  1.101  yamaguch 			    "m_defrag() failed");
   1060  1.101  yamaguch 			break;
   1061    1.1   hannken 		}
   1062    1.1   hannken 	}
   1063   1.34     ozaki 
   1064  1.101  yamaguch 	evcnt_attach_dynamic(&sc->sc_ctrlq.ctrlq_cmd_load_failed, EVCNT_TYPE_MISC,
   1065  1.101  yamaguch 	    NULL, device_xname(sc->sc_dev), "control command dmamap load failed");
   1066  1.101  yamaguch 	evcnt_attach_dynamic(&sc->sc_ctrlq.ctrlq_cmd_failed, EVCNT_TYPE_MISC,
   1067  1.101  yamaguch 	    NULL, device_xname(sc->sc_dev), "control command failed");
   1068  1.101  yamaguch }
   1069    1.1   hannken 
   1070  1.101  yamaguch /*
   1071  1.101  yamaguch  * allocate memory
   1072  1.101  yamaguch  */
   1073  1.101  yamaguch static int
   1074  1.101  yamaguch vioif_dmamap_create(struct vioif_softc *sc, bus_dmamap_t *map,
   1075  1.101  yamaguch     bus_size_t size, int nsegs, const char *usage)
   1076  1.101  yamaguch {
   1077  1.101  yamaguch 	int r;
   1078    1.1   hannken 
   1079  1.101  yamaguch 	r = bus_dmamap_create(virtio_dmat(sc->sc_virtio), size,
   1080  1.101  yamaguch 	    nsegs, size, 0, BUS_DMA_NOWAIT | BUS_DMA_ALLOCNOW, map);
   1081   1.32  jdolecek 
   1082  1.101  yamaguch 	if (r != 0) {
   1083  1.101  yamaguch 		aprint_error_dev(sc->sc_dev, "%s dmamap creation failed, "
   1084  1.101  yamaguch 		    "error code %d\n", usage, r);
   1085   1.55  yamaguch 	}
   1086   1.55  yamaguch 
   1087  1.101  yamaguch 	return r;
   1088  1.101  yamaguch }
   1089   1.63  yamaguch 
   1090  1.101  yamaguch static void
   1091  1.101  yamaguch vioif_dmamap_destroy(struct vioif_softc *sc, bus_dmamap_t *map)
   1092  1.101  yamaguch {
   1093    1.1   hannken 
   1094  1.101  yamaguch 	if (*map) {
   1095  1.101  yamaguch 		bus_dmamap_destroy(virtio_dmat(sc->sc_virtio), *map);
   1096  1.101  yamaguch 		*map = NULL;
   1097  1.101  yamaguch 	}
   1098  1.101  yamaguch }
   1099   1.11     ozaki 
   1100  1.101  yamaguch static int
   1101  1.101  yamaguch vioif_dmamap_create_load(struct vioif_softc *sc, bus_dmamap_t *map,
   1102  1.101  yamaguch     void *buf, bus_size_t size, int nsegs, int rw, const char *usage)
   1103  1.101  yamaguch {
   1104  1.101  yamaguch 	int r;
   1105    1.1   hannken 
   1106  1.101  yamaguch 	r = vioif_dmamap_create(sc, map, size, nsegs, usage);
   1107  1.101  yamaguch 	if (r != 0)
   1108  1.101  yamaguch 		return 1;
   1109    1.1   hannken 
   1110  1.101  yamaguch 	r = bus_dmamap_load(virtio_dmat(sc->sc_virtio), *map, buf,
   1111  1.101  yamaguch 	    size, NULL, rw | BUS_DMA_NOWAIT);
   1112  1.101  yamaguch 	if (r != 0) {
   1113  1.101  yamaguch 		vioif_dmamap_destroy(sc, map);
   1114  1.101  yamaguch 		aprint_error_dev(sc->sc_dev, "%s dmamap load failed. "
   1115  1.101  yamaguch 		    "error code %d\n", usage, r);
   1116   1.43  yamaguch 	}
   1117    1.7     ozaki 
   1118  1.101  yamaguch 	return r;
   1119    1.1   hannken }
   1120    1.1   hannken 
   1121  1.101  yamaguch static void *
   1122  1.101  yamaguch vioif_assign_mem(intptr_t *p, size_t size)
   1123   1.55  yamaguch {
   1124  1.101  yamaguch 	intptr_t rv;
   1125   1.55  yamaguch 
   1126  1.101  yamaguch 	rv = *p;
   1127  1.101  yamaguch 	*p += size;
   1128   1.55  yamaguch 
   1129  1.101  yamaguch 	return (void *)rv;
   1130   1.55  yamaguch }
   1131   1.55  yamaguch 
   1132    1.1   hannken /*
   1133  1.101  yamaguch  * dma memory is used for:
   1134  1.101  yamaguch  *   netq_maps_kva:	 metadata array for received frames (READ) and
   1135  1.101  yamaguch  *			 sent frames (WRITE)
   1136  1.101  yamaguch  *   ctrlq_cmd:		 command to be sent via ctrl vq (WRITE)
   1137  1.101  yamaguch  *   ctrlq_status:	 return value for a command via ctrl vq (READ)
   1138  1.101  yamaguch  *   ctrlq_rx:		 parameter for a VIRTIO_NET_CTRL_RX class command
   1139  1.101  yamaguch  *			 (WRITE)
   1140  1.101  yamaguch  *   ctrlq_mac_tbl_uc:	 unicast MAC address filter for a VIRTIO_NET_CTRL_MAC
   1141  1.101  yamaguch  *			 class command (WRITE)
   1142  1.101  yamaguch  *   ctrlq_mac_tbl_mc:	 multicast MAC address filter for a VIRTIO_NET_CTRL_MAC
   1143  1.101  yamaguch  *			 class command (WRITE)
   1144  1.101  yamaguch  * ctrlq_* structures are allocated only one each; they are protected by
   1145  1.101  yamaguch  * ctrlq_inuse variable and ctrlq_wait condvar.
   1146    1.1   hannken  */
   1147    1.1   hannken static int
   1148  1.101  yamaguch vioif_alloc_mems(struct vioif_softc *sc)
   1149    1.1   hannken {
   1150   1.33     ozaki 	struct virtio_softc *vsc = sc->sc_virtio;
   1151   1.98  yamaguch 	struct vioif_netqueue *netq;
   1152   1.43  yamaguch 	struct vioif_ctrlqueue *ctrlq = &sc->sc_ctrlq;
   1153  1.101  yamaguch 	struct vioif_net_map *maps;
   1154  1.101  yamaguch 	unsigned int vq_num;
   1155  1.101  yamaguch 	int r, rsegs;
   1156  1.101  yamaguch 	bus_size_t dmamemsize;
   1157  1.101  yamaguch 	size_t qid, i, netq_num, kmemsize;
   1158  1.101  yamaguch 	void *vaddr;
   1159  1.101  yamaguch 	intptr_t p;
   1160  1.101  yamaguch 
   1161  1.101  yamaguch 	netq_num = sc->sc_max_nvq_pairs * 2;
   1162    1.1   hannken 
   1163  1.101  yamaguch 	/* allocate DMA memory */
   1164  1.101  yamaguch 	dmamemsize = 0;
   1165    1.7     ozaki 
   1166  1.101  yamaguch 	for (qid = 0; qid < netq_num; qid++) {
   1167  1.101  yamaguch 		maps = sc->sc_netqs[qid].netq_maps;
   1168  1.101  yamaguch 		vq_num = sc->sc_netqs[qid].netq_vq->vq_num;
   1169  1.101  yamaguch 		dmamemsize += sizeof(*maps[0].vnm_hdr) * vq_num;
   1170   1.71  yamaguch 	}
   1171   1.71  yamaguch 
   1172  1.101  yamaguch 	if (sc->sc_has_ctrl) {
   1173  1.101  yamaguch 		dmamemsize += sizeof(struct virtio_net_ctrl_cmd);
   1174  1.101  yamaguch 		dmamemsize += sizeof(struct virtio_net_ctrl_status);
   1175  1.101  yamaguch 		dmamemsize += sizeof(struct virtio_net_ctrl_rx);
   1176  1.101  yamaguch 		dmamemsize += sizeof(struct virtio_net_ctrl_mac_tbl)
   1177  1.101  yamaguch 		    + ETHER_ADDR_LEN;
   1178  1.101  yamaguch 		dmamemsize += sizeof(struct virtio_net_ctrl_mac_tbl)
   1179  1.101  yamaguch 		    + ETHER_ADDR_LEN * VIRTIO_NET_CTRL_MAC_MAXENTRIES;
   1180  1.101  yamaguch 		dmamemsize += sizeof(struct virtio_net_ctrl_mac_addr);
   1181  1.101  yamaguch 		dmamemsize += sizeof(struct virtio_net_ctrl_mq);
   1182  1.101  yamaguch 	}
   1183   1.46  yamaguch 
   1184  1.101  yamaguch 	r = bus_dmamem_alloc(virtio_dmat(vsc), dmamemsize, 0, 0,
   1185  1.101  yamaguch 	    &sc->sc_segs[0], 1, &rsegs, BUS_DMA_NOWAIT);
   1186  1.101  yamaguch 	if (r != 0) {
   1187  1.101  yamaguch 		aprint_error_dev(sc->sc_dev,
   1188  1.107  nakayama 		    "DMA memory allocation failed, size %" PRIuBUSSIZE ", "
   1189  1.101  yamaguch 		    "error code %d\n", dmamemsize, r);
   1190  1.101  yamaguch 		goto err_none;
   1191  1.101  yamaguch 	}
   1192  1.101  yamaguch 	r = bus_dmamem_map(virtio_dmat(vsc), &sc->sc_segs[0], 1,
   1193  1.101  yamaguch 	    dmamemsize, &vaddr, BUS_DMA_NOWAIT);
   1194  1.101  yamaguch 	if (r != 0) {
   1195  1.101  yamaguch 		aprint_error_dev(sc->sc_dev,
   1196  1.101  yamaguch 		    "DMA memory map failed, error code %d\n", r);
   1197  1.101  yamaguch 		goto err_dmamem_alloc;
   1198   1.46  yamaguch 	}
   1199   1.46  yamaguch 
   1200  1.101  yamaguch 	/* assign DMA memory */
   1201  1.101  yamaguch 	memset(vaddr, 0, dmamemsize);
   1202  1.101  yamaguch 	sc->sc_dmamem = vaddr;
   1203  1.101  yamaguch 	p = (intptr_t) vaddr;
   1204   1.47  yamaguch 
   1205  1.101  yamaguch 	for (qid = 0; qid < netq_num; qid++) {
   1206  1.101  yamaguch 		netq = &sc->sc_netqs[qid];
   1207  1.101  yamaguch 		maps = netq->netq_maps;
   1208  1.101  yamaguch 		vq_num = netq->netq_vq->vq_num;
   1209   1.46  yamaguch 
   1210  1.101  yamaguch 		netq->netq_maps_kva = vioif_assign_mem(&p,
   1211  1.101  yamaguch 		    sizeof(*maps[0].vnm_hdr) * vq_num);
   1212  1.101  yamaguch 	}
   1213   1.46  yamaguch 
   1214  1.101  yamaguch 	if (sc->sc_has_ctrl) {
   1215  1.101  yamaguch 		ctrlq->ctrlq_cmd = vioif_assign_mem(&p,
   1216  1.101  yamaguch 		    sizeof(*ctrlq->ctrlq_cmd));
   1217  1.101  yamaguch 		ctrlq->ctrlq_status = vioif_assign_mem(&p,
   1218  1.101  yamaguch 		    sizeof(*ctrlq->ctrlq_status));
   1219  1.101  yamaguch 		ctrlq->ctrlq_rx = vioif_assign_mem(&p,
   1220  1.101  yamaguch 		    sizeof(*ctrlq->ctrlq_rx));
   1221  1.101  yamaguch 		ctrlq->ctrlq_mac_tbl_uc = vioif_assign_mem(&p,
   1222  1.101  yamaguch 		    sizeof(*ctrlq->ctrlq_mac_tbl_uc)
   1223  1.101  yamaguch 		    + ETHER_ADDR_LEN);
   1224  1.101  yamaguch 		ctrlq->ctrlq_mac_tbl_mc = vioif_assign_mem(&p,
   1225  1.101  yamaguch 		    sizeof(*ctrlq->ctrlq_mac_tbl_mc)
   1226  1.101  yamaguch 		    + ETHER_ADDR_LEN * VIRTIO_NET_CTRL_MAC_MAXENTRIES);
   1227  1.101  yamaguch 		ctrlq->ctrlq_mac_addr = vioif_assign_mem(&p,
   1228  1.101  yamaguch 		    sizeof(*ctrlq->ctrlq_mac_addr));
   1229  1.101  yamaguch 		ctrlq->ctrlq_mq = vioif_assign_mem(&p, sizeof(*ctrlq->ctrlq_mq));
   1230   1.76  yamaguch 	}
   1231   1.33     ozaki 
   1232  1.101  yamaguch 	/* allocate kmem */
   1233  1.101  yamaguch 	kmemsize = 0;
   1234   1.85  yamaguch 
   1235  1.101  yamaguch 	for (qid = 0; qid < netq_num; qid++) {
   1236  1.101  yamaguch 		netq = &sc->sc_netqs[qid];
   1237  1.101  yamaguch 		vq_num = netq->netq_vq->vq_num;
   1238   1.55  yamaguch 
   1239  1.101  yamaguch 		kmemsize += sizeof(netq->netq_maps[0]) * vq_num;
   1240   1.85  yamaguch 	}
   1241   1.85  yamaguch 
   1242  1.101  yamaguch 	vaddr = kmem_zalloc(kmemsize, KM_SLEEP);
   1243  1.101  yamaguch 	sc->sc_kmem = vaddr;
   1244   1.98  yamaguch 
   1245  1.101  yamaguch 	/* assign allocated kmem */
   1246  1.101  yamaguch 	p = (intptr_t) vaddr;
   1247   1.46  yamaguch 
   1248  1.101  yamaguch 	for (qid = 0; qid < netq_num; qid++) {
   1249  1.101  yamaguch 		netq = &sc->sc_netqs[qid];
   1250  1.101  yamaguch 		vq_num = netq->netq_vq->vq_num;
   1251   1.85  yamaguch 
   1252  1.101  yamaguch 		netq->netq_maps = vioif_assign_mem(&p,
   1253  1.101  yamaguch 		    sizeof(netq->netq_maps[0]) * vq_num);
   1254   1.85  yamaguch 	}
   1255    1.1   hannken 
   1256  1.101  yamaguch 	/* prepare dmamaps */
   1257  1.101  yamaguch 	for (qid = 0; qid < netq_num; qid++) {
   1258  1.101  yamaguch 		static const struct {
   1259  1.101  yamaguch 			const char	*msg_hdr;
   1260  1.101  yamaguch 			const char	*msg_payload;
   1261  1.101  yamaguch 			int		 dma_flag;
   1262  1.101  yamaguch 			bus_size_t	 dma_size;
   1263  1.101  yamaguch 			int		 dma_nsegs;
   1264  1.101  yamaguch 		} dmaparams[VIOIF_NETQ_IDX] = {
   1265  1.101  yamaguch 			[VIOIF_NETQ_RX] = {
   1266  1.101  yamaguch 				.msg_hdr	= "rx header",
   1267  1.101  yamaguch 				.msg_payload	= "rx payload",
   1268  1.101  yamaguch 				.dma_flag	= BUS_DMA_READ,
   1269  1.101  yamaguch 				.dma_size	= MCLBYTES - ETHER_ALIGN,
   1270  1.101  yamaguch 				.dma_nsegs	= 1,
   1271  1.101  yamaguch 			},
   1272  1.101  yamaguch 			[VIOIF_NETQ_TX] = {
   1273  1.101  yamaguch 				.msg_hdr	= "tx header",
   1274  1.101  yamaguch 				.msg_payload	= "tx payload",
   1275  1.101  yamaguch 				.dma_flag	= BUS_DMA_WRITE,
   1276  1.101  yamaguch 				.dma_size	= ETHER_MAX_LEN,
   1277  1.101  yamaguch 				.dma_nsegs	= VIRTIO_NET_TX_MAXNSEGS,
   1278  1.101  yamaguch 			}
   1279  1.101  yamaguch 		};
   1280    1.1   hannken 
   1281  1.101  yamaguch 		struct virtio_net_hdr *hdrs;
   1282  1.101  yamaguch 		int dir;
   1283  1.111     isaki 		int nsegs;
   1284   1.46  yamaguch 
   1285  1.101  yamaguch 		dir = VIOIF_NETQ_DIR(qid);
   1286  1.101  yamaguch 		netq = &sc->sc_netqs[qid];
   1287  1.101  yamaguch 		vq_num = netq->netq_vq->vq_num;
   1288  1.101  yamaguch 		maps = netq->netq_maps;
   1289  1.101  yamaguch 		hdrs = netq->netq_maps_kva;
   1290  1.111     isaki 		nsegs = uimin(dmaparams[dir].dma_nsegs, vq_num - 1/*hdr*/);
   1291    1.7     ozaki 
   1292  1.101  yamaguch 		for (i = 0; i < vq_num; i++) {
   1293  1.101  yamaguch 			maps[i].vnm_hdr = &hdrs[i];
   1294  1.101  yamaguch 
   1295  1.101  yamaguch 			r = vioif_dmamap_create_load(sc, &maps[i].vnm_hdr_map,
   1296  1.101  yamaguch 			    maps[i].vnm_hdr, sc->sc_hdr_size, 1,
   1297  1.101  yamaguch 			    dmaparams[dir].dma_flag, dmaparams[dir].msg_hdr);
   1298  1.101  yamaguch 			if (r != 0)
   1299  1.101  yamaguch 				goto err_reqs;
   1300   1.98  yamaguch 
   1301  1.101  yamaguch 			r = vioif_dmamap_create(sc, &maps[i].vnm_mbuf_map,
   1302  1.111     isaki 			    dmaparams[dir].dma_size, nsegs,
   1303  1.101  yamaguch 			    dmaparams[dir].msg_payload);
   1304  1.101  yamaguch 			if (r != 0)
   1305  1.101  yamaguch 				goto err_reqs;
   1306  1.101  yamaguch 		}
   1307  1.101  yamaguch 	}
   1308    1.7     ozaki 
   1309  1.101  yamaguch 	if (sc->sc_has_ctrl) {
   1310  1.101  yamaguch 		/* control vq class & command */
   1311  1.101  yamaguch 		r = vioif_dmamap_create_load(sc, &ctrlq->ctrlq_cmd_dmamap,
   1312  1.101  yamaguch 		    ctrlq->ctrlq_cmd, sizeof(*ctrlq->ctrlq_cmd), 1,
   1313  1.101  yamaguch 		    BUS_DMA_WRITE, "control command");
   1314  1.101  yamaguch 		if (r != 0)
   1315  1.101  yamaguch 			goto err_reqs;
   1316    1.1   hannken 
   1317  1.101  yamaguch 		r = vioif_dmamap_create_load(sc, &ctrlq->ctrlq_status_dmamap,
   1318  1.101  yamaguch 		    ctrlq->ctrlq_status, sizeof(*ctrlq->ctrlq_status), 1,
   1319  1.101  yamaguch 		    BUS_DMA_READ, "control status");
   1320  1.101  yamaguch 		if (r != 0)
   1321  1.101  yamaguch 			goto err_reqs;
   1322    1.1   hannken 
   1323  1.101  yamaguch 		/* control vq rx mode command parameter */
   1324  1.101  yamaguch 		r = vioif_dmamap_create_load(sc, &ctrlq->ctrlq_rx_dmamap,
   1325  1.101  yamaguch 		    ctrlq->ctrlq_rx, sizeof(*ctrlq->ctrlq_rx), 1,
   1326  1.101  yamaguch 		    BUS_DMA_WRITE, "rx mode control command");
   1327  1.101  yamaguch 		if (r != 0)
   1328  1.101  yamaguch 			goto err_reqs;
   1329   1.46  yamaguch 
   1330  1.101  yamaguch 		/* multiqueue set command */
   1331  1.101  yamaguch 		r = vioif_dmamap_create_load(sc, &ctrlq->ctrlq_mq_dmamap,
   1332  1.101  yamaguch 		    ctrlq->ctrlq_mq, sizeof(*ctrlq->ctrlq_mq), 1,
   1333  1.101  yamaguch 		    BUS_DMA_WRITE, "multiqueue set command");
   1334  1.101  yamaguch 		if (r != 0)
   1335  1.101  yamaguch 			goto err_reqs;
   1336   1.36  jdolecek 
   1337  1.101  yamaguch 		/* control vq MAC filter table for unicast */
   1338  1.101  yamaguch 		/* do not load now since its length is variable */
   1339  1.101  yamaguch 		r = vioif_dmamap_create(sc, &ctrlq->ctrlq_tbl_uc_dmamap,
   1340  1.101  yamaguch 		    sizeof(*ctrlq->ctrlq_mac_tbl_uc)
   1341  1.101  yamaguch 		    + ETHER_ADDR_LEN, 1,
   1342  1.101  yamaguch 		    "unicast MAC address filter command");
   1343  1.101  yamaguch 		if (r != 0)
   1344  1.101  yamaguch 			goto err_reqs;
   1345   1.95  yamaguch 
   1346  1.101  yamaguch 		/* control vq MAC filter table for multicast */
   1347  1.101  yamaguch 		r = vioif_dmamap_create(sc, &ctrlq->ctrlq_tbl_mc_dmamap,
   1348  1.101  yamaguch 		    sizeof(*ctrlq->ctrlq_mac_tbl_mc)
   1349  1.101  yamaguch 		    + ETHER_ADDR_LEN * VIRTIO_NET_CTRL_MAC_MAXENTRIES, 1,
   1350  1.101  yamaguch 		    "multicast MAC address filter command");
   1351  1.101  yamaguch 		if (r != 0)
   1352  1.101  yamaguch 			goto err_reqs;
   1353   1.36  jdolecek 
   1354  1.101  yamaguch 		/* control vq MAC address set command */
   1355  1.101  yamaguch 		r = vioif_dmamap_create_load(sc,
   1356  1.101  yamaguch 		    &ctrlq->ctrlq_mac_addr_dmamap,
   1357  1.101  yamaguch 		    ctrlq->ctrlq_mac_addr,
   1358  1.101  yamaguch 		    sizeof(*ctrlq->ctrlq_mac_addr), 1,
   1359  1.101  yamaguch 		    BUS_DMA_WRITE, "mac addr set command");
   1360  1.101  yamaguch 		if (r != 0)
   1361  1.101  yamaguch 			goto err_reqs;
   1362  1.101  yamaguch 	}
   1363   1.36  jdolecek 
   1364  1.101  yamaguch 	return 0;
   1365   1.36  jdolecek 
   1366  1.101  yamaguch err_reqs:
   1367  1.101  yamaguch 	vioif_dmamap_destroy(sc, &ctrlq->ctrlq_tbl_mc_dmamap);
   1368  1.101  yamaguch 	vioif_dmamap_destroy(sc, &ctrlq->ctrlq_tbl_uc_dmamap);
   1369  1.101  yamaguch 	vioif_dmamap_destroy(sc, &ctrlq->ctrlq_rx_dmamap);
   1370  1.101  yamaguch 	vioif_dmamap_destroy(sc, &ctrlq->ctrlq_status_dmamap);
   1371  1.101  yamaguch 	vioif_dmamap_destroy(sc, &ctrlq->ctrlq_cmd_dmamap);
   1372  1.101  yamaguch 	vioif_dmamap_destroy(sc, &ctrlq->ctrlq_mac_addr_dmamap);
   1373  1.101  yamaguch 	for (qid = 0; qid < netq_num; qid++) {
   1374  1.101  yamaguch 		vq_num = sc->sc_netqs[qid].netq_vq->vq_num;
   1375  1.101  yamaguch 		maps = sc->sc_netqs[qid].netq_maps;
   1376   1.99  yamaguch 
   1377  1.101  yamaguch 		for (i = 0; i < vq_num; i++) {
   1378  1.101  yamaguch 			vioif_dmamap_destroy(sc, &maps[i].vnm_mbuf_map);
   1379  1.101  yamaguch 			vioif_dmamap_destroy(sc, &maps[i].vnm_hdr_map);
   1380    1.1   hannken 		}
   1381    1.1   hannken 	}
   1382  1.101  yamaguch 	if (sc->sc_kmem) {
   1383  1.101  yamaguch 		kmem_free(sc->sc_kmem, kmemsize);
   1384  1.101  yamaguch 		sc->sc_kmem = NULL;
   1385    1.1   hannken 	}
   1386  1.101  yamaguch 	bus_dmamem_unmap(virtio_dmat(vsc), sc->sc_dmamem, dmamemsize);
   1387  1.101  yamaguch err_dmamem_alloc:
   1388  1.101  yamaguch 	bus_dmamem_free(virtio_dmat(vsc), &sc->sc_segs[0], 1);
   1389  1.101  yamaguch err_none:
   1390  1.101  yamaguch 	return -1;
   1391   1.46  yamaguch }
   1392   1.46  yamaguch 
   1393   1.46  yamaguch static void
   1394  1.101  yamaguch vioif_alloc_queues(struct vioif_softc *sc)
   1395   1.46  yamaguch {
   1396  1.101  yamaguch 	int nvq_pairs = sc->sc_max_nvq_pairs;
   1397  1.101  yamaguch 	size_t nvqs, netq_num;
   1398  1.101  yamaguch 
   1399  1.101  yamaguch 	KASSERT(nvq_pairs <= VIRTIO_NET_CTRL_MQ_VQ_PAIRS_MAX);
   1400   1.46  yamaguch 
   1401  1.101  yamaguch 	nvqs = netq_num = sc->sc_max_nvq_pairs * 2;
   1402  1.101  yamaguch 	if (sc->sc_has_ctrl)
   1403  1.101  yamaguch 		nvqs++;
   1404   1.46  yamaguch 
   1405  1.101  yamaguch 	sc->sc_vqs = kmem_zalloc(sizeof(sc->sc_vqs[0]) * nvqs, KM_SLEEP);
   1406  1.106  yamaguch 	sc->sc_netqs = kmem_zalloc(sizeof(sc->sc_netqs[0]) * netq_num,
   1407  1.101  yamaguch 	    KM_SLEEP);
   1408   1.46  yamaguch }
   1409   1.46  yamaguch 
   1410   1.46  yamaguch static void
   1411  1.101  yamaguch vioif_free_queues(struct vioif_softc *sc)
   1412   1.46  yamaguch {
   1413  1.101  yamaguch 	size_t nvqs, netq_num;
   1414   1.46  yamaguch 
   1415  1.101  yamaguch 	nvqs = netq_num = sc->sc_max_nvq_pairs * 2;
   1416  1.101  yamaguch 	if (sc->sc_ctrlq.ctrlq_vq)
   1417  1.101  yamaguch 		nvqs++;
   1418   1.46  yamaguch 
   1419  1.101  yamaguch 	kmem_free(sc->sc_netqs, sizeof(sc->sc_netqs[0]) * netq_num);
   1420  1.101  yamaguch 	kmem_free(sc->sc_vqs, sizeof(sc->sc_vqs[0]) * nvqs);
   1421  1.101  yamaguch 	sc->sc_netqs = NULL;
   1422  1.101  yamaguch 	sc->sc_vqs = NULL;
   1423   1.46  yamaguch }
   1424   1.46  yamaguch 
   1425  1.101  yamaguch /*
   1426  1.101  yamaguch  * Network queues
   1427  1.101  yamaguch  */
   1428  1.101  yamaguch static int
   1429  1.101  yamaguch vioif_netqueue_init(struct vioif_softc *sc, struct virtio_softc *vsc,
   1430  1.101  yamaguch     size_t qid, u_int softint_flags)
   1431   1.46  yamaguch {
   1432  1.101  yamaguch 	static const struct {
   1433  1.101  yamaguch 		const char	*dirname;
   1434  1.101  yamaguch 		int		 segsize;
   1435  1.101  yamaguch 		int		 nsegs;
   1436  1.101  yamaguch 		int 		(*intrhand)(void *);
   1437  1.101  yamaguch 		void		(*sihand)(void *);
   1438  1.101  yamaguch 	} params[VIOIF_NETQ_IDX] = {
   1439  1.101  yamaguch 		[VIOIF_NETQ_RX] = {
   1440  1.101  yamaguch 			.dirname	= "rx",
   1441  1.101  yamaguch 			.segsize	= MCLBYTES,
   1442  1.101  yamaguch 			.nsegs		= 2,
   1443  1.101  yamaguch 			.intrhand	= vioif_rx_intr,
   1444  1.101  yamaguch 			.sihand		= vioif_rx_handle,
   1445  1.101  yamaguch 		},
   1446  1.101  yamaguch 		[VIOIF_NETQ_TX] = {
   1447  1.101  yamaguch 			.dirname	= "tx",
   1448  1.101  yamaguch 			.segsize	= ETHER_MAX_LEN - ETHER_HDR_LEN,
   1449  1.101  yamaguch 			.nsegs		= 2,
   1450  1.101  yamaguch 			.intrhand	= vioif_tx_intr,
   1451  1.101  yamaguch 			.sihand		= vioif_tx_handle,
   1452  1.101  yamaguch 		}
   1453  1.101  yamaguch 	};
   1454   1.46  yamaguch 
   1455  1.101  yamaguch 	struct virtqueue *vq;
   1456  1.101  yamaguch 	struct vioif_netqueue *netq;
   1457  1.101  yamaguch 	struct vioif_tx_context *txc;
   1458  1.101  yamaguch 	struct vioif_rx_context *rxc;
   1459  1.101  yamaguch 	char qname[32];
   1460  1.101  yamaguch 	int r, dir;
   1461   1.46  yamaguch 
   1462  1.101  yamaguch 	txc = NULL;
   1463  1.101  yamaguch 	rxc = NULL;
   1464  1.101  yamaguch 	netq = &sc->sc_netqs[qid];
   1465  1.101  yamaguch 	vq = &sc->sc_vqs[qid];
   1466  1.101  yamaguch 	dir = VIOIF_NETQ_DIR(qid);
   1467   1.46  yamaguch 
   1468  1.101  yamaguch 	netq->netq_vq = &sc->sc_vqs[qid];
   1469  1.101  yamaguch 	netq->netq_stopping = false;
   1470  1.101  yamaguch 	netq->netq_running_handle = false;
   1471   1.46  yamaguch 
   1472  1.101  yamaguch 	snprintf(qname, sizeof(qname), "%s%zu",
   1473  1.101  yamaguch 	    params[dir].dirname, VIOIF_NETQ_PAIRIDX(qid));
   1474  1.101  yamaguch 	snprintf(netq->netq_evgroup, sizeof(netq->netq_evgroup),
   1475  1.101  yamaguch 	    "%s-%s", device_xname(sc->sc_dev), qname);
   1476   1.46  yamaguch 
   1477  1.101  yamaguch 	mutex_init(&netq->netq_lock, MUTEX_DEFAULT, IPL_NET);
   1478  1.104  yamaguch 	virtio_init_vq(vsc, vq, qid, params[dir].intrhand, netq);
   1479  1.104  yamaguch 
   1480  1.104  yamaguch 	r = virtio_alloc_vq(vsc, vq,
   1481  1.101  yamaguch 	    params[dir].segsize + sc->sc_hdr_size,
   1482  1.101  yamaguch 	    params[dir].nsegs, qname);
   1483  1.101  yamaguch 	if (r != 0)
   1484  1.101  yamaguch 		goto err;
   1485  1.101  yamaguch 	netq->netq_vq = vq;
   1486   1.46  yamaguch 
   1487  1.101  yamaguch 	netq->netq_softint = softint_establish(softint_flags,
   1488  1.101  yamaguch 	    params[dir].sihand, netq);
   1489  1.101  yamaguch 	if (netq->netq_softint == NULL) {
   1490  1.101  yamaguch 		aprint_error_dev(sc->sc_dev,
   1491  1.101  yamaguch 		    "couldn't establish %s softint\n",
   1492  1.101  yamaguch 		    params[dir].dirname);
   1493  1.101  yamaguch 		goto err;
   1494   1.46  yamaguch 	}
   1495  1.101  yamaguch 	vioif_work_set(&netq->netq_work, params[dir].sihand, netq);
   1496   1.46  yamaguch 
   1497  1.101  yamaguch 	switch (dir) {
   1498  1.101  yamaguch 	case VIOIF_NETQ_RX:
   1499  1.101  yamaguch 		rxc = kmem_zalloc(sizeof(*rxc), KM_SLEEP);
   1500  1.101  yamaguch 		netq->netq_ctx = rxc;
   1501  1.101  yamaguch 		/* nothing to do */
   1502  1.101  yamaguch 		break;
   1503  1.101  yamaguch 	case VIOIF_NETQ_TX:
   1504  1.101  yamaguch 		txc = kmem_zalloc(sizeof(*txc), KM_SLEEP);
   1505  1.101  yamaguch 		netq->netq_ctx = (void *)txc;
   1506  1.101  yamaguch 		txc->txc_deferred_transmit = softint_establish(softint_flags,
   1507  1.101  yamaguch 		    vioif_deferred_transmit, netq);
   1508  1.101  yamaguch 		if (txc->txc_deferred_transmit == NULL) {
   1509  1.101  yamaguch 			aprint_error_dev(sc->sc_dev,
   1510  1.101  yamaguch 			    "couldn't establish softint for "
   1511  1.101  yamaguch 			    "tx deferred transmit\n");
   1512  1.101  yamaguch 			goto err;
   1513  1.101  yamaguch 		}
   1514  1.101  yamaguch 		txc->txc_link_active = VIOIF_IS_LINK_ACTIVE(sc);
   1515  1.102  yamaguch 		txc->txc_no_free_slots = false;
   1516  1.101  yamaguch 		txc->txc_intrq = pcq_create(vq->vq_num, KM_SLEEP);
   1517  1.101  yamaguch 		break;
   1518   1.46  yamaguch 	}
   1519   1.46  yamaguch 
   1520   1.46  yamaguch 	return 0;
   1521   1.46  yamaguch 
   1522  1.101  yamaguch err:
   1523  1.101  yamaguch 	netq->netq_ctx = NULL;
   1524    1.7     ozaki 
   1525  1.101  yamaguch 	if (rxc != NULL) {
   1526  1.101  yamaguch 		kmem_free(rxc, sizeof(*rxc));
   1527  1.101  yamaguch 	}
   1528    1.1   hannken 
   1529  1.101  yamaguch 	if (txc != NULL) {
   1530  1.101  yamaguch 		if (txc->txc_deferred_transmit != NULL)
   1531  1.101  yamaguch 			softint_disestablish(txc->txc_deferred_transmit);
   1532  1.101  yamaguch 		if (txc->txc_intrq != NULL)
   1533  1.101  yamaguch 			pcq_destroy(txc->txc_intrq);
   1534  1.101  yamaguch 		kmem_free(txc, sizeof(txc));
   1535  1.101  yamaguch 	}
   1536    1.1   hannken 
   1537  1.101  yamaguch 	vioif_work_set(&netq->netq_work, NULL, NULL);
   1538  1.101  yamaguch 	if (netq->netq_softint != NULL) {
   1539  1.101  yamaguch 		softint_disestablish(netq->netq_softint);
   1540  1.101  yamaguch 		netq->netq_softint = NULL;
   1541    1.1   hannken 	}
   1542    1.1   hannken 
   1543  1.101  yamaguch 	virtio_free_vq(vsc, vq);
   1544  1.101  yamaguch 	mutex_destroy(&netq->netq_lock);
   1545  1.101  yamaguch 	netq->netq_vq = NULL;
   1546    1.1   hannken 
   1547  1.101  yamaguch 	return -1;
   1548    1.1   hannken }
   1549    1.1   hannken 
   1550  1.101  yamaguch static void
   1551  1.101  yamaguch vioif_netqueue_teardown(struct vioif_softc *sc, struct virtio_softc *vsc,
   1552  1.101  yamaguch     size_t qid)
   1553    1.1   hannken {
   1554   1.98  yamaguch 	struct vioif_netqueue *netq;
   1555  1.101  yamaguch 	struct vioif_rx_context *rxc;
   1556  1.101  yamaguch 	struct vioif_tx_context *txc;
   1557  1.101  yamaguch 	int dir;
   1558  1.101  yamaguch 
   1559  1.101  yamaguch 	netq = &sc->sc_netqs[qid];
   1560  1.101  yamaguch 
   1561  1.101  yamaguch 	if (netq->netq_vq == NULL)
   1562  1.101  yamaguch 		return;
   1563  1.101  yamaguch 
   1564  1.101  yamaguch 	netq = &sc->sc_netqs[qid];
   1565  1.101  yamaguch 	dir = VIOIF_NETQ_DIR(qid);
   1566  1.101  yamaguch 	switch (dir) {
   1567  1.101  yamaguch 	case VIOIF_NETQ_RX:
   1568  1.101  yamaguch 		rxc = netq->netq_ctx;
   1569  1.101  yamaguch 		netq->netq_ctx = NULL;
   1570  1.101  yamaguch 		kmem_free(rxc, sizeof(*rxc));
   1571  1.101  yamaguch 		break;
   1572  1.101  yamaguch 	case VIOIF_NETQ_TX:
   1573  1.101  yamaguch 		txc = netq->netq_ctx;
   1574  1.101  yamaguch 		netq->netq_ctx = NULL;
   1575  1.101  yamaguch 		softint_disestablish(txc->txc_deferred_transmit);
   1576  1.101  yamaguch 		pcq_destroy(txc->txc_intrq);
   1577  1.101  yamaguch 		kmem_free(txc, sizeof(*txc));
   1578  1.101  yamaguch 		break;
   1579  1.101  yamaguch 	}
   1580    1.1   hannken 
   1581  1.101  yamaguch 	softint_disestablish(netq->netq_softint);
   1582  1.101  yamaguch 	virtio_free_vq(vsc, netq->netq_vq);
   1583  1.101  yamaguch 	mutex_destroy(&netq->netq_lock);
   1584  1.101  yamaguch 	netq->netq_vq = NULL;
   1585    1.1   hannken }
   1586    1.1   hannken 
   1587   1.98  yamaguch static void
   1588   1.98  yamaguch vioif_net_sched_handle(struct vioif_softc *sc, struct vioif_netqueue *netq)
   1589   1.98  yamaguch {
   1590   1.98  yamaguch 
   1591   1.98  yamaguch 	KASSERT(mutex_owned(&netq->netq_lock));
   1592   1.98  yamaguch 	KASSERT(!netq->netq_stopping);
   1593   1.98  yamaguch 
   1594   1.98  yamaguch 	if (netq->netq_workqueue) {
   1595   1.98  yamaguch 		vioif_work_add(sc->sc_txrx_workqueue, &netq->netq_work);
   1596   1.98  yamaguch 	} else {
   1597   1.98  yamaguch 		softint_schedule(netq->netq_softint);
   1598   1.98  yamaguch 	}
   1599   1.98  yamaguch }
   1600   1.98  yamaguch 
   1601   1.99  yamaguch static int
   1602   1.99  yamaguch vioif_net_load_mbuf(struct virtio_softc *vsc, struct vioif_net_map *map,
   1603   1.99  yamaguch    struct mbuf *m, int dma_flags)
   1604   1.99  yamaguch {
   1605   1.99  yamaguch 	int r;
   1606   1.99  yamaguch 
   1607   1.99  yamaguch 	KASSERT(map->vnm_mbuf == NULL);
   1608   1.99  yamaguch 
   1609   1.99  yamaguch 	r = bus_dmamap_load_mbuf(virtio_dmat(vsc),
   1610   1.99  yamaguch 	    map->vnm_mbuf_map, m, dma_flags | BUS_DMA_NOWAIT);
   1611   1.99  yamaguch 	if (r == 0) {
   1612   1.99  yamaguch 		map->vnm_mbuf = m;
   1613   1.99  yamaguch 	}
   1614   1.99  yamaguch 
   1615   1.99  yamaguch 	return r;
   1616   1.99  yamaguch }
   1617   1.99  yamaguch 
   1618   1.99  yamaguch static void
   1619   1.99  yamaguch vioif_net_unload_mbuf(struct virtio_softc *vsc, struct vioif_net_map *map)
   1620   1.99  yamaguch {
   1621   1.99  yamaguch 
   1622   1.99  yamaguch 	KASSERT(map->vnm_mbuf != NULL);
   1623   1.99  yamaguch 	bus_dmamap_unload(virtio_dmat(vsc), map->vnm_mbuf_map);
   1624   1.99  yamaguch 	map->vnm_mbuf = NULL;
   1625   1.99  yamaguch }
   1626   1.99  yamaguch 
   1627   1.99  yamaguch static int
   1628   1.99  yamaguch vioif_net_enqueue(struct virtio_softc *vsc, struct virtqueue *vq,
   1629   1.99  yamaguch     int slot, struct vioif_net_map *map, int dma_ops, bool is_write)
   1630   1.99  yamaguch {
   1631   1.99  yamaguch 	int r;
   1632   1.99  yamaguch 
   1633   1.99  yamaguch 	KASSERT(map->vnm_mbuf != NULL);
   1634   1.99  yamaguch 
   1635   1.99  yamaguch 	/* This should actually never fail */
   1636   1.99  yamaguch 	r = virtio_enqueue_reserve(vsc, vq, slot,
   1637   1.99  yamaguch 	    map->vnm_mbuf_map->dm_nsegs + 1);
   1638   1.99  yamaguch 	if (r != 0) {
   1639   1.99  yamaguch 		/* slot already freed by virtio_enqueue_reserve */
   1640   1.99  yamaguch 		return r;
   1641   1.99  yamaguch 	}
   1642   1.99  yamaguch 
   1643   1.99  yamaguch 	bus_dmamap_sync(virtio_dmat(vsc), map->vnm_mbuf_map,
   1644   1.99  yamaguch 	    0, map->vnm_mbuf_map->dm_mapsize, dma_ops);
   1645   1.99  yamaguch 	bus_dmamap_sync(virtio_dmat(vsc), map->vnm_hdr_map,
   1646   1.99  yamaguch 	    0, map->vnm_hdr_map->dm_mapsize, dma_ops);
   1647   1.99  yamaguch 
   1648   1.99  yamaguch 	virtio_enqueue(vsc, vq, slot, map->vnm_hdr_map, is_write);
   1649   1.99  yamaguch 	virtio_enqueue(vsc, vq, slot, map->vnm_mbuf_map, is_write);
   1650   1.99  yamaguch 	virtio_enqueue_commit(vsc, vq, slot, false);
   1651   1.99  yamaguch 
   1652   1.99  yamaguch 	return 0;
   1653   1.99  yamaguch }
   1654   1.99  yamaguch 
   1655   1.99  yamaguch static int
   1656   1.99  yamaguch vioif_net_enqueue_tx(struct virtio_softc *vsc, struct virtqueue *vq,
   1657   1.99  yamaguch     int slot, struct vioif_net_map *map)
   1658   1.99  yamaguch {
   1659   1.99  yamaguch 
   1660   1.99  yamaguch 	return vioif_net_enqueue(vsc, vq, slot, map,
   1661   1.99  yamaguch 	    BUS_DMASYNC_PREWRITE, true);
   1662   1.99  yamaguch }
   1663   1.99  yamaguch 
   1664   1.99  yamaguch static int
   1665   1.99  yamaguch vioif_net_enqueue_rx(struct virtio_softc *vsc, struct virtqueue *vq,
   1666   1.99  yamaguch     int slot, struct vioif_net_map *map)
   1667   1.99  yamaguch {
   1668   1.99  yamaguch 
   1669   1.99  yamaguch 	return vioif_net_enqueue(vsc, vq, slot, map,
   1670   1.99  yamaguch 	    BUS_DMASYNC_PREREAD, false);
   1671   1.99  yamaguch }
   1672   1.99  yamaguch 
   1673  1.101  yamaguch static struct mbuf *
   1674  1.101  yamaguch vioif_net_dequeue_commit(struct virtio_softc *vsc, struct virtqueue *vq,
   1675  1.101  yamaguch    int slot, struct vioif_net_map *map, int dma_flags)
   1676  1.101  yamaguch {
   1677  1.101  yamaguch 	struct mbuf *m;
   1678  1.101  yamaguch 
   1679  1.101  yamaguch 	m = map->vnm_mbuf;
   1680  1.101  yamaguch 	KASSERT(m != NULL);
   1681  1.101  yamaguch 	map->vnm_mbuf = NULL;
   1682  1.101  yamaguch 
   1683  1.101  yamaguch 	bus_dmamap_sync(virtio_dmat(vsc), map->vnm_hdr_map,
   1684  1.101  yamaguch 	    0, map->vnm_hdr_map->dm_mapsize, dma_flags);
   1685  1.101  yamaguch 	bus_dmamap_sync(virtio_dmat(vsc), map->vnm_mbuf_map,
   1686  1.101  yamaguch 	    0, map->vnm_mbuf_map->dm_mapsize, dma_flags);
   1687  1.101  yamaguch 
   1688  1.101  yamaguch 	bus_dmamap_unload(virtio_dmat(vsc), map->vnm_mbuf_map);
   1689  1.101  yamaguch 	virtio_dequeue_commit(vsc, vq, slot);
   1690  1.101  yamaguch 
   1691  1.101  yamaguch 	return m;
   1692  1.101  yamaguch }
   1693  1.101  yamaguch 
   1694  1.101  yamaguch static void
   1695  1.101  yamaguch vioif_net_intr_enable(struct vioif_softc *sc, struct virtio_softc *vsc)
   1696  1.101  yamaguch {
   1697  1.101  yamaguch 	struct vioif_netqueue *netq;
   1698  1.101  yamaguch 	size_t i, act_qnum;
   1699  1.101  yamaguch 	int enqueued;
   1700  1.101  yamaguch 
   1701  1.101  yamaguch 	act_qnum = sc->sc_act_nvq_pairs * 2;
   1702  1.101  yamaguch 	for (i = 0; i < act_qnum; i++) {
   1703  1.101  yamaguch 		netq = &sc->sc_netqs[i];
   1704  1.101  yamaguch 
   1705  1.101  yamaguch 		KASSERT(!netq->netq_stopping);
   1706  1.101  yamaguch 		KASSERT(!netq->netq_running_handle);
   1707  1.101  yamaguch 
   1708  1.101  yamaguch 		enqueued = virtio_start_vq_intr(vsc, netq->netq_vq);
   1709  1.101  yamaguch 		if (enqueued != 0) {
   1710  1.101  yamaguch 			virtio_stop_vq_intr(vsc, netq->netq_vq);
   1711  1.101  yamaguch 
   1712  1.101  yamaguch 			mutex_enter(&netq->netq_lock);
   1713  1.101  yamaguch 			netq->netq_running_handle = true;
   1714  1.101  yamaguch 			vioif_net_sched_handle(sc, netq);
   1715  1.101  yamaguch 			mutex_exit(&netq->netq_lock);
   1716  1.101  yamaguch 		}
   1717  1.101  yamaguch 	}
   1718  1.101  yamaguch }
   1719  1.101  yamaguch 
   1720  1.101  yamaguch static void
   1721  1.101  yamaguch vioif_net_intr_disable(struct vioif_softc *sc, struct virtio_softc *vsc)
   1722  1.101  yamaguch {
   1723  1.101  yamaguch 	struct vioif_netqueue *netq;
   1724  1.101  yamaguch 	size_t i, act_qnum;
   1725  1.101  yamaguch 
   1726  1.101  yamaguch 	act_qnum = sc->sc_act_nvq_pairs * 2;
   1727  1.101  yamaguch 	for (i = 0; i < act_qnum; i++) {
   1728  1.101  yamaguch 		netq = &sc->sc_netqs[i];
   1729  1.101  yamaguch 
   1730  1.101  yamaguch 		virtio_stop_vq_intr(vsc, netq->netq_vq);
   1731  1.101  yamaguch 	}
   1732  1.101  yamaguch }
   1733  1.101  yamaguch 
   1734  1.101  yamaguch /*
   1735  1.101  yamaguch  * Receive implementation
   1736  1.101  yamaguch  */
   1737  1.101  yamaguch /* enqueue mbufs to receive slots */
   1738  1.101  yamaguch static void
   1739  1.101  yamaguch vioif_populate_rx_mbufs_locked(struct vioif_softc *sc, struct vioif_netqueue *netq)
   1740  1.101  yamaguch {
   1741  1.101  yamaguch 	struct virtqueue *vq = netq->netq_vq;
   1742  1.101  yamaguch 	struct virtio_softc *vsc = vq->vq_owner;
   1743  1.101  yamaguch 	struct vioif_rx_context *rxc;
   1744  1.101  yamaguch 	struct vioif_net_map *map;
   1745  1.101  yamaguch 	struct mbuf *m;
   1746  1.101  yamaguch 	int i, r, ndone = 0;
   1747  1.101  yamaguch 
   1748  1.101  yamaguch 	KASSERT(mutex_owned(&netq->netq_lock));
   1749  1.101  yamaguch 
   1750  1.101  yamaguch 	rxc = netq->netq_ctx;
   1751  1.101  yamaguch 
   1752  1.101  yamaguch 	for (i = 0; i < vq->vq_num; i++) {
   1753  1.101  yamaguch 		int slot;
   1754  1.101  yamaguch 		r = virtio_enqueue_prep(vsc, vq, &slot);
   1755  1.101  yamaguch 		if (r == EAGAIN)
   1756  1.101  yamaguch 			break;
   1757  1.101  yamaguch 		if (__predict_false(r != 0))
   1758  1.101  yamaguch 			panic("enqueue_prep for rx buffers");
   1759  1.101  yamaguch 
   1760  1.101  yamaguch 		MGETHDR(m, M_DONTWAIT, MT_DATA);
   1761  1.101  yamaguch 		if (m == NULL) {
   1762  1.101  yamaguch 			virtio_enqueue_abort(vsc, vq, slot);
   1763  1.101  yamaguch 			rxc->rxc_mbuf_enobufs.ev_count++;
   1764  1.101  yamaguch 			break;
   1765  1.101  yamaguch 		}
   1766  1.101  yamaguch 		MCLGET(m, M_DONTWAIT);
   1767  1.101  yamaguch 		if ((m->m_flags & M_EXT) == 0) {
   1768  1.101  yamaguch 			virtio_enqueue_abort(vsc, vq, slot);
   1769  1.101  yamaguch 			m_freem(m);
   1770  1.101  yamaguch 			rxc->rxc_mbuf_enobufs.ev_count++;
   1771  1.101  yamaguch 			break;
   1772  1.101  yamaguch 		}
   1773  1.101  yamaguch 
   1774  1.101  yamaguch 		m->m_len = m->m_pkthdr.len = MCLBYTES;
   1775  1.101  yamaguch 		m_adj(m, ETHER_ALIGN);
   1776  1.101  yamaguch 
   1777  1.101  yamaguch 		map = &netq->netq_maps[slot];
   1778  1.101  yamaguch 		r = vioif_net_load_mbuf(vsc, map, m, BUS_DMA_READ);
   1779  1.101  yamaguch 		if (r != 0) {
   1780  1.101  yamaguch 			virtio_enqueue_abort(vsc, vq, slot);
   1781  1.101  yamaguch 			m_freem(m);
   1782  1.101  yamaguch 			netq->netq_mbuf_load_failed.ev_count++;
   1783  1.101  yamaguch 			break;
   1784  1.101  yamaguch 		}
   1785  1.101  yamaguch 
   1786  1.101  yamaguch 		r = vioif_net_enqueue_rx(vsc, vq, slot, map);
   1787  1.101  yamaguch 		if (r != 0) {
   1788  1.101  yamaguch 			vioif_net_unload_mbuf(vsc, map);
   1789  1.101  yamaguch 			netq->netq_enqueue_failed.ev_count++;
   1790  1.101  yamaguch 			m_freem(m);
   1791  1.101  yamaguch 			/* slot already freed by vioif_net_enqueue_rx */
   1792  1.101  yamaguch 			break;
   1793  1.101  yamaguch 		}
   1794  1.101  yamaguch 
   1795  1.101  yamaguch 		ndone++;
   1796  1.101  yamaguch 	}
   1797  1.101  yamaguch 
   1798  1.101  yamaguch 	if (ndone > 0)
   1799  1.101  yamaguch 		vioif_notify(vsc, vq);
   1800  1.101  yamaguch }
   1801  1.101  yamaguch 
   1802  1.101  yamaguch /* dequeue received packets */
   1803  1.101  yamaguch static bool
   1804  1.101  yamaguch vioif_rx_deq_locked(struct vioif_softc *sc, struct virtio_softc *vsc,
   1805  1.101  yamaguch     struct vioif_netqueue *netq, u_int limit, size_t *ndeqp)
   1806  1.101  yamaguch {
   1807  1.101  yamaguch 	struct virtqueue *vq = netq->netq_vq;
   1808  1.101  yamaguch 	struct ifnet *ifp = &sc->sc_ethercom.ec_if;
   1809  1.101  yamaguch 	struct vioif_net_map *map;
   1810  1.101  yamaguch 	struct mbuf *m;
   1811  1.101  yamaguch 	int slot, len;
   1812  1.101  yamaguch 	bool more;
   1813  1.101  yamaguch 	size_t ndeq;
   1814  1.101  yamaguch 
   1815  1.101  yamaguch 	KASSERT(mutex_owned(&netq->netq_lock));
   1816  1.101  yamaguch 
   1817  1.101  yamaguch 	more = false;
   1818  1.101  yamaguch 	ndeq = 0;
   1819  1.101  yamaguch 
   1820  1.101  yamaguch 	if (virtio_vq_is_enqueued(vsc, vq) == false)
   1821  1.101  yamaguch 		goto done;
   1822  1.101  yamaguch 
   1823  1.101  yamaguch 	for (;;ndeq++) {
   1824  1.101  yamaguch 		if (ndeq >= limit) {
   1825  1.101  yamaguch 			more = true;
   1826  1.101  yamaguch 			break;
   1827  1.101  yamaguch 		}
   1828  1.101  yamaguch 
   1829  1.101  yamaguch 		if (virtio_dequeue(vsc, vq, &slot, &len) != 0)
   1830  1.101  yamaguch 			break;
   1831  1.101  yamaguch 
   1832  1.101  yamaguch 		map = &netq->netq_maps[slot];
   1833  1.101  yamaguch 		KASSERT(map->vnm_mbuf != NULL);
   1834  1.101  yamaguch 		m = vioif_net_dequeue_commit(vsc, vq, slot,
   1835  1.101  yamaguch 		    map, BUS_DMASYNC_POSTREAD);
   1836  1.101  yamaguch 		KASSERT(m != NULL);
   1837  1.101  yamaguch 
   1838  1.101  yamaguch 		m->m_len = m->m_pkthdr.len = len - sc->sc_hdr_size;
   1839  1.101  yamaguch 		m_set_rcvif(m, ifp);
   1840  1.101  yamaguch 		if_percpuq_enqueue(ifp->if_percpuq, m);
   1841  1.101  yamaguch 	}
   1842  1.101  yamaguch 
   1843  1.101  yamaguch done:
   1844  1.101  yamaguch 	if (ndeqp != NULL)
   1845  1.101  yamaguch 		*ndeqp = ndeq;
   1846  1.101  yamaguch 
   1847  1.101  yamaguch 	return more;
   1848  1.101  yamaguch }
   1849  1.101  yamaguch 
   1850  1.101  yamaguch static void
   1851  1.101  yamaguch vioif_rx_queue_clear(struct vioif_softc *sc, struct virtio_softc *vsc,
   1852  1.101  yamaguch     struct vioif_netqueue *netq)
   1853  1.101  yamaguch {
   1854  1.101  yamaguch 	struct vioif_net_map *map;
   1855  1.101  yamaguch 	struct mbuf *m;
   1856  1.101  yamaguch 	unsigned int i, vq_num;
   1857  1.101  yamaguch 	bool more;
   1858  1.101  yamaguch 
   1859  1.101  yamaguch 	mutex_enter(&netq->netq_lock);
   1860  1.101  yamaguch 
   1861  1.101  yamaguch 	vq_num = netq->netq_vq->vq_num;
   1862  1.101  yamaguch 	for (;;) {
   1863  1.101  yamaguch 		more = vioif_rx_deq_locked(sc, vsc, netq, vq_num, NULL);
   1864  1.101  yamaguch 		if (more == false)
   1865  1.101  yamaguch 			break;
   1866  1.101  yamaguch 	}
   1867  1.101  yamaguch 
   1868  1.101  yamaguch 	for (i = 0; i < vq_num; i++) {
   1869  1.101  yamaguch 		map = &netq->netq_maps[i];
   1870  1.101  yamaguch 
   1871  1.101  yamaguch 		m = map->vnm_mbuf;
   1872  1.101  yamaguch 		if (m == NULL)
   1873  1.101  yamaguch 			continue;
   1874  1.101  yamaguch 
   1875  1.101  yamaguch 		vioif_net_unload_mbuf(vsc, map);
   1876  1.101  yamaguch 		m_freem(m);
   1877  1.101  yamaguch 	}
   1878  1.101  yamaguch 	mutex_exit(&netq->netq_lock);
   1879  1.101  yamaguch }
   1880  1.101  yamaguch 
   1881  1.101  yamaguch static void
   1882  1.101  yamaguch vioif_rx_handle_locked(void *xnetq, u_int limit)
   1883  1.101  yamaguch {
   1884  1.101  yamaguch 	struct vioif_netqueue *netq = xnetq;
   1885  1.101  yamaguch 	struct virtqueue *vq = netq->netq_vq;
   1886  1.101  yamaguch 	struct virtio_softc *vsc = vq->vq_owner;
   1887  1.101  yamaguch 	struct vioif_softc *sc = device_private(virtio_child(vsc));
   1888  1.101  yamaguch 	bool more;
   1889  1.101  yamaguch 	int enqueued;
   1890  1.101  yamaguch 	size_t ndeq;
   1891  1.101  yamaguch 
   1892  1.101  yamaguch 	KASSERT(mutex_owned(&netq->netq_lock));
   1893  1.101  yamaguch 	KASSERT(!netq->netq_stopping);
   1894  1.101  yamaguch 
   1895  1.101  yamaguch 	more = vioif_rx_deq_locked(sc, vsc, netq, limit, &ndeq);
   1896  1.101  yamaguch 	if (ndeq > 0)
   1897  1.101  yamaguch 		vioif_populate_rx_mbufs_locked(sc, netq);
   1898  1.101  yamaguch 
   1899  1.101  yamaguch 	if (more) {
   1900  1.101  yamaguch 		vioif_net_sched_handle(sc, netq);
   1901  1.101  yamaguch 		return;
   1902  1.101  yamaguch 	}
   1903  1.101  yamaguch 
   1904  1.101  yamaguch 	enqueued = virtio_start_vq_intr(vsc, netq->netq_vq);
   1905  1.101  yamaguch 	if (enqueued != 0) {
   1906  1.101  yamaguch 		virtio_stop_vq_intr(vsc, netq->netq_vq);
   1907  1.101  yamaguch 		vioif_net_sched_handle(sc, netq);
   1908  1.101  yamaguch 		return;
   1909  1.101  yamaguch 	}
   1910  1.101  yamaguch 
   1911  1.101  yamaguch 	netq->netq_running_handle = false;
   1912  1.101  yamaguch }
   1913  1.101  yamaguch 
   1914  1.101  yamaguch static int
   1915  1.101  yamaguch vioif_rx_intr(void *arg)
   1916   1.99  yamaguch {
   1917  1.101  yamaguch 	struct vioif_netqueue *netq = arg;
   1918  1.101  yamaguch 	struct virtqueue *vq = netq->netq_vq;
   1919  1.101  yamaguch 	struct virtio_softc *vsc = vq->vq_owner;
   1920  1.101  yamaguch 	struct vioif_softc *sc = device_private(virtio_child(vsc));
   1921  1.101  yamaguch 	u_int limit;
   1922   1.99  yamaguch 
   1923  1.101  yamaguch 	mutex_enter(&netq->netq_lock);
   1924   1.99  yamaguch 
   1925  1.101  yamaguch 	/* handler is already running in softint/workqueue */
   1926  1.101  yamaguch 	if (netq->netq_running_handle)
   1927  1.101  yamaguch 		goto done;
   1928   1.99  yamaguch 
   1929  1.108  yamaguch 	if (netq->netq_stopping)
   1930  1.108  yamaguch 		goto done;
   1931  1.108  yamaguch 
   1932  1.101  yamaguch 	netq->netq_running_handle = true;
   1933   1.99  yamaguch 
   1934  1.101  yamaguch 	limit = sc->sc_rx_intr_process_limit;
   1935  1.101  yamaguch 	virtio_stop_vq_intr(vsc, vq);
   1936  1.101  yamaguch 	vioif_rx_handle_locked(netq, limit);
   1937   1.99  yamaguch 
   1938  1.101  yamaguch done:
   1939  1.101  yamaguch 	mutex_exit(&netq->netq_lock);
   1940  1.101  yamaguch 	return 1;
   1941   1.99  yamaguch }
   1942   1.99  yamaguch 
   1943   1.99  yamaguch static void
   1944  1.101  yamaguch vioif_rx_handle(void *xnetq)
   1945   1.99  yamaguch {
   1946  1.101  yamaguch 	struct vioif_netqueue *netq = xnetq;
   1947  1.101  yamaguch 	struct virtqueue *vq = netq->netq_vq;
   1948  1.101  yamaguch 	struct virtio_softc *vsc = vq->vq_owner;
   1949  1.101  yamaguch 	struct vioif_softc *sc = device_private(virtio_child(vsc));
   1950  1.101  yamaguch 	u_int limit;
   1951   1.99  yamaguch 
   1952  1.101  yamaguch 	mutex_enter(&netq->netq_lock);
   1953   1.99  yamaguch 
   1954  1.101  yamaguch 	KASSERT(netq->netq_running_handle);
   1955   1.99  yamaguch 
   1956  1.101  yamaguch 	if (netq->netq_stopping) {
   1957  1.101  yamaguch 		netq->netq_running_handle = false;
   1958  1.101  yamaguch 		goto done;
   1959   1.99  yamaguch 	}
   1960   1.99  yamaguch 
   1961  1.101  yamaguch 	limit = sc->sc_rx_process_limit;
   1962  1.101  yamaguch 	vioif_rx_handle_locked(netq, limit);
   1963   1.99  yamaguch 
   1964  1.101  yamaguch done:
   1965  1.101  yamaguch 	mutex_exit(&netq->netq_lock);
   1966   1.99  yamaguch }
   1967   1.99  yamaguch 
   1968    1.1   hannken /*
   1969  1.110    andvar  * Transmission implementation
   1970    1.1   hannken  */
   1971  1.101  yamaguch /* enqueue mbufs to send */
   1972    1.1   hannken static void
   1973  1.101  yamaguch vioif_send_common_locked(struct ifnet *ifp, struct vioif_netqueue *netq,
   1974  1.101  yamaguch     bool is_transmit)
   1975   1.12     ozaki {
   1976  1.101  yamaguch 	struct vioif_softc *sc = ifp->if_softc;
   1977  1.101  yamaguch 	struct virtio_softc *vsc = sc->sc_virtio;
   1978   1.98  yamaguch 	struct virtqueue *vq = netq->netq_vq;
   1979  1.101  yamaguch 	struct vioif_tx_context *txc;
   1980   1.95  yamaguch 	struct vioif_net_map *map;
   1981   1.89  yamaguch 	struct mbuf *m;
   1982  1.101  yamaguch 	int queued = 0;
   1983    1.1   hannken 
   1984   1.98  yamaguch 	KASSERT(mutex_owned(&netq->netq_lock));
   1985   1.98  yamaguch 
   1986  1.101  yamaguch 	if (netq->netq_stopping ||
   1987  1.101  yamaguch 	    !ISSET(ifp->if_flags, IFF_RUNNING))
   1988  1.101  yamaguch 		return;
   1989  1.101  yamaguch 
   1990  1.101  yamaguch 	txc = netq->netq_ctx;
   1991  1.101  yamaguch 
   1992  1.102  yamaguch 	if (!txc->txc_link_active ||
   1993  1.102  yamaguch 	    txc->txc_no_free_slots)
   1994  1.101  yamaguch 		return;
   1995    1.7     ozaki 
   1996  1.101  yamaguch 	for (;;) {
   1997  1.101  yamaguch 		int slot, r;
   1998    1.1   hannken 		r = virtio_enqueue_prep(vsc, vq, &slot);
   1999  1.102  yamaguch 		if (r == EAGAIN) {
   2000  1.102  yamaguch 			txc->txc_no_free_slots = true;
   2001    1.1   hannken 			break;
   2002  1.102  yamaguch 		}
   2003   1.97  yamaguch 		if (__predict_false(r != 0))
   2004  1.101  yamaguch 			panic("enqueue_prep for tx buffers");
   2005  1.101  yamaguch 
   2006  1.101  yamaguch 		if (is_transmit)
   2007  1.101  yamaguch 			m = pcq_get(txc->txc_intrq);
   2008  1.101  yamaguch 		else
   2009  1.101  yamaguch 			IFQ_DEQUEUE(&ifp->if_snd, m);
   2010   1.89  yamaguch 
   2011   1.89  yamaguch 		if (m == NULL) {
   2012   1.91  yamaguch 			virtio_enqueue_abort(vsc, vq, slot);
   2013   1.91  yamaguch 			break;
   2014   1.91  yamaguch 		}
   2015   1.89  yamaguch 
   2016  1.101  yamaguch 		map = &netq->netq_maps[slot];
   2017  1.101  yamaguch 		KASSERT(map->vnm_mbuf == NULL);
   2018   1.89  yamaguch 
   2019  1.101  yamaguch 		r = vioif_net_load_mbuf(vsc, map, m, BUS_DMA_WRITE);
   2020   1.91  yamaguch 		if (r != 0) {
   2021  1.101  yamaguch 			/* maybe just too fragmented */
   2022  1.101  yamaguch 			struct mbuf *newm;
   2023  1.101  yamaguch 
   2024  1.101  yamaguch 			newm = m_defrag(m, M_NOWAIT);
   2025  1.101  yamaguch 			if (newm != NULL) {
   2026  1.101  yamaguch 				m = newm;
   2027  1.101  yamaguch 				r = vioif_net_load_mbuf(vsc, map, m,
   2028  1.101  yamaguch 				    BUS_DMA_WRITE);
   2029  1.101  yamaguch 			} else {
   2030  1.101  yamaguch 				txc->txc_defrag_failed.ev_count++;
   2031  1.101  yamaguch 				r = -1;
   2032  1.101  yamaguch 			}
   2033  1.101  yamaguch 
   2034  1.101  yamaguch 			if (r != 0) {
   2035  1.101  yamaguch 				netq->netq_mbuf_load_failed.ev_count++;
   2036  1.101  yamaguch 				m_freem(m);
   2037  1.101  yamaguch 				if_statinc(ifp, if_oerrors);
   2038  1.101  yamaguch 				virtio_enqueue_abort(vsc, vq, slot);
   2039  1.101  yamaguch 				continue;
   2040  1.101  yamaguch 			}
   2041    1.1   hannken 		}
   2042   1.89  yamaguch 
   2043  1.101  yamaguch 		memset(map->vnm_hdr, 0, sc->sc_hdr_size);
   2044  1.101  yamaguch 
   2045  1.101  yamaguch 		r = vioif_net_enqueue_tx(vsc, vq, slot, map);
   2046    1.1   hannken 		if (r != 0) {
   2047  1.101  yamaguch 			netq->netq_enqueue_failed.ev_count++;
   2048   1.99  yamaguch 			vioif_net_unload_mbuf(vsc, map);
   2049   1.89  yamaguch 			m_freem(m);
   2050  1.101  yamaguch 			/* slot already freed by vioif_net_enqueue_tx */
   2051  1.101  yamaguch 
   2052  1.101  yamaguch 			if_statinc(ifp, if_oerrors);
   2053  1.101  yamaguch 			continue;
   2054    1.1   hannken 		}
   2055   1.95  yamaguch 
   2056  1.101  yamaguch 		queued++;
   2057  1.101  yamaguch 		bpf_mtap(ifp, m, BPF_D_OUT);
   2058    1.1   hannken 	}
   2059   1.99  yamaguch 
   2060  1.101  yamaguch 	if (queued > 0) {
   2061   1.99  yamaguch 		vioif_notify(vsc, vq);
   2062  1.101  yamaguch 		ifp->if_timer = 5;
   2063   1.93  yamaguch 	}
   2064    1.7     ozaki }
   2065    1.7     ozaki 
   2066  1.101  yamaguch /* dequeue sent mbufs */
   2067   1.55  yamaguch static bool
   2068  1.101  yamaguch vioif_tx_deq_locked(struct vioif_softc *sc, struct virtio_softc *vsc,
   2069  1.102  yamaguch     struct vioif_netqueue *netq, u_int limit, size_t *ndeqp)
   2070    1.7     ozaki {
   2071   1.98  yamaguch 	struct virtqueue *vq = netq->netq_vq;
   2072    1.1   hannken 	struct ifnet *ifp = &sc->sc_ethercom.ec_if;
   2073   1.95  yamaguch 	struct vioif_net_map *map;
   2074    1.1   hannken 	struct mbuf *m;
   2075    1.1   hannken 	int slot, len;
   2076  1.102  yamaguch 	bool more;
   2077  1.102  yamaguch 	size_t ndeq;
   2078    1.1   hannken 
   2079   1.98  yamaguch 	KASSERT(mutex_owned(&netq->netq_lock));
   2080    1.7     ozaki 
   2081  1.102  yamaguch 	more = false;
   2082  1.102  yamaguch 	ndeq = 0;
   2083  1.102  yamaguch 
   2084   1.54  yamaguch 	if (virtio_vq_is_enqueued(vsc, vq) == false)
   2085  1.102  yamaguch 		goto done;
   2086   1.55  yamaguch 
   2087  1.102  yamaguch 	for (;;ndeq++) {
   2088  1.101  yamaguch 		if (limit-- == 0) {
   2089   1.55  yamaguch 			more = true;
   2090   1.55  yamaguch 			break;
   2091   1.55  yamaguch 		}
   2092   1.55  yamaguch 
   2093   1.55  yamaguch 		if (virtio_dequeue(vsc, vq, &slot, &len) != 0)
   2094   1.55  yamaguch 			break;
   2095   1.55  yamaguch 
   2096   1.98  yamaguch 		map = &netq->netq_maps[slot];
   2097   1.95  yamaguch 		KASSERT(map->vnm_mbuf != NULL);
   2098   1.99  yamaguch 		m = vioif_net_dequeue_commit(vsc, vq, slot,
   2099  1.101  yamaguch 		    map, BUS_DMASYNC_POSTWRITE);
   2100   1.99  yamaguch 		KASSERT(m != NULL);
   2101   1.95  yamaguch 
   2102  1.101  yamaguch 		if_statinc(ifp, if_opackets);
   2103  1.101  yamaguch 		m_freem(m);
   2104   1.70     skrll 	}
   2105   1.87  yamaguch 
   2106  1.102  yamaguch done:
   2107  1.102  yamaguch 	if (ndeqp != NULL)
   2108  1.102  yamaguch 		*ndeqp = ndeq;
   2109  1.101  yamaguch 	return more;
   2110   1.66   reinoud }
   2111   1.66   reinoud 
   2112  1.101  yamaguch static void
   2113  1.101  yamaguch vioif_tx_queue_clear(struct vioif_softc *sc, struct virtio_softc *vsc,
   2114  1.101  yamaguch     struct vioif_netqueue *netq)
   2115    1.1   hannken {
   2116  1.102  yamaguch 	struct vioif_tx_context *txc;
   2117  1.101  yamaguch 	struct vioif_net_map *map;
   2118  1.101  yamaguch 	struct mbuf *m;
   2119  1.101  yamaguch 	unsigned int i, vq_num;
   2120  1.101  yamaguch 	bool more;
   2121   1.55  yamaguch 
   2122   1.98  yamaguch 	mutex_enter(&netq->netq_lock);
   2123    1.7     ozaki 
   2124  1.102  yamaguch 	txc = netq->netq_ctx;
   2125  1.101  yamaguch 	vq_num = netq->netq_vq->vq_num;
   2126  1.102  yamaguch 
   2127  1.101  yamaguch 	for (;;) {
   2128  1.102  yamaguch 		more = vioif_tx_deq_locked(sc, vsc, netq, vq_num, NULL);
   2129  1.101  yamaguch 		if (more == false)
   2130  1.101  yamaguch 			break;
   2131  1.101  yamaguch 	}
   2132   1.55  yamaguch 
   2133  1.101  yamaguch 	for (i = 0; i < vq_num; i++) {
   2134  1.101  yamaguch 		map = &netq->netq_maps[i];
   2135   1.55  yamaguch 
   2136  1.101  yamaguch 		m = map->vnm_mbuf;
   2137  1.101  yamaguch 		if (m == NULL)
   2138  1.101  yamaguch 			continue;
   2139   1.55  yamaguch 
   2140  1.101  yamaguch 		vioif_net_unload_mbuf(vsc, map);
   2141  1.101  yamaguch 		m_freem(m);
   2142  1.101  yamaguch 	}
   2143  1.102  yamaguch 
   2144  1.102  yamaguch 	txc->txc_no_free_slots = false;
   2145  1.102  yamaguch 
   2146   1.98  yamaguch 	mutex_exit(&netq->netq_lock);
   2147   1.55  yamaguch }
   2148   1.55  yamaguch 
   2149   1.55  yamaguch static void
   2150  1.101  yamaguch vioif_start_locked(struct ifnet *ifp, struct vioif_netqueue *netq)
   2151   1.55  yamaguch {
   2152   1.55  yamaguch 
   2153  1.101  yamaguch 	/*
   2154  1.101  yamaguch 	 * ifp->if_obytes and ifp->if_omcasts are added in if_transmit()@if.c.
   2155  1.101  yamaguch 	 */
   2156  1.101  yamaguch 	vioif_send_common_locked(ifp, netq, false);
   2157  1.101  yamaguch 
   2158  1.101  yamaguch }
   2159   1.85  yamaguch 
   2160  1.101  yamaguch static void
   2161  1.101  yamaguch vioif_transmit_locked(struct ifnet *ifp, struct vioif_netqueue *netq)
   2162  1.101  yamaguch {
   2163   1.55  yamaguch 
   2164  1.101  yamaguch 	vioif_send_common_locked(ifp, netq, true);
   2165  1.101  yamaguch }
   2166    1.1   hannken 
   2167  1.101  yamaguch static void
   2168  1.101  yamaguch vioif_deferred_transmit(void *arg)
   2169  1.101  yamaguch {
   2170  1.101  yamaguch 	struct vioif_netqueue *netq = arg;
   2171  1.101  yamaguch 	struct virtio_softc *vsc = netq->netq_vq->vq_owner;
   2172  1.101  yamaguch 	struct vioif_softc *sc = device_private(virtio_child(vsc));
   2173  1.101  yamaguch 	struct ifnet *ifp = &sc->sc_ethercom.ec_if;
   2174    1.1   hannken 
   2175  1.101  yamaguch 	mutex_enter(&netq->netq_lock);
   2176  1.101  yamaguch 	vioif_send_common_locked(ifp, netq, true);
   2177   1.98  yamaguch 	mutex_exit(&netq->netq_lock);
   2178    1.1   hannken }
   2179    1.1   hannken 
   2180   1.66   reinoud static void
   2181   1.98  yamaguch vioif_tx_handle_locked(struct vioif_netqueue *netq, u_int limit)
   2182   1.66   reinoud {
   2183   1.98  yamaguch 	struct virtqueue *vq = netq->netq_vq;
   2184   1.98  yamaguch 	struct vioif_tx_context *txc = netq->netq_ctx;
   2185   1.66   reinoud 	struct virtio_softc *vsc = vq->vq_owner;
   2186   1.66   reinoud 	struct vioif_softc *sc = device_private(virtio_child(vsc));
   2187   1.66   reinoud 	struct ifnet *ifp = &sc->sc_ethercom.ec_if;
   2188   1.66   reinoud 	bool more;
   2189   1.87  yamaguch 	int enqueued;
   2190  1.102  yamaguch 	size_t ndeq;
   2191   1.66   reinoud 
   2192   1.98  yamaguch 	KASSERT(mutex_owned(&netq->netq_lock));
   2193   1.98  yamaguch 	KASSERT(!netq->netq_stopping);
   2194   1.66   reinoud 
   2195  1.102  yamaguch 	more = vioif_tx_deq_locked(sc, vsc, netq, limit, &ndeq);
   2196  1.102  yamaguch 	if (txc->txc_no_free_slots && ndeq > 0) {
   2197  1.102  yamaguch 		txc->txc_no_free_slots = false;
   2198  1.102  yamaguch 		softint_schedule(txc->txc_deferred_transmit);
   2199  1.102  yamaguch 	}
   2200  1.102  yamaguch 
   2201   1.66   reinoud 	if (more) {
   2202   1.98  yamaguch 		vioif_net_sched_handle(sc, netq);
   2203   1.66   reinoud 		return;
   2204   1.66   reinoud 	}
   2205   1.66   reinoud 
   2206   1.87  yamaguch 	enqueued = (virtio_features(vsc) & VIRTIO_F_RING_EVENT_IDX) ?
   2207   1.87  yamaguch 	    virtio_postpone_intr_smart(vsc, vq):
   2208   1.87  yamaguch 	    virtio_start_vq_intr(vsc, vq);
   2209   1.87  yamaguch 	if (enqueued != 0) {
   2210   1.87  yamaguch 		virtio_stop_vq_intr(vsc, vq);
   2211   1.98  yamaguch 		vioif_net_sched_handle(sc, netq);
   2212   1.66   reinoud 		return;
   2213   1.66   reinoud 	}
   2214   1.66   reinoud 
   2215   1.98  yamaguch 	netq->netq_running_handle = false;
   2216   1.84  yamaguch 
   2217   1.66   reinoud 	/* for ALTQ */
   2218  1.102  yamaguch 	if (netq == &sc->sc_netqs[VIOIF_NETQ_TXQID(0)])
   2219   1.66   reinoud 		if_schedule_deferred_start(ifp);
   2220  1.102  yamaguch 
   2221   1.98  yamaguch 	softint_schedule(txc->txc_deferred_transmit);
   2222   1.66   reinoud }
   2223   1.66   reinoud 
   2224    1.1   hannken static int
   2225   1.54  yamaguch vioif_tx_intr(void *arg)
   2226    1.1   hannken {
   2227   1.98  yamaguch 	struct vioif_netqueue *netq = arg;
   2228   1.98  yamaguch 	struct virtqueue *vq = netq->netq_vq;
   2229    1.1   hannken 	struct virtio_softc *vsc = vq->vq_owner;
   2230   1.32  jdolecek 	struct vioif_softc *sc = device_private(virtio_child(vsc));
   2231   1.55  yamaguch 	u_int limit;
   2232   1.55  yamaguch 
   2233   1.98  yamaguch 	mutex_enter(&netq->netq_lock);
   2234    1.7     ozaki 
   2235   1.84  yamaguch 	/* tx handler is already running in softint/workqueue */
   2236   1.98  yamaguch 	if (netq->netq_running_handle)
   2237   1.84  yamaguch 		goto done;
   2238    1.7     ozaki 
   2239   1.98  yamaguch 	if (netq->netq_stopping)
   2240   1.84  yamaguch 		goto done;
   2241   1.55  yamaguch 
   2242   1.98  yamaguch 	netq->netq_running_handle = true;
   2243   1.55  yamaguch 
   2244   1.84  yamaguch 	virtio_stop_vq_intr(vsc, vq);
   2245   1.98  yamaguch 	netq->netq_workqueue = sc->sc_txrx_workqueue_sysctl;
   2246   1.98  yamaguch 	limit = sc->sc_tx_intr_process_limit;
   2247   1.98  yamaguch 	vioif_tx_handle_locked(netq, limit);
   2248    1.7     ozaki 
   2249   1.84  yamaguch done:
   2250   1.98  yamaguch 	mutex_exit(&netq->netq_lock);
   2251   1.55  yamaguch 	return 1;
   2252   1.55  yamaguch }
   2253   1.55  yamaguch 
   2254   1.55  yamaguch static void
   2255   1.98  yamaguch vioif_tx_handle(void *xnetq)
   2256   1.55  yamaguch {
   2257   1.98  yamaguch 	struct vioif_netqueue *netq = xnetq;
   2258   1.98  yamaguch 	struct virtqueue *vq = netq->netq_vq;
   2259   1.55  yamaguch 	struct virtio_softc *vsc = vq->vq_owner;
   2260   1.55  yamaguch 	struct vioif_softc *sc = device_private(virtio_child(vsc));
   2261   1.55  yamaguch 	u_int limit;
   2262   1.55  yamaguch 
   2263   1.98  yamaguch 	mutex_enter(&netq->netq_lock);
   2264   1.85  yamaguch 
   2265   1.98  yamaguch 	KASSERT(netq->netq_running_handle);
   2266   1.85  yamaguch 
   2267   1.98  yamaguch 	if (netq->netq_stopping) {
   2268   1.98  yamaguch 		netq->netq_running_handle = false;
   2269   1.85  yamaguch 		goto done;
   2270   1.85  yamaguch 	}
   2271   1.85  yamaguch 
   2272   1.55  yamaguch 	limit = sc->sc_tx_process_limit;
   2273   1.98  yamaguch 	vioif_tx_handle_locked(netq, limit);
   2274   1.55  yamaguch 
   2275   1.85  yamaguch done:
   2276   1.98  yamaguch 	mutex_exit(&netq->netq_lock);
   2277    1.7     ozaki }
   2278    1.7     ozaki 
   2279    1.1   hannken /*
   2280    1.1   hannken  * Control vq
   2281    1.1   hannken  */
   2282    1.1   hannken /* issue a VIRTIO_NET_CTRL_RX class command and wait for completion */
   2283   1.44  yamaguch static void
   2284   1.44  yamaguch vioif_ctrl_acquire(struct vioif_softc *sc)
   2285    1.1   hannken {
   2286   1.43  yamaguch 	struct vioif_ctrlqueue *ctrlq = &sc->sc_ctrlq;
   2287    1.1   hannken 
   2288   1.43  yamaguch 	mutex_enter(&ctrlq->ctrlq_wait_lock);
   2289   1.43  yamaguch 	while (ctrlq->ctrlq_inuse != FREE)
   2290   1.43  yamaguch 		cv_wait(&ctrlq->ctrlq_wait, &ctrlq->ctrlq_wait_lock);
   2291   1.43  yamaguch 	ctrlq->ctrlq_inuse = INUSE;
   2292   1.44  yamaguch 	ctrlq->ctrlq_owner = curlwp;
   2293   1.44  yamaguch 	mutex_exit(&ctrlq->ctrlq_wait_lock);
   2294   1.44  yamaguch }
   2295   1.44  yamaguch 
   2296   1.44  yamaguch static void
   2297   1.44  yamaguch vioif_ctrl_release(struct vioif_softc *sc)
   2298   1.44  yamaguch {
   2299   1.44  yamaguch 	struct vioif_ctrlqueue *ctrlq = &sc->sc_ctrlq;
   2300   1.44  yamaguch 
   2301   1.44  yamaguch 	KASSERT(ctrlq->ctrlq_inuse != FREE);
   2302   1.44  yamaguch 	KASSERT(ctrlq->ctrlq_owner == curlwp);
   2303   1.44  yamaguch 
   2304   1.44  yamaguch 	mutex_enter(&ctrlq->ctrlq_wait_lock);
   2305   1.44  yamaguch 	ctrlq->ctrlq_inuse = FREE;
   2306   1.46  yamaguch 	ctrlq->ctrlq_owner = NULL;
   2307   1.44  yamaguch 	cv_signal(&ctrlq->ctrlq_wait);
   2308   1.43  yamaguch 	mutex_exit(&ctrlq->ctrlq_wait_lock);
   2309   1.44  yamaguch }
   2310   1.44  yamaguch 
   2311   1.44  yamaguch static int
   2312   1.44  yamaguch vioif_ctrl_load_cmdspec(struct vioif_softc *sc,
   2313   1.44  yamaguch     struct vioif_ctrl_cmdspec *specs, int nspecs)
   2314   1.44  yamaguch {
   2315   1.44  yamaguch 	struct virtio_softc *vsc = sc->sc_virtio;
   2316   1.44  yamaguch 	int i, r, loaded;
   2317   1.44  yamaguch 
   2318   1.44  yamaguch 	loaded = 0;
   2319   1.44  yamaguch 	for (i = 0; i < nspecs; i++) {
   2320   1.44  yamaguch 		r = bus_dmamap_load(virtio_dmat(vsc),
   2321   1.44  yamaguch 		    specs[i].dmamap, specs[i].buf, specs[i].bufsize,
   2322   1.48   msaitoh 		    NULL, BUS_DMA_WRITE | BUS_DMA_NOWAIT);
   2323   1.44  yamaguch 		if (r) {
   2324   1.63  yamaguch 			sc->sc_ctrlq.ctrlq_cmd_load_failed.ev_count++;
   2325   1.44  yamaguch 			goto err;
   2326   1.44  yamaguch 		}
   2327   1.44  yamaguch 		loaded++;
   2328   1.44  yamaguch 
   2329   1.44  yamaguch 	}
   2330   1.44  yamaguch 
   2331   1.44  yamaguch 	return r;
   2332   1.44  yamaguch 
   2333   1.44  yamaguch err:
   2334   1.44  yamaguch 	for (i = 0; i < loaded; i++) {
   2335   1.44  yamaguch 		bus_dmamap_unload(virtio_dmat(vsc), specs[i].dmamap);
   2336   1.44  yamaguch 	}
   2337   1.44  yamaguch 
   2338   1.44  yamaguch 	return r;
   2339   1.44  yamaguch }
   2340   1.44  yamaguch 
   2341   1.44  yamaguch static void
   2342   1.44  yamaguch vioif_ctrl_unload_cmdspec(struct vioif_softc *sc,
   2343   1.44  yamaguch     struct vioif_ctrl_cmdspec *specs, int nspecs)
   2344   1.44  yamaguch {
   2345   1.44  yamaguch 	struct virtio_softc *vsc = sc->sc_virtio;
   2346   1.44  yamaguch 	int i;
   2347   1.44  yamaguch 
   2348   1.44  yamaguch 	for (i = 0; i < nspecs; i++) {
   2349   1.44  yamaguch 		bus_dmamap_unload(virtio_dmat(vsc), specs[i].dmamap);
   2350   1.44  yamaguch 	}
   2351   1.44  yamaguch }
   2352   1.44  yamaguch 
   2353   1.44  yamaguch static int
   2354   1.44  yamaguch vioif_ctrl_send_command(struct vioif_softc *sc, uint8_t class, uint8_t cmd,
   2355   1.44  yamaguch     struct vioif_ctrl_cmdspec *specs, int nspecs)
   2356   1.44  yamaguch {
   2357   1.44  yamaguch 	struct vioif_ctrlqueue *ctrlq = &sc->sc_ctrlq;
   2358   1.44  yamaguch 	struct virtqueue *vq = ctrlq->ctrlq_vq;
   2359   1.44  yamaguch 	struct virtio_softc *vsc = sc->sc_virtio;
   2360   1.44  yamaguch 	int i, r, slot;
   2361   1.43  yamaguch 
   2362   1.44  yamaguch 	ctrlq->ctrlq_cmd->class = class;
   2363   1.43  yamaguch 	ctrlq->ctrlq_cmd->command = cmd;
   2364    1.1   hannken 
   2365   1.43  yamaguch 	bus_dmamap_sync(virtio_dmat(vsc), ctrlq->ctrlq_cmd_dmamap,
   2366   1.50  christos 	    0, sizeof(struct virtio_net_ctrl_cmd), BUS_DMASYNC_PREWRITE);
   2367   1.44  yamaguch 	for (i = 0; i < nspecs; i++) {
   2368   1.44  yamaguch 		bus_dmamap_sync(virtio_dmat(vsc), specs[i].dmamap,
   2369   1.50  christos 		    0, specs[i].bufsize, BUS_DMASYNC_PREWRITE);
   2370   1.44  yamaguch 	}
   2371   1.43  yamaguch 	bus_dmamap_sync(virtio_dmat(vsc), ctrlq->ctrlq_status_dmamap,
   2372   1.50  christos 	    0, sizeof(struct virtio_net_ctrl_status), BUS_DMASYNC_PREREAD);
   2373    1.1   hannken 
   2374   1.66   reinoud 	/* we need to explicitly (re)start vq intr when using RING EVENT IDX */
   2375   1.66   reinoud 	if (virtio_features(vsc) & VIRTIO_F_RING_EVENT_IDX)
   2376   1.66   reinoud 		virtio_start_vq_intr(vsc, ctrlq->ctrlq_vq);
   2377   1.66   reinoud 
   2378    1.1   hannken 	r = virtio_enqueue_prep(vsc, vq, &slot);
   2379    1.1   hannken 	if (r != 0)
   2380    1.1   hannken 		panic("%s: control vq busy!?", device_xname(sc->sc_dev));
   2381   1.44  yamaguch 	r = virtio_enqueue_reserve(vsc, vq, slot, nspecs + 2);
   2382    1.1   hannken 	if (r != 0)
   2383    1.1   hannken 		panic("%s: control vq busy!?", device_xname(sc->sc_dev));
   2384   1.43  yamaguch 	virtio_enqueue(vsc, vq, slot, ctrlq->ctrlq_cmd_dmamap, true);
   2385   1.44  yamaguch 	for (i = 0; i < nspecs; i++) {
   2386   1.44  yamaguch 		virtio_enqueue(vsc, vq, slot, specs[i].dmamap, true);
   2387   1.44  yamaguch 	}
   2388   1.43  yamaguch 	virtio_enqueue(vsc, vq, slot, ctrlq->ctrlq_status_dmamap, false);
   2389    1.1   hannken 	virtio_enqueue_commit(vsc, vq, slot, true);
   2390    1.1   hannken 
   2391    1.1   hannken 	/* wait for done */
   2392   1.43  yamaguch 	mutex_enter(&ctrlq->ctrlq_wait_lock);
   2393   1.43  yamaguch 	while (ctrlq->ctrlq_inuse != DONE)
   2394   1.43  yamaguch 		cv_wait(&ctrlq->ctrlq_wait, &ctrlq->ctrlq_wait_lock);
   2395   1.43  yamaguch 	mutex_exit(&ctrlq->ctrlq_wait_lock);
   2396  1.109    andvar 	/* already dequeued */
   2397    1.1   hannken 
   2398   1.43  yamaguch 	bus_dmamap_sync(virtio_dmat(vsc), ctrlq->ctrlq_cmd_dmamap, 0,
   2399   1.50  christos 	    sizeof(struct virtio_net_ctrl_cmd), BUS_DMASYNC_POSTWRITE);
   2400   1.44  yamaguch 	for (i = 0; i < nspecs; i++) {
   2401   1.44  yamaguch 		bus_dmamap_sync(virtio_dmat(vsc), specs[i].dmamap, 0,
   2402   1.50  christos 		    specs[i].bufsize, BUS_DMASYNC_POSTWRITE);
   2403   1.44  yamaguch 	}
   2404   1.43  yamaguch 	bus_dmamap_sync(virtio_dmat(vsc), ctrlq->ctrlq_status_dmamap, 0,
   2405   1.50  christos 	    sizeof(struct virtio_net_ctrl_status), BUS_DMASYNC_POSTREAD);
   2406    1.1   hannken 
   2407   1.43  yamaguch 	if (ctrlq->ctrlq_status->ack == VIRTIO_NET_OK)
   2408    1.1   hannken 		r = 0;
   2409    1.1   hannken 	else {
   2410   1.63  yamaguch 		device_printf(sc->sc_dev, "failed setting rx mode\n");
   2411   1.63  yamaguch 		sc->sc_ctrlq.ctrlq_cmd_failed.ev_count++;
   2412    1.1   hannken 		r = EIO;
   2413    1.1   hannken 	}
   2414    1.1   hannken 
   2415   1.44  yamaguch 	return r;
   2416   1.44  yamaguch }
   2417   1.44  yamaguch 
   2418  1.101  yamaguch /* ctrl vq interrupt; wake up the command issuer */
   2419  1.101  yamaguch static int
   2420  1.101  yamaguch vioif_ctrl_intr(void *arg)
   2421  1.101  yamaguch {
   2422  1.101  yamaguch 	struct vioif_ctrlqueue *ctrlq = arg;
   2423  1.101  yamaguch 	struct virtqueue *vq = ctrlq->ctrlq_vq;
   2424  1.101  yamaguch 	struct virtio_softc *vsc = vq->vq_owner;
   2425  1.101  yamaguch 	int r, slot;
   2426  1.101  yamaguch 
   2427  1.101  yamaguch 	if (virtio_vq_is_enqueued(vsc, vq) == false)
   2428  1.101  yamaguch 		return 0;
   2429  1.101  yamaguch 
   2430  1.101  yamaguch 	r = virtio_dequeue(vsc, vq, &slot, NULL);
   2431  1.101  yamaguch 	if (r == ENOENT)
   2432  1.101  yamaguch 		return 0;
   2433  1.101  yamaguch 	virtio_dequeue_commit(vsc, vq, slot);
   2434  1.101  yamaguch 
   2435  1.101  yamaguch 	mutex_enter(&ctrlq->ctrlq_wait_lock);
   2436  1.101  yamaguch 	ctrlq->ctrlq_inuse = DONE;
   2437  1.101  yamaguch 	cv_signal(&ctrlq->ctrlq_wait);
   2438  1.101  yamaguch 	mutex_exit(&ctrlq->ctrlq_wait_lock);
   2439  1.101  yamaguch 
   2440  1.101  yamaguch 	return 1;
   2441  1.101  yamaguch }
   2442  1.101  yamaguch 
   2443   1.44  yamaguch static int
   2444   1.44  yamaguch vioif_ctrl_rx(struct vioif_softc *sc, int cmd, bool onoff)
   2445   1.44  yamaguch {
   2446   1.44  yamaguch 	struct virtio_net_ctrl_rx *rx = sc->sc_ctrlq.ctrlq_rx;
   2447   1.44  yamaguch 	struct vioif_ctrl_cmdspec specs[1];
   2448   1.44  yamaguch 	int r;
   2449   1.44  yamaguch 
   2450   1.44  yamaguch 	if (!sc->sc_has_ctrl)
   2451   1.44  yamaguch 		return ENOTSUP;
   2452   1.44  yamaguch 
   2453   1.44  yamaguch 	vioif_ctrl_acquire(sc);
   2454   1.44  yamaguch 
   2455   1.44  yamaguch 	rx->onoff = onoff;
   2456   1.44  yamaguch 	specs[0].dmamap = sc->sc_ctrlq.ctrlq_rx_dmamap;
   2457   1.44  yamaguch 	specs[0].buf = rx;
   2458   1.44  yamaguch 	specs[0].bufsize = sizeof(*rx);
   2459   1.44  yamaguch 
   2460   1.44  yamaguch 	r = vioif_ctrl_send_command(sc, VIRTIO_NET_CTRL_RX, cmd,
   2461   1.44  yamaguch 	    specs, __arraycount(specs));
   2462    1.3  christos 
   2463   1.44  yamaguch 	vioif_ctrl_release(sc);
   2464    1.1   hannken 	return r;
   2465    1.1   hannken }
   2466    1.1   hannken 
   2467    1.1   hannken static int
   2468    1.1   hannken vioif_set_promisc(struct vioif_softc *sc, bool onoff)
   2469    1.1   hannken {
   2470   1.50  christos 	return vioif_ctrl_rx(sc, VIRTIO_NET_CTRL_RX_PROMISC, onoff);
   2471    1.1   hannken }
   2472    1.1   hannken 
   2473    1.1   hannken static int
   2474    1.1   hannken vioif_set_allmulti(struct vioif_softc *sc, bool onoff)
   2475    1.1   hannken {
   2476   1.50  christos 	return vioif_ctrl_rx(sc, VIRTIO_NET_CTRL_RX_ALLMULTI, onoff);
   2477    1.1   hannken }
   2478    1.1   hannken 
   2479    1.1   hannken static int
   2480  1.101  yamaguch vioif_ctrl_mq_vq_pairs_set(struct vioif_softc *sc, int nvq_pairs)
   2481    1.1   hannken {
   2482  1.101  yamaguch 	struct virtio_net_ctrl_mq *mq = sc->sc_ctrlq.ctrlq_mq;
   2483  1.101  yamaguch 	struct vioif_ctrl_cmdspec specs[1];
   2484   1.44  yamaguch 	int r;
   2485   1.44  yamaguch 
   2486   1.32  jdolecek 	if (!sc->sc_has_ctrl)
   2487    1.1   hannken 		return ENOTSUP;
   2488    1.1   hannken 
   2489  1.101  yamaguch 	if (nvq_pairs <= 1)
   2490  1.101  yamaguch 		return EINVAL;
   2491  1.101  yamaguch 
   2492   1.44  yamaguch 	vioif_ctrl_acquire(sc);
   2493    1.1   hannken 
   2494  1.101  yamaguch 	mq->virtqueue_pairs = virtio_rw16(sc->sc_virtio, nvq_pairs);
   2495  1.101  yamaguch 	specs[0].dmamap = sc->sc_ctrlq.ctrlq_mq_dmamap;
   2496  1.101  yamaguch 	specs[0].buf = mq;
   2497  1.101  yamaguch 	specs[0].bufsize = sizeof(*mq);
   2498    1.1   hannken 
   2499   1.44  yamaguch 	r = vioif_ctrl_send_command(sc,
   2500  1.101  yamaguch 	    VIRTIO_NET_CTRL_MQ, VIRTIO_NET_CTRL_MQ_VQ_PAIRS_SET,
   2501  1.101  yamaguch 	    specs, __arraycount(specs));
   2502    1.1   hannken 
   2503   1.44  yamaguch 	vioif_ctrl_release(sc);
   2504    1.3  christos 
   2505    1.1   hannken 	return r;
   2506    1.1   hannken }
   2507    1.1   hannken 
   2508   1.46  yamaguch static int
   2509   1.74  yamaguch vioif_set_mac_addr(struct vioif_softc *sc)
   2510   1.74  yamaguch {
   2511   1.74  yamaguch 	struct virtio_net_ctrl_mac_addr *ma =
   2512   1.74  yamaguch 	    sc->sc_ctrlq.ctrlq_mac_addr;
   2513   1.74  yamaguch 	struct vioif_ctrl_cmdspec specs[1];
   2514   1.74  yamaguch 	struct ifnet *ifp = &sc->sc_ethercom.ec_if;
   2515   1.74  yamaguch 	int nspecs = __arraycount(specs);
   2516   1.78  yamaguch 	uint64_t features;
   2517   1.74  yamaguch 	int r;
   2518   1.78  yamaguch 	size_t i;
   2519   1.74  yamaguch 
   2520   1.74  yamaguch 	if (!sc->sc_has_ctrl)
   2521   1.74  yamaguch 		return ENOTSUP;
   2522   1.74  yamaguch 
   2523   1.78  yamaguch 	if (memcmp(CLLADDR(ifp->if_sadl), sc->sc_mac,
   2524   1.78  yamaguch 	    ETHER_ADDR_LEN) == 0) {
   2525   1.78  yamaguch 		return 0;
   2526   1.78  yamaguch 	}
   2527   1.74  yamaguch 
   2528   1.78  yamaguch 	memcpy(sc->sc_mac, CLLADDR(ifp->if_sadl), ETHER_ADDR_LEN);
   2529   1.74  yamaguch 
   2530   1.78  yamaguch 	features = virtio_features(sc->sc_virtio);
   2531   1.78  yamaguch 	if (features & VIRTIO_NET_F_CTRL_MAC_ADDR) {
   2532   1.78  yamaguch 		vioif_ctrl_acquire(sc);
   2533   1.78  yamaguch 
   2534   1.78  yamaguch 		memcpy(ma->mac, sc->sc_mac, ETHER_ADDR_LEN);
   2535   1.78  yamaguch 		specs[0].dmamap = sc->sc_ctrlq.ctrlq_mac_addr_dmamap;
   2536   1.78  yamaguch 		specs[0].buf = ma;
   2537   1.78  yamaguch 		specs[0].bufsize = sizeof(*ma);
   2538   1.78  yamaguch 
   2539   1.78  yamaguch 		r = vioif_ctrl_send_command(sc,
   2540   1.78  yamaguch 		    VIRTIO_NET_CTRL_MAC, VIRTIO_NET_CTRL_MAC_ADDR_SET,
   2541   1.78  yamaguch 		    specs, nspecs);
   2542   1.74  yamaguch 
   2543   1.78  yamaguch 		vioif_ctrl_release(sc);
   2544   1.78  yamaguch 	} else {
   2545   1.78  yamaguch 		for (i = 0; i < __arraycount(sc->sc_mac); i++) {
   2546   1.78  yamaguch 			virtio_write_device_config_1(sc->sc_virtio,
   2547   1.78  yamaguch 			    VIRTIO_NET_CONFIG_MAC + i, sc->sc_mac[i]);
   2548   1.78  yamaguch 		}
   2549   1.78  yamaguch 		r = 0;
   2550   1.78  yamaguch 	}
   2551   1.74  yamaguch 
   2552   1.74  yamaguch 	return r;
   2553   1.74  yamaguch }
   2554   1.74  yamaguch 
   2555   1.74  yamaguch static int
   2556  1.101  yamaguch vioif_set_rx_filter(struct vioif_softc *sc)
   2557    1.1   hannken {
   2558  1.101  yamaguch 	/* filter already set in ctrlq->ctrlq_mac_tbl */
   2559  1.101  yamaguch 	struct virtio_softc *vsc = sc->sc_virtio;
   2560  1.101  yamaguch 	struct virtio_net_ctrl_mac_tbl *mac_tbl_uc, *mac_tbl_mc;
   2561  1.101  yamaguch 	struct vioif_ctrl_cmdspec specs[2];
   2562  1.101  yamaguch 	int nspecs = __arraycount(specs);
   2563  1.101  yamaguch 	int r;
   2564    1.1   hannken 
   2565  1.101  yamaguch 	mac_tbl_uc = sc->sc_ctrlq.ctrlq_mac_tbl_uc;
   2566  1.101  yamaguch 	mac_tbl_mc = sc->sc_ctrlq.ctrlq_mac_tbl_mc;
   2567   1.54  yamaguch 
   2568  1.101  yamaguch 	if (!sc->sc_has_ctrl)
   2569  1.101  yamaguch 		return ENOTSUP;
   2570    1.1   hannken 
   2571  1.101  yamaguch 	vioif_ctrl_acquire(sc);
   2572    1.1   hannken 
   2573  1.101  yamaguch 	specs[0].dmamap = sc->sc_ctrlq.ctrlq_tbl_uc_dmamap;
   2574  1.101  yamaguch 	specs[0].buf = mac_tbl_uc;
   2575  1.101  yamaguch 	specs[0].bufsize = sizeof(*mac_tbl_uc)
   2576  1.101  yamaguch 	    + (ETHER_ADDR_LEN * virtio_rw32(vsc, mac_tbl_uc->nentries));
   2577    1.1   hannken 
   2578  1.101  yamaguch 	specs[1].dmamap = sc->sc_ctrlq.ctrlq_tbl_mc_dmamap;
   2579  1.101  yamaguch 	specs[1].buf = mac_tbl_mc;
   2580  1.101  yamaguch 	specs[1].bufsize = sizeof(*mac_tbl_mc)
   2581  1.101  yamaguch 	    + (ETHER_ADDR_LEN * virtio_rw32(vsc, mac_tbl_mc->nentries));
   2582   1.75  yamaguch 
   2583  1.101  yamaguch 	r = vioif_ctrl_load_cmdspec(sc, specs, nspecs);
   2584  1.101  yamaguch 	if (r != 0)
   2585  1.101  yamaguch 		goto out;
   2586   1.75  yamaguch 
   2587  1.101  yamaguch 	r = vioif_ctrl_send_command(sc,
   2588  1.101  yamaguch 	    VIRTIO_NET_CTRL_MAC, VIRTIO_NET_CTRL_MAC_TABLE_SET,
   2589  1.101  yamaguch 	    specs, nspecs);
   2590   1.75  yamaguch 
   2591  1.101  yamaguch 	vioif_ctrl_unload_cmdspec(sc, specs, nspecs);
   2592   1.75  yamaguch 
   2593  1.101  yamaguch out:
   2594  1.101  yamaguch 	vioif_ctrl_release(sc);
   2595   1.75  yamaguch 
   2596  1.101  yamaguch 	return r;
   2597   1.75  yamaguch }
   2598   1.75  yamaguch 
   2599    1.1   hannken /*
   2600    1.1   hannken  * If multicast filter small enough (<=MAXENTRIES) set rx filter
   2601    1.1   hannken  * If large multicast filter exist use ALLMULTI
   2602    1.1   hannken  * If setting rx filter fails fall back to ALLMULTI
   2603    1.1   hannken  */
   2604    1.1   hannken static int
   2605    1.1   hannken vioif_rx_filter(struct vioif_softc *sc)
   2606    1.1   hannken {
   2607   1.66   reinoud 	struct virtio_softc *vsc = sc->sc_virtio;
   2608   1.48   msaitoh 	struct ethercom *ec = &sc->sc_ethercom;
   2609   1.48   msaitoh 	struct ifnet *ifp = &ec->ec_if;
   2610    1.1   hannken 	struct ether_multi *enm;
   2611    1.1   hannken 	struct ether_multistep step;
   2612   1.43  yamaguch 	struct vioif_ctrlqueue *ctrlq = &sc->sc_ctrlq;
   2613    1.1   hannken 	int nentries;
   2614   1.75  yamaguch 	bool allmulti = 0;
   2615    1.1   hannken 	int r;
   2616    1.1   hannken 
   2617   1.75  yamaguch 	if (!sc->sc_has_ctrl) {
   2618   1.75  yamaguch 		goto set_ifflags;
   2619    1.1   hannken 	}
   2620    1.1   hannken 
   2621   1.74  yamaguch 	memcpy(ctrlq->ctrlq_mac_tbl_uc->macs[0],
   2622   1.74  yamaguch 	    CLLADDR(ifp->if_sadl), ETHER_ADDR_LEN);
   2623   1.74  yamaguch 
   2624   1.75  yamaguch 	nentries = 0;
   2625   1.75  yamaguch 	allmulti = false;
   2626   1.75  yamaguch 
   2627   1.48   msaitoh 	ETHER_LOCK(ec);
   2628   1.75  yamaguch 	for (ETHER_FIRST_MULTI(step, ec, enm); enm != NULL;
   2629   1.75  yamaguch 	    ETHER_NEXT_MULTI(step, enm)) {
   2630    1.1   hannken 		if (nentries >= VIRTIO_NET_CTRL_MAC_MAXENTRIES) {
   2631   1.75  yamaguch 			allmulti = true;
   2632   1.75  yamaguch 			break;
   2633    1.1   hannken 		}
   2634   1.50  christos 		if (memcmp(enm->enm_addrlo, enm->enm_addrhi, ETHER_ADDR_LEN)) {
   2635   1.75  yamaguch 			allmulti = true;
   2636   1.75  yamaguch 			break;
   2637    1.1   hannken 		}
   2638   1.75  yamaguch 
   2639   1.43  yamaguch 		memcpy(ctrlq->ctrlq_mac_tbl_mc->macs[nentries],
   2640   1.50  christos 		    enm->enm_addrlo, ETHER_ADDR_LEN);
   2641   1.75  yamaguch 		nentries++;
   2642    1.1   hannken 	}
   2643   1.48   msaitoh 	ETHER_UNLOCK(ec);
   2644   1.30     ozaki 
   2645   1.74  yamaguch 	r = vioif_set_mac_addr(sc);
   2646   1.74  yamaguch 	if (r != 0) {
   2647   1.74  yamaguch 		log(LOG_WARNING, "%s: couldn't set MAC address\n",
   2648   1.74  yamaguch 		    ifp->if_xname);
   2649   1.74  yamaguch 	}
   2650   1.74  yamaguch 
   2651   1.75  yamaguch 	if (!allmulti) {
   2652   1.74  yamaguch 		ctrlq->ctrlq_mac_tbl_uc->nentries = virtio_rw32(vsc, 1);
   2653   1.66   reinoud 		ctrlq->ctrlq_mac_tbl_mc->nentries = virtio_rw32(vsc, nentries);
   2654    1.1   hannken 		r = vioif_set_rx_filter(sc);
   2655    1.1   hannken 		if (r != 0) {
   2656   1.75  yamaguch 			allmulti = true; /* fallback */
   2657    1.1   hannken 		}
   2658   1.75  yamaguch 	}
   2659   1.75  yamaguch 
   2660   1.75  yamaguch 	if (allmulti) {
   2661   1.66   reinoud 		ctrlq->ctrlq_mac_tbl_uc->nentries = virtio_rw32(vsc, 0);
   2662   1.66   reinoud 		ctrlq->ctrlq_mac_tbl_mc->nentries = virtio_rw32(vsc, 0);
   2663    1.1   hannken 		r = vioif_set_rx_filter(sc);
   2664    1.1   hannken 		if (r != 0) {
   2665   1.75  yamaguch 			log(LOG_DEBUG, "%s: couldn't clear RX filter\n",
   2666   1.75  yamaguch 			    ifp->if_xname);
   2667   1.75  yamaguch 			/* what to do on failure? */
   2668    1.1   hannken 		}
   2669   1.75  yamaguch 
   2670   1.75  yamaguch 		ifp->if_flags |= IFF_ALLMULTI;
   2671    1.1   hannken 	}
   2672   1.75  yamaguch 
   2673   1.75  yamaguch set_ifflags:
   2674   1.75  yamaguch 	r = vioif_ifflags(sc);
   2675    1.1   hannken 
   2676    1.1   hannken 	return r;
   2677    1.1   hannken }
   2678    1.1   hannken 
   2679  1.101  yamaguch /*
   2680  1.101  yamaguch  * VM configuration changes
   2681  1.101  yamaguch  */
   2682  1.101  yamaguch static int
   2683  1.101  yamaguch vioif_config_change(struct virtio_softc *vsc)
   2684  1.101  yamaguch {
   2685  1.101  yamaguch 	struct vioif_softc *sc = device_private(virtio_child(vsc));
   2686  1.101  yamaguch 
   2687  1.101  yamaguch 	softint_schedule(sc->sc_cfg_softint);
   2688  1.101  yamaguch 	return 0;
   2689  1.101  yamaguch }
   2690  1.101  yamaguch 
   2691  1.101  yamaguch static void
   2692  1.101  yamaguch vioif_cfg_softint(void *arg)
   2693  1.101  yamaguch {
   2694  1.101  yamaguch 	struct vioif_softc *sc = arg;
   2695  1.101  yamaguch 	struct ifnet *ifp = &sc->sc_ethercom.ec_if;
   2696  1.101  yamaguch 
   2697  1.101  yamaguch 	vioif_update_link_status(sc);
   2698  1.101  yamaguch 	vioif_start(ifp);
   2699  1.101  yamaguch }
   2700  1.101  yamaguch 
   2701   1.82  knakahar static int
   2702   1.82  knakahar vioif_get_link_status(struct vioif_softc *sc)
   2703   1.34     ozaki {
   2704   1.34     ozaki 	struct virtio_softc *vsc = sc->sc_virtio;
   2705   1.34     ozaki 	uint16_t status;
   2706   1.34     ozaki 
   2707   1.34     ozaki 	if (virtio_features(vsc) & VIRTIO_NET_F_STATUS)
   2708   1.34     ozaki 		status = virtio_read_device_config_2(vsc,
   2709   1.34     ozaki 		    VIRTIO_NET_CONFIG_STATUS);
   2710   1.34     ozaki 	else
   2711   1.34     ozaki 		status = VIRTIO_NET_S_LINK_UP;
   2712   1.34     ozaki 
   2713   1.82  knakahar 	if ((status & VIRTIO_NET_S_LINK_UP) != 0)
   2714   1.82  knakahar 		return LINK_STATE_UP;
   2715   1.82  knakahar 
   2716   1.82  knakahar 	return LINK_STATE_DOWN;
   2717   1.34     ozaki }
   2718   1.34     ozaki 
   2719   1.34     ozaki static void
   2720   1.34     ozaki vioif_update_link_status(struct vioif_softc *sc)
   2721   1.34     ozaki {
   2722   1.34     ozaki 	struct ifnet *ifp = &sc->sc_ethercom.ec_if;
   2723   1.98  yamaguch 	struct vioif_netqueue *netq;
   2724   1.98  yamaguch 	struct vioif_tx_context *txc;
   2725   1.82  knakahar 	bool active;
   2726   1.46  yamaguch 	int link, i;
   2727   1.34     ozaki 
   2728   1.62  yamaguch 	mutex_enter(&sc->sc_lock);
   2729   1.62  yamaguch 
   2730   1.82  knakahar 	link = vioif_get_link_status(sc);
   2731   1.34     ozaki 
   2732   1.82  knakahar 	if (link == sc->sc_link_state)
   2733   1.82  knakahar 		goto done;
   2734   1.34     ozaki 
   2735   1.82  knakahar 	sc->sc_link_state = link;
   2736   1.82  knakahar 
   2737   1.82  knakahar 	active = VIOIF_IS_LINK_ACTIVE(sc);
   2738   1.82  knakahar 	for (i = 0; i < sc->sc_act_nvq_pairs; i++) {
   2739   1.98  yamaguch 		netq = &sc->sc_netqs[VIOIF_NETQ_TXQID(i)];
   2740   1.34     ozaki 
   2741   1.98  yamaguch 		mutex_enter(&netq->netq_lock);
   2742   1.98  yamaguch 		txc = netq->netq_ctx;
   2743   1.98  yamaguch 		txc->txc_link_active = active;
   2744   1.98  yamaguch 		mutex_exit(&netq->netq_lock);
   2745   1.34     ozaki 	}
   2746   1.34     ozaki 
   2747   1.82  knakahar 	if_link_state_change(ifp, sc->sc_link_state);
   2748   1.62  yamaguch 
   2749   1.82  knakahar done:
   2750   1.62  yamaguch 	mutex_exit(&sc->sc_lock);
   2751   1.34     ozaki }
   2752   1.34     ozaki 
   2753  1.101  yamaguch static void
   2754  1.101  yamaguch vioif_workq_work(struct work *wk, void *context)
   2755   1.34     ozaki {
   2756  1.101  yamaguch 	struct vioif_work *work;
   2757   1.34     ozaki 
   2758  1.101  yamaguch 	work = container_of(wk, struct vioif_work, cookie);
   2759   1.34     ozaki 
   2760  1.101  yamaguch 	atomic_store_relaxed(&work->added, 0);
   2761  1.101  yamaguch 	work->func(work->arg);
   2762   1.34     ozaki }
   2763   1.34     ozaki 
   2764   1.55  yamaguch static struct workqueue *
   2765   1.55  yamaguch vioif_workq_create(const char *name, pri_t prio, int ipl, int flags)
   2766   1.55  yamaguch {
   2767   1.55  yamaguch 	struct workqueue *wq;
   2768   1.55  yamaguch 	int error;
   2769   1.55  yamaguch 
   2770   1.55  yamaguch 	error = workqueue_create(&wq, name, vioif_workq_work, NULL,
   2771   1.55  yamaguch 	    prio, ipl, flags);
   2772   1.55  yamaguch 
   2773   1.55  yamaguch 	if (error)
   2774   1.55  yamaguch 		return NULL;
   2775   1.55  yamaguch 
   2776   1.55  yamaguch 	return wq;
   2777   1.55  yamaguch }
   2778   1.55  yamaguch 
   2779   1.55  yamaguch static void
   2780   1.55  yamaguch vioif_workq_destroy(struct workqueue *wq)
   2781   1.55  yamaguch {
   2782   1.55  yamaguch 
   2783   1.55  yamaguch 	workqueue_destroy(wq);
   2784   1.55  yamaguch }
   2785   1.55  yamaguch 
   2786   1.55  yamaguch static void
   2787   1.55  yamaguch vioif_work_set(struct vioif_work *work, void (*func)(void *), void *arg)
   2788   1.55  yamaguch {
   2789   1.55  yamaguch 
   2790   1.55  yamaguch 	memset(work, 0, sizeof(*work));
   2791   1.55  yamaguch 	work->func = func;
   2792   1.55  yamaguch 	work->arg = arg;
   2793   1.55  yamaguch }
   2794   1.55  yamaguch 
   2795   1.55  yamaguch static void
   2796   1.55  yamaguch vioif_work_add(struct workqueue *wq, struct vioif_work *work)
   2797   1.55  yamaguch {
   2798   1.55  yamaguch 
   2799   1.55  yamaguch 	if (atomic_load_relaxed(&work->added) != 0)
   2800   1.55  yamaguch 		return;
   2801   1.55  yamaguch 
   2802   1.55  yamaguch 	atomic_store_relaxed(&work->added, 1);
   2803   1.55  yamaguch 	kpreempt_disable();
   2804   1.55  yamaguch 	workqueue_enqueue(wq, &work->cookie, NULL);
   2805   1.55  yamaguch 	kpreempt_enable();
   2806   1.55  yamaguch }
   2807   1.55  yamaguch 
   2808   1.55  yamaguch static void
   2809   1.55  yamaguch vioif_work_wait(struct workqueue *wq, struct vioif_work *work)
   2810   1.55  yamaguch {
   2811   1.55  yamaguch 
   2812   1.55  yamaguch 	workqueue_wait(wq, &work->cookie);
   2813   1.55  yamaguch }
   2814   1.55  yamaguch 
   2815   1.26  pgoyette MODULE(MODULE_CLASS_DRIVER, if_vioif, "virtio");
   2816   1.48   msaitoh 
   2817   1.26  pgoyette #ifdef _MODULE
   2818   1.26  pgoyette #include "ioconf.c"
   2819   1.26  pgoyette #endif
   2820   1.48   msaitoh 
   2821   1.48   msaitoh static int
   2822   1.26  pgoyette if_vioif_modcmd(modcmd_t cmd, void *opaque)
   2823   1.26  pgoyette {
   2824   1.26  pgoyette 	int error = 0;
   2825   1.48   msaitoh 
   2826   1.26  pgoyette #ifdef _MODULE
   2827   1.26  pgoyette 	switch (cmd) {
   2828   1.26  pgoyette 	case MODULE_CMD_INIT:
   2829   1.48   msaitoh 		error = config_init_component(cfdriver_ioconf_if_vioif,
   2830   1.48   msaitoh 		    cfattach_ioconf_if_vioif, cfdata_ioconf_if_vioif);
   2831   1.26  pgoyette 		break;
   2832   1.26  pgoyette 	case MODULE_CMD_FINI:
   2833   1.26  pgoyette 		error = config_fini_component(cfdriver_ioconf_if_vioif,
   2834   1.26  pgoyette 		    cfattach_ioconf_if_vioif, cfdata_ioconf_if_vioif);
   2835   1.26  pgoyette 		break;
   2836   1.26  pgoyette 	default:
   2837   1.26  pgoyette 		error = ENOTTY;
   2838   1.48   msaitoh 		break;
   2839   1.26  pgoyette 	}
   2840   1.26  pgoyette #endif
   2841   1.48   msaitoh 
   2842   1.26  pgoyette 	return error;
   2843   1.26  pgoyette }
   2844