xref: /src/sys/dev/pci/if_vmx.c

/*	$NetBSD: if_vmx.c,v 1.13 2023/08/28 09:50:12 msaitoh Exp $	*/
/*	$OpenBSD: if_vmx.c,v 1.16 2014/01/22 06:04:17 brad Exp $	*/

/*
 * Copyright (c) 2013 Tsubai Masanari
 * Copyright (c) 2013 Bryan Venteicher <bryanv (at) FreeBSD.org>
 *
 * Permission to use, copy, modify, and distribute this software for any
 * purpose with or without fee is hereby granted, provided that the above
 * copyright notice and this permission notice appear in all copies.
 *
 * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
 * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
 * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
 * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
 * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
 * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
 * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
 */

#include <sys/cdefs.h>
__KERNEL_RCSID(0, "$NetBSD: if_vmx.c,v 1.13 2023/08/28 09:50:12 msaitoh Exp $");

#ifdef _KERNEL_OPT
#include "opt_if_vmx.h"
#endif

#include <sys/param.h>
#include <sys/cpu.h>
#include <sys/kernel.h>
#include <sys/kmem.h>
#include <sys/bitops.h>
#include <sys/bus.h>
#include <sys/device.h>
#include <sys/mbuf.h>
#include <sys/module.h>
#include <sys/sockio.h>
#include <sys/pcq.h>
#include <sys/workqueue.h>
#include <sys/interrupt.h>

#include <net/bpf.h>
#include <net/if.h>
#include <net/if_ether.h>
#include <net/if_media.h>

#include <netinet/if_inarp.h>
#include <netinet/in_systm.h>	/* for <netinet/ip.h> */
#include <netinet/in.h>		/* for <netinet/ip.h> */
#include <netinet/ip.h>		/* for struct ip */
#include <netinet/ip6.h>	/* for struct ip6_hdr */
#include <netinet/tcp.h>	/* for struct tcphdr */
#include <netinet/udp.h>	/* for struct udphdr */

#include <dev/pci/pcivar.h>
#include <dev/pci/pcireg.h>
#include <dev/pci/pcidevs.h>

#include <dev/pci/if_vmxreg.h>

#define VMXNET3_DRIVER_VERSION 0x00010000

/*
 * Max descriptors per Tx packet. We must limit the size of the
 * any TSO packets based on the number of segments.
 */
#define VMXNET3_TX_MAXSEGS		32
#define VMXNET3_TX_MAXSIZE		(VMXNET3_TX_MAXSEGS * MCLBYTES)

/*
 * Maximum support Tx segments size. The length field in the
 * Tx descriptor is 14 bits.
 */
#define VMXNET3_TX_MAXSEGSIZE		(1 << 14)

/*
 * The maximum number of Rx segments we accept.
 */
#define VMXNET3_MAX_RX_SEGS		0	/* no segments */

/*
 * Predetermined size of the multicast MACs filter table. If the
 * number of multicast addresses exceeds this size, then the
 * ALL_MULTI mode is use instead.
 */
#define VMXNET3_MULTICAST_MAX		32

/*
 * Our Tx watchdog timeout.
 */
#define VMXNET3_WATCHDOG_TIMEOUT	5

/*
 * Default value for vmx_intr_{rx,tx}_process_limit which is used for
 * max number of packets to process for interrupt handler
 */
#define VMXNET3_RX_INTR_PROCESS_LIMIT 0U
#define VMXNET3_TX_INTR_PROCESS_LIMIT 256

/*
 * Default value for vmx_{rx,tx}_process_limit which is used for
 * max number of packets to process for deferred processing
 */
#define VMXNET3_RX_PROCESS_LIMIT 256
#define VMXNET3_TX_PROCESS_LIMIT 256

#define VMXNET3_WORKQUEUE_PRI PRI_SOFTNET

/*
 * IP protocols that we can perform Tx checksum offloading of.
 */
#define VMXNET3_CSUM_OFFLOAD \
    (M_CSUM_TCPv4 | M_CSUM_UDPv4)
#define VMXNET3_CSUM_OFFLOAD_IPV6 \
    (M_CSUM_TCPv6 | M_CSUM_UDPv6)

#define VMXNET3_CSUM_ALL_OFFLOAD \
    (VMXNET3_CSUM_OFFLOAD | VMXNET3_CSUM_OFFLOAD_IPV6 | M_CSUM_TSOv4 | M_CSUM_TSOv6)

#define VMXNET3_RXRINGS_PERQ 2

#define VMXNET3_CORE_LOCK(_sc)		mutex_enter((_sc)->vmx_mtx)
#define VMXNET3_CORE_UNLOCK(_sc)	mutex_exit((_sc)->vmx_mtx)
#define VMXNET3_CORE_LOCK_ASSERT(_sc)	mutex_owned((_sc)->vmx_mtx)

#define VMXNET3_RXQ_LOCK(_rxq)		mutex_enter((_rxq)->vxrxq_mtx)
#define VMXNET3_RXQ_UNLOCK(_rxq)	mutex_exit((_rxq)->vxrxq_mtx)
#define VMXNET3_RXQ_LOCK_ASSERT(_rxq)		\
    mutex_owned((_rxq)->vxrxq_mtx)

#define VMXNET3_TXQ_LOCK(_txq)		mutex_enter((_txq)->vxtxq_mtx)
#define VMXNET3_TXQ_TRYLOCK(_txq)	mutex_tryenter((_txq)->vxtxq_mtx)
#define VMXNET3_TXQ_UNLOCK(_txq)	mutex_exit((_txq)->vxtxq_mtx)
#define VMXNET3_TXQ_LOCK_ASSERT(_txq)		\
    mutex_owned((_txq)->vxtxq_mtx)

struct vmxnet3_dma_alloc {
	bus_addr_t dma_paddr;
	void *dma_vaddr;
	bus_dmamap_t dma_map;
	bus_size_t dma_size;
	bus_dma_segment_t dma_segs[1];
};

struct vmxnet3_txbuf {
	bus_dmamap_t vtxb_dmamap;
	struct mbuf *vtxb_m;
};

struct vmxnet3_txring {
	struct vmxnet3_txbuf *vxtxr_txbuf;
	struct vmxnet3_txdesc *vxtxr_txd;
	u_int vxtxr_head;
	u_int vxtxr_next;
	u_int vxtxr_ndesc;
	int vxtxr_gen;
	struct vmxnet3_dma_alloc vxtxr_dma;
};

struct vmxnet3_rxbuf {
	bus_dmamap_t vrxb_dmamap;
	struct mbuf *vrxb_m;
};

struct vmxnet3_rxring {
	struct vmxnet3_rxbuf *vxrxr_rxbuf;
	struct vmxnet3_rxdesc *vxrxr_rxd;
	u_int vxrxr_fill;
	u_int vxrxr_ndesc;
	int vxrxr_gen;
	int vxrxr_rid;
	struct vmxnet3_dma_alloc vxrxr_dma;
	bus_dmamap_t vxrxr_spare_dmap;
};

struct vmxnet3_comp_ring {
	union {
		struct vmxnet3_txcompdesc *txcd;
		struct vmxnet3_rxcompdesc *rxcd;
	} vxcr_u;
	u_int vxcr_next;
	u_int vxcr_ndesc;
	int vxcr_gen;
	struct vmxnet3_dma_alloc vxcr_dma;
};

struct vmxnet3_txq_stats {
	uint64_t vmtxs_csum;
	uint64_t vmtxs_tso;
	uint64_t vmtxs_full;
	uint64_t vmtxs_offload_failed;
};

struct vmxnet3_txqueue {
	kmutex_t *vxtxq_mtx;
	struct vmxnet3_softc *vxtxq_sc;
	int vxtxq_watchdog;
	pcq_t *vxtxq_interq;
	struct vmxnet3_txring vxtxq_cmd_ring;
	struct vmxnet3_comp_ring vxtxq_comp_ring;
	struct vmxnet3_txq_stats vxtxq_stats;
	struct vmxnet3_txq_shared *vxtxq_ts;
	char vxtxq_name[16];

	void *vxtxq_si;

	struct evcnt vxtxq_intr;
	struct evcnt vxtxq_defer;
	struct evcnt vxtxq_deferreq;
	struct evcnt vxtxq_pcqdrop;
	struct evcnt vxtxq_transmitdef;
	struct evcnt vxtxq_watchdogto;
	struct evcnt vxtxq_defragged;
	struct evcnt vxtxq_defrag_failed;

	bool vxtxq_stopping;
};


struct vmxnet3_rxqueue {
	kmutex_t *vxrxq_mtx;
	struct vmxnet3_softc *vxrxq_sc;
	struct mbuf *vxrxq_mhead;
	struct mbuf *vxrxq_mtail;
	struct vmxnet3_rxring vxrxq_cmd_ring[VMXNET3_RXRINGS_PERQ];
	struct vmxnet3_comp_ring vxrxq_comp_ring;
	struct vmxnet3_rxq_shared *vxrxq_rs;
	char vxrxq_name[16];

	struct evcnt vxrxq_intr;
	struct evcnt vxrxq_defer;
	struct evcnt vxrxq_deferreq;
	struct evcnt vxrxq_mgetcl_failed;
	struct evcnt vxrxq_mbuf_load_failed;

	bool vxrxq_stopping;
};

struct vmxnet3_queue {
	int vxq_id;
	int vxq_intr_idx;

	struct vmxnet3_txqueue vxq_txqueue;
	struct vmxnet3_rxqueue vxq_rxqueue;

	void *vxq_si;
	bool vxq_workqueue;
	bool vxq_wq_enqueued;
	struct work vxq_wq_cookie;
};

struct vmxnet3_softc {
	device_t vmx_dev;
	struct ethercom vmx_ethercom;
	struct ifmedia vmx_media;
	struct vmxnet3_driver_shared *vmx_ds;
	int vmx_flags;
#define VMXNET3_FLAG_NO_MSIX	(1 << 0)
#define VMXNET3_FLAG_RSS	(1 << 1)
#define VMXNET3_FLAG_ATTACHED	(1 << 2)

	struct vmxnet3_queue *vmx_queue;

	struct pci_attach_args *vmx_pa;
	pci_chipset_tag_t vmx_pc;

	bus_space_tag_t vmx_iot0;
	bus_space_tag_t vmx_iot1;
	bus_space_handle_t vmx_ioh0;
	bus_space_handle_t vmx_ioh1;
	bus_size_t vmx_ios0;
	bus_size_t vmx_ios1;
	bus_dma_tag_t vmx_dmat;

	int vmx_link_active;
	int vmx_ntxqueues;
	int vmx_nrxqueues;
	int vmx_ntxdescs;
	int vmx_nrxdescs;
	int vmx_max_rxsegs;

	struct evcnt vmx_event_intr;
	struct evcnt vmx_event_link;
	struct evcnt vmx_event_txqerror;
	struct evcnt vmx_event_rxqerror;
	struct evcnt vmx_event_dic;
	struct evcnt vmx_event_debug;

	int vmx_intr_type;
	int vmx_intr_mask_mode;
	int vmx_event_intr_idx;
	int vmx_nintrs;
	pci_intr_handle_t *vmx_intrs;	/* legacy use vmx_intrs[0] */
	void *vmx_ihs[VMXNET3_MAX_INTRS];

	kmutex_t *vmx_mtx;

	int vmx_if_flags;
	bool vmx_promisc;
	bool vmx_mcastactive;
	uint8_t *vmx_mcast;
	void *vmx_qs;
	struct vmxnet3_rss_shared *vmx_rss;
	callout_t vmx_tick;
	struct vmxnet3_dma_alloc vmx_ds_dma;
	struct vmxnet3_dma_alloc vmx_qs_dma;
	struct vmxnet3_dma_alloc vmx_mcast_dma;
	struct vmxnet3_dma_alloc vmx_rss_dma;
	int vmx_max_ntxqueues;
	int vmx_max_nrxqueues;
	uint8_t vmx_lladdr[ETHER_ADDR_LEN];

	u_int vmx_rx_intr_process_limit;
	u_int vmx_tx_intr_process_limit;
	u_int vmx_rx_process_limit;
	u_int vmx_tx_process_limit;
	struct sysctllog *vmx_sysctllog;

	bool vmx_txrx_workqueue;
	struct workqueue *vmx_queue_wq;

	struct workqueue *vmx_reset_wq;
	struct work vmx_reset_work;
	bool vmx_reset_pending;
};

#define VMXNET3_STAT

#ifdef VMXNET3_STAT
struct {
	u_int txhead;
	u_int txdone;
	u_int maxtxlen;
	u_int rxdone;
	u_int rxfill;
	u_int intr;
} vmxstat;
#endif

typedef enum {
	VMXNET3_BARRIER_RD,
	VMXNET3_BARRIER_WR,
} vmxnet3_barrier_t;

#define JUMBO_LEN (MCLBYTES - ETHER_ALIGN)	/* XXX */
#define DMAADDR(map) ((map)->dm_segs[0].ds_addr)

#define vtophys(va) 0		/* XXX ok? */

static int vmxnet3_match(device_t, cfdata_t, void *);
static void vmxnet3_attach(device_t, device_t, void *);
static int vmxnet3_detach(device_t, int);

static int vmxnet3_alloc_pci_resources(struct vmxnet3_softc *);
static void vmxnet3_free_pci_resources(struct vmxnet3_softc *);
static int vmxnet3_check_version(struct vmxnet3_softc *);
static void vmxnet3_check_multiqueue(struct vmxnet3_softc *);

static int vmxnet3_alloc_msix_interrupts(struct vmxnet3_softc *);
static int vmxnet3_alloc_msi_interrupts(struct vmxnet3_softc *);
static int vmxnet3_alloc_legacy_interrupts(struct vmxnet3_softc *);
static int vmxnet3_alloc_interrupts(struct vmxnet3_softc *);
static void vmxnet3_free_interrupts(struct vmxnet3_softc *);

static int vmxnet3_setup_msix_interrupts(struct vmxnet3_softc *);
static int vmxnet3_setup_msi_interrupt(struct vmxnet3_softc *);
static int vmxnet3_setup_legacy_interrupt(struct vmxnet3_softc *);
static void vmxnet3_set_interrupt_idx(struct vmxnet3_softc *);
static int vmxnet3_setup_interrupts(struct vmxnet3_softc *);
static int vmxnet3_setup_sysctl(struct vmxnet3_softc *);

static int vmxnet3_setup_stats(struct vmxnet3_softc *);
static void vmxnet3_teardown_stats(struct vmxnet3_softc *);

static int vmxnet3_init_rxq(struct vmxnet3_softc *, int);
static int vmxnet3_init_txq(struct vmxnet3_softc *, int);
static int vmxnet3_alloc_rxtx_queues(struct vmxnet3_softc *);
static void vmxnet3_destroy_rxq(struct vmxnet3_rxqueue *);
static void vmxnet3_destroy_txq(struct vmxnet3_txqueue *);
static void vmxnet3_free_rxtx_queues(struct vmxnet3_softc *);

static int vmxnet3_alloc_shared_data(struct vmxnet3_softc *);
static void vmxnet3_free_shared_data(struct vmxnet3_softc *);
static int vmxnet3_alloc_txq_data(struct vmxnet3_softc *);
static void vmxnet3_free_txq_data(struct vmxnet3_softc *);
static int vmxnet3_alloc_rxq_data(struct vmxnet3_softc *);
static void vmxnet3_free_rxq_data(struct vmxnet3_softc *);
static int vmxnet3_alloc_queue_data(struct vmxnet3_softc *);
static void vmxnet3_free_queue_data(struct vmxnet3_softc *);
static int vmxnet3_alloc_mcast_table(struct vmxnet3_softc *);
static void vmxnet3_free_mcast_table(struct vmxnet3_softc *);
static void vmxnet3_init_shared_data(struct vmxnet3_softc *);
static void vmxnet3_reinit_rss_shared_data(struct vmxnet3_softc *);
static void vmxnet3_reinit_shared_data(struct vmxnet3_softc *);
static int vmxnet3_alloc_data(struct vmxnet3_softc *);
static void vmxnet3_free_data(struct vmxnet3_softc *);
static int vmxnet3_setup_interface(struct vmxnet3_softc *);

static void vmxnet3_evintr(struct vmxnet3_softc *);
static bool vmxnet3_txq_eof(struct vmxnet3_txqueue *, u_int);
static int vmxnet3_newbuf(struct vmxnet3_softc *, struct vmxnet3_rxqueue *,
    struct vmxnet3_rxring *);
static void vmxnet3_rxq_eof_discard(struct vmxnet3_rxqueue *,
    struct vmxnet3_rxring *, int);
static void vmxnet3_rxq_discard_chain(struct vmxnet3_rxqueue *);
static void vmxnet3_rx_csum(struct vmxnet3_rxcompdesc *, struct mbuf *);
static void vmxnet3_rxq_input(struct vmxnet3_rxqueue *,
    struct vmxnet3_rxcompdesc *, struct mbuf *);
static bool vmxnet3_rxq_eof(struct vmxnet3_rxqueue *, u_int);
static int vmxnet3_legacy_intr(void *);
static int vmxnet3_txrxq_intr(void *);
static void vmxnet3_handle_queue(void *);
static void vmxnet3_handle_queue_work(struct work *, void *);
static int vmxnet3_event_intr(void *);

static void vmxnet3_txstop(struct vmxnet3_softc *, struct vmxnet3_txqueue *);
static void vmxnet3_rxstop(struct vmxnet3_softc *, struct vmxnet3_rxqueue *);
static void vmxnet3_stop_locked(struct vmxnet3_softc *);
static void vmxnet3_stop_rendezvous(struct vmxnet3_softc *);
static void vmxnet3_stop(struct ifnet *, int);

static void vmxnet3_txinit(struct vmxnet3_softc *, struct vmxnet3_txqueue *);
static int vmxnet3_rxinit(struct vmxnet3_softc *, struct vmxnet3_rxqueue *);
static int vmxnet3_reinit_queues(struct vmxnet3_softc *);
static int vmxnet3_enable_device(struct vmxnet3_softc *);
static void vmxnet3_reinit_rxfilters(struct vmxnet3_softc *);
static int vmxnet3_reinit(struct vmxnet3_softc *);

static int vmxnet3_init_locked(struct vmxnet3_softc *);
static int vmxnet3_init(struct ifnet *);

static int vmxnet3_txq_offload_ctx(struct vmxnet3_txqueue *, struct mbuf *, int *, int *);
static int vmxnet3_txq_load_mbuf(struct vmxnet3_txqueue *, struct mbuf **, bus_dmamap_t);
static void vmxnet3_txq_unload_mbuf(struct vmxnet3_txqueue *, bus_dmamap_t);
static int vmxnet3_txq_encap(struct vmxnet3_txqueue *, struct mbuf **);
static void vmxnet3_start_locked(struct ifnet *);
static void vmxnet3_start(struct ifnet *);
static void vmxnet3_transmit_locked(struct ifnet *, struct vmxnet3_txqueue *);
static int vmxnet3_transmit(struct ifnet *, struct mbuf *);
static void vmxnet3_deferred_transmit(void *);

static void vmxnet3_set_rxfilter(struct vmxnet3_softc *);
static int vmxnet3_ioctl(struct ifnet *, u_long, void *);
static int vmxnet3_ifflags_cb(struct ethercom *);

static int vmxnet3_watchdog(struct vmxnet3_txqueue *);
static void vmxnet3_refresh_host_stats(struct vmxnet3_softc *);
static void vmxnet3_tick(void *);
static void vmxnet3_reset_work(struct work *, void *);
static void vmxnet3_if_link_status(struct vmxnet3_softc *);
static bool vmxnet3_cmd_link_status(struct ifnet *);
static void vmxnet3_ifmedia_status(struct ifnet *, struct ifmediareq *);
static int vmxnet3_ifmedia_change(struct ifnet *);
static void vmxnet3_set_lladdr(struct vmxnet3_softc *);
static void vmxnet3_get_lladdr(struct vmxnet3_softc *);

static void vmxnet3_enable_all_intrs(struct vmxnet3_softc *);
static void vmxnet3_disable_all_intrs(struct vmxnet3_softc *);

static int vmxnet3_dma_malloc(struct vmxnet3_softc *, bus_size_t, bus_size_t,
    struct vmxnet3_dma_alloc *);
static void vmxnet3_dma_free(struct vmxnet3_softc *, struct vmxnet3_dma_alloc *);

CFATTACH_DECL3_NEW(vmx, sizeof(struct vmxnet3_softc),
    vmxnet3_match, vmxnet3_attach, vmxnet3_detach, NULL, NULL, NULL, 0);

/* round down to the nearest power of 2 */
static int
vmxnet3_calc_queue_size(int n)
{

	if (__predict_false(n <= 0))
		return 1;

	return (1U << (fls32(n) - 1));
}

static inline void
vmxnet3_write_bar0(struct vmxnet3_softc *sc, bus_size_t r, uint32_t v)
{

	bus_space_write_4(sc->vmx_iot0, sc->vmx_ioh0, r, v);
}

static inline uint32_t
vmxnet3_read_bar1(struct vmxnet3_softc *sc, bus_size_t r)
{

	return (bus_space_read_4(sc->vmx_iot1, sc->vmx_ioh1, r));
}

static inline void
vmxnet3_write_bar1(struct vmxnet3_softc *sc, bus_size_t r, uint32_t v)
{

	bus_space_write_4(sc->vmx_iot1, sc->vmx_ioh1, r, v);
}

static inline void
vmxnet3_write_cmd(struct vmxnet3_softc *sc, uint32_t cmd)
{

	vmxnet3_write_bar1(sc, VMXNET3_BAR1_CMD, cmd);
}

static inline uint32_t
vmxnet3_read_cmd(struct vmxnet3_softc *sc, uint32_t cmd)
{

	vmxnet3_write_cmd(sc, cmd);
	return (vmxnet3_read_bar1(sc, VMXNET3_BAR1_CMD));
}

static inline void
vmxnet3_enable_intr(struct vmxnet3_softc *sc, int irq)
{
	vmxnet3_write_bar0(sc, VMXNET3_BAR0_IMASK(irq), 0);
}

static inline void
vmxnet3_disable_intr(struct vmxnet3_softc *sc, int irq)
{
	vmxnet3_write_bar0(sc, VMXNET3_BAR0_IMASK(irq), 1);
}

static inline void
vmxnet3_rxr_increment_fill(struct vmxnet3_rxring *rxr)
{

	if (++rxr->vxrxr_fill == rxr->vxrxr_ndesc) {
		rxr->vxrxr_fill = 0;
		rxr->vxrxr_gen ^= 1;
	}
}

static inline int
vmxnet3_txring_avail(struct vmxnet3_txring *txr)
{
	int avail = txr->vxtxr_next - txr->vxtxr_head - 1;
	return (avail < 0 ? (int)txr->vxtxr_ndesc + avail : avail);
}

/*
 * Since this is a purely paravirtualized device, we do not have
 * to worry about DMA coherency. But at times, we must make sure
 * both the compiler and CPU do not reorder memory operations.
 */
static inline void
vmxnet3_barrier(struct vmxnet3_softc *sc, vmxnet3_barrier_t type)
{

	switch (type) {
	case VMXNET3_BARRIER_RD:
		membar_consumer();
		break;
	case VMXNET3_BARRIER_WR:
		membar_producer();
		break;
	default:
		panic("%s: bad barrier type %d", __func__, type);
	}
}

static int
vmxnet3_match(device_t parent, cfdata_t match, void *aux)
{
	struct pci_attach_args *pa = (struct pci_attach_args *)aux;

	if (PCI_VENDOR(pa->pa_id) == PCI_VENDOR_VMWARE &&
	    PCI_PRODUCT(pa->pa_id) == PCI_PRODUCT_VMWARE_VMXNET3)
		return 1;

	return 0;
}

static void
vmxnet3_attach(device_t parent, device_t self, void *aux)
{
	struct vmxnet3_softc *sc = device_private(self);
	struct pci_attach_args *pa = aux;
	pcireg_t preg;
	int error;
	int candidate;

	sc->vmx_dev = self;
	sc->vmx_pa = pa;
	sc->vmx_pc = pa->pa_pc;
	if (pci_dma64_available(pa))
		sc->vmx_dmat = pa->pa_dmat64;
	else
		sc->vmx_dmat = pa->pa_dmat;

	pci_aprint_devinfo_fancy(pa, "Ethernet controller", "vmxnet3", 1);

	preg = pci_conf_read(pa->pa_pc, pa->pa_tag, PCI_COMMAND_STATUS_REG);
	preg |= PCI_COMMAND_MASTER_ENABLE;
	pci_conf_write(pa->pa_pc, pa->pa_tag, PCI_COMMAND_STATUS_REG, preg);

	sc->vmx_mtx = mutex_obj_alloc(MUTEX_DEFAULT, IPL_NET);
	callout_init(&sc->vmx_tick, CALLOUT_MPSAFE);

	candidate = MIN(MIN(VMXNET3_MAX_TX_QUEUES, VMXNET3_MAX_RX_QUEUES),
	    ncpu);
	sc->vmx_max_ntxqueues = sc->vmx_max_nrxqueues =
	    vmxnet3_calc_queue_size(candidate);
	sc->vmx_ntxdescs = 512;
	sc->vmx_nrxdescs = 256;
	sc->vmx_max_rxsegs = VMXNET3_MAX_RX_SEGS;

	error = vmxnet3_alloc_pci_resources(sc);
	if (error)
		return;

	error = vmxnet3_check_version(sc);
	if (error)
		return;

	error = vmxnet3_alloc_rxtx_queues(sc);
	if (error)
		return;

	error = vmxnet3_alloc_interrupts(sc);
	if (error)
		return;

	vmxnet3_check_multiqueue(sc);

	error = vmxnet3_alloc_data(sc);
	if (error)
		return;

	error = vmxnet3_setup_interface(sc);
	if (error)
		return;

	error = vmxnet3_setup_interrupts(sc);
	if (error)
		return;

	error = vmxnet3_setup_sysctl(sc);
	if (error)
		return;

	error = vmxnet3_setup_stats(sc);
	if (error)
		return;

	char buf[128];
	snprintf(buf, sizeof(buf), "%s_reset", device_xname(sc->vmx_dev));
	error = workqueue_create(&sc->vmx_reset_wq, "%s_reset",
	    vmxnet3_reset_work, sc, VMXNET3_WORKQUEUE_PRI, IPL_SOFTCLOCK,
	    WQ_MPSAFE);
	if (error) {
		aprint_error_dev(sc->vmx_dev,
		    "failed to create reset workqueue: %d\n",
		    error);
		return;
	}

	sc->vmx_flags |= VMXNET3_FLAG_ATTACHED;
}

static int
vmxnet3_detach(device_t self, int flags)
{
	struct vmxnet3_softc *sc;
	struct ifnet *ifp;

	sc = device_private(self);
	ifp = &sc->vmx_ethercom.ec_if;

	if (sc->vmx_flags & VMXNET3_FLAG_ATTACHED) {
		VMXNET3_CORE_LOCK(sc);
		vmxnet3_stop_locked(sc);
		callout_halt(&sc->vmx_tick, sc->vmx_mtx);
		callout_destroy(&sc->vmx_tick);
		VMXNET3_CORE_UNLOCK(sc);

		ether_ifdetach(ifp);
		if_detach(ifp);
		ifmedia_fini(&sc->vmx_media);
	}

	vmxnet3_teardown_stats(sc);
	sysctl_teardown(&sc->vmx_sysctllog);

	vmxnet3_free_interrupts(sc);

	vmxnet3_free_data(sc);
	vmxnet3_free_pci_resources(sc);
	vmxnet3_free_rxtx_queues(sc);

	if (sc->vmx_mtx)
		mutex_obj_free(sc->vmx_mtx);

	return (0);
}

static int
vmxnet3_alloc_pci_resources(struct vmxnet3_softc *sc)
{
	struct pci_attach_args *pa = sc->vmx_pa;
	pcireg_t memtype;

	memtype = pci_mapreg_type(pa->pa_pc, pa->pa_tag, PCI_BAR(0));
	if (pci_mapreg_map(pa, PCI_BAR(0), memtype, 0, &sc->vmx_iot0, &sc->vmx_ioh0,
	    NULL, &sc->vmx_ios0)) {
		aprint_error_dev(sc->vmx_dev, "failed to map BAR0\n");
		return (ENXIO);
	}
	memtype = pci_mapreg_type(pa->pa_pc, pa->pa_tag, PCI_BAR(1));
	if (pci_mapreg_map(pa, PCI_BAR(1), memtype, 0, &sc->vmx_iot1, &sc->vmx_ioh1,
	    NULL, &sc->vmx_ios1)) {
		aprint_error_dev(sc->vmx_dev, "failed to map BAR1\n");
		return (ENXIO);
	}

	if (!pci_get_capability(pa->pa_pc, pa->pa_tag, PCI_CAP_MSIX, NULL, NULL)) {
		sc->vmx_flags |= VMXNET3_FLAG_NO_MSIX;
		return (0);
	}

	return (0);
}

static void
vmxnet3_free_pci_resources(struct vmxnet3_softc *sc)
{

	if (sc->vmx_ios0) {
		bus_space_unmap(sc->vmx_iot0, sc->vmx_ioh0, sc->vmx_ios0);
		sc->vmx_ios0 = 0;
	}

	if (sc->vmx_ios1) {
		bus_space_unmap(sc->vmx_iot1, sc->vmx_ioh1, sc->vmx_ios1);
		sc->vmx_ios1 = 0;
	}
}

static int
vmxnet3_check_version(struct vmxnet3_softc *sc)
{
	u_int ver;

	ver = vmxnet3_read_bar1(sc, VMXNET3_BAR1_VRRS);
	if ((ver & 0x1) == 0) {
		aprint_error_dev(sc->vmx_dev,
		    "unsupported hardware version 0x%x\n", ver);
		return (ENOTSUP);
	}
	vmxnet3_write_bar1(sc, VMXNET3_BAR1_VRRS, 1);

	ver = vmxnet3_read_bar1(sc, VMXNET3_BAR1_UVRS);
	if ((ver & 0x1) == 0) {
		aprint_error_dev(sc->vmx_dev,
		    "incompatiable UPT version 0x%x\n", ver);
		return (ENOTSUP);
	}
	vmxnet3_write_bar1(sc, VMXNET3_BAR1_UVRS, 1);

	return (0);
}

static void
vmxnet3_check_multiqueue(struct vmxnet3_softc *sc)
{

	if (sc->vmx_intr_type != VMXNET3_IT_MSIX)
		goto out;

	/* Just use the maximum configured for now. */
	sc->vmx_nrxqueues = sc->vmx_max_nrxqueues;
	sc->vmx_ntxqueues = sc->vmx_max_ntxqueues;

	if (sc->vmx_nrxqueues > 1)
		sc->vmx_flags |= VMXNET3_FLAG_RSS;

	return;

out:
	sc->vmx_ntxqueues = 1;
	sc->vmx_nrxqueues = 1;
}

static int
vmxnet3_alloc_msix_interrupts(struct vmxnet3_softc *sc)
{
	int required;
	struct pci_attach_args *pa = sc->vmx_pa;

	if (sc->vmx_flags & VMXNET3_FLAG_NO_MSIX)
		return (1);

	/* Allocate an additional vector for the events interrupt. */
	required = MIN(sc->vmx_max_ntxqueues, sc->vmx_max_nrxqueues) + 1;

	if (pci_msix_count(pa->pa_pc, pa->pa_tag) < required)
		return (1);

	if (pci_msix_alloc_exact(pa, &sc->vmx_intrs, required) == 0) {
		sc->vmx_nintrs = required;
		return (0);
	}

	return (1);
}

static int
vmxnet3_alloc_msi_interrupts(struct vmxnet3_softc *sc)
{
	int nmsi, required;
	struct pci_attach_args *pa = sc->vmx_pa;

	required = 1;

	nmsi = pci_msi_count(pa->pa_pc, pa->pa_tag);
	if (nmsi < required)
		return (1);

	if (pci_msi_alloc_exact(pa, &sc->vmx_intrs, required) == 0) {
		sc->vmx_nintrs = required;
		return (0);
	}

	return (1);
}

static int
vmxnet3_alloc_legacy_interrupts(struct vmxnet3_softc *sc)
{

	if (pci_intx_alloc(sc->vmx_pa, &sc->vmx_intrs) == 0) {
		sc->vmx_nintrs = 1;
		return (0);
	}

	return (1);
}

static int
vmxnet3_alloc_interrupts(struct vmxnet3_softc *sc)
{
	u_int config;
	int error;

	config = vmxnet3_read_cmd(sc, VMXNET3_CMD_GET_INTRCFG);

	sc->vmx_intr_type = config & 0x03;
	sc->vmx_intr_mask_mode = (config >> 2) & 0x03;

	switch (sc->vmx_intr_type) {
	case VMXNET3_IT_AUTO:
		sc->vmx_intr_type = VMXNET3_IT_MSIX;
		/* FALLTHROUGH */
	case VMXNET3_IT_MSIX:
		error = vmxnet3_alloc_msix_interrupts(sc);
		if (error == 0)
			break;
		sc->vmx_intr_type = VMXNET3_IT_MSI;
		/* FALLTHROUGH */
	case VMXNET3_IT_MSI:
		error = vmxnet3_alloc_msi_interrupts(sc);
		if (error == 0)
			break;
		sc->vmx_intr_type = VMXNET3_IT_LEGACY;
		/* FALLTHROUGH */
	case VMXNET3_IT_LEGACY:
		error = vmxnet3_alloc_legacy_interrupts(sc);
		if (error == 0)
			break;
		/* FALLTHROUGH */
	default:
		sc->vmx_intr_type = -1;
		aprint_error_dev(sc->vmx_dev, "cannot allocate any interrupt resources\n");
		return (ENXIO);
	}

	return (error);
}

static void
vmxnet3_free_interrupts(struct vmxnet3_softc *sc)
{
	pci_chipset_tag_t pc = sc->vmx_pc;
	int i;

	workqueue_destroy(sc->vmx_queue_wq);
	for (i = 0; i < sc->vmx_ntxqueues; i++) {
		struct vmxnet3_queue *vmxq =  &sc->vmx_queue[i];

		softint_disestablish(vmxq->vxq_si);
		vmxq->vxq_si = NULL;
	}
	for (i = 0; i < sc->vmx_nintrs; i++) {
		pci_intr_disestablish(pc, sc->vmx_ihs[i]);
	}
	pci_intr_release(pc, sc->vmx_intrs, sc->vmx_nintrs);
}

static int
vmxnet3_setup_msix_interrupts(struct vmxnet3_softc *sc)
{
	pci_chipset_tag_t pc = sc->vmx_pa->pa_pc;
	struct vmxnet3_queue *vmxq;
	pci_intr_handle_t *intr;
	void **ihs;
	int intr_idx, i, use_queues, error;
	kcpuset_t *affinity;
	const char *intrstr;
	char intrbuf[PCI_INTRSTR_LEN];
	char xnamebuf[32];

	intr = sc->vmx_intrs;
	intr_idx = 0;
	ihs = sc->vmx_ihs;

	/* See vmxnet3_alloc_msix_interrupts() */
	use_queues = MIN(sc->vmx_max_ntxqueues, sc->vmx_max_nrxqueues);
	for (i = 0; i < use_queues; i++, intr++, ihs++, intr_idx++) {
		snprintf(xnamebuf, 32, "%s: txrx %d", device_xname(sc->vmx_dev), i);

		vmxq = &sc->vmx_queue[i];

		intrstr = pci_intr_string(pc, *intr, intrbuf, sizeof(intrbuf));

		pci_intr_setattr(pc, intr, PCI_INTR_MPSAFE, true);
		*ihs = pci_intr_establish_xname(pc, *intr, IPL_NET,
		    vmxnet3_txrxq_intr, vmxq, xnamebuf);
		if (*ihs == NULL) {
			aprint_error_dev(sc->vmx_dev,
			    "unable to establish txrx interrupt at %s\n", intrstr);
			return (-1);
		}
		aprint_normal_dev(sc->vmx_dev, "txrx interrupting at %s\n", intrstr);

		kcpuset_create(&affinity, true);
		kcpuset_set(affinity, intr_idx % ncpu);
		error = interrupt_distribute(*ihs, affinity, NULL);
		if (error) {
			aprint_normal_dev(sc->vmx_dev,
			    "%s cannot be changed affinity, use default CPU\n",
			    intrstr);
		}
		kcpuset_destroy(affinity);

		vmxq->vxq_si = softint_establish(SOFTINT_NET | SOFTINT_MPSAFE,
		    vmxnet3_handle_queue, vmxq);
		if (vmxq->vxq_si == NULL) {
			aprint_error_dev(sc->vmx_dev,
			    "softint_establish for vxq_si failed\n");
			return (-1);
		}

		vmxq->vxq_intr_idx = intr_idx;
	}
	snprintf(xnamebuf, MAXCOMLEN, "%s_tx_rx", device_xname(sc->vmx_dev));
	error = workqueue_create(&sc->vmx_queue_wq, xnamebuf,
	    vmxnet3_handle_queue_work, sc, VMXNET3_WORKQUEUE_PRI, IPL_NET,
	    WQ_PERCPU | WQ_MPSAFE);
	if (error) {
		aprint_error_dev(sc->vmx_dev, "workqueue_create failed\n");
		return (-1);
	}
	sc->vmx_txrx_workqueue = false;

	intrstr = pci_intr_string(pc, *intr, intrbuf, sizeof(intrbuf));

	snprintf(xnamebuf, 32, "%s: link", device_xname(sc->vmx_dev));
	pci_intr_setattr(pc, intr, PCI_INTR_MPSAFE, true);
	*ihs = pci_intr_establish_xname(pc, *intr, IPL_NET,
	    vmxnet3_event_intr, sc, xnamebuf);
	if (*ihs == NULL) {
		aprint_error_dev(sc->vmx_dev,
		    "unable to establish event interrupt at %s\n", intrstr);
		return (-1);
	}
	aprint_normal_dev(sc->vmx_dev, "event interrupting at %s\n", intrstr);

	sc->vmx_event_intr_idx = intr_idx;

	return (0);
}

static int
vmxnet3_setup_msi_interrupt(struct vmxnet3_softc *sc)
{
	pci_chipset_tag_t pc = sc->vmx_pa->pa_pc;
	pci_intr_handle_t *intr;
	void **ihs;
	struct vmxnet3_queue *vmxq;
	int i;
	const char *intrstr;
	char intrbuf[PCI_INTRSTR_LEN];
	char xnamebuf[32];

	intr = &sc->vmx_intrs[0];
	ihs = sc->vmx_ihs;
	vmxq = &sc->vmx_queue[0];

	intrstr = pci_intr_string(pc, *intr, intrbuf, sizeof(intrbuf));

	snprintf(xnamebuf, 32, "%s: msi", device_xname(sc->vmx_dev));
	pci_intr_setattr(pc, intr, PCI_INTR_MPSAFE, true);
	*ihs = pci_intr_establish_xname(pc, *intr, IPL_NET,
	    vmxnet3_legacy_intr, sc, xnamebuf);
	if (*ihs == NULL) {
		aprint_error_dev(sc->vmx_dev,
		    "unable to establish interrupt at %s\n", intrstr);
		return (-1);
	}
	aprint_normal_dev(sc->vmx_dev, "interrupting at %s\n", intrstr);

	vmxq->vxq_si = softint_establish(SOFTINT_NET | SOFTINT_MPSAFE,
	    vmxnet3_handle_queue, vmxq);
	if (vmxq->vxq_si == NULL) {
		aprint_error_dev(sc->vmx_dev,
		    "softint_establish for vxq_si failed\n");
		return (-1);
	}

	for (i = 0; i < MIN(sc->vmx_nrxqueues, sc->vmx_nrxqueues); i++)
		sc->vmx_queue[i].vxq_intr_idx = 0;
	sc->vmx_event_intr_idx = 0;

	return (0);
}

static int
vmxnet3_setup_legacy_interrupt(struct vmxnet3_softc *sc)
{
	pci_chipset_tag_t pc = sc->vmx_pa->pa_pc;
	pci_intr_handle_t *intr;
	void **ihs;
	struct vmxnet3_queue *vmxq;
	int i;
	const char *intrstr;
	char intrbuf[PCI_INTRSTR_LEN];
	char xnamebuf[32];

	intr = &sc->vmx_intrs[0];
	ihs = sc->vmx_ihs;
	vmxq = &sc->vmx_queue[0];

	intrstr = pci_intr_string(pc, *intr, intrbuf, sizeof(intrbuf));

	snprintf(xnamebuf, 32, "%s:legacy", device_xname(sc->vmx_dev));
	pci_intr_setattr(pc, intr, PCI_INTR_MPSAFE, true);
	*ihs = pci_intr_establish_xname(pc, *intr, IPL_NET,
	    vmxnet3_legacy_intr, sc, xnamebuf);
	if (*ihs == NULL) {
		aprint_error_dev(sc->vmx_dev,
		    "unable to establish interrupt at %s\n", intrstr);
		return (-1);
	}
	aprint_normal_dev(sc->vmx_dev, "interrupting at %s\n", intrstr);

	vmxq->vxq_si = softint_establish(SOFTINT_NET | SOFTINT_MPSAFE,
	    vmxnet3_handle_queue, vmxq);
	if (vmxq->vxq_si == NULL) {
		aprint_error_dev(sc->vmx_dev,
		    "softint_establish for vxq_si failed\n");
		return (-1);
	}

	for (i = 0; i < MIN(sc->vmx_nrxqueues, sc->vmx_nrxqueues); i++)
		sc->vmx_queue[i].vxq_intr_idx = 0;
	sc->vmx_event_intr_idx = 0;

	return (0);
}

static void
vmxnet3_set_interrupt_idx(struct vmxnet3_softc *sc)
{
	struct vmxnet3_queue *vmxq;
	struct vmxnet3_txqueue *txq;
	struct vmxnet3_txq_shared *txs;
	struct vmxnet3_rxqueue *rxq;
	struct vmxnet3_rxq_shared *rxs;
	int i;

	sc->vmx_ds->evintr = sc->vmx_event_intr_idx;

	for (i = 0; i < sc->vmx_ntxqueues; i++) {
		vmxq = &sc->vmx_queue[i];
		txq = &vmxq->vxq_txqueue;
		txs = txq->vxtxq_ts;
		txs->intr_idx = vmxq->vxq_intr_idx;
	}

	for (i = 0; i < sc->vmx_nrxqueues; i++) {
		vmxq = &sc->vmx_queue[i];
		rxq = &vmxq->vxq_rxqueue;
		rxs = rxq->vxrxq_rs;
		rxs->intr_idx = vmxq->vxq_intr_idx;
	}
}

static int
vmxnet3_setup_interrupts(struct vmxnet3_softc *sc)
{
	int error;

	switch (sc->vmx_intr_type) {
	case VMXNET3_IT_MSIX:
		error = vmxnet3_setup_msix_interrupts(sc);
		break;
	case VMXNET3_IT_MSI:
		error = vmxnet3_setup_msi_interrupt(sc);
		break;
	case VMXNET3_IT_LEGACY:
		error = vmxnet3_setup_legacy_interrupt(sc);
		break;
	default:
		panic("%s: invalid interrupt type %d", __func__,
		    sc->vmx_intr_type);
	}

	if (error == 0)
		vmxnet3_set_interrupt_idx(sc);

	return (error);
}

static int
vmxnet3_init_rxq(struct vmxnet3_softc *sc, int q)
{
	struct vmxnet3_rxqueue *rxq;
	struct vmxnet3_rxring *rxr;
	int i;

	rxq = &sc->vmx_queue[q].vxq_rxqueue;

	snprintf(rxq->vxrxq_name, sizeof(rxq->vxrxq_name), "%s-rx%d",
	    device_xname(sc->vmx_dev), q);
	rxq->vxrxq_mtx = mutex_obj_alloc(MUTEX_DEFAULT, IPL_NET /* XXX */);

	rxq->vxrxq_sc = sc;

	for (i = 0; i < VMXNET3_RXRINGS_PERQ; i++) {
		rxr = &rxq->vxrxq_cmd_ring[i];
		rxr->vxrxr_rid = i;
		rxr->vxrxr_ndesc = sc->vmx_nrxdescs;
		rxr->vxrxr_rxbuf = kmem_zalloc(rxr->vxrxr_ndesc *
		    sizeof(struct vmxnet3_rxbuf), KM_SLEEP);

		rxq->vxrxq_comp_ring.vxcr_ndesc += sc->vmx_nrxdescs;
	}

	rxq->vxrxq_stopping = true;

	return (0);
}

static int
vmxnet3_init_txq(struct vmxnet3_softc *sc, int q)
{
	struct vmxnet3_txqueue *txq;
	struct vmxnet3_txring *txr;

	txq = &sc->vmx_queue[q].vxq_txqueue;
	txr = &txq->vxtxq_cmd_ring;

	snprintf(txq->vxtxq_name, sizeof(txq->vxtxq_name), "%s-tx%d",
	    device_xname(sc->vmx_dev), q);
	txq->vxtxq_mtx = mutex_obj_alloc(MUTEX_DEFAULT, IPL_NET /* XXX */);

	txq->vxtxq_sc = sc;

	txq->vxtxq_si = softint_establish(SOFTINT_NET | SOFTINT_MPSAFE,
	    vmxnet3_deferred_transmit, txq);
	if (txq->vxtxq_si == NULL) {
		mutex_obj_free(txq->vxtxq_mtx);
		aprint_error_dev(sc->vmx_dev,
		    "softint_establish for vxtxq_si failed\n");
		return ENOMEM;
	}

	txr->vxtxr_ndesc = sc->vmx_ntxdescs;
	txr->vxtxr_txbuf = kmem_zalloc(txr->vxtxr_ndesc *
	    sizeof(struct vmxnet3_txbuf), KM_SLEEP);

	txq->vxtxq_comp_ring.vxcr_ndesc = sc->vmx_ntxdescs;

	txq->vxtxq_interq = pcq_create(sc->vmx_ntxdescs, KM_SLEEP);

	txq->vxtxq_stopping = true;

	return (0);
}

static int
vmxnet3_alloc_rxtx_queues(struct vmxnet3_softc *sc)
{
	int i, error, max_nqueues;

	KASSERT(!cpu_intr_p());
	KASSERT(!cpu_softintr_p());

	/*
	 * Only attempt to create multiple queues if MSIX is available.
	 * This check prevents us from allocating queue structures that
	 * we will not use.
	 *
	 * FreeBSD:
	 * MSIX is disabled by default because its apparently broken for
	 * devices passed through by at least ESXi 5.1.
	 * The hw.pci.honor_msi_blacklist tunable must be set to zero for MSIX.
	 */
	if (sc->vmx_flags & VMXNET3_FLAG_NO_MSIX) {
		sc->vmx_max_nrxqueues = 1;
		sc->vmx_max_ntxqueues = 1;
	}

	max_nqueues = MAX(sc->vmx_max_ntxqueues, sc->vmx_max_nrxqueues);
	sc->vmx_queue = kmem_zalloc(sizeof(struct vmxnet3_queue) * max_nqueues,
	    KM_SLEEP);

	for (i = 0; i < max_nqueues; i++) {
		struct vmxnet3_queue *vmxq = &sc->vmx_queue[i];
		vmxq->vxq_id = i;
	}

	for (i = 0; i < sc->vmx_max_nrxqueues; i++) {
		error = vmxnet3_init_rxq(sc, i);
		if (error)
			return (error);
	}

	for (i = 0; i < sc->vmx_max_ntxqueues; i++) {
		error = vmxnet3_init_txq(sc, i);
		if (error)
			return (error);
	}

	return (0);
}

static void
vmxnet3_destroy_rxq(struct vmxnet3_rxqueue *rxq)
{
	struct vmxnet3_rxring *rxr;
	int i;

	rxq->vxrxq_sc = NULL;

	for (i = 0; i < VMXNET3_RXRINGS_PERQ; i++) {
		rxr = &rxq->vxrxq_cmd_ring[i];

		if (rxr->vxrxr_rxbuf != NULL) {
			kmem_free(rxr->vxrxr_rxbuf,
			    rxr->vxrxr_ndesc * sizeof(struct vmxnet3_rxbuf));
			rxr->vxrxr_rxbuf = NULL;
		}
	}

	if (rxq->vxrxq_mtx != NULL)
		mutex_obj_free(rxq->vxrxq_mtx);
}

static void
vmxnet3_destroy_txq(struct vmxnet3_txqueue *txq)
{
	struct vmxnet3_txring *txr;
	struct mbuf *m;

	txr = &txq->vxtxq_cmd_ring;

	txq->vxtxq_sc = NULL;

	softint_disestablish(txq->vxtxq_si);

	while ((m = pcq_get(txq->vxtxq_interq)) != NULL)
		m_freem(m);
	pcq_destroy(txq->vxtxq_interq);

	if (txr->vxtxr_txbuf != NULL) {
		kmem_free(txr->vxtxr_txbuf,
		    txr->vxtxr_ndesc * sizeof(struct vmxnet3_txbuf));
		txr->vxtxr_txbuf = NULL;
	}

	if (txq->vxtxq_mtx != NULL)
		mutex_obj_free(txq->vxtxq_mtx);
}

static void
vmxnet3_free_rxtx_queues(struct vmxnet3_softc *sc)
{
	int i;

	if (sc->vmx_queue != NULL) {
		int max_nqueues;

		for (i = 0; i < sc->vmx_max_nrxqueues; i++)
			vmxnet3_destroy_rxq(&sc->vmx_queue[i].vxq_rxqueue);

		for (i = 0; i < sc->vmx_max_ntxqueues; i++)
			vmxnet3_destroy_txq(&sc->vmx_queue[i].vxq_txqueue);

		max_nqueues = MAX(sc->vmx_max_nrxqueues, sc->vmx_max_ntxqueues);
		kmem_free(sc->vmx_queue,
		    sizeof(struct vmxnet3_queue) * max_nqueues);
	}
}

static int
vmxnet3_alloc_shared_data(struct vmxnet3_softc *sc)
{
	device_t dev;
	uint8_t *kva;
	size_t size;
	int i, error;

	dev = sc->vmx_dev;

	size = sizeof(struct vmxnet3_driver_shared);
	error = vmxnet3_dma_malloc(sc, size, 1, &sc->vmx_ds_dma);
	if (error) {
		device_printf(dev, "cannot alloc shared memory\n");
		return (error);
	}
	sc->vmx_ds = (struct vmxnet3_driver_shared *) sc->vmx_ds_dma.dma_vaddr;

	size = sc->vmx_ntxqueues * sizeof(struct vmxnet3_txq_shared) +
	    sc->vmx_nrxqueues * sizeof(struct vmxnet3_rxq_shared);
	error = vmxnet3_dma_malloc(sc, size, 128, &sc->vmx_qs_dma);
	if (error) {
		device_printf(dev, "cannot alloc queue shared memory\n");
		return (error);
	}
	sc->vmx_qs = (void *) sc->vmx_qs_dma.dma_vaddr;
	kva = sc->vmx_qs;

	for (i = 0; i < sc->vmx_ntxqueues; i++) {
		sc->vmx_queue[i].vxq_txqueue.vxtxq_ts =
		    (struct vmxnet3_txq_shared *) kva;
		kva += sizeof(struct vmxnet3_txq_shared);
	}
	for (i = 0; i < sc->vmx_nrxqueues; i++) {
		sc->vmx_queue[i].vxq_rxqueue.vxrxq_rs =
		    (struct vmxnet3_rxq_shared *) kva;
		kva += sizeof(struct vmxnet3_rxq_shared);
	}

	if (sc->vmx_flags & VMXNET3_FLAG_RSS) {
		size = sizeof(struct vmxnet3_rss_shared);
		error = vmxnet3_dma_malloc(sc, size, 128, &sc->vmx_rss_dma);
		if (error) {
			device_printf(dev, "cannot alloc rss shared memory\n");
			return (error);
		}
		sc->vmx_rss =
		    (struct vmxnet3_rss_shared *) sc->vmx_rss_dma.dma_vaddr;
	}

	return (0);
}

static void
vmxnet3_free_shared_data(struct vmxnet3_softc *sc)
{

	if (sc->vmx_rss != NULL) {
		vmxnet3_dma_free(sc, &sc->vmx_rss_dma);
		sc->vmx_rss = NULL;
	}

	if (sc->vmx_qs != NULL) {
		vmxnet3_dma_free(sc, &sc->vmx_qs_dma);
		sc->vmx_qs = NULL;
	}

	if (sc->vmx_ds != NULL) {
		vmxnet3_dma_free(sc, &sc->vmx_ds_dma);
		sc->vmx_ds = NULL;
	}
}

static int
vmxnet3_alloc_txq_data(struct vmxnet3_softc *sc)
{
	device_t dev;
	struct vmxnet3_txqueue *txq;
	struct vmxnet3_txring *txr;
	struct vmxnet3_comp_ring *txc;
	size_t descsz, compsz;
	u_int i;
	int q, error;

	dev = sc->vmx_dev;

	for (q = 0; q < sc->vmx_ntxqueues; q++) {
		txq = &sc->vmx_queue[q].vxq_txqueue;
		txr = &txq->vxtxq_cmd_ring;
		txc = &txq->vxtxq_comp_ring;

		descsz = txr->vxtxr_ndesc * sizeof(struct vmxnet3_txdesc);
		compsz = txr->vxtxr_ndesc * sizeof(struct vmxnet3_txcompdesc);

		error = vmxnet3_dma_malloc(sc, descsz, 512, &txr->vxtxr_dma);
		if (error) {
			device_printf(dev, "cannot alloc Tx descriptors for "
			    "queue %d error %d\n", q, error);
			return (error);
		}
		txr->vxtxr_txd =
		    (struct vmxnet3_txdesc *) txr->vxtxr_dma.dma_vaddr;

		error = vmxnet3_dma_malloc(sc, compsz, 512, &txc->vxcr_dma);
		if (error) {
			device_printf(dev, "cannot alloc Tx comp descriptors "
			   "for queue %d error %d\n", q, error);
			return (error);
		}
		txc->vxcr_u.txcd =
		    (struct vmxnet3_txcompdesc *) txc->vxcr_dma.dma_vaddr;

		for (i = 0; i < txr->vxtxr_ndesc; i++) {
			error = bus_dmamap_create(sc->vmx_dmat, VMXNET3_TX_MAXSIZE,
			    VMXNET3_TX_MAXSEGS, VMXNET3_TX_MAXSEGSIZE, 0, BUS_DMA_NOWAIT,
			    &txr->vxtxr_txbuf[i].vtxb_dmamap);
			if (error) {
				device_printf(dev, "unable to create Tx buf "
				    "dmamap for queue %d idx %d\n", q, i);
				return (error);
			}
		}
	}

	return (0);
}

static void
vmxnet3_free_txq_data(struct vmxnet3_softc *sc)
{
	struct vmxnet3_txqueue *txq;
	struct vmxnet3_txring *txr;
	struct vmxnet3_comp_ring *txc;
	struct vmxnet3_txbuf *txb;
	u_int i;
	int q;

	for (q = 0; q < sc->vmx_ntxqueues; q++) {
		txq = &sc->vmx_queue[q].vxq_txqueue;
		txr = &txq->vxtxq_cmd_ring;
		txc = &txq->vxtxq_comp_ring;

		for (i = 0; i < txr->vxtxr_ndesc; i++) {
			txb = &txr->vxtxr_txbuf[i];
			if (txb->vtxb_dmamap != NULL) {
				bus_dmamap_destroy(sc->vmx_dmat,
				    txb->vtxb_dmamap);
				txb->vtxb_dmamap = NULL;
			}
		}

		if (txc->vxcr_u.txcd != NULL) {
			vmxnet3_dma_free(sc, &txc->vxcr_dma);
			txc->vxcr_u.txcd = NULL;
		}

		if (txr->vxtxr_txd != NULL) {
			vmxnet3_dma_free(sc, &txr->vxtxr_dma);
			txr->vxtxr_txd = NULL;
		}
	}
}

static int
vmxnet3_alloc_rxq_data(struct vmxnet3_softc *sc)
{
	device_t dev;
	struct vmxnet3_rxqueue *rxq;
	struct vmxnet3_rxring *rxr;
	struct vmxnet3_comp_ring *rxc;
	int descsz, compsz;
	u_int i, j;
	int q, error;

	dev = sc->vmx_dev;

	for (q = 0; q < sc->vmx_nrxqueues; q++) {
		rxq = &sc->vmx_queue[q].vxq_rxqueue;
		rxc = &rxq->vxrxq_comp_ring;
		compsz = 0;

		for (i = 0; i < VMXNET3_RXRINGS_PERQ; i++) {
			rxr = &rxq->vxrxq_cmd_ring[i];

			descsz = rxr->vxrxr_ndesc *
			    sizeof(struct vmxnet3_rxdesc);
			compsz += rxr->vxrxr_ndesc *
			    sizeof(struct vmxnet3_rxcompdesc);

			error = vmxnet3_dma_malloc(sc, descsz, 512,
			    &rxr->vxrxr_dma);
			if (error) {
				device_printf(dev, "cannot allocate Rx "
				    "descriptors for queue %d/%d error %d\n",
				    i, q, error);
				return (error);
			}
			rxr->vxrxr_rxd =
			    (struct vmxnet3_rxdesc *) rxr->vxrxr_dma.dma_vaddr;
		}

		error = vmxnet3_dma_malloc(sc, compsz, 512, &rxc->vxcr_dma);
		if (error) {
			device_printf(dev, "cannot alloc Rx comp descriptors "
			    "for queue %d error %d\n", q, error);
			return (error);
		}
		rxc->vxcr_u.rxcd =
		    (struct vmxnet3_rxcompdesc *) rxc->vxcr_dma.dma_vaddr;

		for (i = 0; i < VMXNET3_RXRINGS_PERQ; i++) {
			rxr = &rxq->vxrxq_cmd_ring[i];

			error = bus_dmamap_create(sc->vmx_dmat, JUMBO_LEN, 1,
			    JUMBO_LEN, 0, BUS_DMA_NOWAIT,
			    &rxr->vxrxr_spare_dmap);
			if (error) {
				device_printf(dev, "unable to create spare "
				    "dmamap for queue %d/%d error %d\n",
				    q, i, error);
				return (error);
			}

			for (j = 0; j < rxr->vxrxr_ndesc; j++) {
				error = bus_dmamap_create(sc->vmx_dmat, JUMBO_LEN, 1,
				    JUMBO_LEN, 0, BUS_DMA_NOWAIT,
				    &rxr->vxrxr_rxbuf[j].vrxb_dmamap);
				if (error) {
					device_printf(dev, "unable to create "
					    "dmamap for queue %d/%d slot %d "
					    "error %d\n",
					    q, i, j, error);
					return (error);
				}
			}
		}
	}

	return (0);
}

static void
vmxnet3_free_rxq_data(struct vmxnet3_softc *sc)
{
	struct vmxnet3_rxqueue *rxq;
	struct vmxnet3_rxring *rxr;
	struct vmxnet3_comp_ring *rxc;
	struct vmxnet3_rxbuf *rxb;
	u_int i, j;
	int q;

	for (q = 0; q < sc->vmx_nrxqueues; q++) {
		rxq = &sc->vmx_queue[q].vxq_rxqueue;
		rxc = &rxq->vxrxq_comp_ring;

		for (i = 0; i < VMXNET3_RXRINGS_PERQ; i++) {
			rxr = &rxq->vxrxq_cmd_ring[i];

			if (rxr->vxrxr_spare_dmap != NULL) {
				bus_dmamap_destroy(sc->vmx_dmat,
				    rxr->vxrxr_spare_dmap);
				rxr->vxrxr_spare_dmap = NULL;
			}

			for (j = 0; j < rxr->vxrxr_ndesc; j++) {
				rxb = &rxr->vxrxr_rxbuf[j];
				if (rxb->vrxb_dmamap != NULL) {
					bus_dmamap_destroy(sc->vmx_dmat,
					    rxb->vrxb_dmamap);
					rxb->vrxb_dmamap = NULL;
				}
			}
		}

		if (rxc->vxcr_u.rxcd != NULL) {
			vmxnet3_dma_free(sc, &rxc->vxcr_dma);
			rxc->vxcr_u.rxcd = NULL;
		}

		for (i = 0; i < VMXNET3_RXRINGS_PERQ; i++) {
			rxr = &rxq->vxrxq_cmd_ring[i];

			if (rxr->vxrxr_rxd != NULL) {
				vmxnet3_dma_free(sc, &rxr->vxrxr_dma);
				rxr->vxrxr_rxd = NULL;
			}
		}
	}
}

static int
vmxnet3_alloc_queue_data(struct vmxnet3_softc *sc)
{
	int error;

	error = vmxnet3_alloc_txq_data(sc);
	if (error)
		return (error);

	error = vmxnet3_alloc_rxq_data(sc);
	if (error)
		return (error);

	return (0);
}

static void
vmxnet3_free_queue_data(struct vmxnet3_softc *sc)
{

	if (sc->vmx_queue != NULL) {
		vmxnet3_free_rxq_data(sc);
		vmxnet3_free_txq_data(sc);
	}
}

static int
vmxnet3_alloc_mcast_table(struct vmxnet3_softc *sc)
{
	int error;

	error = vmxnet3_dma_malloc(sc, VMXNET3_MULTICAST_MAX * ETHER_ADDR_LEN,
	    32, &sc->vmx_mcast_dma);
	if (error)
		device_printf(sc->vmx_dev, "unable to alloc multicast table\n");
	else
		sc->vmx_mcast = sc->vmx_mcast_dma.dma_vaddr;

	return (error);
}

static void
vmxnet3_free_mcast_table(struct vmxnet3_softc *sc)
{

	if (sc->vmx_mcast != NULL) {
		vmxnet3_dma_free(sc, &sc->vmx_mcast_dma);
		sc->vmx_mcast = NULL;
	}
}

static void
vmxnet3_init_shared_data(struct vmxnet3_softc *sc)
{
	struct vmxnet3_driver_shared *ds;
	struct vmxnet3_txqueue *txq;
	struct vmxnet3_txq_shared *txs;
	struct vmxnet3_rxqueue *rxq;
	struct vmxnet3_rxq_shared *rxs;
	int i;

	ds = sc->vmx_ds;

	/*
	 * Initialize fields of the shared data that remains the same across
	 * reinits. Note the shared data is zero'd when allocated.
	 */

	ds->magic = VMXNET3_REV1_MAGIC;

	/* DriverInfo */
	ds->version = VMXNET3_DRIVER_VERSION;
	ds->guest = VMXNET3_GOS_FREEBSD |
#ifdef __LP64__
	    VMXNET3_GOS_64BIT;
#else
	    VMXNET3_GOS_32BIT;
#endif
	ds->vmxnet3_revision = 1;
	ds->upt_version = 1;

	/* Misc. conf */
	ds->driver_data = vtophys(sc);
	ds->driver_data_len = sizeof(struct vmxnet3_softc);
	ds->queue_shared = sc->vmx_qs_dma.dma_paddr;
	ds->queue_shared_len = sc->vmx_qs_dma.dma_size;
	ds->nrxsg_max = sc->vmx_max_rxsegs;

	/* RSS conf */
	if (sc->vmx_flags & VMXNET3_FLAG_RSS) {
		ds->rss.version = 1;
		ds->rss.paddr = sc->vmx_rss_dma.dma_paddr;
		ds->rss.len = sc->vmx_rss_dma.dma_size;
	}

	/* Interrupt control. */
	ds->automask = sc->vmx_intr_mask_mode == VMXNET3_IMM_AUTO;
	ds->nintr = sc->vmx_nintrs;
	ds->evintr = sc->vmx_event_intr_idx;
	ds->ictrl = VMXNET3_ICTRL_DISABLE_ALL;

	for (i = 0; i < sc->vmx_nintrs; i++)
		ds->modlevel[i] = UPT1_IMOD_ADAPTIVE;

	/* Receive filter. */
	ds->mcast_table = sc->vmx_mcast_dma.dma_paddr;
	ds->mcast_tablelen = sc->vmx_mcast_dma.dma_size;

	/* Tx queues */
	for (i = 0; i < sc->vmx_ntxqueues; i++) {
		txq = &sc->vmx_queue[i].vxq_txqueue;
		txs = txq->vxtxq_ts;

		txs->cmd_ring = txq->vxtxq_cmd_ring.vxtxr_dma.dma_paddr;
		txs->cmd_ring_len = txq->vxtxq_cmd_ring.vxtxr_ndesc;
		txs->comp_ring = txq->vxtxq_comp_ring.vxcr_dma.dma_paddr;
		txs->comp_ring_len = txq->vxtxq_comp_ring.vxcr_ndesc;
		txs->driver_data = vtophys(txq);
		txs->driver_data_len = sizeof(struct vmxnet3_txqueue);
	}

	/* Rx queues */
	for (i = 0; i < sc->vmx_nrxqueues; i++) {
		rxq = &sc->vmx_queue[i].vxq_rxqueue;
		rxs = rxq->vxrxq_rs;

		rxs->cmd_ring[0] = rxq->vxrxq_cmd_ring[0].vxrxr_dma.dma_paddr;
		rxs->cmd_ring_len[0] = rxq->vxrxq_cmd_ring[0].vxrxr_ndesc;
		rxs->cmd_ring[1] = rxq->vxrxq_cmd_ring[1].vxrxr_dma.dma_paddr;
		rxs->cmd_ring_len[1] = rxq->vxrxq_cmd_ring[1].vxrxr_ndesc;
		rxs->comp_ring = rxq->vxrxq_comp_ring.vxcr_dma.dma_paddr;
		rxs->comp_ring_len = rxq->vxrxq_comp_ring.vxcr_ndesc;
		rxs->driver_data = vtophys(rxq);
		rxs->driver_data_len = sizeof(struct vmxnet3_rxqueue);
	}
}

static void
vmxnet3_reinit_rss_shared_data(struct vmxnet3_softc *sc)
{
	/*
	 * Use the same key as the Linux driver until FreeBSD can do
	 * RSS (presumably Toeplitz) in software.
	 */
	static const uint8_t rss_key[UPT1_RSS_MAX_KEY_SIZE] = {
	    0x3b, 0x56, 0xd1, 0x56, 0x13, 0x4a, 0xe7, 0xac,
	    0xe8, 0x79, 0x09, 0x75, 0xe8, 0x65, 0x79, 0x28,
	    0x35, 0x12, 0xb9, 0x56, 0x7c, 0x76, 0x4b, 0x70,
	    0xd8, 0x56, 0xa3, 0x18, 0x9b, 0x0a, 0xee, 0xf3,
	    0x96, 0xa6, 0x9f, 0x8f, 0x9e, 0x8c, 0x90, 0xc9,
	};

	struct vmxnet3_rss_shared *rss;
	int i;

	rss = sc->vmx_rss;

	rss->hash_type =
	    UPT1_RSS_HASH_TYPE_IPV4 | UPT1_RSS_HASH_TYPE_TCP_IPV4 |
	    UPT1_RSS_HASH_TYPE_IPV6 | UPT1_RSS_HASH_TYPE_TCP_IPV6;
	rss->hash_func = UPT1_RSS_HASH_FUNC_TOEPLITZ;
	rss->hash_key_size = UPT1_RSS_MAX_KEY_SIZE;
	rss->ind_table_size = UPT1_RSS_MAX_IND_TABLE_SIZE;
	memcpy(rss->hash_key, rss_key, UPT1_RSS_MAX_KEY_SIZE);

	for (i = 0; i < UPT1_RSS_MAX_IND_TABLE_SIZE; i++)
		rss->ind_table[i] = i % sc->vmx_nrxqueues;
}

static void
vmxnet3_reinit_shared_data(struct vmxnet3_softc *sc)
{
	struct ifnet *ifp;
	struct vmxnet3_driver_shared *ds;

	ifp = &sc->vmx_ethercom.ec_if;
	ds = sc->vmx_ds;

	ds->mtu = ifp->if_mtu;
	ds->ntxqueue = sc->vmx_ntxqueues;
	ds->nrxqueue = sc->vmx_nrxqueues;

	ds->upt_features = 0;
	if (ifp->if_capenable &
	    (IFCAP_CSUM_IPv4_Rx | IFCAP_CSUM_TCPv4_Rx | IFCAP_CSUM_UDPv4_Rx |
	    IFCAP_CSUM_TCPv6_Rx | IFCAP_CSUM_UDPv6_Rx))
		ds->upt_features |= UPT1_F_CSUM;
	if (sc->vmx_ethercom.ec_capenable & ETHERCAP_VLAN_HWTAGGING)
		ds->upt_features |= UPT1_F_VLAN;

	if (sc->vmx_flags & VMXNET3_FLAG_RSS) {
		ds->upt_features |= UPT1_F_RSS;
		vmxnet3_reinit_rss_shared_data(sc);
	}

	vmxnet3_write_bar1(sc, VMXNET3_BAR1_DSL, sc->vmx_ds_dma.dma_paddr);
	vmxnet3_write_bar1(sc, VMXNET3_BAR1_DSH,
	    (uint64_t) sc->vmx_ds_dma.dma_paddr >> 32);
}

static int
vmxnet3_alloc_data(struct vmxnet3_softc *sc)
{
	int error;

	error = vmxnet3_alloc_shared_data(sc);
	if (error)
		return (error);

	error = vmxnet3_alloc_queue_data(sc);
	if (error)
		return (error);

	error = vmxnet3_alloc_mcast_table(sc);
	if (error)
		return (error);

	vmxnet3_init_shared_data(sc);

	return (0);
}

static void
vmxnet3_free_data(struct vmxnet3_softc *sc)
{

	vmxnet3_free_mcast_table(sc);
	vmxnet3_free_queue_data(sc);
	vmxnet3_free_shared_data(sc);
}

static int
vmxnet3_setup_interface(struct vmxnet3_softc *sc)
{
	struct ifnet *ifp = &sc->vmx_ethercom.ec_if;

	vmxnet3_get_lladdr(sc);
	aprint_normal_dev(sc->vmx_dev, "Ethernet address %s\n",
	    ether_sprintf(sc->vmx_lladdr));
	vmxnet3_set_lladdr(sc);

	strlcpy(ifp->if_xname, device_xname(sc->vmx_dev), IFNAMSIZ);
	ifp->if_softc = sc;
	ifp->if_flags = IFF_BROADCAST | IFF_MULTICAST | IFF_SIMPLEX;
	ifp->if_extflags = IFEF_MPSAFE;
	ifp->if_ioctl = vmxnet3_ioctl;
	ifp->if_start = vmxnet3_start;
	ifp->if_transmit = vmxnet3_transmit;
	ifp->if_watchdog = NULL;
	ifp->if_init = vmxnet3_init;
	ifp->if_stop = vmxnet3_stop;
	sc->vmx_ethercom.ec_if.if_capabilities |=IFCAP_CSUM_IPv4_Rx |
		    IFCAP_CSUM_TCPv4_Tx | IFCAP_CSUM_TCPv4_Rx |
		    IFCAP_CSUM_UDPv4_Tx | IFCAP_CSUM_UDPv4_Rx |
		    IFCAP_CSUM_TCPv6_Tx | IFCAP_CSUM_TCPv6_Rx |
		    IFCAP_CSUM_UDPv6_Tx | IFCAP_CSUM_UDPv6_Rx;

	ifp->if_capenable = ifp->if_capabilities;

	sc->vmx_ethercom.ec_if.if_capabilities |= IFCAP_TSOv4 | IFCAP_TSOv6;

	sc->vmx_ethercom.ec_capabilities |=
	    ETHERCAP_VLAN_MTU | ETHERCAP_VLAN_HWTAGGING | ETHERCAP_JUMBO_MTU;
	sc->vmx_ethercom.ec_capenable |= ETHERCAP_VLAN_HWTAGGING;

	IFQ_SET_MAXLEN(&ifp->if_snd, sc->vmx_ntxdescs);
	IFQ_SET_READY(&ifp->if_snd);

	/* Initialize ifmedia structures. */
	sc->vmx_ethercom.ec_ifmedia = &sc->vmx_media;
	ifmedia_init_with_lock(&sc->vmx_media, IFM_IMASK, vmxnet3_ifmedia_change,
	    vmxnet3_ifmedia_status, sc->vmx_mtx);
	ifmedia_add(&sc->vmx_media, IFM_ETHER | IFM_AUTO, 0, NULL);
	ifmedia_add(&sc->vmx_media, IFM_ETHER | IFM_10G_T | IFM_FDX, 0, NULL);
	ifmedia_add(&sc->vmx_media, IFM_ETHER | IFM_10G_T, 0, NULL);
	ifmedia_add(&sc->vmx_media, IFM_ETHER | IFM_1000_T | IFM_FDX, 0, NULL);
	ifmedia_add(&sc->vmx_media, IFM_ETHER | IFM_1000_T, 0, NULL);
	ifmedia_set(&sc->vmx_media, IFM_ETHER | IFM_AUTO);

	if_attach(ifp);
	if_deferred_start_init(ifp, NULL);
	ether_ifattach(ifp, sc->vmx_lladdr);
	ether_set_ifflags_cb(&sc->vmx_ethercom, vmxnet3_ifflags_cb);
	vmxnet3_cmd_link_status(ifp);

	/* should set before setting interrupts */
	sc->vmx_rx_intr_process_limit = VMXNET3_RX_INTR_PROCESS_LIMIT;
	sc->vmx_rx_process_limit = VMXNET3_RX_PROCESS_LIMIT;
	sc->vmx_tx_intr_process_limit = VMXNET3_TX_INTR_PROCESS_LIMIT;
	sc->vmx_tx_process_limit = VMXNET3_TX_PROCESS_LIMIT;

	return (0);
}

static int
vmxnet3_setup_sysctl(struct vmxnet3_softc *sc)
{
	const char *devname;
	struct sysctllog **log;
	const struct sysctlnode *rnode, *rxnode, *txnode;
	int error;

	log = &sc->vmx_sysctllog;
	devname = device_xname(sc->vmx_dev);

	error = sysctl_createv(log, 0, NULL, &rnode,
	    0, CTLTYPE_NODE, devname,
	    SYSCTL_DESCR("vmxnet3 information and settings"),
	    NULL, 0, NULL, 0, CTL_HW, CTL_CREATE, CTL_EOL);
	if (error)
		goto out;
	error = sysctl_createv(log, 0, &rnode, NULL,
	    CTLFLAG_READWRITE, CTLTYPE_BOOL, "txrx_workqueue",
	    SYSCTL_DESCR("Use workqueue for packet processing"),
	    NULL, 0, &sc->vmx_txrx_workqueue, 0, CTL_CREATE, CTL_EOL);
	if (error)
		goto out;

	error = sysctl_createv(log, 0, &rnode, &rxnode,
	    0, CTLTYPE_NODE, "rx",
	    SYSCTL_DESCR("vmxnet3 information and settings for Rx"),
	    NULL, 0, NULL, 0, CTL_CREATE, CTL_EOL);
	if (error)
		goto out;
	error = sysctl_createv(log, 0, &rxnode, NULL,
	    CTLFLAG_READWRITE, CTLTYPE_INT, "intr_process_limit",
	    SYSCTL_DESCR("max number of Rx packets to process for interrupt processing"),
	    NULL, 0, &sc->vmx_rx_intr_process_limit, 0, CTL_CREATE, CTL_EOL);
	if (error)
		goto out;
	error = sysctl_createv(log, 0, &rxnode, NULL,
	    CTLFLAG_READWRITE, CTLTYPE_INT, "process_limit",
	    SYSCTL_DESCR("max number of Rx packets to process for deferred processing"),
	    NULL, 0, &sc->vmx_rx_process_limit, 0, CTL_CREATE, CTL_EOL);
	if (error)
		goto out;

	error = sysctl_createv(log, 0, &rnode, &txnode,
	    0, CTLTYPE_NODE, "tx",
	    SYSCTL_DESCR("vmxnet3 information and settings for Tx"),
	    NULL, 0, NULL, 0, CTL_CREATE, CTL_EOL);
	if (error)
		goto out;
	error = sysctl_createv(log, 0, &txnode, NULL,
	    CTLFLAG_READWRITE, CTLTYPE_INT, "intr_process_limit",
	    SYSCTL_DESCR("max number of Tx packets to process for interrupt processing"),
	    NULL, 0, &sc->vmx_tx_intr_process_limit, 0, CTL_CREATE, CTL_EOL);
	if (error)
		goto out;
	error = sysctl_createv(log, 0, &txnode, NULL,
	    CTLFLAG_READWRITE, CTLTYPE_INT, "process_limit",
	    SYSCTL_DESCR("max number of Tx packets to process for deferred processing"),
	    NULL, 0, &sc->vmx_tx_process_limit, 0, CTL_CREATE, CTL_EOL);

out:
	if (error) {
		aprint_error_dev(sc->vmx_dev,
		    "unable to create sysctl node\n");
		sysctl_teardown(log);
	}
	return error;
}

static int
vmxnet3_setup_stats(struct vmxnet3_softc *sc)
{
	struct vmxnet3_queue *vmxq;
	struct vmxnet3_txqueue *txq;
	struct vmxnet3_rxqueue *rxq;
	int i;

	for (i = 0; i < sc->vmx_ntxqueues; i++) {
		vmxq = &sc->vmx_queue[i];
		txq = &vmxq->vxq_txqueue;
		evcnt_attach_dynamic(&txq->vxtxq_intr, EVCNT_TYPE_INTR,
		    NULL, txq->vxtxq_name, "Interrupt on queue");
		evcnt_attach_dynamic(&txq->vxtxq_defer, EVCNT_TYPE_MISC,
		    NULL, txq->vxtxq_name, "Handled queue in softint/workqueue");
		evcnt_attach_dynamic(&txq->vxtxq_deferreq, EVCNT_TYPE_MISC,
		    NULL, txq->vxtxq_name, "Requested in softint/workqueue");
		evcnt_attach_dynamic(&txq->vxtxq_pcqdrop, EVCNT_TYPE_MISC,
		    NULL, txq->vxtxq_name, "Dropped in pcq");
		evcnt_attach_dynamic(&txq->vxtxq_transmitdef, EVCNT_TYPE_MISC,
		    NULL, txq->vxtxq_name, "Deferred transmit");
		evcnt_attach_dynamic(&txq->vxtxq_watchdogto, EVCNT_TYPE_MISC,
		    NULL, txq->vxtxq_name, "Watchdog timeout");
		evcnt_attach_dynamic(&txq->vxtxq_defragged, EVCNT_TYPE_MISC,
		    NULL, txq->vxtxq_name, "m_defrag successed");
		evcnt_attach_dynamic(&txq->vxtxq_defrag_failed, EVCNT_TYPE_MISC,
		    NULL, txq->vxtxq_name, "m_defrag failed");
	}

	for (i = 0; i < sc->vmx_nrxqueues; i++) {
		vmxq = &sc->vmx_queue[i];
		rxq = &vmxq->vxq_rxqueue;
		evcnt_attach_dynamic(&rxq->vxrxq_intr, EVCNT_TYPE_INTR,
		    NULL, rxq->vxrxq_name, "Interrupt on queue");
		evcnt_attach_dynamic(&rxq->vxrxq_defer, EVCNT_TYPE_MISC,
		    NULL, rxq->vxrxq_name, "Handled queue in softint/workqueue");
		evcnt_attach_dynamic(&rxq->vxrxq_deferreq, EVCNT_TYPE_MISC,
		    NULL, rxq->vxrxq_name, "Requested in softint/workqueue");
		evcnt_attach_dynamic(&rxq->vxrxq_mgetcl_failed, EVCNT_TYPE_MISC,
		    NULL, rxq->vxrxq_name, "MCLGET failed");
		evcnt_attach_dynamic(&rxq->vxrxq_mbuf_load_failed, EVCNT_TYPE_MISC,
		    NULL, rxq->vxrxq_name, "bus_dmamap_load_mbuf failed");
	}

	evcnt_attach_dynamic(&sc->vmx_event_intr, EVCNT_TYPE_INTR,
	    NULL, device_xname(sc->vmx_dev), "Interrupt for other events");
	evcnt_attach_dynamic(&sc->vmx_event_link, EVCNT_TYPE_MISC,
	    NULL, device_xname(sc->vmx_dev), "Link status event");
	evcnt_attach_dynamic(&sc->vmx_event_txqerror, EVCNT_TYPE_MISC,
	    NULL, device_xname(sc->vmx_dev), "Tx queue error event");
	evcnt_attach_dynamic(&sc->vmx_event_rxqerror, EVCNT_TYPE_MISC,
	    NULL, device_xname(sc->vmx_dev), "Rx queue error event");
	evcnt_attach_dynamic(&sc->vmx_event_dic, EVCNT_TYPE_MISC,
	    NULL, device_xname(sc->vmx_dev), "Device impl change event");
	evcnt_attach_dynamic(&sc->vmx_event_debug, EVCNT_TYPE_MISC,
	    NULL, device_xname(sc->vmx_dev), "Debug event");

	return 0;
}

static void
vmxnet3_teardown_stats(struct vmxnet3_softc *sc)
{
	struct vmxnet3_queue *vmxq;
	struct vmxnet3_txqueue *txq;
	struct vmxnet3_rxqueue *rxq;
	int i;

	for (i = 0; i < sc->vmx_ntxqueues; i++) {
		vmxq = &sc->vmx_queue[i];
		txq = &vmxq->vxq_txqueue;
		evcnt_detach(&txq->vxtxq_intr);
		evcnt_detach(&txq->vxtxq_defer);
		evcnt_detach(&txq->vxtxq_deferreq);
		evcnt_detach(&txq->vxtxq_pcqdrop);
		evcnt_detach(&txq->vxtxq_transmitdef);
		evcnt_detach(&txq->vxtxq_watchdogto);
		evcnt_detach(&txq->vxtxq_defragged);
		evcnt_detach(&txq->vxtxq_defrag_failed);
	}

	for (i = 0; i < sc->vmx_nrxqueues; i++) {
		vmxq = &sc->vmx_queue[i];
		rxq = &vmxq->vxq_rxqueue;
		evcnt_detach(&rxq->vxrxq_intr);
		evcnt_detach(&rxq->vxrxq_defer);
		evcnt_detach(&rxq->vxrxq_deferreq);
		evcnt_detach(&rxq->vxrxq_mgetcl_failed);
		evcnt_detach(&rxq->vxrxq_mbuf_load_failed);
	}

	evcnt_detach(&sc->vmx_event_intr);
	evcnt_detach(&sc->vmx_event_link);
	evcnt_detach(&sc->vmx_event_txqerror);
	evcnt_detach(&sc->vmx_event_rxqerror);
	evcnt_detach(&sc->vmx_event_dic);
	evcnt_detach(&sc->vmx_event_debug);
}

static void
vmxnet3_evintr(struct vmxnet3_softc *sc)
{
	device_t dev;
	struct vmxnet3_txq_shared *ts;
	struct vmxnet3_rxq_shared *rs;
	uint32_t event;
	int reset;

	dev = sc->vmx_dev;
	reset = 0;

	VMXNET3_CORE_LOCK(sc);

	/* Clear events. */
	event = sc->vmx_ds->event;
	vmxnet3_write_bar1(sc, VMXNET3_BAR1_EVENT, event);

	if (event & VMXNET3_EVENT_LINK) {
		sc->vmx_event_link.ev_count++;
		vmxnet3_if_link_status(sc);
		if (sc->vmx_link_active != 0)
			if_schedule_deferred_start(&sc->vmx_ethercom.ec_if);
	}

	if (event & (VMXNET3_EVENT_TQERROR | VMXNET3_EVENT_RQERROR)) {
		if (event & VMXNET3_EVENT_TQERROR)
			sc->vmx_event_txqerror.ev_count++;
		if (event & VMXNET3_EVENT_RQERROR)
			sc->vmx_event_rxqerror.ev_count++;

		reset = 1;
		vmxnet3_read_cmd(sc, VMXNET3_CMD_GET_STATUS);
		ts = sc->vmx_queue[0].vxq_txqueue.vxtxq_ts;
		if (ts->stopped != 0)
			device_printf(dev, "Tx queue error %#x\n", ts->error);
		rs = sc->vmx_queue[0].vxq_rxqueue.vxrxq_rs;
		if (rs->stopped != 0)
			device_printf(dev, "Rx queue error %#x\n", rs->error);
		device_printf(dev, "Rx/Tx queue error event ... resetting\n");
	}

	if (event & VMXNET3_EVENT_DIC) {
		sc->vmx_event_dic.ev_count++;
		device_printf(dev, "device implementation change event\n");
	}
	if (event & VMXNET3_EVENT_DEBUG) {
		sc->vmx_event_debug.ev_count++;
		device_printf(dev, "debug event\n");
	}

	if (reset != 0)
		vmxnet3_init_locked(sc);

	VMXNET3_CORE_UNLOCK(sc);
}

static bool
vmxnet3_txq_eof(struct vmxnet3_txqueue *txq, u_int limit)
{
	struct vmxnet3_softc *sc;
	struct vmxnet3_txring *txr;
	struct vmxnet3_comp_ring *txc;
	struct vmxnet3_txcompdesc *txcd;
	struct vmxnet3_txbuf *txb;
	struct ifnet *ifp;
	struct mbuf *m;
	u_int sop;
	bool more = false;

	sc = txq->vxtxq_sc;
	txr = &txq->vxtxq_cmd_ring;
	txc = &txq->vxtxq_comp_ring;
	ifp = &sc->vmx_ethercom.ec_if;

	VMXNET3_TXQ_LOCK_ASSERT(txq);

	net_stat_ref_t nsr = IF_STAT_GETREF(ifp);
	for (;;) {
		if (limit-- == 0) {
			more = true;
			break;
		}

		txcd = &txc->vxcr_u.txcd[txc->vxcr_next];
		if (txcd->gen != txc->vxcr_gen)
			break;
		vmxnet3_barrier(sc, VMXNET3_BARRIER_RD);

		if (++txc->vxcr_next == txc->vxcr_ndesc) {
			txc->vxcr_next = 0;
			txc->vxcr_gen ^= 1;
		}

		sop = txr->vxtxr_next;
		txb = &txr->vxtxr_txbuf[sop];

		if ((m = txb->vtxb_m) != NULL) {
			bus_dmamap_sync(sc->vmx_dmat, txb->vtxb_dmamap,
			    0, txb->vtxb_dmamap->dm_mapsize,
			    BUS_DMASYNC_POSTWRITE);
			bus_dmamap_unload(sc->vmx_dmat, txb->vtxb_dmamap);

			if_statinc_ref(nsr, if_opackets);
			if_statadd_ref(nsr, if_obytes, m->m_pkthdr.len);
			if (m->m_flags & M_MCAST)
				if_statinc_ref(nsr, if_omcasts);

			m_freem(m);
			txb->vtxb_m = NULL;
		}

		txr->vxtxr_next = (txcd->eop_idx + 1) % txr->vxtxr_ndesc;
	}
	IF_STAT_PUTREF(ifp);

	if (txr->vxtxr_head == txr->vxtxr_next)
		txq->vxtxq_watchdog = 0;

	return more;
}

static int
vmxnet3_newbuf(struct vmxnet3_softc *sc, struct vmxnet3_rxqueue *rxq,
    struct vmxnet3_rxring *rxr)
{
	struct mbuf *m;
	struct vmxnet3_rxdesc *rxd;
	struct vmxnet3_rxbuf *rxb;
	bus_dma_tag_t tag;
	bus_dmamap_t dmap;
	int idx, btype, error;

	tag = sc->vmx_dmat;
	dmap = rxr->vxrxr_spare_dmap;
	idx = rxr->vxrxr_fill;
	rxd = &rxr->vxrxr_rxd[idx];
	rxb = &rxr->vxrxr_rxbuf[idx];

	/* Don't allocate buffers for ring 2 for now. */
	if (rxr->vxrxr_rid != 0)
		return -1;
	btype = VMXNET3_BTYPE_HEAD;

	MGETHDR(m, M_DONTWAIT, MT_DATA);
	if (m == NULL)
		return (ENOBUFS);

	MCLGET(m, M_DONTWAIT);
	if ((m->m_flags & M_EXT) == 0) {
		rxq->vxrxq_mgetcl_failed.ev_count++;
		m_freem(m);
		return (ENOBUFS);
	}

	m->m_pkthdr.len = m->m_len = JUMBO_LEN;
	m_adj(m, ETHER_ALIGN);

	error = bus_dmamap_load_mbuf(sc->vmx_dmat, dmap, m, BUS_DMA_NOWAIT);
	if (error) {
		m_freem(m);
		rxq->vxrxq_mbuf_load_failed.ev_count++;
		return (error);
	}

	if (rxb->vrxb_m != NULL) {
		bus_dmamap_sync(tag, rxb->vrxb_dmamap,
		    0, rxb->vrxb_dmamap->dm_mapsize,
		    BUS_DMASYNC_POSTREAD);
		bus_dmamap_unload(tag, rxb->vrxb_dmamap);
	}

	rxr->vxrxr_spare_dmap = rxb->vrxb_dmamap;
	rxb->vrxb_dmamap = dmap;
	rxb->vrxb_m = m;

	rxd->addr = DMAADDR(dmap);
	rxd->len = m->m_pkthdr.len;
	rxd->btype = btype;
	rxd->gen = rxr->vxrxr_gen;

	vmxnet3_rxr_increment_fill(rxr);
	return (0);
}

static void
vmxnet3_rxq_eof_discard(struct vmxnet3_rxqueue *rxq,
    struct vmxnet3_rxring *rxr, int idx)
{
	struct vmxnet3_rxdesc *rxd;

	rxd = &rxr->vxrxr_rxd[idx];
	rxd->gen = rxr->vxrxr_gen;
	vmxnet3_rxr_increment_fill(rxr);
}

static void
vmxnet3_rxq_discard_chain(struct vmxnet3_rxqueue *rxq)
{
	struct vmxnet3_softc *sc;
	struct vmxnet3_rxring *rxr;
	struct vmxnet3_comp_ring *rxc;
	struct vmxnet3_rxcompdesc *rxcd;
	int idx, eof;

	sc = rxq->vxrxq_sc;
	rxc = &rxq->vxrxq_comp_ring;

	do {
		rxcd = &rxc->vxcr_u.rxcd[rxc->vxcr_next];
		if (rxcd->gen != rxc->vxcr_gen)
			break;		/* Not expected. */
		vmxnet3_barrier(sc, VMXNET3_BARRIER_RD);

		if (++rxc->vxcr_next == rxc->vxcr_ndesc) {
			rxc->vxcr_next = 0;
			rxc->vxcr_gen ^= 1;
		}

		idx = rxcd->rxd_idx;
		eof = rxcd->eop;
		if (rxcd->qid < sc->vmx_nrxqueues)
			rxr = &rxq->vxrxq_cmd_ring[0];
		else
			rxr = &rxq->vxrxq_cmd_ring[1];
		vmxnet3_rxq_eof_discard(rxq, rxr, idx);
	} while (!eof);
}

static void
vmxnet3_rx_csum(struct vmxnet3_rxcompdesc *rxcd, struct mbuf *m)
{
	if (rxcd->no_csum)
		return;

	if (rxcd->ipv4) {
		m->m_pkthdr.csum_flags |= M_CSUM_IPv4;
		if (rxcd->ipcsum_ok == 0)
			m->m_pkthdr.csum_flags |= M_CSUM_IPv4_BAD;
	}

	if (rxcd->fragment)
		return;

	if (rxcd->tcp) {
		m->m_pkthdr.csum_flags |=
		    rxcd->ipv4 ? M_CSUM_TCPv4 : M_CSUM_TCPv6;
		if ((rxcd->csum_ok) == 0)
			m->m_pkthdr.csum_flags |= M_CSUM_TCP_UDP_BAD;
	}

	if (rxcd->udp) {
		m->m_pkthdr.csum_flags |=
		    rxcd->ipv4 ? M_CSUM_UDPv4 : M_CSUM_UDPv6 ;
		if ((rxcd->csum_ok) == 0)
			m->m_pkthdr.csum_flags |= M_CSUM_TCP_UDP_BAD;
	}
}

static void
vmxnet3_rxq_input(struct vmxnet3_rxqueue *rxq,
    struct vmxnet3_rxcompdesc *rxcd, struct mbuf *m)
{
	struct vmxnet3_softc *sc;
	struct ifnet *ifp;

	sc = rxq->vxrxq_sc;
	ifp = &sc->vmx_ethercom.ec_if;

	if (rxcd->error) {
		if_statinc(ifp, if_ierrors);
		m_freem(m);
		return;
	}

	if (!rxcd->no_csum)
		vmxnet3_rx_csum(rxcd, m);
	if (rxcd->vlan)
		vlan_set_tag(m, rxcd->vtag);

	net_stat_ref_t nsr = IF_STAT_GETREF(ifp);
	if_statinc_ref(nsr, if_ipackets);
	if_statadd_ref(nsr, if_ibytes, m->m_pkthdr.len);
	IF_STAT_PUTREF(ifp);

	if_percpuq_enqueue(ifp->if_percpuq, m);
}

static bool
vmxnet3_rxq_eof(struct vmxnet3_rxqueue *rxq, u_int limit)
{
	struct vmxnet3_softc *sc;
	struct ifnet *ifp;
	struct vmxnet3_rxring *rxr;
	struct vmxnet3_comp_ring *rxc;
	struct vmxnet3_rxdesc *rxd __diagused;
	struct vmxnet3_rxcompdesc *rxcd;
	struct mbuf *m, *m_head, *m_tail;
	u_int idx, length;
	bool more = false;

	sc = rxq->vxrxq_sc;
	ifp = &sc->vmx_ethercom.ec_if;
	rxc = &rxq->vxrxq_comp_ring;

	VMXNET3_RXQ_LOCK_ASSERT(rxq);

	if (rxq->vxrxq_stopping)
		return more;

	m_head = rxq->vxrxq_mhead;
	rxq->vxrxq_mhead = NULL;
	m_tail = rxq->vxrxq_mtail;
	rxq->vxrxq_mtail = NULL;
	KASSERT(m_head == NULL || m_tail != NULL);

	for (;;) {
		if (limit-- == 0) {
			more = true;
			break;
		}

		rxcd = &rxc->vxcr_u.rxcd[rxc->vxcr_next];
		if (rxcd->gen != rxc->vxcr_gen) {
			rxq->vxrxq_mhead = m_head;
			rxq->vxrxq_mtail = m_tail;
			break;
		}
		vmxnet3_barrier(sc, VMXNET3_BARRIER_RD);

		if (++rxc->vxcr_next == rxc->vxcr_ndesc) {
			rxc->vxcr_next = 0;
			rxc->vxcr_gen ^= 1;
		}

		idx = rxcd->rxd_idx;
		length = rxcd->len;
		if (rxcd->qid < sc->vmx_nrxqueues)
			rxr = &rxq->vxrxq_cmd_ring[0];
		else
			rxr = &rxq->vxrxq_cmd_ring[1];
		rxd = &rxr->vxrxr_rxd[idx];

		m = rxr->vxrxr_rxbuf[idx].vrxb_m;
		KASSERT(m != NULL);

		/*
		 * The host may skip descriptors. We detect this when this
		 * descriptor does not match the previous fill index. Catch
		 * up with the host now.
		 */
		if (__predict_false(rxr->vxrxr_fill != idx)) {
			while (rxr->vxrxr_fill != idx) {
				rxr->vxrxr_rxd[rxr->vxrxr_fill].gen =
				    rxr->vxrxr_gen;
				vmxnet3_rxr_increment_fill(rxr);
			}
		}

		if (rxcd->sop) {
			/* start of frame w/o head buffer */
			KASSERT(rxd->btype == VMXNET3_BTYPE_HEAD);
			/* start of frame not in ring 0 */
			KASSERT(rxr == &rxq->vxrxq_cmd_ring[0]);
			/* duplicate start of frame? */
			KASSERT(m_head == NULL);

			if (length == 0) {
				/* Just ignore this descriptor. */
				vmxnet3_rxq_eof_discard(rxq, rxr, idx);
				goto nextp;
			}

			if (vmxnet3_newbuf(sc, rxq, rxr) != 0) {
				if_statinc(ifp, if_iqdrops);
				vmxnet3_rxq_eof_discard(rxq, rxr, idx);
				if (!rxcd->eop)
					vmxnet3_rxq_discard_chain(rxq);
				goto nextp;
			}

			m_set_rcvif(m, ifp);
			m->m_pkthdr.len = m->m_len = length;
			m->m_pkthdr.csum_flags = 0;
			m_head = m_tail = m;

		} else {
			/* non start of frame w/o body buffer */
			KASSERT(rxd->btype == VMXNET3_BTYPE_BODY);
			/* frame not started? */
			KASSERT(m_head != NULL);

			if (vmxnet3_newbuf(sc, rxq, rxr) != 0) {
				if_statinc(ifp, if_iqdrops);
				vmxnet3_rxq_eof_discard(rxq, rxr, idx);
				if (!rxcd->eop)
					vmxnet3_rxq_discard_chain(rxq);
				m_freem(m_head);
				m_head = m_tail = NULL;
				goto nextp;
			}

			m->m_len = length;
			m_head->m_pkthdr.len += length;
			m_tail->m_next = m;
			m_tail = m;
		}

		if (rxcd->eop) {
			vmxnet3_rxq_input(rxq, rxcd, m_head);
			m_head = m_tail = NULL;

			/* Must recheck after dropping the Rx lock. */
			if (rxq->vxrxq_stopping)
				break;
		}

nextp:
		if (__predict_false(rxq->vxrxq_rs->update_rxhead)) {
			int qid = rxcd->qid;
			bus_size_t r;

			idx = (idx + 1) % rxr->vxrxr_ndesc;
			if (qid >= sc->vmx_nrxqueues) {
				qid -= sc->vmx_nrxqueues;
				r = VMXNET3_BAR0_RXH2(qid);
			} else
				r = VMXNET3_BAR0_RXH1(qid);
			vmxnet3_write_bar0(sc, r, idx);
		}
	}

	return more;
}

static inline void
vmxnet3_sched_handle_queue(struct vmxnet3_softc *sc, struct vmxnet3_queue *vmxq)
{

	if (vmxq->vxq_workqueue) {
		/*
		 * When this function is called, "vmxq" is owned by one CPU.
		 * so, atomic operation is not required here.
		 */
		if (!vmxq->vxq_wq_enqueued) {
			vmxq->vxq_wq_enqueued = true;
			workqueue_enqueue(sc->vmx_queue_wq,
			    &vmxq->vxq_wq_cookie, curcpu());
		}
	} else {
		softint_schedule(vmxq->vxq_si);
	}
}

static int
vmxnet3_legacy_intr(void *xsc)
{
	struct vmxnet3_softc *sc;
	struct vmxnet3_queue *vmxq;
	struct vmxnet3_txqueue *txq;
	struct vmxnet3_rxqueue *rxq;
	u_int txlimit, rxlimit;
	bool txmore, rxmore;

	sc = xsc;
	vmxq = &sc->vmx_queue[0];
	txq = &vmxq->vxq_txqueue;
	rxq = &vmxq->vxq_rxqueue;
	txlimit = sc->vmx_tx_intr_process_limit;
	rxlimit = sc->vmx_rx_intr_process_limit;

	if (sc->vmx_intr_type == VMXNET3_IT_LEGACY) {
		if (vmxnet3_read_bar1(sc, VMXNET3_BAR1_INTR) == 0)
			return (0);
	}
	if (sc->vmx_intr_mask_mode == VMXNET3_IMM_ACTIVE)
		vmxnet3_disable_all_intrs(sc);

	if (sc->vmx_ds->event != 0)
		vmxnet3_evintr(sc);

	VMXNET3_TXQ_LOCK(txq);
	txmore = vmxnet3_txq_eof(txq, txlimit);
	VMXNET3_TXQ_UNLOCK(txq);

	VMXNET3_RXQ_LOCK(rxq);
	rxmore = vmxnet3_rxq_eof(rxq, rxlimit);
	VMXNET3_RXQ_UNLOCK(rxq);

	if (txmore || rxmore)
		vmxnet3_sched_handle_queue(sc, vmxq);
	else {
		if_schedule_deferred_start(&sc->vmx_ethercom.ec_if);
		vmxnet3_enable_all_intrs(sc);
	}

	return (1);
}

static int
vmxnet3_txrxq_intr(void *xvmxq)
{
	struct vmxnet3_softc *sc;
	struct vmxnet3_queue *vmxq;
	struct vmxnet3_txqueue *txq;
	struct vmxnet3_rxqueue *rxq;
	u_int txlimit, rxlimit;
	bool txmore, rxmore;

	vmxq = xvmxq;
	txq = &vmxq->vxq_txqueue;
	rxq = &vmxq->vxq_rxqueue;
	sc = txq->vxtxq_sc;
	txlimit = sc->vmx_tx_intr_process_limit;
	rxlimit = sc->vmx_rx_intr_process_limit;
	vmxq->vxq_workqueue = sc->vmx_txrx_workqueue;

	if (sc->vmx_intr_mask_mode == VMXNET3_IMM_ACTIVE)
		vmxnet3_disable_intr(sc, vmxq->vxq_intr_idx);

	VMXNET3_TXQ_LOCK(txq);
	txq->vxtxq_intr.ev_count++;
	txmore = vmxnet3_txq_eof(txq, txlimit);
	VMXNET3_TXQ_UNLOCK(txq);

	VMXNET3_RXQ_LOCK(rxq);
	rxq->vxrxq_intr.ev_count++;
	rxmore = vmxnet3_rxq_eof(rxq, rxlimit);
	VMXNET3_RXQ_UNLOCK(rxq);

	if (txmore || rxmore)
		vmxnet3_sched_handle_queue(sc, vmxq);
	else {
		/* for ALTQ */
		if (vmxq->vxq_id == 0)
			if_schedule_deferred_start(&sc->vmx_ethercom.ec_if);
		softint_schedule(txq->vxtxq_si);

		vmxnet3_enable_intr(sc, vmxq->vxq_intr_idx);
	}

	return (1);
}

static void
vmxnet3_handle_queue(void *xvmxq)
{
	struct vmxnet3_softc *sc;
	struct vmxnet3_queue *vmxq;
	struct vmxnet3_txqueue *txq;
	struct vmxnet3_rxqueue *rxq;
	u_int txlimit, rxlimit;
	bool txmore, rxmore;

	vmxq = xvmxq;
	txq = &vmxq->vxq_txqueue;
	rxq = &vmxq->vxq_rxqueue;
	sc = txq->vxtxq_sc;
	txlimit = sc->vmx_tx_process_limit;
	rxlimit = sc->vmx_rx_process_limit;

	VMXNET3_TXQ_LOCK(txq);
	txq->vxtxq_defer.ev_count++;
	txmore = vmxnet3_txq_eof(txq, txlimit);
	if (txmore)
		txq->vxtxq_deferreq.ev_count++;
	/* for ALTQ */
	if (vmxq->vxq_id == 0)
		if_schedule_deferred_start(&sc->vmx_ethercom.ec_if);
	softint_schedule(txq->vxtxq_si);
	VMXNET3_TXQ_UNLOCK(txq);

	VMXNET3_RXQ_LOCK(rxq);
	rxq->vxrxq_defer.ev_count++;
	rxmore = vmxnet3_rxq_eof(rxq, rxlimit);
	if (rxmore)
		rxq->vxrxq_deferreq.ev_count++;
	VMXNET3_RXQ_UNLOCK(rxq);

	if (txmore || rxmore)
		vmxnet3_sched_handle_queue(sc, vmxq);
	else
		vmxnet3_enable_intr(sc, vmxq->vxq_intr_idx);
}

static void
vmxnet3_handle_queue_work(struct work *wk, void *context)
{
	struct vmxnet3_queue *vmxq;

	vmxq = container_of(wk, struct vmxnet3_queue, vxq_wq_cookie);
	vmxq->vxq_wq_enqueued = false;
	vmxnet3_handle_queue(vmxq);
}

static int
vmxnet3_event_intr(void *xsc)
{
	struct vmxnet3_softc *sc;

	sc = xsc;

	if (sc->vmx_intr_mask_mode == VMXNET3_IMM_ACTIVE)
		vmxnet3_disable_intr(sc, sc->vmx_event_intr_idx);

	sc->vmx_event_intr.ev_count++;

	if (sc->vmx_ds->event != 0)
		vmxnet3_evintr(sc);

	vmxnet3_enable_intr(sc, sc->vmx_event_intr_idx);

	return (1);
}

static void
vmxnet3_txstop(struct vmxnet3_softc *sc, struct vmxnet3_txqueue *txq)
{
	struct vmxnet3_txring *txr;
	struct vmxnet3_txbuf *txb;
	u_int i;

	txr = &txq->vxtxq_cmd_ring;

	for (i = 0; i < txr->vxtxr_ndesc; i++) {
		txb = &txr->vxtxr_txbuf[i];

		if (txb->vtxb_m == NULL)
			continue;

		bus_dmamap_sync(sc->vmx_dmat, txb->vtxb_dmamap,
		    0, txb->vtxb_dmamap->dm_mapsize,
		    BUS_DMASYNC_POSTWRITE);
		bus_dmamap_unload(sc->vmx_dmat, txb->vtxb_dmamap);
		m_freem(txb->vtxb_m);
		txb->vtxb_m = NULL;
	}
}

static void
vmxnet3_rxstop(struct vmxnet3_softc *sc, struct vmxnet3_rxqueue *rxq)
{
	struct vmxnet3_rxring *rxr;
	struct vmxnet3_rxbuf *rxb;
	u_int i, j;

	if (rxq->vxrxq_mhead != NULL) {
		m_freem(rxq->vxrxq_mhead);
		rxq->vxrxq_mhead = NULL;
		rxq->vxrxq_mtail = NULL;
	}

	for (i = 0; i < VMXNET3_RXRINGS_PERQ; i++) {
		rxr = &rxq->vxrxq_cmd_ring[i];

		for (j = 0; j < rxr->vxrxr_ndesc; j++) {
			rxb = &rxr->vxrxr_rxbuf[j];

			if (rxb->vrxb_m == NULL)
				continue;

			bus_dmamap_sync(sc->vmx_dmat, rxb->vrxb_dmamap,
			    0, rxb->vrxb_dmamap->dm_mapsize,
			    BUS_DMASYNC_POSTREAD);
			bus_dmamap_unload(sc->vmx_dmat, rxb->vrxb_dmamap);
			m_freem(rxb->vrxb_m);
			rxb->vrxb_m = NULL;
		}
	}
}

static void
vmxnet3_stop_rendezvous(struct vmxnet3_softc *sc)
{
	struct vmxnet3_rxqueue *rxq;
	struct vmxnet3_txqueue *txq;
	struct vmxnet3_queue *vmxq;
	int i;

	for (i = 0; i < sc->vmx_nrxqueues; i++) {
		rxq = &sc->vmx_queue[i].vxq_rxqueue;
		VMXNET3_RXQ_LOCK(rxq);
		rxq->vxrxq_stopping = true;
		VMXNET3_RXQ_UNLOCK(rxq);
	}
	for (i = 0; i < sc->vmx_ntxqueues; i++) {
		txq = &sc->vmx_queue[i].vxq_txqueue;
		VMXNET3_TXQ_LOCK(txq);
		txq->vxtxq_stopping = true;
		VMXNET3_TXQ_UNLOCK(txq);
	}
	for (i = 0; i < sc->vmx_nrxqueues; i++) {
		vmxq = &sc->vmx_queue[i];
		workqueue_wait(sc->vmx_queue_wq, &vmxq->vxq_wq_cookie);
	}
}

static void
vmxnet3_stop_locked(struct vmxnet3_softc *sc)
{
	struct ifnet *ifp = &sc->vmx_ethercom.ec_if;
	int q;

	VMXNET3_CORE_LOCK_ASSERT(sc);
	KASSERT(IFNET_LOCKED(ifp));

	vmxnet3_stop_rendezvous(sc);

	sc->vmx_mcastactive = false;
	sc->vmx_link_active = 0;
	callout_halt(&sc->vmx_tick, sc->vmx_mtx);

	ifp->if_flags &= ~IFF_RUNNING;

	/* Disable interrupts. */
	vmxnet3_disable_all_intrs(sc);
	vmxnet3_write_cmd(sc, VMXNET3_CMD_DISABLE);

	for (q = 0; q < sc->vmx_ntxqueues; q++)
		vmxnet3_txstop(sc, &sc->vmx_queue[q].vxq_txqueue);
	for (q = 0; q < sc->vmx_nrxqueues; q++)
		vmxnet3_rxstop(sc, &sc->vmx_queue[q].vxq_rxqueue);

	vmxnet3_write_cmd(sc, VMXNET3_CMD_RESET);
}

static void
vmxnet3_stop(struct ifnet *ifp, int disable)
{
	struct vmxnet3_softc *sc = ifp->if_softc;

	KASSERT(IFNET_LOCKED(ifp));

	VMXNET3_CORE_LOCK(sc);
	vmxnet3_stop_locked(sc);
	VMXNET3_CORE_UNLOCK(sc);
}

static void
vmxnet3_txinit(struct vmxnet3_softc *sc, struct vmxnet3_txqueue *txq)
{
	struct vmxnet3_txring *txr;
	struct vmxnet3_comp_ring *txc;

	txr = &txq->vxtxq_cmd_ring;
	txr->vxtxr_head = 0;
	txr->vxtxr_next = 0;
	txr->vxtxr_gen = VMXNET3_INIT_GEN;
	memset(txr->vxtxr_txd, 0,
	    txr->vxtxr_ndesc * sizeof(struct vmxnet3_txdesc));

	txc = &txq->vxtxq_comp_ring;
	txc->vxcr_next = 0;
	txc->vxcr_gen = VMXNET3_INIT_GEN;
	memset(txc->vxcr_u.txcd, 0,
	    txc->vxcr_ndesc * sizeof(struct vmxnet3_txcompdesc));
}

static int
vmxnet3_rxinit(struct vmxnet3_softc *sc, struct vmxnet3_rxqueue *rxq)
{
	struct vmxnet3_rxring *rxr;
	struct vmxnet3_comp_ring *rxc;
	u_int i, populate, idx;
	int error;

	/* LRO and jumbo frame is not supported yet */
	populate = 1;

	for (i = 0; i < populate; i++) {
		rxr = &rxq->vxrxq_cmd_ring[i];
		rxr->vxrxr_fill = 0;
		rxr->vxrxr_gen = VMXNET3_INIT_GEN;
		memset(rxr->vxrxr_rxd, 0,
		    rxr->vxrxr_ndesc * sizeof(struct vmxnet3_rxdesc));

		for (idx = 0; idx < rxr->vxrxr_ndesc; idx++) {
			error = vmxnet3_newbuf(sc, rxq, rxr);
			if (error)
				return (error);
		}
	}

	for (/**/; i < VMXNET3_RXRINGS_PERQ; i++) {
		rxr = &rxq->vxrxq_cmd_ring[i];
		rxr->vxrxr_fill = 0;
		rxr->vxrxr_gen = 0;
		memset(rxr->vxrxr_rxd, 0,
		    rxr->vxrxr_ndesc * sizeof(struct vmxnet3_rxdesc));
	}

	rxc = &rxq->vxrxq_comp_ring;
	rxc->vxcr_next = 0;
	rxc->vxcr_gen = VMXNET3_INIT_GEN;
	memset(rxc->vxcr_u.rxcd, 0,
	    rxc->vxcr_ndesc * sizeof(struct vmxnet3_rxcompdesc));

	return (0);
}

static int
vmxnet3_reinit_queues(struct vmxnet3_softc *sc)
{
	device_t dev;
	int q, error;
	dev = sc->vmx_dev;

	for (q = 0; q < sc->vmx_ntxqueues; q++)
		vmxnet3_txinit(sc, &sc->vmx_queue[q].vxq_txqueue);

	for (q = 0; q < sc->vmx_nrxqueues; q++) {
		error = vmxnet3_rxinit(sc, &sc->vmx_queue[q].vxq_rxqueue);
		if (error) {
			device_printf(dev, "cannot populate Rx queue %d\n", q);
			return (error);
		}
	}

	return (0);
}

static int
vmxnet3_enable_device(struct vmxnet3_softc *sc)
{
	int q;

	if (vmxnet3_read_cmd(sc, VMXNET3_CMD_ENABLE) != 0) {
		device_printf(sc->vmx_dev, "device enable command failed!\n");
		return (1);
	}

	/* Reset the Rx queue heads. */
	for (q = 0; q < sc->vmx_nrxqueues; q++) {
		vmxnet3_write_bar0(sc, VMXNET3_BAR0_RXH1(q), 0);
		vmxnet3_write_bar0(sc, VMXNET3_BAR0_RXH2(q), 0);
	}

	return (0);
}

static void
vmxnet3_reinit_rxfilters(struct vmxnet3_softc *sc)
{

	vmxnet3_set_rxfilter(sc);

	memset(sc->vmx_ds->vlan_filter, 0, sizeof(sc->vmx_ds->vlan_filter));
	vmxnet3_write_cmd(sc, VMXNET3_CMD_VLAN_FILTER);
}

static int
vmxnet3_reinit(struct vmxnet3_softc *sc)
{

	VMXNET3_CORE_LOCK_ASSERT(sc);

	vmxnet3_set_lladdr(sc);
	vmxnet3_reinit_shared_data(sc);

	if (vmxnet3_reinit_queues(sc) != 0)
		return (ENXIO);

	if (vmxnet3_enable_device(sc) != 0)
		return (ENXIO);

	vmxnet3_reinit_rxfilters(sc);

	return (0);
}

static int
vmxnet3_init_locked(struct vmxnet3_softc *sc)
{
	struct ifnet *ifp = &sc->vmx_ethercom.ec_if;
	int q;
	int error;

	KASSERT(IFNET_LOCKED(ifp));
	VMXNET3_CORE_LOCK_ASSERT(sc);

	vmxnet3_stop_locked(sc);

	error = vmxnet3_reinit(sc);
	if (error) {
		vmxnet3_stop_locked(sc);
		return (error);
	}

	ifp->if_flags |= IFF_RUNNING;
	vmxnet3_if_link_status(sc);
	sc->vmx_mcastactive = true;

	vmxnet3_enable_all_intrs(sc);
	callout_reset(&sc->vmx_tick, hz, vmxnet3_tick, sc);

	for (q = 0; q < sc->vmx_ntxqueues; q++) {
		VMXNET3_TXQ_LOCK(&sc->vmx_queue[q].vxq_txqueue);
		sc->vmx_queue[q].vxq_txqueue.vxtxq_stopping = false;
		VMXNET3_TXQ_UNLOCK(&sc->vmx_queue[q].vxq_txqueue);
	}
	for (q = 0; q < sc->vmx_nrxqueues; q++) {
		VMXNET3_RXQ_LOCK(&sc->vmx_queue[q].vxq_rxqueue);
		sc->vmx_queue[q].vxq_rxqueue.vxrxq_stopping = false;
		VMXNET3_RXQ_UNLOCK(&sc->vmx_queue[q].vxq_rxqueue);
	}

	return (0);
}

static int
vmxnet3_init(struct ifnet *ifp)
{
	struct vmxnet3_softc *sc = ifp->if_softc;
	int error;

	KASSERT(IFNET_LOCKED(ifp));

	VMXNET3_CORE_LOCK(sc);
	error = vmxnet3_init_locked(sc);
	VMXNET3_CORE_UNLOCK(sc);

	return (error);
}

static int
vmxnet3_txq_offload_ctx(struct vmxnet3_txqueue *txq, struct mbuf *m,
    int *start, int *csum_start)
{
	struct ether_header *eh;
	struct mbuf *mp;
	int offset, csum_off, iphl, offp;
	bool v4;

	eh = mtod(m, struct ether_header *);
	switch (htons(eh->ether_type)) {
	case ETHERTYPE_IP:
	case ETHERTYPE_IPV6:
		offset = ETHER_HDR_LEN;
		break;
	case ETHERTYPE_VLAN:
		offset = ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN;
		break;
	default:
		m_freem(m);
		return (EINVAL);
	}

	if ((m->m_pkthdr.csum_flags &
	    (M_CSUM_TSOv4 | M_CSUM_UDPv4 | M_CSUM_TCPv4)) != 0) {
		iphl = M_CSUM_DATA_IPv4_IPHL(m->m_pkthdr.csum_data);
		v4 = true;
	} else {
		iphl = M_CSUM_DATA_IPv6_IPHL(m->m_pkthdr.csum_data);
		v4 = false;
	}
	*start = offset + iphl;

	if (m->m_pkthdr.csum_flags &
	    (M_CSUM_TCPv4 | M_CSUM_TCPv6 | M_CSUM_TSOv4 | M_CSUM_TSOv6)) {
		csum_off = offsetof(struct tcphdr, th_sum);
	} else {
		csum_off = offsetof(struct udphdr, uh_sum);
	}

	*csum_start = *start + csum_off;
	mp = m_pulldown(m, 0, *csum_start + 2, &offp);
	if (!mp) {
		/* m is already freed */
		return ENOBUFS;
	}

	if (m->m_pkthdr.csum_flags & (M_CSUM_TSOv4 | M_CSUM_TSOv6)) {
		struct tcphdr *tcp;

		txq->vxtxq_stats.vmtxs_tso++;
		tcp = (void *)(mtod(mp, char *) + offp + *start);

		if (v4) {
			struct ip *ip;

			ip = (void *)(mtod(mp, char *) + offp + offset);
			tcp->th_sum = in_cksum_phdr(ip->ip_src.s_addr,
			    ip->ip_dst.s_addr, htons(IPPROTO_TCP));
		} else {
			struct ip6_hdr *ip6;

			ip6 = (void *)(mtod(mp, char *) + offp + offset);
			tcp->th_sum = in6_cksum_phdr(&ip6->ip6_src,
			    &ip6->ip6_dst, 0, htonl(IPPROTO_TCP));
		}

		/*
		 * For TSO, the size of the protocol header is also
		 * included in the descriptor header size.
		 */
		*start += (tcp->th_off << 2);
	} else
		txq->vxtxq_stats.vmtxs_csum++;

	return (0);
}

static int
vmxnet3_txq_load_mbuf(struct vmxnet3_txqueue *txq, struct mbuf **m0,
    bus_dmamap_t dmap)
{
	struct mbuf *m;
	bus_dma_tag_t tag;
	int error;

	m = *m0;
	tag = txq->vxtxq_sc->vmx_dmat;

	error = bus_dmamap_load_mbuf(tag, dmap, m, BUS_DMA_NOWAIT);
	if (error == 0 || error != EFBIG)
		return (error);

	m = m_defrag(m, M_NOWAIT);
	if (m != NULL) {
		*m0 = m;
		error = bus_dmamap_load_mbuf(tag, dmap, m, BUS_DMA_NOWAIT);
	} else
		error = ENOBUFS;

	if (error) {
		m_freem(*m0);
		*m0 = NULL;
		txq->vxtxq_defrag_failed.ev_count++;
	} else
		txq->vxtxq_defragged.ev_count++;

	return (error);
}

static void
vmxnet3_txq_unload_mbuf(struct vmxnet3_txqueue *txq, bus_dmamap_t dmap)
{

	bus_dmamap_unload(txq->vxtxq_sc->vmx_dmat, dmap);
}

static int
vmxnet3_txq_encap(struct vmxnet3_txqueue *txq, struct mbuf **m0)
{
	struct vmxnet3_softc *sc;
	struct vmxnet3_txring *txr;
	struct vmxnet3_txdesc *txd, *sop;
	struct mbuf *m;
	bus_dmamap_t dmap;
	bus_dma_segment_t *segs;
	int i, gen, start, csum_start, nsegs, error;

	sc = txq->vxtxq_sc;
	start = 0;
	txd = NULL;
	txr = &txq->vxtxq_cmd_ring;
	dmap = txr->vxtxr_txbuf[txr->vxtxr_head].vtxb_dmamap;
	csum_start = 0; /* GCC */

	error = vmxnet3_txq_load_mbuf(txq, m0, dmap);
	if (error)
		return (error);

	nsegs = dmap->dm_nsegs;
	segs = dmap->dm_segs;

	m = *m0;
	KASSERT(m->m_flags & M_PKTHDR);
	KASSERT(nsegs <= VMXNET3_TX_MAXSEGS);

	if (vmxnet3_txring_avail(txr) < nsegs) {
		txq->vxtxq_stats.vmtxs_full++;
		vmxnet3_txq_unload_mbuf(txq, dmap);
		return (ENOSPC);
	} else if (m->m_pkthdr.csum_flags & VMXNET3_CSUM_ALL_OFFLOAD) {
		error = vmxnet3_txq_offload_ctx(txq, m, &start, &csum_start);
		if (error) {
			/* m is already freed */
			txq->vxtxq_stats.vmtxs_offload_failed++;
			vmxnet3_txq_unload_mbuf(txq, dmap);
			*m0 = NULL;
			return (error);
		}
	}

	txr->vxtxr_txbuf[txr->vxtxr_head].vtxb_m = m;
	sop = &txr->vxtxr_txd[txr->vxtxr_head];
	gen = txr->vxtxr_gen ^ 1;	/* Owned by cpu (yet) */

	for (i = 0; i < nsegs; i++) {
		txd = &txr->vxtxr_txd[txr->vxtxr_head];

		txd->addr = segs[i].ds_addr;
		txd->len = segs[i].ds_len;
		txd->gen = gen;
		txd->dtype = 0;
		txd->offload_mode = VMXNET3_OM_NONE;
		txd->offload_pos = 0;
		txd->hlen = 0;
		txd->eop = 0;
		txd->compreq = 0;
		txd->vtag_mode = 0;
		txd->vtag = 0;

		if (++txr->vxtxr_head == txr->vxtxr_ndesc) {
			txr->vxtxr_head = 0;
			txr->vxtxr_gen ^= 1;
		}
		gen = txr->vxtxr_gen;
	}
	txd->eop = 1;
	txd->compreq = 1;

	if (vlan_has_tag(m)) {
		sop->vtag_mode = 1;
		sop->vtag = vlan_get_tag(m);
	}

	if (m->m_pkthdr.csum_flags & (M_CSUM_TSOv4 | M_CSUM_TSOv6)) {
		sop->offload_mode = VMXNET3_OM_TSO;
		sop->hlen = start;
		sop->offload_pos = m->m_pkthdr.segsz;
	} else if (m->m_pkthdr.csum_flags & (VMXNET3_CSUM_OFFLOAD |
	    VMXNET3_CSUM_OFFLOAD_IPV6)) {
		sop->offload_mode = VMXNET3_OM_CSUM;
		sop->hlen = start;
		sop->offload_pos = csum_start;
	}

	/* Finally, change the ownership. */
	vmxnet3_barrier(sc, VMXNET3_BARRIER_WR);
	sop->gen ^= 1;

	txq->vxtxq_ts->npending += nsegs;
	if (txq->vxtxq_ts->npending >= txq->vxtxq_ts->intr_threshold) {
		struct vmxnet3_queue *vmxq;
		vmxq = container_of(txq, struct vmxnet3_queue, vxq_txqueue);
		txq->vxtxq_ts->npending = 0;
		vmxnet3_write_bar0(sc, VMXNET3_BAR0_TXH(vmxq->vxq_id),
		    txr->vxtxr_head);
	}

	return (0);
}

#define VMXNET3_TX_START 1
#define VMXNET3_TX_TRANSMIT 2
static inline void
vmxnet3_tx_common_locked(struct ifnet *ifp, struct vmxnet3_txqueue *txq, int txtype)
{
	struct vmxnet3_softc *sc;
	struct vmxnet3_txring *txr;
	struct mbuf *m_head;
	int tx;

	sc = ifp->if_softc;
	txr = &txq->vxtxq_cmd_ring;
	tx = 0;

	VMXNET3_TXQ_LOCK_ASSERT(txq);

	if (txq->vxtxq_stopping || sc->vmx_link_active == 0)
		return;

	for (;;) {
		if (txtype == VMXNET3_TX_START)
			IFQ_POLL(&ifp->if_snd, m_head);
		else
			m_head = pcq_peek(txq->vxtxq_interq);
		if (m_head == NULL)
			break;

		if (vmxnet3_txring_avail(txr) < VMXNET3_TX_MAXSEGS)
			break;

		if (txtype == VMXNET3_TX_START)
			IFQ_DEQUEUE(&ifp->if_snd, m_head);
		else
			m_head = pcq_get(txq->vxtxq_interq);
		if (m_head == NULL)
			break;

		if (vmxnet3_txq_encap(txq, &m_head) != 0) {
			if (m_head != NULL)
				m_freem(m_head);
			break;
		}

		tx++;
		bpf_mtap(ifp, m_head, BPF_D_OUT);
	}

	if (tx > 0)
		txq->vxtxq_watchdog = VMXNET3_WATCHDOG_TIMEOUT;
}

static void
vmxnet3_start_locked(struct ifnet *ifp)
{
	struct vmxnet3_softc *sc;
	struct vmxnet3_txqueue *txq;

	sc = ifp->if_softc;
	txq = &sc->vmx_queue[0].vxq_txqueue;

	vmxnet3_tx_common_locked(ifp, txq, VMXNET3_TX_START);
}

void
vmxnet3_start(struct ifnet *ifp)
{
	struct vmxnet3_softc *sc;
	struct vmxnet3_txqueue *txq;

	sc = ifp->if_softc;
	txq = &sc->vmx_queue[0].vxq_txqueue;

	VMXNET3_TXQ_LOCK(txq);
	vmxnet3_start_locked(ifp);
	VMXNET3_TXQ_UNLOCK(txq);
}

static int
vmxnet3_select_txqueue(struct ifnet *ifp, struct mbuf *m __unused)
{
	struct vmxnet3_softc *sc;
	u_int cpuid;

	sc = ifp->if_softc;
	cpuid = cpu_index(curcpu());
	/*
	 * Furure work
	 * We should select txqueue to even up the load even if ncpu is
	 * different from sc->vmx_ntxqueues. Currently, the load is not
	 * even, that is, when ncpu is six and ntxqueues is four, the load
	 * of vmx_queue[0] and vmx_queue[1] is higher than vmx_queue[2] and
	 * vmx_queue[3] because CPU#4 always uses vmx_queue[0] and CPU#5 always
	 * uses vmx_queue[1].
	 * Furthermore, we should not use random value to select txqueue to
	 * avoid reordering. We should use flow information of mbuf.
	 */
	return cpuid % sc->vmx_ntxqueues;
}

static void
vmxnet3_transmit_locked(struct ifnet *ifp, struct vmxnet3_txqueue *txq)
{

	vmxnet3_tx_common_locked(ifp, txq, VMXNET3_TX_TRANSMIT);
}

static int
vmxnet3_transmit(struct ifnet *ifp, struct mbuf *m)
{
	struct vmxnet3_softc *sc;
	struct vmxnet3_txqueue *txq;
	int qid;

	qid = vmxnet3_select_txqueue(ifp, m);
	sc = ifp->if_softc;
	txq = &sc->vmx_queue[qid].vxq_txqueue;

	if (__predict_false(!pcq_put(txq->vxtxq_interq, m))) {
		VMXNET3_TXQ_LOCK(txq);
		txq->vxtxq_pcqdrop.ev_count++;
		VMXNET3_TXQ_UNLOCK(txq);
		m_freem(m);
		return ENOBUFS;
	}

#ifdef VMXNET3_ALWAYS_TXDEFER
	kpreempt_disable();
	softint_schedule(txq->vxtxq_si);
	kpreempt_enable();
#else
	if (VMXNET3_TXQ_TRYLOCK(txq)) {
		vmxnet3_transmit_locked(ifp, txq);
		VMXNET3_TXQ_UNLOCK(txq);
	} else {
		kpreempt_disable();
		softint_schedule(txq->vxtxq_si);
		kpreempt_enable();
	}
#endif

	return 0;
}

static void
vmxnet3_deferred_transmit(void *arg)
{
	struct vmxnet3_txqueue *txq = arg;
	struct vmxnet3_softc *sc = txq->vxtxq_sc;
	struct ifnet *ifp = &sc->vmx_ethercom.ec_if;

	VMXNET3_TXQ_LOCK(txq);
	txq->vxtxq_transmitdef.ev_count++;
	if (pcq_peek(txq->vxtxq_interq) != NULL)
		vmxnet3_transmit_locked(ifp, txq);
	VMXNET3_TXQ_UNLOCK(txq);
}

static void
vmxnet3_set_rxfilter(struct vmxnet3_softc *sc)
{
	struct ethercom *ec = &sc->vmx_ethercom;
	struct vmxnet3_driver_shared *ds = sc->vmx_ds;
	struct ether_multi *enm;
	struct ether_multistep step;
	u_int mode;
	uint8_t *p;

	VMXNET3_CORE_LOCK_ASSERT(sc);

	ds->mcast_tablelen = 0;
	ETHER_LOCK(ec);
	CLR(ec->ec_flags, ETHER_F_ALLMULTI);
	ETHER_UNLOCK(ec);

	/*
	 * Always accept broadcast frames.
	 * Always accept frames destined to our station address.
	 */
	mode = VMXNET3_RXMODE_BCAST | VMXNET3_RXMODE_UCAST;

	ETHER_LOCK(ec);
	if (sc->vmx_promisc ||
	    ec->ec_multicnt > VMXNET3_MULTICAST_MAX)
		goto allmulti;

	p = sc->vmx_mcast;
	ETHER_FIRST_MULTI(step, ec, enm);
	while (enm != NULL) {
		if (memcmp(enm->enm_addrlo, enm->enm_addrhi, ETHER_ADDR_LEN)) {
			/*
			 * We must listen to a range of multicast addresses.
			 * For now, just accept all multicasts, rather than
			 * trying to set only those filter bits needed to match
			 * the range.  (At this time, the only use of address
			 * ranges is for IP multicast routing, for which the
			 * range is big enough to require all bits set.)
			 */
			goto allmulti;
		}
		memcpy(p, enm->enm_addrlo, ETHER_ADDR_LEN);

		p += ETHER_ADDR_LEN;

		ETHER_NEXT_MULTI(step, enm);
	}

	if (ec->ec_multicnt > 0) {
		SET(mode, VMXNET3_RXMODE_MCAST);
		ds->mcast_tablelen = p - sc->vmx_mcast;
	}
	ETHER_UNLOCK(ec);

	goto setit;

allmulti:
	SET(ec->ec_flags, ETHER_F_ALLMULTI);
	ETHER_UNLOCK(ec);
	SET(mode, (VMXNET3_RXMODE_ALLMULTI | VMXNET3_RXMODE_MCAST));
	if (sc->vmx_promisc)
		SET(mode, VMXNET3_RXMODE_PROMISC);

setit:
	vmxnet3_write_cmd(sc, VMXNET3_CMD_SET_FILTER);
	ds->rxmode = mode;
	vmxnet3_write_cmd(sc, VMXNET3_CMD_SET_RXMODE);
}

static int
vmxnet3_ioctl(struct ifnet *ifp, u_long cmd, void *data)
{
	struct vmxnet3_softc *sc = ifp->if_softc;
	struct ifreq *ifr = (struct ifreq *)data;
	int s, error = 0;

	switch (cmd) {
	case SIOCADDMULTI:
	case SIOCDELMULTI:
		break;
	default:
		KASSERT(IFNET_LOCKED(ifp));
	}

	switch (cmd) {
	case SIOCSIFMTU: {
		int nmtu = ifr->ifr_mtu;

		if (nmtu < VMXNET3_MIN_MTU || nmtu > VMXNET3_MAX_MTU) {
			error = EINVAL;
			break;
		}
		if (ifp->if_mtu != (uint64_t)nmtu) {
			s = splnet();
			error = ether_ioctl(ifp, cmd, data);
			splx(s);
			if (error == ENETRESET)
				error = vmxnet3_init(ifp);
		}
		break;
	}

	default:
		s = splnet();
		error = ether_ioctl(ifp, cmd, data);
		splx(s);
	}

	if (error == ENETRESET) {
		VMXNET3_CORE_LOCK(sc);
		if (sc->vmx_mcastactive)
			vmxnet3_set_rxfilter(sc);
		VMXNET3_CORE_UNLOCK(sc);
		error = 0;
	}

	return error;
}

static int
vmxnet3_ifflags_cb(struct ethercom *ec)
{
	struct ifnet *ifp = &ec->ec_if;
	struct vmxnet3_softc *sc = ifp->if_softc;
	int error = 0;

	KASSERT(IFNET_LOCKED(ifp));

	VMXNET3_CORE_LOCK(sc);
	const unsigned short changed = ifp->if_flags ^ sc->vmx_if_flags;
	if ((changed & ~(IFF_CANTCHANGE | IFF_DEBUG)) == 0) {
		sc->vmx_if_flags = ifp->if_flags;
		if (changed & IFF_PROMISC) {
			sc->vmx_promisc = ifp->if_flags & IFF_PROMISC;
			error = ENETRESET;
		}
	} else {
		error = ENETRESET;
	}
	VMXNET3_CORE_UNLOCK(sc);

	vmxnet3_if_link_status(sc);

	return error;
}

static int
vmxnet3_watchdog(struct vmxnet3_txqueue *txq)
{
	struct vmxnet3_softc *sc;
	struct vmxnet3_queue *vmxq;

	sc = txq->vxtxq_sc;
	vmxq = container_of(txq, struct vmxnet3_queue, vxq_txqueue);

	VMXNET3_TXQ_LOCK(txq);
	if (txq->vxtxq_watchdog == 0 || --txq->vxtxq_watchdog) {
		VMXNET3_TXQ_UNLOCK(txq);
		return (0);
	}
	txq->vxtxq_watchdogto.ev_count++;
	VMXNET3_TXQ_UNLOCK(txq);

	device_printf(sc->vmx_dev, "watchdog timeout on queue %d\n",
	    vmxq->vxq_id);
	return (1);
}

static void
vmxnet3_refresh_host_stats(struct vmxnet3_softc *sc)
{

	vmxnet3_write_cmd(sc, VMXNET3_CMD_GET_STATS);
}

static void
vmxnet3_tick(void *xsc)
{
	struct vmxnet3_softc *sc;
	int i, timedout;

	sc = xsc;
	timedout = 0;

	VMXNET3_CORE_LOCK(sc);

	vmxnet3_refresh_host_stats(sc);

	for (i = 0; i < sc->vmx_ntxqueues; i++)
		timedout |= vmxnet3_watchdog(&sc->vmx_queue[i].vxq_txqueue);

	if (timedout != 0) {
		if (!sc->vmx_reset_pending) {
			sc->vmx_reset_pending = true;
			workqueue_enqueue(sc->vmx_reset_wq,
			    &sc->vmx_reset_work, NULL);
		}
	} else {
		callout_reset(&sc->vmx_tick, hz, vmxnet3_tick, sc);
	}

	VMXNET3_CORE_UNLOCK(sc);
}

static void
vmxnet3_reset_work(struct work *work, void *arg)
{
	struct vmxnet3_softc *sc = arg;
	struct ifnet *ifp = &sc->vmx_ethercom.ec_if;

	VMXNET3_CORE_LOCK(sc);
	KASSERT(sc->vmx_reset_pending);
	sc->vmx_reset_pending = false;
	VMXNET3_CORE_UNLOCK(sc);

	IFNET_LOCK(ifp);
	(void)vmxnet3_init(ifp);
	IFNET_UNLOCK(ifp);
}

/*
 * update link state of ifnet and softc
 */
static void
vmxnet3_if_link_status(struct vmxnet3_softc *sc)
{
	struct ifnet *ifp = &sc->vmx_ethercom.ec_if;
	u_int link;
	bool up;

	up = vmxnet3_cmd_link_status(ifp);
	if (up) {
		sc->vmx_link_active = 1;
		link = LINK_STATE_UP;
	} else {
		sc->vmx_link_active = 0;
		link = LINK_STATE_DOWN;
	}

	if_link_state_change(ifp, link);
}

/*
 * check vmx(4) state by VMXNET3_CMD and update ifp->if_baudrate
 *   returns
 *       - true:  link up
 *       - flase: link down
 */
static bool
vmxnet3_cmd_link_status(struct ifnet *ifp)
{
	struct vmxnet3_softc *sc = ifp->if_softc;
	u_int x, speed;

	x = vmxnet3_read_cmd(sc, VMXNET3_CMD_GET_LINK);
	if ((x & 1) == 0)
		return false;

	speed = x >> 16;
	ifp->if_baudrate = IF_Mbps(speed);
	return true;
}

static void
vmxnet3_ifmedia_status(struct ifnet *ifp, struct ifmediareq *ifmr)
{
	bool up;

	ifmr->ifm_status = IFM_AVALID;
	ifmr->ifm_active = IFM_ETHER;

	up = vmxnet3_cmd_link_status(ifp);
	if (!up)
		return;

	ifmr->ifm_status |= IFM_ACTIVE;

	if (ifp->if_baudrate >= IF_Gbps(10ULL))
		ifmr->ifm_active |= IFM_10G_T | IFM_FDX;
}

static int
vmxnet3_ifmedia_change(struct ifnet *ifp)
{
	return 0;
}

static void
vmxnet3_set_lladdr(struct vmxnet3_softc *sc)
{
	uint32_t ml, mh;

	ml  = sc->vmx_lladdr[0];
	ml |= sc->vmx_lladdr[1] << 8;
	ml |= sc->vmx_lladdr[2] << 16;
	ml |= sc->vmx_lladdr[3] << 24;
	vmxnet3_write_bar1(sc, VMXNET3_BAR1_MACL, ml);

	mh  = sc->vmx_lladdr[4];
	mh |= sc->vmx_lladdr[5] << 8;
	vmxnet3_write_bar1(sc, VMXNET3_BAR1_MACH, mh);
}

static void
vmxnet3_get_lladdr(struct vmxnet3_softc *sc)
{
	uint32_t ml, mh;

	ml = vmxnet3_read_cmd(sc, VMXNET3_CMD_GET_MACL);
	mh = vmxnet3_read_cmd(sc, VMXNET3_CMD_GET_MACH);

	sc->vmx_lladdr[0] = ml;
	sc->vmx_lladdr[1] = ml >> 8;
	sc->vmx_lladdr[2] = ml >> 16;
	sc->vmx_lladdr[3] = ml >> 24;
	sc->vmx_lladdr[4] = mh;
	sc->vmx_lladdr[5] = mh >> 8;
}

static void
vmxnet3_enable_all_intrs(struct vmxnet3_softc *sc)
{
	int i;

	sc->vmx_ds->ictrl &= ~VMXNET3_ICTRL_DISABLE_ALL;
	for (i = 0; i < sc->vmx_nintrs; i++)
		vmxnet3_enable_intr(sc, i);
}

static void
vmxnet3_disable_all_intrs(struct vmxnet3_softc *sc)
{
	int i;

	sc->vmx_ds->ictrl |= VMXNET3_ICTRL_DISABLE_ALL;
	for (i = 0; i < sc->vmx_nintrs; i++)
		vmxnet3_disable_intr(sc, i);
}

static int
vmxnet3_dma_malloc(struct vmxnet3_softc *sc, bus_size_t size, bus_size_t align,
    struct vmxnet3_dma_alloc *dma)
{
	bus_dma_tag_t t = sc->vmx_dmat;
	bus_dma_segment_t *segs = dma->dma_segs;
	int n, error;

	memset(dma, 0, sizeof(*dma));

	error = bus_dmamem_alloc(t, size, align, 0, segs, 1, &n, BUS_DMA_NOWAIT);
	if (error) {
		aprint_error_dev(sc->vmx_dev, "bus_dmamem_alloc failed: %d\n", error);
		goto fail1;
	}
	KASSERT(n == 1);

	error = bus_dmamem_map(t, segs, 1, size, &dma->dma_vaddr, BUS_DMA_NOWAIT);
	if (error) {
		aprint_error_dev(sc->vmx_dev, "bus_dmamem_map failed: %d\n", error);
		goto fail2;
	}

	error = bus_dmamap_create(t, size, 1, size, 0, BUS_DMA_NOWAIT, &dma->dma_map);
	if (error) {
		aprint_error_dev(sc->vmx_dev, "bus_dmamap_create failed: %d\n", error);
		goto fail3;
	}

	error = bus_dmamap_load(t, dma->dma_map, dma->dma_vaddr, size, NULL,
	    BUS_DMA_NOWAIT);
	if (error) {
		aprint_error_dev(sc->vmx_dev, "bus_dmamap_load failed: %d\n", error);
		goto fail4;
	}

	memset(dma->dma_vaddr, 0, size);
	dma->dma_paddr = DMAADDR(dma->dma_map);
	dma->dma_size = size;

	return (0);
fail4:
	bus_dmamap_destroy(t, dma->dma_map);
fail3:
	bus_dmamem_unmap(t, dma->dma_vaddr, size);
fail2:
	bus_dmamem_free(t, segs, 1);
fail1:
	return (error);
}

static void
vmxnet3_dma_free(struct vmxnet3_softc *sc, struct vmxnet3_dma_alloc *dma)
{
	bus_dma_tag_t t = sc->vmx_dmat;

	bus_dmamap_unload(t, dma->dma_map);
	bus_dmamap_destroy(t, dma->dma_map);
	bus_dmamem_unmap(t, dma->dma_vaddr, dma->dma_size);
	bus_dmamem_free(t, dma->dma_segs, 1);

	memset(dma, 0, sizeof(*dma));
}

MODULE(MODULE_CLASS_DRIVER, if_vmx, "pci");

#ifdef _MODULE
#include "ioconf.c"
#endif

static int
if_vmx_modcmd(modcmd_t cmd, void *opaque)
{
	int error = 0;

	switch (cmd) {
	case MODULE_CMD_INIT:
#ifdef _MODULE
		error = config_init_component(cfdriver_ioconf_if_vmx,
		    cfattach_ioconf_if_vmx, cfdata_ioconf_if_vmx);
#endif
		return error;
	case MODULE_CMD_FINI:
#ifdef _MODULE
		error = config_fini_component(cfdriver_ioconf_if_vmx,
		    cfattach_ioconf_if_vmx, cfdata_ioconf_if_vmx);
#endif
		return error;
	default:
		return ENOTTY;
	}
}