xref: /src/sys/arch/arm/sunxi/sun4i_emac.c

/* $NetBSD: sun4i_emac.c,v 1.15 2022/09/18 02:32:14 thorpej Exp $ */

/*-
 * Copyright (c) 2013-2017 The NetBSD Foundation, Inc.
 * All rights reserved.
 *
 * This code is derived from software contributed to The NetBSD Foundation
 * by Matt Thomas of 3am Software Foundry and Jared McNeill.
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions
 * are met:
 * 1. Redistributions of source code must retain the above copyright
 *    notice, this list of conditions and the following disclaimer.
 * 2. Redistributions in binary form must reproduce the above copyright
 *    notice, this list of conditions and the following disclaimer in the
 *    documentation and/or other materials provided with the distribution.
 *
 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
 * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
 * POSSIBILITY OF SUCH DAMAGE.
 */

#include <sys/cdefs.h>

__KERNEL_RCSID(1, "$NetBSD: sun4i_emac.c,v 1.15 2022/09/18 02:32:14 thorpej Exp $");

#include <sys/param.h>
#include <sys/bus.h>
#include <sys/device.h>
#include <sys/intr.h>
#include <sys/ioctl.h>
#include <sys/mutex.h>
#include <sys/rndsource.h>
#include <sys/kernel.h>
#include <sys/systm.h>

#include <net/bpf.h>
#include <net/if.h>
#include <net/if_dl.h>
#include <net/if_ether.h>
#include <net/if_media.h>

#include <dev/mii/miivar.h>

#include <dev/fdt/fdtvar.h>

#include <arm/sunxi/sunxi_sramc.h>

#define	EMAC_IFNAME	"emac%d"

#define	EMAC_CTL_REG		0x00
#define	 EMAC_CTL_RX_EN			__BIT(2)
#define	 EMAC_CTL_TX_EN			__BIT(1)
#define	 EMAC_CTL_RST			__BIT(0)
#define	EMAC_TX_MODE_REG	0x04
#define	 EMAC_TX_MODE_DMA		__BIT(1)
#define	 EMAC_TX_MODE_ABF_ENA		__BIT(0)
#define	EMAC_TX_FLOW_REG	0x08
#define	EMAC_TX_CTL0_REG	0x0c
#define	EMAC_TX_CTL1_REG	0x10
#define	EMAC_TX_CTL_REG(n)	(EMAC_TX_CTL0_REG+4*(n))
#define	 EMAC_TX_CTL_START		__BIT(0)
#define	EMAC_TX_INS_REG		0x14
#define	EMAC_TX_PL0_REG		0x18
#define	EMAC_TX_PL1_REG		0x1c
#define	EMAC_TX_PL_REG(n)	(EMAC_TX_PL0_REG+4*(n))
#define	EMAC_TX_STA_REG		0x20
#define	EMAC_TX_IO_DATA0_REG	0x24
#define	EMAC_TX_IO_DATA1_REG	0x28
#define	EMAC_TX_IO_DATA_REG(n)	(EMAC_TX_IO_DATA0_REG+4*(n))
#define	EMAC_TX_TSVL0_REG	0x2c
#define	EMAC_TX_TSVH0_REG	0x30
#define	EMAC_TX_TSVL1_REG	0x34
#define	EMAC_TX_TSVH1_REG	0x38
#define	EMAC_RX_CTL_REG		0x3c
#define	 EMAC_RX_CTL_SA_IF		__BIT(25)
#define	 EMAC_RX_CTL_SA			__BIT(24)
#define	 EMAC_RX_CTL_BC0		__BIT(22)
#define	 EMAC_RX_CTL_MHF		__BIT(21)
#define	 EMAC_RX_CTL_MC0		__BIT(20)
#define	 EMAC_RX_CTL_DAF		__BIT(17)
#define	 EMAC_RX_CTL_UCAD		__BIT(16)
#define	 EMAC_RX_CTL_POR		__BIT(8)
#define	 EMAC_RX_CTL_PLE		__BIT(7)
#define	 EMAC_RX_CTL_PCRCE		__BIT(6)
#define	 EMAC_RX_CTL_PCF		__BIT(5)
#define	 EMAC_RX_CTL_PROMISC		__BIT(4)
#define	 EMAC_RX_CTL_FIFO_RESET		__BIT(3)
#define	 EMAC_RX_CTL_DMA		__BIT(2)
#define	 EMAC_RX_CTL_DRQ_MODE		__BIT(1)
#define	 EMAC_RX_CTL_START		__BIT(0)
#define	EMAC_RX_HASH0_REG	0x40
#define	EMAC_RX_HASH1_REG	0x44
#define	EMAC_RX_STA_REG		0x48
#define	 EMAC_RX_STA_PKTOK		__BIT(7)
#define	 EMAC_RX_STA_ALNERR		__BIT(6)
#define	 EMAC_RX_STA_LENERR		__BIT(5)
#define	 EMAC_RX_STA_CRCERR		__BIT(4)
#define	EMAC_RX_IO_DATA_REG	0x4c
#define	EMAC_RX_FBC_REG		0x50
#define	EMAC_INT_CTL_REG	0x54
#define	EMAC_INT_STA_REG	0x58
#define	 EMAC_INT_RX			__BIT(8)
#define	 EMAC_INT_TX1			__BIT(1)
#define	 EMAC_INT_TX0			__BIT(0)
#define	 EMAC_INT_ENABLE		\
		(EMAC_INT_RX | EMAC_INT_TX1 | EMAC_INT_TX0)
#define	EMAC_MAC_CTL0_REG	0x5c
#define	 EMAC_MAC_CTL0_SOFT_RESET	__BIT(15)
#define	 EMAC_MAC_CTL0_TFC		__BIT(3)
#define	 EMAC_MAC_CTL0_RFC		__BIT(2)
#define	EMAC_MAC_CTL1_REG	0x60
#define	 EMAC_MAC_CTL1_ED		__BIT(15)
#define	 EMAC_MAC_CTL1_NB		__BIT(13)
#define	 EMAC_MAC_CTL1_BNB		__BIT(12)
#define	 EMAC_MAC_CTL1_LPE		__BIT(9)
#define	 EMAC_MAC_CTL1_PRE		__BIT(8)
#define	 EMAC_MAC_CTL1_ADP		__BIT(7)
#define	 EMAC_MAC_CTL1_VC		__BIT(6)
#define	 EMAC_MAC_CTL1_PC		__BIT(5)
#define	 EMAC_MAC_CTL1_CRC		__BIT(4)
#define	 EMAC_MAC_CTL1_DCRC		__BIT(3)
#define	 EMAC_MAC_CTL1_HF		__BIT(2)
#define	 EMAC_MAC_CTL1_FLC		__BIT(1)
#define	 EMAC_MAC_CTL1_FD		__BIT(0)
#define	EMAC_MAC_IPGT_REG	0x64
#define	 EMAC_MAC_IPGT_FD		0x15
#define	EMAC_MAC_IPGR_REG	0x68
#define	 EMAC_MAC_IPGR_IPG1		__BITS(15,8)
#define	 EMAC_MAC_IPGR_IPG2		__BITS(7,0)
#define	EMAC_MAC_CLRT_REG	0x6c
#define	 EMAC_MAC_CLRT_CW		__BITS(15,8)
#define	 EMAC_MAC_CLRT_RM		__BITS(7,0)
#define	EMAC_MAC_MAXF_REG	0x70
#define	EMAC_MAC_SUPP_REG	0x74
#define	 EMAC_MAC_SUPP_100M		__BIT(8)
#define	EMAC_MAC_TEST_REG	0x78
#define	EMAC_MAC_MCFG_REG	0x7c
#define	 EMAC_MAC_MCFG_CLK		__BITS(5,2)
#define	EMAC_MAC_MCMD_REG	0x80
#define	EMAC_MAC_MADR_REG	0x84
#define	EMAC_MAC_MWTD_REG	0x88
#define	EMAC_MAC_MRDD_REG	0x8c
#define	EMAC_MAC_MIND_REG	0x90
#define	EMAC_MAC_SSRR_REG	0x94
#define	EMAC_MAC_A0_REG		0x98
#define	EMAC_MAC_A1_REG		0x9c
#define	EMAC_MAC_A2_REG		0xa0

#define	EMAC_RXHDR_STS			__BITS(31,16)
#define	EMAC_RXHDR_LEN			__BITS(15,0)

#define	EMAC_RX_MAGIC		0x0143414d	/* M A C \001 */

#define	EMAC_TXBUF_SIZE		4096

static int sun4i_emac_match(device_t, cfdata_t, void *);
static void sun4i_emac_attach(device_t, device_t, void *);

static int sun4i_emac_intr(void *);
static void sun4i_emac_tick(void *);

static int sun4i_emac_miibus_read_reg(device_t, int, int, uint16_t *);
static int sun4i_emac_miibus_write_reg(device_t, int, int, uint16_t);
static void sun4i_emac_miibus_statchg(struct ifnet *);

static void sun4i_emac_ifstart(struct ifnet *);
static int sun4i_emac_ifioctl(struct ifnet *, u_long, void *);
static int sun4i_emac_ifinit(struct ifnet *);
static void sun4i_emac_ifstop(struct ifnet *, int);
static void sun4i_emac_ifwatchdog(struct ifnet *);

struct sun4i_emac_softc;
static void sun4i_emac_rx_hash(struct sun4i_emac_softc *);

struct sun4i_emac_softc {
	device_t sc_dev;
	int sc_phandle;
	bus_space_tag_t sc_bst;
	bus_space_handle_t sc_bsh;
	bus_dma_tag_t sc_dmat;
	struct ethercom sc_ec;
	struct mii_data sc_mii;
	krndsource_t sc_rnd_source;	/* random source */
	kmutex_t sc_intr_lock;
	uint8_t sc_tx_active;
	callout_t sc_stat_ch;
	void *sc_ih;
	uint32_t sc_txbuf[EMAC_TXBUF_SIZE/4];
};

static const struct device_compatible_entry compat_data[] = {
	{ .compat = "allwinner,sun4i-a10-emac" },
	DEVICE_COMPAT_EOL
};

CFATTACH_DECL_NEW(sun4i_emac, sizeof(struct sun4i_emac_softc),
	sun4i_emac_match, sun4i_emac_attach, NULL, NULL);

static inline uint32_t
sun4i_emac_read(struct sun4i_emac_softc *sc, bus_size_t o)
{
	return bus_space_read_4(sc->sc_bst, sc->sc_bsh, o);
}

static inline void
sun4i_emac_write(struct sun4i_emac_softc *sc, bus_size_t o, uint32_t v)
{
	return bus_space_write_4(sc->sc_bst, sc->sc_bsh, o, v);
}

static inline void
sun4i_emac_clear_set(struct sun4i_emac_softc *sc, bus_size_t o, uint32_t c,
    uint32_t s)
{
	uint32_t v = bus_space_read_4(sc->sc_bst, sc->sc_bsh, o);
	return bus_space_write_4(sc->sc_bst, sc->sc_bsh, o, (v & ~c) | s);
}

static int
sun4i_emac_match(device_t parent, cfdata_t cf, void *aux)
{
	struct fdt_attach_args * const faa = aux;

	return of_compatible_match(faa->faa_phandle, compat_data);
}

static void
sun4i_emac_attach(device_t parent, device_t self, void *aux)
{
	struct sun4i_emac_softc * const sc = device_private(self);
	struct fdt_attach_args * const faa = aux;
	struct ifnet * const ifp = &sc->sc_ec.ec_if;
	struct mii_data * const mii = &sc->sc_mii;
	const int phandle = faa->faa_phandle;
	char enaddr[ETHER_ADDR_LEN];
	const uint8_t *local_addr;
	char intrstr[128];
	struct clk *clk;
	bus_addr_t addr;
	bus_size_t size;
	int len;

	if (fdtbus_get_reg(phandle, 0, &addr, &size) != 0) {
		aprint_error(": cannot get registers\n");
		return;
	}

	if (!fdtbus_intr_str(phandle, 0, intrstr, sizeof(intrstr))) {
		aprint_error(": cannot decode interrupt\n");
		return;
	}

	clk = fdtbus_clock_get_index(phandle, 0);
	if (clk == NULL) {
		aprint_error(": cannot acquire clock\n");
		return;
	}
	if (clk_enable(clk) != 0) {
		aprint_error(": cannot enable clock\n");
		return;
	}

	if (sunxi_sramc_claim(phandle) != 0) {
		aprint_error(": cannot map SRAM to EMAC\n");
		return;
	}

	sc->sc_dev = self;
	sc->sc_phandle = phandle;
	sc->sc_ec.ec_mii = mii;
	sc->sc_bst = faa->faa_bst;
	if (bus_space_map(sc->sc_bst, addr, size, 0, &sc->sc_bsh) != 0) {
		aprint_error(": cannot map registers\n");
		return;
	}
	sc->sc_dmat = faa->faa_dmat;

	mutex_init(&sc->sc_intr_lock, MUTEX_DEFAULT, IPL_NET);
	callout_init(&sc->sc_stat_ch, 0);
	callout_setfunc(&sc->sc_stat_ch, sun4i_emac_tick, sc);

	aprint_naive("\n");
	aprint_normal(": 10/100 Ethernet Controller\n");

	/*
	 * Disable and then clear all interrupts
	 */
	sun4i_emac_write(sc, EMAC_INT_CTL_REG, 0);
	sun4i_emac_write(sc, EMAC_INT_STA_REG,
	    sun4i_emac_read(sc, EMAC_INT_STA_REG));

	sc->sc_ih = fdtbus_intr_establish_xname(phandle, 0, IPL_NET, 0,
	    sun4i_emac_intr, sc, device_xname(self));
	if (sc->sc_ih == NULL) {
		aprint_error_dev(self, "failed to establish interrupt on %s\n",
		    intrstr);
		return;
	}
	aprint_normal_dev(self, "interrupting on %s\n", intrstr);

	local_addr = fdtbus_get_prop(phandle, "local-mac-address", &len);
	if (local_addr && len == ETHER_ADDR_LEN) {
		memcpy(enaddr, local_addr, ETHER_ADDR_LEN);

		uint32_t a1 = ((uint32_t)enaddr[0] << 16) |
			      ((uint32_t)enaddr[1] << 8) |
			       (uint32_t)enaddr[2];
		uint32_t a0 = ((uint32_t)enaddr[3] << 16) |
			      ((uint32_t)enaddr[4] << 8) |
			       (uint32_t)enaddr[5];

		sun4i_emac_write(sc, EMAC_MAC_A1_REG, a1);
		sun4i_emac_write(sc, EMAC_MAC_A0_REG, a0);
	}

	uint32_t a1 = sun4i_emac_read(sc, EMAC_MAC_A1_REG);
	uint32_t a0 = sun4i_emac_read(sc, EMAC_MAC_A0_REG);
	if (a0 != 0 || a1 != 0) {
		enaddr[0] = a1 >> 16;
		enaddr[1] = a1 >>  8;
		enaddr[2] = a1 >>  0;
		enaddr[3] = a0 >> 16;
		enaddr[4] = a0 >>  8;
		enaddr[5] = a0 >>  0;
	}
	aprint_normal_dev(self, "Ethernet address %s\n", ether_sprintf(enaddr));

	snprintf(ifp->if_xname, IFNAMSIZ, EMAC_IFNAME, device_unit(self));
	ifp->if_softc = sc;
	ifp->if_capabilities = 0;
	ifp->if_flags = IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST;
	ifp->if_start = sun4i_emac_ifstart;
	ifp->if_ioctl = sun4i_emac_ifioctl;
	ifp->if_init = sun4i_emac_ifinit;
	ifp->if_stop = sun4i_emac_ifstop;
	ifp->if_watchdog = sun4i_emac_ifwatchdog;
	IFQ_SET_READY(&ifp->if_snd);

	/* 802.1Q VLAN-sized frames are supported */
	sc->sc_ec.ec_capabilities |= ETHERCAP_VLAN_MTU;

	ifmedia_init(&mii->mii_media, 0, ether_mediachange, ether_mediastatus);

	mii->mii_ifp = ifp;
	mii->mii_readreg = sun4i_emac_miibus_read_reg;
	mii->mii_writereg = sun4i_emac_miibus_write_reg;
	mii->mii_statchg = sun4i_emac_miibus_statchg;

	mii_attach(self, mii, 0xffffffff, MII_PHY_ANY, MII_OFFSET_ANY, 0);

	if (LIST_EMPTY(&mii->mii_phys)) {
		aprint_error_dev(self, "no PHY found!\n");
		ifmedia_add(&mii->mii_media, IFM_ETHER | IFM_MANUAL, 0, NULL);
		ifmedia_set(&mii->mii_media, IFM_ETHER | IFM_MANUAL);
	} else {
		ifmedia_set(&mii->mii_media, IFM_ETHER | IFM_AUTO);
	}

	/*
	 * Attach the interface.
	 */
	if_attach(ifp);
	if_deferred_start_init(ifp, NULL);
	ether_ifattach(ifp, enaddr);
	rnd_attach_source(&sc->sc_rnd_source, device_xname(self),
	    RND_TYPE_NET, RND_FLAG_DEFAULT);
}

static inline void
sun4i_emac_int_enable(struct sun4i_emac_softc *sc)
{
	sun4i_emac_clear_set(sc, EMAC_INT_CTL_REG, 0,
	    EMAC_INT_ENABLE);
	sun4i_emac_write(sc, EMAC_INT_STA_REG,
	    sun4i_emac_read(sc, EMAC_INT_STA_REG));
}

int
sun4i_emac_miibus_read_reg(device_t self, int phy, int reg, uint16_t *val)
{
	struct sun4i_emac_softc * const sc = device_private(self);
	int retry = 100;
	int rv = 0;

	sun4i_emac_write(sc, EMAC_MAC_MADR_REG, (phy << 8) | reg);
	sun4i_emac_write(sc, EMAC_MAC_MCMD_REG, 1);

	while (--retry > 0 && (sun4i_emac_read(sc, EMAC_MAC_MIND_REG) & 1) != 0)
		delay(1000);
	if (retry == 0) {
		device_printf(self, "PHY read timeout\n");
		rv = ETIMEDOUT;
	}

	sun4i_emac_write(sc, EMAC_MAC_MCMD_REG, 0);
	*val = sun4i_emac_read(sc, EMAC_MAC_MRDD_REG) & 0xffff;

	return rv;
}

int
sun4i_emac_miibus_write_reg(device_t self, int phy, int reg, uint16_t val)
{
	struct sun4i_emac_softc * const sc = device_private(self);
	int retry = 100;
	int rv = 0;

	sun4i_emac_write(sc, EMAC_MAC_MADR_REG, (phy << 8) | reg);
	sun4i_emac_write(sc, EMAC_MAC_MCMD_REG, 1);

	while (--retry > 0 && (sun4i_emac_read(sc, EMAC_MAC_MIND_REG) & 1) != 0)
		delay(1000);
	if (retry == 0) {
		device_printf(self, "PHY write timeout\n");
		rv = ETIMEDOUT;
	}

	sun4i_emac_write(sc, EMAC_MAC_MCMD_REG, 0);
	sun4i_emac_write(sc, EMAC_MAC_MWTD_REG, val);

	return rv;
}

void
sun4i_emac_miibus_statchg(struct ifnet *ifp)
{
	struct sun4i_emac_softc * const sc = ifp->if_softc;
	struct mii_data * const mii = &sc->sc_mii;
	const u_int media = mii->mii_media_active;

	/*
	 * Set MII interface based on the speed
	 * negotiated by the PHY.
	 */
	switch (IFM_SUBTYPE(media)) {
	case IFM_10_T:
		sun4i_emac_clear_set(sc, EMAC_MAC_SUPP_REG,
		    EMAC_MAC_SUPP_100M, 0);
		break;
	case IFM_100_TX:
		sun4i_emac_clear_set(sc, EMAC_MAC_SUPP_REG,
		    0, EMAC_MAC_SUPP_100M);
		break;
	}

	const bool link = (IFM_SUBTYPE(media) & (IFM_10_T | IFM_100_TX)) != 0;
	if (link) {
		if (media & IFM_FDX) {
			sun4i_emac_clear_set(sc, EMAC_MAC_CTL1_REG,
			    0, EMAC_MAC_CTL1_FD);
		} else {
			sun4i_emac_clear_set(sc, EMAC_MAC_CTL1_REG,
			    EMAC_MAC_CTL1_FD, 0);
		}
	}
}

static void
sun4i_emac_tick(void *softc)
{
	struct sun4i_emac_softc * const sc = softc;
	struct mii_data * const mii = &sc->sc_mii;
	int s;

	s = splnet();
	mii_tick(mii);
	callout_schedule(&sc->sc_stat_ch, hz);
	splx(s);
}

static inline void
sun4i_emac_rxfifo_flush(struct sun4i_emac_softc *sc)
{
	sun4i_emac_clear_set(sc, EMAC_CTL_REG, EMAC_CTL_RX_EN, 0);

	sun4i_emac_clear_set(sc, EMAC_RX_CTL_REG, 0, EMAC_RX_CTL_FIFO_RESET);

	for (;;) {
		uint32_t v0 = sun4i_emac_read(sc, EMAC_RX_CTL_REG);
		if ((v0 & EMAC_RX_CTL_FIFO_RESET) == 0)
			break;
	}

	sun4i_emac_clear_set(sc, EMAC_CTL_REG, 0, EMAC_CTL_RX_EN);
}

static void
sun4i_emac_rxfifo_consume(struct sun4i_emac_softc *sc, size_t len)
{
	for (len = (len + 3) >> 2; len > 0; len--) {
		(void) sun4i_emac_read(sc, EMAC_RX_IO_DATA_REG);
	}
}

static void
sun4i_emac_rxfifo_transfer(struct sun4i_emac_softc *sc, struct mbuf *m)
{
	uint32_t *dp32 = mtod(m, uint32_t *);
	const int len = roundup2(m->m_len, 4);

	bus_space_read_multi_4(sc->sc_bst, sc->sc_bsh,
	    EMAC_RX_IO_DATA_REG, dp32, len / 4);
}

static struct mbuf *
sun4i_emac_mgethdr(struct sun4i_emac_softc *sc, size_t rxlen)
{
	struct mbuf *m = m_gethdr(M_DONTWAIT, MT_DATA);

	if (m == NULL) {
		return NULL;
	}
	if (rxlen + 2 > MHLEN) {
		MCLGET(m, M_DONTWAIT);
		if ((m->m_flags & M_EXT) == 0) {
			m_free(m);
			return NULL;
		}
	}

	m_adj(m, 2);
	m->m_len = rxlen;
	m->m_pkthdr.len = rxlen;
	m_set_rcvif(m, &sc->sc_ec.ec_if);
	m->m_flags |= M_HASFCS;

	return m;
}

static void
sun4i_emac_if_input(struct sun4i_emac_softc *sc, struct mbuf *m)
{
	struct ifnet * const ifp = &sc->sc_ec.ec_if;

	if_percpuq_enqueue(ifp->if_percpuq, m);
}

static void
sun4i_emac_rx_intr(struct sun4i_emac_softc *sc)
{
	for (;;) {
		uint32_t rx_count = sun4i_emac_read(sc, EMAC_RX_FBC_REG);
		struct mbuf *m;

		if (rx_count == 0) {
			rx_count = sun4i_emac_read(sc, EMAC_RX_FBC_REG);
			if (rx_count == 0)
				return;
		}

		uint32_t v = sun4i_emac_read(sc, EMAC_RX_IO_DATA_REG);
		if (v != EMAC_RX_MAGIC) {
			sun4i_emac_rxfifo_flush(sc);
			return;
		}

		uint32_t rxhdr = sun4i_emac_read(sc, EMAC_RX_IO_DATA_REG);
		uint32_t rxlen = __SHIFTOUT(rxhdr, EMAC_RXHDR_LEN);
		uint32_t rxsts = __SHIFTOUT(rxhdr, EMAC_RXHDR_STS);

		if (rxlen < ETHER_MIN_LEN || (rxsts & EMAC_RX_STA_PKTOK) == 0) {
			if_statinc(&sc->sc_ec.ec_if, if_ierrors);
			continue;
		}

		m = sun4i_emac_mgethdr(sc, rxlen);
		if (m == NULL) {
			if_statinc(&sc->sc_ec.ec_if, if_ierrors);
			sun4i_emac_rxfifo_consume(sc, rxlen);
			return;
		}

		sun4i_emac_rxfifo_transfer(sc, m);
		sun4i_emac_if_input(sc, m);
	}
}

static int
sun4i_emac_txfifo_transfer(struct sun4i_emac_softc *sc, struct mbuf *m, u_int slot)
{
	bus_size_t const io_data_reg = EMAC_TX_IO_DATA_REG(0);
	const int len = m->m_pkthdr.len;
	uint32_t *pktdata;

	KASSERT(len > 0 && len <= sizeof(sc->sc_txbuf));

	if (m->m_next != NULL) {
		m_copydata(m, 0, len, sc->sc_txbuf);
		pktdata = sc->sc_txbuf;
	} else {
		pktdata = mtod(m, uint32_t *);
	}

	bus_space_write_multi_4(sc->sc_bst, sc->sc_bsh, io_data_reg,
	    pktdata, roundup2(len, 4) / 4);

	return len;
}

static void
sun4i_emac_tx_enqueue(struct sun4i_emac_softc *sc, struct mbuf *m, u_int slot)
{
	struct ifnet * const ifp = &sc->sc_ec.ec_if;

	sun4i_emac_write(sc, EMAC_TX_INS_REG, slot);

	const int len = sun4i_emac_txfifo_transfer(sc, m, slot);

	bus_size_t const pl_reg = EMAC_TX_PL_REG(slot);
	bus_size_t const ctl_reg = EMAC_TX_CTL_REG(slot);

	sun4i_emac_write(sc, pl_reg, len);
	sun4i_emac_clear_set(sc, ctl_reg, 0, EMAC_TX_CTL_START);

	bpf_mtap(ifp, m, BPF_D_OUT);

	m_freem(m);
}

static void
sun4i_emac_tx_intr(struct sun4i_emac_softc *sc, u_int slot)
{
	sc->sc_tx_active &= ~__BIT(slot);
}

int
sun4i_emac_intr(void *arg)
{
	struct sun4i_emac_softc * const sc = arg;
	struct ifnet * const ifp = &sc->sc_ec.ec_if;

	mutex_enter(&sc->sc_intr_lock);

	uint32_t sts = sun4i_emac_read(sc, EMAC_INT_STA_REG);
	sun4i_emac_write(sc, EMAC_INT_STA_REG, sts);
	rnd_add_uint32(&sc->sc_rnd_source, sts);

	if (sts & EMAC_INT_RX) {
		sun4i_emac_rx_intr(sc);
	}
	if (sts & EMAC_INT_TX0) {
		sun4i_emac_tx_intr(sc, 0);
	}
	if (sts & EMAC_INT_TX1) {
		sun4i_emac_tx_intr(sc, 1);
	}
	if (sts & (EMAC_INT_TX0 | EMAC_INT_TX1)) {
		if (sc->sc_tx_active == 0)
			ifp->if_timer = 0;
		if_schedule_deferred_start(ifp);
	}

	mutex_exit(&sc->sc_intr_lock);

	return 1;
}

void
sun4i_emac_ifstart(struct ifnet *ifp)
{
	struct sun4i_emac_softc * const sc = ifp->if_softc;

	mutex_enter(&sc->sc_intr_lock);

	if ((sc->sc_tx_active & 1) == 0) {
		struct mbuf *m;
		IFQ_DEQUEUE(&ifp->if_snd, m);
		if (m == NULL) {
			mutex_exit(&sc->sc_intr_lock);
			return;
		}
		sun4i_emac_tx_enqueue(sc, m, 0);
		sc->sc_tx_active |= 1;
	}

	if ((sc->sc_tx_active & 2) == 0) {
		struct mbuf *m;
		IFQ_DEQUEUE(&ifp->if_snd, m);
		if (m == NULL) {
			mutex_exit(&sc->sc_intr_lock);
			return;
		}
		sun4i_emac_tx_enqueue(sc, m, 1);
		sc->sc_tx_active |= 2;
	}

	ifp->if_timer = 5;

	mutex_exit(&sc->sc_intr_lock);
}


static int
sun4i_emac_ifioctl(struct ifnet *ifp, u_long cmd, void *data)
{
	struct sun4i_emac_softc * const sc = ifp->if_softc;
	int error;

	switch (cmd) {
	default:
		if ((error = ether_ioctl(ifp, cmd, data)) != ENETRESET)
			break;
		error = 0;
		if (cmd != SIOCADDMULTI && cmd != SIOCDELMULTI)
			break;
		if (ifp->if_flags & IFF_RUNNING) {
			/*
			 * Multicast list has changed; set the hardware filter
			 * accordingly.
			 */
			mutex_enter(&sc->sc_intr_lock);
			sun4i_emac_ifstop(ifp, 0);
			error = sun4i_emac_ifinit(ifp);
			mutex_exit(&sc->sc_intr_lock);
		}
		break;
	}

	return error;
}

static void
sun4i_emac_ifstop(struct ifnet *ifp, int discard)
{
	struct sun4i_emac_softc * const sc = ifp->if_softc;
	struct mii_data * const mii = &sc->sc_mii;

	KASSERT(mutex_owned(&sc->sc_intr_lock));

	callout_stop(&sc->sc_stat_ch);
	mii_down(mii);

	sun4i_emac_write(sc, EMAC_INT_CTL_REG, 0);
	sun4i_emac_write(sc, EMAC_INT_STA_REG,
	    sun4i_emac_read(sc, EMAC_INT_STA_REG));

	sun4i_emac_clear_set(sc, EMAC_CTL_REG,
	    EMAC_CTL_RST | EMAC_CTL_TX_EN | EMAC_CTL_RX_EN, 0);

	ifp->if_flags &= ~IFF_RUNNING;
	ifp->if_timer = 0;
}

int
sun4i_emac_ifinit(struct ifnet *ifp)
{
	struct sun4i_emac_softc * const sc = ifp->if_softc;
	struct mii_data * const mii = &sc->sc_mii;

	sun4i_emac_clear_set(sc, EMAC_RX_CTL_REG,
	    0, EMAC_RX_CTL_FIFO_RESET);

	delay(1);

	sun4i_emac_clear_set(sc, EMAC_MAC_CTL0_REG,
	    EMAC_MAC_CTL0_SOFT_RESET, 0);

	sun4i_emac_clear_set(sc, EMAC_MAC_MCFG_REG,
	    EMAC_MAC_MCFG_CLK, __SHIFTIN(0xd, EMAC_MAC_MCFG_CLK));

	sun4i_emac_write(sc, EMAC_RX_FBC_REG, 0);

	sun4i_emac_write(sc, EMAC_INT_CTL_REG, 0);
	sun4i_emac_write(sc, EMAC_INT_STA_REG,
	    sun4i_emac_read(sc, EMAC_INT_STA_REG));

	delay(1);

	sun4i_emac_clear_set(sc, EMAC_TX_MODE_REG,
	    EMAC_TX_MODE_DMA, EMAC_TX_MODE_ABF_ENA);

	sun4i_emac_clear_set(sc, EMAC_MAC_CTL0_REG,
	    0, EMAC_MAC_CTL0_TFC | EMAC_MAC_CTL0_RFC);

	sun4i_emac_clear_set(sc, EMAC_RX_CTL_REG,
	    EMAC_RX_CTL_DMA, 0);

	sun4i_emac_clear_set(sc, EMAC_MAC_CTL1_REG,
	    0,
	    EMAC_MAC_CTL1_FLC | EMAC_MAC_CTL1_CRC |
	    EMAC_MAC_CTL1_PC);

	sun4i_emac_write(sc, EMAC_MAC_IPGT_REG, EMAC_MAC_IPGT_FD);
	sun4i_emac_write(sc, EMAC_MAC_IPGR_REG,
	    __SHIFTIN(0x0c, EMAC_MAC_IPGR_IPG1) |
	    __SHIFTIN(0x12, EMAC_MAC_IPGR_IPG2));

	sun4i_emac_write(sc, EMAC_MAC_CLRT_REG,
	    __SHIFTIN(0x0f, EMAC_MAC_CLRT_RM) |
	    __SHIFTIN(0x37, EMAC_MAC_CLRT_CW));

	sun4i_emac_write(sc, EMAC_MAC_MAXF_REG, 0x600);

	sun4i_emac_rx_hash(sc);

	sun4i_emac_int_enable(sc);

	ifp->if_flags |= IFF_RUNNING;

	/* Enable RX/TX */
	sun4i_emac_clear_set(sc, EMAC_CTL_REG,
	    0, EMAC_CTL_RST | EMAC_CTL_TX_EN | EMAC_CTL_RX_EN);

	mii_mediachg(mii);
	callout_schedule(&sc->sc_stat_ch, hz);

	return 0;
}

static void
sun4i_emac_ifwatchdog(struct ifnet *ifp)
{
	struct sun4i_emac_softc * const sc = ifp->if_softc;

	device_printf(sc->sc_dev, "device timeout\n");

	if_statinc(ifp, if_oerrors);
	sun4i_emac_ifinit(ifp);
	sun4i_emac_ifstart(ifp);
}

static void
sun4i_emac_rx_hash(struct sun4i_emac_softc *sc)
{
	struct ethercom *ec = &sc->sc_ec;
	struct ifnet * const ifp = &ec->ec_if;
	struct ether_multistep step;
	struct ether_multi *enm;
	uint32_t hash[2];
	uint32_t rxctl;

	rxctl = sun4i_emac_read(sc, EMAC_RX_CTL_REG);
	rxctl &= ~EMAC_RX_CTL_MHF;
	rxctl |= EMAC_RX_CTL_UCAD;
	rxctl |= EMAC_RX_CTL_DAF;
	rxctl |= EMAC_RX_CTL_MC0;
	rxctl |= EMAC_RX_CTL_BC0;
	rxctl |= EMAC_RX_CTL_POR;

	hash[0] = hash[1] = ~0;
	if (ifp->if_flags & IFF_PROMISC) {
		ifp->if_flags |= IFF_ALLMULTI;
		rxctl |= EMAC_RX_CTL_PROMISC;
	} else {
		rxctl &= ~EMAC_RX_CTL_PROMISC;
	}

	if ((ifp->if_flags & IFF_PROMISC) == 0) {
		hash[0] = hash[1] = 0;

		ETHER_LOCK(ec);
		ETHER_FIRST_MULTI(step, ec, enm);
		while (enm != NULL) {
			if (memcmp(enm->enm_addrlo, enm->enm_addrhi,
			    ETHER_ADDR_LEN)) {
				ETHER_UNLOCK(ec);
				/*
				 * We must listen to a range of multicast
				 * addresses. For now, just accept all
				 * multicasts, rather than trying to set only
				 * those filter bits needed to match the range.
				 * (At this time, the only use of address
				 * ranges is for IP multicast routing, for
				 * which the range is big enough to require all
				 * bits set.)
				 */
				hash[0] = hash[1] = ~0;
				ifp->if_flags |= IFF_ALLMULTI;
				goto done;
			}

			u_int crc = ether_crc32_be(enm->enm_addrlo,
			    ETHER_ADDR_LEN);

			/* Just want the 6 most significant bits. */
			crc >>= 26;

			/* Set the corresponding bit in the filter. */
			hash[crc >> 5] |= __BIT(crc & 31);
			ETHER_NEXT_MULTI(step, enm);
		}
		ETHER_UNLOCK(ec);
		ifp->if_flags &= ~IFF_ALLMULTI;
		rxctl |= EMAC_RX_CTL_MHF;
	}

done:

	sun4i_emac_write(sc, EMAC_RX_HASH0_REG, hash[0]);
	sun4i_emac_write(sc, EMAC_RX_HASH1_REG, hash[1]);

	sun4i_emac_write(sc, EMAC_RX_CTL_REG, rxctl);
}