Home | History | Annotate | Line # | Download | only in dev
      1 /*	$NetBSD: if_cnmac.c,v 1.30 2025/10/04 04:44:20 thorpej Exp $	*/
      2 
      3 /*
      4  * Copyright (c) 2007 Internet Initiative Japan, Inc.
      5  * All rights reserved.
      6  *
      7  * Redistribution and use in source and binary forms, with or without
      8  * modification, are permitted provided that the following conditions
      9  * are met:
     10  * 1. Redistributions of source code must retain the above copyright
     11  *    notice, this list of conditions and the following disclaimer.
     12  * 2. Redistributions in binary form must reproduce the above copyright
     13  *    notice, this list of conditions and the following disclaimer in the
     14  *    documentation and/or other materials provided with the distribution.
     15  *
     16  * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
     17  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
     18  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
     19  * ARE DISCLAIMED.  IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
     20  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
     21  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
     22  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
     23  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
     24  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
     25  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
     26  * SUCH DAMAGE.
     27  */
     28 
     29 #include <sys/cdefs.h>
     30 __KERNEL_RCSID(0, "$NetBSD: if_cnmac.c,v 1.30 2025/10/04 04:44:20 thorpej Exp $");
     31 
     32 /*
     33  * If no free send buffer is available, free all the sent buffers and bail out.
     34  */
     35 #define CNMAC_SEND_QUEUE_CHECK
     36 
     37 /* XXX XXX XXX XXX XXX XXX */
     38 
     39 #include <sys/param.h>
     40 #include <sys/systm.h>
     41 #include <sys/pool.h>
     42 #include <sys/mbuf.h>
     43 #include <sys/kernel.h>
     44 #include <sys/socket.h>
     45 #include <sys/ioctl.h>
     46 #include <sys/errno.h>
     47 #include <sys/device.h>
     48 #include <sys/queue.h>
     49 #include <sys/conf.h>
     50 #include <sys/sysctl.h>
     51 #include <sys/syslog.h>
     52 
     53 #include <net/if.h>
     54 #include <net/if_media.h>
     55 #include <net/if_ether.h>
     56 #include <net/route.h>
     57 #include <net/bpf.h>
     58 
     59 #include <netinet/in.h>
     60 #include <netinet/in_systm.h>
     61 #include <netinet/in_var.h>
     62 #include <netinet/ip.h>
     63 
     64 #include <sys/bus.h>
     65 #include <machine/intr.h>
     66 #include <machine/endian.h>
     67 #include <machine/locore.h>
     68 
     69 #include <dev/mii/mii.h>
     70 #include <dev/mii/miivar.h>
     71 
     72 #include <mips/cpuregs.h>
     73 
     74 #include <mips/cavium/octeonreg.h>
     75 #include <mips/cavium/octeonvar.h>
     76 #include <mips/cavium/include/iobusvar.h>
     77 
     78 #include <mips/cavium/dev/octeon_ciureg.h>
     79 #include <mips/cavium/dev/octeon_faureg.h>
     80 #include <mips/cavium/dev/octeon_fpareg.h>
     81 #include <mips/cavium/dev/octeon_gmxreg.h>
     82 #include <mips/cavium/dev/octeon_pipreg.h>
     83 #include <mips/cavium/dev/octeon_powreg.h>
     84 #include <mips/cavium/dev/octeon_fauvar.h>
     85 #include <mips/cavium/dev/octeon_fpavar.h>
     86 #include <mips/cavium/dev/octeon_gmxvar.h>
     87 #include <mips/cavium/dev/octeon_ipdvar.h>
     88 #include <mips/cavium/dev/octeon_pipvar.h>
     89 #include <mips/cavium/dev/octeon_pkovar.h>
     90 #include <mips/cavium/dev/octeon_powvar.h>
     91 #include <mips/cavium/dev/octeon_smivar.h>
     92 
     93 #include <mips/cavium/dev/if_cnmacvar.h>
     94 
     95 /*
     96  * Set the PKO to think command buffers are an odd length.  This makes it so we
     97  * never have to divide a comamnd across two buffers.
     98  */
     99 #define OCTEON_POOL_NWORDS_CMD	\
    100 	    (((uint32_t)OCTEON_POOL_SIZE_CMD / sizeof(uint64_t)) - 1)
    101 #define FPA_COMMAND_BUFFER_POOL_NWORDS	OCTEON_POOL_NWORDS_CMD	/* XXX */
    102 
    103 static void	cnmac_buf_init(struct cnmac_softc *);
    104 
    105 static int	cnmac_match(device_t, struct cfdata *, void *);
    106 static void	cnmac_attach(device_t, device_t, void *);
    107 static void	cnmac_pip_init(struct cnmac_softc *);
    108 static void	cnmac_ipd_init(struct cnmac_softc *);
    109 static void	cnmac_pko_init(struct cnmac_softc *);
    110 
    111 static void	cnmac_board_mac_addr(uint8_t *, size_t, struct cnmac_softc *);
    112 
    113 static int	cnmac_mii_readreg(device_t, int, int, uint16_t *);
    114 static int	cnmac_mii_writereg(device_t, int, int, uint16_t);
    115 static void	cnmac_mii_statchg(struct ifnet *);
    116 
    117 static int	cnmac_mediainit(struct cnmac_softc *);
    118 static void	cnmac_mediastatus(struct ifnet *, struct ifmediareq *);
    119 
    120 static inline void cnmac_send_queue_flush_prefetch(struct cnmac_softc *);
    121 static inline void cnmac_send_queue_flush_fetch(struct cnmac_softc *);
    122 static inline void cnmac_send_queue_flush(struct cnmac_softc *);
    123 static inline void cnmac_send_queue_flush_sync(struct cnmac_softc *);
    124 static void cnmac_send_queue_check_and_flush(struct cnmac_softc *);
    125 static inline int cnmac_send_queue_is_full(struct cnmac_softc *);
    126 static inline void cnmac_send_queue_add(struct cnmac_softc *, struct mbuf *,
    127     uint64_t *);
    128 static inline void cnmac_send_queue_del(struct cnmac_softc *, struct mbuf **,
    129     uint64_t **);
    130 static inline int cnmac_buf_free_work(struct cnmac_softc *, uint64_t *);
    131 static inline void cnmac_buf_ext_free(struct mbuf *, void *, size_t, void *);
    132 
    133 static int	cnmac_ioctl(struct ifnet *, u_long, void *);
    134 static void	cnmac_watchdog(struct ifnet *);
    135 static int	cnmac_init(struct ifnet *);
    136 static void	cnmac_stop(struct ifnet *, int);
    137 static void	cnmac_start(struct ifnet *);
    138 
    139 static inline int cnmac_send_cmd(struct cnmac_softc *, uint64_t, uint64_t,
    140     int *);
    141 static inline uint64_t	cnmac_send_makecmd_w1(int, paddr_t);
    142 static inline uint64_t	cnmac_send_makecmd_w0(uint64_t, uint64_t, size_t, int,
    143     int);
    144 static inline int cnmac_send_makecmd_gbuf(struct cnmac_softc *, struct mbuf *,
    145     uint64_t *, int *);
    146 static inline int cnmac_send_makecmd(struct cnmac_softc *, struct mbuf *,
    147     uint64_t *, uint64_t *, uint64_t *);
    148 static inline int cnmac_send_buf(struct cnmac_softc *, struct mbuf *,
    149     uint64_t *, int *);
    150 static inline int cnmac_send(struct cnmac_softc *, struct mbuf *, int *);
    151 
    152 static int	cnmac_reset(struct cnmac_softc *);
    153 static int	cnmac_configure(struct cnmac_softc *);
    154 static int	cnmac_configure_common(struct cnmac_softc *);
    155 
    156 static void	cnmac_tick_free(void *);
    157 static void	cnmac_tick_misc(void *);
    158 
    159 static inline int cnmac_recv_mbuf(struct cnmac_softc *, uint64_t *,
    160     struct mbuf **);
    161 static inline int cnmac_recv_check(struct cnmac_softc *, uint64_t);
    162 static inline int cnmac_recv(struct cnmac_softc *, uint64_t *);
    163 static int	cnmac_intr(void *);
    164 
    165 /* device parameters */
    166 int		cnmac_param_pko_cmd_w0_n2 = 1;
    167 
    168 CFATTACH_DECL_NEW(cnmac, sizeof(struct cnmac_softc),
    169     cnmac_match, cnmac_attach, NULL, NULL);
    170 
    171 /* ---- buffer management */
    172 
    173 static const struct cnmac_pool_param {
    174 	int			poolno;
    175 	size_t			size;
    176 	size_t			nelems;
    177 } cnmac_pool_params[] = {
    178 #define	_ENTRY(x)	{ OCTEON_POOL_NO_##x, OCTEON_POOL_SIZE_##x, OCTEON_POOL_NELEMS_##x }
    179 	_ENTRY(PKT),
    180 	_ENTRY(WQE),
    181 	_ENTRY(CMD),
    182 	_ENTRY(SG)
    183 #undef	_ENTRY
    184 };
    185 struct octfpa_buf	*cnmac_pools[FPA_NPOOLS];
    186 #define	cnmac_fb_pkt	cnmac_pools[OCTEON_POOL_NO_PKT]
    187 #define	cnmac_fb_wqe	cnmac_pools[OCTEON_POOL_NO_WQE]
    188 #define	cnmac_fb_cmd	cnmac_pools[OCTEON_POOL_NO_CMD]
    189 #define	cnmac_fb_sg	cnmac_pools[OCTEON_POOL_NO_SG]
    190 
    191 static int	cnmac_npowgroups = 0;
    192 
    193 static void
    194 cnmac_buf_init(struct cnmac_softc *sc)
    195 {
    196 	static int once;
    197 	int i;
    198 	const struct cnmac_pool_param *pp;
    199 	struct octfpa_buf *fb;
    200 
    201 	if (once == 1)
    202 		return;
    203 	once = 1;
    204 
    205 	for (i = 0; i < (int)__arraycount(cnmac_pool_params); i++) {
    206 		pp = &cnmac_pool_params[i];
    207 		octfpa_buf_init(pp->poolno, pp->size, pp->nelems, &fb);
    208 		cnmac_pools[i] = fb;
    209 	}
    210 }
    211 
    212 /* ---- autoconf */
    213 
    214 static int
    215 cnmac_match(device_t parent, struct cfdata *match, void *aux)
    216 {
    217 	struct octgmx_attach_args *ga = aux;
    218 
    219 	if (strcmp(match->cf_name, ga->ga_name) != 0) {
    220 		return 0;
    221 	}
    222 	return 1;
    223 }
    224 
    225 static void
    226 cnmac_attach(device_t parent, device_t self, void *aux)
    227 {
    228 	struct cnmac_softc *sc = device_private(self);
    229 	struct octgmx_attach_args *ga = aux;
    230 	struct ifnet *ifp = &sc->sc_ethercom.ec_if;
    231 	prop_dictionary_t dict;
    232 	prop_object_t clk;
    233 	uint8_t enaddr[ETHER_ADDR_LEN];
    234 
    235 	if (cnmac_npowgroups >= OCTEON_POW_GROUP_MAX) {
    236 		printf(": out of POW groups\n");
    237 	}
    238 
    239 	sc->sc_dev = self;
    240 	sc->sc_regt = ga->ga_regt;
    241 	sc->sc_port = ga->ga_portno;
    242 	sc->sc_port_type = ga->ga_port_type;
    243 	sc->sc_gmx = ga->ga_gmx;
    244 	sc->sc_gmx_port = ga->ga_gmx_port;
    245 	sc->sc_smi = ga->ga_smi;
    246 	sc->sc_powgroup = cnmac_npowgroups++;
    247 
    248 	if (sc->sc_port >= CVMSEG_LM_ETHER_COUNT) {
    249 		/*
    250 		 * If we got here, increase CVMSEG_LM_ETHER_COUNT
    251 		 * in octeonvar.h .
    252 		 */
    253 		printf("%s: ERROR out of CVMSEG LM buffers\n",
    254 		    device_xname(self));
    255 		return;
    256 	}
    257 
    258 	sc->sc_init_flag = 0;
    259 	/*
    260 	 * XXXUEBAYASI
    261 	 * Setting PIP_IP_OFFSET[OFFSET] to 8 causes panic ... why???
    262 	 */
    263 	sc->sc_ip_offset = 0/* XXX */;
    264 
    265 	if (MIPS_PRID_IMPL(mips_options.mips_cpu_id) <= MIPS_CN30XX) {
    266 		SET(sc->sc_quirks, CNMAC_QUIRKS_NO_PRE_ALIGN);
    267 		SET(sc->sc_quirks, CNMAC_QUIRKS_NO_RX_INBND);
    268 	}
    269 
    270 	cnmac_board_mac_addr(enaddr, sizeof(enaddr), sc);
    271 	printf("%s: Ethernet address %s\n", device_xname(self),
    272 	    ether_sprintf(enaddr));
    273 
    274 	SIMPLEQ_INIT(&sc->sc_sendq);
    275 	sc->sc_soft_req_thresh = 15/* XXX */;
    276 	sc->sc_ext_callback_cnt = 0;
    277 
    278 	octgmx_stats_init(sc->sc_gmx_port);
    279 
    280 	callout_init(&sc->sc_tick_misc_ch, 0);
    281 	callout_setfunc(&sc->sc_tick_misc_ch, cnmac_tick_misc, sc);
    282 
    283 	callout_init(&sc->sc_tick_free_ch, 0);
    284 	callout_setfunc(&sc->sc_tick_free_ch, cnmac_tick_free, sc);
    285 
    286 	const int dv_unit = device_unit(self);
    287 	octfau_op_init(&sc->sc_fau_done,
    288 	    OCTEON_CVMSEG_ETHER_OFFSET(dv_unit, csm_ether_fau_done),
    289 	    OCT_FAU_REG_ADDR_END - (8 * (dv_unit + 1))/* XXX */);
    290 	octfau_op_set_8(&sc->sc_fau_done, 0);
    291 
    292 	cnmac_pip_init(sc);
    293 	cnmac_ipd_init(sc);
    294 	cnmac_pko_init(sc);
    295 
    296 	cnmac_configure_common(sc);
    297 
    298 	sc->sc_gmx_port->sc_ipd = sc->sc_ipd;
    299 	sc->sc_gmx_port->sc_port_mii = &sc->sc_mii;
    300 	sc->sc_gmx_port->sc_port_ec = &sc->sc_ethercom;
    301 	/* XXX */
    302 	sc->sc_gmx_port->sc_quirks = sc->sc_quirks;
    303 
    304 	/* XXX */
    305 	sc->sc_pow = &octpow_softc;
    306 
    307 	cnmac_mediainit(sc);
    308 
    309 	strncpy(ifp->if_xname, device_xname(self), sizeof(ifp->if_xname));
    310 	ifp->if_softc = sc;
    311 	ifp->if_flags = IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST;
    312 	ifp->if_ioctl = cnmac_ioctl;
    313 	ifp->if_start = cnmac_start;
    314 	ifp->if_watchdog = cnmac_watchdog;
    315 	ifp->if_init = cnmac_init;
    316 	ifp->if_stop = cnmac_stop;
    317 	IFQ_SET_MAXLEN(&ifp->if_snd, uimax(GATHER_QUEUE_SIZE, IFQ_MAXLEN));
    318 	IFQ_SET_READY(&ifp->if_snd);
    319 
    320 
    321 	ifp->if_capabilities =
    322 #if 0	/* XXX: no tx checksum yet */
    323 	    IFCAP_CSUM_IPv4_Tx | IFCAP_CSUM_IPv4_Rx |
    324 	    IFCAP_CSUM_TCPv4_Tx | IFCAP_CSUM_TCPv4_Rx |
    325 	    IFCAP_CSUM_UDPv4_Tx | IFCAP_CSUM_UDPv4_Rx |
    326 	    IFCAP_CSUM_TCPv6_Tx | IFCAP_CSUM_TCPv6_Rx |
    327 	    IFCAP_CSUM_UDPv6_Tx | IFCAP_CSUM_UDPv6_Rx;
    328 #else
    329 	    IFCAP_CSUM_IPv4_Rx | IFCAP_CSUM_TCPv4_Rx | IFCAP_CSUM_UDPv4_Rx |
    330 	    IFCAP_CSUM_TCPv6_Rx | IFCAP_CSUM_UDPv6_Rx;
    331 #endif
    332 
    333 	/* 802.1Q VLAN-sized frames are supported */
    334 	sc->sc_ethercom.ec_capabilities |= ETHERCAP_VLAN_MTU;
    335 
    336 	octgmx_set_mac_addr(sc->sc_gmx_port, enaddr);
    337 
    338 	if_attach(ifp);
    339 	ether_ifattach(ifp, enaddr);
    340 	octgmx_set_filter(sc->sc_gmx_port);
    341 
    342 #if 1
    343 	cnmac_buf_init(sc);
    344 #endif
    345 
    346 	sc->sc_ih = octeon_intr_establish(POW_WORKQ_IRQ(sc->sc_powgroup),
    347 	    IPL_NET, cnmac_intr, sc);
    348 	if (sc->sc_ih == NULL)
    349 		panic("%s: could not set up interrupt", device_xname(self));
    350 
    351 	dict = device_properties(sc->sc_gmx->sc_dev);
    352 
    353 	clk = prop_dictionary_get(dict, "rgmii-tx");
    354 	if (clk)
    355 		sc->sc_gmx_port->sc_clk_tx_setting =
    356 		    prop_number_signed_value(clk);
    357 	clk = prop_dictionary_get(dict, "rgmii-rx");
    358 	if (clk)
    359 		sc->sc_gmx_port->sc_clk_rx_setting =
    360 		    prop_number_signed_value(clk);
    361 }
    362 
    363 /* ---- submodules */
    364 
    365 /* XXX */
    366 static void
    367 cnmac_pip_init(struct cnmac_softc *sc)
    368 {
    369 	struct octpip_attach_args pip_aa;
    370 
    371 	pip_aa.aa_port = sc->sc_port;
    372 	pip_aa.aa_regt = sc->sc_regt;
    373 	pip_aa.aa_tag_type = POW_TAG_TYPE_ORDERED/* XXX */;
    374 	pip_aa.aa_receive_group = sc->sc_powgroup;
    375 	pip_aa.aa_ip_offset = sc->sc_ip_offset;
    376 	octpip_init(&pip_aa, &sc->sc_pip);
    377 	octpip_port_config(sc->sc_pip);
    378 }
    379 
    380 /* XXX */
    381 static void
    382 cnmac_ipd_init(struct cnmac_softc *sc)
    383 {
    384 	struct octipd_attach_args ipd_aa;
    385 
    386 	ipd_aa.aa_port = sc->sc_port;
    387 	ipd_aa.aa_regt = sc->sc_regt;
    388 	ipd_aa.aa_first_mbuff_skip = 184/* XXX */;
    389 	ipd_aa.aa_not_first_mbuff_skip = 0/* XXX */;
    390 	octipd_init(&ipd_aa, &sc->sc_ipd);
    391 }
    392 
    393 /* XXX */
    394 static void
    395 cnmac_pko_init(struct cnmac_softc *sc)
    396 {
    397 	struct octpko_attach_args pko_aa;
    398 
    399 	pko_aa.aa_port = sc->sc_port;
    400 	pko_aa.aa_regt = sc->sc_regt;
    401 	pko_aa.aa_cmdptr = &sc->sc_cmdptr;
    402 	pko_aa.aa_cmd_buf_pool = OCTEON_POOL_NO_CMD;
    403 	pko_aa.aa_cmd_buf_size = OCTEON_POOL_NWORDS_CMD;
    404 	octpko_init(&pko_aa, &sc->sc_pko);
    405 }
    406 
    407 /* ---- XXX */
    408 
    409 #define	ADDR2UINT64(u, a) \
    410 	do { \
    411 		u = \
    412 		    (((uint64_t)a[0] << 40) | ((uint64_t)a[1] << 32) | \
    413 		     ((uint64_t)a[2] << 24) | ((uint64_t)a[3] << 16) | \
    414 		     ((uint64_t)a[4] <<	 8) | ((uint64_t)a[5] <<  0)); \
    415 	} while (0)
    416 #define	UINT642ADDR(a, u) \
    417 	do { \
    418 		a[0] = (uint8_t)((u) >> 40); a[1] = (uint8_t)((u) >> 32); \
    419 		a[2] = (uint8_t)((u) >> 24); a[3] = (uint8_t)((u) >> 16); \
    420 		a[4] = (uint8_t)((u) >>	 8); a[5] = (uint8_t)((u) >>  0); \
    421 	} while (0)
    422 
    423 static void
    424 cnmac_board_mac_addr(uint8_t *enaddr, size_t size, struct cnmac_softc *sc)
    425 {
    426 	bool rv __diagused;
    427 
    428 	rv = ether_getaddr(sc->sc_dev, enaddr);
    429 	KASSERT(rv == true);
    430 }
    431 
    432 /* ---- media */
    433 
    434 static int
    435 cnmac_mii_readreg(device_t self, int phy_addr, int reg, uint16_t *val)
    436 {
    437 	struct cnmac_softc *sc = device_private(self);
    438 
    439 	return octsmi_read(sc->sc_smi, phy_addr, reg, val);
    440 }
    441 
    442 static int
    443 cnmac_mii_writereg(device_t self, int phy_addr, int reg, uint16_t val)
    444 {
    445 	struct cnmac_softc *sc = device_private(self);
    446 
    447 	return octsmi_write(sc->sc_smi, phy_addr, reg, val);
    448 }
    449 
    450 static void
    451 cnmac_mii_statchg(struct ifnet *ifp)
    452 {
    453 	struct cnmac_softc *sc = ifp->if_softc;
    454 
    455 	octpko_port_enable(sc->sc_pko, 0);
    456 	octgmx_port_enable(sc->sc_gmx_port, 0);
    457 
    458 	cnmac_reset(sc);
    459 
    460 	if (ISSET(ifp->if_flags, IFF_RUNNING))
    461 		octgmx_set_filter(sc->sc_gmx_port);
    462 
    463 	octpko_port_enable(sc->sc_pko, 1);
    464 	octgmx_port_enable(sc->sc_gmx_port, 1);
    465 }
    466 
    467 static int
    468 cnmac_mediainit(struct cnmac_softc *sc)
    469 {
    470 	struct ifnet *ifp = &sc->sc_ethercom.ec_if;
    471 	struct mii_data *mii = &sc->sc_mii;
    472 	prop_object_t phy;
    473 
    474 	mii->mii_ifp = ifp;
    475 	mii->mii_readreg = cnmac_mii_readreg;
    476 	mii->mii_writereg = cnmac_mii_writereg;
    477 	mii->mii_statchg = cnmac_mii_statchg;
    478 	sc->sc_ethercom.ec_mii = mii;
    479 
    480 	/* Initialize ifmedia structures. */
    481 	ifmedia_init(&mii->mii_media, 0, ether_mediachange, cnmac_mediastatus);
    482 
    483 	phy = prop_dictionary_get(device_properties(sc->sc_dev), "phy-addr");
    484 	KASSERT(phy != NULL);
    485 
    486 	mii_attach(sc->sc_dev, mii, 0xffffffff, prop_number_signed_value(phy),
    487 	    MII_OFFSET_ANY, MIIF_DOPAUSE);
    488 
    489 	/* XXX XXX XXX */
    490 	if (LIST_FIRST(&mii->mii_phys) != NULL) {
    491 		/* XXX XXX XXX */
    492 		ifmedia_set(&mii->mii_media, IFM_ETHER | IFM_AUTO);
    493 		/* XXX XXX XXX */
    494 	} else {
    495 		/* XXX XXX XXX */
    496 		ifmedia_add(&mii->mii_media, IFM_ETHER | IFM_NONE,
    497 		    MII_MEDIA_NONE, NULL);
    498 		/* XXX XXX XXX */
    499 		ifmedia_set(&mii->mii_media, IFM_ETHER | IFM_NONE);
    500 		/* XXX XXX XXX */
    501 	}
    502 	/* XXX XXX XXX */
    503 
    504 	return 0;
    505 }
    506 
    507 static void
    508 cnmac_mediastatus(struct ifnet *ifp, struct ifmediareq *ifmr)
    509 {
    510 	struct cnmac_softc *sc = ifp->if_softc;
    511 
    512 	mii_pollstat(&sc->sc_mii);
    513 
    514 	ifmr->ifm_status = sc->sc_mii.mii_media_status;
    515 	ifmr->ifm_active = sc->sc_mii.mii_media_active;
    516 	ifmr->ifm_active = (sc->sc_mii.mii_media_active & ~IFM_ETH_FMASK) |
    517 	    sc->sc_gmx_port->sc_port_flowflags;
    518 }
    519 
    520 /* ---- send buffer garbage collection */
    521 
    522 static inline void
    523 cnmac_send_queue_flush_prefetch(struct cnmac_softc *sc)
    524 {
    525 
    526 	KASSERT(sc->sc_prefetch == 0);
    527 	octfau_op_inc_fetch_8(&sc->sc_fau_done, 0);
    528 	sc->sc_prefetch = 1;
    529 }
    530 
    531 static inline void
    532 cnmac_send_queue_flush_fetch(struct cnmac_softc *sc)
    533 {
    534 
    535 	KASSERT(sc->sc_prefetch == 1);
    536 	sc->sc_hard_done_cnt = octfau_op_inc_read_8(&sc->sc_fau_done);
    537 	KASSERT(sc->sc_hard_done_cnt <= 0);
    538 	sc->sc_prefetch = 0;
    539 }
    540 
    541 static inline void
    542 cnmac_send_queue_flush(struct cnmac_softc *sc)
    543 {
    544 	const int64_t sent_count = sc->sc_hard_done_cnt;
    545 	int i;
    546 
    547 	KASSERT(sc->sc_flush == 0);
    548 	KASSERT(sent_count <= 0);
    549 
    550 	for (i = 0; i < 0 - sent_count; i++) {
    551 		struct mbuf *m;
    552 		uint64_t *gbuf;
    553 
    554 		cnmac_send_queue_del(sc, &m, &gbuf);
    555 
    556 		octfpa_buf_put(cnmac_fb_sg, gbuf);
    557 
    558 		m_freem(m);
    559 
    560 		sc->sc_txbusy = false;
    561 	}
    562 
    563 	octfau_op_inc_fetch_8(&sc->sc_fau_done, i);
    564 	sc->sc_flush = i;
    565 }
    566 
    567 static inline void
    568 cnmac_send_queue_flush_sync(struct cnmac_softc *sc)
    569 {
    570 	if (sc->sc_flush == 0)
    571 		return;
    572 
    573 	KASSERT(sc->sc_flush > 0);
    574 
    575 	/* XXX XXX XXX */
    576 	octfau_op_inc_read_8(&sc->sc_fau_done);
    577 	sc->sc_soft_req_cnt -= sc->sc_flush;
    578 	KASSERT(sc->sc_soft_req_cnt >= 0);
    579 	/* XXX XXX XXX */
    580 
    581 	sc->sc_flush = 0;
    582 }
    583 
    584 static inline int
    585 cnmac_send_queue_is_full(struct cnmac_softc *sc)
    586 {
    587 #ifdef CNMAC_SEND_QUEUE_CHECK
    588 	int64_t nofree_cnt;
    589 
    590 	nofree_cnt = sc->sc_soft_req_cnt + sc->sc_hard_done_cnt;
    591 
    592 	if (__predict_false(nofree_cnt == GATHER_QUEUE_SIZE - 1)) {
    593 		cnmac_send_queue_flush(sc);
    594 		cnmac_send_queue_flush_sync(sc);
    595 		return 1;
    596 	}
    597 
    598 #endif
    599 	return 0;
    600 }
    601 
    602 static void
    603 cnmac_send_queue_check_and_flush(struct cnmac_softc *sc)
    604 {
    605 	int s;
    606 
    607 	/* XXX XXX XXX */
    608 	s = splnet();
    609 	if (sc->sc_soft_req_cnt > 0) {
    610 		cnmac_send_queue_flush_prefetch(sc);
    611 		cnmac_send_queue_flush_fetch(sc);
    612 		cnmac_send_queue_flush(sc);
    613 		cnmac_send_queue_flush_sync(sc);
    614 	}
    615 	splx(s);
    616 	/* XXX XXX XXX */
    617 }
    618 
    619 /*
    620  * (Ab)use m_nextpkt and m_paddr to maintain mbuf chain and pointer to gather
    621  * buffer.  Other mbuf members may be used by m_freem(), so don't touch them!
    622  */
    623 
    624 struct _send_queue_entry {
    625 	union {
    626 		struct mbuf _sqe_s_mbuf;
    627 		struct {
    628 			char _sqe_s_entry_pad[offsetof(struct mbuf, m_nextpkt)];
    629 			SIMPLEQ_ENTRY(_send_queue_entry) _sqe_s_entry_entry;
    630 		} _sqe_s_entry;
    631 		struct {
    632 			char _sqe_s_gbuf_pad[offsetof(struct mbuf, m_paddr)];
    633 			uint64_t *_sqe_s_gbuf_gbuf;
    634 		} _sqe_s_gbuf;
    635 	} _sqe_u;
    636 #define	_sqe_entry	_sqe_u._sqe_s_entry._sqe_s_entry_entry
    637 #define	_sqe_gbuf	_sqe_u._sqe_s_gbuf._sqe_s_gbuf_gbuf
    638 };
    639 
    640 static inline void
    641 cnmac_send_queue_add(struct cnmac_softc *sc, struct mbuf *m,
    642     uint64_t *gbuf)
    643 {
    644 	struct _send_queue_entry *sqe = (struct _send_queue_entry *)m;
    645 
    646 	sqe->_sqe_gbuf = gbuf;
    647 	SIMPLEQ_INSERT_TAIL(&sc->sc_sendq, sqe, _sqe_entry);
    648 
    649 	if ((m->m_flags & M_EXT) && m->m_ext.ext_free != NULL)
    650 		sc->sc_ext_callback_cnt++;
    651 }
    652 
    653 static inline void
    654 cnmac_send_queue_del(struct cnmac_softc *sc, struct mbuf **rm, uint64_t **rgbuf)
    655 {
    656 	struct _send_queue_entry *sqe;
    657 
    658 	sqe = SIMPLEQ_FIRST(&sc->sc_sendq);
    659 	KASSERT(sqe != NULL);
    660 	SIMPLEQ_REMOVE_HEAD(&sc->sc_sendq, _sqe_entry);
    661 
    662 	*rm = (void *)sqe;
    663 	*rgbuf = sqe->_sqe_gbuf;
    664 
    665 	if (((*rm)->m_flags & M_EXT) && (*rm)->m_ext.ext_free != NULL) {
    666 		sc->sc_ext_callback_cnt--;
    667 		KASSERT(sc->sc_ext_callback_cnt >= 0);
    668 	}
    669 }
    670 
    671 static inline int
    672 cnmac_buf_free_work(struct cnmac_softc *sc, uint64_t *work)
    673 {
    674 
    675 	/* XXX when jumbo frame */
    676 	if (ISSET(work[2], PIP_WQE_WORD2_IP_BUFS)) {
    677 		paddr_t addr;
    678 		paddr_t start_buffer;
    679 
    680 		addr = work[3] & PIP_WQE_WORD3_ADDR;
    681 		start_buffer = addr & ~(2048 - 1);
    682 
    683 		octfpa_buf_put_paddr(cnmac_fb_pkt, start_buffer);
    684 	}
    685 
    686 	octfpa_buf_put(cnmac_fb_wqe, work);
    687 
    688 	return 0;
    689 }
    690 
    691 static inline void
    692 cnmac_buf_ext_free(struct mbuf *m, void *buf, size_t size, void *arg)
    693 {
    694 	octfpa_buf_put(cnmac_fb_pkt, buf);
    695 
    696 	KASSERT(m != NULL);
    697 
    698 	pool_cache_put(mb_cache, m);
    699 }
    700 
    701 /* ---- ifnet interfaces */
    702 
    703 static int
    704 cnmac_ioctl(struct ifnet *ifp, u_long cmd, void *data)
    705 {
    706 	struct cnmac_softc *sc = ifp->if_softc;
    707 	struct ifreq *ifr = (struct ifreq *)data;
    708 	int s, error;
    709 
    710 	s = splnet();
    711 	switch (cmd) {
    712 	case SIOCSIFMEDIA:
    713 		/* Flow control requires full-duplex mode. */
    714 		if (IFM_SUBTYPE(ifr->ifr_media) == IFM_AUTO ||
    715 		    (ifr->ifr_media & IFM_FDX) == 0) {
    716 			ifr->ifr_media &= ~IFM_ETH_FMASK;
    717 		}
    718 		if (IFM_SUBTYPE(ifr->ifr_media) != IFM_AUTO) {
    719 			if ((ifr->ifr_media & IFM_ETH_FMASK) == IFM_FLOW) {
    720 				ifr->ifr_media |=
    721 				    IFM_ETH_TXPAUSE | IFM_ETH_RXPAUSE;
    722 			}
    723 			sc->sc_gmx_port->sc_port_flowflags =
    724 				ifr->ifr_media & IFM_ETH_FMASK;
    725 		}
    726 		error = ifmedia_ioctl(ifp, ifr, &sc->sc_mii.mii_media, cmd);
    727 		break;
    728 	default:
    729 		error = ether_ioctl(ifp, cmd, data);
    730 		break;
    731 	}
    732 
    733 	if (error == ENETRESET) {
    734 		if (ISSET(ifp->if_flags, IFF_RUNNING))
    735 			octgmx_set_filter(sc->sc_gmx_port);
    736 		error = 0;
    737 	}
    738 
    739 	cnmac_start(ifp);
    740 
    741 	splx(s);
    742 
    743 	return error;
    744 }
    745 
    746 /* ---- send (output) */
    747 
    748 static inline uint64_t
    749 cnmac_send_makecmd_w0(uint64_t fau0, uint64_t fau1, size_t len, int segs,
    750     int ipoffp1)
    751 {
    752 
    753 	return octpko_cmd_word0(
    754 		OCT_FAU_OP_SIZE_64,		/* sz1 */
    755 		OCT_FAU_OP_SIZE_64,		/* sz0 */
    756 		1, fau1, 1, fau0,		/* s1, reg1, s0, reg0 */
    757 		0,				/* le */
    758 		cnmac_param_pko_cmd_w0_n2,	/* n2 */
    759 		1, 0,				/* q, r */
    760 		(segs == 1) ? 0 : 1,		/* g */
    761 		0, 0, 1,			/* ipoffp1, ii, df */
    762 		segs, (int)len);		/* segs, totalbytes */
    763 }
    764 
    765 static inline uint64_t
    766 cnmac_send_makecmd_w1(int size, paddr_t addr)
    767 {
    768 
    769 	return octpko_cmd_word1(
    770 		0, 0,				/* i, back */
    771 		OCTEON_POOL_NO_SG,		/* pool */
    772 		size, addr);			/* size, addr */
    773 }
    774 
    775 static inline int
    776 cnmac_send_makecmd_gbuf(struct cnmac_softc *sc, struct mbuf *m0, uint64_t *gbuf,
    777     int *rsegs)
    778 {
    779 	struct mbuf *m;
    780 	int segs = 0;
    781 	uintptr_t laddr, rlen, nlen;
    782 
    783 	for (m = m0; m != NULL; m = m->m_next) {
    784 
    785 		if (__predict_false(m->m_len == 0))
    786 			continue;
    787 
    788 		/* Aligned 4k */
    789 		laddr = (uintptr_t)m->m_data & (PAGE_SIZE - 1);
    790 
    791 		if (laddr + m->m_len > PAGE_SIZE) {
    792 			/* XXX XXX XXX */
    793 			rlen = PAGE_SIZE - laddr;
    794 			nlen = m->m_len - rlen;
    795 			*(gbuf + segs) = cnmac_send_makecmd_w1(rlen,
    796 			    kvtophys((vaddr_t)m->m_data));
    797 			segs++;
    798 			if (segs > 63) {
    799 				return 1;
    800 			}
    801 			/* XXX XXX XXX */
    802 		} else {
    803 			rlen = 0;
    804 			nlen = m->m_len;
    805 		}
    806 
    807 		*(gbuf + segs) = cnmac_send_makecmd_w1(nlen,
    808 		    kvtophys((vaddr_t)(m->m_data + rlen)));
    809 		segs++;
    810 		if (segs > 63) {
    811 			return 1;
    812 		}
    813 	}
    814 
    815 	KASSERT(m == NULL);
    816 
    817 	*rsegs = segs;
    818 
    819 	return 0;
    820 }
    821 
    822 static inline int
    823 cnmac_send_makecmd(struct cnmac_softc *sc, struct mbuf *m,
    824     uint64_t *gbuf, uint64_t *rpko_cmd_w0, uint64_t *rpko_cmd_w1)
    825 {
    826 	uint64_t pko_cmd_w0, pko_cmd_w1;
    827 	int ipoffp1;
    828 	int segs;
    829 	int result = 0;
    830 
    831 	if (cnmac_send_makecmd_gbuf(sc, m, gbuf, &segs)) {
    832 		log(LOG_WARNING, "%s: there are a lot of number of segments"
    833 		    " of transmission data", device_xname(sc->sc_dev));
    834 		result = 1;
    835 		goto done;
    836 	}
    837 
    838 	/*  Get the IP packet offset for TCP/UDP checksum offloading. */
    839 	ipoffp1 = (m->m_pkthdr.csum_flags & (M_CSUM_TCPv4 | M_CSUM_UDPv4))
    840 	    ? (ETHER_HDR_LEN + 1) : 0;
    841 
    842 	/*
    843 	 * segs == 1	-> link mode (single continuous buffer)
    844 	 *		   WORD1[size] is number of bytes pointed by segment
    845 	 *
    846 	 * segs > 1	-> gather mode (scatter-gather buffer)
    847 	 *		   WORD1[size] is number of segments
    848 	 */
    849 	pko_cmd_w0 = cnmac_send_makecmd_w0(sc->sc_fau_done.fd_regno,
    850 	    0, m->m_pkthdr.len, segs, ipoffp1);
    851 	if (segs == 1) {
    852 		pko_cmd_w1 = cnmac_send_makecmd_w1(
    853 		    m->m_pkthdr.len, kvtophys((vaddr_t)m->m_data));
    854 	} else {
    855 #ifdef __mips_n32
    856 		KASSERT(MIPS_KSEG0_P(gbuf));
    857 		pko_cmd_w1 = cnmac_send_makecmd_w1(segs,
    858 		    MIPS_KSEG0_TO_PHYS(gbuf));
    859 #else
    860 		pko_cmd_w1 = cnmac_send_makecmd_w1(segs,
    861 		    MIPS_XKPHYS_TO_PHYS(gbuf));
    862 #endif
    863 	}
    864 
    865 	*rpko_cmd_w0 = pko_cmd_w0;
    866 	*rpko_cmd_w1 = pko_cmd_w1;
    867 
    868 done:
    869 	return result;
    870 }
    871 
    872 static inline int
    873 cnmac_send_cmd(struct cnmac_softc *sc, uint64_t pko_cmd_w0,
    874     uint64_t pko_cmd_w1, int *pwdc)
    875 {
    876 	uint64_t *cmdptr;
    877 	int result = 0;
    878 
    879 #ifdef __mips_n32
    880 	KASSERT((sc->sc_cmdptr.cmdptr & ~MIPS_PHYS_MASK) == 0);
    881 	cmdptr = (uint64_t *)MIPS_PHYS_TO_KSEG0(sc->sc_cmdptr.cmdptr);
    882 #else
    883 	cmdptr = (uint64_t *)MIPS_PHYS_TO_XKPHYS_CACHED(sc->sc_cmdptr.cmdptr);
    884 #endif
    885 	cmdptr += sc->sc_cmdptr.cmdptr_idx;
    886 
    887 	KASSERT(cmdptr != NULL);
    888 
    889 	*cmdptr++ = pko_cmd_w0;
    890 	*cmdptr++ = pko_cmd_w1;
    891 
    892 	KASSERT(sc->sc_cmdptr.cmdptr_idx + 2 <= FPA_COMMAND_BUFFER_POOL_NWORDS - 1);
    893 
    894 	if (sc->sc_cmdptr.cmdptr_idx + 2 == FPA_COMMAND_BUFFER_POOL_NWORDS - 1) {
    895 		paddr_t buf;
    896 
    897 		buf = octfpa_buf_get_paddr(cnmac_fb_cmd);
    898 		if (buf == 0) {
    899 			log(LOG_WARNING,
    900 			    "%s: can not allocate command buffer from free pool allocator\n",
    901 			    device_xname(sc->sc_dev));
    902 			result = 1;
    903 			goto done;
    904 		}
    905 		*cmdptr++ = buf;
    906 		sc->sc_cmdptr.cmdptr = (uint64_t)buf;
    907 		sc->sc_cmdptr.cmdptr_idx = 0;
    908 	} else {
    909 		sc->sc_cmdptr.cmdptr_idx += 2;
    910 	}
    911 
    912 	*pwdc += 2;
    913 
    914 done:
    915 	return result;
    916 }
    917 
    918 static inline int
    919 cnmac_send_buf(struct cnmac_softc *sc, struct mbuf *m, uint64_t *gbuf,
    920     int *pwdc)
    921 {
    922 	int result = 0, error;
    923 	uint64_t pko_cmd_w0, pko_cmd_w1;
    924 
    925 	error = cnmac_send_makecmd(sc, m, gbuf, &pko_cmd_w0, &pko_cmd_w1);
    926 	if (error != 0) {
    927 		/* Already logging */
    928 		result = error;
    929 		goto done;
    930 	}
    931 
    932 	error = cnmac_send_cmd(sc, pko_cmd_w0, pko_cmd_w1, pwdc);
    933 	if (error != 0) {
    934 		/* Already logging */
    935 		result = error;
    936 	}
    937 
    938 done:
    939 	return result;
    940 }
    941 
    942 static inline int
    943 cnmac_send(struct cnmac_softc *sc, struct mbuf *m, int *pwdc)
    944 {
    945 	paddr_t gaddr = 0;
    946 	uint64_t *gbuf = NULL;
    947 	int result = 0, error;
    948 
    949 	gaddr = octfpa_buf_get_paddr(cnmac_fb_sg);
    950 	if (gaddr == 0) {
    951 		log(LOG_WARNING, "%s: can not allocate gather buffer from "
    952 		    "free pool allocator\n", device_xname(sc->sc_dev));
    953 		result = 1;
    954 		goto done;
    955 	}
    956 
    957 #ifdef __mips_n32
    958 	KASSERT((gaddr & ~MIPS_PHYS_MASK) == 0);
    959 	gbuf = (uint64_t *)(uintptr_t)MIPS_PHYS_TO_KSEG0(gaddr);
    960 #else
    961 	gbuf = (uint64_t *)(uintptr_t)MIPS_PHYS_TO_XKPHYS_CACHED(gaddr);
    962 #endif
    963 
    964 	KASSERT(gbuf != NULL);
    965 
    966 	error = cnmac_send_buf(sc, m, gbuf, pwdc);
    967 	if (error != 0) {
    968 		/* Already logging */
    969 		octfpa_buf_put_paddr(cnmac_fb_sg, gaddr);
    970 		result = error;
    971 		goto done;
    972 	}
    973 
    974 	cnmac_send_queue_add(sc, m, gbuf);
    975 
    976 done:
    977 	return result;
    978 }
    979 
    980 static void
    981 cnmac_start(struct ifnet *ifp)
    982 {
    983 	struct cnmac_softc *sc = ifp->if_softc;
    984 	struct mbuf *m;
    985 	int wdc = 0;
    986 
    987 	/*
    988 	 * Performance tuning
    989 	 * pre-send iobdma request
    990 	 */
    991 	cnmac_send_queue_flush_prefetch(sc);
    992 
    993 	if ((ifp->if_flags & IFF_RUNNING) == 0)
    994 		goto last;
    995 
    996 	if (sc->sc_txbusy)
    997 		goto last;
    998 
    999 	if (__predict_false(!octgmx_link_status(sc->sc_gmx_port)))
   1000 		goto last;
   1001 
   1002 	for (;;) {
   1003 		IFQ_POLL(&ifp->if_snd, m);
   1004 		if (__predict_false(m == NULL))
   1005 			break;
   1006 
   1007 		/* XXX XXX XXX */
   1008 		cnmac_send_queue_flush_fetch(sc);
   1009 
   1010 		/*
   1011 		 * If no free send buffer is available, free all the sent
   1012 		 * buffers and bail out.
   1013 		 */
   1014 		if (cnmac_send_queue_is_full(sc)) {
   1015 			sc->sc_txbusy = true;
   1016 			if (wdc > 0)
   1017 				octpko_op_doorbell_write(sc->sc_port,
   1018 				    sc->sc_port, wdc);
   1019 			callout_schedule(&sc->sc_tick_free_ch, 1);
   1020 			return;
   1021 		}
   1022 		/* XXX XXX XXX */
   1023 
   1024 		IFQ_DEQUEUE(&ifp->if_snd, m);
   1025 
   1026 		bpf_mtap(ifp, m, BPF_D_OUT);
   1027 
   1028 		/* XXX XXX XXX */
   1029 		if (sc->sc_soft_req_cnt > sc->sc_soft_req_thresh)
   1030 			cnmac_send_queue_flush(sc);
   1031 		if (cnmac_send(sc, m, &wdc)) {
   1032 			IF_DROP(&ifp->if_snd);
   1033 			m_freem(m);
   1034 			log(LOG_WARNING,
   1035 			  "%s: failed in the transmission of the packet\n",
   1036 			  device_xname(sc->sc_dev));
   1037 		} else
   1038 			sc->sc_soft_req_cnt++;
   1039 
   1040 		if (sc->sc_flush)
   1041 			cnmac_send_queue_flush_sync(sc);
   1042 		/* XXX XXX XXX */
   1043 
   1044 		/* Send next iobdma request */
   1045 		cnmac_send_queue_flush_prefetch(sc);
   1046 	}
   1047 
   1048 	if (wdc > 0)
   1049 		octpko_op_doorbell_write(sc->sc_port, sc->sc_port, wdc);
   1050 
   1051 last:
   1052 	cnmac_send_queue_flush_fetch(sc);
   1053 	callout_schedule(&sc->sc_tick_free_ch, 1);
   1054 }
   1055 
   1056 static void
   1057 cnmac_watchdog(struct ifnet *ifp)
   1058 {
   1059 	struct cnmac_softc *sc = ifp->if_softc;
   1060 
   1061 	printf("%s: device timeout\n", device_xname(sc->sc_dev));
   1062 
   1063 	cnmac_configure(sc);
   1064 
   1065 	SET(ifp->if_flags, IFF_RUNNING);
   1066 	sc->sc_txbusy = false;
   1067 	ifp->if_timer = 0;
   1068 
   1069 	cnmac_start(ifp);
   1070 }
   1071 
   1072 static int
   1073 cnmac_init(struct ifnet *ifp)
   1074 {
   1075 	struct cnmac_softc *sc = ifp->if_softc;
   1076 
   1077 	/* XXX don't disable commonly used parts!!! XXX */
   1078 	if (sc->sc_init_flag == 0) {
   1079 		/* Cancel any pending I/O. */
   1080 		cnmac_stop(ifp, 0);
   1081 
   1082 		/* Initialize the device */
   1083 		cnmac_configure(sc);
   1084 
   1085 		octpko_enable(sc->sc_pko);
   1086 		octipd_enable(sc->sc_ipd);
   1087 
   1088 		sc->sc_init_flag = 1;
   1089 	} else {
   1090 		octgmx_port_enable(sc->sc_gmx_port, 1);
   1091 	}
   1092 	mii_ifmedia_change(&sc->sc_mii);
   1093 
   1094 	octgmx_set_filter(sc->sc_gmx_port);
   1095 
   1096 	callout_schedule(&sc->sc_tick_misc_ch, hz);
   1097 	callout_schedule(&sc->sc_tick_free_ch, hz);
   1098 
   1099 	SET(ifp->if_flags, IFF_RUNNING);
   1100 	sc->sc_txbusy = false;
   1101 
   1102 	return 0;
   1103 }
   1104 
   1105 static void
   1106 cnmac_stop(struct ifnet *ifp, int disable)
   1107 {
   1108 	struct cnmac_softc *sc = ifp->if_softc;
   1109 
   1110 	callout_stop(&sc->sc_tick_misc_ch);
   1111 	callout_stop(&sc->sc_tick_free_ch);
   1112 
   1113 	mii_down(&sc->sc_mii);
   1114 
   1115 	octgmx_port_enable(sc->sc_gmx_port, 0);
   1116 
   1117 	/* Mark the interface as down and cancel the watchdog timer. */
   1118 	CLR(ifp->if_flags, IFF_RUNNING);
   1119 	sc->sc_txbusy = false;
   1120 	ifp->if_timer = 0;
   1121 }
   1122 
   1123 /* ---- misc */
   1124 
   1125 static int
   1126 cnmac_reset(struct cnmac_softc *sc)
   1127 {
   1128 	octgmx_reset_speed(sc->sc_gmx_port);
   1129 	octgmx_reset_flowctl(sc->sc_gmx_port);
   1130 	octgmx_reset_timing(sc->sc_gmx_port);
   1131 
   1132 	return 0;
   1133 }
   1134 
   1135 static int
   1136 cnmac_configure(struct cnmac_softc *sc)
   1137 {
   1138 	octgmx_port_enable(sc->sc_gmx_port, 0);
   1139 
   1140 	cnmac_reset(sc);
   1141 
   1142 	cnmac_configure_common(sc);
   1143 
   1144 	octpko_port_config(sc->sc_pko);
   1145 	octpko_port_enable(sc->sc_pko, 1);
   1146 	octpow_config(sc->sc_pow, sc->sc_powgroup);
   1147 
   1148 	octgmx_tx_stats_rd_clr(sc->sc_gmx_port, 1);
   1149 	octgmx_rx_stats_rd_clr(sc->sc_gmx_port, 1);
   1150 
   1151 	octgmx_port_enable(sc->sc_gmx_port, 1);
   1152 
   1153 	return 0;
   1154 }
   1155 
   1156 static int
   1157 cnmac_configure_common(struct cnmac_softc *sc)
   1158 {
   1159 	static int once;
   1160 
   1161 	if (once == 1)
   1162 		return 0;
   1163 	once = 1;
   1164 
   1165 	octipd_config(sc->sc_ipd);
   1166 	octpko_config(sc->sc_pko);
   1167 
   1168 	return 0;
   1169 }
   1170 
   1171 /* ---- receive (input) */
   1172 
   1173 static inline int
   1174 cnmac_recv_mbuf(struct cnmac_softc *sc, uint64_t *work, struct mbuf **rm)
   1175 {
   1176 	struct mbuf *m;
   1177 	vaddr_t addr;
   1178 	vaddr_t ext_buf;
   1179 	size_t ext_size;
   1180 	uint64_t word1 = work[1];
   1181 	uint64_t word2 = work[2];
   1182 	uint64_t word3 = work[3];
   1183 
   1184 	MGETHDR(m, M_NOWAIT, MT_DATA);
   1185 	if (m == NULL)
   1186 		return 1;
   1187 
   1188 	octfpa_buf_put(cnmac_fb_wqe, work);
   1189 
   1190 	if (__SHIFTOUT(word2, PIP_WQE_WORD2_IP_BUFS) != 1)
   1191 		panic("%s: expected one buffer, got %" PRId64, __func__,
   1192 		    __SHIFTOUT(word2, PIP_WQE_WORD2_IP_BUFS));
   1193 
   1194 
   1195 #ifdef __mips_n32
   1196 	KASSERT((word3 & ~MIPS_PHYS_MASK) == 0);
   1197 	addr = MIPS_PHYS_TO_KSEG0(word3 & PIP_WQE_WORD3_ADDR);
   1198 #else
   1199 	addr = MIPS_PHYS_TO_XKPHYS_CACHED(word3 & PIP_WQE_WORD3_ADDR);
   1200 #endif
   1201 
   1202 	ext_size = OCTEON_POOL_SIZE_PKT;
   1203 	ext_buf = addr & ~(ext_size - 1);
   1204 	MEXTADD(m, ext_buf, ext_size, 0, cnmac_buf_ext_free, NULL);
   1205 
   1206 	m->m_data = (void *)addr;
   1207 	m->m_len = m->m_pkthdr.len = (word1 & PIP_WQE_WORD1_LEN) >> 48;
   1208 	m_set_rcvif(m, &sc->sc_ethercom.ec_if);
   1209 
   1210 	/* Not readonly buffer */
   1211 	m->m_flags |= M_EXT_RW;
   1212 
   1213 	*rm = m;
   1214 
   1215 	KASSERT(*rm != NULL);
   1216 
   1217 	return 0;
   1218 }
   1219 
   1220 static inline int
   1221 cnmac_recv_check(struct cnmac_softc *sc, uint64_t word2)
   1222 {
   1223 	static struct timeval rxerr_log_interval = { 0, 2500000 };
   1224 	uint64_t opecode;
   1225 
   1226 	if (__predict_true(!ISSET(word2, PIP_WQE_WORD2_NOIP_RE)))
   1227 		return 0;
   1228 
   1229 	opecode = word2 & PIP_WQE_WORD2_NOIP_OPECODE;
   1230 	if ((sc->sc_ethercom.ec_if.if_flags & IFF_DEBUG) &&
   1231 	    ratecheck(&sc->sc_rxerr_log_last, &rxerr_log_interval))
   1232 		log(LOG_DEBUG, "%s: rx error (%"PRId64")\n",
   1233 		    device_xname(sc->sc_dev), opecode);
   1234 
   1235 	/* This error is harmless */
   1236 	if (opecode == PIP_WQE_WORD2_RE_OPCODE_OVRRUN)
   1237 		return 0;
   1238 
   1239 	return 1;
   1240 }
   1241 
   1242 static inline int
   1243 cnmac_recv(struct cnmac_softc *sc, uint64_t *work)
   1244 {
   1245 	struct ifnet *ifp;
   1246 	struct mbuf *m;
   1247 	uint64_t word2;
   1248 
   1249 	KASSERT(sc != NULL);
   1250 	KASSERT(work != NULL);
   1251 
   1252 	word2 = work[2];
   1253 	ifp = &sc->sc_ethercom.ec_if;
   1254 
   1255 	KASSERT(ifp != NULL);
   1256 
   1257 	if (!ISSET(ifp->if_flags, IFF_RUNNING))
   1258 		goto drop;
   1259 
   1260 	if (__predict_false(cnmac_recv_check(sc, word2) != 0)) {
   1261 		if_statinc(ifp, if_ierrors);
   1262 		goto drop;
   1263 	}
   1264 
   1265 	if (__predict_false(cnmac_recv_mbuf(sc, work, &m) != 0)) {
   1266 		if_statinc(ifp, if_ierrors);
   1267 		goto drop;
   1268 	}
   1269 
   1270 	/* work[0] .. work[3] may not be valid any more */
   1271 
   1272 	KASSERT(m != NULL);
   1273 
   1274 	octipd_offload(word2, m->m_data, &m->m_pkthdr.csum_flags);
   1275 
   1276 	if_percpuq_enqueue(ifp->if_percpuq, m);
   1277 
   1278 	return 0;
   1279 
   1280 drop:
   1281 	cnmac_buf_free_work(sc, work);
   1282 	return 1;
   1283 }
   1284 
   1285 static int
   1286 cnmac_intr(void *arg)
   1287 {
   1288 	struct cnmac_softc *sc = arg;
   1289 	uint64_t *work;
   1290 	uint64_t wqmask = __BIT(sc->sc_powgroup);
   1291 	uint32_t coreid = 0;	/* XXX octeon_get_coreid() */
   1292 	uint32_t port;
   1293 
   1294 	_POW_WR8(sc->sc_pow, POW_PP_GRP_MSK_OFFSET(coreid), wqmask);
   1295 
   1296 	octpow_tag_sw_wait();
   1297 	octpow_work_request_async(OCTEON_CVMSEG_OFFSET(csm_pow_intr),
   1298 	    POW_NO_WAIT);
   1299 
   1300 	for (;;) {
   1301 		work = (uint64_t *)octpow_work_response_async(
   1302 		    OCTEON_CVMSEG_OFFSET(csm_pow_intr));
   1303 		if (work == NULL)
   1304 			break;
   1305 
   1306 		octpow_tag_sw_wait();
   1307 		octpow_work_request_async(OCTEON_CVMSEG_OFFSET(csm_pow_intr),
   1308 		    POW_NO_WAIT);
   1309 
   1310 		port = __SHIFTOUT(work[1], PIP_WQE_WORD1_IPRT);
   1311 		if (port != sc->sc_port) {
   1312 			printf("%s: unexpected wqe port %u, should be %u\n",
   1313 			    device_xname(sc->sc_dev), port, sc->sc_port);
   1314 			goto wqe_error;
   1315 		}
   1316 
   1317 		(void)cnmac_recv(sc, work);
   1318 
   1319 		cnmac_send_queue_check_and_flush(sc);
   1320 	}
   1321 
   1322 	_POW_WR8(sc->sc_pow, POW_WQ_INT_OFFSET, wqmask);
   1323 
   1324 	return 1;
   1325 
   1326 wqe_error:
   1327 	printf("word0: 0x%016" PRIx64 "\n", work[0]);
   1328 	printf("word1: 0x%016" PRIx64 "\n", work[1]);
   1329 	printf("word2: 0x%016" PRIx64 "\n", work[2]);
   1330 	printf("word3: 0x%016" PRIx64 "\n", work[3]);
   1331 	panic("wqe_error");
   1332 }
   1333 
   1334 /* ---- tick */
   1335 
   1336 /*
   1337  * cnmac_tick_free
   1338  *
   1339  * => garbage collect send gather buffer / mbuf
   1340  * => called at softclock
   1341  */
   1342 static void
   1343 cnmac_tick_free(void *arg)
   1344 {
   1345 	struct cnmac_softc *sc = arg;
   1346 	int timo;
   1347 
   1348 	cnmac_send_queue_check_and_flush(sc);
   1349 
   1350 	timo = (sc->sc_ext_callback_cnt > 0) ? 1 : hz;
   1351 	callout_schedule(&sc->sc_tick_free_ch, timo);
   1352 }
   1353 
   1354 /*
   1355  * cnmac_tick_misc
   1356  *
   1357  * => collect statistics
   1358  * => check link status
   1359  * => called at softclock
   1360  */
   1361 static void
   1362 cnmac_tick_misc(void *arg)
   1363 {
   1364 	struct cnmac_softc *sc = arg;
   1365 	struct ifnet *ifp;
   1366 	int s;
   1367 
   1368 	s = splnet();
   1369 
   1370 	ifp = &sc->sc_ethercom.ec_if;
   1371 
   1372 	octgmx_stats(sc->sc_gmx_port);
   1373 	octpip_stats(sc->sc_pip, ifp, sc->sc_port);
   1374 	mii_tick(&sc->sc_mii);
   1375 
   1376 	splx(s);
   1377 
   1378 	callout_schedule(&sc->sc_tick_misc_ch, hz);
   1379 }
   1380