Home | History | Annotate | Line # | Download | only in dev
      1 /*	$NetBSD: if_cnmac.c,v 1.29 2022/09/29 07:00:46 skrll Exp $	*/
      2 
      3 /*
      4  * Copyright (c) 2007 Internet Initiative Japan, Inc.
      5  * All rights reserved.
      6  *
      7  * Redistribution and use in source and binary forms, with or without
      8  * modification, are permitted provided that the following conditions
      9  * are met:
     10  * 1. Redistributions of source code must retain the above copyright
     11  *    notice, this list of conditions and the following disclaimer.
     12  * 2. Redistributions in binary form must reproduce the above copyright
     13  *    notice, this list of conditions and the following disclaimer in the
     14  *    documentation and/or other materials provided with the distribution.
     15  *
     16  * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
     17  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
     18  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
     19  * ARE DISCLAIMED.  IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
     20  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
     21  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
     22  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
     23  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
     24  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
     25  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
     26  * SUCH DAMAGE.
     27  */
     28 
     29 #include <sys/cdefs.h>
     30 __KERNEL_RCSID(0, "$NetBSD: if_cnmac.c,v 1.29 2022/09/29 07:00:46 skrll Exp $");
     31 
     32 /*
     33  * If no free send buffer is available, free all the sent buffers and bail out.
     34  */
     35 #define CNMAC_SEND_QUEUE_CHECK
     36 
     37 /* XXX XXX XXX XXX XXX XXX */
     38 
     39 #include <sys/param.h>
     40 #include <sys/systm.h>
     41 #include <sys/pool.h>
     42 #include <sys/mbuf.h>
     43 #include <sys/kernel.h>
     44 #include <sys/socket.h>
     45 #include <sys/ioctl.h>
     46 #include <sys/errno.h>
     47 #include <sys/device.h>
     48 #include <sys/queue.h>
     49 #include <sys/conf.h>
     50 #include <sys/sysctl.h>
     51 #include <sys/syslog.h>
     52 
     53 #include <net/if.h>
     54 #include <net/if_media.h>
     55 #include <net/if_ether.h>
     56 #include <net/route.h>
     57 #include <net/bpf.h>
     58 
     59 #include <netinet/in.h>
     60 #include <netinet/in_systm.h>
     61 #include <netinet/in_var.h>
     62 #include <netinet/ip.h>
     63 
     64 #include <sys/bus.h>
     65 #include <machine/intr.h>
     66 #include <machine/endian.h>
     67 #include <machine/locore.h>
     68 
     69 #include <dev/mii/mii.h>
     70 #include <dev/mii/miivar.h>
     71 
     72 #include <mips/cpuregs.h>
     73 
     74 #include <mips/cavium/octeonreg.h>
     75 #include <mips/cavium/octeonvar.h>
     76 #include <mips/cavium/include/iobusvar.h>
     77 
     78 #include <mips/cavium/dev/octeon_ciureg.h>
     79 #include <mips/cavium/dev/octeon_faureg.h>
     80 #include <mips/cavium/dev/octeon_fpareg.h>
     81 #include <mips/cavium/dev/octeon_gmxreg.h>
     82 #include <mips/cavium/dev/octeon_pipreg.h>
     83 #include <mips/cavium/dev/octeon_powreg.h>
     84 #include <mips/cavium/dev/octeon_fauvar.h>
     85 #include <mips/cavium/dev/octeon_fpavar.h>
     86 #include <mips/cavium/dev/octeon_gmxvar.h>
     87 #include <mips/cavium/dev/octeon_ipdvar.h>
     88 #include <mips/cavium/dev/octeon_pipvar.h>
     89 #include <mips/cavium/dev/octeon_pkovar.h>
     90 #include <mips/cavium/dev/octeon_powvar.h>
     91 #include <mips/cavium/dev/octeon_smivar.h>
     92 
     93 #include <mips/cavium/dev/if_cnmacvar.h>
     94 
     95 /*
     96  * Set the PKO to think command buffers are an odd length.  This makes it so we
     97  * never have to divide a comamnd across two buffers.
     98  */
     99 #define OCTEON_POOL_NWORDS_CMD	\
    100 	    (((uint32_t)OCTEON_POOL_SIZE_CMD / sizeof(uint64_t)) - 1)
    101 #define FPA_COMMAND_BUFFER_POOL_NWORDS	OCTEON_POOL_NWORDS_CMD	/* XXX */
    102 
    103 static void	cnmac_buf_init(struct cnmac_softc *);
    104 
    105 static int	cnmac_match(device_t, struct cfdata *, void *);
    106 static void	cnmac_attach(device_t, device_t, void *);
    107 static void	cnmac_pip_init(struct cnmac_softc *);
    108 static void	cnmac_ipd_init(struct cnmac_softc *);
    109 static void	cnmac_pko_init(struct cnmac_softc *);
    110 
    111 static void	cnmac_board_mac_addr(uint8_t *, size_t, struct cnmac_softc *);
    112 
    113 static int	cnmac_mii_readreg(device_t, int, int, uint16_t *);
    114 static int	cnmac_mii_writereg(device_t, int, int, uint16_t);
    115 static void	cnmac_mii_statchg(struct ifnet *);
    116 
    117 static int	cnmac_mediainit(struct cnmac_softc *);
    118 static void	cnmac_mediastatus(struct ifnet *, struct ifmediareq *);
    119 
    120 static inline void cnmac_send_queue_flush_prefetch(struct cnmac_softc *);
    121 static inline void cnmac_send_queue_flush_fetch(struct cnmac_softc *);
    122 static inline void cnmac_send_queue_flush(struct cnmac_softc *);
    123 static inline void cnmac_send_queue_flush_sync(struct cnmac_softc *);
    124 static void cnmac_send_queue_check_and_flush(struct cnmac_softc *);
    125 static inline int cnmac_send_queue_is_full(struct cnmac_softc *);
    126 static inline void cnmac_send_queue_add(struct cnmac_softc *, struct mbuf *,
    127     uint64_t *);
    128 static inline void cnmac_send_queue_del(struct cnmac_softc *, struct mbuf **,
    129     uint64_t **);
    130 static inline int cnmac_buf_free_work(struct cnmac_softc *, uint64_t *);
    131 static inline void cnmac_buf_ext_free(struct mbuf *, void *, size_t, void *);
    132 
    133 static int	cnmac_ioctl(struct ifnet *, u_long, void *);
    134 static void	cnmac_watchdog(struct ifnet *);
    135 static int	cnmac_init(struct ifnet *);
    136 static void	cnmac_stop(struct ifnet *, int);
    137 static void	cnmac_start(struct ifnet *);
    138 
    139 static inline int cnmac_send_cmd(struct cnmac_softc *, uint64_t, uint64_t,
    140     int *);
    141 static inline uint64_t	cnmac_send_makecmd_w1(int, paddr_t);
    142 static inline uint64_t	cnmac_send_makecmd_w0(uint64_t, uint64_t, size_t, int,
    143     int);
    144 static inline int cnmac_send_makecmd_gbuf(struct cnmac_softc *, struct mbuf *,
    145     uint64_t *, int *);
    146 static inline int cnmac_send_makecmd(struct cnmac_softc *, struct mbuf *,
    147     uint64_t *, uint64_t *, uint64_t *);
    148 static inline int cnmac_send_buf(struct cnmac_softc *, struct mbuf *,
    149     uint64_t *, int *);
    150 static inline int cnmac_send(struct cnmac_softc *, struct mbuf *, int *);
    151 
    152 static int	cnmac_reset(struct cnmac_softc *);
    153 static int	cnmac_configure(struct cnmac_softc *);
    154 static int	cnmac_configure_common(struct cnmac_softc *);
    155 
    156 static void	cnmac_tick_free(void *);
    157 static void	cnmac_tick_misc(void *);
    158 
    159 static inline int cnmac_recv_mbuf(struct cnmac_softc *, uint64_t *,
    160     struct mbuf **);
    161 static inline int cnmac_recv_check(struct cnmac_softc *, uint64_t);
    162 static inline int cnmac_recv(struct cnmac_softc *, uint64_t *);
    163 static int	cnmac_intr(void *);
    164 
    165 /* device parameters */
    166 int		cnmac_param_pko_cmd_w0_n2 = 1;
    167 
    168 CFATTACH_DECL_NEW(cnmac, sizeof(struct cnmac_softc),
    169     cnmac_match, cnmac_attach, NULL, NULL);
    170 
    171 /* ---- buffer management */
    172 
    173 static const struct cnmac_pool_param {
    174 	int			poolno;
    175 	size_t			size;
    176 	size_t			nelems;
    177 } cnmac_pool_params[] = {
    178 #define	_ENTRY(x)	{ OCTEON_POOL_NO_##x, OCTEON_POOL_SIZE_##x, OCTEON_POOL_NELEMS_##x }
    179 	_ENTRY(PKT),
    180 	_ENTRY(WQE),
    181 	_ENTRY(CMD),
    182 	_ENTRY(SG)
    183 #undef	_ENTRY
    184 };
    185 struct octfpa_buf	*cnmac_pools[FPA_NPOOLS];
    186 #define	cnmac_fb_pkt	cnmac_pools[OCTEON_POOL_NO_PKT]
    187 #define	cnmac_fb_wqe	cnmac_pools[OCTEON_POOL_NO_WQE]
    188 #define	cnmac_fb_cmd	cnmac_pools[OCTEON_POOL_NO_CMD]
    189 #define	cnmac_fb_sg	cnmac_pools[OCTEON_POOL_NO_SG]
    190 
    191 static int	cnmac_npowgroups = 0;
    192 
    193 static void
    194 cnmac_buf_init(struct cnmac_softc *sc)
    195 {
    196 	static int once;
    197 	int i;
    198 	const struct cnmac_pool_param *pp;
    199 	struct octfpa_buf *fb;
    200 
    201 	if (once == 1)
    202 		return;
    203 	once = 1;
    204 
    205 	for (i = 0; i < (int)__arraycount(cnmac_pool_params); i++) {
    206 		pp = &cnmac_pool_params[i];
    207 		octfpa_buf_init(pp->poolno, pp->size, pp->nelems, &fb);
    208 		cnmac_pools[i] = fb;
    209 	}
    210 }
    211 
    212 /* ---- autoconf */
    213 
    214 static int
    215 cnmac_match(device_t parent, struct cfdata *match, void *aux)
    216 {
    217 	struct octgmx_attach_args *ga = aux;
    218 
    219 	if (strcmp(match->cf_name, ga->ga_name) != 0) {
    220 		return 0;
    221 	}
    222 	return 1;
    223 }
    224 
    225 static void
    226 cnmac_attach(device_t parent, device_t self, void *aux)
    227 {
    228 	struct cnmac_softc *sc = device_private(self);
    229 	struct octgmx_attach_args *ga = aux;
    230 	struct ifnet *ifp = &sc->sc_ethercom.ec_if;
    231 	prop_dictionary_t dict;
    232 	prop_object_t clk;
    233 	uint8_t enaddr[ETHER_ADDR_LEN];
    234 
    235 	if (cnmac_npowgroups >= OCTEON_POW_GROUP_MAX) {
    236 		printf(": out of POW groups\n");
    237 	}
    238 
    239 	sc->sc_dev = self;
    240 	sc->sc_regt = ga->ga_regt;
    241 	sc->sc_port = ga->ga_portno;
    242 	sc->sc_port_type = ga->ga_port_type;
    243 	sc->sc_gmx = ga->ga_gmx;
    244 	sc->sc_gmx_port = ga->ga_gmx_port;
    245 	sc->sc_smi = ga->ga_smi;
    246 	sc->sc_powgroup = cnmac_npowgroups++;
    247 
    248 	if (sc->sc_port >= CVMSEG_LM_ETHER_COUNT) {
    249 		/*
    250 		 * If we got here, increase CVMSEG_LM_ETHER_COUNT
    251 		 * in octeonvar.h .
    252 		 */
    253 		printf("%s: ERROR out of CVMSEG LM buffers\n",
    254 		    device_xname(self));
    255 		return;
    256 	}
    257 
    258 	sc->sc_init_flag = 0;
    259 	/*
    260 	 * XXXUEBAYASI
    261 	 * Setting PIP_IP_OFFSET[OFFSET] to 8 causes panic ... why???
    262 	 */
    263 	sc->sc_ip_offset = 0/* XXX */;
    264 
    265 	if (MIPS_PRID_IMPL(mips_options.mips_cpu_id) <= MIPS_CN30XX) {
    266 		SET(sc->sc_quirks, CNMAC_QUIRKS_NO_PRE_ALIGN);
    267 		SET(sc->sc_quirks, CNMAC_QUIRKS_NO_RX_INBND);
    268 	}
    269 
    270 	cnmac_board_mac_addr(enaddr, sizeof(enaddr), sc);
    271 	printf("%s: Ethernet address %s\n", device_xname(self),
    272 	    ether_sprintf(enaddr));
    273 
    274 	SIMPLEQ_INIT(&sc->sc_sendq);
    275 	sc->sc_soft_req_thresh = 15/* XXX */;
    276 	sc->sc_ext_callback_cnt = 0;
    277 
    278 	octgmx_stats_init(sc->sc_gmx_port);
    279 
    280 	callout_init(&sc->sc_tick_misc_ch, 0);
    281 	callout_setfunc(&sc->sc_tick_misc_ch, cnmac_tick_misc, sc);
    282 
    283 	callout_init(&sc->sc_tick_free_ch, 0);
    284 	callout_setfunc(&sc->sc_tick_free_ch, cnmac_tick_free, sc);
    285 
    286 	const int dv_unit = device_unit(self);
    287 	octfau_op_init(&sc->sc_fau_done,
    288 	    OCTEON_CVMSEG_ETHER_OFFSET(dv_unit, csm_ether_fau_done),
    289 	    OCT_FAU_REG_ADDR_END - (8 * (dv_unit + 1))/* XXX */);
    290 	octfau_op_set_8(&sc->sc_fau_done, 0);
    291 
    292 	cnmac_pip_init(sc);
    293 	cnmac_ipd_init(sc);
    294 	cnmac_pko_init(sc);
    295 
    296 	cnmac_configure_common(sc);
    297 
    298 	sc->sc_gmx_port->sc_ipd = sc->sc_ipd;
    299 	sc->sc_gmx_port->sc_port_mii = &sc->sc_mii;
    300 	sc->sc_gmx_port->sc_port_ec = &sc->sc_ethercom;
    301 	/* XXX */
    302 	sc->sc_gmx_port->sc_quirks = sc->sc_quirks;
    303 
    304 	/* XXX */
    305 	sc->sc_pow = &octpow_softc;
    306 
    307 	cnmac_mediainit(sc);
    308 
    309 	strncpy(ifp->if_xname, device_xname(self), sizeof(ifp->if_xname));
    310 	ifp->if_softc = sc;
    311 	ifp->if_flags = IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST;
    312 	ifp->if_ioctl = cnmac_ioctl;
    313 	ifp->if_start = cnmac_start;
    314 	ifp->if_watchdog = cnmac_watchdog;
    315 	ifp->if_init = cnmac_init;
    316 	ifp->if_stop = cnmac_stop;
    317 	IFQ_SET_MAXLEN(&ifp->if_snd, uimax(GATHER_QUEUE_SIZE, IFQ_MAXLEN));
    318 	IFQ_SET_READY(&ifp->if_snd);
    319 
    320 
    321 	ifp->if_capabilities =
    322 #if 0	/* XXX: no tx checksum yet */
    323 	    IFCAP_CSUM_IPv4_Tx | IFCAP_CSUM_IPv4_Rx |
    324 	    IFCAP_CSUM_TCPv4_Tx | IFCAP_CSUM_TCPv4_Rx |
    325 	    IFCAP_CSUM_UDPv4_Tx | IFCAP_CSUM_UDPv4_Rx |
    326 	    IFCAP_CSUM_TCPv6_Tx | IFCAP_CSUM_TCPv6_Rx |
    327 	    IFCAP_CSUM_UDPv6_Tx | IFCAP_CSUM_UDPv6_Rx;
    328 #else
    329 	    IFCAP_CSUM_IPv4_Rx | IFCAP_CSUM_TCPv4_Rx | IFCAP_CSUM_UDPv4_Rx |
    330 	    IFCAP_CSUM_TCPv6_Rx | IFCAP_CSUM_UDPv6_Rx;
    331 #endif
    332 
    333 	/* 802.1Q VLAN-sized frames are supported */
    334 	sc->sc_ethercom.ec_capabilities |= ETHERCAP_VLAN_MTU;
    335 
    336 	octgmx_set_mac_addr(sc->sc_gmx_port, enaddr);
    337 
    338 	if_attach(ifp);
    339 	ether_ifattach(ifp, enaddr);
    340 	octgmx_set_filter(sc->sc_gmx_port);
    341 
    342 #if 1
    343 	cnmac_buf_init(sc);
    344 #endif
    345 
    346 	sc->sc_ih = octeon_intr_establish(POW_WORKQ_IRQ(sc->sc_powgroup),
    347 	    IPL_NET, cnmac_intr, sc);
    348 	if (sc->sc_ih == NULL)
    349 		panic("%s: could not set up interrupt", device_xname(self));
    350 
    351 	dict = device_properties(sc->sc_gmx->sc_dev);
    352 
    353 	clk = prop_dictionary_get(dict, "rgmii-tx");
    354 	if (clk)
    355 		sc->sc_gmx_port->sc_clk_tx_setting =
    356 		    prop_number_signed_value(clk);
    357 	clk = prop_dictionary_get(dict, "rgmii-rx");
    358 	if (clk)
    359 		sc->sc_gmx_port->sc_clk_rx_setting =
    360 		    prop_number_signed_value(clk);
    361 }
    362 
    363 /* ---- submodules */
    364 
    365 /* XXX */
    366 static void
    367 cnmac_pip_init(struct cnmac_softc *sc)
    368 {
    369 	struct octpip_attach_args pip_aa;
    370 
    371 	pip_aa.aa_port = sc->sc_port;
    372 	pip_aa.aa_regt = sc->sc_regt;
    373 	pip_aa.aa_tag_type = POW_TAG_TYPE_ORDERED/* XXX */;
    374 	pip_aa.aa_receive_group = sc->sc_powgroup;
    375 	pip_aa.aa_ip_offset = sc->sc_ip_offset;
    376 	octpip_init(&pip_aa, &sc->sc_pip);
    377 	octpip_port_config(sc->sc_pip);
    378 }
    379 
    380 /* XXX */
    381 static void
    382 cnmac_ipd_init(struct cnmac_softc *sc)
    383 {
    384 	struct octipd_attach_args ipd_aa;
    385 
    386 	ipd_aa.aa_port = sc->sc_port;
    387 	ipd_aa.aa_regt = sc->sc_regt;
    388 	ipd_aa.aa_first_mbuff_skip = 184/* XXX */;
    389 	ipd_aa.aa_not_first_mbuff_skip = 0/* XXX */;
    390 	octipd_init(&ipd_aa, &sc->sc_ipd);
    391 }
    392 
    393 /* XXX */
    394 static void
    395 cnmac_pko_init(struct cnmac_softc *sc)
    396 {
    397 	struct octpko_attach_args pko_aa;
    398 
    399 	pko_aa.aa_port = sc->sc_port;
    400 	pko_aa.aa_regt = sc->sc_regt;
    401 	pko_aa.aa_cmdptr = &sc->sc_cmdptr;
    402 	pko_aa.aa_cmd_buf_pool = OCTEON_POOL_NO_CMD;
    403 	pko_aa.aa_cmd_buf_size = OCTEON_POOL_NWORDS_CMD;
    404 	octpko_init(&pko_aa, &sc->sc_pko);
    405 }
    406 
    407 /* ---- XXX */
    408 
    409 #define	ADDR2UINT64(u, a) \
    410 	do { \
    411 		u = \
    412 		    (((uint64_t)a[0] << 40) | ((uint64_t)a[1] << 32) | \
    413 		     ((uint64_t)a[2] << 24) | ((uint64_t)a[3] << 16) | \
    414 		     ((uint64_t)a[4] <<	 8) | ((uint64_t)a[5] <<  0)); \
    415 	} while (0)
    416 #define	UINT642ADDR(a, u) \
    417 	do { \
    418 		a[0] = (uint8_t)((u) >> 40); a[1] = (uint8_t)((u) >> 32); \
    419 		a[2] = (uint8_t)((u) >> 24); a[3] = (uint8_t)((u) >> 16); \
    420 		a[4] = (uint8_t)((u) >>	 8); a[5] = (uint8_t)((u) >>  0); \
    421 	} while (0)
    422 
    423 static void
    424 cnmac_board_mac_addr(uint8_t *enaddr, size_t size, struct cnmac_softc *sc)
    425 {
    426 	prop_dictionary_t dict;
    427 	prop_data_t ea;
    428 
    429 	dict = device_properties(sc->sc_dev);
    430 	KASSERT(dict != NULL);
    431 	ea = prop_dictionary_get(dict, "mac-address");
    432 	KASSERT(ea != NULL);
    433 	memcpy(enaddr, prop_data_value(ea), size);
    434 }
    435 
    436 /* ---- media */
    437 
    438 static int
    439 cnmac_mii_readreg(device_t self, int phy_addr, int reg, uint16_t *val)
    440 {
    441 	struct cnmac_softc *sc = device_private(self);
    442 
    443 	return octsmi_read(sc->sc_smi, phy_addr, reg, val);
    444 }
    445 
    446 static int
    447 cnmac_mii_writereg(device_t self, int phy_addr, int reg, uint16_t val)
    448 {
    449 	struct cnmac_softc *sc = device_private(self);
    450 
    451 	return octsmi_write(sc->sc_smi, phy_addr, reg, val);
    452 }
    453 
    454 static void
    455 cnmac_mii_statchg(struct ifnet *ifp)
    456 {
    457 	struct cnmac_softc *sc = ifp->if_softc;
    458 
    459 	octpko_port_enable(sc->sc_pko, 0);
    460 	octgmx_port_enable(sc->sc_gmx_port, 0);
    461 
    462 	cnmac_reset(sc);
    463 
    464 	if (ISSET(ifp->if_flags, IFF_RUNNING))
    465 		octgmx_set_filter(sc->sc_gmx_port);
    466 
    467 	octpko_port_enable(sc->sc_pko, 1);
    468 	octgmx_port_enable(sc->sc_gmx_port, 1);
    469 }
    470 
    471 static int
    472 cnmac_mediainit(struct cnmac_softc *sc)
    473 {
    474 	struct ifnet *ifp = &sc->sc_ethercom.ec_if;
    475 	struct mii_data *mii = &sc->sc_mii;
    476 	prop_object_t phy;
    477 
    478 	mii->mii_ifp = ifp;
    479 	mii->mii_readreg = cnmac_mii_readreg;
    480 	mii->mii_writereg = cnmac_mii_writereg;
    481 	mii->mii_statchg = cnmac_mii_statchg;
    482 	sc->sc_ethercom.ec_mii = mii;
    483 
    484 	/* Initialize ifmedia structures. */
    485 	ifmedia_init(&mii->mii_media, 0, ether_mediachange, cnmac_mediastatus);
    486 
    487 	phy = prop_dictionary_get(device_properties(sc->sc_dev), "phy-addr");
    488 	KASSERT(phy != NULL);
    489 
    490 	mii_attach(sc->sc_dev, mii, 0xffffffff, prop_number_signed_value(phy),
    491 	    MII_OFFSET_ANY, MIIF_DOPAUSE);
    492 
    493 	/* XXX XXX XXX */
    494 	if (LIST_FIRST(&mii->mii_phys) != NULL) {
    495 		/* XXX XXX XXX */
    496 		ifmedia_set(&mii->mii_media, IFM_ETHER | IFM_AUTO);
    497 		/* XXX XXX XXX */
    498 	} else {
    499 		/* XXX XXX XXX */
    500 		ifmedia_add(&mii->mii_media, IFM_ETHER | IFM_NONE,
    501 		    MII_MEDIA_NONE, NULL);
    502 		/* XXX XXX XXX */
    503 		ifmedia_set(&mii->mii_media, IFM_ETHER | IFM_NONE);
    504 		/* XXX XXX XXX */
    505 	}
    506 	/* XXX XXX XXX */
    507 
    508 	return 0;
    509 }
    510 
    511 static void
    512 cnmac_mediastatus(struct ifnet *ifp, struct ifmediareq *ifmr)
    513 {
    514 	struct cnmac_softc *sc = ifp->if_softc;
    515 
    516 	mii_pollstat(&sc->sc_mii);
    517 
    518 	ifmr->ifm_status = sc->sc_mii.mii_media_status;
    519 	ifmr->ifm_active = sc->sc_mii.mii_media_active;
    520 	ifmr->ifm_active = (sc->sc_mii.mii_media_active & ~IFM_ETH_FMASK) |
    521 	    sc->sc_gmx_port->sc_port_flowflags;
    522 }
    523 
    524 /* ---- send buffer garbage collection */
    525 
    526 static inline void
    527 cnmac_send_queue_flush_prefetch(struct cnmac_softc *sc)
    528 {
    529 
    530 	KASSERT(sc->sc_prefetch == 0);
    531 	octfau_op_inc_fetch_8(&sc->sc_fau_done, 0);
    532 	sc->sc_prefetch = 1;
    533 }
    534 
    535 static inline void
    536 cnmac_send_queue_flush_fetch(struct cnmac_softc *sc)
    537 {
    538 
    539 	KASSERT(sc->sc_prefetch == 1);
    540 	sc->sc_hard_done_cnt = octfau_op_inc_read_8(&sc->sc_fau_done);
    541 	KASSERT(sc->sc_hard_done_cnt <= 0);
    542 	sc->sc_prefetch = 0;
    543 }
    544 
    545 static inline void
    546 cnmac_send_queue_flush(struct cnmac_softc *sc)
    547 {
    548 	const int64_t sent_count = sc->sc_hard_done_cnt;
    549 	int i;
    550 
    551 	KASSERT(sc->sc_flush == 0);
    552 	KASSERT(sent_count <= 0);
    553 
    554 	for (i = 0; i < 0 - sent_count; i++) {
    555 		struct mbuf *m;
    556 		uint64_t *gbuf;
    557 
    558 		cnmac_send_queue_del(sc, &m, &gbuf);
    559 
    560 		octfpa_buf_put(cnmac_fb_sg, gbuf);
    561 
    562 		m_freem(m);
    563 
    564 		sc->sc_txbusy = false;
    565 	}
    566 
    567 	octfau_op_inc_fetch_8(&sc->sc_fau_done, i);
    568 	sc->sc_flush = i;
    569 }
    570 
    571 static inline void
    572 cnmac_send_queue_flush_sync(struct cnmac_softc *sc)
    573 {
    574 	if (sc->sc_flush == 0)
    575 		return;
    576 
    577 	KASSERT(sc->sc_flush > 0);
    578 
    579 	/* XXX XXX XXX */
    580 	octfau_op_inc_read_8(&sc->sc_fau_done);
    581 	sc->sc_soft_req_cnt -= sc->sc_flush;
    582 	KASSERT(sc->sc_soft_req_cnt >= 0);
    583 	/* XXX XXX XXX */
    584 
    585 	sc->sc_flush = 0;
    586 }
    587 
    588 static inline int
    589 cnmac_send_queue_is_full(struct cnmac_softc *sc)
    590 {
    591 #ifdef CNMAC_SEND_QUEUE_CHECK
    592 	int64_t nofree_cnt;
    593 
    594 	nofree_cnt = sc->sc_soft_req_cnt + sc->sc_hard_done_cnt;
    595 
    596 	if (__predict_false(nofree_cnt == GATHER_QUEUE_SIZE - 1)) {
    597 		cnmac_send_queue_flush(sc);
    598 		cnmac_send_queue_flush_sync(sc);
    599 		return 1;
    600 	}
    601 
    602 #endif
    603 	return 0;
    604 }
    605 
    606 static void
    607 cnmac_send_queue_check_and_flush(struct cnmac_softc *sc)
    608 {
    609 	int s;
    610 
    611 	/* XXX XXX XXX */
    612 	s = splnet();
    613 	if (sc->sc_soft_req_cnt > 0) {
    614 		cnmac_send_queue_flush_prefetch(sc);
    615 		cnmac_send_queue_flush_fetch(sc);
    616 		cnmac_send_queue_flush(sc);
    617 		cnmac_send_queue_flush_sync(sc);
    618 	}
    619 	splx(s);
    620 	/* XXX XXX XXX */
    621 }
    622 
    623 /*
    624  * (Ab)use m_nextpkt and m_paddr to maintain mbuf chain and pointer to gather
    625  * buffer.  Other mbuf members may be used by m_freem(), so don't touch them!
    626  */
    627 
    628 struct _send_queue_entry {
    629 	union {
    630 		struct mbuf _sqe_s_mbuf;
    631 		struct {
    632 			char _sqe_s_entry_pad[offsetof(struct mbuf, m_nextpkt)];
    633 			SIMPLEQ_ENTRY(_send_queue_entry) _sqe_s_entry_entry;
    634 		} _sqe_s_entry;
    635 		struct {
    636 			char _sqe_s_gbuf_pad[offsetof(struct mbuf, m_paddr)];
    637 			uint64_t *_sqe_s_gbuf_gbuf;
    638 		} _sqe_s_gbuf;
    639 	} _sqe_u;
    640 #define	_sqe_entry	_sqe_u._sqe_s_entry._sqe_s_entry_entry
    641 #define	_sqe_gbuf	_sqe_u._sqe_s_gbuf._sqe_s_gbuf_gbuf
    642 };
    643 
    644 static inline void
    645 cnmac_send_queue_add(struct cnmac_softc *sc, struct mbuf *m,
    646     uint64_t *gbuf)
    647 {
    648 	struct _send_queue_entry *sqe = (struct _send_queue_entry *)m;
    649 
    650 	sqe->_sqe_gbuf = gbuf;
    651 	SIMPLEQ_INSERT_TAIL(&sc->sc_sendq, sqe, _sqe_entry);
    652 
    653 	if ((m->m_flags & M_EXT) && m->m_ext.ext_free != NULL)
    654 		sc->sc_ext_callback_cnt++;
    655 }
    656 
    657 static inline void
    658 cnmac_send_queue_del(struct cnmac_softc *sc, struct mbuf **rm, uint64_t **rgbuf)
    659 {
    660 	struct _send_queue_entry *sqe;
    661 
    662 	sqe = SIMPLEQ_FIRST(&sc->sc_sendq);
    663 	KASSERT(sqe != NULL);
    664 	SIMPLEQ_REMOVE_HEAD(&sc->sc_sendq, _sqe_entry);
    665 
    666 	*rm = (void *)sqe;
    667 	*rgbuf = sqe->_sqe_gbuf;
    668 
    669 	if (((*rm)->m_flags & M_EXT) && (*rm)->m_ext.ext_free != NULL) {
    670 		sc->sc_ext_callback_cnt--;
    671 		KASSERT(sc->sc_ext_callback_cnt >= 0);
    672 	}
    673 }
    674 
    675 static inline int
    676 cnmac_buf_free_work(struct cnmac_softc *sc, uint64_t *work)
    677 {
    678 
    679 	/* XXX when jumbo frame */
    680 	if (ISSET(work[2], PIP_WQE_WORD2_IP_BUFS)) {
    681 		paddr_t addr;
    682 		paddr_t start_buffer;
    683 
    684 		addr = work[3] & PIP_WQE_WORD3_ADDR;
    685 		start_buffer = addr & ~(2048 - 1);
    686 
    687 		octfpa_buf_put_paddr(cnmac_fb_pkt, start_buffer);
    688 	}
    689 
    690 	octfpa_buf_put(cnmac_fb_wqe, work);
    691 
    692 	return 0;
    693 }
    694 
    695 static inline void
    696 cnmac_buf_ext_free(struct mbuf *m, void *buf, size_t size, void *arg)
    697 {
    698 	octfpa_buf_put(cnmac_fb_pkt, buf);
    699 
    700 	KASSERT(m != NULL);
    701 
    702 	pool_cache_put(mb_cache, m);
    703 }
    704 
    705 /* ---- ifnet interfaces */
    706 
    707 static int
    708 cnmac_ioctl(struct ifnet *ifp, u_long cmd, void *data)
    709 {
    710 	struct cnmac_softc *sc = ifp->if_softc;
    711 	struct ifreq *ifr = (struct ifreq *)data;
    712 	int s, error;
    713 
    714 	s = splnet();
    715 	switch (cmd) {
    716 	case SIOCSIFMEDIA:
    717 		/* Flow control requires full-duplex mode. */
    718 		if (IFM_SUBTYPE(ifr->ifr_media) == IFM_AUTO ||
    719 		    (ifr->ifr_media & IFM_FDX) == 0) {
    720 			ifr->ifr_media &= ~IFM_ETH_FMASK;
    721 		}
    722 		if (IFM_SUBTYPE(ifr->ifr_media) != IFM_AUTO) {
    723 			if ((ifr->ifr_media & IFM_ETH_FMASK) == IFM_FLOW) {
    724 				ifr->ifr_media |=
    725 				    IFM_ETH_TXPAUSE | IFM_ETH_RXPAUSE;
    726 			}
    727 			sc->sc_gmx_port->sc_port_flowflags =
    728 				ifr->ifr_media & IFM_ETH_FMASK;
    729 		}
    730 		error = ifmedia_ioctl(ifp, ifr, &sc->sc_mii.mii_media, cmd);
    731 		break;
    732 	default:
    733 		error = ether_ioctl(ifp, cmd, data);
    734 		break;
    735 	}
    736 
    737 	if (error == ENETRESET) {
    738 		if (ISSET(ifp->if_flags, IFF_RUNNING))
    739 			octgmx_set_filter(sc->sc_gmx_port);
    740 		error = 0;
    741 	}
    742 
    743 	cnmac_start(ifp);
    744 
    745 	splx(s);
    746 
    747 	return error;
    748 }
    749 
    750 /* ---- send (output) */
    751 
    752 static inline uint64_t
    753 cnmac_send_makecmd_w0(uint64_t fau0, uint64_t fau1, size_t len, int segs,
    754     int ipoffp1)
    755 {
    756 
    757 	return octpko_cmd_word0(
    758 		OCT_FAU_OP_SIZE_64,		/* sz1 */
    759 		OCT_FAU_OP_SIZE_64,		/* sz0 */
    760 		1, fau1, 1, fau0,		/* s1, reg1, s0, reg0 */
    761 		0,				/* le */
    762 		cnmac_param_pko_cmd_w0_n2,	/* n2 */
    763 		1, 0,				/* q, r */
    764 		(segs == 1) ? 0 : 1,		/* g */
    765 		0, 0, 1,			/* ipoffp1, ii, df */
    766 		segs, (int)len);		/* segs, totalbytes */
    767 }
    768 
    769 static inline uint64_t
    770 cnmac_send_makecmd_w1(int size, paddr_t addr)
    771 {
    772 
    773 	return octpko_cmd_word1(
    774 		0, 0,				/* i, back */
    775 		OCTEON_POOL_NO_SG,		/* pool */
    776 		size, addr);			/* size, addr */
    777 }
    778 
    779 static inline int
    780 cnmac_send_makecmd_gbuf(struct cnmac_softc *sc, struct mbuf *m0, uint64_t *gbuf,
    781     int *rsegs)
    782 {
    783 	struct mbuf *m;
    784 	int segs = 0;
    785 	uintptr_t laddr, rlen, nlen;
    786 
    787 	for (m = m0; m != NULL; m = m->m_next) {
    788 
    789 		if (__predict_false(m->m_len == 0))
    790 			continue;
    791 
    792 		/* Aligned 4k */
    793 		laddr = (uintptr_t)m->m_data & (PAGE_SIZE - 1);
    794 
    795 		if (laddr + m->m_len > PAGE_SIZE) {
    796 			/* XXX XXX XXX */
    797 			rlen = PAGE_SIZE - laddr;
    798 			nlen = m->m_len - rlen;
    799 			*(gbuf + segs) = cnmac_send_makecmd_w1(rlen,
    800 			    kvtophys((vaddr_t)m->m_data));
    801 			segs++;
    802 			if (segs > 63) {
    803 				return 1;
    804 			}
    805 			/* XXX XXX XXX */
    806 		} else {
    807 			rlen = 0;
    808 			nlen = m->m_len;
    809 		}
    810 
    811 		*(gbuf + segs) = cnmac_send_makecmd_w1(nlen,
    812 		    kvtophys((vaddr_t)(m->m_data + rlen)));
    813 		segs++;
    814 		if (segs > 63) {
    815 			return 1;
    816 		}
    817 	}
    818 
    819 	KASSERT(m == NULL);
    820 
    821 	*rsegs = segs;
    822 
    823 	return 0;
    824 }
    825 
    826 static inline int
    827 cnmac_send_makecmd(struct cnmac_softc *sc, struct mbuf *m,
    828     uint64_t *gbuf, uint64_t *rpko_cmd_w0, uint64_t *rpko_cmd_w1)
    829 {
    830 	uint64_t pko_cmd_w0, pko_cmd_w1;
    831 	int ipoffp1;
    832 	int segs;
    833 	int result = 0;
    834 
    835 	if (cnmac_send_makecmd_gbuf(sc, m, gbuf, &segs)) {
    836 		log(LOG_WARNING, "%s: there are a lot of number of segments"
    837 		    " of transmission data", device_xname(sc->sc_dev));
    838 		result = 1;
    839 		goto done;
    840 	}
    841 
    842 	/*  Get the IP packet offset for TCP/UDP checksum offloading. */
    843 	ipoffp1 = (m->m_pkthdr.csum_flags & (M_CSUM_TCPv4 | M_CSUM_UDPv4))
    844 	    ? (ETHER_HDR_LEN + 1) : 0;
    845 
    846 	/*
    847 	 * segs == 1	-> link mode (single continuous buffer)
    848 	 *		   WORD1[size] is number of bytes pointed by segment
    849 	 *
    850 	 * segs > 1	-> gather mode (scatter-gather buffer)
    851 	 *		   WORD1[size] is number of segments
    852 	 */
    853 	pko_cmd_w0 = cnmac_send_makecmd_w0(sc->sc_fau_done.fd_regno,
    854 	    0, m->m_pkthdr.len, segs, ipoffp1);
    855 	if (segs == 1) {
    856 		pko_cmd_w1 = cnmac_send_makecmd_w1(
    857 		    m->m_pkthdr.len, kvtophys((vaddr_t)m->m_data));
    858 	} else {
    859 #ifdef __mips_n32
    860 		KASSERT(MIPS_KSEG0_P(gbuf));
    861 		pko_cmd_w1 = cnmac_send_makecmd_w1(segs,
    862 		    MIPS_KSEG0_TO_PHYS(gbuf));
    863 #else
    864 		pko_cmd_w1 = cnmac_send_makecmd_w1(segs,
    865 		    MIPS_XKPHYS_TO_PHYS(gbuf));
    866 #endif
    867 	}
    868 
    869 	*rpko_cmd_w0 = pko_cmd_w0;
    870 	*rpko_cmd_w1 = pko_cmd_w1;
    871 
    872 done:
    873 	return result;
    874 }
    875 
    876 static inline int
    877 cnmac_send_cmd(struct cnmac_softc *sc, uint64_t pko_cmd_w0,
    878     uint64_t pko_cmd_w1, int *pwdc)
    879 {
    880 	uint64_t *cmdptr;
    881 	int result = 0;
    882 
    883 #ifdef __mips_n32
    884 	KASSERT((sc->sc_cmdptr.cmdptr & ~MIPS_PHYS_MASK) == 0);
    885 	cmdptr = (uint64_t *)MIPS_PHYS_TO_KSEG0(sc->sc_cmdptr.cmdptr);
    886 #else
    887 	cmdptr = (uint64_t *)MIPS_PHYS_TO_XKPHYS_CACHED(sc->sc_cmdptr.cmdptr);
    888 #endif
    889 	cmdptr += sc->sc_cmdptr.cmdptr_idx;
    890 
    891 	KASSERT(cmdptr != NULL);
    892 
    893 	*cmdptr++ = pko_cmd_w0;
    894 	*cmdptr++ = pko_cmd_w1;
    895 
    896 	KASSERT(sc->sc_cmdptr.cmdptr_idx + 2 <= FPA_COMMAND_BUFFER_POOL_NWORDS - 1);
    897 
    898 	if (sc->sc_cmdptr.cmdptr_idx + 2 == FPA_COMMAND_BUFFER_POOL_NWORDS - 1) {
    899 		paddr_t buf;
    900 
    901 		buf = octfpa_buf_get_paddr(cnmac_fb_cmd);
    902 		if (buf == 0) {
    903 			log(LOG_WARNING,
    904 			    "%s: can not allocate command buffer from free pool allocator\n",
    905 			    device_xname(sc->sc_dev));
    906 			result = 1;
    907 			goto done;
    908 		}
    909 		*cmdptr++ = buf;
    910 		sc->sc_cmdptr.cmdptr = (uint64_t)buf;
    911 		sc->sc_cmdptr.cmdptr_idx = 0;
    912 	} else {
    913 		sc->sc_cmdptr.cmdptr_idx += 2;
    914 	}
    915 
    916 	*pwdc += 2;
    917 
    918 done:
    919 	return result;
    920 }
    921 
    922 static inline int
    923 cnmac_send_buf(struct cnmac_softc *sc, struct mbuf *m, uint64_t *gbuf,
    924     int *pwdc)
    925 {
    926 	int result = 0, error;
    927 	uint64_t pko_cmd_w0, pko_cmd_w1;
    928 
    929 	error = cnmac_send_makecmd(sc, m, gbuf, &pko_cmd_w0, &pko_cmd_w1);
    930 	if (error != 0) {
    931 		/* Already logging */
    932 		result = error;
    933 		goto done;
    934 	}
    935 
    936 	error = cnmac_send_cmd(sc, pko_cmd_w0, pko_cmd_w1, pwdc);
    937 	if (error != 0) {
    938 		/* Already logging */
    939 		result = error;
    940 	}
    941 
    942 done:
    943 	return result;
    944 }
    945 
    946 static inline int
    947 cnmac_send(struct cnmac_softc *sc, struct mbuf *m, int *pwdc)
    948 {
    949 	paddr_t gaddr = 0;
    950 	uint64_t *gbuf = NULL;
    951 	int result = 0, error;
    952 
    953 	gaddr = octfpa_buf_get_paddr(cnmac_fb_sg);
    954 	if (gaddr == 0) {
    955 		log(LOG_WARNING, "%s: can not allocate gather buffer from "
    956 		    "free pool allocator\n", device_xname(sc->sc_dev));
    957 		result = 1;
    958 		goto done;
    959 	}
    960 
    961 #ifdef __mips_n32
    962 	KASSERT((gaddr & ~MIPS_PHYS_MASK) == 0);
    963 	gbuf = (uint64_t *)(uintptr_t)MIPS_PHYS_TO_KSEG0(gaddr);
    964 #else
    965 	gbuf = (uint64_t *)(uintptr_t)MIPS_PHYS_TO_XKPHYS_CACHED(gaddr);
    966 #endif
    967 
    968 	KASSERT(gbuf != NULL);
    969 
    970 	error = cnmac_send_buf(sc, m, gbuf, pwdc);
    971 	if (error != 0) {
    972 		/* Already logging */
    973 		octfpa_buf_put_paddr(cnmac_fb_sg, gaddr);
    974 		result = error;
    975 		goto done;
    976 	}
    977 
    978 	cnmac_send_queue_add(sc, m, gbuf);
    979 
    980 done:
    981 	return result;
    982 }
    983 
    984 static void
    985 cnmac_start(struct ifnet *ifp)
    986 {
    987 	struct cnmac_softc *sc = ifp->if_softc;
    988 	struct mbuf *m;
    989 	int wdc = 0;
    990 
    991 	/*
    992 	 * Performance tuning
    993 	 * pre-send iobdma request
    994 	 */
    995 	cnmac_send_queue_flush_prefetch(sc);
    996 
    997 	if ((ifp->if_flags & IFF_RUNNING) == 0)
    998 		goto last;
    999 
   1000 	if (sc->sc_txbusy)
   1001 		goto last;
   1002 
   1003 	if (__predict_false(!octgmx_link_status(sc->sc_gmx_port)))
   1004 		goto last;
   1005 
   1006 	for (;;) {
   1007 		IFQ_POLL(&ifp->if_snd, m);
   1008 		if (__predict_false(m == NULL))
   1009 			break;
   1010 
   1011 		/* XXX XXX XXX */
   1012 		cnmac_send_queue_flush_fetch(sc);
   1013 
   1014 		/*
   1015 		 * If no free send buffer is available, free all the sent
   1016 		 * buffers and bail out.
   1017 		 */
   1018 		if (cnmac_send_queue_is_full(sc)) {
   1019 			sc->sc_txbusy = true;
   1020 			if (wdc > 0)
   1021 				octpko_op_doorbell_write(sc->sc_port,
   1022 				    sc->sc_port, wdc);
   1023 			callout_schedule(&sc->sc_tick_free_ch, 1);
   1024 			return;
   1025 		}
   1026 		/* XXX XXX XXX */
   1027 
   1028 		IFQ_DEQUEUE(&ifp->if_snd, m);
   1029 
   1030 		bpf_mtap(ifp, m, BPF_D_OUT);
   1031 
   1032 		/* XXX XXX XXX */
   1033 		if (sc->sc_soft_req_cnt > sc->sc_soft_req_thresh)
   1034 			cnmac_send_queue_flush(sc);
   1035 		if (cnmac_send(sc, m, &wdc)) {
   1036 			IF_DROP(&ifp->if_snd);
   1037 			m_freem(m);
   1038 			log(LOG_WARNING,
   1039 			  "%s: failed in the transmission of the packet\n",
   1040 			  device_xname(sc->sc_dev));
   1041 		} else
   1042 			sc->sc_soft_req_cnt++;
   1043 
   1044 		if (sc->sc_flush)
   1045 			cnmac_send_queue_flush_sync(sc);
   1046 		/* XXX XXX XXX */
   1047 
   1048 		/* Send next iobdma request */
   1049 		cnmac_send_queue_flush_prefetch(sc);
   1050 	}
   1051 
   1052 	if (wdc > 0)
   1053 		octpko_op_doorbell_write(sc->sc_port, sc->sc_port, wdc);
   1054 
   1055 last:
   1056 	cnmac_send_queue_flush_fetch(sc);
   1057 	callout_schedule(&sc->sc_tick_free_ch, 1);
   1058 }
   1059 
   1060 static void
   1061 cnmac_watchdog(struct ifnet *ifp)
   1062 {
   1063 	struct cnmac_softc *sc = ifp->if_softc;
   1064 
   1065 	printf("%s: device timeout\n", device_xname(sc->sc_dev));
   1066 
   1067 	cnmac_configure(sc);
   1068 
   1069 	SET(ifp->if_flags, IFF_RUNNING);
   1070 	sc->sc_txbusy = false;
   1071 	ifp->if_timer = 0;
   1072 
   1073 	cnmac_start(ifp);
   1074 }
   1075 
   1076 static int
   1077 cnmac_init(struct ifnet *ifp)
   1078 {
   1079 	struct cnmac_softc *sc = ifp->if_softc;
   1080 
   1081 	/* XXX don't disable commonly used parts!!! XXX */
   1082 	if (sc->sc_init_flag == 0) {
   1083 		/* Cancel any pending I/O. */
   1084 		cnmac_stop(ifp, 0);
   1085 
   1086 		/* Initialize the device */
   1087 		cnmac_configure(sc);
   1088 
   1089 		octpko_enable(sc->sc_pko);
   1090 		octipd_enable(sc->sc_ipd);
   1091 
   1092 		sc->sc_init_flag = 1;
   1093 	} else {
   1094 		octgmx_port_enable(sc->sc_gmx_port, 1);
   1095 	}
   1096 	mii_ifmedia_change(&sc->sc_mii);
   1097 
   1098 	octgmx_set_filter(sc->sc_gmx_port);
   1099 
   1100 	callout_schedule(&sc->sc_tick_misc_ch, hz);
   1101 	callout_schedule(&sc->sc_tick_free_ch, hz);
   1102 
   1103 	SET(ifp->if_flags, IFF_RUNNING);
   1104 	sc->sc_txbusy = false;
   1105 
   1106 	return 0;
   1107 }
   1108 
   1109 static void
   1110 cnmac_stop(struct ifnet *ifp, int disable)
   1111 {
   1112 	struct cnmac_softc *sc = ifp->if_softc;
   1113 
   1114 	callout_stop(&sc->sc_tick_misc_ch);
   1115 	callout_stop(&sc->sc_tick_free_ch);
   1116 
   1117 	mii_down(&sc->sc_mii);
   1118 
   1119 	octgmx_port_enable(sc->sc_gmx_port, 0);
   1120 
   1121 	/* Mark the interface as down and cancel the watchdog timer. */
   1122 	CLR(ifp->if_flags, IFF_RUNNING);
   1123 	sc->sc_txbusy = false;
   1124 	ifp->if_timer = 0;
   1125 }
   1126 
   1127 /* ---- misc */
   1128 
   1129 static int
   1130 cnmac_reset(struct cnmac_softc *sc)
   1131 {
   1132 	octgmx_reset_speed(sc->sc_gmx_port);
   1133 	octgmx_reset_flowctl(sc->sc_gmx_port);
   1134 	octgmx_reset_timing(sc->sc_gmx_port);
   1135 
   1136 	return 0;
   1137 }
   1138 
   1139 static int
   1140 cnmac_configure(struct cnmac_softc *sc)
   1141 {
   1142 	octgmx_port_enable(sc->sc_gmx_port, 0);
   1143 
   1144 	cnmac_reset(sc);
   1145 
   1146 	cnmac_configure_common(sc);
   1147 
   1148 	octpko_port_config(sc->sc_pko);
   1149 	octpko_port_enable(sc->sc_pko, 1);
   1150 	octpow_config(sc->sc_pow, sc->sc_powgroup);
   1151 
   1152 	octgmx_tx_stats_rd_clr(sc->sc_gmx_port, 1);
   1153 	octgmx_rx_stats_rd_clr(sc->sc_gmx_port, 1);
   1154 
   1155 	octgmx_port_enable(sc->sc_gmx_port, 1);
   1156 
   1157 	return 0;
   1158 }
   1159 
   1160 static int
   1161 cnmac_configure_common(struct cnmac_softc *sc)
   1162 {
   1163 	static int once;
   1164 
   1165 	if (once == 1)
   1166 		return 0;
   1167 	once = 1;
   1168 
   1169 	octipd_config(sc->sc_ipd);
   1170 	octpko_config(sc->sc_pko);
   1171 
   1172 	return 0;
   1173 }
   1174 
   1175 /* ---- receive (input) */
   1176 
   1177 static inline int
   1178 cnmac_recv_mbuf(struct cnmac_softc *sc, uint64_t *work, struct mbuf **rm)
   1179 {
   1180 	struct mbuf *m;
   1181 	vaddr_t addr;
   1182 	vaddr_t ext_buf;
   1183 	size_t ext_size;
   1184 	uint64_t word1 = work[1];
   1185 	uint64_t word2 = work[2];
   1186 	uint64_t word3 = work[3];
   1187 
   1188 	MGETHDR(m, M_NOWAIT, MT_DATA);
   1189 	if (m == NULL)
   1190 		return 1;
   1191 
   1192 	octfpa_buf_put(cnmac_fb_wqe, work);
   1193 
   1194 	if (__SHIFTOUT(word2, PIP_WQE_WORD2_IP_BUFS) != 1)
   1195 		panic("%s: expected one buffer, got %" PRId64, __func__,
   1196 		    __SHIFTOUT(word2, PIP_WQE_WORD2_IP_BUFS));
   1197 
   1198 
   1199 #ifdef __mips_n32
   1200 	KASSERT((word3 & ~MIPS_PHYS_MASK) == 0);
   1201 	addr = MIPS_PHYS_TO_KSEG0(word3 & PIP_WQE_WORD3_ADDR);
   1202 #else
   1203 	addr = MIPS_PHYS_TO_XKPHYS_CACHED(word3 & PIP_WQE_WORD3_ADDR);
   1204 #endif
   1205 
   1206 	ext_size = OCTEON_POOL_SIZE_PKT;
   1207 	ext_buf = addr & ~(ext_size - 1);
   1208 	MEXTADD(m, ext_buf, ext_size, 0, cnmac_buf_ext_free, NULL);
   1209 
   1210 	m->m_data = (void *)addr;
   1211 	m->m_len = m->m_pkthdr.len = (word1 & PIP_WQE_WORD1_LEN) >> 48;
   1212 	m_set_rcvif(m, &sc->sc_ethercom.ec_if);
   1213 
   1214 	/* Not readonly buffer */
   1215 	m->m_flags |= M_EXT_RW;
   1216 
   1217 	*rm = m;
   1218 
   1219 	KASSERT(*rm != NULL);
   1220 
   1221 	return 0;
   1222 }
   1223 
   1224 static inline int
   1225 cnmac_recv_check(struct cnmac_softc *sc, uint64_t word2)
   1226 {
   1227 	static struct timeval rxerr_log_interval = { 0, 2500000 };
   1228 	uint64_t opecode;
   1229 
   1230 	if (__predict_true(!ISSET(word2, PIP_WQE_WORD2_NOIP_RE)))
   1231 		return 0;
   1232 
   1233 	opecode = word2 & PIP_WQE_WORD2_NOIP_OPECODE;
   1234 	if ((sc->sc_ethercom.ec_if.if_flags & IFF_DEBUG) &&
   1235 	    ratecheck(&sc->sc_rxerr_log_last, &rxerr_log_interval))
   1236 		log(LOG_DEBUG, "%s: rx error (%"PRId64")\n",
   1237 		    device_xname(sc->sc_dev), opecode);
   1238 
   1239 	/* This error is harmless */
   1240 	if (opecode == PIP_WQE_WORD2_RE_OPCODE_OVRRUN)
   1241 		return 0;
   1242 
   1243 	return 1;
   1244 }
   1245 
   1246 static inline int
   1247 cnmac_recv(struct cnmac_softc *sc, uint64_t *work)
   1248 {
   1249 	struct ifnet *ifp;
   1250 	struct mbuf *m;
   1251 	uint64_t word2;
   1252 
   1253 	KASSERT(sc != NULL);
   1254 	KASSERT(work != NULL);
   1255 
   1256 	word2 = work[2];
   1257 	ifp = &sc->sc_ethercom.ec_if;
   1258 
   1259 	KASSERT(ifp != NULL);
   1260 
   1261 	if (!ISSET(ifp->if_flags, IFF_RUNNING))
   1262 		goto drop;
   1263 
   1264 	if (__predict_false(cnmac_recv_check(sc, word2) != 0)) {
   1265 		if_statinc(ifp, if_ierrors);
   1266 		goto drop;
   1267 	}
   1268 
   1269 	if (__predict_false(cnmac_recv_mbuf(sc, work, &m) != 0)) {
   1270 		if_statinc(ifp, if_ierrors);
   1271 		goto drop;
   1272 	}
   1273 
   1274 	/* work[0] .. work[3] may not be valid any more */
   1275 
   1276 	KASSERT(m != NULL);
   1277 
   1278 	octipd_offload(word2, m->m_data, &m->m_pkthdr.csum_flags);
   1279 
   1280 	if_percpuq_enqueue(ifp->if_percpuq, m);
   1281 
   1282 	return 0;
   1283 
   1284 drop:
   1285 	cnmac_buf_free_work(sc, work);
   1286 	return 1;
   1287 }
   1288 
   1289 static int
   1290 cnmac_intr(void *arg)
   1291 {
   1292 	struct cnmac_softc *sc = arg;
   1293 	uint64_t *work;
   1294 	uint64_t wqmask = __BIT(sc->sc_powgroup);
   1295 	uint32_t coreid = 0;	/* XXX octeon_get_coreid() */
   1296 	uint32_t port;
   1297 
   1298 	_POW_WR8(sc->sc_pow, POW_PP_GRP_MSK_OFFSET(coreid), wqmask);
   1299 
   1300 	octpow_tag_sw_wait();
   1301 	octpow_work_request_async(OCTEON_CVMSEG_OFFSET(csm_pow_intr),
   1302 	    POW_NO_WAIT);
   1303 
   1304 	for (;;) {
   1305 		work = (uint64_t *)octpow_work_response_async(
   1306 		    OCTEON_CVMSEG_OFFSET(csm_pow_intr));
   1307 		if (work == NULL)
   1308 			break;
   1309 
   1310 		octpow_tag_sw_wait();
   1311 		octpow_work_request_async(OCTEON_CVMSEG_OFFSET(csm_pow_intr),
   1312 		    POW_NO_WAIT);
   1313 
   1314 		port = __SHIFTOUT(work[1], PIP_WQE_WORD1_IPRT);
   1315 		if (port != sc->sc_port) {
   1316 			printf("%s: unexpected wqe port %u, should be %u\n",
   1317 			    device_xname(sc->sc_dev), port, sc->sc_port);
   1318 			goto wqe_error;
   1319 		}
   1320 
   1321 		(void)cnmac_recv(sc, work);
   1322 
   1323 		cnmac_send_queue_check_and_flush(sc);
   1324 	}
   1325 
   1326 	_POW_WR8(sc->sc_pow, POW_WQ_INT_OFFSET, wqmask);
   1327 
   1328 	return 1;
   1329 
   1330 wqe_error:
   1331 	printf("word0: 0x%016" PRIx64 "\n", work[0]);
   1332 	printf("word1: 0x%016" PRIx64 "\n", work[1]);
   1333 	printf("word2: 0x%016" PRIx64 "\n", work[2]);
   1334 	printf("word3: 0x%016" PRIx64 "\n", work[3]);
   1335 	panic("wqe_error");
   1336 }
   1337 
   1338 /* ---- tick */
   1339 
   1340 /*
   1341  * cnmac_tick_free
   1342  *
   1343  * => garbage collect send gather buffer / mbuf
   1344  * => called at softclock
   1345  */
   1346 static void
   1347 cnmac_tick_free(void *arg)
   1348 {
   1349 	struct cnmac_softc *sc = arg;
   1350 	int timo;
   1351 
   1352 	cnmac_send_queue_check_and_flush(sc);
   1353 
   1354 	timo = (sc->sc_ext_callback_cnt > 0) ? 1 : hz;
   1355 	callout_schedule(&sc->sc_tick_free_ch, timo);
   1356 }
   1357 
   1358 /*
   1359  * cnmac_tick_misc
   1360  *
   1361  * => collect statistics
   1362  * => check link status
   1363  * => called at softclock
   1364  */
   1365 static void
   1366 cnmac_tick_misc(void *arg)
   1367 {
   1368 	struct cnmac_softc *sc = arg;
   1369 	struct ifnet *ifp;
   1370 	int s;
   1371 
   1372 	s = splnet();
   1373 
   1374 	ifp = &sc->sc_ethercom.ec_if;
   1375 
   1376 	octgmx_stats(sc->sc_gmx_port);
   1377 	octpip_stats(sc->sc_pip, ifp, sc->sc_port);
   1378 	mii_tick(&sc->sc_mii);
   1379 
   1380 	splx(s);
   1381 
   1382 	callout_schedule(&sc->sc_tick_misc_ch, hz);
   1383 }
   1384