bcm53xx_eth.c revision 1.25.2.2
1 /*- 2 * Copyright (c) 2012 The NetBSD Foundation, Inc. 3 * All rights reserved. 4 * 5 * This code is derived from software contributed to The NetBSD Foundation 6 * by Matt Thomas of 3am Software Foundry. 7 * 8 * Redistribution and use in source and binary forms, with or without 9 * modification, are permitted provided that the following conditions 10 * are met: 11 * 1. Redistributions of source code must retain the above copyright 12 * notice, this list of conditions and the following disclaimer. 13 * 2. Redistributions in binary form must reproduce the above copyright 14 * notice, this list of conditions and the following disclaimer in the 15 * documentation and/or other materials provided with the distribution. 16 * 17 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS 18 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED 19 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 20 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS 21 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 22 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 23 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 24 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 25 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 26 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 27 * POSSIBILITY OF SUCH DAMAGE. 28 */ 29 30 #define _ARM32_BUS_DMA_PRIVATE 31 #define GMAC_PRIVATE 32 33 #include "locators.h" 34 #include "opt_broadcom.h" 35 36 #include <sys/cdefs.h> 37 38 __KERNEL_RCSID(1, "$NetBSD: bcm53xx_eth.c,v 1.25.2.2 2014/02/15 16:18:36 matt Exp $"); 39 40 #include <sys/param.h> 41 #include <sys/atomic.h> 42 #include <sys/bus.h> 43 #include <sys/device.h> 44 #include <sys/ioctl.h> 45 #include <sys/intr.h> 46 //#include <sys/kmem.h> 47 #include <sys/malloc.h> 48 #include <sys/mutex.h> 49 #include <sys/socket.h> 50 #include <sys/systm.h> 51 #include <sys/workqueue.h> 52 53 #include <net/if.h> 54 #include <net/if_ether.h> 55 #include <net/if_media.h> 56 57 #include <net/if_dl.h> 58 59 #include <net/bpf.h> 60 61 #include <dev/mii/miivar.h> 62 63 #include <arm/broadcom/bcm53xx_reg.h> 64 #include <arm/broadcom/bcm53xx_var.h> 65 66 //#define BCMETH_MPSAFE 67 68 #ifdef BCMETH_COUNTERS 69 #define BCMETH_EVCNT_ADD(a,b) ((void)((a).ev_count += (b))) 70 #else 71 #define BCMETH_EVCNT_ADD(a,b) do { } while (/*CONSTCOND*/0) 72 #endif 73 #define BCMETH_EVCNT_INCR(a) BCMETH_EVCNT_ADD((a), 1) 74 75 #define BCMETH_MAXTXMBUFS 128 76 #define BCMETH_NTXSEGS 30 77 #define BCMETH_MAXRXMBUFS 255 78 #define BCMETH_MINRXMBUFS 64 79 #define BCMETH_NRXSEGS 1 80 #define BCMETH_RINGSIZE PAGE_SIZE 81 82 #if 1 83 #define BCMETH_RCVMAGIC 0xfeedface 84 #endif 85 86 static int bcmeth_ccb_match(device_t, cfdata_t, void *); 87 static void bcmeth_ccb_attach(device_t, device_t, void *); 88 89 struct bcmeth_txqueue { 90 bus_dmamap_t txq_descmap; 91 struct gmac_txdb *txq_consumer; 92 struct gmac_txdb *txq_producer; 93 struct gmac_txdb *txq_first; 94 struct gmac_txdb *txq_last; 95 struct ifqueue txq_mbufs; 96 struct mbuf *txq_next; 97 size_t txq_free; 98 size_t txq_threshold; 99 size_t txq_lastintr; 100 bus_size_t txq_reg_xmtaddrlo; 101 bus_size_t txq_reg_xmtptr; 102 bus_size_t txq_reg_xmtctl; 103 bus_size_t txq_reg_xmtsts0; 104 bus_size_t txq_reg_xmtsts1; 105 bus_dma_segment_t txq_descmap_seg; 106 }; 107 108 struct bcmeth_rxqueue { 109 bus_dmamap_t rxq_descmap; 110 struct gmac_rxdb *rxq_consumer; 111 struct gmac_rxdb *rxq_producer; 112 struct gmac_rxdb *rxq_first; 113 struct gmac_rxdb *rxq_last; 114 struct mbuf *rxq_mhead; 115 struct mbuf **rxq_mtail; 116 struct mbuf *rxq_mconsumer; 117 size_t rxq_inuse; 118 size_t rxq_threshold; 119 bus_size_t rxq_reg_rcvaddrlo; 120 bus_size_t rxq_reg_rcvptr; 121 bus_size_t rxq_reg_rcvctl; 122 bus_size_t rxq_reg_rcvsts0; 123 bus_size_t rxq_reg_rcvsts1; 124 bus_dma_segment_t rxq_descmap_seg; 125 }; 126 127 struct bcmeth_mapcache { 128 u_int dmc_nmaps; 129 u_int dmc_maxseg; 130 u_int dmc_maxmaps; 131 u_int dmc_maxmapsize; 132 bus_dmamap_t dmc_maps[0]; 133 }; 134 135 struct bcmeth_softc { 136 device_t sc_dev; 137 bus_space_tag_t sc_bst; 138 bus_space_handle_t sc_bsh; 139 bus_dma_tag_t sc_dmat; 140 kmutex_t *sc_lock; 141 kmutex_t *sc_hwlock; 142 struct ethercom sc_ec; 143 #define sc_if sc_ec.ec_if 144 struct ifmedia sc_media; 145 void *sc_soft_ih; 146 void *sc_ih; 147 148 struct bcmeth_rxqueue sc_rxq; 149 struct bcmeth_txqueue sc_txq; 150 151 size_t sc_rcvoffset; 152 uint32_t sc_macaddr[2]; 153 uint32_t sc_maxfrm; 154 uint32_t sc_cmdcfg; 155 uint32_t sc_intmask; 156 uint32_t sc_rcvlazy; 157 volatile uint32_t sc_soft_flags; 158 #define SOFT_RXINTR 0x01 159 #define SOFT_TXINTR 0x02 160 161 #ifdef BCMETH_COUNTERS 162 struct evcnt sc_ev_intr; 163 struct evcnt sc_ev_soft_intr; 164 struct evcnt sc_ev_work; 165 struct evcnt sc_ev_tx_stall; 166 struct evcnt sc_ev_rx_badmagic_lo; 167 struct evcnt sc_ev_rx_badmagic_hi; 168 #endif 169 170 struct ifqueue sc_rx_bufcache; 171 struct bcmeth_mapcache *sc_rx_mapcache; 172 struct bcmeth_mapcache *sc_tx_mapcache; 173 174 struct workqueue *sc_workq; 175 struct work sc_work; 176 177 volatile uint32_t sc_work_flags; 178 #define WORK_RXINTR 0x01 179 #define WORK_RXUNDERFLOW 0x02 180 #define WORK_REINIT 0x04 181 182 uint8_t sc_enaddr[ETHER_ADDR_LEN]; 183 }; 184 185 static void bcmeth_ifstart(struct ifnet *); 186 static void bcmeth_ifwatchdog(struct ifnet *); 187 static int bcmeth_ifinit(struct ifnet *); 188 static void bcmeth_ifstop(struct ifnet *, int); 189 static int bcmeth_ifioctl(struct ifnet *, u_long, void *); 190 191 static int bcmeth_mapcache_create(struct bcmeth_softc *, 192 struct bcmeth_mapcache **, size_t, size_t, size_t); 193 static void bcmeth_mapcache_destroy(struct bcmeth_softc *, 194 struct bcmeth_mapcache *); 195 static bus_dmamap_t bcmeth_mapcache_get(struct bcmeth_softc *, 196 struct bcmeth_mapcache *); 197 static void bcmeth_mapcache_put(struct bcmeth_softc *, 198 struct bcmeth_mapcache *, bus_dmamap_t); 199 200 static int bcmeth_txq_attach(struct bcmeth_softc *, 201 struct bcmeth_txqueue *, u_int); 202 static void bcmeth_txq_purge(struct bcmeth_softc *, 203 struct bcmeth_txqueue *); 204 static void bcmeth_txq_reset(struct bcmeth_softc *, 205 struct bcmeth_txqueue *); 206 static bool bcmeth_txq_consume(struct bcmeth_softc *, 207 struct bcmeth_txqueue *); 208 static bool bcmeth_txq_produce(struct bcmeth_softc *, 209 struct bcmeth_txqueue *, struct mbuf *m); 210 static bool bcmeth_txq_active_p(struct bcmeth_softc *, 211 struct bcmeth_txqueue *); 212 213 static int bcmeth_rxq_attach(struct bcmeth_softc *, 214 struct bcmeth_rxqueue *, u_int); 215 static bool bcmeth_rxq_produce(struct bcmeth_softc *, 216 struct bcmeth_rxqueue *); 217 static void bcmeth_rxq_purge(struct bcmeth_softc *, 218 struct bcmeth_rxqueue *, bool); 219 static void bcmeth_rxq_reset(struct bcmeth_softc *, 220 struct bcmeth_rxqueue *); 221 222 static int bcmeth_intr(void *); 223 #ifdef BCMETH_MPSAFETX 224 static void bcmeth_soft_txintr(struct bcmeth_softc *); 225 #endif 226 static void bcmeth_soft_intr(void *); 227 static void bcmeth_worker(struct work *, void *); 228 229 static int bcmeth_mediachange(struct ifnet *); 230 static void bcmeth_mediastatus(struct ifnet *, struct ifmediareq *); 231 232 static inline uint32_t 233 bcmeth_read_4(struct bcmeth_softc *sc, bus_size_t o) 234 { 235 return bus_space_read_4(sc->sc_bst, sc->sc_bsh, o); 236 } 237 238 static inline void 239 bcmeth_write_4(struct bcmeth_softc *sc, bus_size_t o, uint32_t v) 240 { 241 bus_space_write_4(sc->sc_bst, sc->sc_bsh, o, v); 242 } 243 244 CFATTACH_DECL_NEW(bcmeth_ccb, sizeof(struct bcmeth_softc), 245 bcmeth_ccb_match, bcmeth_ccb_attach, NULL, NULL); 246 247 static int 248 bcmeth_ccb_match(device_t parent, cfdata_t cf, void *aux) 249 { 250 struct bcmccb_attach_args * const ccbaa = aux; 251 const struct bcm_locators * const loc = &ccbaa->ccbaa_loc; 252 253 if (strcmp(cf->cf_name, loc->loc_name)) 254 return 0; 255 256 #ifdef DIAGNOSTIC 257 const int port = cf->cf_loc[BCMCCBCF_PORT]; 258 #endif 259 KASSERT(port == BCMCCBCF_PORT_DEFAULT || port == loc->loc_port); 260 261 return 1; 262 } 263 264 static void 265 bcmeth_ccb_attach(device_t parent, device_t self, void *aux) 266 { 267 struct bcmeth_softc * const sc = device_private(self); 268 struct ethercom * const ec = &sc->sc_ec; 269 struct ifnet * const ifp = &ec->ec_if; 270 struct bcmccb_attach_args * const ccbaa = aux; 271 const struct bcm_locators * const loc = &ccbaa->ccbaa_loc; 272 const char * const xname = device_xname(self); 273 prop_dictionary_t dict = device_properties(self); 274 int error; 275 276 sc->sc_bst = ccbaa->ccbaa_ccb_bst; 277 sc->sc_dmat = ccbaa->ccbaa_dmat; 278 bus_space_subregion(sc->sc_bst, ccbaa->ccbaa_ccb_bsh, 279 loc->loc_offset, loc->loc_size, &sc->sc_bsh); 280 281 /* 282 * We need to use the coherent dma tag for the GMAC. 283 */ 284 sc->sc_dmat = &bcm53xx_coherent_dma_tag; 285 #if _ARM32_NEED_BUS_DMA_BOUNCE 286 if (device_cfdata(self)->cf_flags & 2) { 287 sc->sc_dmat = &bcm53xx_bounce_dma_tag; 288 } 289 #endif 290 291 prop_data_t eaprop = prop_dictionary_get(dict, "mac-address"); 292 if (eaprop == NULL) { 293 uint32_t mac0 = bcmeth_read_4(sc, UNIMAC_MAC_0); 294 uint32_t mac1 = bcmeth_read_4(sc, UNIMAC_MAC_1); 295 if ((mac0 == 0 && mac1 == 0) || (mac1 & 1)) { 296 aprint_error(": mac-address property is missing\n"); 297 return; 298 } 299 sc->sc_enaddr[0] = (mac0 >> 0) & 0xff; 300 sc->sc_enaddr[1] = (mac0 >> 8) & 0xff; 301 sc->sc_enaddr[2] = (mac0 >> 16) & 0xff; 302 sc->sc_enaddr[3] = (mac0 >> 24) & 0xff; 303 sc->sc_enaddr[4] = (mac1 >> 0) & 0xff; 304 sc->sc_enaddr[5] = (mac1 >> 8) & 0xff; 305 } else { 306 KASSERT(prop_object_type(eaprop) == PROP_TYPE_DATA); 307 KASSERT(prop_data_size(eaprop) == ETHER_ADDR_LEN); 308 memcpy(sc->sc_enaddr, prop_data_data_nocopy(eaprop), 309 ETHER_ADDR_LEN); 310 } 311 sc->sc_dev = self; 312 sc->sc_lock = mutex_obj_alloc(MUTEX_DEFAULT, IPL_SOFTNET); 313 sc->sc_hwlock = mutex_obj_alloc(MUTEX_DEFAULT, IPL_VM); 314 315 bcmeth_write_4(sc, GMAC_INTMASK, 0); // disable interrupts 316 317 aprint_naive("\n"); 318 aprint_normal(": Gigabit Ethernet Controller\n"); 319 320 error = bcmeth_rxq_attach(sc, &sc->sc_rxq, 0); 321 if (error) { 322 aprint_error(": failed to init rxq: %d\n", error); 323 return; 324 } 325 326 error = bcmeth_txq_attach(sc, &sc->sc_txq, 0); 327 if (error) { 328 aprint_error(": failed to init txq: %d\n", error); 329 return; 330 } 331 332 error = bcmeth_mapcache_create(sc, &sc->sc_rx_mapcache, 333 BCMETH_MAXRXMBUFS, MCLBYTES, BCMETH_NRXSEGS); 334 if (error) { 335 aprint_error(": failed to allocate rx dmamaps: %d\n", error); 336 return; 337 } 338 339 error = bcmeth_mapcache_create(sc, &sc->sc_tx_mapcache, 340 BCMETH_MAXTXMBUFS, MCLBYTES, BCMETH_NTXSEGS); 341 if (error) { 342 aprint_error(": failed to allocate tx dmamaps: %d\n", error); 343 return; 344 } 345 346 error = workqueue_create(&sc->sc_workq, xname, bcmeth_worker, sc, 347 (PRI_USER + MAXPRI_USER) / 2, IPL_NET, WQ_MPSAFE|WQ_PERCPU); 348 if (error) { 349 aprint_error(": failed to create workqueue: %d\n", error); 350 return; 351 } 352 353 sc->sc_soft_ih = softint_establish(SOFTINT_MPSAFE | SOFTINT_NET, 354 bcmeth_soft_intr, sc); 355 356 sc->sc_ih = intr_establish(loc->loc_intrs[0], IPL_VM, IST_LEVEL, 357 bcmeth_intr, sc); 358 359 if (sc->sc_ih == NULL) { 360 aprint_error_dev(self, "failed to establish interrupt %d\n", 361 loc->loc_intrs[0]); 362 } else { 363 aprint_normal_dev(self, "interrupting on irq %d\n", 364 loc->loc_intrs[0]); 365 } 366 367 aprint_normal_dev(sc->sc_dev, "Ethernet address %s\n", 368 ether_sprintf(sc->sc_enaddr)); 369 370 /* 371 * Since each port in plugged into the switch/flow-accelerator, 372 * we hard code at Gige Full-Duplex with Flow Control enabled. 373 */ 374 int ifmedia = IFM_ETHER|IFM_1000_T|IFM_FDX; 375 //ifmedia |= IFM_FLOW|IFM_ETH_TXPAUSE|IFM_ETH_RXPAUSE; 376 ifmedia_init(&sc->sc_media, IFM_IMASK, bcmeth_mediachange, 377 bcmeth_mediastatus); 378 ifmedia_add(&sc->sc_media, ifmedia, 0, NULL); 379 ifmedia_set(&sc->sc_media, ifmedia); 380 381 ec->ec_capabilities = ETHERCAP_VLAN_MTU | ETHERCAP_JUMBO_MTU; 382 383 strlcpy(ifp->if_xname, xname, IFNAMSIZ); 384 ifp->if_softc = sc; 385 ifp->if_baudrate = IF_Mbps(1000); 386 ifp->if_capabilities = 0; 387 ifp->if_flags = IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST; 388 #ifdef BCMETH_MPSAFE 389 ifp->if_flags2 = IFF2_MPSAFE; 390 #endif 391 ifp->if_ioctl = bcmeth_ifioctl; 392 ifp->if_start = bcmeth_ifstart; 393 ifp->if_watchdog = bcmeth_ifwatchdog; 394 ifp->if_init = bcmeth_ifinit; 395 ifp->if_stop = bcmeth_ifstop; 396 IFQ_SET_READY(&ifp->if_snd); 397 398 bcmeth_ifstop(ifp, true); 399 400 /* 401 * Attach the interface. 402 */ 403 if_attach(ifp); 404 ether_ifattach(ifp, sc->sc_enaddr); 405 406 #ifdef BCMETH_COUNTERS 407 evcnt_attach_dynamic(&sc->sc_ev_intr, EVCNT_TYPE_INTR, 408 NULL, xname, "intr"); 409 evcnt_attach_dynamic(&sc->sc_ev_soft_intr, EVCNT_TYPE_INTR, 410 NULL, xname, "soft intr"); 411 evcnt_attach_dynamic(&sc->sc_ev_work, EVCNT_TYPE_MISC, 412 NULL, xname, "work items"); 413 evcnt_attach_dynamic(&sc->sc_ev_tx_stall, EVCNT_TYPE_MISC, 414 NULL, xname, "tx stalls"); 415 evcnt_attach_dynamic(&sc->sc_ev_rx_badmagic_lo, EVCNT_TYPE_MISC, 416 NULL, xname, "rx badmagic lo"); 417 evcnt_attach_dynamic(&sc->sc_ev_rx_badmagic_hi, EVCNT_TYPE_MISC, 418 NULL, xname, "rx badmagic hi"); 419 #endif 420 } 421 422 static int 423 bcmeth_mediachange(struct ifnet *ifp) 424 { 425 //struct bcmeth_softc * const sc = ifp->if_softc; 426 return 0; 427 } 428 429 static void 430 bcmeth_mediastatus(struct ifnet *ifp, struct ifmediareq *ifm) 431 { 432 //struct bcmeth_softc * const sc = ifp->if_softc; 433 434 ifm->ifm_status = IFM_AVALID | IFM_ACTIVE; 435 ifm->ifm_active = IFM_ETHER | IFM_FDX | IFM_1000_T; 436 } 437 438 static uint64_t 439 bcmeth_macaddr_create(const uint8_t *enaddr) 440 { 441 return (enaddr[3] << 0) // UNIMAC_MAC_0 442 | (enaddr[2] << 8) // UNIMAC_MAC_0 443 | (enaddr[1] << 16) // UNIMAC_MAC_0 444 | ((uint64_t)enaddr[0] << 24) // UNIMAC_MAC_0 445 | ((uint64_t)enaddr[5] << 32) // UNIMAC_MAC_1 446 | ((uint64_t)enaddr[4] << 40); // UNIMAC_MAC_1 447 } 448 449 static int 450 bcmeth_ifinit(struct ifnet *ifp) 451 { 452 struct bcmeth_softc * const sc = ifp->if_softc; 453 int error = 0; 454 455 sc->sc_maxfrm = max(ifp->if_mtu + 32, MCLBYTES); 456 if (ifp->if_mtu > ETHERMTU_JUMBO) 457 return error; 458 459 KASSERT(ifp->if_flags & IFF_UP); 460 461 /* 462 * Stop the interface 463 */ 464 bcmeth_ifstop(ifp, 0); 465 466 /* 467 * Reserve enough space at the front so that we can insert a maxsized 468 * link header and a VLAN tag. Also make sure we have enough room for 469 * the rcvsts field as well. 470 */ 471 KASSERT(ALIGN(max_linkhdr) == max_linkhdr); 472 KASSERTMSG(max_linkhdr > sizeof(struct ether_header), "%u > %zu", 473 max_linkhdr, sizeof(struct ether_header)); 474 sc->sc_rcvoffset = max_linkhdr + 4 - sizeof(struct ether_header); 475 if (sc->sc_rcvoffset <= 4) 476 sc->sc_rcvoffset += 4; 477 KASSERT((sc->sc_rcvoffset & 3) == 2); 478 KASSERT(sc->sc_rcvoffset <= __SHIFTOUT(RCVCTL_RCVOFFSET, RCVCTL_RCVOFFSET)); 479 KASSERT(sc->sc_rcvoffset >= 6); 480 481 /* 482 * If our frame size has changed (or it's our first time through) 483 * destroy the existing transmit mapcache. 484 */ 485 if (sc->sc_tx_mapcache != NULL 486 && sc->sc_maxfrm != sc->sc_tx_mapcache->dmc_maxmapsize) { 487 bcmeth_mapcache_destroy(sc, sc->sc_tx_mapcache); 488 sc->sc_tx_mapcache = NULL; 489 } 490 491 if (sc->sc_tx_mapcache == NULL) { 492 error = bcmeth_mapcache_create(sc, &sc->sc_tx_mapcache, 493 BCMETH_MAXTXMBUFS, sc->sc_maxfrm, BCMETH_NTXSEGS); 494 if (error) 495 return error; 496 } 497 498 sc->sc_cmdcfg = NO_LENGTH_CHECK | PAUSE_IGNORE 499 | __SHIFTIN(ETH_SPEED_1000, ETH_SPEED) 500 | RX_ENA | TX_ENA; 501 502 if (ifp->if_flags & IFF_PROMISC) { 503 sc->sc_cmdcfg |= PROMISC_EN; 504 } else { 505 sc->sc_cmdcfg &= ~PROMISC_EN; 506 } 507 508 const uint8_t * const lladdr = CLLADDR(ifp->if_sadl); 509 const uint64_t macstnaddr = bcmeth_macaddr_create(lladdr); 510 511 /* 512 * We make sure that a received Ethernet packet start on a non-word 513 * boundary so that the packet payload will be on a word boundary. 514 * So to check the destination address we keep around two words to 515 * quickly compare with. 516 */ 517 #if __ARMEL__ 518 sc->sc_macaddr[0] = lladdr[0] | (lladdr[1] << 8); 519 sc->sc_macaddr[1] = lladdr[2] | (lladdr[3] << 8) 520 | (lladdr[4] << 16) | (lladdr[5] << 24); 521 #else 522 sc->sc_macaddr[0] = lladdr[1] | (lladdr[0] << 8); 523 sc->sc_macaddr[1] = lladdr[5] | (lladdr[4] << 8) 524 | (lladdr[1] << 16) | (lladdr[2] << 24); 525 #endif 526 527 sc->sc_intmask = DESCPROTOERR|DATAERR|DESCERR; 528 529 /* 5. Load RCVADDR_LO with new pointer */ 530 bcmeth_rxq_reset(sc, &sc->sc_rxq); 531 532 bcmeth_write_4(sc, sc->sc_rxq.rxq_reg_rcvctl, 533 __SHIFTIN(sc->sc_rcvoffset, RCVCTL_RCVOFFSET) 534 | RCVCTL_PARITY_DIS 535 | RCVCTL_OFLOW_CONTINUE 536 | __SHIFTIN(3, RCVCTL_BURSTLEN)); 537 538 /* 6. Load XMTADDR_LO with new pointer */ 539 bcmeth_txq_reset(sc, &sc->sc_txq); 540 541 bcmeth_write_4(sc, sc->sc_txq.txq_reg_xmtctl, XMTCTL_DMA_ACT_INDEX 542 | XMTCTL_PARITY_DIS 543 | __SHIFTIN(3, XMTCTL_BURSTLEN)); 544 545 /* 7. Setup other UNIMAC registers */ 546 bcmeth_write_4(sc, UNIMAC_FRAME_LEN, sc->sc_maxfrm); 547 bcmeth_write_4(sc, UNIMAC_MAC_0, (uint32_t)(macstnaddr >> 0)); 548 bcmeth_write_4(sc, UNIMAC_MAC_1, (uint32_t)(macstnaddr >> 32)); 549 bcmeth_write_4(sc, UNIMAC_COMMAND_CONFIG, sc->sc_cmdcfg); 550 551 uint32_t devctl = bcmeth_read_4(sc, GMAC_DEVCONTROL); 552 devctl |= RGMII_LINK_STATUS_SEL | NWAY_AUTO_POLL_EN | TXARB_STRICT_MODE; 553 devctl &= ~FLOW_CTRL_MODE; 554 devctl &= ~MIB_RD_RESET_EN; 555 devctl &= ~RXQ_OVERFLOW_CTRL_SEL; 556 devctl &= ~CPU_FLOW_CTRL_ON; 557 bcmeth_write_4(sc, GMAC_DEVCONTROL, devctl); 558 559 /* Setup lazy receive (at most 1ms). */ 560 const struct cpu_softc * const cpu = curcpu()->ci_softc; 561 sc->sc_rcvlazy = __SHIFTIN(4, INTRCVLAZY_FRAMECOUNT) 562 | __SHIFTIN(cpu->cpu_clk.clk_apb / 1000, INTRCVLAZY_TIMEOUT); 563 bcmeth_write_4(sc, GMAC_INTRCVLAZY, sc->sc_rcvlazy); 564 565 /* 11. Enable transmit queues in TQUEUE, and ensure that the transmit scheduling mode is correctly set in TCTRL. */ 566 sc->sc_intmask |= XMTINT_0|XMTUF; 567 bcmeth_write_4(sc, sc->sc_txq.txq_reg_xmtctl, 568 bcmeth_read_4(sc, sc->sc_txq.txq_reg_xmtctl) | XMTCTL_ENABLE); 569 570 571 /* 12. Enable receive queues in RQUEUE, */ 572 sc->sc_intmask |= RCVINT|RCVDESCUF|RCVFIFOOF; 573 bcmeth_write_4(sc, sc->sc_rxq.rxq_reg_rcvctl, 574 bcmeth_read_4(sc, sc->sc_rxq.rxq_reg_rcvctl) | RCVCTL_ENABLE); 575 576 bcmeth_rxq_produce(sc, &sc->sc_rxq); /* fill with rx buffers */ 577 578 #if 0 579 aprint_normal_dev(sc->sc_dev, 580 "devctl=%#x ucmdcfg=%#x xmtctl=%#x rcvctl=%#x\n", 581 devctl, sc->sc_cmdcfg, 582 bcmeth_read_4(sc, sc->sc_txq.txq_reg_xmtctl), 583 bcmeth_read_4(sc, sc->sc_rxq.rxq_reg_rcvctl)); 584 #endif 585 586 sc->sc_soft_flags = 0; 587 588 bcmeth_write_4(sc, GMAC_INTMASK, sc->sc_intmask); 589 590 ifp->if_flags |= IFF_RUNNING; 591 592 return error; 593 } 594 595 static void 596 bcmeth_ifstop(struct ifnet *ifp, int disable) 597 { 598 struct bcmeth_softc * const sc = ifp->if_softc; 599 struct bcmeth_txqueue * const txq = &sc->sc_txq; 600 struct bcmeth_rxqueue * const rxq = &sc->sc_rxq; 601 602 KASSERT(!cpu_intr_p()); 603 604 sc->sc_soft_flags = 0; 605 sc->sc_work_flags = 0; 606 607 /* Disable Rx processing */ 608 bcmeth_write_4(sc, rxq->rxq_reg_rcvctl, 609 bcmeth_read_4(sc, rxq->rxq_reg_rcvctl) & ~RCVCTL_ENABLE); 610 611 /* Disable Tx processing */ 612 bcmeth_write_4(sc, txq->txq_reg_xmtctl, 613 bcmeth_read_4(sc, txq->txq_reg_xmtctl) & ~XMTCTL_ENABLE); 614 615 /* Disable all interrupts */ 616 bcmeth_write_4(sc, GMAC_INTMASK, 0); 617 618 for (;;) { 619 uint32_t tx0 = bcmeth_read_4(sc, txq->txq_reg_xmtsts0); 620 uint32_t rx0 = bcmeth_read_4(sc, rxq->rxq_reg_rcvsts0); 621 if (__SHIFTOUT(tx0, XMTSTATE) == XMTSTATE_DIS 622 && __SHIFTOUT(rx0, RCVSTATE) == RCVSTATE_DIS) 623 break; 624 delay(50); 625 } 626 /* 627 * Now reset the controller. 628 * 629 * 3. Set SW_RESET bit in UNIMAC_COMMAND_CONFIG register 630 * 4. Clear SW_RESET bit in UNIMAC_COMMAND_CONFIG register 631 */ 632 bcmeth_write_4(sc, UNIMAC_COMMAND_CONFIG, SW_RESET); 633 bcmeth_write_4(sc, GMAC_INTSTATUS, ~0); 634 sc->sc_intmask = 0; 635 ifp->if_flags &= ~IFF_RUNNING; 636 637 /* 638 * Let's consume any remaining transmitted packets. And if we are 639 * disabling the interface, purge ourselves of any untransmitted 640 * packets. But don't consume any received packets, just drop them. 641 * If we aren't disabling the interface, save the mbufs in the 642 * receive queue for reuse. 643 */ 644 bcmeth_rxq_purge(sc, &sc->sc_rxq, disable); 645 bcmeth_txq_consume(sc, &sc->sc_txq); 646 if (disable) { 647 bcmeth_txq_purge(sc, &sc->sc_txq); 648 IF_PURGE(&ifp->if_snd); 649 } 650 651 bcmeth_write_4(sc, UNIMAC_COMMAND_CONFIG, 0); 652 } 653 654 static void 655 bcmeth_ifwatchdog(struct ifnet *ifp) 656 { 657 } 658 659 static int 660 bcmeth_ifioctl(struct ifnet *ifp, u_long cmd, void *data) 661 { 662 struct bcmeth_softc *sc = ifp->if_softc; 663 struct ifreq * const ifr = data; 664 const int s = splnet(); 665 int error; 666 667 switch (cmd) { 668 case SIOCSIFMEDIA: 669 case SIOCGIFMEDIA: 670 error = ifmedia_ioctl(ifp, ifr, &sc->sc_media, cmd); 671 break; 672 673 default: 674 error = ether_ioctl(ifp, cmd, data); 675 if (error != ENETRESET) 676 break; 677 678 if (cmd == SIOCADDMULTI || cmd == SIOCDELMULTI) { 679 error = 0; 680 break; 681 } 682 error = bcmeth_ifinit(ifp); 683 break; 684 } 685 686 splx(s); 687 return error; 688 } 689 690 static void 691 bcmeth_rxq_desc_presync( 692 struct bcmeth_softc *sc, 693 struct bcmeth_rxqueue *rxq, 694 struct gmac_rxdb *rxdb, 695 size_t count) 696 { 697 bus_dmamap_sync(sc->sc_dmat, rxq->rxq_descmap, 698 (rxdb - rxq->rxq_first) * sizeof(*rxdb), count * sizeof(*rxdb), 699 BUS_DMASYNC_PREWRITE); 700 } 701 702 static void 703 bcmeth_rxq_desc_postsync( 704 struct bcmeth_softc *sc, 705 struct bcmeth_rxqueue *rxq, 706 struct gmac_rxdb *rxdb, 707 size_t count) 708 { 709 bus_dmamap_sync(sc->sc_dmat, rxq->rxq_descmap, 710 (rxdb - rxq->rxq_first) * sizeof(*rxdb), count * sizeof(*rxdb), 711 BUS_DMASYNC_POSTWRITE); 712 } 713 714 static void 715 bcmeth_txq_desc_presync( 716 struct bcmeth_softc *sc, 717 struct bcmeth_txqueue *txq, 718 struct gmac_txdb *txdb, 719 size_t count) 720 { 721 bus_dmamap_sync(sc->sc_dmat, txq->txq_descmap, 722 (txdb - txq->txq_first) * sizeof(*txdb), count * sizeof(*txdb), 723 BUS_DMASYNC_PREWRITE); 724 } 725 726 static void 727 bcmeth_txq_desc_postsync( 728 struct bcmeth_softc *sc, 729 struct bcmeth_txqueue *txq, 730 struct gmac_txdb *txdb, 731 size_t count) 732 { 733 bus_dmamap_sync(sc->sc_dmat, txq->txq_descmap, 734 (txdb - txq->txq_first) * sizeof(*txdb), count * sizeof(*txdb), 735 BUS_DMASYNC_POSTWRITE); 736 } 737 738 static bus_dmamap_t 739 bcmeth_mapcache_get( 740 struct bcmeth_softc *sc, 741 struct bcmeth_mapcache *dmc) 742 { 743 KASSERT(dmc->dmc_nmaps > 0); 744 KASSERT(dmc->dmc_maps[dmc->dmc_nmaps-1] != NULL); 745 return dmc->dmc_maps[--dmc->dmc_nmaps]; 746 } 747 748 static void 749 bcmeth_mapcache_put( 750 struct bcmeth_softc *sc, 751 struct bcmeth_mapcache *dmc, 752 bus_dmamap_t map) 753 { 754 KASSERT(map != NULL); 755 KASSERT(dmc->dmc_nmaps < dmc->dmc_maxmaps); 756 dmc->dmc_maps[dmc->dmc_nmaps++] = map; 757 } 758 759 static void 760 bcmeth_mapcache_destroy( 761 struct bcmeth_softc *sc, 762 struct bcmeth_mapcache *dmc) 763 { 764 for (u_int i = 0; i < dmc->dmc_maxmaps; i++) { 765 bus_dmamap_destroy(sc->sc_dmat, dmc->dmc_maps[i]); 766 } 767 //const size_t dmc_size = 768 // offsetof(struct bcmeth_mapcache, dmc_maps[dmc->dmc_maxmaps]); 769 //kmem_intr_free(dmc, dmc_size); 770 free(dmc, M_DEVBUF); 771 } 772 773 static int 774 bcmeth_mapcache_create( 775 struct bcmeth_softc *sc, 776 struct bcmeth_mapcache **dmc_p, 777 size_t maxmaps, 778 size_t maxmapsize, 779 size_t maxseg) 780 { 781 const size_t dmc_size = 782 offsetof(struct bcmeth_mapcache, dmc_maps[maxmaps]); 783 struct bcmeth_mapcache * const dmc = 784 malloc(dmc_size, M_DEVBUF, M_NOWAIT|M_ZERO); 785 //kmem_intr_zalloc(dmc_size, KM_NOSLEEP); 786 787 dmc->dmc_maxmaps = maxmaps; 788 dmc->dmc_nmaps = maxmaps; 789 dmc->dmc_maxmapsize = maxmapsize; 790 dmc->dmc_maxseg = maxseg; 791 792 for (u_int i = 0; i < maxmaps; i++) { 793 int error = bus_dmamap_create(sc->sc_dmat, dmc->dmc_maxmapsize, 794 dmc->dmc_maxseg, dmc->dmc_maxmapsize, 0, 795 BUS_DMA_WAITOK|BUS_DMA_ALLOCNOW, &dmc->dmc_maps[i]); 796 if (error) { 797 aprint_error_dev(sc->sc_dev, 798 "failed to creat dma map cache " 799 "entry %u of %zu: %d\n", 800 i, maxmaps, error); 801 while (i-- > 0) { 802 bus_dmamap_destroy(sc->sc_dmat, 803 dmc->dmc_maps[i]); 804 } 805 free(dmc, M_DEVBUF); 806 //kmem_intr_free(dmc, dmc_size); 807 return error; 808 } 809 KASSERT(dmc->dmc_maps[i] != NULL); 810 } 811 812 *dmc_p = dmc; 813 814 return 0; 815 } 816 817 #if 0 818 static void 819 bcmeth_dmamem_free( 820 bus_dma_tag_t dmat, 821 size_t map_size, 822 bus_dma_segment_t *seg, 823 bus_dmamap_t map, 824 void *kvap) 825 { 826 bus_dmamap_destroy(dmat, map); 827 bus_dmamem_unmap(dmat, kvap, map_size); 828 bus_dmamem_free(dmat, seg, 1); 829 } 830 #endif 831 832 static int 833 bcmeth_dmamem_alloc( 834 bus_dma_tag_t dmat, 835 size_t map_size, 836 bus_dma_segment_t *seg, 837 bus_dmamap_t *map, 838 void **kvap) 839 { 840 int error; 841 int nseg; 842 843 *kvap = NULL; 844 *map = NULL; 845 846 error = bus_dmamem_alloc(dmat, map_size, 2*PAGE_SIZE, 0, 847 seg, 1, &nseg, 0); 848 if (error) 849 return error; 850 851 KASSERT(nseg == 1); 852 853 error = bus_dmamem_map(dmat, seg, nseg, map_size, (void **)kvap, 0); 854 if (error == 0) { 855 error = bus_dmamap_create(dmat, map_size, 1, map_size, 0, 0, 856 map); 857 if (error == 0) { 858 error = bus_dmamap_load(dmat, *map, *kvap, map_size, 859 NULL, 0); 860 if (error == 0) 861 return 0; 862 bus_dmamap_destroy(dmat, *map); 863 *map = NULL; 864 } 865 bus_dmamem_unmap(dmat, *kvap, map_size); 866 *kvap = NULL; 867 } 868 bus_dmamem_free(dmat, seg, nseg); 869 return 0; 870 } 871 872 static struct mbuf * 873 bcmeth_rx_buf_alloc( 874 struct bcmeth_softc *sc) 875 { 876 struct mbuf *m = m_gethdr(M_DONTWAIT, MT_DATA); 877 if (m == NULL) { 878 printf("%s:%d: %s\n", __func__, __LINE__, "m_gethdr"); 879 return NULL; 880 } 881 MCLGET(m, M_DONTWAIT); 882 if ((m->m_flags & M_EXT) == 0) { 883 printf("%s:%d: %s\n", __func__, __LINE__, "MCLGET"); 884 m_freem(m); 885 return NULL; 886 } 887 m->m_len = m->m_pkthdr.len = m->m_ext.ext_size; 888 889 bus_dmamap_t map = bcmeth_mapcache_get(sc, sc->sc_rx_mapcache); 890 if (map == NULL) { 891 printf("%s:%d: %s\n", __func__, __LINE__, "map get"); 892 m_freem(m); 893 return NULL; 894 } 895 M_SETCTX(m, map); 896 m->m_len = m->m_pkthdr.len = MCLBYTES; 897 int error = bus_dmamap_load_mbuf(sc->sc_dmat, map, m, 898 BUS_DMA_READ|BUS_DMA_NOWAIT); 899 if (error) { 900 aprint_error_dev(sc->sc_dev, "fail to load rx dmamap: %d\n", 901 error); 902 M_SETCTX(m, NULL); 903 m_freem(m); 904 bcmeth_mapcache_put(sc, sc->sc_rx_mapcache, map); 905 return NULL; 906 } 907 KASSERT(map->dm_mapsize == MCLBYTES); 908 #ifdef BCMETH_RCVMAGIC 909 *mtod(m, uint32_t *) = htole32(BCMETH_RCVMAGIC); 910 bus_dmamap_sync(sc->sc_dmat, map, 0, sizeof(uint32_t), 911 BUS_DMASYNC_PREREAD|BUS_DMASYNC_PREWRITE); 912 bus_dmamap_sync(sc->sc_dmat, map, sizeof(uint32_t), 913 map->dm_mapsize - sizeof(uint32_t), BUS_DMASYNC_PREREAD); 914 #else 915 bus_dmamap_sync(sc->sc_dmat, map, 0, map->dm_mapsize, 916 BUS_DMASYNC_PREREAD); 917 #endif 918 919 return m; 920 } 921 922 static void 923 bcmeth_rx_map_unload( 924 struct bcmeth_softc *sc, 925 struct mbuf *m) 926 { 927 KASSERT(m); 928 for (; m != NULL; m = m->m_next) { 929 bus_dmamap_t map = M_GETCTX(m, bus_dmamap_t); 930 KASSERT(map); 931 KASSERT(map->dm_mapsize == MCLBYTES); 932 bus_dmamap_sync(sc->sc_dmat, map, 0, m->m_len, 933 BUS_DMASYNC_POSTREAD); 934 bus_dmamap_unload(sc->sc_dmat, map); 935 bcmeth_mapcache_put(sc, sc->sc_rx_mapcache, map); 936 M_SETCTX(m, NULL); 937 } 938 } 939 940 static bool 941 bcmeth_rxq_produce( 942 struct bcmeth_softc *sc, 943 struct bcmeth_rxqueue *rxq) 944 { 945 struct gmac_rxdb *producer = rxq->rxq_producer; 946 bool produced = false; 947 948 while (rxq->rxq_inuse < rxq->rxq_threshold) { 949 struct mbuf *m; 950 IF_DEQUEUE(&sc->sc_rx_bufcache, m); 951 if (m == NULL) { 952 m = bcmeth_rx_buf_alloc(sc); 953 if (m == NULL) { 954 printf("%s: bcmeth_rx_buf_alloc failed\n", __func__); 955 break; 956 } 957 } 958 bus_dmamap_t map = M_GETCTX(m, bus_dmamap_t); 959 KASSERT(map); 960 961 producer->rxdb_buflen = htole32(MCLBYTES); 962 producer->rxdb_addrlo = htole32(map->dm_segs[0].ds_addr); 963 producer->rxdb_flags &= htole32(RXDB_FLAG_ET); 964 *rxq->rxq_mtail = m; 965 rxq->rxq_mtail = &m->m_next; 966 m->m_len = MCLBYTES; 967 m->m_next = NULL; 968 rxq->rxq_inuse++; 969 if (++producer == rxq->rxq_last) { 970 membar_producer(); 971 bcmeth_rxq_desc_presync(sc, rxq, rxq->rxq_producer, 972 rxq->rxq_last - rxq->rxq_producer); 973 producer = rxq->rxq_producer = rxq->rxq_first; 974 } 975 produced = true; 976 } 977 if (produced) { 978 membar_producer(); 979 if (producer != rxq->rxq_producer) { 980 bcmeth_rxq_desc_presync(sc, rxq, rxq->rxq_producer, 981 producer - rxq->rxq_producer); 982 rxq->rxq_producer = producer; 983 } 984 bcmeth_write_4(sc, rxq->rxq_reg_rcvptr, 985 rxq->rxq_descmap->dm_segs[0].ds_addr 986 + ((uintptr_t)producer & RCVPTR)); 987 } 988 return true; 989 } 990 991 static void 992 bcmeth_rx_input( 993 struct bcmeth_softc *sc, 994 struct mbuf *m, 995 uint32_t rxdb_flags) 996 { 997 struct ifnet * const ifp = &sc->sc_if; 998 999 bcmeth_rx_map_unload(sc, m); 1000 1001 m_adj(m, sc->sc_rcvoffset); 1002 1003 /* 1004 * If we are in promiscuous mode and this isn't a multicast, check the 1005 * destination address to make sure it matches our own. If it doesn't, 1006 * mark the packet as being received promiscuously. 1007 */ 1008 if ((sc->sc_cmdcfg & PROMISC_EN) 1009 && (m->m_data[0] & 1) == 0 1010 && (*(uint16_t *)&m->m_data[0] != sc->sc_macaddr[0] 1011 || *(uint32_t *)&m->m_data[2] != sc->sc_macaddr[1])) { 1012 m->m_flags |= M_PROMISC; 1013 } 1014 m->m_pkthdr.rcvif = ifp; 1015 1016 ifp->if_ipackets++; 1017 ifp->if_ibytes += m->m_pkthdr.len; 1018 1019 /* 1020 * Let's give it to the network subsystm to deal with. 1021 */ 1022 #ifdef BCMETH_MPSAFE 1023 mutex_exit(sc->sc_lock); 1024 (*ifp->if_input)(ifp, m); 1025 mutex_enter(sc->sc_lock); 1026 #else 1027 int s = splnet(); 1028 bpf_mtap(ifp, m); 1029 (*ifp->if_input)(ifp, m); 1030 splx(s); 1031 #endif 1032 } 1033 1034 static bool 1035 bcmeth_rxq_consume( 1036 struct bcmeth_softc *sc, 1037 struct bcmeth_rxqueue *rxq, 1038 size_t atmost) 1039 { 1040 struct ifnet * const ifp = &sc->sc_if; 1041 struct gmac_rxdb *consumer = rxq->rxq_consumer; 1042 size_t rxconsumed = 0; 1043 bool didconsume = false; 1044 1045 while (atmost-- > 0) { 1046 if (consumer == rxq->rxq_producer) { 1047 KASSERT(rxq->rxq_inuse == 0); 1048 break; 1049 } 1050 1051 uint32_t rcvsts0 = bcmeth_read_4(sc, rxq->rxq_reg_rcvsts0); 1052 uint32_t currdscr = __SHIFTOUT(rcvsts0, RCV_CURRDSCR); 1053 if (consumer == rxq->rxq_first + currdscr) { 1054 break; 1055 } 1056 bcmeth_rxq_desc_postsync(sc, rxq, consumer, 1); 1057 1058 /* 1059 * We own this packet again. Copy the rxsts word from it. 1060 */ 1061 rxconsumed++; 1062 didconsume = true; 1063 uint32_t rxsts; 1064 KASSERT(rxq->rxq_mhead != NULL); 1065 bus_dmamap_t map = M_GETCTX(rxq->rxq_mhead, bus_dmamap_t); 1066 bus_dmamap_sync(sc->sc_dmat, map, 0, arm_dcache_align, 1067 BUS_DMASYNC_POSTREAD); 1068 memcpy(&rxsts, rxq->rxq_mhead->m_data, 4); 1069 rxsts = le32toh(rxsts); 1070 #if 0 1071 KASSERTMSG(rxsts != BCMETH_RCVMAGIC, "currdscr=%u consumer=%zd", 1072 currdscr, consumer - rxq->rxq_first); 1073 #endif 1074 1075 /* 1076 * Get the count of descriptors. Fetch the correct number 1077 * of mbufs. 1078 */ 1079 #ifdef BCMETH_RCVMAGIC 1080 size_t desc_count = rxsts != BCMETH_RCVMAGIC ? __SHIFTOUT(rxsts, RXSTS_DESC_COUNT) + 1 : 1; 1081 #else 1082 size_t desc_count = __SHIFTOUT(rxsts, RXSTS_DESC_COUNT) + 1; 1083 #endif 1084 struct mbuf *m = rxq->rxq_mhead; 1085 struct mbuf *m_last = m; 1086 for (size_t i = 1; i < desc_count; i++) { 1087 if (++consumer == rxq->rxq_last) { 1088 consumer = rxq->rxq_first; 1089 } 1090 KASSERTMSG(consumer != rxq->rxq_first + currdscr, 1091 "i=%zu rxsts=%#x desc_count=%zu currdscr=%u consumer=%zd", 1092 i, rxsts, desc_count, currdscr, 1093 consumer - rxq->rxq_first); 1094 m_last = m_last->m_next; 1095 } 1096 1097 /* 1098 * Now remove it/them from the list of enqueued mbufs. 1099 */ 1100 if ((rxq->rxq_mhead = m_last->m_next) == NULL) 1101 rxq->rxq_mtail = &rxq->rxq_mhead; 1102 m_last->m_next = NULL; 1103 1104 #ifdef BCMETH_RCVMAGIC 1105 if (rxsts == BCMETH_RCVMAGIC) { 1106 ifp->if_ierrors++; 1107 if ((m->m_ext.ext_paddr >> 28) == 8) { 1108 BCMETH_EVCNT_INCR(sc->sc_ev_rx_badmagic_lo); 1109 } else { 1110 BCMETH_EVCNT_INCR( sc->sc_ev_rx_badmagic_hi); 1111 } 1112 IF_ENQUEUE(&sc->sc_rx_bufcache, m); 1113 } else 1114 #endif /* BCMETH_RCVMAGIC */ 1115 if (rxsts & (RXSTS_CRC_ERROR|RXSTS_OVERSIZED|RXSTS_PKT_OVERFLOW)) { 1116 aprint_error_dev(sc->sc_dev, "[%zu]: count=%zu rxsts=%#x\n", 1117 consumer - rxq->rxq_first, desc_count, rxsts); 1118 /* 1119 * We encountered an error, take the mbufs and add them 1120 * to the rx bufcache so we can quickly reuse them. 1121 */ 1122 ifp->if_ierrors++; 1123 do { 1124 struct mbuf *m0 = m->m_next; 1125 m->m_next = NULL; 1126 IF_ENQUEUE(&sc->sc_rx_bufcache, m); 1127 m = m0; 1128 } while (m); 1129 } else { 1130 uint32_t framelen = __SHIFTOUT(rxsts, RXSTS_FRAMELEN); 1131 framelen += sc->sc_rcvoffset; 1132 m->m_pkthdr.len = framelen; 1133 if (desc_count == 1) { 1134 KASSERT(framelen <= MCLBYTES); 1135 m->m_len = framelen; 1136 } else { 1137 m_last->m_len = framelen & (MCLBYTES - 1); 1138 } 1139 1140 #ifdef BCMETH_MPSAFE 1141 /* 1142 * Wrap at the last entry! 1143 */ 1144 if (++consumer == rxq->rxq_last) { 1145 KASSERT(consumer[-1].rxdb_flags & htole32(RXDB_FLAG_ET)); 1146 rxq->rxq_consumer = rxq->rxq_first; 1147 } else { 1148 rxq->rxq_consumer = consumer; 1149 } 1150 rxq->rxq_inuse -= rxconsumed; 1151 #endif /* BCMETH_MPSAFE */ 1152 1153 /* 1154 * Receive the packet (which releases our lock) 1155 */ 1156 bcmeth_rx_input(sc, m, rxsts); 1157 1158 #ifdef BCMETH_MPSAFE 1159 /* 1160 * Since we had to give up our lock, we need to 1161 * refresh these. 1162 */ 1163 consumer = rxq->rxq_consumer; 1164 rxconsumed = 0; 1165 continue; 1166 #endif /* BCMETH_MPSAFE */ 1167 } 1168 1169 /* 1170 * Wrap at the last entry! 1171 */ 1172 if (++consumer == rxq->rxq_last) { 1173 KASSERT(consumer[-1].rxdb_flags & htole32(RXDB_FLAG_ET)); 1174 consumer = rxq->rxq_first; 1175 } 1176 } 1177 1178 /* 1179 * Update queue info. 1180 */ 1181 rxq->rxq_consumer = consumer; 1182 rxq->rxq_inuse -= rxconsumed; 1183 1184 /* 1185 * Did we consume anything? 1186 */ 1187 return didconsume; 1188 } 1189 1190 static void 1191 bcmeth_rxq_purge( 1192 struct bcmeth_softc *sc, 1193 struct bcmeth_rxqueue *rxq, 1194 bool discard) 1195 { 1196 struct mbuf *m; 1197 1198 if ((m = rxq->rxq_mhead) != NULL) { 1199 if (discard) { 1200 bcmeth_rx_map_unload(sc, m); 1201 m_freem(m); 1202 } else { 1203 while (m != NULL) { 1204 struct mbuf *m0 = m->m_next; 1205 m->m_next = NULL; 1206 IF_ENQUEUE(&sc->sc_rx_bufcache, m); 1207 m = m0; 1208 } 1209 } 1210 1211 } 1212 1213 rxq->rxq_mhead = NULL; 1214 rxq->rxq_mtail = &rxq->rxq_mhead; 1215 rxq->rxq_inuse = 0; 1216 } 1217 1218 static void 1219 bcmeth_rxq_reset( 1220 struct bcmeth_softc *sc, 1221 struct bcmeth_rxqueue *rxq) 1222 { 1223 /* 1224 * sync all the descriptors 1225 */ 1226 bcmeth_rxq_desc_postsync(sc, rxq, rxq->rxq_first, 1227 rxq->rxq_last - rxq->rxq_first); 1228 1229 /* 1230 * Make sure we own all descriptors in the ring. 1231 */ 1232 struct gmac_rxdb *rxdb; 1233 for (rxdb = rxq->rxq_first; rxdb < rxq->rxq_last - 1; rxdb++) { 1234 rxdb->rxdb_flags = htole32(RXDB_FLAG_IC); 1235 } 1236 1237 /* 1238 * Last descriptor has the wrap flag. 1239 */ 1240 rxdb->rxdb_flags = htole32(RXDB_FLAG_ET|RXDB_FLAG_IC); 1241 1242 /* 1243 * Reset the producer consumer indexes. 1244 */ 1245 rxq->rxq_consumer = rxq->rxq_first; 1246 rxq->rxq_producer = rxq->rxq_first; 1247 rxq->rxq_inuse = 0; 1248 if (rxq->rxq_threshold < BCMETH_MINRXMBUFS) 1249 rxq->rxq_threshold = BCMETH_MINRXMBUFS; 1250 1251 sc->sc_intmask |= RCVINT|RCVFIFOOF|RCVDESCUF; 1252 1253 /* 1254 * Restart the receiver at the first descriptor 1255 */ 1256 bcmeth_write_4(sc, rxq->rxq_reg_rcvaddrlo, 1257 rxq->rxq_descmap->dm_segs[0].ds_addr); 1258 } 1259 1260 static int 1261 bcmeth_rxq_attach( 1262 struct bcmeth_softc *sc, 1263 struct bcmeth_rxqueue *rxq, 1264 u_int qno) 1265 { 1266 size_t desc_count = BCMETH_RINGSIZE / sizeof(rxq->rxq_first[0]); 1267 int error; 1268 void *descs; 1269 1270 KASSERT(desc_count == 256 || desc_count == 512); 1271 1272 error = bcmeth_dmamem_alloc(sc->sc_dmat, BCMETH_RINGSIZE, 1273 &rxq->rxq_descmap_seg, &rxq->rxq_descmap, &descs); 1274 if (error) 1275 return error; 1276 1277 memset(descs, 0, BCMETH_RINGSIZE); 1278 rxq->rxq_first = descs; 1279 rxq->rxq_last = rxq->rxq_first + desc_count; 1280 rxq->rxq_consumer = descs; 1281 rxq->rxq_producer = descs; 1282 1283 bcmeth_rxq_purge(sc, rxq, true); 1284 bcmeth_rxq_reset(sc, rxq); 1285 1286 rxq->rxq_reg_rcvaddrlo = GMAC_RCVADDR_LOW; 1287 rxq->rxq_reg_rcvctl = GMAC_RCVCONTROL; 1288 rxq->rxq_reg_rcvptr = GMAC_RCVPTR; 1289 rxq->rxq_reg_rcvsts0 = GMAC_RCVSTATUS0; 1290 rxq->rxq_reg_rcvsts1 = GMAC_RCVSTATUS1; 1291 1292 return 0; 1293 } 1294 1295 static bool 1296 bcmeth_txq_active_p( 1297 struct bcmeth_softc * const sc, 1298 struct bcmeth_txqueue *txq) 1299 { 1300 return !IF_IS_EMPTY(&txq->txq_mbufs); 1301 } 1302 1303 static bool 1304 bcmeth_txq_fillable_p( 1305 struct bcmeth_softc * const sc, 1306 struct bcmeth_txqueue *txq) 1307 { 1308 return txq->txq_free >= txq->txq_threshold; 1309 } 1310 1311 static int 1312 bcmeth_txq_attach( 1313 struct bcmeth_softc *sc, 1314 struct bcmeth_txqueue *txq, 1315 u_int qno) 1316 { 1317 size_t desc_count = BCMETH_RINGSIZE / sizeof(txq->txq_first[0]); 1318 int error; 1319 void *descs; 1320 1321 KASSERT(desc_count == 256 || desc_count == 512); 1322 1323 error = bcmeth_dmamem_alloc(sc->sc_dmat, BCMETH_RINGSIZE, 1324 &txq->txq_descmap_seg, &txq->txq_descmap, &descs); 1325 if (error) 1326 return error; 1327 1328 memset(descs, 0, BCMETH_RINGSIZE); 1329 txq->txq_first = descs; 1330 txq->txq_last = txq->txq_first + desc_count; 1331 txq->txq_consumer = descs; 1332 txq->txq_producer = descs; 1333 1334 IFQ_SET_MAXLEN(&txq->txq_mbufs, BCMETH_MAXTXMBUFS); 1335 1336 txq->txq_reg_xmtaddrlo = GMAC_XMTADDR_LOW; 1337 txq->txq_reg_xmtctl = GMAC_XMTCONTROL; 1338 txq->txq_reg_xmtptr = GMAC_XMTPTR; 1339 txq->txq_reg_xmtsts0 = GMAC_XMTSTATUS0; 1340 txq->txq_reg_xmtsts1 = GMAC_XMTSTATUS1; 1341 1342 bcmeth_txq_reset(sc, txq); 1343 1344 return 0; 1345 } 1346 1347 static int 1348 bcmeth_txq_map_load( 1349 struct bcmeth_softc *sc, 1350 struct bcmeth_txqueue *txq, 1351 struct mbuf *m) 1352 { 1353 bus_dmamap_t map; 1354 int error; 1355 1356 map = M_GETCTX(m, bus_dmamap_t); 1357 if (map != NULL) 1358 return 0; 1359 1360 map = bcmeth_mapcache_get(sc, sc->sc_tx_mapcache); 1361 if (map == NULL) 1362 return ENOMEM; 1363 1364 error = bus_dmamap_load_mbuf(sc->sc_dmat, map, m, 1365 BUS_DMA_WRITE | BUS_DMA_NOWAIT); 1366 if (error) 1367 return error; 1368 1369 bus_dmamap_sync(sc->sc_dmat, map, 0, m->m_pkthdr.len, 1370 BUS_DMASYNC_PREWRITE); 1371 M_SETCTX(m, map); 1372 return 0; 1373 } 1374 1375 static void 1376 bcmeth_txq_map_unload( 1377 struct bcmeth_softc *sc, 1378 struct bcmeth_txqueue *txq, 1379 struct mbuf *m) 1380 { 1381 KASSERT(m); 1382 bus_dmamap_t map = M_GETCTX(m, bus_dmamap_t); 1383 bus_dmamap_sync(sc->sc_dmat, map, 0, map->dm_mapsize, 1384 BUS_DMASYNC_POSTWRITE); 1385 bus_dmamap_unload(sc->sc_dmat, map); 1386 bcmeth_mapcache_put(sc, sc->sc_tx_mapcache, map); 1387 } 1388 1389 static bool 1390 bcmeth_txq_produce( 1391 struct bcmeth_softc *sc, 1392 struct bcmeth_txqueue *txq, 1393 struct mbuf *m) 1394 { 1395 bus_dmamap_t map = M_GETCTX(m, bus_dmamap_t); 1396 1397 if (map->dm_nsegs > txq->txq_free) 1398 return false; 1399 1400 /* 1401 * TCP Offload flag must be set in the first descriptor. 1402 */ 1403 struct gmac_txdb *producer = txq->txq_producer; 1404 uint32_t first_flags = TXDB_FLAG_SF; 1405 uint32_t last_flags = TXDB_FLAG_EF; 1406 1407 /* 1408 * If we've produced enough descriptors without consuming any 1409 * we need to ask for an interrupt to reclaim some. 1410 */ 1411 txq->txq_lastintr += map->dm_nsegs; 1412 if (txq->txq_lastintr >= txq->txq_threshold 1413 || txq->txq_mbufs.ifq_len + 1 == txq->txq_mbufs.ifq_maxlen) { 1414 txq->txq_lastintr = 0; 1415 last_flags |= TXDB_FLAG_IC; 1416 } 1417 1418 KASSERT(producer != txq->txq_last); 1419 1420 struct gmac_txdb *start = producer; 1421 size_t count = map->dm_nsegs; 1422 producer->txdb_flags |= htole32(first_flags); 1423 producer->txdb_addrlo = htole32(map->dm_segs[0].ds_addr); 1424 producer->txdb_buflen = htole32(map->dm_segs[0].ds_len); 1425 for (u_int i = 1; i < map->dm_nsegs; i++) { 1426 #if 0 1427 printf("[%zu]: %#x/%#x/%#x/%#x\n", producer - txq->txq_first, 1428 le32toh(producer->txdb_flags), 1429 le32toh(producer->txdb_buflen), 1430 le32toh(producer->txdb_addrlo), 1431 le32toh(producer->txdb_addrhi)); 1432 #endif 1433 if (__predict_false(++producer == txq->txq_last)) { 1434 bcmeth_txq_desc_presync(sc, txq, start, 1435 txq->txq_last - start); 1436 count -= txq->txq_last - start; 1437 producer = txq->txq_first; 1438 start = txq->txq_first; 1439 } 1440 producer->txdb_addrlo = htole32(map->dm_segs[i].ds_addr); 1441 producer->txdb_buflen = htole32(map->dm_segs[i].ds_len); 1442 } 1443 producer->txdb_flags |= htole32(last_flags); 1444 #if 0 1445 printf("[%zu]: %#x/%#x/%#x/%#x\n", producer - txq->txq_first, 1446 le32toh(producer->txdb_flags), le32toh(producer->txdb_buflen), 1447 le32toh(producer->txdb_addrlo), le32toh(producer->txdb_addrhi)); 1448 #endif 1449 if (count) 1450 bcmeth_txq_desc_presync(sc, txq, start, count); 1451 1452 /* 1453 * Reduce free count by the number of segments we consumed. 1454 */ 1455 txq->txq_free -= map->dm_nsegs; 1456 KASSERT(map->dm_nsegs == 1 || txq->txq_producer != producer); 1457 KASSERT(map->dm_nsegs == 1 || (txq->txq_producer->txdb_flags & htole32(TXDB_FLAG_EF)) == 0); 1458 KASSERT(producer->txdb_flags & htole32(TXDB_FLAG_EF)); 1459 1460 #if 0 1461 printf("%s: mbuf %p: produced a %u byte packet in %u segments (%zd..%zd)\n", 1462 __func__, m, m->m_pkthdr.len, map->dm_nsegs, 1463 txq->txq_producer - txq->txq_first, producer - txq->txq_first); 1464 #endif 1465 1466 if (producer + 1 == txq->txq_last) 1467 txq->txq_producer = txq->txq_first; 1468 else 1469 txq->txq_producer = producer + 1; 1470 IF_ENQUEUE(&txq->txq_mbufs, m); 1471 1472 /* 1473 * Let the transmitter know there's more to do 1474 */ 1475 bcmeth_write_4(sc, txq->txq_reg_xmtptr, 1476 txq->txq_descmap->dm_segs[0].ds_addr 1477 + ((uintptr_t)txq->txq_producer & XMT_LASTDSCR)); 1478 1479 return true; 1480 } 1481 1482 static struct mbuf * 1483 bcmeth_copy_packet(struct mbuf *m) 1484 { 1485 struct mbuf *mext = NULL; 1486 size_t misalignment = 0; 1487 size_t hlen = 0; 1488 1489 for (mext = m; mext != NULL; mext = mext->m_next) { 1490 if (mext->m_flags & M_EXT) { 1491 misalignment = mtod(mext, vaddr_t) & arm_dcache_align; 1492 break; 1493 } 1494 hlen += m->m_len; 1495 } 1496 1497 struct mbuf *n = m->m_next; 1498 if (m != mext && hlen + misalignment <= MHLEN && false) { 1499 KASSERT(m->m_pktdat <= m->m_data && m->m_data <= &m->m_pktdat[MHLEN - m->m_len]); 1500 size_t oldoff = m->m_data - m->m_pktdat; 1501 size_t off; 1502 if (mext == NULL) { 1503 off = (oldoff + hlen > MHLEN) ? 0 : oldoff; 1504 } else { 1505 off = MHLEN - (hlen + misalignment); 1506 } 1507 KASSERT(off + hlen + misalignment <= MHLEN); 1508 if (((oldoff ^ off) & arm_dcache_align) != 0 || off < oldoff) { 1509 memmove(&m->m_pktdat[off], m->m_data, m->m_len); 1510 m->m_data = &m->m_pktdat[off]; 1511 } 1512 m_copydata(n, 0, hlen - m->m_len, &m->m_data[m->m_len]); 1513 m->m_len = hlen; 1514 m->m_next = mext; 1515 while (n != mext) { 1516 n = m_free(n); 1517 } 1518 return m; 1519 } 1520 1521 struct mbuf *m0 = m_gethdr(M_DONTWAIT, m->m_type); 1522 if (m0 == NULL) { 1523 return NULL; 1524 } 1525 M_COPY_PKTHDR(m0, m); 1526 MCLAIM(m0, m->m_owner); 1527 if (m0->m_pkthdr.len > MHLEN) { 1528 MCLGET(m0, M_DONTWAIT); 1529 if ((m0->m_flags & M_EXT) == 0) { 1530 m_freem(m0); 1531 return NULL; 1532 } 1533 } 1534 m0->m_len = m->m_pkthdr.len; 1535 m_copydata(m, 0, m0->m_len, mtod(m0, void *)); 1536 m_freem(m); 1537 return m0; 1538 } 1539 1540 static bool 1541 bcmeth_txq_enqueue( 1542 struct bcmeth_softc *sc, 1543 struct bcmeth_txqueue *txq) 1544 { 1545 for (;;) { 1546 if (IF_QFULL(&txq->txq_mbufs)) 1547 return false; 1548 struct mbuf *m = txq->txq_next; 1549 if (m == NULL) { 1550 int s = splnet(); 1551 IF_DEQUEUE(&sc->sc_if.if_snd, m); 1552 splx(s); 1553 if (m == NULL) 1554 return true; 1555 M_SETCTX(m, NULL); 1556 } else { 1557 txq->txq_next = NULL; 1558 } 1559 /* 1560 * If LINK2 is set and this packet uses multiple mbufs, 1561 * consolidate it into a single mbuf. 1562 */ 1563 if (m->m_next != NULL && (sc->sc_if.if_flags & IFF_LINK2)) { 1564 struct mbuf *m0 = bcmeth_copy_packet(m); 1565 if (m0 == NULL) { 1566 txq->txq_next = m; 1567 return true; 1568 } 1569 m = m0; 1570 } 1571 int error = bcmeth_txq_map_load(sc, txq, m); 1572 if (error) { 1573 aprint_error_dev(sc->sc_dev, 1574 "discarded packet due to " 1575 "dmamap load failure: %d\n", error); 1576 m_freem(m); 1577 continue; 1578 } 1579 KASSERT(txq->txq_next == NULL); 1580 if (!bcmeth_txq_produce(sc, txq, m)) { 1581 txq->txq_next = m; 1582 return false; 1583 } 1584 KASSERT(txq->txq_next == NULL); 1585 } 1586 } 1587 1588 static bool 1589 bcmeth_txq_consume( 1590 struct bcmeth_softc *sc, 1591 struct bcmeth_txqueue *txq) 1592 { 1593 struct ifnet * const ifp = &sc->sc_if; 1594 struct gmac_txdb *consumer = txq->txq_consumer; 1595 size_t txfree = 0; 1596 1597 #if 0 1598 printf("%s: entry: free=%zu\n", __func__, txq->txq_free); 1599 #endif 1600 1601 for (;;) { 1602 if (consumer == txq->txq_producer) { 1603 txq->txq_consumer = consumer; 1604 txq->txq_free += txfree; 1605 txq->txq_lastintr -= min(txq->txq_lastintr, txfree); 1606 #if 0 1607 printf("%s: empty: freed %zu descriptors going from %zu to %zu\n", 1608 __func__, txfree, txq->txq_free - txfree, txq->txq_free); 1609 #endif 1610 KASSERT(txq->txq_lastintr == 0); 1611 KASSERT(txq->txq_free == txq->txq_last - txq->txq_first - 1); 1612 return true; 1613 } 1614 bcmeth_txq_desc_postsync(sc, txq, consumer, 1); 1615 uint32_t s0 = bcmeth_read_4(sc, txq->txq_reg_xmtsts0); 1616 if (consumer == txq->txq_first + __SHIFTOUT(s0, XMT_CURRDSCR)) { 1617 txq->txq_consumer = consumer; 1618 txq->txq_free += txfree; 1619 txq->txq_lastintr -= min(txq->txq_lastintr, txfree); 1620 #if 0 1621 printf("%s: freed %zu descriptors\n", 1622 __func__, txfree); 1623 #endif 1624 return bcmeth_txq_fillable_p(sc, txq); 1625 } 1626 1627 /* 1628 * If this is the last descriptor in the chain, get the 1629 * mbuf, free its dmamap, and free the mbuf chain itself. 1630 */ 1631 const uint32_t txdb_flags = le32toh(consumer->txdb_flags); 1632 if (txdb_flags & TXDB_FLAG_EF) { 1633 struct mbuf *m; 1634 1635 IF_DEQUEUE(&txq->txq_mbufs, m); 1636 KASSERT(m); 1637 bcmeth_txq_map_unload(sc, txq, m); 1638 #if 0 1639 printf("%s: mbuf %p: consumed a %u byte packet\n", 1640 __func__, m, m->m_pkthdr.len); 1641 #endif 1642 bpf_mtap(ifp, m); 1643 ifp->if_opackets++; 1644 ifp->if_obytes += m->m_pkthdr.len; 1645 if (m->m_flags & M_MCAST) 1646 ifp->if_omcasts++; 1647 m_freem(m); 1648 } 1649 1650 /* 1651 * We own this packet again. Clear all flags except wrap. 1652 */ 1653 txfree++; 1654 1655 /* 1656 * Wrap at the last entry! 1657 */ 1658 if (txdb_flags & TXDB_FLAG_ET) { 1659 consumer->txdb_flags = htole32(TXDB_FLAG_ET); 1660 KASSERT(consumer + 1 == txq->txq_last); 1661 consumer = txq->txq_first; 1662 } else { 1663 consumer->txdb_flags = 0; 1664 consumer++; 1665 KASSERT(consumer < txq->txq_last); 1666 } 1667 } 1668 } 1669 1670 static void 1671 bcmeth_txq_purge( 1672 struct bcmeth_softc *sc, 1673 struct bcmeth_txqueue *txq) 1674 { 1675 struct mbuf *m; 1676 KASSERT((bcmeth_read_4(sc, UNIMAC_COMMAND_CONFIG) & TX_ENA) == 0); 1677 1678 for (;;) { 1679 IF_DEQUEUE(&txq->txq_mbufs, m); 1680 if (m == NULL) 1681 break; 1682 bcmeth_txq_map_unload(sc, txq, m); 1683 m_freem(m); 1684 } 1685 if ((m = txq->txq_next) != NULL) { 1686 txq->txq_next = NULL; 1687 bcmeth_txq_map_unload(sc, txq, m); 1688 m_freem(m); 1689 } 1690 } 1691 1692 static void 1693 bcmeth_txq_reset( 1694 struct bcmeth_softc *sc, 1695 struct bcmeth_txqueue *txq) 1696 { 1697 /* 1698 * sync all the descriptors 1699 */ 1700 bcmeth_txq_desc_postsync(sc, txq, txq->txq_first, 1701 txq->txq_last - txq->txq_first); 1702 1703 /* 1704 * Make sure we own all descriptors in the ring. 1705 */ 1706 struct gmac_txdb *txdb; 1707 for (txdb = txq->txq_first; txdb < txq->txq_last - 1; txdb++) { 1708 txdb->txdb_flags = 0; 1709 } 1710 1711 /* 1712 * Last descriptor has the wrap flag. 1713 */ 1714 txdb->txdb_flags = htole32(TXDB_FLAG_ET); 1715 1716 /* 1717 * Reset the producer consumer indexes. 1718 */ 1719 txq->txq_consumer = txq->txq_first; 1720 txq->txq_producer = txq->txq_first; 1721 txq->txq_free = txq->txq_last - txq->txq_first - 1; 1722 txq->txq_threshold = txq->txq_free / 2; 1723 txq->txq_lastintr = 0; 1724 1725 /* 1726 * What do we want to get interrupted on? 1727 */ 1728 sc->sc_intmask |= XMTINT_0 | XMTUF; 1729 1730 /* 1731 * Restart the transmiter at the first descriptor 1732 */ 1733 bcmeth_write_4(sc, txq->txq_reg_xmtaddrlo, 1734 txq->txq_descmap->dm_segs->ds_addr); 1735 } 1736 1737 static void 1738 bcmeth_ifstart(struct ifnet *ifp) 1739 { 1740 struct bcmeth_softc * const sc = ifp->if_softc; 1741 1742 if (__predict_false((ifp->if_flags & IFF_RUNNING) == 0)) { 1743 return; 1744 } 1745 1746 #ifdef BCMETH_MPSAFETX 1747 if (cpu_intr_p()) { 1748 #endif 1749 atomic_or_uint(&sc->sc_soft_flags, SOFT_TXINTR); 1750 softint_schedule(sc->sc_soft_ih); 1751 #ifdef BCMETH_MPSAFETX 1752 } else { 1753 /* 1754 * Either we are in a softintr thread already or some other 1755 * thread so just borrow it to do the send and save ourselves 1756 * the overhead of a fast soft int. 1757 */ 1758 bcmeth_soft_txintr(sc); 1759 } 1760 #endif 1761 } 1762 1763 int 1764 bcmeth_intr(void *arg) 1765 { 1766 struct bcmeth_softc * const sc = arg; 1767 uint32_t soft_flags = 0; 1768 uint32_t work_flags = 0; 1769 int rv = 0; 1770 1771 mutex_enter(sc->sc_hwlock); 1772 1773 uint32_t intmask = sc->sc_intmask; 1774 BCMETH_EVCNT_INCR(sc->sc_ev_intr); 1775 1776 for (;;) { 1777 uint32_t intstatus = bcmeth_read_4(sc, GMAC_INTSTATUS); 1778 intstatus &= intmask; 1779 bcmeth_write_4(sc, GMAC_INTSTATUS, intstatus); /* write 1 to clear */ 1780 if (intstatus == 0) { 1781 break; 1782 } 1783 #if 0 1784 aprint_normal_dev(sc->sc_dev, "%s: intstatus=%#x intmask=%#x\n", 1785 __func__, intstatus, bcmeth_read_4(sc, GMAC_INTMASK)); 1786 #endif 1787 if (intstatus & RCVINT) { 1788 struct bcmeth_rxqueue * const rxq = &sc->sc_rxq; 1789 intmask &= ~RCVINT; 1790 1791 uint32_t rcvsts0 = bcmeth_read_4(sc, rxq->rxq_reg_rcvsts0); 1792 uint32_t descs = __SHIFTOUT(rcvsts0, RCV_CURRDSCR); 1793 if (descs < rxq->rxq_consumer - rxq->rxq_first) { 1794 /* 1795 * We wrapped at the end so count how far 1796 * we are from the end. 1797 */ 1798 descs += rxq->rxq_last - rxq->rxq_consumer; 1799 } else { 1800 descs -= rxq->rxq_consumer - rxq->rxq_first; 1801 } 1802 /* 1803 * If we "timedout" we can't be hogging so use 1804 * softints. If we exceeded then we might hogging 1805 * so let the workqueue deal with them. 1806 */ 1807 const uint32_t framecount = __SHIFTOUT(sc->sc_rcvlazy, INTRCVLAZY_FRAMECOUNT); 1808 if (descs < framecount 1809 || (curcpu()->ci_curlwp->l_flag & LW_IDLE)) { 1810 soft_flags |= SOFT_RXINTR; 1811 } else { 1812 work_flags |= WORK_RXINTR; 1813 } 1814 } 1815 1816 if (intstatus & XMTINT_0) { 1817 intmask &= ~XMTINT_0; 1818 soft_flags |= SOFT_TXINTR; 1819 } 1820 1821 if (intstatus & RCVDESCUF) { 1822 intmask &= ~RCVDESCUF; 1823 work_flags |= WORK_RXUNDERFLOW; 1824 } 1825 1826 intstatus &= intmask; 1827 if (intstatus) { 1828 aprint_error_dev(sc->sc_dev, 1829 "intr: intstatus=%#x\n", intstatus); 1830 aprint_error_dev(sc->sc_dev, 1831 "rcvbase=%p/%#lx rcvptr=%#x rcvsts=%#x/%#x\n", 1832 sc->sc_rxq.rxq_first, 1833 sc->sc_rxq.rxq_descmap->dm_segs[0].ds_addr, 1834 bcmeth_read_4(sc, sc->sc_rxq.rxq_reg_rcvptr), 1835 bcmeth_read_4(sc, sc->sc_rxq.rxq_reg_rcvsts0), 1836 bcmeth_read_4(sc, sc->sc_rxq.rxq_reg_rcvsts1)); 1837 aprint_error_dev(sc->sc_dev, 1838 "xmtbase=%p/%#lx xmtptr=%#x xmtsts=%#x/%#x\n", 1839 sc->sc_txq.txq_first, 1840 sc->sc_txq.txq_descmap->dm_segs[0].ds_addr, 1841 bcmeth_read_4(sc, sc->sc_txq.txq_reg_xmtptr), 1842 bcmeth_read_4(sc, sc->sc_txq.txq_reg_xmtsts0), 1843 bcmeth_read_4(sc, sc->sc_txq.txq_reg_xmtsts1)); 1844 intmask &= ~intstatus; 1845 work_flags |= WORK_REINIT; 1846 break; 1847 } 1848 } 1849 1850 if (intmask != sc->sc_intmask) { 1851 bcmeth_write_4(sc, GMAC_INTMASK, sc->sc_intmask); 1852 } 1853 1854 if (work_flags) { 1855 if (sc->sc_work_flags == 0) { 1856 workqueue_enqueue(sc->sc_workq, &sc->sc_work, NULL); 1857 } 1858 atomic_or_32(&sc->sc_work_flags, work_flags); 1859 rv = 1; 1860 } 1861 1862 if (soft_flags) { 1863 if (sc->sc_soft_flags == 0) { 1864 softint_schedule(sc->sc_soft_ih); 1865 } 1866 atomic_or_32(&sc->sc_soft_flags, soft_flags); 1867 rv = 1; 1868 } 1869 1870 mutex_exit(sc->sc_hwlock); 1871 1872 return rv; 1873 } 1874 1875 #ifdef BCMETH_MPSAFETX 1876 void 1877 bcmeth_soft_txintr(struct bcmeth_softc *sc) 1878 { 1879 mutex_enter(sc->sc_lock); 1880 /* 1881 * Let's do what we came here for. Consume transmitted 1882 * packets off the the transmit ring. 1883 */ 1884 if (!bcmeth_txq_consume(sc, &sc->sc_txq) 1885 || !bcmeth_txq_enqueue(sc, &sc->sc_txq)) { 1886 BCMETH_EVCNT_INCR(sc->sc_ev_tx_stall); 1887 sc->sc_if.if_flags |= IFF_OACTIVE; 1888 } else { 1889 sc->sc_if.if_flags &= ~IFF_OACTIVE; 1890 } 1891 if (sc->sc_if.if_flags & IFF_RUNNING) { 1892 mutex_spin_enter(sc->sc_hwlock); 1893 sc->sc_intmask |= XMTINT_0; 1894 bcmeth_write_4(sc, GMAC_INTMASK, sc->sc_intmask); 1895 mutex_spin_exit(sc->sc_hwlock); 1896 } 1897 mutex_exit(sc->sc_lock); 1898 } 1899 #endif /* BCMETH_MPSAFETX */ 1900 1901 void 1902 bcmeth_soft_intr(void *arg) 1903 { 1904 struct bcmeth_softc * const sc = arg; 1905 struct ifnet * const ifp = &sc->sc_if; 1906 uint32_t intmask = 0; 1907 1908 mutex_enter(sc->sc_lock); 1909 1910 u_int soft_flags = atomic_swap_uint(&sc->sc_soft_flags, 0); 1911 1912 BCMETH_EVCNT_INCR(sc->sc_ev_soft_intr); 1913 1914 if ((soft_flags & SOFT_TXINTR) 1915 || bcmeth_txq_active_p(sc, &sc->sc_txq)) { 1916 /* 1917 * Let's do what we came here for. Consume transmitted 1918 * packets off the the transmit ring. 1919 */ 1920 if (!bcmeth_txq_consume(sc, &sc->sc_txq) 1921 || !bcmeth_txq_enqueue(sc, &sc->sc_txq)) { 1922 BCMETH_EVCNT_INCR(sc->sc_ev_tx_stall); 1923 ifp->if_flags |= IFF_OACTIVE; 1924 } else { 1925 ifp->if_flags &= ~IFF_OACTIVE; 1926 } 1927 intmask |= XMTINT_0; 1928 } 1929 1930 if (soft_flags & SOFT_RXINTR) { 1931 /* 1932 * Let's consume 1933 */ 1934 while (bcmeth_rxq_consume(sc, &sc->sc_rxq, 1935 sc->sc_rxq.rxq_threshold / 4)) { 1936 /* 1937 * We've consumed a quarter of the ring and still have 1938 * more to do. Refill the ring. 1939 */ 1940 bcmeth_rxq_produce(sc, &sc->sc_rxq); 1941 } 1942 intmask |= RCVINT; 1943 } 1944 1945 if (ifp->if_flags & IFF_RUNNING) { 1946 bcmeth_rxq_produce(sc, &sc->sc_rxq); 1947 mutex_spin_enter(sc->sc_hwlock); 1948 sc->sc_intmask |= intmask; 1949 bcmeth_write_4(sc, GMAC_INTMASK, sc->sc_intmask); 1950 mutex_spin_exit(sc->sc_hwlock); 1951 } 1952 1953 mutex_exit(sc->sc_lock); 1954 } 1955 1956 void 1957 bcmeth_worker(struct work *wk, void *arg) 1958 { 1959 struct bcmeth_softc * const sc = arg; 1960 struct ifnet * const ifp = &sc->sc_if; 1961 uint32_t intmask = 0; 1962 1963 mutex_enter(sc->sc_lock); 1964 1965 BCMETH_EVCNT_INCR(sc->sc_ev_work); 1966 1967 uint32_t work_flags = atomic_swap_32(&sc->sc_work_flags, 0); 1968 if (work_flags & WORK_REINIT) { 1969 int s = splnet(); 1970 sc->sc_soft_flags = 0; 1971 bcmeth_ifinit(ifp); 1972 splx(s); 1973 work_flags &= ~WORK_RXUNDERFLOW; 1974 } 1975 1976 if (work_flags & WORK_RXUNDERFLOW) { 1977 struct bcmeth_rxqueue * const rxq = &sc->sc_rxq; 1978 size_t threshold = 5 * rxq->rxq_threshold / 4; 1979 if (threshold >= rxq->rxq_last - rxq->rxq_first) { 1980 threshold = rxq->rxq_last - rxq->rxq_first - 1; 1981 } else { 1982 intmask |= RCVDESCUF; 1983 } 1984 aprint_normal_dev(sc->sc_dev, 1985 "increasing receive buffers from %zu to %zu\n", 1986 rxq->rxq_threshold, threshold); 1987 rxq->rxq_threshold = threshold; 1988 } 1989 1990 if (work_flags & WORK_RXINTR) { 1991 /* 1992 * Let's consume 1993 */ 1994 while (bcmeth_rxq_consume(sc, &sc->sc_rxq, 1995 sc->sc_rxq.rxq_threshold / 4)) { 1996 /* 1997 * We've consumed a quarter of the ring and still have 1998 * more to do. Refill the ring. 1999 */ 2000 bcmeth_rxq_produce(sc, &sc->sc_rxq); 2001 } 2002 intmask |= RCVINT; 2003 } 2004 2005 if (ifp->if_flags & IFF_RUNNING) { 2006 bcmeth_rxq_produce(sc, &sc->sc_rxq); 2007 #if 0 2008 uint32_t intstatus = bcmeth_read_4(sc, GMAC_INTSTATUS); 2009 if (intstatus & RCVINT) { 2010 bcmeth_write_4(sc, GMAC_INTSTATUS, RCVINT); 2011 work_flags |= WORK_RXINTR; 2012 continue; 2013 } 2014 #endif 2015 mutex_spin_enter(sc->sc_hwlock); 2016 sc->sc_intmask |= intmask; 2017 bcmeth_write_4(sc, GMAC_INTMASK, sc->sc_intmask); 2018 mutex_spin_exit(sc->sc_hwlock); 2019 } 2020 2021 mutex_exit(sc->sc_lock); 2022 } 2023
Indexes created Wed Oct 22 00:09:40 GMT 2025