bcm53xx_eth.c revision 1.43
1 /*- 2 * Copyright (c) 2012 The NetBSD Foundation, Inc. 3 * All rights reserved. 4 * 5 * This code is derived from software contributed to The NetBSD Foundation 6 * by Matt Thomas of 3am Software Foundry. 7 * 8 * Redistribution and use in source and binary forms, with or without 9 * modification, are permitted provided that the following conditions 10 * are met: 11 * 1. Redistributions of source code must retain the above copyright 12 * notice, this list of conditions and the following disclaimer. 13 * 2. Redistributions in binary form must reproduce the above copyright 14 * notice, this list of conditions and the following disclaimer in the 15 * documentation and/or other materials provided with the distribution. 16 * 17 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS 18 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED 19 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 20 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS 21 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 22 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 23 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 24 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 25 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 26 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 27 * POSSIBILITY OF SUCH DAMAGE. 28 */ 29 30 #define _ARM32_BUS_DMA_PRIVATE 31 #define GMAC_PRIVATE 32 33 #include "locators.h" 34 #include "opt_broadcom.h" 35 36 #include <sys/cdefs.h> 37 38 __KERNEL_RCSID(1, "$NetBSD: bcm53xx_eth.c,v 1.43 2024/02/16 12:08:29 skrll Exp $"); 39 40 #include <sys/param.h> 41 #include <sys/atomic.h> 42 #include <sys/bus.h> 43 #include <sys/device.h> 44 #include <sys/ioctl.h> 45 #include <sys/intr.h> 46 #include <sys/kmem.h> 47 #include <sys/mutex.h> 48 #include <sys/socket.h> 49 #include <sys/systm.h> 50 #include <sys/workqueue.h> 51 52 #include <net/if.h> 53 #include <net/if_ether.h> 54 #include <net/if_media.h> 55 #include <net/if_dl.h> 56 #include <net/bpf.h> 57 58 #include <dev/mii/miivar.h> 59 60 #include <arm/locore.h> 61 62 #include <arm/broadcom/bcm53xx_reg.h> 63 #include <arm/broadcom/bcm53xx_var.h> 64 65 //#define BCMETH_MPSAFE 66 67 #ifdef BCMETH_COUNTERS 68 #define BCMETH_EVCNT_ADD(a, b) ((void)((a).ev_count += (b))) 69 #else 70 #define BCMETH_EVCNT_ADD(a, b) do { } while (/*CONSTCOND*/0) 71 #endif 72 #define BCMETH_EVCNT_INCR(a) BCMETH_EVCNT_ADD((a), 1) 73 74 #define BCMETH_MAXTXMBUFS 128 75 #define BCMETH_NTXSEGS 30 76 #define BCMETH_MAXRXMBUFS 255 77 #define BCMETH_MINRXMBUFS 64 78 #define BCMETH_NRXSEGS 1 79 #define BCMETH_RINGSIZE PAGE_SIZE 80 81 #if 1 82 #define BCMETH_RCVMAGIC 0xfeedface 83 #endif 84 85 static int bcmeth_ccb_match(device_t, cfdata_t, void *); 86 static void bcmeth_ccb_attach(device_t, device_t, void *); 87 88 struct bcmeth_txqueue { 89 bus_dmamap_t txq_descmap; 90 struct gmac_txdb *txq_consumer; 91 struct gmac_txdb *txq_producer; 92 struct gmac_txdb *txq_first; 93 struct gmac_txdb *txq_last; 94 struct ifqueue txq_mbufs; 95 struct mbuf *txq_next; 96 size_t txq_free; 97 size_t txq_threshold; 98 size_t txq_lastintr; 99 bus_size_t txq_reg_xmtaddrlo; 100 bus_size_t txq_reg_xmtptr; 101 bus_size_t txq_reg_xmtctl; 102 bus_size_t txq_reg_xmtsts0; 103 bus_size_t txq_reg_xmtsts1; 104 bus_dma_segment_t txq_descmap_seg; 105 }; 106 107 struct bcmeth_rxqueue { 108 bus_dmamap_t rxq_descmap; 109 struct gmac_rxdb *rxq_consumer; 110 struct gmac_rxdb *rxq_producer; 111 struct gmac_rxdb *rxq_first; 112 struct gmac_rxdb *rxq_last; 113 struct mbuf *rxq_mhead; 114 struct mbuf **rxq_mtail; 115 struct mbuf *rxq_mconsumer; 116 size_t rxq_inuse; 117 size_t rxq_threshold; 118 bus_size_t rxq_reg_rcvaddrlo; 119 bus_size_t rxq_reg_rcvptr; 120 bus_size_t rxq_reg_rcvctl; 121 bus_size_t rxq_reg_rcvsts0; 122 bus_size_t rxq_reg_rcvsts1; 123 bus_dma_segment_t rxq_descmap_seg; 124 }; 125 126 struct bcmeth_mapcache { 127 u_int dmc_nmaps; 128 u_int dmc_maxseg; 129 u_int dmc_maxmaps; 130 u_int dmc_maxmapsize; 131 bus_dmamap_t dmc_maps[0]; 132 }; 133 134 struct bcmeth_softc { 135 device_t sc_dev; 136 bus_space_tag_t sc_bst; 137 bus_space_handle_t sc_bsh; 138 bus_dma_tag_t sc_dmat; 139 kmutex_t *sc_lock; 140 kmutex_t *sc_hwlock; 141 struct ethercom sc_ec; 142 #define sc_if sc_ec.ec_if 143 struct ifmedia sc_media; 144 void *sc_soft_ih; 145 void *sc_ih; 146 147 struct bcmeth_rxqueue sc_rxq; 148 struct bcmeth_txqueue sc_txq; 149 150 size_t sc_rcvoffset; 151 uint32_t sc_macaddr[2]; 152 uint32_t sc_maxfrm; 153 uint32_t sc_cmdcfg; 154 uint32_t sc_intmask; 155 uint32_t sc_rcvlazy; 156 volatile uint32_t sc_soft_flags; 157 #define SOFT_RXINTR 0x01 158 #define SOFT_TXINTR 0x02 159 160 #ifdef BCMETH_COUNTERS 161 struct evcnt sc_ev_intr; 162 struct evcnt sc_ev_soft_intr; 163 struct evcnt sc_ev_work; 164 struct evcnt sc_ev_tx_stall; 165 struct evcnt sc_ev_rx_badmagic_lo; 166 struct evcnt sc_ev_rx_badmagic_hi; 167 #endif 168 169 struct ifqueue sc_rx_bufcache; 170 struct bcmeth_mapcache *sc_rx_mapcache; 171 struct bcmeth_mapcache *sc_tx_mapcache; 172 173 struct workqueue *sc_workq; 174 struct work sc_work; 175 176 volatile uint32_t sc_work_flags; 177 #define WORK_RXINTR 0x01 178 #define WORK_RXUNDERFLOW 0x02 179 #define WORK_REINIT 0x04 180 181 uint8_t sc_enaddr[ETHER_ADDR_LEN]; 182 }; 183 184 static void bcmeth_ifstart(struct ifnet *); 185 static void bcmeth_ifwatchdog(struct ifnet *); 186 static int bcmeth_ifinit(struct ifnet *); 187 static void bcmeth_ifstop(struct ifnet *, int); 188 static int bcmeth_ifioctl(struct ifnet *, u_long, void *); 189 190 static int bcmeth_mapcache_create(struct bcmeth_softc *, 191 struct bcmeth_mapcache **, size_t, size_t, size_t); 192 static void bcmeth_mapcache_destroy(struct bcmeth_softc *, 193 struct bcmeth_mapcache *); 194 static bus_dmamap_t bcmeth_mapcache_get(struct bcmeth_softc *, 195 struct bcmeth_mapcache *); 196 static void bcmeth_mapcache_put(struct bcmeth_softc *, 197 struct bcmeth_mapcache *, bus_dmamap_t); 198 199 static int bcmeth_txq_attach(struct bcmeth_softc *, 200 struct bcmeth_txqueue *, u_int); 201 static void bcmeth_txq_purge(struct bcmeth_softc *, 202 struct bcmeth_txqueue *); 203 static void bcmeth_txq_reset(struct bcmeth_softc *, 204 struct bcmeth_txqueue *); 205 static bool bcmeth_txq_consume(struct bcmeth_softc *, 206 struct bcmeth_txqueue *); 207 static bool bcmeth_txq_produce(struct bcmeth_softc *, 208 struct bcmeth_txqueue *, struct mbuf *m); 209 static bool bcmeth_txq_active_p(struct bcmeth_softc *, 210 struct bcmeth_txqueue *); 211 212 static int bcmeth_rxq_attach(struct bcmeth_softc *, 213 struct bcmeth_rxqueue *, u_int); 214 static bool bcmeth_rxq_produce(struct bcmeth_softc *, 215 struct bcmeth_rxqueue *); 216 static void bcmeth_rxq_purge(struct bcmeth_softc *, 217 struct bcmeth_rxqueue *, bool); 218 static void bcmeth_rxq_reset(struct bcmeth_softc *, 219 struct bcmeth_rxqueue *); 220 221 static int bcmeth_intr(void *); 222 #ifdef BCMETH_MPSAFETX 223 static void bcmeth_soft_txintr(struct bcmeth_softc *); 224 #endif 225 static void bcmeth_soft_intr(void *); 226 static void bcmeth_worker(struct work *, void *); 227 228 static int bcmeth_mediachange(struct ifnet *); 229 static void bcmeth_mediastatus(struct ifnet *, struct ifmediareq *); 230 231 static inline uint32_t 232 bcmeth_read_4(struct bcmeth_softc *sc, bus_size_t o) 233 { 234 return bus_space_read_4(sc->sc_bst, sc->sc_bsh, o); 235 } 236 237 static inline void 238 bcmeth_write_4(struct bcmeth_softc *sc, bus_size_t o, uint32_t v) 239 { 240 bus_space_write_4(sc->sc_bst, sc->sc_bsh, o, v); 241 } 242 243 CFATTACH_DECL_NEW(bcmeth_ccb, sizeof(struct bcmeth_softc), 244 bcmeth_ccb_match, bcmeth_ccb_attach, NULL, NULL); 245 246 static int 247 bcmeth_ccb_match(device_t parent, cfdata_t cf, void *aux) 248 { 249 struct bcmccb_attach_args * const ccbaa = aux; 250 const struct bcm_locators * const loc = &ccbaa->ccbaa_loc; 251 252 if (strcmp(cf->cf_name, loc->loc_name)) 253 return 0; 254 255 const int port __diagused = cf->cf_loc[BCMCCBCF_PORT]; 256 KASSERT(port == BCMCCBCF_PORT_DEFAULT || port == loc->loc_port); 257 258 return 1; 259 } 260 261 static void 262 bcmeth_ccb_attach(device_t parent, device_t self, void *aux) 263 { 264 struct bcmeth_softc * const sc = device_private(self); 265 struct ethercom * const ec = &sc->sc_ec; 266 struct ifnet * const ifp = &ec->ec_if; 267 struct bcmccb_attach_args * const ccbaa = aux; 268 const struct bcm_locators * const loc = &ccbaa->ccbaa_loc; 269 const char * const xname = device_xname(self); 270 prop_dictionary_t dict = device_properties(self); 271 int error; 272 273 sc->sc_bst = ccbaa->ccbaa_ccb_bst; 274 sc->sc_dmat = ccbaa->ccbaa_dmat; 275 bus_space_subregion(sc->sc_bst, ccbaa->ccbaa_ccb_bsh, 276 loc->loc_offset, loc->loc_size, &sc->sc_bsh); 277 278 /* 279 * We need to use the coherent dma tag for the GMAC. 280 */ 281 sc->sc_dmat = &bcm53xx_coherent_dma_tag; 282 #if _ARM32_NEED_BUS_DMA_BOUNCE 283 if (device_cfdata(self)->cf_flags & 2) { 284 sc->sc_dmat = &bcm53xx_bounce_dma_tag; 285 } 286 #endif 287 288 prop_data_t eaprop = prop_dictionary_get(dict, "mac-address"); 289 if (eaprop == NULL) { 290 uint32_t mac0 = bcmeth_read_4(sc, UNIMAC_MAC_0); 291 uint32_t mac1 = bcmeth_read_4(sc, UNIMAC_MAC_1); 292 if ((mac0 == 0 && mac1 == 0) || (mac1 & 1)) { 293 aprint_error(": mac-address property is missing\n"); 294 return; 295 } 296 sc->sc_enaddr[0] = (mac0 >> 0) & 0xff; 297 sc->sc_enaddr[1] = (mac0 >> 8) & 0xff; 298 sc->sc_enaddr[2] = (mac0 >> 16) & 0xff; 299 sc->sc_enaddr[3] = (mac0 >> 24) & 0xff; 300 sc->sc_enaddr[4] = (mac1 >> 0) & 0xff; 301 sc->sc_enaddr[5] = (mac1 >> 8) & 0xff; 302 } else { 303 KASSERT(prop_object_type(eaprop) == PROP_TYPE_DATA); 304 KASSERT(prop_data_size(eaprop) == ETHER_ADDR_LEN); 305 memcpy(sc->sc_enaddr, prop_data_data_nocopy(eaprop), 306 ETHER_ADDR_LEN); 307 } 308 sc->sc_dev = self; 309 sc->sc_lock = mutex_obj_alloc(MUTEX_DEFAULT, IPL_SOFTNET); 310 sc->sc_hwlock = mutex_obj_alloc(MUTEX_DEFAULT, IPL_VM); 311 312 bcmeth_write_4(sc, GMAC_INTMASK, 0); // disable interrupts 313 314 aprint_naive("\n"); 315 aprint_normal(": Gigabit Ethernet Controller\n"); 316 317 error = bcmeth_rxq_attach(sc, &sc->sc_rxq, 0); 318 if (error) { 319 aprint_error(": failed to init rxq: %d\n", error); 320 goto fail_1; 321 } 322 323 error = bcmeth_txq_attach(sc, &sc->sc_txq, 0); 324 if (error) { 325 aprint_error(": failed to init txq: %d\n", error); 326 goto fail_1; 327 } 328 329 error = bcmeth_mapcache_create(sc, &sc->sc_rx_mapcache, 330 BCMETH_MAXRXMBUFS, MCLBYTES, BCMETH_NRXSEGS); 331 if (error) { 332 aprint_error(": failed to allocate rx dmamaps: %d\n", error); 333 goto fail_1; 334 } 335 336 error = bcmeth_mapcache_create(sc, &sc->sc_tx_mapcache, 337 BCMETH_MAXTXMBUFS, MCLBYTES, BCMETH_NTXSEGS); 338 if (error) { 339 aprint_error(": failed to allocate tx dmamaps: %d\n", error); 340 goto fail_1; 341 } 342 343 error = workqueue_create(&sc->sc_workq, xname, bcmeth_worker, sc, 344 (PRI_USER + MAXPRI_USER) / 2, IPL_NET, WQ_MPSAFE|WQ_PERCPU); 345 if (error) { 346 aprint_error(": failed to create workqueue: %d\n", error); 347 goto fail_2; 348 } 349 350 sc->sc_soft_ih = softint_establish(SOFTINT_MPSAFE | SOFTINT_NET, 351 bcmeth_soft_intr, sc); 352 353 if (sc->sc_ih == NULL) { 354 aprint_error_dev(self, "failed to establish interrupt %d\n", 355 loc->loc_intrs[0]); 356 goto fail_3; 357 } 358 359 sc->sc_ih = intr_establish(loc->loc_intrs[0], IPL_VM, IST_LEVEL, 360 bcmeth_intr, sc); 361 362 if (sc->sc_ih == NULL) { 363 aprint_error_dev(self, "failed to establish interrupt %d\n", 364 loc->loc_intrs[0]); 365 goto fail_4; 366 } else { 367 aprint_normal_dev(self, "interrupting on irq %d\n", 368 loc->loc_intrs[0]); 369 } 370 371 aprint_normal_dev(sc->sc_dev, "Ethernet address %s\n", 372 ether_sprintf(sc->sc_enaddr)); 373 374 /* 375 * Since each port in plugged into the switch/flow-accelerator, 376 * we hard code at Gige Full-Duplex with Flow Control enabled. 377 */ 378 int ifmedia = IFM_ETHER | IFM_1000_T | IFM_FDX; 379 //ifmedia |= IFM_FLOW | IFM_ETH_TXPAUSE | IFM_ETH_RXPAUSE; 380 ec->ec_ifmedia = &sc->sc_media; 381 ifmedia_init(&sc->sc_media, IFM_IMASK, bcmeth_mediachange, 382 bcmeth_mediastatus); 383 ifmedia_add(&sc->sc_media, ifmedia, 0, NULL); 384 ifmedia_set(&sc->sc_media, ifmedia); 385 386 ec->ec_capabilities = ETHERCAP_VLAN_MTU | ETHERCAP_JUMBO_MTU; 387 388 strlcpy(ifp->if_xname, xname, IFNAMSIZ); 389 ifp->if_softc = sc; 390 ifp->if_baudrate = IF_Mbps(1000); 391 ifp->if_capabilities = 0; 392 ifp->if_flags = IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST; 393 #ifdef BCMETH_MPSAFE 394 ifp->if_flags2 = IFF2_MPSAFE; 395 #endif 396 ifp->if_ioctl = bcmeth_ifioctl; 397 ifp->if_start = bcmeth_ifstart; 398 ifp->if_watchdog = bcmeth_ifwatchdog; 399 ifp->if_init = bcmeth_ifinit; 400 ifp->if_stop = bcmeth_ifstop; 401 IFQ_SET_READY(&ifp->if_snd); 402 403 bcmeth_ifstop(ifp, true); 404 405 /* 406 * Attach the interface. 407 */ 408 if_initialize(ifp); 409 ether_ifattach(ifp, sc->sc_enaddr); 410 if_register(ifp); 411 412 #ifdef BCMETH_COUNTERS 413 evcnt_attach_dynamic(&sc->sc_ev_intr, EVCNT_TYPE_INTR, 414 NULL, xname, "intr"); 415 evcnt_attach_dynamic(&sc->sc_ev_soft_intr, EVCNT_TYPE_INTR, 416 NULL, xname, "soft intr"); 417 evcnt_attach_dynamic(&sc->sc_ev_work, EVCNT_TYPE_MISC, 418 NULL, xname, "work items"); 419 evcnt_attach_dynamic(&sc->sc_ev_tx_stall, EVCNT_TYPE_MISC, 420 NULL, xname, "tx stalls"); 421 evcnt_attach_dynamic(&sc->sc_ev_rx_badmagic_lo, EVCNT_TYPE_MISC, 422 NULL, xname, "rx badmagic lo"); 423 evcnt_attach_dynamic(&sc->sc_ev_rx_badmagic_hi, EVCNT_TYPE_MISC, 424 NULL, xname, "rx badmagic hi"); 425 #endif 426 427 return; 428 429 fail_4: 430 intr_disestablish(sc->sc_ih); 431 fail_3: 432 softint_disestablish(sc->sc_soft_ih); 433 fail_2: 434 workqueue_destroy(sc->sc_workq); 435 fail_1: 436 mutex_obj_free(sc->sc_lock); 437 mutex_obj_free(sc->sc_hwlock); 438 } 439 440 static int 441 bcmeth_mediachange(struct ifnet *ifp) 442 { 443 //struct bcmeth_softc * const sc = ifp->if_softc; 444 return 0; 445 } 446 447 static void 448 bcmeth_mediastatus(struct ifnet *ifp, struct ifmediareq *ifm) 449 { 450 //struct bcmeth_softc * const sc = ifp->if_softc; 451 452 ifm->ifm_status = IFM_AVALID | IFM_ACTIVE; 453 ifm->ifm_active = IFM_ETHER | IFM_FDX | IFM_1000_T; 454 } 455 456 static uint64_t 457 bcmeth_macaddr_create(const uint8_t *enaddr) 458 { 459 return (enaddr[3] << 0) // UNIMAC_MAC_0 460 | (enaddr[2] << 8) // UNIMAC_MAC_0 461 | (enaddr[1] << 16) // UNIMAC_MAC_0 462 | ((uint64_t)enaddr[0] << 24) // UNIMAC_MAC_0 463 | ((uint64_t)enaddr[5] << 32) // UNIMAC_MAC_1 464 | ((uint64_t)enaddr[4] << 40); // UNIMAC_MAC_1 465 } 466 467 static int 468 bcmeth_ifinit(struct ifnet *ifp) 469 { 470 struct bcmeth_softc * const sc = ifp->if_softc; 471 int error = 0; 472 473 sc->sc_maxfrm = uimax(ifp->if_mtu + 32, MCLBYTES); 474 if (ifp->if_mtu > ETHERMTU_JUMBO) 475 return error; 476 477 KASSERT(ifp->if_flags & IFF_UP); 478 479 /* 480 * Stop the interface 481 */ 482 bcmeth_ifstop(ifp, 0); 483 484 /* 485 * Reserve enough space at the front so that we can insert a maxsized 486 * link header and a VLAN tag. Also make sure we have enough room for 487 * the rcvsts field as well. 488 */ 489 KASSERT(ALIGN(max_linkhdr) == max_linkhdr); 490 KASSERTMSG(max_linkhdr > sizeof(struct ether_header), "%u > %zu", 491 max_linkhdr, sizeof(struct ether_header)); 492 sc->sc_rcvoffset = max_linkhdr + 4 - sizeof(struct ether_header); 493 if (sc->sc_rcvoffset <= 4) 494 sc->sc_rcvoffset += 4; 495 KASSERT((sc->sc_rcvoffset & 3) == 2); 496 KASSERT(sc->sc_rcvoffset <= __SHIFTOUT(RCVCTL_RCVOFFSET, RCVCTL_RCVOFFSET)); 497 KASSERT(sc->sc_rcvoffset >= 6); 498 499 /* 500 * If our frame size has changed (or it's our first time through) 501 * destroy the existing transmit mapcache. 502 */ 503 if (sc->sc_tx_mapcache != NULL 504 && sc->sc_maxfrm != sc->sc_tx_mapcache->dmc_maxmapsize) { 505 bcmeth_mapcache_destroy(sc, sc->sc_tx_mapcache); 506 sc->sc_tx_mapcache = NULL; 507 } 508 509 if (sc->sc_tx_mapcache == NULL) { 510 error = bcmeth_mapcache_create(sc, &sc->sc_tx_mapcache, 511 BCMETH_MAXTXMBUFS, sc->sc_maxfrm, BCMETH_NTXSEGS); 512 if (error) 513 return error; 514 } 515 516 sc->sc_cmdcfg = NO_LENGTH_CHECK | PAUSE_IGNORE 517 | __SHIFTIN(ETH_SPEED_1000, ETH_SPEED) 518 | RX_ENA | TX_ENA; 519 520 if (ifp->if_flags & IFF_PROMISC) { 521 sc->sc_cmdcfg |= PROMISC_EN; 522 } else { 523 sc->sc_cmdcfg &= ~PROMISC_EN; 524 } 525 526 const uint8_t * const lladdr = CLLADDR(ifp->if_sadl); 527 const uint64_t macstnaddr = bcmeth_macaddr_create(lladdr); 528 529 /* 530 * We make sure that a received Ethernet packet start on a non-word 531 * boundary so that the packet payload will be on a word boundary. 532 * So to check the destination address we keep around two words to 533 * quickly compare with. 534 */ 535 #if __ARMEL__ 536 sc->sc_macaddr[0] = lladdr[0] | (lladdr[1] << 8); 537 sc->sc_macaddr[1] = lladdr[2] | (lladdr[3] << 8) 538 | (lladdr[4] << 16) | (lladdr[5] << 24); 539 #else 540 sc->sc_macaddr[0] = lladdr[1] | (lladdr[0] << 8); 541 sc->sc_macaddr[1] = lladdr[5] | (lladdr[4] << 8) 542 | (lladdr[1] << 16) | (lladdr[2] << 24); 543 #endif 544 545 sc->sc_intmask = DESCPROTOERR | DATAERR | DESCERR; 546 547 /* 5. Load RCVADDR_LO with new pointer */ 548 bcmeth_rxq_reset(sc, &sc->sc_rxq); 549 550 bcmeth_write_4(sc, sc->sc_rxq.rxq_reg_rcvctl, 551 __SHIFTIN(sc->sc_rcvoffset, RCVCTL_RCVOFFSET) 552 | RCVCTL_PARITY_DIS 553 | RCVCTL_OFLOW_CONTINUE 554 | __SHIFTIN(3, RCVCTL_BURSTLEN)); 555 556 /* 6. Load XMTADDR_LO with new pointer */ 557 bcmeth_txq_reset(sc, &sc->sc_txq); 558 559 bcmeth_write_4(sc, sc->sc_txq.txq_reg_xmtctl, XMTCTL_DMA_ACT_INDEX 560 | XMTCTL_PARITY_DIS 561 | __SHIFTIN(3, XMTCTL_BURSTLEN)); 562 563 /* 7. Setup other UNIMAC registers */ 564 bcmeth_write_4(sc, UNIMAC_FRAME_LEN, sc->sc_maxfrm); 565 bcmeth_write_4(sc, UNIMAC_MAC_0, (uint32_t)(macstnaddr >> 0)); 566 bcmeth_write_4(sc, UNIMAC_MAC_1, (uint32_t)(macstnaddr >> 32)); 567 bcmeth_write_4(sc, UNIMAC_COMMAND_CONFIG, sc->sc_cmdcfg); 568 569 uint32_t devctl = bcmeth_read_4(sc, GMAC_DEVCONTROL); 570 devctl |= RGMII_LINK_STATUS_SEL | NWAY_AUTO_POLL_EN | TXARB_STRICT_MODE; 571 devctl &= ~FLOW_CTRL_MODE; 572 devctl &= ~MIB_RD_RESET_EN; 573 devctl &= ~RXQ_OVERFLOW_CTRL_SEL; 574 devctl &= ~CPU_FLOW_CTRL_ON; 575 bcmeth_write_4(sc, GMAC_DEVCONTROL, devctl); 576 577 /* Setup lazy receive (at most 1ms). */ 578 const struct cpu_softc * const cpu = curcpu()->ci_softc; 579 sc->sc_rcvlazy = __SHIFTIN(4, INTRCVLAZY_FRAMECOUNT) 580 | __SHIFTIN(cpu->cpu_clk.clk_apb / 1000, INTRCVLAZY_TIMEOUT); 581 bcmeth_write_4(sc, GMAC_INTRCVLAZY, sc->sc_rcvlazy); 582 583 /* 11. Enable transmit queues in TQUEUE, and ensure that the transmit scheduling mode is correctly set in TCTRL. */ 584 sc->sc_intmask |= XMTINT_0 | XMTUF; 585 bcmeth_write_4(sc, sc->sc_txq.txq_reg_xmtctl, 586 bcmeth_read_4(sc, sc->sc_txq.txq_reg_xmtctl) | XMTCTL_ENABLE); 587 588 589 /* 12. Enable receive queues in RQUEUE, */ 590 sc->sc_intmask |= RCVINT | RCVDESCUF | RCVFIFOOF; 591 bcmeth_write_4(sc, sc->sc_rxq.rxq_reg_rcvctl, 592 bcmeth_read_4(sc, sc->sc_rxq.rxq_reg_rcvctl) | RCVCTL_ENABLE); 593 594 bcmeth_rxq_produce(sc, &sc->sc_rxq); /* fill with rx buffers */ 595 596 #if 0 597 aprint_normal_dev(sc->sc_dev, 598 "devctl=%#x ucmdcfg=%#x xmtctl=%#x rcvctl=%#x\n", 599 devctl, sc->sc_cmdcfg, 600 bcmeth_read_4(sc, sc->sc_txq.txq_reg_xmtctl), 601 bcmeth_read_4(sc, sc->sc_rxq.rxq_reg_rcvctl)); 602 #endif 603 604 sc->sc_soft_flags = 0; 605 606 bcmeth_write_4(sc, GMAC_INTMASK, sc->sc_intmask); 607 608 ifp->if_flags |= IFF_RUNNING; 609 610 return error; 611 } 612 613 static void 614 bcmeth_ifstop(struct ifnet *ifp, int disable) 615 { 616 struct bcmeth_softc * const sc = ifp->if_softc; 617 struct bcmeth_txqueue * const txq = &sc->sc_txq; 618 struct bcmeth_rxqueue * const rxq = &sc->sc_rxq; 619 620 KASSERT(!cpu_intr_p()); 621 622 sc->sc_soft_flags = 0; 623 sc->sc_work_flags = 0; 624 625 /* Disable Rx processing */ 626 bcmeth_write_4(sc, rxq->rxq_reg_rcvctl, 627 bcmeth_read_4(sc, rxq->rxq_reg_rcvctl) & ~RCVCTL_ENABLE); 628 629 /* Disable Tx processing */ 630 bcmeth_write_4(sc, txq->txq_reg_xmtctl, 631 bcmeth_read_4(sc, txq->txq_reg_xmtctl) & ~XMTCTL_ENABLE); 632 633 /* Disable all interrupts */ 634 bcmeth_write_4(sc, GMAC_INTMASK, 0); 635 636 for (;;) { 637 uint32_t tx0 = bcmeth_read_4(sc, txq->txq_reg_xmtsts0); 638 uint32_t rx0 = bcmeth_read_4(sc, rxq->rxq_reg_rcvsts0); 639 if (__SHIFTOUT(tx0, XMTSTATE) == XMTSTATE_DIS 640 && __SHIFTOUT(rx0, RCVSTATE) == RCVSTATE_DIS) 641 break; 642 delay(50); 643 } 644 /* 645 * Now reset the controller. 646 * 647 * 3. Set SW_RESET bit in UNIMAC_COMMAND_CONFIG register 648 * 4. Clear SW_RESET bit in UNIMAC_COMMAND_CONFIG register 649 */ 650 bcmeth_write_4(sc, UNIMAC_COMMAND_CONFIG, SW_RESET); 651 bcmeth_write_4(sc, GMAC_INTSTATUS, ~0); 652 sc->sc_intmask = 0; 653 ifp->if_flags &= ~IFF_RUNNING; 654 655 /* 656 * Let's consume any remaining transmitted packets. And if we are 657 * disabling the interface, purge ourselves of any untransmitted 658 * packets. But don't consume any received packets, just drop them. 659 * If we aren't disabling the interface, save the mbufs in the 660 * receive queue for reuse. 661 */ 662 bcmeth_rxq_purge(sc, &sc->sc_rxq, disable); 663 bcmeth_txq_consume(sc, &sc->sc_txq); 664 if (disable) { 665 bcmeth_txq_purge(sc, &sc->sc_txq); 666 IF_PURGE(&ifp->if_snd); 667 } 668 669 bcmeth_write_4(sc, UNIMAC_COMMAND_CONFIG, 0); 670 } 671 672 static void 673 bcmeth_ifwatchdog(struct ifnet *ifp) 674 { 675 } 676 677 static int 678 bcmeth_ifioctl(struct ifnet *ifp, u_long cmd, void *data) 679 { 680 const int s = splnet(); 681 int error; 682 683 switch (cmd) { 684 default: 685 error = ether_ioctl(ifp, cmd, data); 686 if (error != ENETRESET) 687 break; 688 689 if (cmd == SIOCADDMULTI || cmd == SIOCDELMULTI) { 690 error = 0; 691 break; 692 } 693 error = bcmeth_ifinit(ifp); 694 break; 695 } 696 697 splx(s); 698 return error; 699 } 700 701 static void 702 bcmeth_rxq_desc_presync( 703 struct bcmeth_softc *sc, 704 struct bcmeth_rxqueue *rxq, 705 struct gmac_rxdb *rxdb, 706 size_t count) 707 { 708 bus_dmamap_sync(sc->sc_dmat, rxq->rxq_descmap, 709 (rxdb - rxq->rxq_first) * sizeof(*rxdb), count * sizeof(*rxdb), 710 BUS_DMASYNC_PREWRITE); 711 } 712 713 static void 714 bcmeth_rxq_desc_postsync( 715 struct bcmeth_softc *sc, 716 struct bcmeth_rxqueue *rxq, 717 struct gmac_rxdb *rxdb, 718 size_t count) 719 { 720 bus_dmamap_sync(sc->sc_dmat, rxq->rxq_descmap, 721 (rxdb - rxq->rxq_first) * sizeof(*rxdb), count * sizeof(*rxdb), 722 BUS_DMASYNC_POSTWRITE); 723 } 724 725 static void 726 bcmeth_txq_desc_presync( 727 struct bcmeth_softc *sc, 728 struct bcmeth_txqueue *txq, 729 struct gmac_txdb *txdb, 730 size_t count) 731 { 732 bus_dmamap_sync(sc->sc_dmat, txq->txq_descmap, 733 (txdb - txq->txq_first) * sizeof(*txdb), count * sizeof(*txdb), 734 BUS_DMASYNC_PREWRITE); 735 } 736 737 static void 738 bcmeth_txq_desc_postsync( 739 struct bcmeth_softc *sc, 740 struct bcmeth_txqueue *txq, 741 struct gmac_txdb *txdb, 742 size_t count) 743 { 744 bus_dmamap_sync(sc->sc_dmat, txq->txq_descmap, 745 (txdb - txq->txq_first) * sizeof(*txdb), count * sizeof(*txdb), 746 BUS_DMASYNC_POSTWRITE); 747 } 748 749 static bus_dmamap_t 750 bcmeth_mapcache_get( 751 struct bcmeth_softc *sc, 752 struct bcmeth_mapcache *dmc) 753 { 754 KASSERT(dmc->dmc_nmaps > 0); 755 KASSERT(dmc->dmc_maps[dmc->dmc_nmaps-1] != NULL); 756 return dmc->dmc_maps[--dmc->dmc_nmaps]; 757 } 758 759 static void 760 bcmeth_mapcache_put( 761 struct bcmeth_softc *sc, 762 struct bcmeth_mapcache *dmc, 763 bus_dmamap_t map) 764 { 765 KASSERT(map != NULL); 766 KASSERT(dmc->dmc_nmaps < dmc->dmc_maxmaps); 767 dmc->dmc_maps[dmc->dmc_nmaps++] = map; 768 } 769 770 static void 771 bcmeth_mapcache_destroy( 772 struct bcmeth_softc *sc, 773 struct bcmeth_mapcache *dmc) 774 { 775 const size_t dmc_size = 776 offsetof(struct bcmeth_mapcache, dmc_maps[dmc->dmc_maxmaps]); 777 778 for (u_int i = 0; i < dmc->dmc_maxmaps; i++) { 779 bus_dmamap_destroy(sc->sc_dmat, dmc->dmc_maps[i]); 780 } 781 kmem_intr_free(dmc, dmc_size); 782 } 783 784 static int 785 bcmeth_mapcache_create( 786 struct bcmeth_softc *sc, 787 struct bcmeth_mapcache **dmc_p, 788 size_t maxmaps, 789 size_t maxmapsize, 790 size_t maxseg) 791 { 792 const size_t dmc_size = 793 offsetof(struct bcmeth_mapcache, dmc_maps[maxmaps]); 794 struct bcmeth_mapcache * const dmc = 795 kmem_intr_zalloc(dmc_size, KM_NOSLEEP); 796 797 dmc->dmc_maxmaps = maxmaps; 798 dmc->dmc_nmaps = maxmaps; 799 dmc->dmc_maxmapsize = maxmapsize; 800 dmc->dmc_maxseg = maxseg; 801 802 for (u_int i = 0; i < maxmaps; i++) { 803 int error = bus_dmamap_create(sc->sc_dmat, dmc->dmc_maxmapsize, 804 dmc->dmc_maxseg, dmc->dmc_maxmapsize, 0, 805 BUS_DMA_WAITOK | BUS_DMA_ALLOCNOW, &dmc->dmc_maps[i]); 806 if (error) { 807 aprint_error_dev(sc->sc_dev, 808 "failed to creat dma map cache " 809 "entry %u of %zu: %d\n", 810 i, maxmaps, error); 811 while (i-- > 0) { 812 bus_dmamap_destroy(sc->sc_dmat, 813 dmc->dmc_maps[i]); 814 } 815 kmem_intr_free(dmc, dmc_size); 816 return error; 817 } 818 KASSERT(dmc->dmc_maps[i] != NULL); 819 } 820 821 *dmc_p = dmc; 822 823 return 0; 824 } 825 826 #if 0 827 static void 828 bcmeth_dmamem_free( 829 bus_dma_tag_t dmat, 830 size_t map_size, 831 bus_dma_segment_t *seg, 832 bus_dmamap_t map, 833 void *kvap) 834 { 835 bus_dmamap_destroy(dmat, map); 836 bus_dmamem_unmap(dmat, kvap, map_size); 837 bus_dmamem_free(dmat, seg, 1); 838 } 839 #endif 840 841 static int 842 bcmeth_dmamem_alloc( 843 bus_dma_tag_t dmat, 844 size_t map_size, 845 bus_dma_segment_t *seg, 846 bus_dmamap_t *map, 847 void **kvap) 848 { 849 int error; 850 int nseg; 851 852 *kvap = NULL; 853 *map = NULL; 854 855 error = bus_dmamem_alloc(dmat, map_size, 2*PAGE_SIZE, 0, 856 seg, 1, &nseg, 0); 857 if (error) 858 return error; 859 860 KASSERT(nseg == 1); 861 862 error = bus_dmamem_map(dmat, seg, nseg, map_size, (void **)kvap, 0); 863 if (error == 0) { 864 error = bus_dmamap_create(dmat, map_size, 1, map_size, 0, 0, 865 map); 866 if (error == 0) { 867 error = bus_dmamap_load(dmat, *map, *kvap, map_size, 868 NULL, 0); 869 if (error == 0) 870 return 0; 871 bus_dmamap_destroy(dmat, *map); 872 *map = NULL; 873 } 874 bus_dmamem_unmap(dmat, *kvap, map_size); 875 *kvap = NULL; 876 } 877 bus_dmamem_free(dmat, seg, nseg); 878 return 0; 879 } 880 881 static struct mbuf * 882 bcmeth_rx_buf_alloc( 883 struct bcmeth_softc *sc) 884 { 885 struct mbuf *m = m_gethdr(M_DONTWAIT, MT_DATA); 886 if (m == NULL) { 887 printf("%s:%d: %s\n", __func__, __LINE__, "m_gethdr"); 888 return NULL; 889 } 890 MCLGET(m, M_DONTWAIT); 891 if ((m->m_flags & M_EXT) == 0) { 892 printf("%s:%d: %s\n", __func__, __LINE__, "MCLGET"); 893 m_freem(m); 894 return NULL; 895 } 896 m->m_len = m->m_pkthdr.len = m->m_ext.ext_size; 897 898 bus_dmamap_t map = bcmeth_mapcache_get(sc, sc->sc_rx_mapcache); 899 if (map == NULL) { 900 printf("%s:%d: %s\n", __func__, __LINE__, "map get"); 901 m_freem(m); 902 return NULL; 903 } 904 M_SETCTX(m, map); 905 m->m_len = m->m_pkthdr.len = MCLBYTES; 906 int error = bus_dmamap_load_mbuf(sc->sc_dmat, map, m, 907 BUS_DMA_READ | BUS_DMA_NOWAIT); 908 if (error) { 909 aprint_error_dev(sc->sc_dev, "fail to load rx dmamap: %d\n", 910 error); 911 M_SETCTX(m, NULL); 912 m_freem(m); 913 bcmeth_mapcache_put(sc, sc->sc_rx_mapcache, map); 914 return NULL; 915 } 916 KASSERT(map->dm_mapsize == MCLBYTES); 917 #ifdef BCMETH_RCVMAGIC 918 *mtod(m, uint32_t *) = htole32(BCMETH_RCVMAGIC); 919 bus_dmamap_sync(sc->sc_dmat, map, 0, sizeof(uint32_t), 920 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); 921 bus_dmamap_sync(sc->sc_dmat, map, sizeof(uint32_t), 922 map->dm_mapsize - sizeof(uint32_t), BUS_DMASYNC_PREREAD); 923 #else 924 bus_dmamap_sync(sc->sc_dmat, map, 0, map->dm_mapsize, 925 BUS_DMASYNC_PREREAD); 926 #endif 927 928 return m; 929 } 930 931 static void 932 bcmeth_rx_map_unload( 933 struct bcmeth_softc *sc, 934 struct mbuf *m) 935 { 936 KASSERT(m); 937 for (; m != NULL; m = m->m_next) { 938 bus_dmamap_t map = M_GETCTX(m, bus_dmamap_t); 939 KASSERT(map); 940 KASSERT(map->dm_mapsize == MCLBYTES); 941 bus_dmamap_sync(sc->sc_dmat, map, 0, m->m_len, 942 BUS_DMASYNC_POSTREAD); 943 bus_dmamap_unload(sc->sc_dmat, map); 944 bcmeth_mapcache_put(sc, sc->sc_rx_mapcache, map); 945 M_SETCTX(m, NULL); 946 } 947 } 948 949 static bool 950 bcmeth_rxq_produce( 951 struct bcmeth_softc *sc, 952 struct bcmeth_rxqueue *rxq) 953 { 954 struct gmac_rxdb *producer = rxq->rxq_producer; 955 bool produced = false; 956 957 while (rxq->rxq_inuse < rxq->rxq_threshold) { 958 struct mbuf *m; 959 IF_DEQUEUE(&sc->sc_rx_bufcache, m); 960 if (m == NULL) { 961 m = bcmeth_rx_buf_alloc(sc); 962 if (m == NULL) { 963 printf("%s: bcmeth_rx_buf_alloc failed\n", 964 __func__); 965 break; 966 } 967 } 968 bus_dmamap_t map = M_GETCTX(m, bus_dmamap_t); 969 KASSERT(map); 970 971 producer->rxdb_buflen = htole32(MCLBYTES); 972 producer->rxdb_addrlo = htole32(map->dm_segs[0].ds_addr); 973 producer->rxdb_flags &= htole32(RXDB_FLAG_ET); 974 *rxq->rxq_mtail = m; 975 rxq->rxq_mtail = &m->m_next; 976 m->m_len = MCLBYTES; 977 m->m_next = NULL; 978 rxq->rxq_inuse++; 979 if (++producer == rxq->rxq_last) { 980 membar_producer(); 981 bcmeth_rxq_desc_presync(sc, rxq, rxq->rxq_producer, 982 rxq->rxq_last - rxq->rxq_producer); 983 producer = rxq->rxq_producer = rxq->rxq_first; 984 } 985 produced = true; 986 } 987 if (produced) { 988 membar_producer(); 989 if (producer != rxq->rxq_producer) { 990 bcmeth_rxq_desc_presync(sc, rxq, rxq->rxq_producer, 991 producer - rxq->rxq_producer); 992 rxq->rxq_producer = producer; 993 } 994 bcmeth_write_4(sc, rxq->rxq_reg_rcvptr, 995 rxq->rxq_descmap->dm_segs[0].ds_addr 996 + ((uintptr_t)producer & RCVPTR)); 997 } 998 return true; 999 } 1000 1001 static void 1002 bcmeth_rx_input( 1003 struct bcmeth_softc *sc, 1004 struct mbuf *m, 1005 uint32_t rxdb_flags) 1006 { 1007 struct ifnet * const ifp = &sc->sc_if; 1008 1009 bcmeth_rx_map_unload(sc, m); 1010 1011 m_adj(m, sc->sc_rcvoffset); 1012 1013 /* 1014 * If we are in promiscuous mode and this isn't a multicast, check the 1015 * destination address to make sure it matches our own. If it doesn't, 1016 * mark the packet as being received promiscuously. 1017 */ 1018 if ((sc->sc_cmdcfg & PROMISC_EN) 1019 && (m->m_data[0] & 1) == 0 1020 && (*(uint16_t *)&m->m_data[0] != sc->sc_macaddr[0] 1021 || *(uint32_t *)&m->m_data[2] != sc->sc_macaddr[1])) { 1022 m->m_flags |= M_PROMISC; 1023 } 1024 m_set_rcvif(m, ifp); 1025 1026 /* 1027 * Let's give it to the network subsystm to deal with. 1028 */ 1029 #ifdef BCMETH_MPSAFE 1030 mutex_exit(sc->sc_lock); 1031 if_input(ifp, m); 1032 mutex_enter(sc->sc_lock); 1033 #else 1034 int s = splnet(); 1035 if_input(ifp, m); 1036 splx(s); 1037 #endif 1038 } 1039 1040 static bool 1041 bcmeth_rxq_consume( 1042 struct bcmeth_softc *sc, 1043 struct bcmeth_rxqueue *rxq, 1044 size_t atmost) 1045 { 1046 struct ifnet * const ifp = &sc->sc_if; 1047 struct gmac_rxdb *consumer = rxq->rxq_consumer; 1048 size_t rxconsumed = 0; 1049 bool didconsume = false; 1050 1051 while (atmost-- > 0) { 1052 if (consumer == rxq->rxq_producer) { 1053 KASSERT(rxq->rxq_inuse == 0); 1054 break; 1055 } 1056 1057 uint32_t rcvsts0 = bcmeth_read_4(sc, rxq->rxq_reg_rcvsts0); 1058 uint32_t currdscr = __SHIFTOUT(rcvsts0, RCV_CURRDSCR); 1059 if (consumer == rxq->rxq_first + currdscr) { 1060 break; 1061 } 1062 bcmeth_rxq_desc_postsync(sc, rxq, consumer, 1); 1063 1064 /* 1065 * We own this packet again. Copy the rxsts word from it. 1066 */ 1067 rxconsumed++; 1068 didconsume = true; 1069 uint32_t rxsts; 1070 KASSERT(rxq->rxq_mhead != NULL); 1071 bus_dmamap_t map = M_GETCTX(rxq->rxq_mhead, bus_dmamap_t); 1072 bus_dmamap_sync(sc->sc_dmat, map, 0, arm_dcache_align, 1073 BUS_DMASYNC_POSTREAD); 1074 memcpy(&rxsts, rxq->rxq_mhead->m_data, 4); 1075 rxsts = le32toh(rxsts); 1076 #if 0 1077 KASSERTMSG(rxsts != BCMETH_RCVMAGIC, "currdscr=%u consumer=%zd", 1078 currdscr, consumer - rxq->rxq_first); 1079 #endif 1080 1081 /* 1082 * Get the count of descriptors. Fetch the correct number 1083 * of mbufs. 1084 */ 1085 #ifdef BCMETH_RCVMAGIC 1086 size_t desc_count = rxsts != BCMETH_RCVMAGIC 1087 ? __SHIFTOUT(rxsts, RXSTS_DESC_COUNT) + 1 : 1; 1088 #else 1089 size_t desc_count = __SHIFTOUT(rxsts, RXSTS_DESC_COUNT) + 1; 1090 #endif 1091 struct mbuf *m = rxq->rxq_mhead; 1092 struct mbuf *m_last = m; 1093 for (size_t i = 1; i < desc_count; i++) { 1094 if (++consumer == rxq->rxq_last) { 1095 consumer = rxq->rxq_first; 1096 } 1097 KASSERTMSG(consumer != rxq->rxq_first + currdscr, 1098 "i=%zu rxsts=%#x desc_count=%zu currdscr=%u " 1099 "consumer=%zd", i, rxsts, desc_count, currdscr, 1100 consumer - rxq->rxq_first); 1101 m_last = m_last->m_next; 1102 } 1103 1104 /* 1105 * Now remove it/them from the list of enqueued mbufs. 1106 */ 1107 if ((rxq->rxq_mhead = m_last->m_next) == NULL) 1108 rxq->rxq_mtail = &rxq->rxq_mhead; 1109 m_last->m_next = NULL; 1110 1111 #ifdef BCMETH_RCVMAGIC 1112 if (rxsts == BCMETH_RCVMAGIC) { 1113 if_statinc(ifp, if_ierrors); 1114 if ((m->m_ext.ext_paddr >> 28) == 8) { 1115 BCMETH_EVCNT_INCR(sc->sc_ev_rx_badmagic_lo); 1116 } else { 1117 BCMETH_EVCNT_INCR( sc->sc_ev_rx_badmagic_hi); 1118 } 1119 IF_ENQUEUE(&sc->sc_rx_bufcache, m); 1120 } else 1121 #endif /* BCMETH_RCVMAGIC */ 1122 if (rxsts 1123 & (RXSTS_CRC_ERROR |RXSTS_OVERSIZED |RXSTS_PKT_OVERFLOW)) { 1124 aprint_error_dev(sc->sc_dev, 1125 "[%zu]: count=%zu rxsts=%#x\n", 1126 consumer - rxq->rxq_first, desc_count, rxsts); 1127 /* 1128 * We encountered an error, take the mbufs and add them 1129 * to the rx bufcache so we can quickly reuse them. 1130 */ 1131 if_statinc(ifp, if_ierrors); 1132 do { 1133 struct mbuf *m0 = m->m_next; 1134 m->m_next = NULL; 1135 IF_ENQUEUE(&sc->sc_rx_bufcache, m); 1136 m = m0; 1137 } while (m); 1138 } else { 1139 uint32_t framelen = __SHIFTOUT(rxsts, RXSTS_FRAMELEN); 1140 framelen += sc->sc_rcvoffset; 1141 m->m_pkthdr.len = framelen; 1142 if (desc_count == 1) { 1143 KASSERT(framelen <= MCLBYTES); 1144 m->m_len = framelen; 1145 } else { 1146 m_last->m_len = framelen & (MCLBYTES - 1); 1147 } 1148 1149 #ifdef BCMETH_MPSAFE 1150 /* 1151 * Wrap at the last entry! 1152 */ 1153 if (++consumer == rxq->rxq_last) { 1154 KASSERT(consumer[-1].rxdb_flags 1155 & htole32(RXDB_FLAG_ET)); 1156 rxq->rxq_consumer = rxq->rxq_first; 1157 } else { 1158 rxq->rxq_consumer = consumer; 1159 } 1160 rxq->rxq_inuse -= rxconsumed; 1161 #endif /* BCMETH_MPSAFE */ 1162 1163 /* 1164 * Receive the packet (which releases our lock) 1165 */ 1166 bcmeth_rx_input(sc, m, rxsts); 1167 1168 #ifdef BCMETH_MPSAFE 1169 /* 1170 * Since we had to give up our lock, we need to 1171 * refresh these. 1172 */ 1173 consumer = rxq->rxq_consumer; 1174 rxconsumed = 0; 1175 continue; 1176 #endif /* BCMETH_MPSAFE */ 1177 } 1178 1179 /* 1180 * Wrap at the last entry! 1181 */ 1182 if (++consumer == rxq->rxq_last) { 1183 KASSERT(consumer[-1].rxdb_flags & htole32(RXDB_FLAG_ET)); 1184 consumer = rxq->rxq_first; 1185 } 1186 } 1187 1188 /* 1189 * Update queue info. 1190 */ 1191 rxq->rxq_consumer = consumer; 1192 rxq->rxq_inuse -= rxconsumed; 1193 1194 /* 1195 * Did we consume anything? 1196 */ 1197 return didconsume; 1198 } 1199 1200 static void 1201 bcmeth_rxq_purge( 1202 struct bcmeth_softc *sc, 1203 struct bcmeth_rxqueue *rxq, 1204 bool discard) 1205 { 1206 struct mbuf *m; 1207 1208 if ((m = rxq->rxq_mhead) != NULL) { 1209 if (discard) { 1210 bcmeth_rx_map_unload(sc, m); 1211 m_freem(m); 1212 } else { 1213 while (m != NULL) { 1214 struct mbuf *m0 = m->m_next; 1215 m->m_next = NULL; 1216 IF_ENQUEUE(&sc->sc_rx_bufcache, m); 1217 m = m0; 1218 } 1219 } 1220 } 1221 1222 rxq->rxq_mhead = NULL; 1223 rxq->rxq_mtail = &rxq->rxq_mhead; 1224 rxq->rxq_inuse = 0; 1225 } 1226 1227 static void 1228 bcmeth_rxq_reset( 1229 struct bcmeth_softc *sc, 1230 struct bcmeth_rxqueue *rxq) 1231 { 1232 /* 1233 * sync all the descriptors 1234 */ 1235 bcmeth_rxq_desc_postsync(sc, rxq, rxq->rxq_first, 1236 rxq->rxq_last - rxq->rxq_first); 1237 1238 /* 1239 * Make sure we own all descriptors in the ring. 1240 */ 1241 struct gmac_rxdb *rxdb; 1242 for (rxdb = rxq->rxq_first; rxdb < rxq->rxq_last - 1; rxdb++) { 1243 rxdb->rxdb_flags = htole32(RXDB_FLAG_IC); 1244 } 1245 1246 /* 1247 * Last descriptor has the wrap flag. 1248 */ 1249 rxdb->rxdb_flags = htole32(RXDB_FLAG_ET | RXDB_FLAG_IC); 1250 1251 /* 1252 * Reset the producer consumer indexes. 1253 */ 1254 rxq->rxq_consumer = rxq->rxq_first; 1255 rxq->rxq_producer = rxq->rxq_first; 1256 rxq->rxq_inuse = 0; 1257 if (rxq->rxq_threshold < BCMETH_MINRXMBUFS) 1258 rxq->rxq_threshold = BCMETH_MINRXMBUFS; 1259 1260 sc->sc_intmask |= RCVINT | RCVFIFOOF | RCVDESCUF; 1261 1262 /* 1263 * Restart the receiver at the first descriptor 1264 */ 1265 bcmeth_write_4(sc, rxq->rxq_reg_rcvaddrlo, 1266 rxq->rxq_descmap->dm_segs[0].ds_addr); 1267 } 1268 1269 static int 1270 bcmeth_rxq_attach( 1271 struct bcmeth_softc *sc, 1272 struct bcmeth_rxqueue *rxq, 1273 u_int qno) 1274 { 1275 size_t desc_count = BCMETH_RINGSIZE / sizeof(rxq->rxq_first[0]); 1276 int error; 1277 void *descs; 1278 1279 KASSERT(desc_count == 256 || desc_count == 512); 1280 1281 error = bcmeth_dmamem_alloc(sc->sc_dmat, BCMETH_RINGSIZE, 1282 &rxq->rxq_descmap_seg, &rxq->rxq_descmap, &descs); 1283 if (error) 1284 return error; 1285 1286 memset(descs, 0, BCMETH_RINGSIZE); 1287 rxq->rxq_first = descs; 1288 rxq->rxq_last = rxq->rxq_first + desc_count; 1289 rxq->rxq_consumer = descs; 1290 rxq->rxq_producer = descs; 1291 1292 bcmeth_rxq_purge(sc, rxq, true); 1293 bcmeth_rxq_reset(sc, rxq); 1294 1295 rxq->rxq_reg_rcvaddrlo = GMAC_RCVADDR_LOW; 1296 rxq->rxq_reg_rcvctl = GMAC_RCVCONTROL; 1297 rxq->rxq_reg_rcvptr = GMAC_RCVPTR; 1298 rxq->rxq_reg_rcvsts0 = GMAC_RCVSTATUS0; 1299 rxq->rxq_reg_rcvsts1 = GMAC_RCVSTATUS1; 1300 1301 return 0; 1302 } 1303 1304 static bool 1305 bcmeth_txq_active_p( 1306 struct bcmeth_softc * const sc, 1307 struct bcmeth_txqueue *txq) 1308 { 1309 return !IF_IS_EMPTY(&txq->txq_mbufs); 1310 } 1311 1312 static bool 1313 bcmeth_txq_fillable_p( 1314 struct bcmeth_softc * const sc, 1315 struct bcmeth_txqueue *txq) 1316 { 1317 return txq->txq_free >= txq->txq_threshold; 1318 } 1319 1320 static int 1321 bcmeth_txq_attach( 1322 struct bcmeth_softc *sc, 1323 struct bcmeth_txqueue *txq, 1324 u_int qno) 1325 { 1326 size_t desc_count = BCMETH_RINGSIZE / sizeof(txq->txq_first[0]); 1327 int error; 1328 void *descs; 1329 1330 KASSERT(desc_count == 256 || desc_count == 512); 1331 1332 error = bcmeth_dmamem_alloc(sc->sc_dmat, BCMETH_RINGSIZE, 1333 &txq->txq_descmap_seg, &txq->txq_descmap, &descs); 1334 if (error) 1335 return error; 1336 1337 memset(descs, 0, BCMETH_RINGSIZE); 1338 txq->txq_first = descs; 1339 txq->txq_last = txq->txq_first + desc_count; 1340 txq->txq_consumer = descs; 1341 txq->txq_producer = descs; 1342 1343 IFQ_SET_MAXLEN(&txq->txq_mbufs, BCMETH_MAXTXMBUFS); 1344 1345 txq->txq_reg_xmtaddrlo = GMAC_XMTADDR_LOW; 1346 txq->txq_reg_xmtctl = GMAC_XMTCONTROL; 1347 txq->txq_reg_xmtptr = GMAC_XMTPTR; 1348 txq->txq_reg_xmtsts0 = GMAC_XMTSTATUS0; 1349 txq->txq_reg_xmtsts1 = GMAC_XMTSTATUS1; 1350 1351 bcmeth_txq_reset(sc, txq); 1352 1353 return 0; 1354 } 1355 1356 static int 1357 bcmeth_txq_map_load( 1358 struct bcmeth_softc *sc, 1359 struct bcmeth_txqueue *txq, 1360 struct mbuf *m) 1361 { 1362 bus_dmamap_t map; 1363 int error; 1364 1365 map = M_GETCTX(m, bus_dmamap_t); 1366 if (map != NULL) 1367 return 0; 1368 1369 map = bcmeth_mapcache_get(sc, sc->sc_tx_mapcache); 1370 if (map == NULL) 1371 return ENOMEM; 1372 1373 error = bus_dmamap_load_mbuf(sc->sc_dmat, map, m, 1374 BUS_DMA_WRITE | BUS_DMA_NOWAIT); 1375 if (error) 1376 return error; 1377 1378 bus_dmamap_sync(sc->sc_dmat, map, 0, m->m_pkthdr.len, 1379 BUS_DMASYNC_PREWRITE); 1380 M_SETCTX(m, map); 1381 return 0; 1382 } 1383 1384 static void 1385 bcmeth_txq_map_unload( 1386 struct bcmeth_softc *sc, 1387 struct bcmeth_txqueue *txq, 1388 struct mbuf *m) 1389 { 1390 KASSERT(m); 1391 bus_dmamap_t map = M_GETCTX(m, bus_dmamap_t); 1392 bus_dmamap_sync(sc->sc_dmat, map, 0, map->dm_mapsize, 1393 BUS_DMASYNC_POSTWRITE); 1394 bus_dmamap_unload(sc->sc_dmat, map); 1395 bcmeth_mapcache_put(sc, sc->sc_tx_mapcache, map); 1396 } 1397 1398 static bool 1399 bcmeth_txq_produce( 1400 struct bcmeth_softc *sc, 1401 struct bcmeth_txqueue *txq, 1402 struct mbuf *m) 1403 { 1404 bus_dmamap_t map = M_GETCTX(m, bus_dmamap_t); 1405 1406 if (map->dm_nsegs > txq->txq_free) 1407 return false; 1408 1409 /* 1410 * TCP Offload flag must be set in the first descriptor. 1411 */ 1412 struct gmac_txdb *producer = txq->txq_producer; 1413 uint32_t first_flags = TXDB_FLAG_SF; 1414 uint32_t last_flags = TXDB_FLAG_EF; 1415 1416 /* 1417 * If we've produced enough descriptors without consuming any 1418 * we need to ask for an interrupt to reclaim some. 1419 */ 1420 txq->txq_lastintr += map->dm_nsegs; 1421 if (txq->txq_lastintr >= txq->txq_threshold 1422 || txq->txq_mbufs.ifq_len + 1 == txq->txq_mbufs.ifq_maxlen) { 1423 txq->txq_lastintr = 0; 1424 last_flags |= TXDB_FLAG_IC; 1425 } 1426 1427 KASSERT(producer != txq->txq_last); 1428 1429 struct gmac_txdb *start = producer; 1430 size_t count = map->dm_nsegs; 1431 producer->txdb_flags |= htole32(first_flags); 1432 producer->txdb_addrlo = htole32(map->dm_segs[0].ds_addr); 1433 producer->txdb_buflen = htole32(map->dm_segs[0].ds_len); 1434 for (u_int i = 1; i < map->dm_nsegs; i++) { 1435 #if 0 1436 printf("[%zu]: %#x/%#x/%#x/%#x\n", producer - txq->txq_first, 1437 le32toh(producer->txdb_flags), 1438 le32toh(producer->txdb_buflen), 1439 le32toh(producer->txdb_addrlo), 1440 le32toh(producer->txdb_addrhi)); 1441 #endif 1442 if (__predict_false(++producer == txq->txq_last)) { 1443 bcmeth_txq_desc_presync(sc, txq, start, 1444 txq->txq_last - start); 1445 count -= txq->txq_last - start; 1446 producer = txq->txq_first; 1447 start = txq->txq_first; 1448 } 1449 producer->txdb_addrlo = htole32(map->dm_segs[i].ds_addr); 1450 producer->txdb_buflen = htole32(map->dm_segs[i].ds_len); 1451 } 1452 producer->txdb_flags |= htole32(last_flags); 1453 #if 0 1454 printf("[%zu]: %#x/%#x/%#x/%#x\n", producer - txq->txq_first, 1455 le32toh(producer->txdb_flags), le32toh(producer->txdb_buflen), 1456 le32toh(producer->txdb_addrlo), le32toh(producer->txdb_addrhi)); 1457 #endif 1458 if (count) 1459 bcmeth_txq_desc_presync(sc, txq, start, count); 1460 1461 /* 1462 * Reduce free count by the number of segments we consumed. 1463 */ 1464 txq->txq_free -= map->dm_nsegs; 1465 KASSERT(map->dm_nsegs == 1 || txq->txq_producer != producer); 1466 KASSERT(map->dm_nsegs == 1 1467 || (txq->txq_producer->txdb_flags & htole32(TXDB_FLAG_EF)) == 0); 1468 KASSERT(producer->txdb_flags & htole32(TXDB_FLAG_EF)); 1469 1470 #if 0 1471 printf("%s: mbuf %p: produced a %u byte packet in %u segments " 1472 "(%zd..%zd)\n", __func__, m, m->m_pkthdr.len, map->dm_nsegs, 1473 txq->txq_producer - txq->txq_first, producer - txq->txq_first); 1474 #endif 1475 1476 if (producer + 1 == txq->txq_last) 1477 txq->txq_producer = txq->txq_first; 1478 else 1479 txq->txq_producer = producer + 1; 1480 IF_ENQUEUE(&txq->txq_mbufs, m); 1481 1482 /* 1483 * Let the transmitter know there's more to do 1484 */ 1485 bcmeth_write_4(sc, txq->txq_reg_xmtptr, 1486 txq->txq_descmap->dm_segs[0].ds_addr 1487 + ((uintptr_t)txq->txq_producer & XMT_LASTDSCR)); 1488 1489 return true; 1490 } 1491 1492 static struct mbuf * 1493 bcmeth_copy_packet(struct mbuf *m) 1494 { 1495 struct mbuf *mext = NULL; 1496 size_t misalignment = 0; 1497 size_t hlen = 0; 1498 1499 for (mext = m; mext != NULL; mext = mext->m_next) { 1500 if (mext->m_flags & M_EXT) { 1501 misalignment = mtod(mext, vaddr_t) & arm_dcache_align; 1502 break; 1503 } 1504 hlen += m->m_len; 1505 } 1506 1507 struct mbuf *n = m->m_next; 1508 if (m != mext && hlen + misalignment <= MHLEN && false) { 1509 KASSERT(m->m_pktdat <= m->m_data 1510 && m->m_data <= &m->m_pktdat[MHLEN - m->m_len]); 1511 size_t oldoff = m->m_data - m->m_pktdat; 1512 size_t off; 1513 if (mext == NULL) { 1514 off = (oldoff + hlen > MHLEN) ? 0 : oldoff; 1515 } else { 1516 off = MHLEN - (hlen + misalignment); 1517 } 1518 KASSERT(off + hlen + misalignment <= MHLEN); 1519 if (((oldoff ^ off) & arm_dcache_align) != 0 || off < oldoff) { 1520 memmove(&m->m_pktdat[off], m->m_data, m->m_len); 1521 m->m_data = &m->m_pktdat[off]; 1522 } 1523 m_copydata(n, 0, hlen - m->m_len, &m->m_data[m->m_len]); 1524 m->m_len = hlen; 1525 m->m_next = mext; 1526 while (n != mext) { 1527 n = m_free(n); 1528 } 1529 return m; 1530 } 1531 1532 struct mbuf *m0 = m_gethdr(M_DONTWAIT, m->m_type); 1533 if (m0 == NULL) { 1534 return NULL; 1535 } 1536 m_copy_pkthdr(m0, m); 1537 MCLAIM(m0, m->m_owner); 1538 if (m0->m_pkthdr.len > MHLEN) { 1539 MCLGET(m0, M_DONTWAIT); 1540 if ((m0->m_flags & M_EXT) == 0) { 1541 m_freem(m0); 1542 return NULL; 1543 } 1544 } 1545 m0->m_len = m->m_pkthdr.len; 1546 m_copydata(m, 0, m0->m_len, mtod(m0, void *)); 1547 m_freem(m); 1548 return m0; 1549 } 1550 1551 static bool 1552 bcmeth_txq_enqueue( 1553 struct bcmeth_softc *sc, 1554 struct bcmeth_txqueue *txq) 1555 { 1556 for (;;) { 1557 if (IF_QFULL(&txq->txq_mbufs)) 1558 return false; 1559 struct mbuf *m = txq->txq_next; 1560 if (m == NULL) { 1561 int s = splnet(); 1562 IF_DEQUEUE(&sc->sc_if.if_snd, m); 1563 splx(s); 1564 if (m == NULL) 1565 return true; 1566 M_SETCTX(m, NULL); 1567 } else { 1568 txq->txq_next = NULL; 1569 } 1570 /* 1571 * If LINK2 is set and this packet uses multiple mbufs, 1572 * consolidate it into a single mbuf. 1573 */ 1574 if (m->m_next != NULL && (sc->sc_if.if_flags & IFF_LINK2)) { 1575 struct mbuf *m0 = bcmeth_copy_packet(m); 1576 if (m0 == NULL) { 1577 txq->txq_next = m; 1578 return true; 1579 } 1580 m = m0; 1581 } 1582 int error = bcmeth_txq_map_load(sc, txq, m); 1583 if (error) { 1584 aprint_error_dev(sc->sc_dev, 1585 "discarded packet due to " 1586 "dmamap load failure: %d\n", error); 1587 m_freem(m); 1588 continue; 1589 } 1590 KASSERT(txq->txq_next == NULL); 1591 if (!bcmeth_txq_produce(sc, txq, m)) { 1592 txq->txq_next = m; 1593 return false; 1594 } 1595 KASSERT(txq->txq_next == NULL); 1596 } 1597 } 1598 1599 static bool 1600 bcmeth_txq_consume( 1601 struct bcmeth_softc *sc, 1602 struct bcmeth_txqueue *txq) 1603 { 1604 struct ifnet * const ifp = &sc->sc_if; 1605 struct gmac_txdb *consumer = txq->txq_consumer; 1606 size_t txfree = 0; 1607 1608 #if 0 1609 printf("%s: entry: free=%zu\n", __func__, txq->txq_free); 1610 #endif 1611 1612 for (;;) { 1613 if (consumer == txq->txq_producer) { 1614 txq->txq_consumer = consumer; 1615 txq->txq_free += txfree; 1616 txq->txq_lastintr -= uimin(txq->txq_lastintr, txfree); 1617 #if 0 1618 printf("%s: empty: freed %zu descriptors going from " 1619 "%zu to %zu\n", __func__, txfree, 1620 txq->txq_free - txfree, txq->txq_free); 1621 #endif 1622 KASSERT(txq->txq_lastintr == 0); 1623 KASSERT(txq->txq_free 1624 == txq->txq_last - txq->txq_first - 1); 1625 return true; 1626 } 1627 bcmeth_txq_desc_postsync(sc, txq, consumer, 1); 1628 uint32_t s0 = bcmeth_read_4(sc, txq->txq_reg_xmtsts0); 1629 if (consumer == txq->txq_first + __SHIFTOUT(s0, XMT_CURRDSCR)) { 1630 txq->txq_consumer = consumer; 1631 txq->txq_free += txfree; 1632 txq->txq_lastintr -= uimin(txq->txq_lastintr, txfree); 1633 #if 0 1634 printf("%s: freed %zu descriptors\n", 1635 __func__, txfree); 1636 #endif 1637 return bcmeth_txq_fillable_p(sc, txq); 1638 } 1639 1640 /* 1641 * If this is the last descriptor in the chain, get the 1642 * mbuf, free its dmamap, and free the mbuf chain itself. 1643 */ 1644 const uint32_t txdb_flags = le32toh(consumer->txdb_flags); 1645 if (txdb_flags & TXDB_FLAG_EF) { 1646 struct mbuf *m; 1647 1648 IF_DEQUEUE(&txq->txq_mbufs, m); 1649 KASSERT(m); 1650 bcmeth_txq_map_unload(sc, txq, m); 1651 #if 0 1652 printf("%s: mbuf %p: consumed a %u byte packet\n", 1653 __func__, m, m->m_pkthdr.len); 1654 #endif 1655 bpf_mtap(ifp, m, BPF_D_OUT); 1656 if_statinc(ifp, if_opackets); 1657 if_statadd(ifp, if_obytes, m->m_pkthdr.len); 1658 if (m->m_flags & M_MCAST) 1659 if_statinc(ifp, if_omcasts); 1660 m_freem(m); 1661 } 1662 1663 /* 1664 * We own this packet again. Clear all flags except wrap. 1665 */ 1666 txfree++; 1667 1668 /* 1669 * Wrap at the last entry! 1670 */ 1671 if (txdb_flags & TXDB_FLAG_ET) { 1672 consumer->txdb_flags = htole32(TXDB_FLAG_ET); 1673 KASSERT(consumer + 1 == txq->txq_last); 1674 consumer = txq->txq_first; 1675 } else { 1676 consumer->txdb_flags = 0; 1677 consumer++; 1678 KASSERT(consumer < txq->txq_last); 1679 } 1680 } 1681 } 1682 1683 static void 1684 bcmeth_txq_purge( 1685 struct bcmeth_softc *sc, 1686 struct bcmeth_txqueue *txq) 1687 { 1688 struct mbuf *m; 1689 KASSERT((bcmeth_read_4(sc, UNIMAC_COMMAND_CONFIG) & TX_ENA) == 0); 1690 1691 for (;;) { 1692 IF_DEQUEUE(&txq->txq_mbufs, m); 1693 if (m == NULL) 1694 break; 1695 bcmeth_txq_map_unload(sc, txq, m); 1696 m_freem(m); 1697 } 1698 if ((m = txq->txq_next) != NULL) { 1699 txq->txq_next = NULL; 1700 bcmeth_txq_map_unload(sc, txq, m); 1701 m_freem(m); 1702 } 1703 } 1704 1705 static void 1706 bcmeth_txq_reset( 1707 struct bcmeth_softc *sc, 1708 struct bcmeth_txqueue *txq) 1709 { 1710 /* 1711 * sync all the descriptors 1712 */ 1713 bcmeth_txq_desc_postsync(sc, txq, txq->txq_first, 1714 txq->txq_last - txq->txq_first); 1715 1716 /* 1717 * Make sure we own all descriptors in the ring. 1718 */ 1719 struct gmac_txdb *txdb; 1720 for (txdb = txq->txq_first; txdb < txq->txq_last - 1; txdb++) { 1721 txdb->txdb_flags = 0; 1722 } 1723 1724 /* 1725 * Last descriptor has the wrap flag. 1726 */ 1727 txdb->txdb_flags = htole32(TXDB_FLAG_ET); 1728 1729 /* 1730 * Reset the producer consumer indexes. 1731 */ 1732 txq->txq_consumer = txq->txq_first; 1733 txq->txq_producer = txq->txq_first; 1734 txq->txq_free = txq->txq_last - txq->txq_first - 1; 1735 txq->txq_threshold = txq->txq_free / 2; 1736 txq->txq_lastintr = 0; 1737 1738 /* 1739 * What do we want to get interrupted on? 1740 */ 1741 sc->sc_intmask |= XMTINT_0 | XMTUF; 1742 1743 /* 1744 * Restart the transmiter at the first descriptor 1745 */ 1746 bcmeth_write_4(sc, txq->txq_reg_xmtaddrlo, 1747 txq->txq_descmap->dm_segs->ds_addr); 1748 } 1749 1750 static void 1751 bcmeth_ifstart(struct ifnet *ifp) 1752 { 1753 struct bcmeth_softc * const sc = ifp->if_softc; 1754 1755 if (__predict_false((ifp->if_flags & IFF_RUNNING) == 0)) { 1756 return; 1757 } 1758 1759 #ifdef BCMETH_MPSAFETX 1760 if (cpu_intr_p()) { 1761 #endif 1762 atomic_or_uint(&sc->sc_soft_flags, SOFT_TXINTR); 1763 softint_schedule(sc->sc_soft_ih); 1764 #ifdef BCMETH_MPSAFETX 1765 } else { 1766 /* 1767 * Either we are in a softintr thread already or some other 1768 * thread so just borrow it to do the send and save ourselves 1769 * the overhead of a fast soft int. 1770 */ 1771 bcmeth_soft_txintr(sc); 1772 } 1773 #endif 1774 } 1775 1776 int 1777 bcmeth_intr(void *arg) 1778 { 1779 struct bcmeth_softc * const sc = arg; 1780 uint32_t soft_flags = 0; 1781 uint32_t work_flags = 0; 1782 int rv = 0; 1783 1784 mutex_enter(sc->sc_hwlock); 1785 1786 uint32_t intmask = sc->sc_intmask; 1787 BCMETH_EVCNT_INCR(sc->sc_ev_intr); 1788 1789 for (;;) { 1790 uint32_t intstatus = bcmeth_read_4(sc, GMAC_INTSTATUS); 1791 intstatus &= intmask; 1792 bcmeth_write_4(sc, GMAC_INTSTATUS, intstatus); /* write 1 to clear */ 1793 if (intstatus == 0) { 1794 break; 1795 } 1796 #if 0 1797 aprint_normal_dev(sc->sc_dev, "%s: intstatus=%#x intmask=%#x\n", 1798 __func__, intstatus, bcmeth_read_4(sc, GMAC_INTMASK)); 1799 #endif 1800 if (intstatus & RCVINT) { 1801 struct bcmeth_rxqueue * const rxq = &sc->sc_rxq; 1802 intmask &= ~RCVINT; 1803 1804 uint32_t rcvsts0 = bcmeth_read_4(sc, rxq->rxq_reg_rcvsts0); 1805 uint32_t descs = __SHIFTOUT(rcvsts0, RCV_CURRDSCR); 1806 if (descs < rxq->rxq_consumer - rxq->rxq_first) { 1807 /* 1808 * We wrapped at the end so count how far 1809 * we are from the end. 1810 */ 1811 descs += rxq->rxq_last - rxq->rxq_consumer; 1812 } else { 1813 descs -= rxq->rxq_consumer - rxq->rxq_first; 1814 } 1815 /* 1816 * If we "timedout" we can't be hogging so use 1817 * softints. If we exceeded then we might hogging 1818 * so let the workqueue deal with them. 1819 */ 1820 const uint32_t framecount = __SHIFTOUT(sc->sc_rcvlazy, 1821 INTRCVLAZY_FRAMECOUNT); 1822 if (descs < framecount 1823 || (curcpu()->ci_curlwp->l_flag & LW_IDLE)) { 1824 soft_flags |= SOFT_RXINTR; 1825 } else { 1826 work_flags |= WORK_RXINTR; 1827 } 1828 } 1829 1830 if (intstatus & XMTINT_0) { 1831 intmask &= ~XMTINT_0; 1832 soft_flags |= SOFT_TXINTR; 1833 } 1834 1835 if (intstatus & RCVDESCUF) { 1836 intmask &= ~RCVDESCUF; 1837 work_flags |= WORK_RXUNDERFLOW; 1838 } 1839 1840 intstatus &= intmask; 1841 if (intstatus) { 1842 aprint_error_dev(sc->sc_dev, 1843 "intr: intstatus=%#x\n", intstatus); 1844 aprint_error_dev(sc->sc_dev, 1845 "rcvbase=%p/%#lx rcvptr=%#x rcvsts=%#x/%#x\n", 1846 sc->sc_rxq.rxq_first, 1847 sc->sc_rxq.rxq_descmap->dm_segs[0].ds_addr, 1848 bcmeth_read_4(sc, sc->sc_rxq.rxq_reg_rcvptr), 1849 bcmeth_read_4(sc, sc->sc_rxq.rxq_reg_rcvsts0), 1850 bcmeth_read_4(sc, sc->sc_rxq.rxq_reg_rcvsts1)); 1851 aprint_error_dev(sc->sc_dev, 1852 "xmtbase=%p/%#lx xmtptr=%#x xmtsts=%#x/%#x\n", 1853 sc->sc_txq.txq_first, 1854 sc->sc_txq.txq_descmap->dm_segs[0].ds_addr, 1855 bcmeth_read_4(sc, sc->sc_txq.txq_reg_xmtptr), 1856 bcmeth_read_4(sc, sc->sc_txq.txq_reg_xmtsts0), 1857 bcmeth_read_4(sc, sc->sc_txq.txq_reg_xmtsts1)); 1858 intmask &= ~intstatus; 1859 work_flags |= WORK_REINIT; 1860 break; 1861 } 1862 } 1863 1864 if (intmask != sc->sc_intmask) { 1865 bcmeth_write_4(sc, GMAC_INTMASK, sc->sc_intmask); 1866 } 1867 1868 if (work_flags) { 1869 if (sc->sc_work_flags == 0) { 1870 workqueue_enqueue(sc->sc_workq, &sc->sc_work, NULL); 1871 } 1872 atomic_or_32(&sc->sc_work_flags, work_flags); 1873 rv = 1; 1874 } 1875 1876 if (soft_flags) { 1877 if (sc->sc_soft_flags == 0) { 1878 softint_schedule(sc->sc_soft_ih); 1879 } 1880 atomic_or_32(&sc->sc_soft_flags, soft_flags); 1881 rv = 1; 1882 } 1883 1884 mutex_exit(sc->sc_hwlock); 1885 1886 return rv; 1887 } 1888 1889 #ifdef BCMETH_MPSAFETX 1890 void 1891 bcmeth_soft_txintr(struct bcmeth_softc *sc) 1892 { 1893 mutex_enter(sc->sc_lock); 1894 /* 1895 * Let's do what we came here for. Consume transmitted 1896 * packets off the transmit ring. 1897 */ 1898 if (!bcmeth_txq_consume(sc, &sc->sc_txq) 1899 || !bcmeth_txq_enqueue(sc, &sc->sc_txq)) { 1900 BCMETH_EVCNT_INCR(sc->sc_ev_tx_stall); 1901 } 1902 if (sc->sc_if.if_flags & IFF_RUNNING) { 1903 mutex_spin_enter(sc->sc_hwlock); 1904 sc->sc_intmask |= XMTINT_0; 1905 bcmeth_write_4(sc, GMAC_INTMASK, sc->sc_intmask); 1906 mutex_spin_exit(sc->sc_hwlock); 1907 } 1908 mutex_exit(sc->sc_lock); 1909 } 1910 #endif /* BCMETH_MPSAFETX */ 1911 1912 void 1913 bcmeth_soft_intr(void *arg) 1914 { 1915 struct bcmeth_softc * const sc = arg; 1916 struct ifnet * const ifp = &sc->sc_if; 1917 uint32_t intmask = 0; 1918 1919 mutex_enter(sc->sc_lock); 1920 1921 u_int soft_flags = atomic_swap_uint(&sc->sc_soft_flags, 0); 1922 1923 BCMETH_EVCNT_INCR(sc->sc_ev_soft_intr); 1924 1925 if ((soft_flags & SOFT_TXINTR) 1926 || bcmeth_txq_active_p(sc, &sc->sc_txq)) { 1927 /* 1928 * Let's do what we came here for. Consume transmitted 1929 * packets off the transmit ring. 1930 */ 1931 if (!bcmeth_txq_consume(sc, &sc->sc_txq) 1932 || !bcmeth_txq_enqueue(sc, &sc->sc_txq)) { 1933 BCMETH_EVCNT_INCR(sc->sc_ev_tx_stall); 1934 } 1935 intmask |= XMTINT_0; 1936 } 1937 1938 if (soft_flags & SOFT_RXINTR) { 1939 /* 1940 * Let's consume 1941 */ 1942 while (bcmeth_rxq_consume(sc, &sc->sc_rxq, 1943 sc->sc_rxq.rxq_threshold / 4)) { 1944 /* 1945 * We've consumed a quarter of the ring and still have 1946 * more to do. Refill the ring. 1947 */ 1948 bcmeth_rxq_produce(sc, &sc->sc_rxq); 1949 } 1950 intmask |= RCVINT; 1951 } 1952 1953 if (ifp->if_flags & IFF_RUNNING) { 1954 bcmeth_rxq_produce(sc, &sc->sc_rxq); 1955 mutex_spin_enter(sc->sc_hwlock); 1956 sc->sc_intmask |= intmask; 1957 bcmeth_write_4(sc, GMAC_INTMASK, sc->sc_intmask); 1958 mutex_spin_exit(sc->sc_hwlock); 1959 } 1960 1961 mutex_exit(sc->sc_lock); 1962 } 1963 1964 void 1965 bcmeth_worker(struct work *wk, void *arg) 1966 { 1967 struct bcmeth_softc * const sc = arg; 1968 struct ifnet * const ifp = &sc->sc_if; 1969 uint32_t intmask = 0; 1970 1971 mutex_enter(sc->sc_lock); 1972 1973 BCMETH_EVCNT_INCR(sc->sc_ev_work); 1974 1975 uint32_t work_flags = atomic_swap_32(&sc->sc_work_flags, 0); 1976 if (work_flags & WORK_REINIT) { 1977 int s = splnet(); 1978 sc->sc_soft_flags = 0; 1979 bcmeth_ifinit(ifp); 1980 splx(s); 1981 work_flags &= ~WORK_RXUNDERFLOW; 1982 } 1983 1984 if (work_flags & WORK_RXUNDERFLOW) { 1985 struct bcmeth_rxqueue * const rxq = &sc->sc_rxq; 1986 size_t threshold = 5 * rxq->rxq_threshold / 4; 1987 if (threshold >= rxq->rxq_last - rxq->rxq_first) { 1988 threshold = rxq->rxq_last - rxq->rxq_first - 1; 1989 } else { 1990 intmask |= RCVDESCUF; 1991 } 1992 aprint_normal_dev(sc->sc_dev, 1993 "increasing receive buffers from %zu to %zu\n", 1994 rxq->rxq_threshold, threshold); 1995 rxq->rxq_threshold = threshold; 1996 } 1997 1998 if (work_flags & WORK_RXINTR) { 1999 /* 2000 * Let's consume 2001 */ 2002 while (bcmeth_rxq_consume(sc, &sc->sc_rxq, 2003 sc->sc_rxq.rxq_threshold / 4)) { 2004 /* 2005 * We've consumed a quarter of the ring and still have 2006 * more to do. Refill the ring. 2007 */ 2008 bcmeth_rxq_produce(sc, &sc->sc_rxq); 2009 } 2010 intmask |= RCVINT; 2011 } 2012 2013 if (ifp->if_flags & IFF_RUNNING) { 2014 bcmeth_rxq_produce(sc, &sc->sc_rxq); 2015 #if 0 2016 uint32_t intstatus = bcmeth_read_4(sc, GMAC_INTSTATUS); 2017 if (intstatus & RCVINT) { 2018 bcmeth_write_4(sc, GMAC_INTSTATUS, RCVINT); 2019 work_flags |= WORK_RXINTR; 2020 continue; 2021 } 2022 #endif 2023 mutex_spin_enter(sc->sc_hwlock); 2024 sc->sc_intmask |= intmask; 2025 bcmeth_write_4(sc, GMAC_INTMASK, sc->sc_intmask); 2026 mutex_spin_exit(sc->sc_hwlock); 2027 } 2028 2029 mutex_exit(sc->sc_lock); 2030 } 2031
Indexes created Mon Sep 29 21:09:56 GMT 2025