bcm53xx_eth.c revision 1.31.2.2
1 /*- 2 * Copyright (c) 2012 The NetBSD Foundation, Inc. 3 * All rights reserved. 4 * 5 * This code is derived from software contributed to The NetBSD Foundation 6 * by Matt Thomas of 3am Software Foundry. 7 * 8 * Redistribution and use in source and binary forms, with or without 9 * modification, are permitted provided that the following conditions 10 * are met: 11 * 1. Redistributions of source code must retain the above copyright 12 * notice, this list of conditions and the following disclaimer. 13 * 2. Redistributions in binary form must reproduce the above copyright 14 * notice, this list of conditions and the following disclaimer in the 15 * documentation and/or other materials provided with the distribution. 16 * 17 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS 18 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED 19 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 20 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS 21 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 22 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 23 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 24 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 25 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 26 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 27 * POSSIBILITY OF SUCH DAMAGE. 28 */ 29 30 #define _ARM32_BUS_DMA_PRIVATE 31 #define GMAC_PRIVATE 32 33 #include "locators.h" 34 #include "opt_broadcom.h" 35 36 #include <sys/cdefs.h> 37 38 __KERNEL_RCSID(1, "$NetBSD: bcm53xx_eth.c,v 1.31.2.2 2020/04/08 14:07:28 martin Exp $"); 39 40 #include <sys/param.h> 41 #include <sys/atomic.h> 42 #include <sys/bus.h> 43 #include <sys/device.h> 44 #include <sys/ioctl.h> 45 #include <sys/intr.h> 46 #include <sys/kmem.h> 47 #include <sys/mutex.h> 48 #include <sys/socket.h> 49 #include <sys/systm.h> 50 #include <sys/workqueue.h> 51 52 #include <net/if.h> 53 #include <net/if_ether.h> 54 #include <net/if_media.h> 55 #include <net/if_dl.h> 56 #include <net/bpf.h> 57 58 #include <dev/mii/miivar.h> 59 60 #include <arm/locore.h> 61 62 #include <arm/broadcom/bcm53xx_reg.h> 63 #include <arm/broadcom/bcm53xx_var.h> 64 65 //#define BCMETH_MPSAFE 66 67 #ifdef BCMETH_COUNTERS 68 #define BCMETH_EVCNT_ADD(a, b) ((void)((a).ev_count += (b))) 69 #else 70 #define BCMETH_EVCNT_ADD(a, b) do { } while (/*CONSTCOND*/0) 71 #endif 72 #define BCMETH_EVCNT_INCR(a) BCMETH_EVCNT_ADD((a), 1) 73 74 #define BCMETH_MAXTXMBUFS 128 75 #define BCMETH_NTXSEGS 30 76 #define BCMETH_MAXRXMBUFS 255 77 #define BCMETH_MINRXMBUFS 64 78 #define BCMETH_NRXSEGS 1 79 #define BCMETH_RINGSIZE PAGE_SIZE 80 81 #if 1 82 #define BCMETH_RCVMAGIC 0xfeedface 83 #endif 84 85 static int bcmeth_ccb_match(device_t, cfdata_t, void *); 86 static void bcmeth_ccb_attach(device_t, device_t, void *); 87 88 struct bcmeth_txqueue { 89 bus_dmamap_t txq_descmap; 90 struct gmac_txdb *txq_consumer; 91 struct gmac_txdb *txq_producer; 92 struct gmac_txdb *txq_first; 93 struct gmac_txdb *txq_last; 94 struct ifqueue txq_mbufs; 95 struct mbuf *txq_next; 96 size_t txq_free; 97 size_t txq_threshold; 98 size_t txq_lastintr; 99 bus_size_t txq_reg_xmtaddrlo; 100 bus_size_t txq_reg_xmtptr; 101 bus_size_t txq_reg_xmtctl; 102 bus_size_t txq_reg_xmtsts0; 103 bus_size_t txq_reg_xmtsts1; 104 bus_dma_segment_t txq_descmap_seg; 105 }; 106 107 struct bcmeth_rxqueue { 108 bus_dmamap_t rxq_descmap; 109 struct gmac_rxdb *rxq_consumer; 110 struct gmac_rxdb *rxq_producer; 111 struct gmac_rxdb *rxq_first; 112 struct gmac_rxdb *rxq_last; 113 struct mbuf *rxq_mhead; 114 struct mbuf **rxq_mtail; 115 struct mbuf *rxq_mconsumer; 116 size_t rxq_inuse; 117 size_t rxq_threshold; 118 bus_size_t rxq_reg_rcvaddrlo; 119 bus_size_t rxq_reg_rcvptr; 120 bus_size_t rxq_reg_rcvctl; 121 bus_size_t rxq_reg_rcvsts0; 122 bus_size_t rxq_reg_rcvsts1; 123 bus_dma_segment_t rxq_descmap_seg; 124 }; 125 126 struct bcmeth_mapcache { 127 u_int dmc_nmaps; 128 u_int dmc_maxseg; 129 u_int dmc_maxmaps; 130 u_int dmc_maxmapsize; 131 bus_dmamap_t dmc_maps[0]; 132 }; 133 134 struct bcmeth_softc { 135 device_t sc_dev; 136 bus_space_tag_t sc_bst; 137 bus_space_handle_t sc_bsh; 138 bus_dma_tag_t sc_dmat; 139 kmutex_t *sc_lock; 140 kmutex_t *sc_hwlock; 141 struct ethercom sc_ec; 142 #define sc_if sc_ec.ec_if 143 struct ifmedia sc_media; 144 void *sc_soft_ih; 145 void *sc_ih; 146 147 struct bcmeth_rxqueue sc_rxq; 148 struct bcmeth_txqueue sc_txq; 149 150 size_t sc_rcvoffset; 151 uint32_t sc_macaddr[2]; 152 uint32_t sc_maxfrm; 153 uint32_t sc_cmdcfg; 154 uint32_t sc_intmask; 155 uint32_t sc_rcvlazy; 156 volatile uint32_t sc_soft_flags; 157 #define SOFT_RXINTR 0x01 158 #define SOFT_TXINTR 0x02 159 160 #ifdef BCMETH_COUNTERS 161 struct evcnt sc_ev_intr; 162 struct evcnt sc_ev_soft_intr; 163 struct evcnt sc_ev_work; 164 struct evcnt sc_ev_tx_stall; 165 struct evcnt sc_ev_rx_badmagic_lo; 166 struct evcnt sc_ev_rx_badmagic_hi; 167 #endif 168 169 struct ifqueue sc_rx_bufcache; 170 struct bcmeth_mapcache *sc_rx_mapcache; 171 struct bcmeth_mapcache *sc_tx_mapcache; 172 173 struct workqueue *sc_workq; 174 struct work sc_work; 175 176 volatile uint32_t sc_work_flags; 177 #define WORK_RXINTR 0x01 178 #define WORK_RXUNDERFLOW 0x02 179 #define WORK_REINIT 0x04 180 181 uint8_t sc_enaddr[ETHER_ADDR_LEN]; 182 }; 183 184 static void bcmeth_ifstart(struct ifnet *); 185 static void bcmeth_ifwatchdog(struct ifnet *); 186 static int bcmeth_ifinit(struct ifnet *); 187 static void bcmeth_ifstop(struct ifnet *, int); 188 static int bcmeth_ifioctl(struct ifnet *, u_long, void *); 189 190 static int bcmeth_mapcache_create(struct bcmeth_softc *, 191 struct bcmeth_mapcache **, size_t, size_t, size_t); 192 static void bcmeth_mapcache_destroy(struct bcmeth_softc *, 193 struct bcmeth_mapcache *); 194 static bus_dmamap_t bcmeth_mapcache_get(struct bcmeth_softc *, 195 struct bcmeth_mapcache *); 196 static void bcmeth_mapcache_put(struct bcmeth_softc *, 197 struct bcmeth_mapcache *, bus_dmamap_t); 198 199 static int bcmeth_txq_attach(struct bcmeth_softc *, 200 struct bcmeth_txqueue *, u_int); 201 static void bcmeth_txq_purge(struct bcmeth_softc *, 202 struct bcmeth_txqueue *); 203 static void bcmeth_txq_reset(struct bcmeth_softc *, 204 struct bcmeth_txqueue *); 205 static bool bcmeth_txq_consume(struct bcmeth_softc *, 206 struct bcmeth_txqueue *); 207 static bool bcmeth_txq_produce(struct bcmeth_softc *, 208 struct bcmeth_txqueue *, struct mbuf *m); 209 static bool bcmeth_txq_active_p(struct bcmeth_softc *, 210 struct bcmeth_txqueue *); 211 212 static int bcmeth_rxq_attach(struct bcmeth_softc *, 213 struct bcmeth_rxqueue *, u_int); 214 static bool bcmeth_rxq_produce(struct bcmeth_softc *, 215 struct bcmeth_rxqueue *); 216 static void bcmeth_rxq_purge(struct bcmeth_softc *, 217 struct bcmeth_rxqueue *, bool); 218 static void bcmeth_rxq_reset(struct bcmeth_softc *, 219 struct bcmeth_rxqueue *); 220 221 static int bcmeth_intr(void *); 222 #ifdef BCMETH_MPSAFETX 223 static void bcmeth_soft_txintr(struct bcmeth_softc *); 224 #endif 225 static void bcmeth_soft_intr(void *); 226 static void bcmeth_worker(struct work *, void *); 227 228 static int bcmeth_mediachange(struct ifnet *); 229 static void bcmeth_mediastatus(struct ifnet *, struct ifmediareq *); 230 231 static inline uint32_t 232 bcmeth_read_4(struct bcmeth_softc *sc, bus_size_t o) 233 { 234 return bus_space_read_4(sc->sc_bst, sc->sc_bsh, o); 235 } 236 237 static inline void 238 bcmeth_write_4(struct bcmeth_softc *sc, bus_size_t o, uint32_t v) 239 { 240 bus_space_write_4(sc->sc_bst, sc->sc_bsh, o, v); 241 } 242 243 CFATTACH_DECL_NEW(bcmeth_ccb, sizeof(struct bcmeth_softc), 244 bcmeth_ccb_match, bcmeth_ccb_attach, NULL, NULL); 245 246 static int 247 bcmeth_ccb_match(device_t parent, cfdata_t cf, void *aux) 248 { 249 struct bcmccb_attach_args * const ccbaa = aux; 250 const struct bcm_locators * const loc = &ccbaa->ccbaa_loc; 251 252 if (strcmp(cf->cf_name, loc->loc_name)) 253 return 0; 254 255 #ifdef DIAGNOSTIC 256 const int port = cf->cf_loc[BCMCCBCF_PORT]; 257 #endif 258 KASSERT(port == BCMCCBCF_PORT_DEFAULT || port == loc->loc_port); 259 260 return 1; 261 } 262 263 static void 264 bcmeth_ccb_attach(device_t parent, device_t self, void *aux) 265 { 266 struct bcmeth_softc * const sc = device_private(self); 267 struct ethercom * const ec = &sc->sc_ec; 268 struct ifnet * const ifp = &ec->ec_if; 269 struct bcmccb_attach_args * const ccbaa = aux; 270 const struct bcm_locators * const loc = &ccbaa->ccbaa_loc; 271 const char * const xname = device_xname(self); 272 prop_dictionary_t dict = device_properties(self); 273 int error; 274 275 sc->sc_bst = ccbaa->ccbaa_ccb_bst; 276 sc->sc_dmat = ccbaa->ccbaa_dmat; 277 bus_space_subregion(sc->sc_bst, ccbaa->ccbaa_ccb_bsh, 278 loc->loc_offset, loc->loc_size, &sc->sc_bsh); 279 280 /* 281 * We need to use the coherent dma tag for the GMAC. 282 */ 283 sc->sc_dmat = &bcm53xx_coherent_dma_tag; 284 #if _ARM32_NEED_BUS_DMA_BOUNCE 285 if (device_cfdata(self)->cf_flags & 2) { 286 sc->sc_dmat = &bcm53xx_bounce_dma_tag; 287 } 288 #endif 289 290 prop_data_t eaprop = prop_dictionary_get(dict, "mac-address"); 291 if (eaprop == NULL) { 292 uint32_t mac0 = bcmeth_read_4(sc, UNIMAC_MAC_0); 293 uint32_t mac1 = bcmeth_read_4(sc, UNIMAC_MAC_1); 294 if ((mac0 == 0 && mac1 == 0) || (mac1 & 1)) { 295 aprint_error(": mac-address property is missing\n"); 296 return; 297 } 298 sc->sc_enaddr[0] = (mac0 >> 0) & 0xff; 299 sc->sc_enaddr[1] = (mac0 >> 8) & 0xff; 300 sc->sc_enaddr[2] = (mac0 >> 16) & 0xff; 301 sc->sc_enaddr[3] = (mac0 >> 24) & 0xff; 302 sc->sc_enaddr[4] = (mac1 >> 0) & 0xff; 303 sc->sc_enaddr[5] = (mac1 >> 8) & 0xff; 304 } else { 305 KASSERT(prop_object_type(eaprop) == PROP_TYPE_DATA); 306 KASSERT(prop_data_size(eaprop) == ETHER_ADDR_LEN); 307 memcpy(sc->sc_enaddr, prop_data_data_nocopy(eaprop), 308 ETHER_ADDR_LEN); 309 } 310 sc->sc_dev = self; 311 sc->sc_lock = mutex_obj_alloc(MUTEX_DEFAULT, IPL_SOFTNET); 312 sc->sc_hwlock = mutex_obj_alloc(MUTEX_DEFAULT, IPL_VM); 313 314 bcmeth_write_4(sc, GMAC_INTMASK, 0); // disable interrupts 315 316 aprint_naive("\n"); 317 aprint_normal(": Gigabit Ethernet Controller\n"); 318 319 error = bcmeth_rxq_attach(sc, &sc->sc_rxq, 0); 320 if (error) { 321 aprint_error(": failed to init rxq: %d\n", error); 322 goto fail_1; 323 } 324 325 error = bcmeth_txq_attach(sc, &sc->sc_txq, 0); 326 if (error) { 327 aprint_error(": failed to init txq: %d\n", error); 328 goto fail_1; 329 } 330 331 error = bcmeth_mapcache_create(sc, &sc->sc_rx_mapcache, 332 BCMETH_MAXRXMBUFS, MCLBYTES, BCMETH_NRXSEGS); 333 if (error) { 334 aprint_error(": failed to allocate rx dmamaps: %d\n", error); 335 goto fail_1; 336 } 337 338 error = bcmeth_mapcache_create(sc, &sc->sc_tx_mapcache, 339 BCMETH_MAXTXMBUFS, MCLBYTES, BCMETH_NTXSEGS); 340 if (error) { 341 aprint_error(": failed to allocate tx dmamaps: %d\n", error); 342 goto fail_1; 343 } 344 345 error = workqueue_create(&sc->sc_workq, xname, bcmeth_worker, sc, 346 (PRI_USER + MAXPRI_USER) / 2, IPL_NET, WQ_MPSAFE|WQ_PERCPU); 347 if (error) { 348 aprint_error(": failed to create workqueue: %d\n", error); 349 goto fail_2; 350 } 351 352 sc->sc_soft_ih = softint_establish(SOFTINT_MPSAFE | SOFTINT_NET, 353 bcmeth_soft_intr, sc); 354 355 if (sc->sc_ih == NULL) { 356 aprint_error_dev(self, "failed to establish interrupt %d\n", 357 loc->loc_intrs[0]); 358 goto fail_3; 359 } 360 361 sc->sc_ih = intr_establish(loc->loc_intrs[0], IPL_VM, IST_LEVEL, 362 bcmeth_intr, sc); 363 364 if (sc->sc_ih == NULL) { 365 aprint_error_dev(self, "failed to establish interrupt %d\n", 366 loc->loc_intrs[0]); 367 goto fail_4; 368 } else { 369 aprint_normal_dev(self, "interrupting on irq %d\n", 370 loc->loc_intrs[0]); 371 } 372 373 aprint_normal_dev(sc->sc_dev, "Ethernet address %s\n", 374 ether_sprintf(sc->sc_enaddr)); 375 376 /* 377 * Since each port in plugged into the switch/flow-accelerator, 378 * we hard code at Gige Full-Duplex with Flow Control enabled. 379 */ 380 int ifmedia = IFM_ETHER | IFM_1000_T | IFM_FDX; 381 //ifmedia |= IFM_FLOW | IFM_ETH_TXPAUSE | IFM_ETH_RXPAUSE; 382 ec->ec_ifmedia = &sc->sc_media; 383 ifmedia_init(&sc->sc_media, IFM_IMASK, bcmeth_mediachange, 384 bcmeth_mediastatus); 385 ifmedia_add(&sc->sc_media, ifmedia, 0, NULL); 386 ifmedia_set(&sc->sc_media, ifmedia); 387 388 ec->ec_capabilities = ETHERCAP_VLAN_MTU | ETHERCAP_JUMBO_MTU; 389 390 strlcpy(ifp->if_xname, xname, IFNAMSIZ); 391 ifp->if_softc = sc; 392 ifp->if_baudrate = IF_Mbps(1000); 393 ifp->if_capabilities = 0; 394 ifp->if_flags = IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST; 395 #ifdef BCMETH_MPSAFE 396 ifp->if_flags2 = IFF2_MPSAFE; 397 #endif 398 ifp->if_ioctl = bcmeth_ifioctl; 399 ifp->if_start = bcmeth_ifstart; 400 ifp->if_watchdog = bcmeth_ifwatchdog; 401 ifp->if_init = bcmeth_ifinit; 402 ifp->if_stop = bcmeth_ifstop; 403 IFQ_SET_READY(&ifp->if_snd); 404 405 bcmeth_ifstop(ifp, true); 406 407 /* 408 * Attach the interface. 409 */ 410 error = if_initialize(ifp); 411 if (error != 0) { 412 aprint_error_dev(sc->sc_dev, "if_initialize failed(%d)\n", 413 error); 414 goto fail_5; 415 } 416 ether_ifattach(ifp, sc->sc_enaddr); 417 if_register(ifp); 418 419 #ifdef BCMETH_COUNTERS 420 evcnt_attach_dynamic(&sc->sc_ev_intr, EVCNT_TYPE_INTR, 421 NULL, xname, "intr"); 422 evcnt_attach_dynamic(&sc->sc_ev_soft_intr, EVCNT_TYPE_INTR, 423 NULL, xname, "soft intr"); 424 evcnt_attach_dynamic(&sc->sc_ev_work, EVCNT_TYPE_MISC, 425 NULL, xname, "work items"); 426 evcnt_attach_dynamic(&sc->sc_ev_tx_stall, EVCNT_TYPE_MISC, 427 NULL, xname, "tx stalls"); 428 evcnt_attach_dynamic(&sc->sc_ev_rx_badmagic_lo, EVCNT_TYPE_MISC, 429 NULL, xname, "rx badmagic lo"); 430 evcnt_attach_dynamic(&sc->sc_ev_rx_badmagic_hi, EVCNT_TYPE_MISC, 431 NULL, xname, "rx badmagic hi"); 432 #endif 433 434 return; 435 436 fail_5: 437 ifmedia_removeall(&sc->sc_media); 438 fail_4: 439 intr_disestablish(sc->sc_ih); 440 fail_3: 441 softint_disestablish(sc->sc_soft_ih); 442 fail_2: 443 workqueue_destroy(sc->sc_workq); 444 fail_1: 445 mutex_obj_free(sc->sc_lock); 446 mutex_obj_free(sc->sc_hwlock); 447 } 448 449 static int 450 bcmeth_mediachange(struct ifnet *ifp) 451 { 452 //struct bcmeth_softc * const sc = ifp->if_softc; 453 return 0; 454 } 455 456 static void 457 bcmeth_mediastatus(struct ifnet *ifp, struct ifmediareq *ifm) 458 { 459 //struct bcmeth_softc * const sc = ifp->if_softc; 460 461 ifm->ifm_status = IFM_AVALID | IFM_ACTIVE; 462 ifm->ifm_active = IFM_ETHER | IFM_FDX | IFM_1000_T; 463 } 464 465 static uint64_t 466 bcmeth_macaddr_create(const uint8_t *enaddr) 467 { 468 return (enaddr[3] << 0) // UNIMAC_MAC_0 469 | (enaddr[2] << 8) // UNIMAC_MAC_0 470 | (enaddr[1] << 16) // UNIMAC_MAC_0 471 | ((uint64_t)enaddr[0] << 24) // UNIMAC_MAC_0 472 | ((uint64_t)enaddr[5] << 32) // UNIMAC_MAC_1 473 | ((uint64_t)enaddr[4] << 40); // UNIMAC_MAC_1 474 } 475 476 static int 477 bcmeth_ifinit(struct ifnet *ifp) 478 { 479 struct bcmeth_softc * const sc = ifp->if_softc; 480 int error = 0; 481 482 sc->sc_maxfrm = uimax(ifp->if_mtu + 32, MCLBYTES); 483 if (ifp->if_mtu > ETHERMTU_JUMBO) 484 return error; 485 486 KASSERT(ifp->if_flags & IFF_UP); 487 488 /* 489 * Stop the interface 490 */ 491 bcmeth_ifstop(ifp, 0); 492 493 /* 494 * Reserve enough space at the front so that we can insert a maxsized 495 * link header and a VLAN tag. Also make sure we have enough room for 496 * the rcvsts field as well. 497 */ 498 KASSERT(ALIGN(max_linkhdr) == max_linkhdr); 499 KASSERTMSG(max_linkhdr > sizeof(struct ether_header), "%u > %zu", 500 max_linkhdr, sizeof(struct ether_header)); 501 sc->sc_rcvoffset = max_linkhdr + 4 - sizeof(struct ether_header); 502 if (sc->sc_rcvoffset <= 4) 503 sc->sc_rcvoffset += 4; 504 KASSERT((sc->sc_rcvoffset & 3) == 2); 505 KASSERT(sc->sc_rcvoffset <= __SHIFTOUT(RCVCTL_RCVOFFSET, RCVCTL_RCVOFFSET)); 506 KASSERT(sc->sc_rcvoffset >= 6); 507 508 /* 509 * If our frame size has changed (or it's our first time through) 510 * destroy the existing transmit mapcache. 511 */ 512 if (sc->sc_tx_mapcache != NULL 513 && sc->sc_maxfrm != sc->sc_tx_mapcache->dmc_maxmapsize) { 514 bcmeth_mapcache_destroy(sc, sc->sc_tx_mapcache); 515 sc->sc_tx_mapcache = NULL; 516 } 517 518 if (sc->sc_tx_mapcache == NULL) { 519 error = bcmeth_mapcache_create(sc, &sc->sc_tx_mapcache, 520 BCMETH_MAXTXMBUFS, sc->sc_maxfrm, BCMETH_NTXSEGS); 521 if (error) 522 return error; 523 } 524 525 sc->sc_cmdcfg = NO_LENGTH_CHECK | PAUSE_IGNORE 526 | __SHIFTIN(ETH_SPEED_1000, ETH_SPEED) 527 | RX_ENA | TX_ENA; 528 529 if (ifp->if_flags & IFF_PROMISC) { 530 sc->sc_cmdcfg |= PROMISC_EN; 531 } else { 532 sc->sc_cmdcfg &= ~PROMISC_EN; 533 } 534 535 const uint8_t * const lladdr = CLLADDR(ifp->if_sadl); 536 const uint64_t macstnaddr = bcmeth_macaddr_create(lladdr); 537 538 /* 539 * We make sure that a received Ethernet packet start on a non-word 540 * boundary so that the packet payload will be on a word boundary. 541 * So to check the destination address we keep around two words to 542 * quickly compare with. 543 */ 544 #if __ARMEL__ 545 sc->sc_macaddr[0] = lladdr[0] | (lladdr[1] << 8); 546 sc->sc_macaddr[1] = lladdr[2] | (lladdr[3] << 8) 547 | (lladdr[4] << 16) | (lladdr[5] << 24); 548 #else 549 sc->sc_macaddr[0] = lladdr[1] | (lladdr[0] << 8); 550 sc->sc_macaddr[1] = lladdr[5] | (lladdr[4] << 8) 551 | (lladdr[1] << 16) | (lladdr[2] << 24); 552 #endif 553 554 sc->sc_intmask = DESCPROTOERR | DATAERR | DESCERR; 555 556 /* 5. Load RCVADDR_LO with new pointer */ 557 bcmeth_rxq_reset(sc, &sc->sc_rxq); 558 559 bcmeth_write_4(sc, sc->sc_rxq.rxq_reg_rcvctl, 560 __SHIFTIN(sc->sc_rcvoffset, RCVCTL_RCVOFFSET) 561 | RCVCTL_PARITY_DIS 562 | RCVCTL_OFLOW_CONTINUE 563 | __SHIFTIN(3, RCVCTL_BURSTLEN)); 564 565 /* 6. Load XMTADDR_LO with new pointer */ 566 bcmeth_txq_reset(sc, &sc->sc_txq); 567 568 bcmeth_write_4(sc, sc->sc_txq.txq_reg_xmtctl, XMTCTL_DMA_ACT_INDEX 569 | XMTCTL_PARITY_DIS 570 | __SHIFTIN(3, XMTCTL_BURSTLEN)); 571 572 /* 7. Setup other UNIMAC registers */ 573 bcmeth_write_4(sc, UNIMAC_FRAME_LEN, sc->sc_maxfrm); 574 bcmeth_write_4(sc, UNIMAC_MAC_0, (uint32_t)(macstnaddr >> 0)); 575 bcmeth_write_4(sc, UNIMAC_MAC_1, (uint32_t)(macstnaddr >> 32)); 576 bcmeth_write_4(sc, UNIMAC_COMMAND_CONFIG, sc->sc_cmdcfg); 577 578 uint32_t devctl = bcmeth_read_4(sc, GMAC_DEVCONTROL); 579 devctl |= RGMII_LINK_STATUS_SEL | NWAY_AUTO_POLL_EN | TXARB_STRICT_MODE; 580 devctl &= ~FLOW_CTRL_MODE; 581 devctl &= ~MIB_RD_RESET_EN; 582 devctl &= ~RXQ_OVERFLOW_CTRL_SEL; 583 devctl &= ~CPU_FLOW_CTRL_ON; 584 bcmeth_write_4(sc, GMAC_DEVCONTROL, devctl); 585 586 /* Setup lazy receive (at most 1ms). */ 587 const struct cpu_softc * const cpu = curcpu()->ci_softc; 588 sc->sc_rcvlazy = __SHIFTIN(4, INTRCVLAZY_FRAMECOUNT) 589 | __SHIFTIN(cpu->cpu_clk.clk_apb / 1000, INTRCVLAZY_TIMEOUT); 590 bcmeth_write_4(sc, GMAC_INTRCVLAZY, sc->sc_rcvlazy); 591 592 /* 11. Enable transmit queues in TQUEUE, and ensure that the transmit scheduling mode is correctly set in TCTRL. */ 593 sc->sc_intmask |= XMTINT_0 | XMTUF; 594 bcmeth_write_4(sc, sc->sc_txq.txq_reg_xmtctl, 595 bcmeth_read_4(sc, sc->sc_txq.txq_reg_xmtctl) | XMTCTL_ENABLE); 596 597 598 /* 12. Enable receive queues in RQUEUE, */ 599 sc->sc_intmask |= RCVINT | RCVDESCUF | RCVFIFOOF; 600 bcmeth_write_4(sc, sc->sc_rxq.rxq_reg_rcvctl, 601 bcmeth_read_4(sc, sc->sc_rxq.rxq_reg_rcvctl) | RCVCTL_ENABLE); 602 603 bcmeth_rxq_produce(sc, &sc->sc_rxq); /* fill with rx buffers */ 604 605 #if 0 606 aprint_normal_dev(sc->sc_dev, 607 "devctl=%#x ucmdcfg=%#x xmtctl=%#x rcvctl=%#x\n", 608 devctl, sc->sc_cmdcfg, 609 bcmeth_read_4(sc, sc->sc_txq.txq_reg_xmtctl), 610 bcmeth_read_4(sc, sc->sc_rxq.rxq_reg_rcvctl)); 611 #endif 612 613 sc->sc_soft_flags = 0; 614 615 bcmeth_write_4(sc, GMAC_INTMASK, sc->sc_intmask); 616 617 ifp->if_flags |= IFF_RUNNING; 618 619 return error; 620 } 621 622 static void 623 bcmeth_ifstop(struct ifnet *ifp, int disable) 624 { 625 struct bcmeth_softc * const sc = ifp->if_softc; 626 struct bcmeth_txqueue * const txq = &sc->sc_txq; 627 struct bcmeth_rxqueue * const rxq = &sc->sc_rxq; 628 629 KASSERT(!cpu_intr_p()); 630 631 sc->sc_soft_flags = 0; 632 sc->sc_work_flags = 0; 633 634 /* Disable Rx processing */ 635 bcmeth_write_4(sc, rxq->rxq_reg_rcvctl, 636 bcmeth_read_4(sc, rxq->rxq_reg_rcvctl) & ~RCVCTL_ENABLE); 637 638 /* Disable Tx processing */ 639 bcmeth_write_4(sc, txq->txq_reg_xmtctl, 640 bcmeth_read_4(sc, txq->txq_reg_xmtctl) & ~XMTCTL_ENABLE); 641 642 /* Disable all interrupts */ 643 bcmeth_write_4(sc, GMAC_INTMASK, 0); 644 645 for (;;) { 646 uint32_t tx0 = bcmeth_read_4(sc, txq->txq_reg_xmtsts0); 647 uint32_t rx0 = bcmeth_read_4(sc, rxq->rxq_reg_rcvsts0); 648 if (__SHIFTOUT(tx0, XMTSTATE) == XMTSTATE_DIS 649 && __SHIFTOUT(rx0, RCVSTATE) == RCVSTATE_DIS) 650 break; 651 delay(50); 652 } 653 /* 654 * Now reset the controller. 655 * 656 * 3. Set SW_RESET bit in UNIMAC_COMMAND_CONFIG register 657 * 4. Clear SW_RESET bit in UNIMAC_COMMAND_CONFIG register 658 */ 659 bcmeth_write_4(sc, UNIMAC_COMMAND_CONFIG, SW_RESET); 660 bcmeth_write_4(sc, GMAC_INTSTATUS, ~0); 661 sc->sc_intmask = 0; 662 ifp->if_flags &= ~IFF_RUNNING; 663 664 /* 665 * Let's consume any remaining transmitted packets. And if we are 666 * disabling the interface, purge ourselves of any untransmitted 667 * packets. But don't consume any received packets, just drop them. 668 * If we aren't disabling the interface, save the mbufs in the 669 * receive queue for reuse. 670 */ 671 bcmeth_rxq_purge(sc, &sc->sc_rxq, disable); 672 bcmeth_txq_consume(sc, &sc->sc_txq); 673 if (disable) { 674 bcmeth_txq_purge(sc, &sc->sc_txq); 675 IF_PURGE(&ifp->if_snd); 676 } 677 678 bcmeth_write_4(sc, UNIMAC_COMMAND_CONFIG, 0); 679 } 680 681 static void 682 bcmeth_ifwatchdog(struct ifnet *ifp) 683 { 684 } 685 686 static int 687 bcmeth_ifioctl(struct ifnet *ifp, u_long cmd, void *data) 688 { 689 const int s = splnet(); 690 int error; 691 692 switch (cmd) { 693 default: 694 error = ether_ioctl(ifp, cmd, data); 695 if (error != ENETRESET) 696 break; 697 698 if (cmd == SIOCADDMULTI || cmd == SIOCDELMULTI) { 699 error = 0; 700 break; 701 } 702 error = bcmeth_ifinit(ifp); 703 break; 704 } 705 706 splx(s); 707 return error; 708 } 709 710 static void 711 bcmeth_rxq_desc_presync( 712 struct bcmeth_softc *sc, 713 struct bcmeth_rxqueue *rxq, 714 struct gmac_rxdb *rxdb, 715 size_t count) 716 { 717 bus_dmamap_sync(sc->sc_dmat, rxq->rxq_descmap, 718 (rxdb - rxq->rxq_first) * sizeof(*rxdb), count * sizeof(*rxdb), 719 BUS_DMASYNC_PREWRITE); 720 } 721 722 static void 723 bcmeth_rxq_desc_postsync( 724 struct bcmeth_softc *sc, 725 struct bcmeth_rxqueue *rxq, 726 struct gmac_rxdb *rxdb, 727 size_t count) 728 { 729 bus_dmamap_sync(sc->sc_dmat, rxq->rxq_descmap, 730 (rxdb - rxq->rxq_first) * sizeof(*rxdb), count * sizeof(*rxdb), 731 BUS_DMASYNC_POSTWRITE); 732 } 733 734 static void 735 bcmeth_txq_desc_presync( 736 struct bcmeth_softc *sc, 737 struct bcmeth_txqueue *txq, 738 struct gmac_txdb *txdb, 739 size_t count) 740 { 741 bus_dmamap_sync(sc->sc_dmat, txq->txq_descmap, 742 (txdb - txq->txq_first) * sizeof(*txdb), count * sizeof(*txdb), 743 BUS_DMASYNC_PREWRITE); 744 } 745 746 static void 747 bcmeth_txq_desc_postsync( 748 struct bcmeth_softc *sc, 749 struct bcmeth_txqueue *txq, 750 struct gmac_txdb *txdb, 751 size_t count) 752 { 753 bus_dmamap_sync(sc->sc_dmat, txq->txq_descmap, 754 (txdb - txq->txq_first) * sizeof(*txdb), count * sizeof(*txdb), 755 BUS_DMASYNC_POSTWRITE); 756 } 757 758 static bus_dmamap_t 759 bcmeth_mapcache_get( 760 struct bcmeth_softc *sc, 761 struct bcmeth_mapcache *dmc) 762 { 763 KASSERT(dmc->dmc_nmaps > 0); 764 KASSERT(dmc->dmc_maps[dmc->dmc_nmaps-1] != NULL); 765 return dmc->dmc_maps[--dmc->dmc_nmaps]; 766 } 767 768 static void 769 bcmeth_mapcache_put( 770 struct bcmeth_softc *sc, 771 struct bcmeth_mapcache *dmc, 772 bus_dmamap_t map) 773 { 774 KASSERT(map != NULL); 775 KASSERT(dmc->dmc_nmaps < dmc->dmc_maxmaps); 776 dmc->dmc_maps[dmc->dmc_nmaps++] = map; 777 } 778 779 static void 780 bcmeth_mapcache_destroy( 781 struct bcmeth_softc *sc, 782 struct bcmeth_mapcache *dmc) 783 { 784 const size_t dmc_size = 785 offsetof(struct bcmeth_mapcache, dmc_maps[dmc->dmc_maxmaps]); 786 787 for (u_int i = 0; i < dmc->dmc_maxmaps; i++) { 788 bus_dmamap_destroy(sc->sc_dmat, dmc->dmc_maps[i]); 789 } 790 kmem_intr_free(dmc, dmc_size); 791 } 792 793 static int 794 bcmeth_mapcache_create( 795 struct bcmeth_softc *sc, 796 struct bcmeth_mapcache **dmc_p, 797 size_t maxmaps, 798 size_t maxmapsize, 799 size_t maxseg) 800 { 801 const size_t dmc_size = 802 offsetof(struct bcmeth_mapcache, dmc_maps[maxmaps]); 803 struct bcmeth_mapcache * const dmc = 804 kmem_intr_zalloc(dmc_size, KM_NOSLEEP); 805 806 dmc->dmc_maxmaps = maxmaps; 807 dmc->dmc_nmaps = maxmaps; 808 dmc->dmc_maxmapsize = maxmapsize; 809 dmc->dmc_maxseg = maxseg; 810 811 for (u_int i = 0; i < maxmaps; i++) { 812 int error = bus_dmamap_create(sc->sc_dmat, dmc->dmc_maxmapsize, 813 dmc->dmc_maxseg, dmc->dmc_maxmapsize, 0, 814 BUS_DMA_WAITOK | BUS_DMA_ALLOCNOW, &dmc->dmc_maps[i]); 815 if (error) { 816 aprint_error_dev(sc->sc_dev, 817 "failed to creat dma map cache " 818 "entry %u of %zu: %d\n", 819 i, maxmaps, error); 820 while (i-- > 0) { 821 bus_dmamap_destroy(sc->sc_dmat, 822 dmc->dmc_maps[i]); 823 } 824 kmem_intr_free(dmc, dmc_size); 825 return error; 826 } 827 KASSERT(dmc->dmc_maps[i] != NULL); 828 } 829 830 *dmc_p = dmc; 831 832 return 0; 833 } 834 835 #if 0 836 static void 837 bcmeth_dmamem_free( 838 bus_dma_tag_t dmat, 839 size_t map_size, 840 bus_dma_segment_t *seg, 841 bus_dmamap_t map, 842 void *kvap) 843 { 844 bus_dmamap_destroy(dmat, map); 845 bus_dmamem_unmap(dmat, kvap, map_size); 846 bus_dmamem_free(dmat, seg, 1); 847 } 848 #endif 849 850 static int 851 bcmeth_dmamem_alloc( 852 bus_dma_tag_t dmat, 853 size_t map_size, 854 bus_dma_segment_t *seg, 855 bus_dmamap_t *map, 856 void **kvap) 857 { 858 int error; 859 int nseg; 860 861 *kvap = NULL; 862 *map = NULL; 863 864 error = bus_dmamem_alloc(dmat, map_size, 2*PAGE_SIZE, 0, 865 seg, 1, &nseg, 0); 866 if (error) 867 return error; 868 869 KASSERT(nseg == 1); 870 871 error = bus_dmamem_map(dmat, seg, nseg, map_size, (void **)kvap, 0); 872 if (error == 0) { 873 error = bus_dmamap_create(dmat, map_size, 1, map_size, 0, 0, 874 map); 875 if (error == 0) { 876 error = bus_dmamap_load(dmat, *map, *kvap, map_size, 877 NULL, 0); 878 if (error == 0) 879 return 0; 880 bus_dmamap_destroy(dmat, *map); 881 *map = NULL; 882 } 883 bus_dmamem_unmap(dmat, *kvap, map_size); 884 *kvap = NULL; 885 } 886 bus_dmamem_free(dmat, seg, nseg); 887 return 0; 888 } 889 890 static struct mbuf * 891 bcmeth_rx_buf_alloc( 892 struct bcmeth_softc *sc) 893 { 894 struct mbuf *m = m_gethdr(M_DONTWAIT, MT_DATA); 895 if (m == NULL) { 896 printf("%s:%d: %s\n", __func__, __LINE__, "m_gethdr"); 897 return NULL; 898 } 899 MCLGET(m, M_DONTWAIT); 900 if ((m->m_flags & M_EXT) == 0) { 901 printf("%s:%d: %s\n", __func__, __LINE__, "MCLGET"); 902 m_freem(m); 903 return NULL; 904 } 905 m->m_len = m->m_pkthdr.len = m->m_ext.ext_size; 906 907 bus_dmamap_t map = bcmeth_mapcache_get(sc, sc->sc_rx_mapcache); 908 if (map == NULL) { 909 printf("%s:%d: %s\n", __func__, __LINE__, "map get"); 910 m_freem(m); 911 return NULL; 912 } 913 M_SETCTX(m, map); 914 m->m_len = m->m_pkthdr.len = MCLBYTES; 915 int error = bus_dmamap_load_mbuf(sc->sc_dmat, map, m, 916 BUS_DMA_READ | BUS_DMA_NOWAIT); 917 if (error) { 918 aprint_error_dev(sc->sc_dev, "fail to load rx dmamap: %d\n", 919 error); 920 M_SETCTX(m, NULL); 921 m_freem(m); 922 bcmeth_mapcache_put(sc, sc->sc_rx_mapcache, map); 923 return NULL; 924 } 925 KASSERT(map->dm_mapsize == MCLBYTES); 926 #ifdef BCMETH_RCVMAGIC 927 *mtod(m, uint32_t *) = htole32(BCMETH_RCVMAGIC); 928 bus_dmamap_sync(sc->sc_dmat, map, 0, sizeof(uint32_t), 929 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); 930 bus_dmamap_sync(sc->sc_dmat, map, sizeof(uint32_t), 931 map->dm_mapsize - sizeof(uint32_t), BUS_DMASYNC_PREREAD); 932 #else 933 bus_dmamap_sync(sc->sc_dmat, map, 0, map->dm_mapsize, 934 BUS_DMASYNC_PREREAD); 935 #endif 936 937 return m; 938 } 939 940 static void 941 bcmeth_rx_map_unload( 942 struct bcmeth_softc *sc, 943 struct mbuf *m) 944 { 945 KASSERT(m); 946 for (; m != NULL; m = m->m_next) { 947 bus_dmamap_t map = M_GETCTX(m, bus_dmamap_t); 948 KASSERT(map); 949 KASSERT(map->dm_mapsize == MCLBYTES); 950 bus_dmamap_sync(sc->sc_dmat, map, 0, m->m_len, 951 BUS_DMASYNC_POSTREAD); 952 bus_dmamap_unload(sc->sc_dmat, map); 953 bcmeth_mapcache_put(sc, sc->sc_rx_mapcache, map); 954 M_SETCTX(m, NULL); 955 } 956 } 957 958 static bool 959 bcmeth_rxq_produce( 960 struct bcmeth_softc *sc, 961 struct bcmeth_rxqueue *rxq) 962 { 963 struct gmac_rxdb *producer = rxq->rxq_producer; 964 bool produced = false; 965 966 while (rxq->rxq_inuse < rxq->rxq_threshold) { 967 struct mbuf *m; 968 IF_DEQUEUE(&sc->sc_rx_bufcache, m); 969 if (m == NULL) { 970 m = bcmeth_rx_buf_alloc(sc); 971 if (m == NULL) { 972 printf("%s: bcmeth_rx_buf_alloc failed\n", 973 __func__); 974 break; 975 } 976 } 977 bus_dmamap_t map = M_GETCTX(m, bus_dmamap_t); 978 KASSERT(map); 979 980 producer->rxdb_buflen = htole32(MCLBYTES); 981 producer->rxdb_addrlo = htole32(map->dm_segs[0].ds_addr); 982 producer->rxdb_flags &= htole32(RXDB_FLAG_ET); 983 *rxq->rxq_mtail = m; 984 rxq->rxq_mtail = &m->m_next; 985 m->m_len = MCLBYTES; 986 m->m_next = NULL; 987 rxq->rxq_inuse++; 988 if (++producer == rxq->rxq_last) { 989 membar_producer(); 990 bcmeth_rxq_desc_presync(sc, rxq, rxq->rxq_producer, 991 rxq->rxq_last - rxq->rxq_producer); 992 producer = rxq->rxq_producer = rxq->rxq_first; 993 } 994 produced = true; 995 } 996 if (produced) { 997 membar_producer(); 998 if (producer != rxq->rxq_producer) { 999 bcmeth_rxq_desc_presync(sc, rxq, rxq->rxq_producer, 1000 producer - rxq->rxq_producer); 1001 rxq->rxq_producer = producer; 1002 } 1003 bcmeth_write_4(sc, rxq->rxq_reg_rcvptr, 1004 rxq->rxq_descmap->dm_segs[0].ds_addr 1005 + ((uintptr_t)producer & RCVPTR)); 1006 } 1007 return true; 1008 } 1009 1010 static void 1011 bcmeth_rx_input( 1012 struct bcmeth_softc *sc, 1013 struct mbuf *m, 1014 uint32_t rxdb_flags) 1015 { 1016 struct ifnet * const ifp = &sc->sc_if; 1017 1018 bcmeth_rx_map_unload(sc, m); 1019 1020 m_adj(m, sc->sc_rcvoffset); 1021 1022 /* 1023 * If we are in promiscuous mode and this isn't a multicast, check the 1024 * destination address to make sure it matches our own. If it doesn't, 1025 * mark the packet as being received promiscuously. 1026 */ 1027 if ((sc->sc_cmdcfg & PROMISC_EN) 1028 && (m->m_data[0] & 1) == 0 1029 && (*(uint16_t *)&m->m_data[0] != sc->sc_macaddr[0] 1030 || *(uint32_t *)&m->m_data[2] != sc->sc_macaddr[1])) { 1031 m->m_flags |= M_PROMISC; 1032 } 1033 m_set_rcvif(m, ifp); 1034 1035 ifp->if_ibytes += m->m_pkthdr.len; 1036 1037 /* 1038 * Let's give it to the network subsystm to deal with. 1039 */ 1040 #ifdef BCMETH_MPSAFE 1041 mutex_exit(sc->sc_lock); 1042 if_input(ifp, m); 1043 mutex_enter(sc->sc_lock); 1044 #else 1045 int s = splnet(); 1046 if_input(ifp, m); 1047 splx(s); 1048 #endif 1049 } 1050 1051 static bool 1052 bcmeth_rxq_consume( 1053 struct bcmeth_softc *sc, 1054 struct bcmeth_rxqueue *rxq, 1055 size_t atmost) 1056 { 1057 struct ifnet * const ifp = &sc->sc_if; 1058 struct gmac_rxdb *consumer = rxq->rxq_consumer; 1059 size_t rxconsumed = 0; 1060 bool didconsume = false; 1061 1062 while (atmost-- > 0) { 1063 if (consumer == rxq->rxq_producer) { 1064 KASSERT(rxq->rxq_inuse == 0); 1065 break; 1066 } 1067 1068 uint32_t rcvsts0 = bcmeth_read_4(sc, rxq->rxq_reg_rcvsts0); 1069 uint32_t currdscr = __SHIFTOUT(rcvsts0, RCV_CURRDSCR); 1070 if (consumer == rxq->rxq_first + currdscr) { 1071 break; 1072 } 1073 bcmeth_rxq_desc_postsync(sc, rxq, consumer, 1); 1074 1075 /* 1076 * We own this packet again. Copy the rxsts word from it. 1077 */ 1078 rxconsumed++; 1079 didconsume = true; 1080 uint32_t rxsts; 1081 KASSERT(rxq->rxq_mhead != NULL); 1082 bus_dmamap_t map = M_GETCTX(rxq->rxq_mhead, bus_dmamap_t); 1083 bus_dmamap_sync(sc->sc_dmat, map, 0, arm_dcache_align, 1084 BUS_DMASYNC_POSTREAD); 1085 memcpy(&rxsts, rxq->rxq_mhead->m_data, 4); 1086 rxsts = le32toh(rxsts); 1087 #if 0 1088 KASSERTMSG(rxsts != BCMETH_RCVMAGIC, "currdscr=%u consumer=%zd", 1089 currdscr, consumer - rxq->rxq_first); 1090 #endif 1091 1092 /* 1093 * Get the count of descriptors. Fetch the correct number 1094 * of mbufs. 1095 */ 1096 #ifdef BCMETH_RCVMAGIC 1097 size_t desc_count = rxsts != BCMETH_RCVMAGIC 1098 ? __SHIFTOUT(rxsts, RXSTS_DESC_COUNT) + 1 : 1; 1099 #else 1100 size_t desc_count = __SHIFTOUT(rxsts, RXSTS_DESC_COUNT) + 1; 1101 #endif 1102 struct mbuf *m = rxq->rxq_mhead; 1103 struct mbuf *m_last = m; 1104 for (size_t i = 1; i < desc_count; i++) { 1105 if (++consumer == rxq->rxq_last) { 1106 consumer = rxq->rxq_first; 1107 } 1108 KASSERTMSG(consumer != rxq->rxq_first + currdscr, 1109 "i=%zu rxsts=%#x desc_count=%zu currdscr=%u " 1110 "consumer=%zd", i, rxsts, desc_count, currdscr, 1111 consumer - rxq->rxq_first); 1112 m_last = m_last->m_next; 1113 } 1114 1115 /* 1116 * Now remove it/them from the list of enqueued mbufs. 1117 */ 1118 if ((rxq->rxq_mhead = m_last->m_next) == NULL) 1119 rxq->rxq_mtail = &rxq->rxq_mhead; 1120 m_last->m_next = NULL; 1121 1122 #ifdef BCMETH_RCVMAGIC 1123 if (rxsts == BCMETH_RCVMAGIC) { 1124 if_statinc(ifp, if_ierrors); 1125 if ((m->m_ext.ext_paddr >> 28) == 8) { 1126 BCMETH_EVCNT_INCR(sc->sc_ev_rx_badmagic_lo); 1127 } else { 1128 BCMETH_EVCNT_INCR( sc->sc_ev_rx_badmagic_hi); 1129 } 1130 IF_ENQUEUE(&sc->sc_rx_bufcache, m); 1131 } else 1132 #endif /* BCMETH_RCVMAGIC */ 1133 if (rxsts 1134 & (RXSTS_CRC_ERROR |RXSTS_OVERSIZED |RXSTS_PKT_OVERFLOW)) { 1135 aprint_error_dev(sc->sc_dev, 1136 "[%zu]: count=%zu rxsts=%#x\n", 1137 consumer - rxq->rxq_first, desc_count, rxsts); 1138 /* 1139 * We encountered an error, take the mbufs and add them 1140 * to the rx bufcache so we can quickly reuse them. 1141 */ 1142 if_statinc(ifp, if_ierrors); 1143 do { 1144 struct mbuf *m0 = m->m_next; 1145 m->m_next = NULL; 1146 IF_ENQUEUE(&sc->sc_rx_bufcache, m); 1147 m = m0; 1148 } while (m); 1149 } else { 1150 uint32_t framelen = __SHIFTOUT(rxsts, RXSTS_FRAMELEN); 1151 framelen += sc->sc_rcvoffset; 1152 m->m_pkthdr.len = framelen; 1153 if (desc_count == 1) { 1154 KASSERT(framelen <= MCLBYTES); 1155 m->m_len = framelen; 1156 } else { 1157 m_last->m_len = framelen & (MCLBYTES - 1); 1158 } 1159 1160 #ifdef BCMETH_MPSAFE 1161 /* 1162 * Wrap at the last entry! 1163 */ 1164 if (++consumer == rxq->rxq_last) { 1165 KASSERT(consumer[-1].rxdb_flags 1166 & htole32(RXDB_FLAG_ET)); 1167 rxq->rxq_consumer = rxq->rxq_first; 1168 } else { 1169 rxq->rxq_consumer = consumer; 1170 } 1171 rxq->rxq_inuse -= rxconsumed; 1172 #endif /* BCMETH_MPSAFE */ 1173 1174 /* 1175 * Receive the packet (which releases our lock) 1176 */ 1177 bcmeth_rx_input(sc, m, rxsts); 1178 1179 #ifdef BCMETH_MPSAFE 1180 /* 1181 * Since we had to give up our lock, we need to 1182 * refresh these. 1183 */ 1184 consumer = rxq->rxq_consumer; 1185 rxconsumed = 0; 1186 continue; 1187 #endif /* BCMETH_MPSAFE */ 1188 } 1189 1190 /* 1191 * Wrap at the last entry! 1192 */ 1193 if (++consumer == rxq->rxq_last) { 1194 KASSERT(consumer[-1].rxdb_flags & htole32(RXDB_FLAG_ET)); 1195 consumer = rxq->rxq_first; 1196 } 1197 } 1198 1199 /* 1200 * Update queue info. 1201 */ 1202 rxq->rxq_consumer = consumer; 1203 rxq->rxq_inuse -= rxconsumed; 1204 1205 /* 1206 * Did we consume anything? 1207 */ 1208 return didconsume; 1209 } 1210 1211 static void 1212 bcmeth_rxq_purge( 1213 struct bcmeth_softc *sc, 1214 struct bcmeth_rxqueue *rxq, 1215 bool discard) 1216 { 1217 struct mbuf *m; 1218 1219 if ((m = rxq->rxq_mhead) != NULL) { 1220 if (discard) { 1221 bcmeth_rx_map_unload(sc, m); 1222 m_freem(m); 1223 } else { 1224 while (m != NULL) { 1225 struct mbuf *m0 = m->m_next; 1226 m->m_next = NULL; 1227 IF_ENQUEUE(&sc->sc_rx_bufcache, m); 1228 m = m0; 1229 } 1230 } 1231 } 1232 1233 rxq->rxq_mhead = NULL; 1234 rxq->rxq_mtail = &rxq->rxq_mhead; 1235 rxq->rxq_inuse = 0; 1236 } 1237 1238 static void 1239 bcmeth_rxq_reset( 1240 struct bcmeth_softc *sc, 1241 struct bcmeth_rxqueue *rxq) 1242 { 1243 /* 1244 * sync all the descriptors 1245 */ 1246 bcmeth_rxq_desc_postsync(sc, rxq, rxq->rxq_first, 1247 rxq->rxq_last - rxq->rxq_first); 1248 1249 /* 1250 * Make sure we own all descriptors in the ring. 1251 */ 1252 struct gmac_rxdb *rxdb; 1253 for (rxdb = rxq->rxq_first; rxdb < rxq->rxq_last - 1; rxdb++) { 1254 rxdb->rxdb_flags = htole32(RXDB_FLAG_IC); 1255 } 1256 1257 /* 1258 * Last descriptor has the wrap flag. 1259 */ 1260 rxdb->rxdb_flags = htole32(RXDB_FLAG_ET | RXDB_FLAG_IC); 1261 1262 /* 1263 * Reset the producer consumer indexes. 1264 */ 1265 rxq->rxq_consumer = rxq->rxq_first; 1266 rxq->rxq_producer = rxq->rxq_first; 1267 rxq->rxq_inuse = 0; 1268 if (rxq->rxq_threshold < BCMETH_MINRXMBUFS) 1269 rxq->rxq_threshold = BCMETH_MINRXMBUFS; 1270 1271 sc->sc_intmask |= RCVINT | RCVFIFOOF | RCVDESCUF; 1272 1273 /* 1274 * Restart the receiver at the first descriptor 1275 */ 1276 bcmeth_write_4(sc, rxq->rxq_reg_rcvaddrlo, 1277 rxq->rxq_descmap->dm_segs[0].ds_addr); 1278 } 1279 1280 static int 1281 bcmeth_rxq_attach( 1282 struct bcmeth_softc *sc, 1283 struct bcmeth_rxqueue *rxq, 1284 u_int qno) 1285 { 1286 size_t desc_count = BCMETH_RINGSIZE / sizeof(rxq->rxq_first[0]); 1287 int error; 1288 void *descs; 1289 1290 KASSERT(desc_count == 256 || desc_count == 512); 1291 1292 error = bcmeth_dmamem_alloc(sc->sc_dmat, BCMETH_RINGSIZE, 1293 &rxq->rxq_descmap_seg, &rxq->rxq_descmap, &descs); 1294 if (error) 1295 return error; 1296 1297 memset(descs, 0, BCMETH_RINGSIZE); 1298 rxq->rxq_first = descs; 1299 rxq->rxq_last = rxq->rxq_first + desc_count; 1300 rxq->rxq_consumer = descs; 1301 rxq->rxq_producer = descs; 1302 1303 bcmeth_rxq_purge(sc, rxq, true); 1304 bcmeth_rxq_reset(sc, rxq); 1305 1306 rxq->rxq_reg_rcvaddrlo = GMAC_RCVADDR_LOW; 1307 rxq->rxq_reg_rcvctl = GMAC_RCVCONTROL; 1308 rxq->rxq_reg_rcvptr = GMAC_RCVPTR; 1309 rxq->rxq_reg_rcvsts0 = GMAC_RCVSTATUS0; 1310 rxq->rxq_reg_rcvsts1 = GMAC_RCVSTATUS1; 1311 1312 return 0; 1313 } 1314 1315 static bool 1316 bcmeth_txq_active_p( 1317 struct bcmeth_softc * const sc, 1318 struct bcmeth_txqueue *txq) 1319 { 1320 return !IF_IS_EMPTY(&txq->txq_mbufs); 1321 } 1322 1323 static bool 1324 bcmeth_txq_fillable_p( 1325 struct bcmeth_softc * const sc, 1326 struct bcmeth_txqueue *txq) 1327 { 1328 return txq->txq_free >= txq->txq_threshold; 1329 } 1330 1331 static int 1332 bcmeth_txq_attach( 1333 struct bcmeth_softc *sc, 1334 struct bcmeth_txqueue *txq, 1335 u_int qno) 1336 { 1337 size_t desc_count = BCMETH_RINGSIZE / sizeof(txq->txq_first[0]); 1338 int error; 1339 void *descs; 1340 1341 KASSERT(desc_count == 256 || desc_count == 512); 1342 1343 error = bcmeth_dmamem_alloc(sc->sc_dmat, BCMETH_RINGSIZE, 1344 &txq->txq_descmap_seg, &txq->txq_descmap, &descs); 1345 if (error) 1346 return error; 1347 1348 memset(descs, 0, BCMETH_RINGSIZE); 1349 txq->txq_first = descs; 1350 txq->txq_last = txq->txq_first + desc_count; 1351 txq->txq_consumer = descs; 1352 txq->txq_producer = descs; 1353 1354 IFQ_SET_MAXLEN(&txq->txq_mbufs, BCMETH_MAXTXMBUFS); 1355 1356 txq->txq_reg_xmtaddrlo = GMAC_XMTADDR_LOW; 1357 txq->txq_reg_xmtctl = GMAC_XMTCONTROL; 1358 txq->txq_reg_xmtptr = GMAC_XMTPTR; 1359 txq->txq_reg_xmtsts0 = GMAC_XMTSTATUS0; 1360 txq->txq_reg_xmtsts1 = GMAC_XMTSTATUS1; 1361 1362 bcmeth_txq_reset(sc, txq); 1363 1364 return 0; 1365 } 1366 1367 static int 1368 bcmeth_txq_map_load( 1369 struct bcmeth_softc *sc, 1370 struct bcmeth_txqueue *txq, 1371 struct mbuf *m) 1372 { 1373 bus_dmamap_t map; 1374 int error; 1375 1376 map = M_GETCTX(m, bus_dmamap_t); 1377 if (map != NULL) 1378 return 0; 1379 1380 map = bcmeth_mapcache_get(sc, sc->sc_tx_mapcache); 1381 if (map == NULL) 1382 return ENOMEM; 1383 1384 error = bus_dmamap_load_mbuf(sc->sc_dmat, map, m, 1385 BUS_DMA_WRITE | BUS_DMA_NOWAIT); 1386 if (error) 1387 return error; 1388 1389 bus_dmamap_sync(sc->sc_dmat, map, 0, m->m_pkthdr.len, 1390 BUS_DMASYNC_PREWRITE); 1391 M_SETCTX(m, map); 1392 return 0; 1393 } 1394 1395 static void 1396 bcmeth_txq_map_unload( 1397 struct bcmeth_softc *sc, 1398 struct bcmeth_txqueue *txq, 1399 struct mbuf *m) 1400 { 1401 KASSERT(m); 1402 bus_dmamap_t map = M_GETCTX(m, bus_dmamap_t); 1403 bus_dmamap_sync(sc->sc_dmat, map, 0, map->dm_mapsize, 1404 BUS_DMASYNC_POSTWRITE); 1405 bus_dmamap_unload(sc->sc_dmat, map); 1406 bcmeth_mapcache_put(sc, sc->sc_tx_mapcache, map); 1407 } 1408 1409 static bool 1410 bcmeth_txq_produce( 1411 struct bcmeth_softc *sc, 1412 struct bcmeth_txqueue *txq, 1413 struct mbuf *m) 1414 { 1415 bus_dmamap_t map = M_GETCTX(m, bus_dmamap_t); 1416 1417 if (map->dm_nsegs > txq->txq_free) 1418 return false; 1419 1420 /* 1421 * TCP Offload flag must be set in the first descriptor. 1422 */ 1423 struct gmac_txdb *producer = txq->txq_producer; 1424 uint32_t first_flags = TXDB_FLAG_SF; 1425 uint32_t last_flags = TXDB_FLAG_EF; 1426 1427 /* 1428 * If we've produced enough descriptors without consuming any 1429 * we need to ask for an interrupt to reclaim some. 1430 */ 1431 txq->txq_lastintr += map->dm_nsegs; 1432 if (txq->txq_lastintr >= txq->txq_threshold 1433 || txq->txq_mbufs.ifq_len + 1 == txq->txq_mbufs.ifq_maxlen) { 1434 txq->txq_lastintr = 0; 1435 last_flags |= TXDB_FLAG_IC; 1436 } 1437 1438 KASSERT(producer != txq->txq_last); 1439 1440 struct gmac_txdb *start = producer; 1441 size_t count = map->dm_nsegs; 1442 producer->txdb_flags |= htole32(first_flags); 1443 producer->txdb_addrlo = htole32(map->dm_segs[0].ds_addr); 1444 producer->txdb_buflen = htole32(map->dm_segs[0].ds_len); 1445 for (u_int i = 1; i < map->dm_nsegs; i++) { 1446 #if 0 1447 printf("[%zu]: %#x/%#x/%#x/%#x\n", producer - txq->txq_first, 1448 le32toh(producer->txdb_flags), 1449 le32toh(producer->txdb_buflen), 1450 le32toh(producer->txdb_addrlo), 1451 le32toh(producer->txdb_addrhi)); 1452 #endif 1453 if (__predict_false(++producer == txq->txq_last)) { 1454 bcmeth_txq_desc_presync(sc, txq, start, 1455 txq->txq_last - start); 1456 count -= txq->txq_last - start; 1457 producer = txq->txq_first; 1458 start = txq->txq_first; 1459 } 1460 producer->txdb_addrlo = htole32(map->dm_segs[i].ds_addr); 1461 producer->txdb_buflen = htole32(map->dm_segs[i].ds_len); 1462 } 1463 producer->txdb_flags |= htole32(last_flags); 1464 #if 0 1465 printf("[%zu]: %#x/%#x/%#x/%#x\n", producer - txq->txq_first, 1466 le32toh(producer->txdb_flags), le32toh(producer->txdb_buflen), 1467 le32toh(producer->txdb_addrlo), le32toh(producer->txdb_addrhi)); 1468 #endif 1469 if (count) 1470 bcmeth_txq_desc_presync(sc, txq, start, count); 1471 1472 /* 1473 * Reduce free count by the number of segments we consumed. 1474 */ 1475 txq->txq_free -= map->dm_nsegs; 1476 KASSERT(map->dm_nsegs == 1 || txq->txq_producer != producer); 1477 KASSERT(map->dm_nsegs == 1 1478 || (txq->txq_producer->txdb_flags & htole32(TXDB_FLAG_EF)) == 0); 1479 KASSERT(producer->txdb_flags & htole32(TXDB_FLAG_EF)); 1480 1481 #if 0 1482 printf("%s: mbuf %p: produced a %u byte packet in %u segments " 1483 "(%zd..%zd)\n", __func__, m, m->m_pkthdr.len, map->dm_nsegs, 1484 txq->txq_producer - txq->txq_first, producer - txq->txq_first); 1485 #endif 1486 1487 if (producer + 1 == txq->txq_last) 1488 txq->txq_producer = txq->txq_first; 1489 else 1490 txq->txq_producer = producer + 1; 1491 IF_ENQUEUE(&txq->txq_mbufs, m); 1492 1493 /* 1494 * Let the transmitter know there's more to do 1495 */ 1496 bcmeth_write_4(sc, txq->txq_reg_xmtptr, 1497 txq->txq_descmap->dm_segs[0].ds_addr 1498 + ((uintptr_t)txq->txq_producer & XMT_LASTDSCR)); 1499 1500 return true; 1501 } 1502 1503 static struct mbuf * 1504 bcmeth_copy_packet(struct mbuf *m) 1505 { 1506 struct mbuf *mext = NULL; 1507 size_t misalignment = 0; 1508 size_t hlen = 0; 1509 1510 for (mext = m; mext != NULL; mext = mext->m_next) { 1511 if (mext->m_flags & M_EXT) { 1512 misalignment = mtod(mext, vaddr_t) & arm_dcache_align; 1513 break; 1514 } 1515 hlen += m->m_len; 1516 } 1517 1518 struct mbuf *n = m->m_next; 1519 if (m != mext && hlen + misalignment <= MHLEN && false) { 1520 KASSERT(m->m_pktdat <= m->m_data 1521 && m->m_data <= &m->m_pktdat[MHLEN - m->m_len]); 1522 size_t oldoff = m->m_data - m->m_pktdat; 1523 size_t off; 1524 if (mext == NULL) { 1525 off = (oldoff + hlen > MHLEN) ? 0 : oldoff; 1526 } else { 1527 off = MHLEN - (hlen + misalignment); 1528 } 1529 KASSERT(off + hlen + misalignment <= MHLEN); 1530 if (((oldoff ^ off) & arm_dcache_align) != 0 || off < oldoff) { 1531 memmove(&m->m_pktdat[off], m->m_data, m->m_len); 1532 m->m_data = &m->m_pktdat[off]; 1533 } 1534 m_copydata(n, 0, hlen - m->m_len, &m->m_data[m->m_len]); 1535 m->m_len = hlen; 1536 m->m_next = mext; 1537 while (n != mext) { 1538 n = m_free(n); 1539 } 1540 return m; 1541 } 1542 1543 struct mbuf *m0 = m_gethdr(M_DONTWAIT, m->m_type); 1544 if (m0 == NULL) { 1545 return NULL; 1546 } 1547 m_copy_pkthdr(m0, m); 1548 MCLAIM(m0, m->m_owner); 1549 if (m0->m_pkthdr.len > MHLEN) { 1550 MCLGET(m0, M_DONTWAIT); 1551 if ((m0->m_flags & M_EXT) == 0) { 1552 m_freem(m0); 1553 return NULL; 1554 } 1555 } 1556 m0->m_len = m->m_pkthdr.len; 1557 m_copydata(m, 0, m0->m_len, mtod(m0, void *)); 1558 m_freem(m); 1559 return m0; 1560 } 1561 1562 static bool 1563 bcmeth_txq_enqueue( 1564 struct bcmeth_softc *sc, 1565 struct bcmeth_txqueue *txq) 1566 { 1567 for (;;) { 1568 if (IF_QFULL(&txq->txq_mbufs)) 1569 return false; 1570 struct mbuf *m = txq->txq_next; 1571 if (m == NULL) { 1572 int s = splnet(); 1573 IF_DEQUEUE(&sc->sc_if.if_snd, m); 1574 splx(s); 1575 if (m == NULL) 1576 return true; 1577 M_SETCTX(m, NULL); 1578 } else { 1579 txq->txq_next = NULL; 1580 } 1581 /* 1582 * If LINK2 is set and this packet uses multiple mbufs, 1583 * consolidate it into a single mbuf. 1584 */ 1585 if (m->m_next != NULL && (sc->sc_if.if_flags & IFF_LINK2)) { 1586 struct mbuf *m0 = bcmeth_copy_packet(m); 1587 if (m0 == NULL) { 1588 txq->txq_next = m; 1589 return true; 1590 } 1591 m = m0; 1592 } 1593 int error = bcmeth_txq_map_load(sc, txq, m); 1594 if (error) { 1595 aprint_error_dev(sc->sc_dev, 1596 "discarded packet due to " 1597 "dmamap load failure: %d\n", error); 1598 m_freem(m); 1599 continue; 1600 } 1601 KASSERT(txq->txq_next == NULL); 1602 if (!bcmeth_txq_produce(sc, txq, m)) { 1603 txq->txq_next = m; 1604 return false; 1605 } 1606 KASSERT(txq->txq_next == NULL); 1607 } 1608 } 1609 1610 static bool 1611 bcmeth_txq_consume( 1612 struct bcmeth_softc *sc, 1613 struct bcmeth_txqueue *txq) 1614 { 1615 struct ifnet * const ifp = &sc->sc_if; 1616 struct gmac_txdb *consumer = txq->txq_consumer; 1617 size_t txfree = 0; 1618 1619 #if 0 1620 printf("%s: entry: free=%zu\n", __func__, txq->txq_free); 1621 #endif 1622 1623 for (;;) { 1624 if (consumer == txq->txq_producer) { 1625 txq->txq_consumer = consumer; 1626 txq->txq_free += txfree; 1627 txq->txq_lastintr -= uimin(txq->txq_lastintr, txfree); 1628 #if 0 1629 printf("%s: empty: freed %zu descriptors going from " 1630 "%zu to %zu\n", __func__, txfree, 1631 txq->txq_free - txfree, txq->txq_free); 1632 #endif 1633 KASSERT(txq->txq_lastintr == 0); 1634 KASSERT(txq->txq_free 1635 == txq->txq_last - txq->txq_first - 1); 1636 return true; 1637 } 1638 bcmeth_txq_desc_postsync(sc, txq, consumer, 1); 1639 uint32_t s0 = bcmeth_read_4(sc, txq->txq_reg_xmtsts0); 1640 if (consumer == txq->txq_first + __SHIFTOUT(s0, XMT_CURRDSCR)) { 1641 txq->txq_consumer = consumer; 1642 txq->txq_free += txfree; 1643 txq->txq_lastintr -= uimin(txq->txq_lastintr, txfree); 1644 #if 0 1645 printf("%s: freed %zu descriptors\n", 1646 __func__, txfree); 1647 #endif 1648 return bcmeth_txq_fillable_p(sc, txq); 1649 } 1650 1651 /* 1652 * If this is the last descriptor in the chain, get the 1653 * mbuf, free its dmamap, and free the mbuf chain itself. 1654 */ 1655 const uint32_t txdb_flags = le32toh(consumer->txdb_flags); 1656 if (txdb_flags & TXDB_FLAG_EF) { 1657 struct mbuf *m; 1658 1659 IF_DEQUEUE(&txq->txq_mbufs, m); 1660 KASSERT(m); 1661 bcmeth_txq_map_unload(sc, txq, m); 1662 #if 0 1663 printf("%s: mbuf %p: consumed a %u byte packet\n", 1664 __func__, m, m->m_pkthdr.len); 1665 #endif 1666 bpf_mtap(ifp, m, BPF_D_OUT); 1667 if_statinc(ifp, if_opackets); 1668 if_statadd(ifp, if_obytes, m->m_pkthdr.len); 1669 if (m->m_flags & M_MCAST) 1670 if_statinc(ifp, if_omcasts); 1671 m_freem(m); 1672 } 1673 1674 /* 1675 * We own this packet again. Clear all flags except wrap. 1676 */ 1677 txfree++; 1678 1679 /* 1680 * Wrap at the last entry! 1681 */ 1682 if (txdb_flags & TXDB_FLAG_ET) { 1683 consumer->txdb_flags = htole32(TXDB_FLAG_ET); 1684 KASSERT(consumer + 1 == txq->txq_last); 1685 consumer = txq->txq_first; 1686 } else { 1687 consumer->txdb_flags = 0; 1688 consumer++; 1689 KASSERT(consumer < txq->txq_last); 1690 } 1691 } 1692 } 1693 1694 static void 1695 bcmeth_txq_purge( 1696 struct bcmeth_softc *sc, 1697 struct bcmeth_txqueue *txq) 1698 { 1699 struct mbuf *m; 1700 KASSERT((bcmeth_read_4(sc, UNIMAC_COMMAND_CONFIG) & TX_ENA) == 0); 1701 1702 for (;;) { 1703 IF_DEQUEUE(&txq->txq_mbufs, m); 1704 if (m == NULL) 1705 break; 1706 bcmeth_txq_map_unload(sc, txq, m); 1707 m_freem(m); 1708 } 1709 if ((m = txq->txq_next) != NULL) { 1710 txq->txq_next = NULL; 1711 bcmeth_txq_map_unload(sc, txq, m); 1712 m_freem(m); 1713 } 1714 } 1715 1716 static void 1717 bcmeth_txq_reset( 1718 struct bcmeth_softc *sc, 1719 struct bcmeth_txqueue *txq) 1720 { 1721 /* 1722 * sync all the descriptors 1723 */ 1724 bcmeth_txq_desc_postsync(sc, txq, txq->txq_first, 1725 txq->txq_last - txq->txq_first); 1726 1727 /* 1728 * Make sure we own all descriptors in the ring. 1729 */ 1730 struct gmac_txdb *txdb; 1731 for (txdb = txq->txq_first; txdb < txq->txq_last - 1; txdb++) { 1732 txdb->txdb_flags = 0; 1733 } 1734 1735 /* 1736 * Last descriptor has the wrap flag. 1737 */ 1738 txdb->txdb_flags = htole32(TXDB_FLAG_ET); 1739 1740 /* 1741 * Reset the producer consumer indexes. 1742 */ 1743 txq->txq_consumer = txq->txq_first; 1744 txq->txq_producer = txq->txq_first; 1745 txq->txq_free = txq->txq_last - txq->txq_first - 1; 1746 txq->txq_threshold = txq->txq_free / 2; 1747 txq->txq_lastintr = 0; 1748 1749 /* 1750 * What do we want to get interrupted on? 1751 */ 1752 sc->sc_intmask |= XMTINT_0 | XMTUF; 1753 1754 /* 1755 * Restart the transmiter at the first descriptor 1756 */ 1757 bcmeth_write_4(sc, txq->txq_reg_xmtaddrlo, 1758 txq->txq_descmap->dm_segs->ds_addr); 1759 } 1760 1761 static void 1762 bcmeth_ifstart(struct ifnet *ifp) 1763 { 1764 struct bcmeth_softc * const sc = ifp->if_softc; 1765 1766 if (__predict_false((ifp->if_flags & IFF_RUNNING) == 0)) { 1767 return; 1768 } 1769 1770 #ifdef BCMETH_MPSAFETX 1771 if (cpu_intr_p()) { 1772 #endif 1773 atomic_or_uint(&sc->sc_soft_flags, SOFT_TXINTR); 1774 softint_schedule(sc->sc_soft_ih); 1775 #ifdef BCMETH_MPSAFETX 1776 } else { 1777 /* 1778 * Either we are in a softintr thread already or some other 1779 * thread so just borrow it to do the send and save ourselves 1780 * the overhead of a fast soft int. 1781 */ 1782 bcmeth_soft_txintr(sc); 1783 } 1784 #endif 1785 } 1786 1787 int 1788 bcmeth_intr(void *arg) 1789 { 1790 struct bcmeth_softc * const sc = arg; 1791 uint32_t soft_flags = 0; 1792 uint32_t work_flags = 0; 1793 int rv = 0; 1794 1795 mutex_enter(sc->sc_hwlock); 1796 1797 uint32_t intmask = sc->sc_intmask; 1798 BCMETH_EVCNT_INCR(sc->sc_ev_intr); 1799 1800 for (;;) { 1801 uint32_t intstatus = bcmeth_read_4(sc, GMAC_INTSTATUS); 1802 intstatus &= intmask; 1803 bcmeth_write_4(sc, GMAC_INTSTATUS, intstatus); /* write 1 to clear */ 1804 if (intstatus == 0) { 1805 break; 1806 } 1807 #if 0 1808 aprint_normal_dev(sc->sc_dev, "%s: intstatus=%#x intmask=%#x\n", 1809 __func__, intstatus, bcmeth_read_4(sc, GMAC_INTMASK)); 1810 #endif 1811 if (intstatus & RCVINT) { 1812 struct bcmeth_rxqueue * const rxq = &sc->sc_rxq; 1813 intmask &= ~RCVINT; 1814 1815 uint32_t rcvsts0 = bcmeth_read_4(sc, rxq->rxq_reg_rcvsts0); 1816 uint32_t descs = __SHIFTOUT(rcvsts0, RCV_CURRDSCR); 1817 if (descs < rxq->rxq_consumer - rxq->rxq_first) { 1818 /* 1819 * We wrapped at the end so count how far 1820 * we are from the end. 1821 */ 1822 descs += rxq->rxq_last - rxq->rxq_consumer; 1823 } else { 1824 descs -= rxq->rxq_consumer - rxq->rxq_first; 1825 } 1826 /* 1827 * If we "timedout" we can't be hogging so use 1828 * softints. If we exceeded then we might hogging 1829 * so let the workqueue deal with them. 1830 */ 1831 const uint32_t framecount = __SHIFTOUT(sc->sc_rcvlazy, 1832 INTRCVLAZY_FRAMECOUNT); 1833 if (descs < framecount 1834 || (curcpu()->ci_curlwp->l_flag & LW_IDLE)) { 1835 soft_flags |= SOFT_RXINTR; 1836 } else { 1837 work_flags |= WORK_RXINTR; 1838 } 1839 } 1840 1841 if (intstatus & XMTINT_0) { 1842 intmask &= ~XMTINT_0; 1843 soft_flags |= SOFT_TXINTR; 1844 } 1845 1846 if (intstatus & RCVDESCUF) { 1847 intmask &= ~RCVDESCUF; 1848 work_flags |= WORK_RXUNDERFLOW; 1849 } 1850 1851 intstatus &= intmask; 1852 if (intstatus) { 1853 aprint_error_dev(sc->sc_dev, 1854 "intr: intstatus=%#x\n", intstatus); 1855 aprint_error_dev(sc->sc_dev, 1856 "rcvbase=%p/%#lx rcvptr=%#x rcvsts=%#x/%#x\n", 1857 sc->sc_rxq.rxq_first, 1858 sc->sc_rxq.rxq_descmap->dm_segs[0].ds_addr, 1859 bcmeth_read_4(sc, sc->sc_rxq.rxq_reg_rcvptr), 1860 bcmeth_read_4(sc, sc->sc_rxq.rxq_reg_rcvsts0), 1861 bcmeth_read_4(sc, sc->sc_rxq.rxq_reg_rcvsts1)); 1862 aprint_error_dev(sc->sc_dev, 1863 "xmtbase=%p/%#lx xmtptr=%#x xmtsts=%#x/%#x\n", 1864 sc->sc_txq.txq_first, 1865 sc->sc_txq.txq_descmap->dm_segs[0].ds_addr, 1866 bcmeth_read_4(sc, sc->sc_txq.txq_reg_xmtptr), 1867 bcmeth_read_4(sc, sc->sc_txq.txq_reg_xmtsts0), 1868 bcmeth_read_4(sc, sc->sc_txq.txq_reg_xmtsts1)); 1869 intmask &= ~intstatus; 1870 work_flags |= WORK_REINIT; 1871 break; 1872 } 1873 } 1874 1875 if (intmask != sc->sc_intmask) { 1876 bcmeth_write_4(sc, GMAC_INTMASK, sc->sc_intmask); 1877 } 1878 1879 if (work_flags) { 1880 if (sc->sc_work_flags == 0) { 1881 workqueue_enqueue(sc->sc_workq, &sc->sc_work, NULL); 1882 } 1883 atomic_or_32(&sc->sc_work_flags, work_flags); 1884 rv = 1; 1885 } 1886 1887 if (soft_flags) { 1888 if (sc->sc_soft_flags == 0) { 1889 softint_schedule(sc->sc_soft_ih); 1890 } 1891 atomic_or_32(&sc->sc_soft_flags, soft_flags); 1892 rv = 1; 1893 } 1894 1895 mutex_exit(sc->sc_hwlock); 1896 1897 return rv; 1898 } 1899 1900 #ifdef BCMETH_MPSAFETX 1901 void 1902 bcmeth_soft_txintr(struct bcmeth_softc *sc) 1903 { 1904 mutex_enter(sc->sc_lock); 1905 /* 1906 * Let's do what we came here for. Consume transmitted 1907 * packets off the transmit ring. 1908 */ 1909 if (!bcmeth_txq_consume(sc, &sc->sc_txq) 1910 || !bcmeth_txq_enqueue(sc, &sc->sc_txq)) { 1911 BCMETH_EVCNT_INCR(sc->sc_ev_tx_stall); 1912 sc->sc_if.if_flags |= IFF_OACTIVE; 1913 } else { 1914 sc->sc_if.if_flags &= ~IFF_OACTIVE; 1915 } 1916 if (sc->sc_if.if_flags & IFF_RUNNING) { 1917 mutex_spin_enter(sc->sc_hwlock); 1918 sc->sc_intmask |= XMTINT_0; 1919 bcmeth_write_4(sc, GMAC_INTMASK, sc->sc_intmask); 1920 mutex_spin_exit(sc->sc_hwlock); 1921 } 1922 mutex_exit(sc->sc_lock); 1923 } 1924 #endif /* BCMETH_MPSAFETX */ 1925 1926 void 1927 bcmeth_soft_intr(void *arg) 1928 { 1929 struct bcmeth_softc * const sc = arg; 1930 struct ifnet * const ifp = &sc->sc_if; 1931 uint32_t intmask = 0; 1932 1933 mutex_enter(sc->sc_lock); 1934 1935 u_int soft_flags = atomic_swap_uint(&sc->sc_soft_flags, 0); 1936 1937 BCMETH_EVCNT_INCR(sc->sc_ev_soft_intr); 1938 1939 if ((soft_flags & SOFT_TXINTR) 1940 || bcmeth_txq_active_p(sc, &sc->sc_txq)) { 1941 /* 1942 * Let's do what we came here for. Consume transmitted 1943 * packets off the transmit ring. 1944 */ 1945 if (!bcmeth_txq_consume(sc, &sc->sc_txq) 1946 || !bcmeth_txq_enqueue(sc, &sc->sc_txq)) { 1947 BCMETH_EVCNT_INCR(sc->sc_ev_tx_stall); 1948 ifp->if_flags |= IFF_OACTIVE; 1949 } else { 1950 ifp->if_flags &= ~IFF_OACTIVE; 1951 } 1952 intmask |= XMTINT_0; 1953 } 1954 1955 if (soft_flags & SOFT_RXINTR) { 1956 /* 1957 * Let's consume 1958 */ 1959 while (bcmeth_rxq_consume(sc, &sc->sc_rxq, 1960 sc->sc_rxq.rxq_threshold / 4)) { 1961 /* 1962 * We've consumed a quarter of the ring and still have 1963 * more to do. Refill the ring. 1964 */ 1965 bcmeth_rxq_produce(sc, &sc->sc_rxq); 1966 } 1967 intmask |= RCVINT; 1968 } 1969 1970 if (ifp->if_flags & IFF_RUNNING) { 1971 bcmeth_rxq_produce(sc, &sc->sc_rxq); 1972 mutex_spin_enter(sc->sc_hwlock); 1973 sc->sc_intmask |= intmask; 1974 bcmeth_write_4(sc, GMAC_INTMASK, sc->sc_intmask); 1975 mutex_spin_exit(sc->sc_hwlock); 1976 } 1977 1978 mutex_exit(sc->sc_lock); 1979 } 1980 1981 void 1982 bcmeth_worker(struct work *wk, void *arg) 1983 { 1984 struct bcmeth_softc * const sc = arg; 1985 struct ifnet * const ifp = &sc->sc_if; 1986 uint32_t intmask = 0; 1987 1988 mutex_enter(sc->sc_lock); 1989 1990 BCMETH_EVCNT_INCR(sc->sc_ev_work); 1991 1992 uint32_t work_flags = atomic_swap_32(&sc->sc_work_flags, 0); 1993 if (work_flags & WORK_REINIT) { 1994 int s = splnet(); 1995 sc->sc_soft_flags = 0; 1996 bcmeth_ifinit(ifp); 1997 splx(s); 1998 work_flags &= ~WORK_RXUNDERFLOW; 1999 } 2000 2001 if (work_flags & WORK_RXUNDERFLOW) { 2002 struct bcmeth_rxqueue * const rxq = &sc->sc_rxq; 2003 size_t threshold = 5 * rxq->rxq_threshold / 4; 2004 if (threshold >= rxq->rxq_last - rxq->rxq_first) { 2005 threshold = rxq->rxq_last - rxq->rxq_first - 1; 2006 } else { 2007 intmask |= RCVDESCUF; 2008 } 2009 aprint_normal_dev(sc->sc_dev, 2010 "increasing receive buffers from %zu to %zu\n", 2011 rxq->rxq_threshold, threshold); 2012 rxq->rxq_threshold = threshold; 2013 } 2014 2015 if (work_flags & WORK_RXINTR) { 2016 /* 2017 * Let's consume 2018 */ 2019 while (bcmeth_rxq_consume(sc, &sc->sc_rxq, 2020 sc->sc_rxq.rxq_threshold / 4)) { 2021 /* 2022 * We've consumed a quarter of the ring and still have 2023 * more to do. Refill the ring. 2024 */ 2025 bcmeth_rxq_produce(sc, &sc->sc_rxq); 2026 } 2027 intmask |= RCVINT; 2028 } 2029 2030 if (ifp->if_flags & IFF_RUNNING) { 2031 bcmeth_rxq_produce(sc, &sc->sc_rxq); 2032 #if 0 2033 uint32_t intstatus = bcmeth_read_4(sc, GMAC_INTSTATUS); 2034 if (intstatus & RCVINT) { 2035 bcmeth_write_4(sc, GMAC_INTSTATUS, RCVINT); 2036 work_flags |= WORK_RXINTR; 2037 continue; 2038 } 2039 #endif 2040 mutex_spin_enter(sc->sc_hwlock); 2041 sc->sc_intmask |= intmask; 2042 bcmeth_write_4(sc, GMAC_INTMASK, sc->sc_intmask); 2043 mutex_spin_exit(sc->sc_hwlock); 2044 } 2045 2046 mutex_exit(sc->sc_lock); 2047 } 2048
Indexes created Mon Oct 20 01:09:56 GMT 2025