bcm53xx_eth.c revision 1.8
1 /*- 2 * Copyright (c) 2012 The NetBSD Foundation, Inc. 3 * All rights reserved. 4 * 5 * This code is derived from software contributed to The NetBSD Foundation 6 * by Matt Thomas of 3am Software Foundry. 7 * 8 * Redistribution and use in source and binary forms, with or without 9 * modification, are permitted provided that the following conditions 10 * are met: 11 * 1. Redistributions of source code must retain the above copyright 12 * notice, this list of conditions and the following disclaimer. 13 * 2. Redistributions in binary form must reproduce the above copyright 14 * notice, this list of conditions and the following disclaimer in the 15 * documentation and/or other materials provided with the distribution. 16 * 17 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS 18 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED 19 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 20 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS 21 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 22 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 23 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 24 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 25 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 26 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 27 * POSSIBILITY OF SUCH DAMAGE. 28 */ 29 30 #define GMAC_PRIVATE 31 32 #include "locators.h" 33 34 #include <sys/cdefs.h> 35 36 __KERNEL_RCSID(1, "$NetBSD: bcm53xx_eth.c,v 1.8 2012/10/07 20:14:08 matt Exp $"); 37 38 #include <sys/param.h> 39 #include <sys/atomic.h> 40 #include <sys/bus.h> 41 #include <sys/device.h> 42 #include <sys/ioctl.h> 43 #include <sys/intr.h> 44 #include <sys/kmem.h> 45 #include <sys/mutex.h> 46 #include <sys/socket.h> 47 #include <sys/systm.h> 48 #include <sys/workqueue.h> 49 50 #include <net/if.h> 51 #include <net/if_ether.h> 52 #include <net/if_media.h> 53 54 #include <net/if_dl.h> 55 56 #include <net/bpf.h> 57 58 #include <dev/mii/miivar.h> 59 60 #include <arm/broadcom/bcm53xx_reg.h> 61 #include <arm/broadcom/bcm53xx_var.h> 62 63 #define BCMETH_RCVOFFSET 6 64 #define BCMETH_MAXTXMBUFS 32 65 #define BCMETH_NTXSEGS 30 66 #define BCMETH_MAXRXMBUFS 255 67 #define BCMETH_MINRXMBUFS 64 68 #define BCMETH_NRXSEGS 1 69 #define BCMETH_RINGSIZE PAGE_SIZE 70 71 static int bcmeth_ccb_match(device_t, cfdata_t, void *); 72 static void bcmeth_ccb_attach(device_t, device_t, void *); 73 74 struct bcmeth_txqueue { 75 bus_dmamap_t txq_descmap; 76 struct gmac_txdb *txq_consumer; 77 struct gmac_txdb *txq_producer; 78 struct gmac_txdb *txq_first; 79 struct gmac_txdb *txq_last; 80 struct ifqueue txq_mbufs; 81 struct mbuf *txq_next; 82 size_t txq_free; 83 size_t txq_threshold; 84 size_t txq_lastintr; 85 bus_size_t txq_reg_xmtaddrlo; 86 bus_size_t txq_reg_xmtptr; 87 bus_size_t txq_reg_xmtctl; 88 bus_size_t txq_reg_xmtsts0; 89 bus_dma_segment_t txq_descmap_seg; 90 }; 91 92 struct bcmeth_rxqueue { 93 bus_dmamap_t rxq_descmap; 94 struct gmac_rxdb *rxq_consumer; 95 struct gmac_rxdb *rxq_producer; 96 struct gmac_rxdb *rxq_first; 97 struct gmac_rxdb *rxq_last; 98 struct mbuf *rxq_mhead; 99 struct mbuf **rxq_mtail; 100 struct mbuf *rxq_mconsumer; 101 size_t rxq_inuse; 102 size_t rxq_threshold; 103 bus_size_t rxq_reg_rcvaddrlo; 104 bus_size_t rxq_reg_rcvptr; 105 bus_size_t rxq_reg_rcvctl; 106 bus_size_t rxq_reg_rcvsts0; 107 bus_dma_segment_t rxq_descmap_seg; 108 }; 109 110 struct bcmeth_mapcache { 111 u_int dmc_nmaps; 112 u_int dmc_maxseg; 113 u_int dmc_maxmaps; 114 u_int dmc_maxmapsize; 115 bus_dmamap_t dmc_maps[0]; 116 }; 117 118 struct bcmeth_softc { 119 device_t sc_dev; 120 bus_space_tag_t sc_bst; 121 bus_space_handle_t sc_bsh; 122 bus_dma_tag_t sc_dmat; 123 kmutex_t *sc_lock; 124 kmutex_t *sc_hwlock; 125 struct ethercom sc_ec; 126 #define sc_if sc_ec.ec_if 127 struct ifmedia sc_media; 128 void *sc_soft_ih; 129 void *sc_ih; 130 131 struct bcmeth_rxqueue sc_rxq; 132 struct bcmeth_txqueue sc_txq; 133 134 uint32_t sc_maxfrm; 135 uint32_t sc_cmdcfg; 136 uint32_t sc_intmask; 137 uint32_t sc_rcvlazy; 138 volatile uint32_t sc_soft_flags; 139 #define SOFT_RXINTR 0x01 140 #define SOFT_TXINTR 0x02 141 142 struct evcnt sc_ev_intr; 143 struct evcnt sc_ev_soft_intr; 144 struct evcnt sc_ev_work;; 145 struct evcnt sc_ev_tx_stall; 146 147 struct ifqueue sc_rx_bufcache; 148 struct bcmeth_mapcache *sc_rx_mapcache; 149 struct bcmeth_mapcache *sc_tx_mapcache; 150 151 struct workqueue *sc_workq; 152 struct work sc_work; 153 154 volatile uint32_t sc_work_flags; 155 #define WORK_RXINTR 0x01 156 #define WORK_RXUNDERFLOW 0x02 157 #define WORK_REINIT 0x04 158 159 uint8_t sc_enaddr[ETHER_ADDR_LEN]; 160 }; 161 162 static void bcmeth_ifstart(struct ifnet *); 163 static void bcmeth_ifwatchdog(struct ifnet *); 164 static int bcmeth_ifinit(struct ifnet *); 165 static void bcmeth_ifstop(struct ifnet *, int); 166 static int bcmeth_ifioctl(struct ifnet *, u_long, void *); 167 168 static int bcmeth_mapcache_create(struct bcmeth_softc *, 169 struct bcmeth_mapcache **, size_t, size_t, size_t); 170 static void bcmeth_mapcache_destroy(struct bcmeth_softc *, 171 struct bcmeth_mapcache *); 172 static bus_dmamap_t bcmeth_mapcache_get(struct bcmeth_softc *, 173 struct bcmeth_mapcache *); 174 static void bcmeth_mapcache_put(struct bcmeth_softc *, 175 struct bcmeth_mapcache *, bus_dmamap_t); 176 177 static int bcmeth_txq_attach(struct bcmeth_softc *, 178 struct bcmeth_txqueue *, u_int); 179 static void bcmeth_txq_purge(struct bcmeth_softc *, 180 struct bcmeth_txqueue *); 181 static void bcmeth_txq_reset(struct bcmeth_softc *, 182 struct bcmeth_txqueue *); 183 static bool bcmeth_txq_consume(struct bcmeth_softc *, 184 struct bcmeth_txqueue *); 185 static bool bcmeth_txq_produce(struct bcmeth_softc *, 186 struct bcmeth_txqueue *, struct mbuf *m); 187 static bool bcmeth_txq_active_p(struct bcmeth_softc *, 188 struct bcmeth_txqueue *); 189 190 static int bcmeth_rxq_attach(struct bcmeth_softc *, 191 struct bcmeth_rxqueue *, u_int); 192 static bool bcmeth_rxq_produce(struct bcmeth_softc *, 193 struct bcmeth_rxqueue *); 194 static void bcmeth_rxq_purge(struct bcmeth_softc *, 195 struct bcmeth_rxqueue *, bool); 196 static void bcmeth_rxq_reset(struct bcmeth_softc *, 197 struct bcmeth_rxqueue *); 198 199 static int bcmeth_intr(void *); 200 static void bcmeth_soft_intr(void *); 201 static void bcmeth_worker(struct work *, void *); 202 203 static int bcmeth_mediachange(struct ifnet *); 204 static void bcmeth_mediastatus(struct ifnet *, struct ifmediareq *); 205 206 static inline uint32_t 207 bcmeth_read_4(struct bcmeth_softc *sc, bus_size_t o) 208 { 209 return bus_space_read_4(sc->sc_bst, sc->sc_bsh, o); 210 } 211 212 static inline void 213 bcmeth_write_4(struct bcmeth_softc *sc, bus_size_t o, uint32_t v) 214 { 215 bus_space_write_4(sc->sc_bst, sc->sc_bsh, o, v); 216 } 217 218 CFATTACH_DECL_NEW(bcmeth_ccb, sizeof(struct bcmeth_softc), 219 bcmeth_ccb_match, bcmeth_ccb_attach, NULL, NULL); 220 221 static int 222 bcmeth_ccb_match(device_t parent, cfdata_t cf, void *aux) 223 { 224 struct bcmccb_attach_args * const ccbaa = aux; 225 const struct bcm_locators * const loc = &ccbaa->ccbaa_loc; 226 227 if (strcmp(cf->cf_name, loc->loc_name)) 228 return 0; 229 230 #ifdef DIAGNOSTIC 231 const int port = cf->cf_loc[BCMCCBCF_PORT]; 232 #endif 233 KASSERT(port == BCMCCBCF_PORT_DEFAULT || port == loc->loc_port); 234 235 return 1; 236 } 237 238 static void 239 bcmeth_ccb_attach(device_t parent, device_t self, void *aux) 240 { 241 struct bcmeth_softc * const sc = device_private(self); 242 struct ethercom * const ec = &sc->sc_ec; 243 struct ifnet * const ifp = &ec->ec_if; 244 struct bcmccb_attach_args * const ccbaa = aux; 245 const struct bcm_locators * const loc = &ccbaa->ccbaa_loc; 246 const char * const xname = device_xname(self); 247 prop_dictionary_t dict = device_properties(self); 248 int error; 249 250 sc->sc_bst = ccbaa->ccbaa_ccb_bst; 251 sc->sc_dmat = ccbaa->ccbaa_dmat; 252 bus_space_subregion(sc->sc_bst, ccbaa->ccbaa_ccb_bsh, 253 loc->loc_offset, loc->loc_size, &sc->sc_bsh); 254 255 prop_data_t eaprop = prop_dictionary_get(dict, "mac-address"); 256 if (eaprop == NULL) { 257 uint32_t mac0 = bcmeth_read_4(sc, UNIMAC_MAC_0); 258 uint32_t mac1 = bcmeth_read_4(sc, UNIMAC_MAC_1); 259 if ((mac0 == 0 && mac1 == 0) || (mac1 & 1)) { 260 aprint_error(": mac-address property is missing\n"); 261 return; 262 } 263 sc->sc_enaddr[0] = (mac0 >> 0) & 0xff; 264 sc->sc_enaddr[1] = (mac0 >> 8) & 0xff; 265 sc->sc_enaddr[2] = (mac0 >> 16) & 0xff; 266 sc->sc_enaddr[3] = (mac0 >> 24) & 0xff; 267 sc->sc_enaddr[4] = (mac1 >> 0) & 0xff; 268 sc->sc_enaddr[5] = (mac1 >> 8) & 0xff; 269 } else { 270 KASSERT(prop_object_type(eaprop) == PROP_TYPE_DATA); 271 KASSERT(prop_data_size(eaprop) == ETHER_ADDR_LEN); 272 memcpy(sc->sc_enaddr, prop_data_data_nocopy(eaprop), 273 ETHER_ADDR_LEN); 274 } 275 sc->sc_dev = self; 276 sc->sc_lock = mutex_obj_alloc(MUTEX_DEFAULT, IPL_SOFTNET); 277 sc->sc_hwlock = mutex_obj_alloc(MUTEX_DEFAULT, IPL_VM); 278 279 bcmeth_write_4(sc, GMAC_INTMASK, 0); // disable interrupts 280 281 aprint_naive("\n"); 282 aprint_normal(": Gigabit Ethernet Controller\n"); 283 284 error = bcmeth_rxq_attach(sc, &sc->sc_rxq, 0); 285 if (error) { 286 aprint_error(": failed to init rxq: %d\n", error); 287 return; 288 } 289 290 error = bcmeth_txq_attach(sc, &sc->sc_txq, 0); 291 if (error) { 292 aprint_error(": failed to init txq: %d\n", error); 293 return; 294 } 295 296 error = bcmeth_mapcache_create(sc, &sc->sc_rx_mapcache, 297 BCMETH_MAXRXMBUFS, MCLBYTES, BCMETH_NRXSEGS); 298 if (error) { 299 aprint_error(": failed to allocate rx dmamaps: %d\n", error); 300 return; 301 } 302 303 error = bcmeth_mapcache_create(sc, &sc->sc_tx_mapcache, 304 BCMETH_MAXTXMBUFS, MCLBYTES, BCMETH_NTXSEGS); 305 if (error) { 306 aprint_error(": failed to allocate tx dmamaps: %d\n", error); 307 return; 308 } 309 310 error = workqueue_create(&sc->sc_workq, xname, bcmeth_worker, sc, 311 (PRI_USER + MAXPRI_USER) / 2, IPL_SOFTNET, WQ_MPSAFE|WQ_PERCPU); 312 if (error) { 313 aprint_error(": failed to create workqueue: %d\n", error); 314 return; 315 } 316 317 sc->sc_soft_ih = softint_establish(SOFTINT_MPSAFE | SOFTINT_NET, 318 bcmeth_soft_intr, sc); 319 320 sc->sc_ih = intr_establish(loc->loc_intrs[0], IPL_VM, IST_LEVEL, 321 bcmeth_intr, sc); 322 323 if (sc->sc_ih == NULL) { 324 aprint_error_dev(self, "failed to establish interrupt %d\n", 325 loc->loc_intrs[0]); 326 } else { 327 aprint_normal_dev(self, "interrupting on irq %d\n", 328 loc->loc_intrs[0]); 329 } 330 331 aprint_normal_dev(sc->sc_dev, "Ethernet address %s\n", 332 ether_sprintf(sc->sc_enaddr)); 333 334 /* 335 * Since each port in plugged into the switch/flow-accelerator, 336 * we hard code at Gige Full-Duplex with Flow Control enabled. 337 */ 338 int ifmedia = IFM_ETHER|IFM_1000_T|IFM_FDX; 339 //ifmedia |= IFM_FLOW|IFM_ETH_TXPAUSE|IFM_ETH_RXPAUSE; 340 ifmedia_init(&sc->sc_media, IFM_IMASK, bcmeth_mediachange, 341 bcmeth_mediastatus); 342 ifmedia_add(&sc->sc_media, ifmedia, 0, NULL); 343 ifmedia_set(&sc->sc_media, ifmedia); 344 345 ec->ec_capabilities = ETHERCAP_VLAN_MTU | ETHERCAP_JUMBO_MTU; 346 347 strlcpy(ifp->if_xname, xname, IFNAMSIZ); 348 ifp->if_softc = sc; 349 ifp->if_baudrate = IF_Mbps(1000); 350 ifp->if_capabilities = 0; 351 ifp->if_flags = IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST; 352 ifp->if_ioctl = bcmeth_ifioctl; 353 ifp->if_start = bcmeth_ifstart; 354 ifp->if_watchdog = bcmeth_ifwatchdog; 355 ifp->if_init = bcmeth_ifinit; 356 ifp->if_stop = bcmeth_ifstop; 357 IFQ_SET_READY(&ifp->if_snd); 358 359 bcmeth_ifstop(ifp, true); 360 361 /* 362 * Attach the interface. 363 */ 364 if_attach(ifp); 365 ether_ifattach(ifp, sc->sc_enaddr); 366 367 evcnt_attach_dynamic(&sc->sc_ev_intr, EVCNT_TYPE_INTR, 368 NULL, xname, "intr"); 369 evcnt_attach_dynamic(&sc->sc_ev_soft_intr, EVCNT_TYPE_INTR, 370 NULL, xname, "soft intr"); 371 evcnt_attach_dynamic(&sc->sc_ev_work, EVCNT_TYPE_MISC, 372 NULL, xname, "work items"); 373 evcnt_attach_dynamic(&sc->sc_ev_tx_stall, EVCNT_TYPE_MISC, 374 NULL, xname, "tx stalls"); 375 } 376 377 static int 378 bcmeth_mediachange(struct ifnet *ifp) 379 { 380 //struct bcmeth_softc * const sc = ifp->if_softc; 381 return 0; 382 } 383 384 static void 385 bcmeth_mediastatus(struct ifnet *ifp, struct ifmediareq *ifm) 386 { 387 //struct bcmeth_softc * const sc = ifp->if_softc; 388 389 ifm->ifm_status = IFM_AVALID | IFM_ACTIVE; 390 ifm->ifm_active = IFM_ETHER | IFM_FDX | IFM_1000_T; 391 } 392 393 static uint64_t 394 bcmeth_macaddr_create(const uint8_t *enaddr) 395 { 396 return (enaddr[3] << 0) // UNIMAC_MAC_0 397 | (enaddr[2] << 8) // UNIMAC_MAC_0 398 | (enaddr[1] << 16) // UNIMAC_MAC_0 399 | (enaddr[0] << 24) // UNIMAC_MAC_0 400 | ((uint64_t)enaddr[5] << 32) // UNIMAC_MAC_1 401 | ((uint64_t)enaddr[4] << 40); // UNIMAC_MAC_1 402 } 403 404 static int 405 bcmeth_ifinit(struct ifnet *ifp) 406 { 407 struct bcmeth_softc * const sc = ifp->if_softc; 408 int error = 0; 409 410 sc->sc_maxfrm = max(ifp->if_mtu + 32, MCLBYTES); 411 if (ifp->if_mtu > ETHERMTU_JUMBO) 412 return error; 413 414 KASSERT(ifp->if_flags & IFF_UP); 415 416 /* 417 * Stop the interface 418 */ 419 bcmeth_ifstop(ifp, 0); 420 421 /* 422 * If our frame size has changed (or it's our first time through) 423 * destroy the existing transmit mapcache. 424 */ 425 if (sc->sc_tx_mapcache != NULL 426 && sc->sc_maxfrm != sc->sc_tx_mapcache->dmc_maxmapsize) { 427 bcmeth_mapcache_destroy(sc, sc->sc_tx_mapcache); 428 sc->sc_tx_mapcache = NULL; 429 } 430 431 if (sc->sc_tx_mapcache == NULL) { 432 error = bcmeth_mapcache_create(sc, &sc->sc_tx_mapcache, 433 BCMETH_MAXTXMBUFS, sc->sc_maxfrm, BCMETH_NTXSEGS); 434 if (error) 435 return error; 436 } 437 438 sc->sc_cmdcfg = NO_LENGTH_CHECK | PAUSE_IGNORE 439 | __SHIFTIN(ETH_SPEED_1000, ETH_SPEED) 440 | RX_ENA | TX_ENA; 441 442 if (ifp->if_flags & IFF_PROMISC) { 443 sc->sc_cmdcfg |= PROMISC_EN; 444 } else { 445 sc->sc_cmdcfg &= ~PROMISC_EN; 446 } 447 448 const uint64_t macstnaddr = 449 bcmeth_macaddr_create(CLLADDR(ifp->if_sadl)); 450 451 sc->sc_intmask = DESCPROTOERR|DATAERR|DESCERR; 452 453 /* 5. Load RCVADDR_LO with new pointer */ 454 bcmeth_rxq_reset(sc, &sc->sc_rxq); 455 456 bcmeth_write_4(sc, sc->sc_rxq.rxq_reg_rcvctl, 457 __SHIFTIN(BCMETH_RCVOFFSET, RCVCTL_RCVOFFSET) 458 | RCVCTL_PARITY_DIS 459 | RCVCTL_OFLOW_CONTINUE 460 | __SHIFTIN(4, RCVCTL_BURSTLEN)); 461 462 /* 6. Load XMTADDR_LO with new pointer */ 463 bcmeth_txq_reset(sc, &sc->sc_txq); 464 465 bcmeth_write_4(sc, sc->sc_txq.txq_reg_xmtctl, XMTCTL_DMA_ACT_INDEX 466 | XMTCTL_PARITY_DIS 467 | __SHIFTIN(4, XMTCTL_BURSTLEN)); 468 469 /* 7. Setup other UNIMAC registers */ 470 bcmeth_write_4(sc, UNIMAC_FRAME_LEN, sc->sc_maxfrm); 471 bcmeth_write_4(sc, UNIMAC_MAC_0, (uint32_t)(macstnaddr >> 0)); 472 bcmeth_write_4(sc, UNIMAC_MAC_1, (uint32_t)(macstnaddr >> 32)); 473 bcmeth_write_4(sc, UNIMAC_COMMAND_CONFIG, sc->sc_cmdcfg); 474 475 uint32_t devctl = bcmeth_read_4(sc, GMAC_DEVCONTROL); 476 devctl |= RGMII_LINK_STATUS_SEL | NWAY_AUTO_POLL_EN | TXARB_STRICT_MODE; 477 devctl &= ~FLOW_CTRL_MODE; 478 devctl &= ~MIB_RD_RESET_EN; 479 devctl &= ~RXQ_OVERFLOW_CTRL_SEL; 480 devctl &= ~CPU_FLOW_CTRL_ON; 481 bcmeth_write_4(sc, GMAC_DEVCONTROL, devctl); 482 483 /* Setup lazy receive (at most 1ms). */ 484 sc->sc_rcvlazy = __SHIFTIN(4, INTRCVLAZY_FRAMECOUNT) 485 | __SHIFTIN(125000000 / 1000, INTRCVLAZY_TIMEOUT); 486 bcmeth_write_4(sc, GMAC_INTRCVLAZY, sc->sc_rcvlazy); 487 488 /* 11. Enable transmit queues in TQUEUE, and ensure that the transmit scheduling mode is correctly set in TCTRL. */ 489 sc->sc_intmask |= XMTINT_0|XMTUF; 490 bcmeth_write_4(sc, sc->sc_txq.txq_reg_xmtctl, 491 bcmeth_read_4(sc, sc->sc_txq.txq_reg_xmtctl) | XMTCTL_ENABLE); 492 493 494 /* 12. Enable receive queues in RQUEUE, */ 495 sc->sc_intmask |= RCVINT|RCVDESCUF|RCVFIFOOF; 496 bcmeth_write_4(sc, sc->sc_rxq.rxq_reg_rcvctl, 497 bcmeth_read_4(sc, sc->sc_rxq.rxq_reg_rcvctl) | RCVCTL_ENABLE); 498 499 bcmeth_rxq_produce(sc, &sc->sc_rxq); /* fill with rx buffers */ 500 501 #if 0 502 aprint_normal_dev(sc->sc_dev, 503 "devctl=%#x ucmdcfg=%#x xmtctl=%#x rcvctl=%#x\n", 504 devctl, sc->sc_cmdcfg, 505 bcmeth_read_4(sc, sc->sc_txq.txq_reg_xmtctl), 506 bcmeth_read_4(sc, sc->sc_rxq.rxq_reg_rcvctl)); 507 #endif 508 509 sc->sc_soft_flags = 0; 510 511 bcmeth_write_4(sc, GMAC_INTMASK, sc->sc_intmask); 512 513 ifp->if_flags |= IFF_RUNNING; 514 515 return error; 516 } 517 518 static void 519 bcmeth_ifstop(struct ifnet *ifp, int disable) 520 { 521 struct bcmeth_softc * const sc = ifp->if_softc; 522 struct bcmeth_txqueue * const txq = &sc->sc_txq; 523 struct bcmeth_rxqueue * const rxq = &sc->sc_rxq; 524 525 KASSERT(!cpu_intr_p()); 526 527 sc->sc_soft_flags = 0; 528 529 /* Disable Rx processing */ 530 bcmeth_write_4(sc, rxq->rxq_reg_rcvctl, 531 bcmeth_read_4(sc, rxq->rxq_reg_rcvctl) & ~RCVCTL_ENABLE); 532 533 /* Disable Tx processing */ 534 bcmeth_write_4(sc, txq->txq_reg_xmtctl, 535 bcmeth_read_4(sc, txq->txq_reg_xmtctl) & ~XMTCTL_ENABLE); 536 537 /* Disable all interrupts */ 538 bcmeth_write_4(sc, GMAC_INTMASK, 0); 539 540 for (;;) { 541 uint32_t tx0 = bcmeth_read_4(sc, txq->txq_reg_xmtsts0); 542 uint32_t rx0 = bcmeth_read_4(sc, rxq->rxq_reg_rcvsts0); 543 if (__SHIFTOUT(tx0, XMTSTATE) == XMTSTATE_DIS 544 && __SHIFTOUT(rx0, RCVSTATE) == RCVSTATE_DIS) 545 break; 546 delay(50); 547 } 548 /* 549 * Now reset the controller. 550 * 551 * 3. Set SW_RESET bit in UNIMAC_COMMAND_CONFIG register 552 * 4. Clear SW_RESET bit in UNIMAC_COMMAND_CONFIG register 553 */ 554 bcmeth_write_4(sc, UNIMAC_COMMAND_CONFIG, SW_RESET); 555 bcmeth_write_4(sc, GMAC_INTSTATUS, ~0); 556 sc->sc_intmask = 0; 557 ifp->if_flags &= ~IFF_RUNNING; 558 559 /* 560 * Let's consume any remaining transmitted packets. And if we are 561 * disabling the interface, purge ourselves of any untransmitted 562 * packets. But don't consume any received packets, just drop them. 563 * If we aren't disabling the interface, save the mbufs in the 564 * receive queue for reuse. 565 */ 566 bcmeth_rxq_purge(sc, &sc->sc_rxq, disable); 567 bcmeth_txq_consume(sc, &sc->sc_txq); 568 if (disable) { 569 bcmeth_txq_purge(sc, &sc->sc_txq); 570 IF_PURGE(&ifp->if_snd); 571 } 572 573 bcmeth_write_4(sc, UNIMAC_COMMAND_CONFIG, 0); 574 } 575 576 static void 577 bcmeth_ifwatchdog(struct ifnet *ifp) 578 { 579 } 580 581 static int 582 bcmeth_ifioctl(struct ifnet *ifp, u_long cmd, void *data) 583 { 584 struct bcmeth_softc *sc = ifp->if_softc; 585 struct ifreq * const ifr = data; 586 const int s = splnet(); 587 int error; 588 589 switch (cmd) { 590 case SIOCSIFMEDIA: 591 case SIOCGIFMEDIA: 592 error = ifmedia_ioctl(ifp, ifr, &sc->sc_media, cmd); 593 break; 594 595 default: 596 error = ether_ioctl(ifp, cmd, data); 597 if (error != ENETRESET) 598 break; 599 600 if (cmd == SIOCADDMULTI || cmd == SIOCDELMULTI) { 601 error = 0; 602 break; 603 } 604 error = bcmeth_ifinit(ifp); 605 break; 606 } 607 608 splx(s); 609 return error; 610 } 611 612 static void 613 bcmeth_rxq_desc_presync( 614 struct bcmeth_softc *sc, 615 struct bcmeth_rxqueue *rxq, 616 struct gmac_rxdb *rxdb, 617 size_t count) 618 { 619 bus_dmamap_sync(sc->sc_dmat, rxq->rxq_descmap, 620 (rxdb - rxq->rxq_first) * sizeof(*rxdb), count * sizeof(*rxdb), 621 BUS_DMASYNC_PREWRITE); 622 } 623 624 static void 625 bcmeth_rxq_desc_postsync( 626 struct bcmeth_softc *sc, 627 struct bcmeth_rxqueue *rxq, 628 struct gmac_rxdb *rxdb, 629 size_t count) 630 { 631 bus_dmamap_sync(sc->sc_dmat, rxq->rxq_descmap, 632 (rxdb - rxq->rxq_first) * sizeof(*rxdb), count * sizeof(*rxdb), 633 BUS_DMASYNC_POSTWRITE); 634 } 635 636 static void 637 bcmeth_txq_desc_presync( 638 struct bcmeth_softc *sc, 639 struct bcmeth_txqueue *txq, 640 struct gmac_txdb *txdb, 641 size_t count) 642 { 643 bus_dmamap_sync(sc->sc_dmat, txq->txq_descmap, 644 (txdb - txq->txq_first) * sizeof(*txdb), count * sizeof(*txdb), 645 BUS_DMASYNC_PREWRITE); 646 } 647 648 static void 649 bcmeth_txq_desc_postsync( 650 struct bcmeth_softc *sc, 651 struct bcmeth_txqueue *txq, 652 struct gmac_txdb *txdb, 653 size_t count) 654 { 655 bus_dmamap_sync(sc->sc_dmat, txq->txq_descmap, 656 (txdb - txq->txq_first) * sizeof(*txdb), count * sizeof(*txdb), 657 BUS_DMASYNC_POSTWRITE); 658 } 659 660 static bus_dmamap_t 661 bcmeth_mapcache_get( 662 struct bcmeth_softc *sc, 663 struct bcmeth_mapcache *dmc) 664 { 665 KASSERT(dmc->dmc_nmaps > 0); 666 KASSERT(dmc->dmc_maps[dmc->dmc_nmaps-1] != NULL); 667 return dmc->dmc_maps[--dmc->dmc_nmaps]; 668 } 669 670 static void 671 bcmeth_mapcache_put( 672 struct bcmeth_softc *sc, 673 struct bcmeth_mapcache *dmc, 674 bus_dmamap_t map) 675 { 676 KASSERT(map != NULL); 677 KASSERT(dmc->dmc_nmaps < dmc->dmc_maxmaps); 678 dmc->dmc_maps[dmc->dmc_nmaps++] = map; 679 } 680 681 static void 682 bcmeth_mapcache_destroy( 683 struct bcmeth_softc *sc, 684 struct bcmeth_mapcache *dmc) 685 { 686 const size_t dmc_size = 687 offsetof(struct bcmeth_mapcache, dmc_maps[dmc->dmc_maxmaps]); 688 689 for (u_int i = 0; i < dmc->dmc_maxmaps; i++) { 690 bus_dmamap_destroy(sc->sc_dmat, dmc->dmc_maps[i]); 691 } 692 kmem_intr_free(dmc, dmc_size); 693 } 694 695 static int 696 bcmeth_mapcache_create( 697 struct bcmeth_softc *sc, 698 struct bcmeth_mapcache **dmc_p, 699 size_t maxmaps, 700 size_t maxmapsize, 701 size_t maxseg) 702 { 703 const size_t dmc_size = 704 offsetof(struct bcmeth_mapcache, dmc_maps[maxmaps]); 705 struct bcmeth_mapcache * const dmc = 706 kmem_intr_zalloc(dmc_size, KM_NOSLEEP); 707 708 dmc->dmc_maxmaps = maxmaps; 709 dmc->dmc_nmaps = maxmaps; 710 dmc->dmc_maxmapsize = maxmapsize; 711 dmc->dmc_maxseg = maxseg; 712 713 for (u_int i = 0; i < maxmaps; i++) { 714 int error = bus_dmamap_create(sc->sc_dmat, dmc->dmc_maxmapsize, 715 dmc->dmc_maxseg, dmc->dmc_maxmapsize, 0, 716 BUS_DMA_WAITOK|BUS_DMA_ALLOCNOW, &dmc->dmc_maps[i]); 717 if (error) { 718 aprint_error_dev(sc->sc_dev, 719 "failed to creat dma map cache " 720 "entry %u of %zu: %d\n", 721 i, maxmaps, error); 722 while (i-- > 0) { 723 bus_dmamap_destroy(sc->sc_dmat, 724 dmc->dmc_maps[i]); 725 } 726 kmem_intr_free(dmc, dmc_size); 727 return error; 728 } 729 KASSERT(dmc->dmc_maps[i] != NULL); 730 } 731 732 *dmc_p = dmc; 733 734 return 0; 735 } 736 737 #if 0 738 static void 739 bcmeth_dmamem_free( 740 bus_dma_tag_t dmat, 741 size_t map_size, 742 bus_dma_segment_t *seg, 743 bus_dmamap_t map, 744 void *kvap) 745 { 746 bus_dmamap_destroy(dmat, map); 747 bus_dmamem_unmap(dmat, kvap, map_size); 748 bus_dmamem_free(dmat, seg, 1); 749 } 750 #endif 751 752 static int 753 bcmeth_dmamem_alloc( 754 bus_dma_tag_t dmat, 755 size_t map_size, 756 bus_dma_segment_t *seg, 757 bus_dmamap_t *map, 758 void **kvap) 759 { 760 int error; 761 int nseg; 762 763 *kvap = NULL; 764 *map = NULL; 765 766 error = bus_dmamem_alloc(dmat, map_size, PAGE_SIZE, 0, 767 seg, 1, &nseg, 0); 768 if (error) 769 return error; 770 771 KASSERT(nseg == 1); 772 773 error = bus_dmamem_map(dmat, seg, nseg, map_size, (void **)kvap, 774 BUS_DMA_COHERENT); 775 if (error == 0) { 776 error = bus_dmamap_create(dmat, map_size, 1, map_size, 0, 0, 777 map); 778 if (error == 0) { 779 error = bus_dmamap_load(dmat, *map, *kvap, map_size, 780 NULL, 0); 781 if (error == 0) 782 return 0; 783 bus_dmamap_destroy(dmat, *map); 784 *map = NULL; 785 } 786 bus_dmamem_unmap(dmat, *kvap, map_size); 787 *kvap = NULL; 788 } 789 bus_dmamem_free(dmat, seg, nseg); 790 return 0; 791 } 792 793 static struct mbuf * 794 bcmeth_rx_buf_alloc( 795 struct bcmeth_softc *sc) 796 { 797 struct mbuf *m = m_gethdr(M_DONTWAIT, MT_DATA); 798 if (m == NULL) { 799 printf("%s:%d: %s\n", __func__, __LINE__, "m_gethdr"); 800 return NULL; 801 } 802 MCLGET(m, M_DONTWAIT); 803 if ((m->m_flags & M_EXT) == 0) { 804 printf("%s:%d: %s\n", __func__, __LINE__, "MCLGET"); 805 m_freem(m); 806 return NULL; 807 } 808 m->m_len = m->m_pkthdr.len = m->m_ext.ext_size; 809 810 bus_dmamap_t map = bcmeth_mapcache_get(sc, sc->sc_rx_mapcache); 811 if (map == NULL) { 812 printf("%s:%d: %s\n", __func__, __LINE__, "map get"); 813 m_freem(m); 814 return NULL; 815 } 816 M_SETCTX(m, map); 817 m->m_len = m->m_pkthdr.len = MCLBYTES; 818 int error = bus_dmamap_load_mbuf(sc->sc_dmat, map, m, 819 BUS_DMA_READ|BUS_DMA_NOWAIT); 820 if (error) { 821 aprint_error_dev(sc->sc_dev, "fail to load rx dmamap: %d\n", 822 error); 823 M_SETCTX(m, NULL); 824 m_freem(m); 825 bcmeth_mapcache_put(sc, sc->sc_rx_mapcache, map); 826 return NULL; 827 } 828 KASSERT(map->dm_mapsize == MCLBYTES); 829 bus_dmamap_sync(sc->sc_dmat, map, 0, map->dm_mapsize, 830 BUS_DMASYNC_PREREAD); 831 832 return m; 833 } 834 835 static void 836 bcmeth_rx_map_unload( 837 struct bcmeth_softc *sc, 838 struct mbuf *m) 839 { 840 KASSERT(m); 841 for (; m != NULL; m = m->m_next) { 842 bus_dmamap_t map = M_GETCTX(m, bus_dmamap_t); 843 KASSERT(map); 844 KASSERT(map->dm_mapsize == MCLBYTES); 845 bus_dmamap_sync(sc->sc_dmat, map, 0, m->m_len, 846 BUS_DMASYNC_POSTREAD); 847 bus_dmamap_unload(sc->sc_dmat, map); 848 bcmeth_mapcache_put(sc, sc->sc_rx_mapcache, map); 849 M_SETCTX(m, NULL); 850 } 851 } 852 853 static bool 854 bcmeth_rxq_produce( 855 struct bcmeth_softc *sc, 856 struct bcmeth_rxqueue *rxq) 857 { 858 struct gmac_rxdb *producer = rxq->rxq_producer; 859 bool produced = false; 860 861 while (rxq->rxq_inuse < rxq->rxq_threshold) { 862 struct mbuf *m; 863 IF_DEQUEUE(&sc->sc_rx_bufcache, m); 864 if (m == NULL) { 865 m = bcmeth_rx_buf_alloc(sc); 866 if (m == NULL) { 867 printf("%s: bcmeth_rx_buf_alloc failed\n", __func__); 868 break; 869 } 870 } 871 bus_dmamap_t map = M_GETCTX(m, bus_dmamap_t); 872 KASSERT(map); 873 874 producer->rxdb_buflen = MCLBYTES; 875 producer->rxdb_addrlo = map->dm_segs[0].ds_addr; 876 producer->rxdb_flags &= RXDB_FLAG_ET; 877 *rxq->rxq_mtail = m; 878 rxq->rxq_mtail = &m->m_next; 879 m->m_len = MCLBYTES; 880 m->m_next = NULL; 881 rxq->rxq_inuse++; 882 if (++producer == rxq->rxq_last) { 883 membar_producer(); 884 bcmeth_rxq_desc_presync(sc, rxq, rxq->rxq_producer, 885 rxq->rxq_last - rxq->rxq_producer); 886 producer = rxq->rxq_producer = rxq->rxq_first; 887 } 888 produced = true; 889 } 890 if (produced) { 891 membar_producer(); 892 if (producer != rxq->rxq_producer) { 893 bcmeth_rxq_desc_presync(sc, rxq, rxq->rxq_producer, 894 producer - rxq->rxq_producer); 895 rxq->rxq_producer = producer; 896 } 897 bcmeth_write_4(sc, rxq->rxq_reg_rcvptr, 898 rxq->rxq_descmap->dm_segs[0].ds_addr 899 + ((uintptr_t)producer & RCVPTR)); 900 } 901 return true; 902 } 903 904 static void 905 bcmeth_rx_input( 906 struct bcmeth_softc *sc, 907 struct mbuf *m, 908 uint32_t rxdb_flags) 909 { 910 struct ifnet * const ifp = &sc->sc_if; 911 912 bcmeth_rx_map_unload(sc, m); 913 914 m_adj(m, BCMETH_RCVOFFSET); 915 916 switch (__SHIFTOUT(rxdb_flags, RXSTS_PKTTYPE)) { 917 case RXSTS_PKTTYPE_UC: 918 break; 919 case RXSTS_PKTTYPE_MC: 920 m->m_flags |= M_MCAST; 921 break; 922 case RXSTS_PKTTYPE_BC: 923 m->m_flags |= M_BCAST|M_MCAST; 924 break; 925 default: 926 if (sc->sc_cmdcfg & PROMISC_EN) 927 m->m_flags |= M_PROMISC; 928 break; 929 } 930 m->m_pkthdr.rcvif = ifp; 931 932 ifp->if_ipackets++; 933 ifp->if_ibytes += m->m_pkthdr.len; 934 935 /* 936 * Let's give it to the network subsystm to deal with. 937 */ 938 int s = splnet(); 939 bpf_mtap(ifp, m); 940 (*ifp->if_input)(ifp, m); 941 splx(s); 942 } 943 944 static void 945 bcmeth_rxq_consume( 946 struct bcmeth_softc *sc, 947 struct bcmeth_rxqueue *rxq) 948 { 949 struct ifnet * const ifp = &sc->sc_if; 950 struct gmac_rxdb *consumer = rxq->rxq_consumer; 951 size_t rxconsumed = 0; 952 953 for (;;) { 954 if (consumer == rxq->rxq_producer) { 955 rxq->rxq_consumer = consumer; 956 rxq->rxq_inuse -= rxconsumed; 957 KASSERT(rxq->rxq_inuse == 0); 958 return; 959 } 960 961 uint32_t rcvsts0 = bcmeth_read_4(sc, rxq->rxq_reg_rcvsts0); 962 uint32_t currdscr = __SHIFTOUT(rcvsts0, RCV_CURRDSCR); 963 if (consumer == rxq->rxq_first + currdscr) { 964 rxq->rxq_consumer = consumer; 965 rxq->rxq_inuse -= rxconsumed; 966 return; 967 } 968 bcmeth_rxq_desc_postsync(sc, rxq, consumer, 1); 969 970 /* 971 * We own this packet again. Copy the rxsts word from it. 972 */ 973 rxconsumed++; 974 uint32_t rxsts; 975 KASSERT(rxq->rxq_mhead != NULL); 976 bus_dmamap_t map = M_GETCTX(rxq->rxq_mhead, bus_dmamap_t); 977 bus_dmamap_sync(sc->sc_dmat, map, 0, arm_dcache_align, 978 BUS_DMASYNC_POSTREAD); 979 memcpy(&rxsts, rxq->rxq_mhead->m_data, 4); 980 981 /* 982 * Get the count of descriptors. Fetch the correct number 983 * of mbufs. 984 */ 985 size_t desc_count = __SHIFTOUT(rxsts, RXSTS_DESC_COUNT) + 1; 986 struct mbuf *m = rxq->rxq_mhead; 987 struct mbuf *m_last = m; 988 for (size_t i = 1; i < desc_count; i++) { 989 if (++consumer == rxq->rxq_last) { 990 consumer = rxq->rxq_first; 991 } 992 KASSERT(consumer != rxq->rxq_first + currdscr); 993 m_last = m_last->m_next; 994 } 995 996 /* 997 * Now remove it/them from the list of enqueued mbufs. 998 */ 999 if ((rxq->rxq_mhead = m_last->m_next) == NULL) 1000 rxq->rxq_mtail = &rxq->rxq_mhead; 1001 m_last->m_next = NULL; 1002 1003 if (rxsts & (RXSTS_CRC_ERROR|RXSTS_OVERSIZED|RXSTS_PKT_OVERFLOW)) { 1004 aprint_error_dev(sc->sc_dev, "[%zu]: count=%zu rxsts=%#x\n", 1005 consumer - rxq->rxq_first, desc_count, rxsts); 1006 /* 1007 * We encountered an error, take the mbufs and add them 1008 * to the rx bufcache so we can quickly reuse them. 1009 */ 1010 ifp->if_ierrors++; 1011 do { 1012 struct mbuf *m0 = m->m_next; 1013 m->m_next = NULL; 1014 IF_ENQUEUE(&sc->sc_rx_bufcache, m); 1015 m = m0; 1016 } while (m); 1017 } else { 1018 uint32_t framelen = __SHIFTOUT(rxsts, RXSTS_FRAMELEN); 1019 framelen += BCMETH_RCVOFFSET; 1020 m->m_pkthdr.len = framelen; 1021 if (desc_count == 1) { 1022 KASSERT(framelen <= MCLBYTES); 1023 m->m_len = framelen; 1024 } else { 1025 m_last->m_len = framelen & (MCLBYTES - 1); 1026 } 1027 bcmeth_rx_input(sc, m, rxsts); 1028 } 1029 1030 /* 1031 * Wrap at the last entry! 1032 */ 1033 if (++consumer == rxq->rxq_last) { 1034 KASSERT(consumer[-1].rxdb_flags & RXDB_FLAG_ET); 1035 consumer = rxq->rxq_first; 1036 } 1037 } 1038 } 1039 1040 static void 1041 bcmeth_rxq_purge( 1042 struct bcmeth_softc *sc, 1043 struct bcmeth_rxqueue *rxq, 1044 bool discard) 1045 { 1046 struct mbuf *m; 1047 1048 if ((m = rxq->rxq_mhead) != NULL) { 1049 if (discard) { 1050 bcmeth_rx_map_unload(sc, m); 1051 m_freem(m); 1052 } else { 1053 while (m != NULL) { 1054 struct mbuf *m0 = m->m_next; 1055 m->m_next = NULL; 1056 IF_ENQUEUE(&sc->sc_rx_bufcache, m); 1057 m = m0; 1058 } 1059 } 1060 1061 } 1062 1063 rxq->rxq_mhead = NULL; 1064 rxq->rxq_mtail = &rxq->rxq_mhead; 1065 rxq->rxq_inuse = 0; 1066 } 1067 1068 static void 1069 bcmeth_rxq_reset( 1070 struct bcmeth_softc *sc, 1071 struct bcmeth_rxqueue *rxq) 1072 { 1073 /* 1074 * sync all the descriptors 1075 */ 1076 bcmeth_rxq_desc_postsync(sc, rxq, rxq->rxq_first, 1077 rxq->rxq_last - rxq->rxq_first); 1078 1079 /* 1080 * Make sure we own all descriptors in the ring. 1081 */ 1082 struct gmac_rxdb *rxdb; 1083 for (rxdb = rxq->rxq_first; rxdb < rxq->rxq_last - 1; rxdb++) { 1084 rxdb->rxdb_flags = RXDB_FLAG_IC; 1085 } 1086 1087 /* 1088 * Last descriptor has the wrap flag. 1089 */ 1090 rxdb->rxdb_flags = RXDB_FLAG_ET|RXDB_FLAG_IC; 1091 1092 /* 1093 * Reset the producer consumer indexes. 1094 */ 1095 rxq->rxq_consumer = rxq->rxq_first; 1096 rxq->rxq_producer = rxq->rxq_first; 1097 rxq->rxq_inuse = 0; 1098 if (rxq->rxq_threshold < BCMETH_MINRXMBUFS) 1099 rxq->rxq_threshold = BCMETH_MINRXMBUFS; 1100 1101 sc->sc_intmask |= RCVINT|RCVFIFOOF|RCVDESCUF; 1102 1103 /* 1104 * Restart the receiver at the first descriptor 1105 */ 1106 bcmeth_write_4(sc, rxq->rxq_reg_rcvaddrlo, 1107 rxq->rxq_descmap->dm_segs[0].ds_addr); 1108 } 1109 1110 static int 1111 bcmeth_rxq_attach( 1112 struct bcmeth_softc *sc, 1113 struct bcmeth_rxqueue *rxq, 1114 u_int qno) 1115 { 1116 size_t desc_count = BCMETH_RINGSIZE / sizeof(rxq->rxq_first[0]); 1117 int error; 1118 void *descs; 1119 1120 KASSERT(desc_count == 256 || desc_count == 512); 1121 1122 error = bcmeth_dmamem_alloc(sc->sc_dmat, BCMETH_RINGSIZE, 1123 &rxq->rxq_descmap_seg, &rxq->rxq_descmap, &descs); 1124 if (error) 1125 return error; 1126 1127 memset(descs, 0, BCMETH_RINGSIZE); 1128 rxq->rxq_first = descs; 1129 rxq->rxq_last = rxq->rxq_first + desc_count; 1130 rxq->rxq_consumer = descs; 1131 rxq->rxq_producer = descs; 1132 1133 bcmeth_rxq_purge(sc, rxq, true); 1134 bcmeth_rxq_reset(sc, rxq); 1135 1136 rxq->rxq_reg_rcvaddrlo = GMAC_RCVADDR_LOW; 1137 rxq->rxq_reg_rcvctl = GMAC_RCVCONTROL; 1138 rxq->rxq_reg_rcvptr = GMAC_RCVPTR; 1139 rxq->rxq_reg_rcvsts0 = GMAC_RCVSTATUS0; 1140 1141 return 0; 1142 } 1143 1144 static bool 1145 bcmeth_txq_active_p( 1146 struct bcmeth_softc * const sc, 1147 struct bcmeth_txqueue *txq) 1148 { 1149 return !IF_IS_EMPTY(&txq->txq_mbufs); 1150 } 1151 1152 static bool 1153 bcmeth_txq_fillable_p( 1154 struct bcmeth_softc * const sc, 1155 struct bcmeth_txqueue *txq) 1156 { 1157 return txq->txq_free >= txq->txq_threshold; 1158 } 1159 1160 static int 1161 bcmeth_txq_attach( 1162 struct bcmeth_softc *sc, 1163 struct bcmeth_txqueue *txq, 1164 u_int qno) 1165 { 1166 size_t desc_count = BCMETH_RINGSIZE / sizeof(txq->txq_first[0]); 1167 int error; 1168 void *descs; 1169 1170 KASSERT(desc_count == 256 || desc_count == 512); 1171 1172 error = bcmeth_dmamem_alloc(sc->sc_dmat, BCMETH_RINGSIZE, 1173 &txq->txq_descmap_seg, &txq->txq_descmap, &descs); 1174 if (error) 1175 return error; 1176 1177 memset(descs, 0, BCMETH_RINGSIZE); 1178 txq->txq_first = descs; 1179 txq->txq_last = txq->txq_first + desc_count; 1180 txq->txq_consumer = descs; 1181 txq->txq_producer = descs; 1182 1183 IFQ_SET_MAXLEN(&txq->txq_mbufs, BCMETH_MAXTXMBUFS); 1184 1185 txq->txq_reg_xmtaddrlo = GMAC_XMTADDR_LOW; 1186 txq->txq_reg_xmtctl = GMAC_XMTCONTROL; 1187 txq->txq_reg_xmtptr = GMAC_XMTPTR; 1188 txq->txq_reg_xmtsts0 = GMAC_XMTSTATUS0; 1189 1190 bcmeth_txq_reset(sc, txq); 1191 1192 return 0; 1193 } 1194 1195 static int 1196 bcmeth_txq_map_load( 1197 struct bcmeth_softc *sc, 1198 struct bcmeth_txqueue *txq, 1199 struct mbuf *m) 1200 { 1201 bus_dmamap_t map; 1202 int error; 1203 1204 map = M_GETCTX(m, bus_dmamap_t); 1205 if (map != NULL) 1206 return 0; 1207 1208 map = bcmeth_mapcache_get(sc, sc->sc_tx_mapcache); 1209 if (map == NULL) 1210 return ENOMEM; 1211 1212 error = bus_dmamap_load_mbuf(sc->sc_dmat, map, m, 1213 BUS_DMA_WRITE | BUS_DMA_NOWAIT); 1214 if (error) 1215 return error; 1216 1217 bus_dmamap_sync(sc->sc_dmat, map, 0, m->m_pkthdr.len, 1218 BUS_DMASYNC_PREWRITE); 1219 M_SETCTX(m, map); 1220 return 0; 1221 } 1222 1223 static void 1224 bcmeth_txq_map_unload( 1225 struct bcmeth_softc *sc, 1226 struct bcmeth_txqueue *txq, 1227 struct mbuf *m) 1228 { 1229 KASSERT(m); 1230 bus_dmamap_t map = M_GETCTX(m, bus_dmamap_t); 1231 bus_dmamap_sync(sc->sc_dmat, map, 0, map->dm_mapsize, 1232 BUS_DMASYNC_POSTWRITE); 1233 bus_dmamap_unload(sc->sc_dmat, map); 1234 bcmeth_mapcache_put(sc, sc->sc_tx_mapcache, map); 1235 } 1236 1237 static bool 1238 bcmeth_txq_produce( 1239 struct bcmeth_softc *sc, 1240 struct bcmeth_txqueue *txq, 1241 struct mbuf *m) 1242 { 1243 bus_dmamap_t map = M_GETCTX(m, bus_dmamap_t); 1244 1245 if (map->dm_nsegs > txq->txq_free) 1246 return false; 1247 1248 /* 1249 * TCP Offload flag must be set in the first descriptor. 1250 */ 1251 struct gmac_txdb *producer = txq->txq_producer; 1252 uint32_t first_flags = TXDB_FLAG_SF; 1253 uint32_t last_flags = TXDB_FLAG_EF; 1254 1255 /* 1256 * If we've produced enough descriptors without consuming any 1257 * we need to ask for an interrupt to reclaim some. 1258 */ 1259 txq->txq_lastintr += map->dm_nsegs; 1260 if (txq->txq_lastintr >= txq->txq_threshold 1261 || txq->txq_mbufs.ifq_len + 1 == txq->txq_mbufs.ifq_maxlen) { 1262 txq->txq_lastintr = 0; 1263 last_flags |= TXDB_FLAG_IC; 1264 } 1265 1266 KASSERT(producer != txq->txq_last); 1267 1268 struct gmac_txdb *start = producer; 1269 size_t count = map->dm_nsegs; 1270 producer->txdb_flags |= first_flags; 1271 producer->txdb_addrlo = map->dm_segs[0].ds_addr; 1272 producer->txdb_buflen = map->dm_segs[0].ds_len; 1273 for (u_int i = 1; i < map->dm_nsegs; i++) { 1274 #if 0 1275 printf("[%zu]: %#x/%#x/%#x/%#x\n", producer - txq->txq_first, 1276 producer->txdb_flags, producer->txdb_buflen, 1277 producer->txdb_addrlo, producer->txdb_addrhi); 1278 #endif 1279 if (__predict_false(++producer == txq->txq_last)) { 1280 bcmeth_txq_desc_presync(sc, txq, start, 1281 txq->txq_last - start); 1282 count -= txq->txq_last - start; 1283 producer = txq->txq_first; 1284 start = txq->txq_first; 1285 } 1286 producer->txdb_addrlo = map->dm_segs[i].ds_addr; 1287 producer->txdb_buflen = map->dm_segs[i].ds_len; 1288 } 1289 producer->txdb_flags |= last_flags; 1290 #if 0 1291 printf("[%zu]: %#x/%#x/%#x/%#x\n", producer - txq->txq_first, 1292 producer->txdb_flags, producer->txdb_buflen, 1293 producer->txdb_addrlo, producer->txdb_addrhi); 1294 #endif 1295 bcmeth_txq_desc_presync(sc, txq, start, count); 1296 1297 /* 1298 * Reduce free count by the number of segments we consumed. 1299 */ 1300 txq->txq_free -= map->dm_nsegs; 1301 KASSERT(map->dm_nsegs == 1 || txq->txq_producer != producer); 1302 KASSERT(map->dm_nsegs == 1 || (txq->txq_producer->txdb_flags & TXDB_FLAG_EF) == 0); 1303 KASSERT(producer->txdb_flags & TXDB_FLAG_EF); 1304 1305 #if 0 1306 printf("%s: mbuf %p: produced a %u byte packet in %u segments (%zd..%zd)\n", 1307 __func__, m, m->m_pkthdr.len, map->dm_nsegs, 1308 txq->txq_producer - txq->txq_first, producer - txq->txq_first); 1309 #endif 1310 1311 if (++producer == txq->txq_last) 1312 txq->txq_producer = txq->txq_first; 1313 else 1314 txq->txq_producer = producer; 1315 IF_ENQUEUE(&txq->txq_mbufs, m); 1316 bpf_mtap(&sc->sc_if, m); 1317 1318 /* 1319 * Let the transmitter know there's more to do 1320 */ 1321 bcmeth_write_4(sc, txq->txq_reg_xmtptr, 1322 txq->txq_descmap->dm_segs[0].ds_addr 1323 + ((uintptr_t)txq->txq_producer & XMT_LASTDSCR)); 1324 1325 return true; 1326 } 1327 1328 static bool 1329 bcmeth_txq_enqueue( 1330 struct bcmeth_softc *sc, 1331 struct bcmeth_txqueue *txq) 1332 { 1333 for (;;) { 1334 if (IF_QFULL(&txq->txq_mbufs)) 1335 return false; 1336 struct mbuf *m = txq->txq_next; 1337 if (m == NULL) { 1338 int s = splnet(); 1339 IF_DEQUEUE(&sc->sc_if.if_snd, m); 1340 splx(s); 1341 if (m == NULL) 1342 return true; 1343 M_SETCTX(m, NULL); 1344 } else { 1345 txq->txq_next = NULL; 1346 } 1347 int error = bcmeth_txq_map_load(sc, txq, m); 1348 if (error) { 1349 aprint_error_dev(sc->sc_dev, 1350 "discarded packet due to " 1351 "dmamap load failure: %d\n", error); 1352 m_freem(m); 1353 continue; 1354 } 1355 KASSERT(txq->txq_next == NULL); 1356 if (!bcmeth_txq_produce(sc, txq, m)) { 1357 txq->txq_next = m; 1358 return false; 1359 } 1360 KASSERT(txq->txq_next == NULL); 1361 } 1362 } 1363 1364 static bool 1365 bcmeth_txq_consume( 1366 struct bcmeth_softc *sc, 1367 struct bcmeth_txqueue *txq) 1368 { 1369 struct ifnet * const ifp = &sc->sc_if; 1370 struct gmac_txdb *consumer = txq->txq_consumer; 1371 size_t txfree = 0; 1372 1373 #if 0 1374 printf("%s: entry: free=%zu\n", __func__, txq->txq_free); 1375 #endif 1376 1377 for (;;) { 1378 if (consumer == txq->txq_producer) { 1379 txq->txq_consumer = consumer; 1380 txq->txq_free += txfree; 1381 txq->txq_lastintr -= min(txq->txq_lastintr, txfree); 1382 #if 0 1383 printf("%s: empty: freed %zu descriptors going from %zu to %zu\n", 1384 __func__, txfree, txq->txq_free - txfree, txq->txq_free); 1385 #endif 1386 KASSERT(txq->txq_lastintr == 0); 1387 KASSERT(txq->txq_free == txq->txq_last - txq->txq_first - 1); 1388 return true; 1389 } 1390 bcmeth_txq_desc_postsync(sc, txq, consumer, 1); 1391 uint32_t s0 = bcmeth_read_4(sc, txq->txq_reg_xmtsts0); 1392 if (consumer == txq->txq_first + __SHIFTOUT(s0, XMT_CURRDSCR)) { 1393 txq->txq_consumer = consumer; 1394 txq->txq_free += txfree; 1395 txq->txq_lastintr -= min(txq->txq_lastintr, txfree); 1396 #if 0 1397 printf("%s: freed %zu descriptors\n", 1398 __func__, txfree); 1399 #endif 1400 return bcmeth_txq_fillable_p(sc, txq); 1401 } 1402 1403 /* 1404 * If this is the last descriptor in the chain, get the 1405 * mbuf, free its dmamap, and free the mbuf chain itself. 1406 */ 1407 const uint32_t txdb_flags = consumer->txdb_flags; 1408 if (txdb_flags & TXDB_FLAG_EF) { 1409 struct mbuf *m; 1410 1411 IF_DEQUEUE(&txq->txq_mbufs, m); 1412 KASSERT(m); 1413 bcmeth_txq_map_unload(sc, txq, m); 1414 #if 0 1415 printf("%s: mbuf %p: consumed a %u byte packet\n", 1416 __func__, m, m->m_pkthdr.len); 1417 #endif 1418 ifp->if_opackets++; 1419 ifp->if_obytes += m->m_pkthdr.len; 1420 if (m->m_flags & M_MCAST) 1421 ifp->if_omcasts++; 1422 m_freem(m); 1423 } 1424 1425 /* 1426 * We own this packet again. Clear all flags except wrap. 1427 */ 1428 txfree++; 1429 1430 /* 1431 * Wrap at the last entry! 1432 */ 1433 if (txdb_flags & TXDB_FLAG_ET) { 1434 consumer->txdb_flags = TXDB_FLAG_ET; 1435 KASSERT(consumer + 1 == txq->txq_last); 1436 consumer = txq->txq_first; 1437 } else { 1438 consumer->txdb_flags = 0; 1439 consumer++; 1440 KASSERT(consumer < txq->txq_last); 1441 } 1442 } 1443 } 1444 1445 static void 1446 bcmeth_txq_purge( 1447 struct bcmeth_softc *sc, 1448 struct bcmeth_txqueue *txq) 1449 { 1450 struct mbuf *m; 1451 KASSERT((bcmeth_read_4(sc, UNIMAC_COMMAND_CONFIG) & TX_ENA) == 0); 1452 1453 for (;;) { 1454 IF_DEQUEUE(&txq->txq_mbufs, m); 1455 if (m == NULL) 1456 break; 1457 bcmeth_txq_map_unload(sc, txq, m); 1458 m_freem(m); 1459 } 1460 if ((m = txq->txq_next) != NULL) { 1461 txq->txq_next = NULL; 1462 bcmeth_txq_map_unload(sc, txq, m); 1463 m_freem(m); 1464 } 1465 } 1466 1467 static void 1468 bcmeth_txq_reset( 1469 struct bcmeth_softc *sc, 1470 struct bcmeth_txqueue *txq) 1471 { 1472 /* 1473 * sync all the descriptors 1474 */ 1475 bcmeth_txq_desc_postsync(sc, txq, txq->txq_first, 1476 txq->txq_last - txq->txq_first); 1477 1478 /* 1479 * Make sure we own all descriptors in the ring. 1480 */ 1481 struct gmac_txdb *txdb; 1482 for (txdb = txq->txq_first; txdb < txq->txq_last - 1; txdb++) { 1483 txdb->txdb_flags = 0; 1484 } 1485 1486 /* 1487 * Last descriptor has the wrap flag. 1488 */ 1489 txdb->txdb_flags = TXDB_FLAG_ET; 1490 1491 /* 1492 * Reset the producer consumer indexes. 1493 */ 1494 txq->txq_consumer = txq->txq_first; 1495 txq->txq_producer = txq->txq_first; 1496 txq->txq_free = txq->txq_last - txq->txq_first - 1; 1497 txq->txq_threshold = txq->txq_free / 2; 1498 txq->txq_lastintr = 0; 1499 1500 /* 1501 * What do we want to get interrupted on? 1502 */ 1503 sc->sc_intmask |= XMTINT_0 | XMTUF; 1504 1505 /* 1506 * Restart the transmiter at the first descriptor 1507 */ 1508 bcmeth_write_4(sc, txq->txq_reg_xmtaddrlo, 1509 txq->txq_descmap->dm_segs->ds_addr); 1510 } 1511 1512 static void 1513 bcmeth_ifstart(struct ifnet *ifp) 1514 { 1515 struct bcmeth_softc * const sc = ifp->if_softc; 1516 1517 atomic_or_uint(&sc->sc_soft_flags, SOFT_TXINTR); 1518 softint_schedule(sc->sc_soft_ih); 1519 } 1520 1521 int 1522 bcmeth_intr(void *arg) 1523 { 1524 struct bcmeth_softc * const sc = arg; 1525 uint32_t soft_flags = 0; 1526 uint32_t work_flags = 0; 1527 int rv = 0; 1528 1529 mutex_enter(sc->sc_hwlock); 1530 1531 sc->sc_ev_intr.ev_count++; 1532 1533 for (;;) { 1534 uint32_t intstatus = bcmeth_read_4(sc, GMAC_INTSTATUS); 1535 intstatus &= sc->sc_intmask; 1536 bcmeth_write_4(sc, GMAC_INTSTATUS, intstatus); /* write 1 to clear */ 1537 if (intstatus == 0) { 1538 break; 1539 } 1540 #if 0 1541 aprint_normal_dev(sc->sc_dev, "%s: intstatus=%#x intmask=%#x\n", 1542 __func__, intstatus, bcmeth_read_4(sc, GMAC_INTMASK)); 1543 #endif 1544 if (intstatus & RCVINT) { 1545 struct bcmeth_rxqueue * const rxq = &sc->sc_rxq; 1546 intstatus &= ~RCVINT; 1547 sc->sc_intmask &= ~RCVINT; 1548 1549 uint32_t rcvsts0 = bcmeth_read_4(sc, rxq->rxq_reg_rcvsts0); 1550 uint32_t descs = __SHIFTOUT(rcvsts0, RCV_CURRDSCR); 1551 if (descs < rxq->rxq_consumer - rxq->rxq_first) { 1552 /* 1553 * We wrapped at the end so count how far 1554 * we are from the end. 1555 */ 1556 descs += rxq->rxq_last - rxq->rxq_consumer; 1557 } else { 1558 descs -= rxq->rxq_consumer - rxq->rxq_first; 1559 } 1560 /* 1561 * If we "timedout" we can't be hogging so use 1562 * softints. If we exceeded then we might hogging 1563 * so let the workqueue deal with them. 1564 */ 1565 const uint32_t framecount = __SHIFTOUT(sc->sc_rcvlazy, INTRCVLAZY_FRAMECOUNT); 1566 if (descs < framecount) { 1567 soft_flags |= SOFT_RXINTR; 1568 } else { 1569 work_flags |= WORK_RXINTR; 1570 } 1571 } 1572 1573 if (intstatus & XMTINT_0) { 1574 intstatus &= ~XMTINT_0; 1575 sc->sc_intmask &= ~XMTINT_0; 1576 soft_flags |= SOFT_TXINTR; 1577 } 1578 1579 if (intstatus & RCVDESCUF) { 1580 intstatus &= ~RCVDESCUF; 1581 sc->sc_intmask &= ~RCVDESCUF; 1582 work_flags |= WORK_RXUNDERFLOW; 1583 } 1584 1585 if (intstatus) { 1586 aprint_error_dev(sc->sc_dev, "intr: intstatus=%#x\n", 1587 intstatus); 1588 Debugger(); 1589 sc->sc_intmask &= ~intstatus; 1590 work_flags |= WORK_REINIT; 1591 break; 1592 } 1593 } 1594 1595 if (work_flags | soft_flags) { 1596 bcmeth_write_4(sc, GMAC_INTMASK, sc->sc_intmask); 1597 } 1598 1599 if (work_flags) { 1600 if (sc->sc_work_flags == 0) { 1601 workqueue_enqueue(sc->sc_workq, &sc->sc_work, NULL); 1602 } 1603 atomic_or_32(&sc->sc_work_flags, work_flags); 1604 rv = 1; 1605 } 1606 1607 if (soft_flags) { 1608 if (sc->sc_soft_flags == 0) { 1609 softint_schedule(sc->sc_soft_ih); 1610 } 1611 atomic_or_32(&sc->sc_soft_flags, soft_flags); 1612 rv = 1; 1613 } 1614 1615 mutex_exit(sc->sc_hwlock); 1616 1617 return rv; 1618 } 1619 1620 void 1621 bcmeth_soft_intr(void *arg) 1622 { 1623 struct bcmeth_softc * const sc = arg; 1624 struct ifnet * const ifp = &sc->sc_if; 1625 1626 mutex_enter(sc->sc_lock); 1627 1628 u_int soft_flags = atomic_swap_uint(&sc->sc_soft_flags, 0); 1629 1630 sc->sc_ev_soft_intr.ev_count++; 1631 1632 if ((soft_flags & SOFT_TXINTR) 1633 || bcmeth_txq_active_p(sc, &sc->sc_txq)) { 1634 /* 1635 * Let's do what we came here for. Consume transmitted 1636 * packets off the the transmit ring. 1637 */ 1638 if (!bcmeth_txq_consume(sc, &sc->sc_txq) 1639 || !bcmeth_txq_enqueue(sc, &sc->sc_txq)) { 1640 sc->sc_ev_tx_stall.ev_count++; 1641 ifp->if_flags |= IFF_OACTIVE; 1642 } else { 1643 ifp->if_flags &= ~IFF_OACTIVE; 1644 } 1645 sc->sc_intmask |= XMTINT_0; 1646 } 1647 1648 if (soft_flags & SOFT_RXINTR) { 1649 /* 1650 * Let's consume 1651 */ 1652 bcmeth_rxq_consume(sc, &sc->sc_rxq); 1653 sc->sc_intmask |= RCVINT; 1654 } 1655 1656 if (ifp->if_flags & IFF_RUNNING) { 1657 bcmeth_rxq_produce(sc, &sc->sc_rxq); 1658 bcmeth_write_4(sc, GMAC_INTMASK, sc->sc_intmask); 1659 } 1660 1661 mutex_exit(sc->sc_lock); 1662 } 1663 1664 void 1665 bcmeth_worker(struct work *wk, void *arg) 1666 { 1667 struct bcmeth_softc * const sc = arg; 1668 struct ifnet * const ifp = &sc->sc_if; 1669 1670 mutex_enter(sc->sc_lock); 1671 1672 sc->sc_ev_work.ev_count++; 1673 1674 uint32_t work_flags = atomic_swap_32(&sc->sc_work_flags, 0); 1675 if (work_flags & WORK_REINIT) { 1676 int s = splnet(); 1677 sc->sc_soft_flags = 0; 1678 bcmeth_ifinit(ifp); 1679 splx(s); 1680 work_flags &= ~WORK_RXUNDERFLOW; 1681 } 1682 1683 if (work_flags & WORK_RXUNDERFLOW) { 1684 struct bcmeth_rxqueue * const rxq = &sc->sc_rxq; 1685 size_t threshold = 5 * rxq->rxq_threshold / 4; 1686 if (threshold >= rxq->rxq_last - rxq->rxq_first) { 1687 threshold = rxq->rxq_last - rxq->rxq_first - 1; 1688 } else { 1689 sc->sc_intmask |= RCVDESCUF; 1690 } 1691 aprint_normal_dev(sc->sc_dev, 1692 "increasing receive buffers from %zu to %zu\n", 1693 rxq->rxq_threshold, threshold); 1694 rxq->rxq_threshold = threshold; 1695 } 1696 1697 if (work_flags & WORK_RXINTR) { 1698 /* 1699 * Let's consume 1700 */ 1701 bcmeth_rxq_consume(sc, &sc->sc_rxq); 1702 sc->sc_intmask |= RCVINT; 1703 } 1704 1705 if (ifp->if_flags & IFF_RUNNING) { 1706 bcmeth_rxq_produce(sc, &sc->sc_rxq); 1707 bcmeth_write_4(sc, GMAC_INTMASK, sc->sc_intmask); 1708 } 1709 1710 mutex_exit(sc->sc_lock); 1711 } 1712
Indexes created Sat Sep 27 22:09:54 GMT 2025