if_igc.c revision 1.20
1 /* $NetBSD: if_igc.c,v 1.20 2025/06/01 05:28:42 rin Exp $ */ 2 /* $OpenBSD: if_igc.c,v 1.13 2023/04/28 10:18:57 bluhm Exp $ */ 3 /*- 4 * SPDX-License-Identifier: BSD-2-Clause 5 * 6 * Copyright (c) 2016 Nicole Graziano <nicole (at) nextbsd.org> 7 * All rights reserved. 8 * Copyright (c) 2021 Rubicon Communications, LLC (Netgate) 9 * 10 * Redistribution and use in source and binary forms, with or without 11 * modification, are permitted provided that the following conditions 12 * are met: 13 * 1. Redistributions of source code must retain the above copyright 14 * notice, this list of conditions and the following disclaimer. 15 * 2. Redistributions in binary form must reproduce the above copyright 16 * notice, this list of conditions and the following disclaimer in the 17 * documentation and/or other materials provided with the distribution. 18 * 19 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 20 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 21 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 22 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 23 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 24 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 25 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 26 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 27 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 28 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 29 * SUCH DAMAGE. 30 */ 31 32 #include <sys/cdefs.h> 33 __KERNEL_RCSID(0, "$NetBSD: if_igc.c,v 1.20 2025/06/01 05:28:42 rin Exp $"); 34 35 #ifdef _KERNEL_OPT 36 #include "opt_if_igc.h" 37 #if 0 /* notyet */ 38 #include "vlan.h" 39 #endif 40 #endif 41 42 #include <sys/param.h> 43 #include <sys/systm.h> 44 #include <sys/bus.h> 45 #include <sys/cpu.h> 46 #include <sys/device.h> 47 #include <sys/endian.h> 48 #include <sys/intr.h> 49 #include <sys/interrupt.h> 50 #include <sys/kernel.h> 51 #include <sys/kmem.h> 52 #include <sys/mbuf.h> 53 #include <sys/mutex.h> 54 #include <sys/socket.h> 55 #include <sys/workqueue.h> 56 #include <sys/xcall.h> 57 58 #include <net/bpf.h> 59 #include <net/if.h> 60 #include <net/if_ether.h> 61 #include <net/if_media.h> 62 #include <net/if_vlanvar.h> 63 #include <net/rss_config.h> 64 65 #include <netinet/in.h> 66 #include <netinet/ip.h> 67 #include <netinet/ip6.h> 68 #include <netinet/tcp.h> 69 70 #include <dev/pci/pcivar.h> 71 #include <dev/pci/pcireg.h> 72 #include <dev/pci/pcidevs.h> 73 74 #include <dev/pci/igc/if_igc.h> 75 #include <dev/pci/igc/igc_evcnt.h> 76 #include <dev/pci/igc/igc_hw.h> 77 #include <dev/mii/miivar.h> 78 79 #define IGC_WORKQUEUE_PRI PRI_SOFTNET 80 81 #ifndef IGC_RX_INTR_PROCESS_LIMIT_DEFAULT 82 #define IGC_RX_INTR_PROCESS_LIMIT_DEFAULT 0 83 #endif 84 #ifndef IGC_TX_INTR_PROCESS_LIMIT_DEFAULT 85 #define IGC_TX_INTR_PROCESS_LIMIT_DEFAULT 0 86 #endif 87 88 #ifndef IGC_RX_PROCESS_LIMIT_DEFAULT 89 #define IGC_RX_PROCESS_LIMIT_DEFAULT 256 90 #endif 91 #ifndef IGC_TX_PROCESS_LIMIT_DEFAULT 92 #define IGC_TX_PROCESS_LIMIT_DEFAULT 256 93 #endif 94 95 #define htolem32(p, x) (*((uint32_t *)(p)) = htole32(x)) 96 #define htolem64(p, x) (*((uint64_t *)(p)) = htole64(x)) 97 98 static const struct igc_product { 99 pci_vendor_id_t igcp_vendor; 100 pci_product_id_t igcp_product; 101 const char *igcp_name; 102 } igc_products[] = { 103 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_I225_IT, 104 "Intel(R) Ethernet Controller I225-IT(2)" }, 105 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_I226_LM, 106 "Intel(R) Ethernet Controller I226-LM" }, 107 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_I226_V, 108 "Intel(R) Ethernet Controller I226-V" }, 109 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_I226_IT, 110 "Intel(R) Ethernet Controller I226-IT" }, 111 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_I221_V, 112 "Intel(R) Ethernet Controller I221-V" }, 113 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_I226_BLANK_NVM, 114 "Intel(R) Ethernet Controller I226(blankNVM)" }, 115 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_I225_LM, 116 "Intel(R) Ethernet Controller I225-LM" }, 117 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_I225_V, 118 "Intel(R) Ethernet Controller I225-V" }, 119 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_I220_V, 120 "Intel(R) Ethernet Controller I220-V" }, 121 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_I225_I, 122 "Intel(R) Ethernet Controller I225-I" }, 123 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_I225_BLANK_NVM, 124 "Intel(R) Ethernet Controller I225(blankNVM)" }, 125 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_I225_K, 126 "Intel(R) Ethernet Controller I225-K" }, 127 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_I225_K2, 128 "Intel(R) Ethernet Controller I225-K(2)" }, 129 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_I226_K, 130 "Intel(R) Ethernet Controller I226-K" }, 131 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_I225_LMVP, 132 "Intel(R) Ethernet Controller I225-LMvP(2)" }, 133 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_I226_LMVP, 134 "Intel(R) Ethernet Controller I226-LMvP" }, 135 { 0, 0, NULL }, 136 }; 137 138 #define IGC_DF_CFG 0x1 139 #define IGC_DF_TX 0x2 140 #define IGC_DF_RX 0x4 141 #define IGC_DF_MISC 0x8 142 143 #ifdef IGC_DEBUG_FLAGS 144 int igc_debug_flags = IGC_DEBUG_FLAGS; 145 #else 146 int igc_debug_flags = 0; 147 #endif 148 149 #define DPRINTF(flag, fmt, args...) do { \ 150 if (igc_debug_flags & (IGC_DF_ ## flag)) \ 151 printf("%s: %d: " fmt, __func__, __LINE__, ##args); \ 152 } while (0) 153 154 /********************************************************************* 155 * Function Prototypes 156 *********************************************************************/ 157 static int igc_match(device_t, cfdata_t, void *); 158 static void igc_attach(device_t, device_t, void *); 159 static int igc_detach(device_t, int); 160 161 static void igc_identify_hardware(struct igc_softc *); 162 static int igc_adjust_nqueues(struct igc_softc *); 163 static int igc_allocate_pci_resources(struct igc_softc *); 164 static int igc_allocate_interrupts(struct igc_softc *); 165 static int igc_allocate_queues(struct igc_softc *); 166 static void igc_free_pci_resources(struct igc_softc *); 167 static void igc_free_interrupts(struct igc_softc *); 168 static void igc_free_queues(struct igc_softc *); 169 static void igc_reset(struct igc_softc *); 170 static void igc_init_dmac(struct igc_softc *, uint32_t); 171 static int igc_setup_interrupts(struct igc_softc *); 172 static void igc_attach_counters(struct igc_softc *sc); 173 static void igc_detach_counters(struct igc_softc *sc); 174 static void igc_update_counters(struct igc_softc *sc); 175 static void igc_clear_counters(struct igc_softc *sc); 176 static int igc_setup_msix(struct igc_softc *); 177 static int igc_setup_msi(struct igc_softc *); 178 static int igc_setup_intx(struct igc_softc *); 179 static int igc_dma_malloc(struct igc_softc *, bus_size_t, 180 struct igc_dma_alloc *); 181 static void igc_dma_free(struct igc_softc *, struct igc_dma_alloc *); 182 static void igc_setup_interface(struct igc_softc *); 183 184 static int igc_init(struct ifnet *); 185 static int igc_init_locked(struct igc_softc *); 186 static void igc_start(struct ifnet *); 187 static int igc_transmit(struct ifnet *, struct mbuf *); 188 static void igc_tx_common_locked(struct ifnet *, struct tx_ring *, int); 189 static bool igc_txeof(struct tx_ring *, u_int); 190 static void igc_intr_barrier(struct igc_softc *); 191 static void igc_stop(struct ifnet *, int); 192 static void igc_stop_locked(struct igc_softc *); 193 static int igc_ioctl(struct ifnet *, u_long, void *); 194 #ifdef IF_RXR 195 static int igc_rxrinfo(struct igc_softc *, struct if_rxrinfo *); 196 #endif 197 static void igc_rxfill(struct rx_ring *); 198 static void igc_rxrefill(struct rx_ring *, int); 199 static bool igc_rxeof(struct rx_ring *, u_int); 200 static int igc_rx_checksum(struct igc_queue *, uint64_t, uint32_t, 201 uint32_t); 202 static void igc_watchdog(struct ifnet *); 203 static void igc_tick(void *); 204 static void igc_media_status(struct ifnet *, struct ifmediareq *); 205 static int igc_media_change(struct ifnet *); 206 static int igc_ifflags_cb(struct ethercom *); 207 static void igc_set_filter(struct igc_softc *); 208 static void igc_update_link_status(struct igc_softc *); 209 static int igc_get_buf(struct rx_ring *, int, bool); 210 static bool igc_tx_ctx_setup(struct tx_ring *, struct mbuf *, int, 211 uint32_t *, uint32_t *); 212 213 static void igc_configure_queues(struct igc_softc *); 214 static void igc_set_queues(struct igc_softc *, uint32_t, uint32_t, int); 215 static void igc_enable_queue(struct igc_softc *, uint32_t); 216 static void igc_enable_intr(struct igc_softc *); 217 static void igc_disable_intr(struct igc_softc *); 218 static int igc_intr_link(void *); 219 static int igc_intr_queue(void *); 220 static int igc_intr(void *); 221 static void igc_handle_queue(void *); 222 static void igc_handle_queue_work(struct work *, void *); 223 static void igc_sched_handle_queue(struct igc_softc *, struct igc_queue *); 224 static void igc_barrier_handle_queue(struct igc_softc *); 225 226 static int igc_allocate_transmit_buffers(struct tx_ring *); 227 static int igc_setup_transmit_structures(struct igc_softc *); 228 static int igc_setup_transmit_ring(struct tx_ring *); 229 static void igc_initialize_transmit_unit(struct igc_softc *); 230 static void igc_free_transmit_structures(struct igc_softc *); 231 static void igc_free_transmit_buffers(struct tx_ring *); 232 static void igc_withdraw_transmit_packets(struct tx_ring *, bool); 233 static int igc_allocate_receive_buffers(struct rx_ring *); 234 static int igc_setup_receive_structures(struct igc_softc *); 235 static int igc_setup_receive_ring(struct rx_ring *); 236 static void igc_initialize_receive_unit(struct igc_softc *); 237 static void igc_free_receive_structures(struct igc_softc *); 238 static void igc_free_receive_buffers(struct rx_ring *); 239 static void igc_clear_receive_status(struct rx_ring *); 240 static void igc_initialize_rss_mapping(struct igc_softc *); 241 242 static void igc_get_hw_control(struct igc_softc *); 243 static void igc_release_hw_control(struct igc_softc *); 244 static int igc_is_valid_ether_addr(uint8_t *); 245 static void igc_print_devinfo(struct igc_softc *); 246 247 CFATTACH_DECL3_NEW(igc, sizeof(struct igc_softc), 248 igc_match, igc_attach, igc_detach, NULL, NULL, NULL, 0); 249 250 static inline int 251 igc_txdesc_incr(struct igc_softc *sc, int id) 252 { 253 254 if (++id == sc->num_tx_desc) 255 id = 0; 256 return id; 257 } 258 259 static inline int __unused 260 igc_txdesc_decr(struct igc_softc *sc, int id) 261 { 262 263 if (--id < 0) 264 id = sc->num_tx_desc - 1; 265 return id; 266 } 267 268 static inline void 269 igc_txdesc_sync(struct tx_ring *txr, int id, int ops) 270 { 271 272 bus_dmamap_sync(txr->txdma.dma_tag, txr->txdma.dma_map, 273 id * sizeof(union igc_adv_tx_desc), sizeof(union igc_adv_tx_desc), 274 ops); 275 } 276 277 static inline int 278 igc_rxdesc_incr(struct igc_softc *sc, int id) 279 { 280 281 if (++id == sc->num_rx_desc) 282 id = 0; 283 return id; 284 } 285 286 static inline int 287 igc_rxdesc_decr(struct igc_softc *sc, int id) 288 { 289 290 if (--id < 0) 291 id = sc->num_rx_desc - 1; 292 return id; 293 } 294 295 static inline void 296 igc_rxdesc_sync(struct rx_ring *rxr, int id, int ops) 297 { 298 299 bus_dmamap_sync(rxr->rxdma.dma_tag, rxr->rxdma.dma_map, 300 id * sizeof(union igc_adv_rx_desc), sizeof(union igc_adv_rx_desc), 301 ops); 302 } 303 304 static const struct igc_product * 305 igc_lookup(const struct pci_attach_args *pa) 306 { 307 const struct igc_product *igcp; 308 309 for (igcp = igc_products; igcp->igcp_name != NULL; igcp++) { 310 if (PCI_VENDOR(pa->pa_id) == igcp->igcp_vendor && 311 PCI_PRODUCT(pa->pa_id) == igcp->igcp_product) 312 return igcp; 313 } 314 return NULL; 315 } 316 317 /********************************************************************* 318 * Device identification routine 319 * 320 * igc_match determines if the driver should be loaded on 321 * adapter based on PCI vendor/device id of the adapter. 322 * 323 * return 0 on success, positive on failure 324 *********************************************************************/ 325 static int 326 igc_match(device_t parent, cfdata_t match, void *aux) 327 { 328 struct pci_attach_args *pa = aux; 329 330 if (igc_lookup(pa) != NULL) 331 return 1; 332 333 return 0; 334 } 335 336 /********************************************************************* 337 * Device initialization routine 338 * 339 * The attach entry point is called when the driver is being loaded. 340 * This routine identifies the type of hardware, allocates all resources 341 * and initializes the hardware. 342 * 343 * return 0 on success, positive on failure 344 *********************************************************************/ 345 static void 346 igc_attach(device_t parent, device_t self, void *aux) 347 { 348 struct pci_attach_args *pa = aux; 349 struct igc_softc *sc = device_private(self); 350 struct igc_hw *hw = &sc->hw; 351 352 const struct igc_product *igcp = igc_lookup(pa); 353 KASSERT(igcp != NULL); 354 355 sc->sc_dev = self; 356 callout_init(&sc->sc_tick_ch, CALLOUT_MPSAFE); 357 callout_setfunc(&sc->sc_tick_ch, igc_tick, sc); 358 sc->sc_core_stopping = false; 359 360 sc->osdep.os_sc = sc; 361 sc->osdep.os_pa = *pa; 362 #ifndef __aarch64__ 363 /* 364 * XXX PR port-arm/57643 365 * 64-bit DMA does not work at least for LX2K with 32/64GB memory. 366 * smmu(4) support may be required. 367 */ 368 if (pci_dma64_available(pa)) { 369 aprint_verbose(", 64-bit DMA"); 370 sc->osdep.os_dmat = pa->pa_dmat64; 371 } else 372 #endif 373 { 374 aprint_verbose(", 32-bit DMA"); 375 sc->osdep.os_dmat = pa->pa_dmat; 376 } 377 378 pci_aprint_devinfo_fancy(pa, "Ethernet controller", igcp->igcp_name, 1); 379 380 /* Determine hardware and mac info */ 381 igc_identify_hardware(sc); 382 383 sc->num_tx_desc = IGC_DEFAULT_TXD; 384 sc->num_rx_desc = IGC_DEFAULT_RXD; 385 386 /* Setup PCI resources */ 387 if (igc_allocate_pci_resources(sc)) { 388 aprint_error_dev(sc->sc_dev, 389 "unable to allocate PCI resources\n"); 390 goto err_pci; 391 } 392 393 if (igc_allocate_interrupts(sc)) { 394 aprint_error_dev(sc->sc_dev, "unable to allocate interrupts\n"); 395 goto err_pci; 396 } 397 398 /* Allocate TX/RX queues */ 399 if (igc_allocate_queues(sc)) { 400 aprint_error_dev(sc->sc_dev, "unable to allocate queues\n"); 401 goto err_alloc_intr; 402 } 403 404 /* Do shared code initialization */ 405 if (igc_setup_init_funcs(hw, true)) { 406 aprint_error_dev(sc->sc_dev, "unable to initialize\n"); 407 goto err_alloc_intr; 408 } 409 410 hw->mac.autoneg = DO_AUTO_NEG; 411 hw->phy.autoneg_wait_to_complete = false; 412 hw->phy.autoneg_advertised = AUTONEG_ADV_DEFAULT; 413 414 /* Copper options. */ 415 if (hw->phy.media_type == igc_media_type_copper) 416 hw->phy.mdix = AUTO_ALL_MODES; 417 418 /* Set the max frame size. */ 419 sc->hw.mac.max_frame_size = 9234; 420 421 /* Allocate multicast array memory. */ 422 sc->mta = kmem_alloc(IGC_MTA_LEN, KM_SLEEP); 423 424 /* Check SOL/IDER usage. */ 425 if (igc_check_reset_block(hw)) { 426 aprint_error_dev(sc->sc_dev, 427 "PHY reset is blocked due to SOL/IDER session\n"); 428 } 429 430 /* Disable Energy Efficient Ethernet. */ 431 sc->hw.dev_spec._i225.eee_disable = true; 432 433 igc_reset_hw(hw); 434 435 /* Make sure we have a good EEPROM before we read from it. */ 436 if (igc_validate_nvm_checksum(hw) < 0) { 437 /* 438 * Some PCI-E parts fail the first check due to 439 * the link being in sleep state, call it again, 440 * if it fails a second time its a real issue. 441 */ 442 if (igc_validate_nvm_checksum(hw) < 0) { 443 aprint_error_dev(sc->sc_dev, 444 "EEPROM checksum invalid\n"); 445 goto err_late; 446 } 447 } 448 449 /* Copy the permanent MAC address out of the EEPROM. */ 450 if (igc_read_mac_addr(hw) < 0) { 451 aprint_error_dev(sc->sc_dev, 452 "unable to read MAC address from EEPROM\n"); 453 goto err_late; 454 } 455 456 if (!igc_is_valid_ether_addr(hw->mac.addr)) { 457 aprint_error_dev(sc->sc_dev, "invalid MAC address\n"); 458 goto err_late; 459 } 460 461 if (igc_setup_interrupts(sc)) 462 goto err_late; 463 464 /* Attach counters. */ 465 igc_attach_counters(sc); 466 467 /* Setup OS specific network interface. */ 468 igc_setup_interface(sc); 469 470 igc_print_devinfo(sc); 471 472 igc_reset(sc); 473 hw->mac.get_link_status = true; 474 igc_update_link_status(sc); 475 476 /* The driver can now take control from firmware. */ 477 igc_get_hw_control(sc); 478 479 aprint_normal_dev(sc->sc_dev, "Ethernet address %s\n", 480 ether_sprintf(sc->hw.mac.addr)); 481 482 if (pmf_device_register(self, NULL, NULL)) 483 pmf_class_network_register(self, &sc->sc_ec.ec_if); 484 else 485 aprint_error_dev(self, "couldn't establish power handler\n"); 486 487 return; 488 489 err_late: 490 igc_release_hw_control(sc); 491 err_alloc_intr: 492 igc_free_interrupts(sc); 493 err_pci: 494 igc_free_pci_resources(sc); 495 kmem_free(sc->mta, IGC_MTA_LEN); 496 } 497 498 /********************************************************************* 499 * Device removal routine 500 * 501 * The detach entry point is called when the driver is being removed. 502 * This routine stops the adapter and deallocates all the resources 503 * that were allocated for driver operation. 504 * 505 * return 0 on success, positive on failure 506 *********************************************************************/ 507 static int 508 igc_detach(device_t self, int flags) 509 { 510 struct igc_softc *sc = device_private(self); 511 struct ifnet *ifp = &sc->sc_ec.ec_if; 512 513 mutex_enter(&sc->sc_core_lock); 514 igc_stop_locked(sc); 515 mutex_exit(&sc->sc_core_lock); 516 517 igc_detach_counters(sc); 518 519 igc_free_queues(sc); 520 521 igc_phy_hw_reset(&sc->hw); 522 igc_release_hw_control(sc); 523 524 ether_ifdetach(ifp); 525 if_detach(ifp); 526 ifmedia_fini(&sc->media); 527 528 igc_free_interrupts(sc); 529 igc_free_pci_resources(sc); 530 kmem_free(sc->mta, IGC_MTA_LEN); 531 532 mutex_destroy(&sc->sc_core_lock); 533 534 return 0; 535 } 536 537 static void 538 igc_identify_hardware(struct igc_softc *sc) 539 { 540 struct igc_osdep *os = &sc->osdep; 541 struct pci_attach_args *pa = &os->os_pa; 542 543 /* Save off the information about this board. */ 544 sc->hw.device_id = PCI_PRODUCT(pa->pa_id); 545 546 /* Do shared code init and setup. */ 547 if (igc_set_mac_type(&sc->hw)) { 548 aprint_error_dev(sc->sc_dev, "unable to identify hardware\n"); 549 return; 550 } 551 } 552 553 static int 554 igc_allocate_pci_resources(struct igc_softc *sc) 555 { 556 struct igc_osdep *os = &sc->osdep; 557 struct pci_attach_args *pa = &os->os_pa; 558 559 /* 560 * Enable bus mastering and memory-mapped I/O for sure. 561 */ 562 pcireg_t csr = 563 pci_conf_read(pa->pa_pc, pa->pa_tag, PCI_COMMAND_STATUS_REG); 564 csr |= PCI_COMMAND_MASTER_ENABLE | PCI_COMMAND_MEM_ENABLE; 565 pci_conf_write(pa->pa_pc, pa->pa_tag, PCI_COMMAND_STATUS_REG, csr); 566 567 const pcireg_t memtype = 568 pci_mapreg_type(pa->pa_pc, pa->pa_tag, IGC_PCIREG); 569 if (pci_mapreg_map(pa, IGC_PCIREG, memtype, 0, &os->os_memt, 570 &os->os_memh, &os->os_membase, &os->os_memsize)) { 571 aprint_error_dev(sc->sc_dev, "unable to map registers\n"); 572 return ENXIO; 573 } 574 575 sc->hw.hw_addr = os->os_membase; 576 sc->hw.back = os; 577 578 return 0; 579 } 580 581 static int __unused 582 igc_adjust_nqueues(struct igc_softc *sc) 583 { 584 struct pci_attach_args *pa = &sc->osdep.os_pa; 585 int nqueues = MIN(IGC_MAX_NQUEUES, ncpu); 586 587 const int nmsix = pci_msix_count(pa->pa_pc, pa->pa_tag); 588 if (nmsix <= 1) 589 nqueues = 1; 590 else if (nmsix < nqueues + 1) 591 nqueues = nmsix - 1; 592 593 return nqueues; 594 } 595 596 static int 597 igc_allocate_interrupts(struct igc_softc *sc) 598 { 599 struct pci_attach_args *pa = &sc->osdep.os_pa; 600 int error; 601 602 #ifndef IGC_DISABLE_MSIX 603 const int nqueues = igc_adjust_nqueues(sc); 604 if (nqueues > 1) { 605 sc->sc_nintrs = nqueues + 1; 606 error = pci_msix_alloc_exact(pa, &sc->sc_intrs, sc->sc_nintrs); 607 if (!error) { 608 sc->sc_nqueues = nqueues; 609 sc->sc_intr_type = PCI_INTR_TYPE_MSIX; 610 return 0; 611 } 612 } 613 #endif 614 615 /* fallback to MSI */ 616 sc->sc_nintrs = sc->sc_nqueues = 1; 617 618 #ifndef IGC_DISABLE_MSI 619 error = pci_msi_alloc_exact(pa, &sc->sc_intrs, sc->sc_nintrs); 620 if (!error) { 621 sc->sc_intr_type = PCI_INTR_TYPE_MSI; 622 return 0; 623 } 624 #endif 625 626 /* fallback to INTx */ 627 628 error = pci_intx_alloc(pa, &sc->sc_intrs); 629 if (!error) { 630 sc->sc_intr_type = PCI_INTR_TYPE_INTX; 631 return 0; 632 } 633 634 return error; 635 } 636 637 static int 638 igc_allocate_queues(struct igc_softc *sc) 639 { 640 device_t dev = sc->sc_dev; 641 int rxconf = 0, txconf = 0; 642 643 /* Allocate the top level queue structs. */ 644 sc->queues = 645 kmem_zalloc(sc->sc_nqueues * sizeof(struct igc_queue), KM_SLEEP); 646 647 /* Allocate the TX ring. */ 648 sc->tx_rings = 649 kmem_zalloc(sc->sc_nqueues * sizeof(struct tx_ring), KM_SLEEP); 650 651 /* Allocate the RX ring. */ 652 sc->rx_rings = 653 kmem_zalloc(sc->sc_nqueues * sizeof(struct rx_ring), KM_SLEEP); 654 655 /* Set up the TX queues. */ 656 for (int iq = 0; iq < sc->sc_nqueues; iq++, txconf++) { 657 struct tx_ring *txr = &sc->tx_rings[iq]; 658 const int tsize = roundup2( 659 sc->num_tx_desc * sizeof(union igc_adv_tx_desc), 660 IGC_DBA_ALIGN); 661 662 txr->sc = sc; 663 txr->txr_igcq = &sc->queues[iq]; 664 txr->me = iq; 665 if (igc_dma_malloc(sc, tsize, &txr->txdma)) { 666 aprint_error_dev(dev, 667 "unable to allocate TX descriptor\n"); 668 goto fail; 669 } 670 txr->tx_base = (union igc_adv_tx_desc *)txr->txdma.dma_vaddr; 671 memset(txr->tx_base, 0, tsize); 672 } 673 674 /* Prepare transmit descriptors and buffers. */ 675 if (igc_setup_transmit_structures(sc)) { 676 aprint_error_dev(dev, "unable to setup transmit structures\n"); 677 goto fail; 678 } 679 680 /* Set up the RX queues. */ 681 for (int iq = 0; iq < sc->sc_nqueues; iq++, rxconf++) { 682 struct rx_ring *rxr = &sc->rx_rings[iq]; 683 const int rsize = roundup2( 684 sc->num_rx_desc * sizeof(union igc_adv_rx_desc), 685 IGC_DBA_ALIGN); 686 687 rxr->sc = sc; 688 rxr->rxr_igcq = &sc->queues[iq]; 689 rxr->me = iq; 690 #ifdef OPENBSD 691 timeout_set(&rxr->rx_refill, igc_rxrefill, rxr); 692 #endif 693 if (igc_dma_malloc(sc, rsize, &rxr->rxdma)) { 694 aprint_error_dev(dev, 695 "unable to allocate RX descriptor\n"); 696 goto fail; 697 } 698 rxr->rx_base = (union igc_adv_rx_desc *)rxr->rxdma.dma_vaddr; 699 memset(rxr->rx_base, 0, rsize); 700 } 701 702 sc->rx_mbuf_sz = MCLBYTES; 703 /* Prepare receive descriptors and buffers. */ 704 if (igc_setup_receive_structures(sc)) { 705 aprint_error_dev(sc->sc_dev, 706 "unable to setup receive structures\n"); 707 goto fail; 708 } 709 710 /* Set up the queue holding structs. */ 711 for (int iq = 0; iq < sc->sc_nqueues; iq++) { 712 struct igc_queue *q = &sc->queues[iq]; 713 714 q->sc = sc; 715 q->txr = &sc->tx_rings[iq]; 716 q->rxr = &sc->rx_rings[iq]; 717 } 718 719 return 0; 720 721 fail: 722 for (struct rx_ring *rxr = sc->rx_rings; rxconf > 0; rxr++, rxconf--) 723 igc_dma_free(sc, &rxr->rxdma); 724 for (struct tx_ring *txr = sc->tx_rings; txconf > 0; txr++, txconf--) 725 igc_dma_free(sc, &txr->txdma); 726 727 kmem_free(sc->rx_rings, sc->sc_nqueues * sizeof(struct rx_ring)); 728 sc->rx_rings = NULL; 729 kmem_free(sc->tx_rings, sc->sc_nqueues * sizeof(struct tx_ring)); 730 sc->tx_rings = NULL; 731 kmem_free(sc->queues, sc->sc_nqueues * sizeof(struct igc_queue)); 732 sc->queues = NULL; 733 734 return ENOMEM; 735 } 736 737 static void 738 igc_free_pci_resources(struct igc_softc *sc) 739 { 740 struct igc_osdep *os = &sc->osdep; 741 742 if (os->os_membase != 0) 743 bus_space_unmap(os->os_memt, os->os_memh, os->os_memsize); 744 os->os_membase = 0; 745 } 746 747 static void 748 igc_free_interrupts(struct igc_softc *sc) 749 { 750 struct pci_attach_args *pa = &sc->osdep.os_pa; 751 pci_chipset_tag_t pc = pa->pa_pc; 752 753 for (int i = 0; i < sc->sc_nintrs; i++) { 754 if (sc->sc_ihs[i] != NULL) { 755 pci_intr_disestablish(pc, sc->sc_ihs[i]); 756 sc->sc_ihs[i] = NULL; 757 } 758 } 759 pci_intr_release(pc, sc->sc_intrs, sc->sc_nintrs); 760 } 761 762 static void 763 igc_free_queues(struct igc_softc *sc) 764 { 765 766 igc_free_receive_structures(sc); 767 for (int iq = 0; iq < sc->sc_nqueues; iq++) { 768 struct rx_ring *rxr = &sc->rx_rings[iq]; 769 770 igc_dma_free(sc, &rxr->rxdma); 771 } 772 773 igc_free_transmit_structures(sc); 774 for (int iq = 0; iq < sc->sc_nqueues; iq++) { 775 struct tx_ring *txr = &sc->tx_rings[iq]; 776 777 igc_dma_free(sc, &txr->txdma); 778 } 779 780 kmem_free(sc->rx_rings, sc->sc_nqueues * sizeof(struct rx_ring)); 781 kmem_free(sc->tx_rings, sc->sc_nqueues * sizeof(struct tx_ring)); 782 kmem_free(sc->queues, sc->sc_nqueues * sizeof(struct igc_queue)); 783 } 784 785 /********************************************************************* 786 * 787 * Initialize the hardware to a configuration as specified by the 788 * adapter structure. 789 * 790 **********************************************************************/ 791 static void 792 igc_reset(struct igc_softc *sc) 793 { 794 struct igc_hw *hw = &sc->hw; 795 796 /* Let the firmware know the OS is in control */ 797 igc_get_hw_control(sc); 798 799 /* 800 * Packet Buffer Allocation (PBA) 801 * Writing PBA sets the receive portion of the buffer 802 * the remainder is used for the transmit buffer. 803 */ 804 const uint32_t pba = IGC_PBA_34K; 805 806 /* 807 * These parameters control the automatic generation (Tx) and 808 * response (Rx) to Ethernet PAUSE frames. 809 * - High water mark should allow for at least two frames to be 810 * received after sending an XOFF. 811 * - Low water mark works best when it is very near the high water mark. 812 * This allows the receiver to restart by sending XON when it has 813 * drained a bit. Here we use an arbitrary value of 1500 which will 814 * restart after one full frame is pulled from the buffer. There 815 * could be several smaller frames in the buffer and if so they will 816 * not trigger the XON until their total number reduces the buffer 817 * by 1500. 818 * - The pause time is fairly large at 1000 x 512ns = 512 usec. 819 */ 820 const uint16_t rx_buffer_size = (pba & 0xffff) << 10; 821 822 hw->fc.high_water = rx_buffer_size - 823 roundup2(sc->hw.mac.max_frame_size, 1024); 824 /* 16-byte granularity */ 825 hw->fc.low_water = hw->fc.high_water - 16; 826 827 if (sc->fc) /* locally set flow control value? */ 828 hw->fc.requested_mode = sc->fc; 829 else 830 hw->fc.requested_mode = igc_fc_full; 831 832 hw->fc.pause_time = IGC_FC_PAUSE_TIME; 833 834 hw->fc.send_xon = true; 835 836 /* Issue a global reset */ 837 igc_reset_hw(hw); 838 IGC_WRITE_REG(hw, IGC_WUC, 0); 839 840 /* and a re-init */ 841 if (igc_init_hw(hw) < 0) { 842 aprint_error_dev(sc->sc_dev, "unable to reset hardware\n"); 843 return; 844 } 845 846 /* Setup DMA Coalescing */ 847 igc_init_dmac(sc, pba); 848 849 IGC_WRITE_REG(hw, IGC_VET, ETHERTYPE_VLAN); 850 igc_get_phy_info(hw); 851 igc_check_for_link(hw); 852 } 853 854 /********************************************************************* 855 * 856 * Initialize the DMA Coalescing feature 857 * 858 **********************************************************************/ 859 static void 860 igc_init_dmac(struct igc_softc *sc, uint32_t pba) 861 { 862 struct igc_hw *hw = &sc->hw; 863 const uint16_t max_frame_size = sc->hw.mac.max_frame_size; 864 uint32_t reg, status; 865 866 if (sc->dmac == 0) { /* Disabling it */ 867 reg = ~IGC_DMACR_DMAC_EN; /* XXXRO */ 868 IGC_WRITE_REG(hw, IGC_DMACR, reg); 869 DPRINTF(MISC, "DMA coalescing disabled\n"); 870 return; 871 } else { 872 device_printf(sc->sc_dev, "DMA coalescing enabled\n"); 873 } 874 875 /* Set starting threshold */ 876 IGC_WRITE_REG(hw, IGC_DMCTXTH, 0); 877 878 uint16_t hwm = 64 * pba - max_frame_size / 16; 879 if (hwm < 64 * (pba - 6)) 880 hwm = 64 * (pba - 6); 881 reg = IGC_READ_REG(hw, IGC_FCRTC); 882 reg &= ~IGC_FCRTC_RTH_COAL_MASK; 883 reg |= (hwm << IGC_FCRTC_RTH_COAL_SHIFT) & IGC_FCRTC_RTH_COAL_MASK; 884 IGC_WRITE_REG(hw, IGC_FCRTC, reg); 885 886 uint32_t dmac = pba - max_frame_size / 512; 887 if (dmac < pba - 10) 888 dmac = pba - 10; 889 reg = IGC_READ_REG(hw, IGC_DMACR); 890 reg &= ~IGC_DMACR_DMACTHR_MASK; 891 reg |= (dmac << IGC_DMACR_DMACTHR_SHIFT) & IGC_DMACR_DMACTHR_MASK; 892 893 /* transition to L0x or L1 if available..*/ 894 reg |= IGC_DMACR_DMAC_EN | IGC_DMACR_DMAC_LX_MASK; 895 896 /* Check if status is 2.5Gb backplane connection 897 * before configuration of watchdog timer, which is 898 * in msec values in 12.8usec intervals 899 * watchdog timer= msec values in 32usec intervals 900 * for non 2.5Gb connection 901 */ 902 status = IGC_READ_REG(hw, IGC_STATUS); 903 if ((status & IGC_STATUS_2P5_SKU) && 904 !(status & IGC_STATUS_2P5_SKU_OVER)) 905 reg |= (sc->dmac * 5) >> 6; 906 else 907 reg |= sc->dmac >> 5; 908 909 IGC_WRITE_REG(hw, IGC_DMACR, reg); 910 911 IGC_WRITE_REG(hw, IGC_DMCRTRH, 0); 912 913 /* Set the interval before transition */ 914 reg = IGC_READ_REG(hw, IGC_DMCTLX); 915 reg |= IGC_DMCTLX_DCFLUSH_DIS; 916 917 /* 918 * in 2.5Gb connection, TTLX unit is 0.4 usec 919 * which is 0x4*2 = 0xA. But delay is still 4 usec 920 */ 921 status = IGC_READ_REG(hw, IGC_STATUS); 922 if ((status & IGC_STATUS_2P5_SKU) && 923 !(status & IGC_STATUS_2P5_SKU_OVER)) 924 reg |= 0xA; 925 else 926 reg |= 0x4; 927 928 IGC_WRITE_REG(hw, IGC_DMCTLX, reg); 929 930 /* free space in tx packet buffer to wake from DMA coal */ 931 IGC_WRITE_REG(hw, IGC_DMCTXTH, 932 (IGC_TXPBSIZE - (2 * max_frame_size)) >> 6); 933 934 /* make low power state decision controlled by DMA coal */ 935 reg = IGC_READ_REG(hw, IGC_PCIEMISC); 936 reg &= ~IGC_PCIEMISC_LX_DECISION; 937 IGC_WRITE_REG(hw, IGC_PCIEMISC, reg); 938 } 939 940 static int 941 igc_setup_interrupts(struct igc_softc *sc) 942 { 943 int error; 944 945 switch (sc->sc_intr_type) { 946 case PCI_INTR_TYPE_MSIX: 947 error = igc_setup_msix(sc); 948 break; 949 case PCI_INTR_TYPE_MSI: 950 error = igc_setup_msi(sc); 951 break; 952 case PCI_INTR_TYPE_INTX: 953 error = igc_setup_intx(sc); 954 break; 955 default: 956 panic("%s: invalid interrupt type: %d", 957 device_xname(sc->sc_dev), sc->sc_intr_type); 958 } 959 960 return error; 961 } 962 963 static void 964 igc_attach_counters(struct igc_softc *sc) 965 { 966 #ifdef IGC_EVENT_COUNTERS 967 968 /* Global counters */ 969 sc->sc_global_evcnts = kmem_zalloc( 970 IGC_GLOBAL_COUNTERS * sizeof(sc->sc_global_evcnts[0]), KM_SLEEP); 971 972 for (int cnt = 0; cnt < IGC_GLOBAL_COUNTERS; cnt++) { 973 evcnt_attach_dynamic(&sc->sc_global_evcnts[cnt], 974 igc_global_counters[cnt].type, NULL, 975 device_xname(sc->sc_dev), igc_global_counters[cnt].name); 976 } 977 978 /* Driver counters */ 979 sc->sc_driver_evcnts = kmem_zalloc( 980 IGC_DRIVER_COUNTERS * sizeof(sc->sc_driver_evcnts[0]), KM_SLEEP); 981 982 for (int cnt = 0; cnt < IGC_DRIVER_COUNTERS; cnt++) { 983 evcnt_attach_dynamic(&sc->sc_driver_evcnts[cnt], 984 igc_driver_counters[cnt].type, NULL, 985 device_xname(sc->sc_dev), igc_driver_counters[cnt].name); 986 } 987 988 for (int iq = 0; iq < sc->sc_nqueues; iq++) { 989 struct igc_queue *q = &sc->queues[iq]; 990 991 q->igcq_driver_counters = kmem_zalloc( 992 IGC_DRIVER_COUNTERS * sizeof(q->igcq_driver_counters[0]), 993 KM_SLEEP); 994 } 995 996 /* Queue counters */ 997 for (int iq = 0; iq < sc->sc_nqueues; iq++) { 998 struct igc_queue *q = &sc->queues[iq]; 999 1000 snprintf(q->igcq_queue_evname, sizeof(q->igcq_queue_evname), 1001 "%s q%d", device_xname(sc->sc_dev), iq); 1002 1003 q->igcq_queue_evcnts = kmem_zalloc( 1004 IGC_QUEUE_COUNTERS * sizeof(q->igcq_queue_evcnts[0]), 1005 KM_SLEEP); 1006 1007 for (int cnt = 0; cnt < IGC_QUEUE_COUNTERS; cnt++) { 1008 evcnt_attach_dynamic(&q->igcq_queue_evcnts[cnt], 1009 igc_queue_counters[cnt].type, NULL, 1010 q->igcq_queue_evname, igc_queue_counters[cnt].name); 1011 } 1012 } 1013 1014 /* MAC counters */ 1015 snprintf(sc->sc_mac_evname, sizeof(sc->sc_mac_evname), 1016 "%s Mac Statistics", device_xname(sc->sc_dev)); 1017 1018 sc->sc_mac_evcnts = kmem_zalloc( 1019 IGC_MAC_COUNTERS * sizeof(sc->sc_mac_evcnts[0]), KM_SLEEP); 1020 1021 for (int cnt = 0; cnt < IGC_MAC_COUNTERS; cnt++) { 1022 evcnt_attach_dynamic(&sc->sc_mac_evcnts[cnt], EVCNT_TYPE_MISC, 1023 NULL, sc->sc_mac_evname, igc_mac_counters[cnt].name); 1024 } 1025 #endif 1026 } 1027 1028 static void 1029 igc_detach_counters(struct igc_softc *sc) 1030 { 1031 #ifdef IGC_EVENT_COUNTERS 1032 1033 /* Global counters */ 1034 for (int cnt = 0; cnt < IGC_GLOBAL_COUNTERS; cnt++) 1035 evcnt_detach(&sc->sc_global_evcnts[cnt]); 1036 1037 kmem_free(sc->sc_global_evcnts, 1038 IGC_GLOBAL_COUNTERS * sizeof(sc->sc_global_evcnts[0])); 1039 1040 /* Driver counters */ 1041 for (int iq = 0; iq < sc->sc_nqueues; iq++) { 1042 struct igc_queue *q = &sc->queues[iq]; 1043 1044 kmem_free(q->igcq_driver_counters, 1045 IGC_DRIVER_COUNTERS * sizeof(q->igcq_driver_counters[0])); 1046 } 1047 1048 for (int cnt = 0; cnt < IGC_DRIVER_COUNTERS; cnt++) 1049 evcnt_detach(&sc->sc_driver_evcnts[cnt]); 1050 1051 kmem_free(sc->sc_driver_evcnts, 1052 IGC_DRIVER_COUNTERS * sizeof(sc->sc_driver_evcnts[0])); 1053 1054 /* Queue counters */ 1055 for (int iq = 0; iq < sc->sc_nqueues; iq++) { 1056 struct igc_queue *q = &sc->queues[iq]; 1057 1058 for (int cnt = 0; cnt < IGC_QUEUE_COUNTERS; cnt++) 1059 evcnt_detach(&q->igcq_queue_evcnts[cnt]); 1060 1061 kmem_free(q->igcq_queue_evcnts, 1062 IGC_QUEUE_COUNTERS * sizeof(q->igcq_queue_evcnts[0])); 1063 } 1064 1065 /* MAC statistics */ 1066 for (int cnt = 0; cnt < IGC_MAC_COUNTERS; cnt++) 1067 evcnt_detach(&sc->sc_mac_evcnts[cnt]); 1068 1069 kmem_free(sc->sc_mac_evcnts, 1070 IGC_MAC_COUNTERS * sizeof(sc->sc_mac_evcnts[0])); 1071 #endif 1072 } 1073 1074 /* 1075 * XXX 1076 * FreeBSD uses 4-byte-wise read for 64-bit counters, while Linux just 1077 * drops hi words. 1078 */ 1079 static inline uint64_t __unused 1080 igc_read_mac_counter(struct igc_hw *hw, bus_size_t reg, bool is64) 1081 { 1082 uint64_t val; 1083 1084 val = IGC_READ_REG(hw, reg); 1085 if (is64) 1086 val += ((uint64_t)IGC_READ_REG(hw, reg + 4)) << 32; 1087 return val; 1088 } 1089 1090 static void 1091 igc_update_counters(struct igc_softc *sc) 1092 { 1093 #ifdef IGC_EVENT_COUNTERS 1094 1095 /* Global counters: nop */ 1096 1097 /* Driver counters */ 1098 uint64_t sum[IGC_DRIVER_COUNTERS]; 1099 1100 memset(sum, 0, sizeof(sum)); 1101 for (int iq = 0; iq < sc->sc_nqueues; iq++) { 1102 struct igc_queue *q = &sc->queues[iq]; 1103 1104 for (int cnt = 0; cnt < IGC_DRIVER_COUNTERS; cnt++) { 1105 sum[cnt] += IGC_QUEUE_DRIVER_COUNTER_VAL(q, cnt); 1106 IGC_QUEUE_DRIVER_COUNTER_STORE(q, cnt, 0); 1107 } 1108 } 1109 1110 for (int cnt = 0; cnt < IGC_DRIVER_COUNTERS; cnt++) 1111 IGC_DRIVER_COUNTER_ADD(sc, cnt, sum[cnt]); 1112 1113 /* Queue counters: nop */ 1114 1115 /* Mac statistics */ 1116 struct igc_hw *hw = &sc->hw; 1117 struct ifnet *ifp = &sc->sc_ec.ec_if; 1118 uint64_t iqdrops = 0; 1119 1120 for (int cnt = 0; cnt < IGC_MAC_COUNTERS; cnt++) { 1121 uint64_t val; 1122 bus_size_t regaddr = igc_mac_counters[cnt].reg; 1123 1124 val = igc_read_mac_counter(hw, regaddr, 1125 igc_mac_counters[cnt].is64); 1126 IGC_MAC_COUNTER_ADD(sc, cnt, val); 1127 /* XXX Count MPC to iqdrops. */ 1128 if (regaddr == IGC_MPC) 1129 iqdrops += val; 1130 } 1131 1132 for (int iq = 0; iq < sc->sc_nqueues; iq++) { 1133 uint32_t val; 1134 1135 /* XXX RQDPC should be visible via evcnt(9). */ 1136 val = IGC_READ_REG(hw, IGC_RQDPC(iq)); 1137 1138 /* RQDPC is not cleard on read. */ 1139 if (val != 0) 1140 IGC_WRITE_REG(hw, IGC_RQDPC(iq), 0); 1141 iqdrops += val; 1142 } 1143 1144 if (iqdrops != 0) 1145 if_statadd(ifp, if_iqdrops, iqdrops); 1146 #endif 1147 } 1148 1149 static void 1150 igc_clear_counters(struct igc_softc *sc) 1151 { 1152 #ifdef IGC_EVENT_COUNTERS 1153 1154 /* Global counters */ 1155 for (int cnt = 0; cnt < IGC_GLOBAL_COUNTERS; cnt++) 1156 IGC_GLOBAL_COUNTER_STORE(sc, cnt, 0); 1157 1158 /* Driver counters */ 1159 for (int iq = 0; iq < sc->sc_nqueues; iq++) { 1160 struct igc_queue *q = &sc->queues[iq]; 1161 1162 for (int cnt = 0; cnt < IGC_DRIVER_COUNTERS; cnt++) 1163 IGC_QUEUE_DRIVER_COUNTER_STORE(q, cnt, 0); 1164 } 1165 1166 for (int cnt = 0; cnt < IGC_DRIVER_COUNTERS; cnt++) 1167 IGC_DRIVER_COUNTER_STORE(sc, cnt, 0); 1168 1169 /* Queue counters */ 1170 for (int iq = 0; iq < sc->sc_nqueues; iq++) { 1171 struct igc_queue *q = &sc->queues[iq]; 1172 1173 for (int cnt = 0; cnt < IGC_QUEUE_COUNTERS; cnt++) 1174 IGC_QUEUE_COUNTER_STORE(q, cnt, 0); 1175 } 1176 1177 /* Mac statistics */ 1178 struct igc_hw *hw = &sc->hw; 1179 1180 for (int cnt = 0; cnt < IGC_MAC_COUNTERS; cnt++) { 1181 (void)igc_read_mac_counter(hw, igc_mac_counters[cnt].reg, 1182 igc_mac_counters[cnt].is64); 1183 IGC_MAC_COUNTER_STORE(sc, cnt, 0); 1184 } 1185 #endif 1186 } 1187 1188 static int 1189 igc_setup_msix(struct igc_softc *sc) 1190 { 1191 pci_chipset_tag_t pc = sc->osdep.os_pa.pa_pc; 1192 device_t dev = sc->sc_dev; 1193 pci_intr_handle_t *intrs; 1194 void **ihs; 1195 const char *intrstr; 1196 char intrbuf[PCI_INTRSTR_LEN]; 1197 char xnamebuf[MAX(32, MAXCOMLEN)]; 1198 int iq, error; 1199 1200 for (iq = 0, intrs = sc->sc_intrs, ihs = sc->sc_ihs; 1201 iq < sc->sc_nqueues; iq++, intrs++, ihs++) { 1202 struct igc_queue *q = &sc->queues[iq]; 1203 1204 snprintf(xnamebuf, sizeof(xnamebuf), "%s: txrx %d", 1205 device_xname(dev), iq); 1206 1207 intrstr = pci_intr_string(pc, *intrs, intrbuf, sizeof(intrbuf)); 1208 1209 pci_intr_setattr(pc, intrs, PCI_INTR_MPSAFE, true); 1210 *ihs = pci_intr_establish_xname(pc, *intrs, IPL_NET, 1211 igc_intr_queue, q, xnamebuf); 1212 if (*ihs == NULL) { 1213 aprint_error_dev(dev, 1214 "unable to establish txrx interrupt at %s\n", 1215 intrstr); 1216 return ENOBUFS; 1217 } 1218 aprint_normal_dev(dev, "txrx interrupting at %s\n", intrstr); 1219 1220 kcpuset_t *affinity; 1221 kcpuset_create(&affinity, true); 1222 kcpuset_set(affinity, iq % ncpu); 1223 error = interrupt_distribute(*ihs, affinity, NULL); 1224 if (error) { 1225 aprint_normal_dev(dev, 1226 "%s: unable to change affinity, use default CPU\n", 1227 intrstr); 1228 } 1229 kcpuset_destroy(affinity); 1230 1231 q->igcq_si = softint_establish(SOFTINT_NET | SOFTINT_MPSAFE, 1232 igc_handle_queue, q); 1233 if (q->igcq_si == NULL) { 1234 aprint_error_dev(dev, 1235 "%s: unable to establish softint\n", intrstr); 1236 return ENOBUFS; 1237 } 1238 1239 q->msix = iq; 1240 q->eims = 1 << iq; 1241 } 1242 1243 snprintf(xnamebuf, MAXCOMLEN, "%s_tx_rx", device_xname(dev)); 1244 error = workqueue_create(&sc->sc_queue_wq, xnamebuf, 1245 igc_handle_queue_work, sc, IGC_WORKQUEUE_PRI, IPL_NET, 1246 WQ_PERCPU | WQ_MPSAFE); 1247 if (error) { 1248 aprint_error_dev(dev, "workqueue_create failed\n"); 1249 return ENOBUFS; 1250 } 1251 sc->sc_txrx_workqueue = false; 1252 1253 intrstr = pci_intr_string(pc, *intrs, intrbuf, sizeof(intrbuf)); 1254 snprintf(xnamebuf, sizeof(xnamebuf), "%s: link", device_xname(dev)); 1255 pci_intr_setattr(pc, intrs, PCI_INTR_MPSAFE, true); 1256 *ihs = pci_intr_establish_xname(pc, *intrs, IPL_NET, 1257 igc_intr_link, sc, xnamebuf); 1258 if (*ihs == NULL) { 1259 aprint_error_dev(dev, 1260 "unable to establish link interrupt at %s\n", intrstr); 1261 return ENOBUFS; 1262 } 1263 aprint_normal_dev(dev, "link interrupting at %s\n", intrstr); 1264 /* use later in igc_configure_queues() */ 1265 sc->linkvec = iq; 1266 1267 return 0; 1268 } 1269 1270 static int 1271 igc_setup_msi(struct igc_softc *sc) 1272 { 1273 pci_chipset_tag_t pc = sc->osdep.os_pa.pa_pc; 1274 device_t dev = sc->sc_dev; 1275 pci_intr_handle_t *intr = sc->sc_intrs; 1276 void **ihs = sc->sc_ihs; 1277 const char *intrstr; 1278 char intrbuf[PCI_INTRSTR_LEN]; 1279 char xnamebuf[MAX(32, MAXCOMLEN)]; 1280 int error; 1281 1282 intrstr = pci_intr_string(pc, *intr, intrbuf, sizeof(intrbuf)); 1283 1284 snprintf(xnamebuf, sizeof(xnamebuf), "%s: msi", device_xname(dev)); 1285 pci_intr_setattr(pc, intr, PCI_INTR_MPSAFE, true); 1286 *ihs = pci_intr_establish_xname(pc, *intr, IPL_NET, 1287 igc_intr, sc, xnamebuf); 1288 if (*ihs == NULL) { 1289 aprint_error_dev(dev, 1290 "unable to establish interrupt at %s\n", intrstr); 1291 return ENOBUFS; 1292 } 1293 aprint_normal_dev(dev, "interrupting at %s\n", intrstr); 1294 1295 struct igc_queue *iq = sc->queues; 1296 iq->igcq_si = softint_establish(SOFTINT_NET | SOFTINT_MPSAFE, 1297 igc_handle_queue, iq); 1298 if (iq->igcq_si == NULL) { 1299 aprint_error_dev(dev, 1300 "%s: unable to establish softint\n", intrstr); 1301 return ENOBUFS; 1302 } 1303 1304 snprintf(xnamebuf, MAXCOMLEN, "%s_tx_rx", device_xname(dev)); 1305 error = workqueue_create(&sc->sc_queue_wq, xnamebuf, 1306 igc_handle_queue_work, sc, IGC_WORKQUEUE_PRI, IPL_NET, 1307 WQ_PERCPU | WQ_MPSAFE); 1308 if (error) { 1309 aprint_error_dev(dev, "workqueue_create failed\n"); 1310 return ENOBUFS; 1311 } 1312 sc->sc_txrx_workqueue = false; 1313 1314 sc->queues[0].msix = 0; 1315 sc->linkvec = 0; 1316 1317 return 0; 1318 } 1319 1320 static int 1321 igc_setup_intx(struct igc_softc *sc) 1322 { 1323 pci_chipset_tag_t pc = sc->osdep.os_pa.pa_pc; 1324 device_t dev = sc->sc_dev; 1325 pci_intr_handle_t *intr = sc->sc_intrs; 1326 void **ihs = sc->sc_ihs; 1327 const char *intrstr; 1328 char intrbuf[PCI_INTRSTR_LEN]; 1329 char xnamebuf[32]; 1330 1331 intrstr = pci_intr_string(pc, *intr, intrbuf, sizeof(intrbuf)); 1332 1333 snprintf(xnamebuf, sizeof(xnamebuf), "%s:intx", device_xname(dev)); 1334 pci_intr_setattr(pc, intr, PCI_INTR_MPSAFE, true); 1335 *ihs = pci_intr_establish_xname(pc, *intr, IPL_NET, 1336 igc_intr, sc, xnamebuf); 1337 if (*ihs == NULL) { 1338 aprint_error_dev(dev, 1339 "unable to establish interrupt at %s\n", intrstr); 1340 return ENOBUFS; 1341 } 1342 aprint_normal_dev(dev, "interrupting at %s\n", intrstr); 1343 1344 struct igc_queue *iq = sc->queues; 1345 iq->igcq_si = softint_establish(SOFTINT_NET | SOFTINT_MPSAFE, 1346 igc_handle_queue, iq); 1347 if (iq->igcq_si == NULL) { 1348 aprint_error_dev(dev, 1349 "%s: unable to establish softint\n", intrstr); 1350 return ENOBUFS; 1351 } 1352 1353 /* create workqueue? */ 1354 sc->sc_txrx_workqueue = false; 1355 1356 sc->queues[0].msix = 0; 1357 sc->linkvec = 0; 1358 1359 return 0; 1360 } 1361 1362 static int 1363 igc_dma_malloc(struct igc_softc *sc, bus_size_t size, struct igc_dma_alloc *dma) 1364 { 1365 struct igc_osdep *os = &sc->osdep; 1366 1367 dma->dma_tag = os->os_dmat; 1368 1369 if (bus_dmamap_create(dma->dma_tag, size, 1, size, 0, 1370 BUS_DMA_WAITOK | BUS_DMA_ALLOCNOW, &dma->dma_map)) 1371 return 1; 1372 if (bus_dmamem_alloc(dma->dma_tag, size, PAGE_SIZE, 0, &dma->dma_seg, 1373 1, &dma->dma_nseg, BUS_DMA_WAITOK)) 1374 goto destroy; 1375 /* 1376 * XXXRO 1377 * 1378 * Coherent mapping for descriptors is required for now. 1379 * 1380 * Both TX and RX descriptors are 16-byte length, which is shorter 1381 * than dcache lines on modern CPUs. Therefore, sync for a descriptor 1382 * may overwrite DMA read for descriptors in the same cache line. 1383 * 1384 * Can't we avoid this by use cache-line-aligned descriptors at once? 1385 */ 1386 if (bus_dmamem_map(dma->dma_tag, &dma->dma_seg, dma->dma_nseg, size, 1387 &dma->dma_vaddr, BUS_DMA_WAITOK | BUS_DMA_COHERENT /* XXXRO */)) 1388 goto free; 1389 if (bus_dmamap_load(dma->dma_tag, dma->dma_map, dma->dma_vaddr, size, 1390 NULL, BUS_DMA_WAITOK)) 1391 goto unmap; 1392 1393 dma->dma_size = size; 1394 1395 return 0; 1396 unmap: 1397 bus_dmamem_unmap(dma->dma_tag, dma->dma_vaddr, size); 1398 free: 1399 bus_dmamem_free(dma->dma_tag, &dma->dma_seg, dma->dma_nseg); 1400 destroy: 1401 bus_dmamap_destroy(dma->dma_tag, dma->dma_map); 1402 dma->dma_map = NULL; 1403 dma->dma_tag = NULL; 1404 return 1; 1405 } 1406 1407 static void 1408 igc_dma_free(struct igc_softc *sc, struct igc_dma_alloc *dma) 1409 { 1410 1411 if (dma->dma_tag == NULL) 1412 return; 1413 1414 if (dma->dma_map != NULL) { 1415 bus_dmamap_sync(dma->dma_tag, dma->dma_map, 0, 1416 dma->dma_map->dm_mapsize, 1417 BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE); 1418 bus_dmamap_unload(dma->dma_tag, dma->dma_map); 1419 bus_dmamem_unmap(dma->dma_tag, dma->dma_vaddr, dma->dma_size); 1420 bus_dmamem_free(dma->dma_tag, &dma->dma_seg, dma->dma_nseg); 1421 bus_dmamap_destroy(dma->dma_tag, dma->dma_map); 1422 dma->dma_map = NULL; 1423 } 1424 } 1425 1426 /********************************************************************* 1427 * 1428 * Setup networking device structure and register an interface. 1429 * 1430 **********************************************************************/ 1431 static void 1432 igc_setup_interface(struct igc_softc *sc) 1433 { 1434 struct ifnet *ifp = &sc->sc_ec.ec_if; 1435 1436 strlcpy(ifp->if_xname, device_xname(sc->sc_dev), sizeof(ifp->if_xname)); 1437 ifp->if_softc = sc; 1438 ifp->if_flags = IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST; 1439 ifp->if_extflags = IFEF_MPSAFE; 1440 ifp->if_ioctl = igc_ioctl; 1441 ifp->if_start = igc_start; 1442 if (sc->sc_nqueues > 1) 1443 ifp->if_transmit = igc_transmit; 1444 ifp->if_watchdog = igc_watchdog; 1445 ifp->if_init = igc_init; 1446 ifp->if_stop = igc_stop; 1447 1448 ifp->if_capabilities = IFCAP_TSOv4 | IFCAP_TSOv6; 1449 1450 ifp->if_capabilities |= 1451 IFCAP_CSUM_IPv4_Tx | IFCAP_CSUM_IPv4_Rx | 1452 IFCAP_CSUM_TCPv4_Tx | IFCAP_CSUM_TCPv4_Rx | 1453 IFCAP_CSUM_UDPv4_Tx | IFCAP_CSUM_UDPv4_Rx | 1454 IFCAP_CSUM_TCPv6_Tx | IFCAP_CSUM_TCPv6_Rx | 1455 IFCAP_CSUM_UDPv6_Tx | IFCAP_CSUM_UDPv6_Rx; 1456 1457 ifp->if_capenable = 0; 1458 1459 sc->sc_ec.ec_capabilities |= 1460 ETHERCAP_JUMBO_MTU | ETHERCAP_VLAN_MTU; 1461 1462 IFQ_SET_MAXLEN(&ifp->if_snd, sc->num_tx_desc - 1); 1463 IFQ_SET_READY(&ifp->if_snd); 1464 1465 #if NVLAN > 0 1466 sc->sc_ec.ec_capabilities |= ETHERCAP_VLAN_HWTAGGING; 1467 #endif 1468 1469 mutex_init(&sc->sc_core_lock, MUTEX_DEFAULT, IPL_NET); 1470 1471 /* Initialize ifmedia structures. */ 1472 sc->sc_ec.ec_ifmedia = &sc->media; 1473 ifmedia_init_with_lock(&sc->media, IFM_IMASK, igc_media_change, 1474 igc_media_status, &sc->sc_core_lock); 1475 ifmedia_add(&sc->media, IFM_ETHER | IFM_10_T, 0, NULL); 1476 ifmedia_add(&sc->media, IFM_ETHER | IFM_10_T | IFM_FDX, 0, NULL); 1477 ifmedia_add(&sc->media, IFM_ETHER | IFM_100_TX, 0, NULL); 1478 ifmedia_add(&sc->media, IFM_ETHER | IFM_100_TX | IFM_FDX, 0, NULL); 1479 ifmedia_add(&sc->media, IFM_ETHER | IFM_1000_T | IFM_FDX, 0, NULL); 1480 ifmedia_add(&sc->media, IFM_ETHER | IFM_2500_T | IFM_FDX, 0, NULL); 1481 ifmedia_add(&sc->media, IFM_ETHER | IFM_AUTO, 0, NULL); 1482 ifmedia_set(&sc->media, IFM_ETHER | IFM_AUTO); 1483 1484 sc->sc_rx_intr_process_limit = IGC_RX_INTR_PROCESS_LIMIT_DEFAULT; 1485 sc->sc_tx_intr_process_limit = IGC_TX_INTR_PROCESS_LIMIT_DEFAULT; 1486 sc->sc_rx_process_limit = IGC_RX_PROCESS_LIMIT_DEFAULT; 1487 sc->sc_tx_process_limit = IGC_TX_PROCESS_LIMIT_DEFAULT; 1488 1489 if_initialize(ifp); 1490 sc->sc_ipq = if_percpuq_create(ifp); 1491 if_deferred_start_init(ifp, NULL); 1492 ether_ifattach(ifp, sc->hw.mac.addr); 1493 ether_set_ifflags_cb(&sc->sc_ec, igc_ifflags_cb); 1494 if_register(ifp); 1495 } 1496 1497 static int 1498 igc_init(struct ifnet *ifp) 1499 { 1500 struct igc_softc *sc = ifp->if_softc; 1501 int error; 1502 1503 mutex_enter(&sc->sc_core_lock); 1504 error = igc_init_locked(sc); 1505 mutex_exit(&sc->sc_core_lock); 1506 1507 return error; 1508 } 1509 1510 static int 1511 igc_init_locked(struct igc_softc *sc) 1512 { 1513 struct ethercom *ec = &sc->sc_ec; 1514 struct ifnet *ifp = &ec->ec_if; 1515 1516 DPRINTF(CFG, "called\n"); 1517 1518 KASSERT(mutex_owned(&sc->sc_core_lock)); 1519 1520 if (ISSET(ifp->if_flags, IFF_RUNNING)) 1521 igc_stop_locked(sc); 1522 1523 /* Put the address into the receive address array. */ 1524 igc_rar_set(&sc->hw, sc->hw.mac.addr, 0); 1525 1526 /* Initialize the hardware. */ 1527 igc_reset(sc); 1528 igc_update_link_status(sc); 1529 1530 /* Setup VLAN support, basic and offload if available. */ 1531 IGC_WRITE_REG(&sc->hw, IGC_VET, ETHERTYPE_VLAN); 1532 1533 igc_initialize_transmit_unit(sc); 1534 igc_initialize_receive_unit(sc); 1535 1536 if (ec->ec_capenable & ETHERCAP_VLAN_HWTAGGING) { 1537 uint32_t ctrl = IGC_READ_REG(&sc->hw, IGC_CTRL); 1538 ctrl |= IGC_CTRL_VME; 1539 IGC_WRITE_REG(&sc->hw, IGC_CTRL, ctrl); 1540 } 1541 1542 /* Setup multicast table. */ 1543 igc_set_filter(sc); 1544 1545 igc_clear_hw_cntrs_base_generic(&sc->hw); 1546 1547 if (sc->sc_intr_type == PCI_INTR_TYPE_MSIX) 1548 igc_configure_queues(sc); 1549 1550 /* This clears any pending interrupts */ 1551 IGC_READ_REG(&sc->hw, IGC_ICR); 1552 IGC_WRITE_REG(&sc->hw, IGC_ICS, IGC_ICS_LSC); 1553 1554 /* The driver can now take control from firmware. */ 1555 igc_get_hw_control(sc); 1556 1557 /* Set Energy Efficient Ethernet. */ 1558 igc_set_eee_i225(&sc->hw, true, true, true); 1559 1560 for (int iq = 0; iq < sc->sc_nqueues; iq++) { 1561 struct rx_ring *rxr = &sc->rx_rings[iq]; 1562 1563 mutex_enter(&rxr->rxr_lock); 1564 igc_rxfill(rxr); 1565 mutex_exit(&rxr->rxr_lock); 1566 } 1567 1568 sc->sc_core_stopping = false; 1569 1570 ifp->if_flags |= IFF_RUNNING; 1571 1572 /* Save last flags for the callback */ 1573 sc->sc_if_flags = ifp->if_flags; 1574 1575 callout_schedule(&sc->sc_tick_ch, hz); 1576 1577 igc_enable_intr(sc); 1578 1579 return 0; 1580 } 1581 1582 static inline int 1583 igc_load_mbuf(struct igc_queue *q, bus_dma_tag_t dmat, bus_dmamap_t map, 1584 struct mbuf *m) 1585 { 1586 int error; 1587 1588 error = bus_dmamap_load_mbuf(dmat, map, m, 1589 BUS_DMA_WRITE | BUS_DMA_NOWAIT); 1590 1591 if (__predict_false(error == EFBIG)) { 1592 IGC_DRIVER_EVENT(q, txdma_efbig, 1); 1593 m = m_defrag(m, M_NOWAIT); 1594 if (__predict_false(m == NULL)) { 1595 IGC_DRIVER_EVENT(q, txdma_defrag, 1); 1596 return ENOBUFS; 1597 } 1598 error = bus_dmamap_load_mbuf(dmat, map, m, 1599 BUS_DMA_WRITE | BUS_DMA_NOWAIT); 1600 } 1601 1602 switch (error) { 1603 case 0: 1604 break; 1605 case ENOMEM: 1606 IGC_DRIVER_EVENT(q, txdma_enomem, 1); 1607 break; 1608 case EINVAL: 1609 IGC_DRIVER_EVENT(q, txdma_einval, 1); 1610 break; 1611 case EAGAIN: 1612 IGC_DRIVER_EVENT(q, txdma_eagain, 1); 1613 break; 1614 default: 1615 IGC_DRIVER_EVENT(q, txdma_other, 1); 1616 break; 1617 } 1618 1619 return error; 1620 } 1621 1622 #define IGC_TX_START 1 1623 #define IGC_TX_TRANSMIT 2 1624 1625 static void 1626 igc_start(struct ifnet *ifp) 1627 { 1628 struct igc_softc *sc = ifp->if_softc; 1629 1630 if (__predict_false(!sc->link_active)) { 1631 IFQ_PURGE(&ifp->if_snd); 1632 return; 1633 } 1634 1635 struct tx_ring *txr = &sc->tx_rings[0]; /* queue 0 */ 1636 mutex_enter(&txr->txr_lock); 1637 igc_tx_common_locked(ifp, txr, IGC_TX_START); 1638 mutex_exit(&txr->txr_lock); 1639 } 1640 1641 static inline u_int 1642 igc_select_txqueue(struct igc_softc *sc, struct mbuf *m __unused) 1643 { 1644 const u_int cpuid = cpu_index(curcpu()); 1645 1646 return cpuid % sc->sc_nqueues; 1647 } 1648 1649 static int 1650 igc_transmit(struct ifnet *ifp, struct mbuf *m) 1651 { 1652 struct igc_softc *sc = ifp->if_softc; 1653 const u_int qid = igc_select_txqueue(sc, m); 1654 struct tx_ring *txr = &sc->tx_rings[qid]; 1655 struct igc_queue *q = txr->txr_igcq; 1656 1657 if (__predict_false(!pcq_put(txr->txr_interq, m))) { 1658 IGC_QUEUE_EVENT(q, tx_pcq_drop, 1); 1659 m_freem(m); 1660 return ENOBUFS; 1661 } 1662 1663 mutex_enter(&txr->txr_lock); 1664 igc_tx_common_locked(ifp, txr, IGC_TX_TRANSMIT); 1665 mutex_exit(&txr->txr_lock); 1666 1667 return 0; 1668 } 1669 1670 static void 1671 igc_tx_common_locked(struct ifnet *ifp, struct tx_ring *txr, int caller) 1672 { 1673 struct igc_softc *sc = ifp->if_softc; 1674 struct igc_queue *q = txr->txr_igcq; 1675 net_stat_ref_t nsr = IF_STAT_GETREF(ifp); 1676 int prod, free, last = -1; 1677 bool post = false; 1678 1679 prod = txr->next_avail_desc; 1680 free = txr->next_to_clean; 1681 if (free <= prod) 1682 free += sc->num_tx_desc; 1683 free -= prod; 1684 1685 DPRINTF(TX, "%s: begin: msix %d prod %d n2c %d free %d\n", 1686 caller == IGC_TX_TRANSMIT ? "transmit" : "start", 1687 txr->me, prod, txr->next_to_clean, free); 1688 1689 for (;;) { 1690 struct mbuf *m; 1691 1692 if (__predict_false(free <= IGC_MAX_SCATTER)) { 1693 IGC_QUEUE_EVENT(q, tx_no_desc, 1); 1694 break; 1695 } 1696 1697 if (caller == IGC_TX_TRANSMIT) 1698 m = pcq_get(txr->txr_interq); 1699 else 1700 IFQ_DEQUEUE(&ifp->if_snd, m); 1701 if (__predict_false(m == NULL)) 1702 break; 1703 1704 struct igc_tx_buf *txbuf = &txr->tx_buffers[prod]; 1705 bus_dmamap_t map = txbuf->map; 1706 1707 if (__predict_false( 1708 igc_load_mbuf(q, txr->txdma.dma_tag, map, m))) { 1709 if (caller == IGC_TX_TRANSMIT) 1710 IGC_QUEUE_EVENT(q, tx_pcq_drop, 1); 1711 m_freem(m); 1712 if_statinc_ref(ifp, nsr, if_oerrors); 1713 continue; 1714 } 1715 1716 uint32_t ctx_cmd_type_len = 0, olinfo_status = 0; 1717 if (igc_tx_ctx_setup(txr, m, prod, &ctx_cmd_type_len, 1718 &olinfo_status)) { 1719 IGC_QUEUE_EVENT(q, tx_ctx, 1); 1720 /* Consume the first descriptor */ 1721 prod = igc_txdesc_incr(sc, prod); 1722 free--; 1723 } 1724 1725 bus_dmamap_sync(txr->txdma.dma_tag, map, 0, 1726 map->dm_mapsize, BUS_DMASYNC_PREWRITE); 1727 1728 for (int i = 0; i < map->dm_nsegs; i++) { 1729 union igc_adv_tx_desc *txdesc = &txr->tx_base[prod]; 1730 1731 uint32_t cmd_type_len = ctx_cmd_type_len | 1732 IGC_ADVTXD_DCMD_IFCS | IGC_ADVTXD_DTYP_DATA | 1733 IGC_ADVTXD_DCMD_DEXT | map->dm_segs[i].ds_len; 1734 if (i == map->dm_nsegs - 1) { 1735 cmd_type_len |= 1736 IGC_ADVTXD_DCMD_EOP | IGC_ADVTXD_DCMD_RS; 1737 } 1738 1739 igc_txdesc_sync(txr, prod, 1740 BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE); 1741 htolem64(&txdesc->read.buffer_addr, 1742 map->dm_segs[i].ds_addr); 1743 htolem32(&txdesc->read.cmd_type_len, cmd_type_len); 1744 htolem32(&txdesc->read.olinfo_status, olinfo_status); 1745 igc_txdesc_sync(txr, prod, 1746 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); 1747 1748 last = prod; 1749 prod = igc_txdesc_incr(sc, prod); 1750 } 1751 1752 txbuf->m_head = m; 1753 txbuf->eop_index = last; 1754 1755 bpf_mtap(ifp, m, BPF_D_OUT); 1756 1757 if_statadd_ref(ifp, nsr, if_obytes, m->m_pkthdr.len); 1758 if (m->m_flags & M_MCAST) 1759 if_statinc_ref(ifp, nsr, if_omcasts); 1760 IGC_QUEUE_EVENT(q, tx_packets, 1); 1761 IGC_QUEUE_EVENT(q, tx_bytes, m->m_pkthdr.len); 1762 1763 free -= map->dm_nsegs; 1764 post = true; 1765 } 1766 1767 if (post) { 1768 txr->next_avail_desc = prod; 1769 IGC_WRITE_REG(&sc->hw, IGC_TDT(txr->me), prod); 1770 } 1771 1772 DPRINTF(TX, "%s: done : msix %d prod %d n2c %d free %d\n", 1773 caller == IGC_TX_TRANSMIT ? "transmit" : "start", 1774 txr->me, prod, txr->next_to_clean, free); 1775 1776 IF_STAT_PUTREF(ifp); 1777 } 1778 1779 static bool 1780 igc_txeof(struct tx_ring *txr, u_int limit) 1781 { 1782 struct igc_softc *sc = txr->sc; 1783 struct ifnet *ifp = &sc->sc_ec.ec_if; 1784 int cons, prod; 1785 bool more = false; 1786 1787 prod = txr->next_avail_desc; 1788 cons = txr->next_to_clean; 1789 1790 if (cons == prod) { 1791 DPRINTF(TX, "false: msix %d cons %d prod %d\n", 1792 txr->me, cons, prod); 1793 return false; 1794 } 1795 1796 do { 1797 struct igc_tx_buf *txbuf = &txr->tx_buffers[cons]; 1798 const int last = txbuf->eop_index; 1799 1800 membar_consumer(); /* XXXRO necessary? */ 1801 1802 KASSERT(last != -1); 1803 union igc_adv_tx_desc *txdesc = &txr->tx_base[last]; 1804 igc_txdesc_sync(txr, last, BUS_DMASYNC_POSTREAD); 1805 const uint32_t status = le32toh(txdesc->wb.status); 1806 igc_txdesc_sync(txr, last, BUS_DMASYNC_PREREAD); 1807 1808 if (!(status & IGC_TXD_STAT_DD)) 1809 break; 1810 1811 if (limit-- == 0) { 1812 more = true; 1813 DPRINTF(TX, "pending TX " 1814 "msix %d cons %d last %d prod %d " 1815 "status 0x%08x\n", 1816 txr->me, cons, last, prod, status); 1817 break; 1818 } 1819 1820 DPRINTF(TX, "handled TX " 1821 "msix %d cons %d last %d prod %d " 1822 "status 0x%08x\n", 1823 txr->me, cons, last, prod, status); 1824 1825 if_statinc(ifp, if_opackets); 1826 1827 bus_dmamap_t map = txbuf->map; 1828 bus_dmamap_sync(txr->txdma.dma_tag, map, 0, map->dm_mapsize, 1829 BUS_DMASYNC_POSTWRITE); 1830 bus_dmamap_unload(txr->txdma.dma_tag, map); 1831 m_freem(txbuf->m_head); 1832 1833 txbuf->m_head = NULL; 1834 txbuf->eop_index = -1; 1835 1836 cons = igc_txdesc_incr(sc, last); 1837 } while (cons != prod); 1838 1839 txr->next_to_clean = cons; 1840 1841 return more; 1842 } 1843 1844 static void 1845 igc_intr_barrier(struct igc_softc *sc __unused) 1846 { 1847 1848 xc_barrier(0); 1849 } 1850 1851 static void 1852 igc_stop(struct ifnet *ifp, int disable) 1853 { 1854 struct igc_softc *sc = ifp->if_softc; 1855 1856 mutex_enter(&sc->sc_core_lock); 1857 igc_stop_locked(sc); 1858 mutex_exit(&sc->sc_core_lock); 1859 } 1860 1861 /********************************************************************* 1862 * 1863 * This routine disables all traffic on the adapter by issuing a 1864 * global reset on the MAC. 1865 * 1866 **********************************************************************/ 1867 static void 1868 igc_stop_locked(struct igc_softc *sc) 1869 { 1870 struct ifnet *ifp = &sc->sc_ec.ec_if; 1871 1872 DPRINTF(CFG, "called\n"); 1873 1874 KASSERT(mutex_owned(&sc->sc_core_lock)); 1875 1876 /* 1877 * If stopping processing has already started, do nothing. 1878 */ 1879 if ((ifp->if_flags & IFF_RUNNING) == 0) 1880 return; 1881 1882 /* Tell the stack that the interface is no longer active. */ 1883 ifp->if_flags &= ~IFF_RUNNING; 1884 1885 /* 1886 * igc_handle_queue() can enable interrupts, so wait for completion of 1887 * last igc_handle_queue() after unset IFF_RUNNING. 1888 */ 1889 mutex_exit(&sc->sc_core_lock); 1890 igc_barrier_handle_queue(sc); 1891 mutex_enter(&sc->sc_core_lock); 1892 1893 sc->sc_core_stopping = true; 1894 1895 igc_disable_intr(sc); 1896 1897 callout_halt(&sc->sc_tick_ch, &sc->sc_core_lock); 1898 1899 igc_reset_hw(&sc->hw); 1900 IGC_WRITE_REG(&sc->hw, IGC_WUC, 0); 1901 1902 /* 1903 * Wait for completion of interrupt handlers. 1904 */ 1905 mutex_exit(&sc->sc_core_lock); 1906 igc_intr_barrier(sc); 1907 mutex_enter(&sc->sc_core_lock); 1908 1909 igc_update_link_status(sc); 1910 1911 for (int iq = 0; iq < sc->sc_nqueues; iq++) { 1912 struct tx_ring *txr = &sc->tx_rings[iq]; 1913 1914 igc_withdraw_transmit_packets(txr, false); 1915 } 1916 1917 for (int iq = 0; iq < sc->sc_nqueues; iq++) { 1918 struct rx_ring *rxr = &sc->rx_rings[iq]; 1919 1920 igc_clear_receive_status(rxr); 1921 } 1922 1923 /* Save last flags for the callback */ 1924 sc->sc_if_flags = ifp->if_flags; 1925 } 1926 1927 /********************************************************************* 1928 * Ioctl entry point 1929 * 1930 * igc_ioctl is called when the user wants to configure the 1931 * interface. 1932 * 1933 * return 0 on success, positive on failure 1934 **********************************************************************/ 1935 static int 1936 igc_ioctl(struct ifnet * ifp, u_long cmd, void *data) 1937 { 1938 struct igc_softc *sc __unused = ifp->if_softc; 1939 int s; 1940 int error; 1941 1942 DPRINTF(CFG, "cmd 0x%016lx\n", cmd); 1943 1944 switch (cmd) { 1945 case SIOCADDMULTI: 1946 case SIOCDELMULTI: 1947 break; 1948 default: 1949 KASSERT(IFNET_LOCKED(ifp)); 1950 } 1951 1952 if (cmd == SIOCZIFDATA) { 1953 mutex_enter(&sc->sc_core_lock); 1954 igc_clear_counters(sc); 1955 mutex_exit(&sc->sc_core_lock); 1956 } 1957 1958 switch (cmd) { 1959 #ifdef IF_RXR 1960 case SIOCGIFRXR: 1961 s = splnet(); 1962 error = igc_rxrinfo(sc, (struct if_rxrinfo *)ifr->ifr_data); 1963 splx(s); 1964 break; 1965 #endif 1966 default: 1967 s = splnet(); 1968 error = ether_ioctl(ifp, cmd, data); 1969 splx(s); 1970 break; 1971 } 1972 1973 if (error != ENETRESET) 1974 return error; 1975 1976 error = 0; 1977 1978 if (cmd == SIOCSIFCAP) 1979 error = if_init(ifp); 1980 else if ((cmd == SIOCADDMULTI) || (cmd == SIOCDELMULTI)) { 1981 mutex_enter(&sc->sc_core_lock); 1982 if (sc->sc_if_flags & IFF_RUNNING) { 1983 /* 1984 * Multicast list has changed; set the hardware filter 1985 * accordingly. 1986 */ 1987 igc_disable_intr(sc); 1988 igc_set_filter(sc); 1989 igc_enable_intr(sc); 1990 } 1991 mutex_exit(&sc->sc_core_lock); 1992 } 1993 1994 return error; 1995 } 1996 1997 #ifdef IF_RXR 1998 static int 1999 igc_rxrinfo(struct igc_softc *sc, struct if_rxrinfo *ifri) 2000 { 2001 struct if_rxring_info *ifr, ifr1; 2002 int error; 2003 2004 if (sc->sc_nqueues > 1) { 2005 ifr = kmem_zalloc(sc->sc_nqueues * sizeof(*ifr), KM_SLEEP); 2006 } else { 2007 ifr = &ifr1; 2008 memset(ifr, 0, sizeof(*ifr)); 2009 } 2010 2011 for (int iq = 0; iq < sc->sc_nqueues; iq++) { 2012 struct rx_ring *rxr = &sc->rx_rings[iq]; 2013 2014 ifr[iq].ifr_size = MCLBYTES; 2015 snprintf(ifr[iq].ifr_name, sizeof(ifr[iq].ifr_name), "%d", iq); 2016 ifr[iq].ifr_info = rxr->rx_ring; 2017 } 2018 2019 error = if_rxr_info_ioctl(ifri, sc->sc_nqueues, ifr); 2020 if (sc->sc_nqueues > 1) 2021 kmem_free(ifr, sc->sc_nqueues * sizeof(*ifr)); 2022 2023 return error; 2024 } 2025 #endif 2026 2027 static void 2028 igc_rxfill(struct rx_ring *rxr) 2029 { 2030 struct igc_softc *sc = rxr->sc; 2031 int id; 2032 2033 for (id = 0; id < sc->num_rx_desc; id++) { 2034 if (igc_get_buf(rxr, id, false)) { 2035 panic("%s: msix=%d i=%d\n", __func__, rxr->me, id); 2036 } 2037 } 2038 2039 id = sc->num_rx_desc - 1; 2040 rxr->last_desc_filled = id; 2041 IGC_WRITE_REG(&sc->hw, IGC_RDT(rxr->me), id); 2042 rxr->next_to_check = 0; 2043 } 2044 2045 static void 2046 igc_rxrefill(struct rx_ring *rxr, int end) 2047 { 2048 struct igc_softc *sc = rxr->sc; 2049 int id; 2050 2051 for (id = rxr->next_to_check; id != end; id = igc_rxdesc_incr(sc, id)) { 2052 if (igc_get_buf(rxr, id, true)) { 2053 /* XXXRO */ 2054 panic("%s: msix=%d id=%d\n", __func__, rxr->me, id); 2055 } 2056 } 2057 2058 id = igc_rxdesc_decr(sc, id); 2059 DPRINTF(RX, "%s RDT %d id %d\n", 2060 rxr->last_desc_filled == id ? "same" : "diff", 2061 rxr->last_desc_filled, id); 2062 rxr->last_desc_filled = id; 2063 IGC_WRITE_REG(&sc->hw, IGC_RDT(rxr->me), id); 2064 } 2065 2066 /********************************************************************* 2067 * 2068 * This routine executes in interrupt context. It replenishes 2069 * the mbufs in the descriptor and sends data which has been 2070 * dma'ed into host memory to upper layer. 2071 * 2072 *********************************************************************/ 2073 static bool 2074 igc_rxeof(struct rx_ring *rxr, u_int limit) 2075 { 2076 struct igc_softc *sc = rxr->sc; 2077 struct igc_queue *q = rxr->rxr_igcq; 2078 struct ifnet *ifp = &sc->sc_ec.ec_if; 2079 int id; 2080 bool more = false; 2081 2082 id = rxr->next_to_check; 2083 for (;;) { 2084 union igc_adv_rx_desc *rxdesc = &rxr->rx_base[id]; 2085 struct igc_rx_buf *rxbuf, *nxbuf; 2086 struct mbuf *mp, *m; 2087 2088 igc_rxdesc_sync(rxr, id, 2089 BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE); 2090 2091 const uint32_t staterr = le32toh(rxdesc->wb.upper.status_error); 2092 2093 if (!ISSET(staterr, IGC_RXD_STAT_DD)) { 2094 igc_rxdesc_sync(rxr, id, 2095 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); 2096 break; 2097 } 2098 2099 if (limit-- == 0) { 2100 igc_rxdesc_sync(rxr, id, 2101 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); 2102 DPRINTF(RX, "more=true\n"); 2103 more = true; 2104 break; 2105 } 2106 2107 /* Zero out the receive descriptors status. */ 2108 rxdesc->wb.upper.status_error = 0; 2109 2110 /* Pull the mbuf off the ring. */ 2111 rxbuf = &rxr->rx_buffers[id]; 2112 bus_dmamap_t map = rxbuf->map; 2113 bus_dmamap_sync(rxr->rxdma.dma_tag, map, 2114 0, map->dm_mapsize, BUS_DMASYNC_POSTREAD); 2115 bus_dmamap_unload(rxr->rxdma.dma_tag, map); 2116 2117 mp = rxbuf->buf; 2118 rxbuf->buf = NULL; 2119 2120 const bool eop = staterr & IGC_RXD_STAT_EOP; 2121 const uint16_t len = le16toh(rxdesc->wb.upper.length); 2122 2123 #if NVLAN > 0 2124 const uint16_t vtag = le16toh(rxdesc->wb.upper.vlan); 2125 #endif 2126 2127 const uint32_t ptype = le32toh(rxdesc->wb.lower.lo_dword.data) & 2128 IGC_PKTTYPE_MASK; 2129 2130 const uint32_t hash __unused = 2131 le32toh(rxdesc->wb.lower.hi_dword.rss); 2132 const uint16_t hashtype __unused = 2133 le16toh(rxdesc->wb.lower.lo_dword.hs_rss.pkt_info) & 2134 IGC_RXDADV_RSSTYPE_MASK; 2135 2136 igc_rxdesc_sync(rxr, id, 2137 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); 2138 2139 if (__predict_false(staterr & IGC_RXDEXT_STATERR_RXE)) { 2140 m_freem(rxbuf->fmp); 2141 rxbuf->fmp = NULL; 2142 2143 m_freem(mp); 2144 m = NULL; 2145 2146 if_statinc(ifp, if_ierrors); 2147 IGC_QUEUE_EVENT(q, rx_discard, 1); 2148 2149 DPRINTF(RX, "ierrors++\n"); 2150 2151 goto next_desc; 2152 } 2153 2154 if (__predict_false(mp == NULL)) { 2155 panic("%s: igc_rxeof: NULL mbuf in slot %d " 2156 "(filled %d)", device_xname(sc->sc_dev), 2157 id, rxr->last_desc_filled); 2158 } 2159 2160 if (!eop) { 2161 /* 2162 * Figure out the next descriptor of this frame. 2163 */ 2164 int nextp = igc_rxdesc_incr(sc, id); 2165 2166 nxbuf = &rxr->rx_buffers[nextp]; 2167 /* 2168 * TODO prefetch(nxbuf); 2169 */ 2170 } 2171 2172 mp->m_len = len; 2173 2174 m = rxbuf->fmp; 2175 rxbuf->fmp = NULL; 2176 2177 if (m != NULL) { 2178 m->m_pkthdr.len += mp->m_len; 2179 } else { 2180 m = mp; 2181 m->m_pkthdr.len = mp->m_len; 2182 #if NVLAN > 0 2183 if (staterr & IGC_RXD_STAT_VP) 2184 vlan_set_tag(m, vtag); 2185 #endif 2186 } 2187 2188 /* Pass the head pointer on */ 2189 if (!eop) { 2190 nxbuf->fmp = m; 2191 m = NULL; 2192 mp->m_next = nxbuf->buf; 2193 } else { 2194 m_set_rcvif(m, ifp); 2195 2196 m->m_pkthdr.csum_flags = igc_rx_checksum(q, 2197 ifp->if_capenable, staterr, ptype); 2198 2199 #ifdef notyet 2200 if (hashtype != IGC_RXDADV_RSSTYPE_NONE) { 2201 m->m_pkthdr.ph_flowid = hash; 2202 SET(m->m_pkthdr.csum_flags, M_FLOWID); 2203 } 2204 ml_enqueue(&ml, m); 2205 #endif 2206 2207 if_percpuq_enqueue(sc->sc_ipq, m); 2208 2209 if_statinc(ifp, if_ipackets); 2210 IGC_QUEUE_EVENT(q, rx_packets, 1); 2211 IGC_QUEUE_EVENT(q, rx_bytes, m->m_pkthdr.len); 2212 } 2213 next_desc: 2214 /* Advance our pointers to the next descriptor. */ 2215 id = igc_rxdesc_incr(sc, id); 2216 } 2217 2218 DPRINTF(RX, "fill queue[%d]\n", rxr->me); 2219 igc_rxrefill(rxr, id); 2220 2221 DPRINTF(RX, "%s n2c %d id %d\n", 2222 rxr->next_to_check == id ? "same" : "diff", 2223 rxr->next_to_check, id); 2224 rxr->next_to_check = id; 2225 2226 #ifdef OPENBSD 2227 if (!(staterr & IGC_RXD_STAT_DD)) 2228 return 0; 2229 #endif 2230 2231 return more; 2232 } 2233 2234 /********************************************************************* 2235 * 2236 * Verify that the hardware indicated that the checksum is valid. 2237 * Inform the stack about the status of checksum so that stack 2238 * doesn't spend time verifying the checksum. 2239 * 2240 *********************************************************************/ 2241 static int 2242 igc_rx_checksum(struct igc_queue *q, uint64_t capenable, uint32_t staterr, 2243 uint32_t ptype) 2244 { 2245 const uint16_t status = (uint16_t)staterr; 2246 const uint8_t errors = (uint8_t)(staterr >> 24); 2247 int flags = 0; 2248 2249 if ((status & IGC_RXD_STAT_IPCS) != 0 && 2250 (capenable & IFCAP_CSUM_IPv4_Rx) != 0) { 2251 IGC_DRIVER_EVENT(q, rx_ipcs, 1); 2252 flags |= M_CSUM_IPv4; 2253 if (__predict_false((errors & IGC_RXD_ERR_IPE) != 0)) { 2254 IGC_DRIVER_EVENT(q, rx_ipcs_bad, 1); 2255 flags |= M_CSUM_IPv4_BAD; 2256 } 2257 } 2258 2259 if ((status & IGC_RXD_STAT_TCPCS) != 0) { 2260 IGC_DRIVER_EVENT(q, rx_tcpcs, 1); 2261 if ((capenable & IFCAP_CSUM_TCPv4_Rx) != 0) 2262 flags |= M_CSUM_TCPv4; 2263 if ((capenable & IFCAP_CSUM_TCPv6_Rx) != 0) 2264 flags |= M_CSUM_TCPv6; 2265 } 2266 2267 if ((status & IGC_RXD_STAT_UDPCS) != 0) { 2268 IGC_DRIVER_EVENT(q, rx_udpcs, 1); 2269 if ((capenable & IFCAP_CSUM_UDPv4_Rx) != 0) 2270 flags |= M_CSUM_UDPv4; 2271 if ((capenable & IFCAP_CSUM_UDPv6_Rx) != 0) 2272 flags |= M_CSUM_UDPv6; 2273 } 2274 2275 if (__predict_false((errors & IGC_RXD_ERR_TCPE) != 0)) { 2276 IGC_DRIVER_EVENT(q, rx_l4cs_bad, 1); 2277 if ((flags & ~M_CSUM_IPv4) != 0) 2278 flags |= M_CSUM_TCP_UDP_BAD; 2279 } 2280 2281 return flags; 2282 } 2283 2284 static void 2285 igc_watchdog(struct ifnet * ifp) 2286 { 2287 } 2288 2289 static void 2290 igc_tick(void *arg) 2291 { 2292 struct igc_softc *sc = arg; 2293 2294 mutex_enter(&sc->sc_core_lock); 2295 2296 if (__predict_false(sc->sc_core_stopping)) { 2297 mutex_exit(&sc->sc_core_lock); 2298 return; 2299 } 2300 2301 /* XXX watchdog */ 2302 if (0) { 2303 IGC_GLOBAL_EVENT(sc, watchdog, 1); 2304 } 2305 2306 igc_update_counters(sc); 2307 2308 mutex_exit(&sc->sc_core_lock); 2309 2310 callout_schedule(&sc->sc_tick_ch, hz); 2311 } 2312 2313 /********************************************************************* 2314 * 2315 * Media Ioctl callback 2316 * 2317 * This routine is called whenever the user queries the status of 2318 * the interface using ifconfig. 2319 * 2320 **********************************************************************/ 2321 static void 2322 igc_media_status(struct ifnet *ifp, struct ifmediareq *ifmr) 2323 { 2324 struct igc_softc *sc = ifp->if_softc; 2325 struct igc_hw *hw = &sc->hw; 2326 2327 igc_update_link_status(sc); 2328 2329 ifmr->ifm_status = IFM_AVALID; 2330 ifmr->ifm_active = IFM_ETHER; 2331 2332 if (!sc->link_active) { 2333 ifmr->ifm_active |= IFM_NONE; 2334 return; 2335 } 2336 2337 ifmr->ifm_status |= IFM_ACTIVE; 2338 2339 switch (sc->link_speed) { 2340 case 10: 2341 ifmr->ifm_active |= IFM_10_T; 2342 break; 2343 case 100: 2344 ifmr->ifm_active |= IFM_100_TX; 2345 break; 2346 case 1000: 2347 ifmr->ifm_active |= IFM_1000_T; 2348 break; 2349 case 2500: 2350 ifmr->ifm_active |= IFM_2500_T; 2351 break; 2352 } 2353 2354 if (sc->link_duplex == FULL_DUPLEX) 2355 ifmr->ifm_active |= IFM_FDX; 2356 else 2357 ifmr->ifm_active |= IFM_HDX; 2358 2359 switch (hw->fc.current_mode) { 2360 case igc_fc_tx_pause: 2361 ifmr->ifm_active |= IFM_FLOW | IFM_ETH_TXPAUSE; 2362 break; 2363 case igc_fc_rx_pause: 2364 ifmr->ifm_active |= IFM_FLOW | IFM_ETH_RXPAUSE; 2365 break; 2366 case igc_fc_full: 2367 ifmr->ifm_active |= IFM_FLOW | 2368 IFM_ETH_TXPAUSE | IFM_ETH_RXPAUSE; 2369 break; 2370 case igc_fc_none: 2371 default: 2372 break; 2373 } 2374 } 2375 2376 /********************************************************************* 2377 * 2378 * Media Ioctl callback 2379 * 2380 * This routine is called when the user changes speed/duplex using 2381 * media/mediopt option with ifconfig. 2382 * 2383 **********************************************************************/ 2384 static int 2385 igc_media_change(struct ifnet *ifp) 2386 { 2387 struct igc_softc *sc = ifp->if_softc; 2388 struct ifmedia *ifm = &sc->media; 2389 2390 if (IFM_TYPE(ifm->ifm_media) != IFM_ETHER) 2391 return EINVAL; 2392 2393 sc->hw.mac.autoneg = DO_AUTO_NEG; 2394 2395 switch (IFM_SUBTYPE(ifm->ifm_media)) { 2396 case IFM_AUTO: 2397 sc->hw.phy.autoneg_advertised = AUTONEG_ADV_DEFAULT; 2398 break; 2399 case IFM_2500_T: 2400 sc->hw.phy.autoneg_advertised = ADVERTISE_2500_FULL; 2401 break; 2402 case IFM_1000_T: 2403 sc->hw.phy.autoneg_advertised = ADVERTISE_1000_FULL; 2404 break; 2405 case IFM_100_TX: 2406 if ((ifm->ifm_media & IFM_GMASK) == IFM_FDX) 2407 sc->hw.phy.autoneg_advertised = ADVERTISE_100_FULL; 2408 else 2409 sc->hw.phy.autoneg_advertised = ADVERTISE_100_HALF; 2410 break; 2411 case IFM_10_T: 2412 if ((ifm->ifm_media & IFM_GMASK) == IFM_FDX) 2413 sc->hw.phy.autoneg_advertised = ADVERTISE_10_FULL; 2414 else 2415 sc->hw.phy.autoneg_advertised = ADVERTISE_10_HALF; 2416 break; 2417 default: 2418 return EINVAL; 2419 } 2420 2421 igc_init_locked(sc); 2422 2423 return 0; 2424 } 2425 2426 static int 2427 igc_ifflags_cb(struct ethercom *ec) 2428 { 2429 struct ifnet *ifp = &ec->ec_if; 2430 struct igc_softc *sc = ifp->if_softc; 2431 int rc = 0; 2432 u_short iffchange; 2433 bool needreset = false; 2434 2435 DPRINTF(CFG, "called\n"); 2436 2437 KASSERT(IFNET_LOCKED(ifp)); 2438 2439 mutex_enter(&sc->sc_core_lock); 2440 2441 /* 2442 * Check for if_flags. 2443 * Main usage is to prevent linkdown when opening bpf. 2444 */ 2445 iffchange = ifp->if_flags ^ sc->sc_if_flags; 2446 sc->sc_if_flags = ifp->if_flags; 2447 if ((iffchange & ~(IFF_CANTCHANGE | IFF_DEBUG)) != 0) { 2448 needreset = true; 2449 goto ec; 2450 } 2451 2452 /* iff related updates */ 2453 if ((iffchange & IFF_PROMISC) != 0) 2454 igc_set_filter(sc); 2455 2456 #ifdef notyet 2457 igc_set_vlan(sc); 2458 #endif 2459 2460 ec: 2461 #ifdef notyet 2462 /* Check for ec_capenable. */ 2463 ecchange = ec->ec_capenable ^ sc->sc_ec_capenable; 2464 sc->sc_ec_capenable = ec->ec_capenable; 2465 if ((ecchange & ~ETHERCAP_SOMETHING) != 0) { 2466 needreset = true; 2467 goto out; 2468 } 2469 #endif 2470 if (needreset) 2471 rc = ENETRESET; 2472 2473 mutex_exit(&sc->sc_core_lock); 2474 2475 return rc; 2476 } 2477 2478 static void 2479 igc_set_filter(struct igc_softc *sc) 2480 { 2481 struct ethercom *ec = &sc->sc_ec; 2482 uint32_t rctl; 2483 2484 rctl = IGC_READ_REG(&sc->hw, IGC_RCTL); 2485 rctl &= ~(IGC_RCTL_BAM |IGC_RCTL_UPE | IGC_RCTL_MPE); 2486 2487 if ((sc->sc_if_flags & IFF_BROADCAST) != 0) 2488 rctl |= IGC_RCTL_BAM; 2489 if ((sc->sc_if_flags & IFF_PROMISC) != 0) { 2490 DPRINTF(CFG, "promisc\n"); 2491 rctl |= IGC_RCTL_UPE; 2492 ETHER_LOCK(ec); 2493 allmulti: 2494 ec->ec_flags |= ETHER_F_ALLMULTI; 2495 ETHER_UNLOCK(ec); 2496 rctl |= IGC_RCTL_MPE; 2497 } else { 2498 struct ether_multistep step; 2499 struct ether_multi *enm; 2500 int mcnt = 0; 2501 2502 memset(sc->mta, 0, IGC_MTA_LEN); 2503 2504 ETHER_LOCK(ec); 2505 ETHER_FIRST_MULTI(step, ec, enm); 2506 while (enm != NULL) { 2507 if (((memcmp(enm->enm_addrlo, enm->enm_addrhi, 2508 ETHER_ADDR_LEN)) != 0) || 2509 (mcnt >= MAX_NUM_MULTICAST_ADDRESSES)) { 2510 /* 2511 * We must listen to a range of multicast 2512 * addresses. For now, just accept all 2513 * multicasts, rather than trying to set only 2514 * those filter bits needed to match the range. 2515 * (At this time, the only use of address 2516 * ranges is for IP multicast routing, for 2517 * which the range is big enough to require all 2518 * bits set.) 2519 */ 2520 goto allmulti; 2521 } 2522 DPRINTF(CFG, "%d: %s\n", mcnt, 2523 ether_sprintf(enm->enm_addrlo)); 2524 memcpy(&sc->mta[mcnt * ETHER_ADDR_LEN], 2525 enm->enm_addrlo, ETHER_ADDR_LEN); 2526 2527 mcnt++; 2528 ETHER_NEXT_MULTI(step, enm); 2529 } 2530 ec->ec_flags &= ~ETHER_F_ALLMULTI; 2531 ETHER_UNLOCK(ec); 2532 2533 DPRINTF(CFG, "hw filter\n"); 2534 igc_update_mc_addr_list(&sc->hw, sc->mta, mcnt); 2535 } 2536 2537 IGC_WRITE_REG(&sc->hw, IGC_RCTL, rctl); 2538 } 2539 2540 static void 2541 igc_update_link_status(struct igc_softc *sc) 2542 { 2543 struct ifnet *ifp = &sc->sc_ec.ec_if; 2544 struct igc_hw *hw = &sc->hw; 2545 2546 if (hw->mac.get_link_status == true) 2547 igc_check_for_link(hw); 2548 2549 if (IGC_READ_REG(&sc->hw, IGC_STATUS) & IGC_STATUS_LU) { 2550 if (sc->link_active == 0) { 2551 igc_get_speed_and_duplex(hw, &sc->link_speed, 2552 &sc->link_duplex); 2553 sc->link_active = 1; 2554 ifp->if_baudrate = IF_Mbps(sc->link_speed); 2555 if_link_state_change(ifp, LINK_STATE_UP); 2556 } 2557 } else { 2558 if (sc->link_active == 1) { 2559 ifp->if_baudrate = sc->link_speed = 0; 2560 sc->link_duplex = 0; 2561 sc->link_active = 0; 2562 if_link_state_change(ifp, LINK_STATE_DOWN); 2563 } 2564 } 2565 } 2566 2567 /********************************************************************* 2568 * 2569 * Get a buffer from system mbuf buffer pool. 2570 * 2571 **********************************************************************/ 2572 static int 2573 igc_get_buf(struct rx_ring *rxr, int id, bool strict) 2574 { 2575 struct igc_softc *sc = rxr->sc; 2576 struct igc_queue *q = rxr->rxr_igcq; 2577 struct igc_rx_buf *rxbuf = &rxr->rx_buffers[id]; 2578 bus_dmamap_t map = rxbuf->map; 2579 struct mbuf *m; 2580 int error; 2581 2582 if (__predict_false(rxbuf->buf)) { 2583 if (strict) { 2584 DPRINTF(RX, "slot %d already has an mbuf\n", id); 2585 return EINVAL; 2586 } 2587 return 0; 2588 } 2589 2590 MGETHDR(m, M_DONTWAIT, MT_DATA); 2591 if (__predict_false(m == NULL)) { 2592 enobuf: 2593 IGC_QUEUE_EVENT(q, rx_no_mbuf, 1); 2594 return ENOBUFS; 2595 } 2596 MCLAIM(m, &sc->sc_ec.ec_rx_mowner); 2597 2598 MCLGET(m, M_DONTWAIT); 2599 if (__predict_false(!(m->m_flags & M_EXT))) { 2600 m_freem(m); 2601 goto enobuf; 2602 } 2603 2604 m->m_len = m->m_pkthdr.len = sc->rx_mbuf_sz; 2605 2606 error = bus_dmamap_load_mbuf(rxr->rxdma.dma_tag, map, m, 2607 BUS_DMA_READ | BUS_DMA_NOWAIT); 2608 if (error) { 2609 m_freem(m); 2610 return error; 2611 } 2612 2613 bus_dmamap_sync(rxr->rxdma.dma_tag, map, 0, 2614 map->dm_mapsize, BUS_DMASYNC_PREREAD); 2615 rxbuf->buf = m; 2616 2617 union igc_adv_rx_desc *rxdesc = &rxr->rx_base[id]; 2618 igc_rxdesc_sync(rxr, id, BUS_DMASYNC_POSTWRITE | BUS_DMASYNC_POSTREAD); 2619 rxdesc->read.pkt_addr = htole64(map->dm_segs[0].ds_addr); 2620 igc_rxdesc_sync(rxr, id, BUS_DMASYNC_PREWRITE | BUS_DMASYNC_PREREAD); 2621 2622 return 0; 2623 } 2624 2625 static void 2626 igc_configure_queues(struct igc_softc *sc) 2627 { 2628 struct igc_hw *hw = &sc->hw; 2629 uint32_t ivar; 2630 2631 /* First turn on RSS capability */ 2632 IGC_WRITE_REG(hw, IGC_GPIE, IGC_GPIE_MSIX_MODE | IGC_GPIE_EIAME | 2633 IGC_GPIE_PBA | IGC_GPIE_NSICR); 2634 2635 /* Set the starting interrupt rate */ 2636 uint32_t newitr = (4000000 / MAX_INTS_PER_SEC) & 0x7FFC; 2637 newitr |= IGC_EITR_CNT_IGNR; 2638 2639 /* Turn on MSI-X */ 2640 uint32_t newmask = 0; 2641 for (int iq = 0; iq < sc->sc_nqueues; iq++) { 2642 struct igc_queue *q = &sc->queues[iq]; 2643 2644 /* RX entries */ 2645 igc_set_queues(sc, iq, q->msix, 0); 2646 /* TX entries */ 2647 igc_set_queues(sc, iq, q->msix, 1); 2648 newmask |= q->eims; 2649 IGC_WRITE_REG(hw, IGC_EITR(q->msix), newitr); 2650 } 2651 sc->msix_queuesmask = newmask; 2652 2653 #if 1 2654 ivar = IGC_READ_REG_ARRAY(hw, IGC_IVAR0, 0); 2655 DPRINTF(CFG, "ivar(0)=0x%x\n", ivar); 2656 ivar = IGC_READ_REG_ARRAY(hw, IGC_IVAR0, 1); 2657 DPRINTF(CFG, "ivar(1)=0x%x\n", ivar); 2658 #endif 2659 2660 /* And for the link interrupt */ 2661 ivar = (sc->linkvec | IGC_IVAR_VALID) << 8; 2662 sc->msix_linkmask = 1 << sc->linkvec; 2663 IGC_WRITE_REG(hw, IGC_IVAR_MISC, ivar); 2664 } 2665 2666 static void 2667 igc_set_queues(struct igc_softc *sc, uint32_t entry, uint32_t vector, int type) 2668 { 2669 struct igc_hw *hw = &sc->hw; 2670 const uint32_t index = entry >> 1; 2671 uint32_t ivar = IGC_READ_REG_ARRAY(hw, IGC_IVAR0, index); 2672 2673 if (type) { 2674 if (entry & 1) { 2675 ivar &= 0x00FFFFFF; 2676 ivar |= (vector | IGC_IVAR_VALID) << 24; 2677 } else { 2678 ivar &= 0xFFFF00FF; 2679 ivar |= (vector | IGC_IVAR_VALID) << 8; 2680 } 2681 } else { 2682 if (entry & 1) { 2683 ivar &= 0xFF00FFFF; 2684 ivar |= (vector | IGC_IVAR_VALID) << 16; 2685 } else { 2686 ivar &= 0xFFFFFF00; 2687 ivar |= vector | IGC_IVAR_VALID; 2688 } 2689 } 2690 IGC_WRITE_REG_ARRAY(hw, IGC_IVAR0, index, ivar); 2691 } 2692 2693 static void 2694 igc_enable_queue(struct igc_softc *sc, uint32_t eims) 2695 { 2696 IGC_WRITE_REG(&sc->hw, IGC_EIMS, eims); 2697 } 2698 2699 static void 2700 igc_enable_intr(struct igc_softc *sc) 2701 { 2702 struct igc_hw *hw = &sc->hw; 2703 2704 if (sc->sc_intr_type == PCI_INTR_TYPE_MSIX) { 2705 const uint32_t mask = sc->msix_queuesmask | sc->msix_linkmask; 2706 2707 IGC_WRITE_REG(hw, IGC_EIAC, mask); 2708 IGC_WRITE_REG(hw, IGC_EIAM, mask); 2709 IGC_WRITE_REG(hw, IGC_EIMS, mask); 2710 IGC_WRITE_REG(hw, IGC_IMS, IGC_IMS_LSC); 2711 } else { 2712 IGC_WRITE_REG(hw, IGC_IMS, IMS_ENABLE_MASK); 2713 } 2714 IGC_WRITE_FLUSH(hw); 2715 } 2716 2717 static void 2718 igc_disable_intr(struct igc_softc *sc) 2719 { 2720 struct igc_hw *hw = &sc->hw; 2721 2722 if (sc->sc_intr_type == PCI_INTR_TYPE_MSIX) { 2723 IGC_WRITE_REG(hw, IGC_EIMC, 0xffffffff); 2724 IGC_WRITE_REG(hw, IGC_EIAC, 0); 2725 } 2726 IGC_WRITE_REG(hw, IGC_IMC, 0xffffffff); 2727 IGC_WRITE_FLUSH(hw); 2728 } 2729 2730 static int 2731 igc_intr_link(void *arg) 2732 { 2733 struct igc_softc *sc = (struct igc_softc *)arg; 2734 const uint32_t reg_icr = IGC_READ_REG(&sc->hw, IGC_ICR); 2735 2736 IGC_GLOBAL_EVENT(sc, link, 1); 2737 2738 if (reg_icr & IGC_ICR_LSC) { 2739 mutex_enter(&sc->sc_core_lock); 2740 sc->hw.mac.get_link_status = true; 2741 igc_update_link_status(sc); 2742 mutex_exit(&sc->sc_core_lock); 2743 } 2744 2745 IGC_WRITE_REG(&sc->hw, IGC_IMS, IGC_IMS_LSC); 2746 IGC_WRITE_REG(&sc->hw, IGC_EIMS, sc->msix_linkmask); 2747 2748 return 1; 2749 } 2750 2751 static int 2752 igc_intr_queue(void *arg) 2753 { 2754 struct igc_queue *iq = arg; 2755 struct igc_softc *sc = iq->sc; 2756 struct ifnet *ifp = &sc->sc_ec.ec_if; 2757 struct rx_ring *rxr = iq->rxr; 2758 struct tx_ring *txr = iq->txr; 2759 const u_int txlimit = sc->sc_tx_intr_process_limit, 2760 rxlimit = sc->sc_rx_intr_process_limit; 2761 bool txmore, rxmore; 2762 2763 IGC_QUEUE_EVENT(iq, irqs, 1); 2764 2765 if (__predict_false(!ISSET(ifp->if_flags, IFF_RUNNING))) 2766 return 0; 2767 2768 mutex_enter(&txr->txr_lock); 2769 txmore = igc_txeof(txr, txlimit); 2770 mutex_exit(&txr->txr_lock); 2771 mutex_enter(&rxr->rxr_lock); 2772 rxmore = igc_rxeof(rxr, rxlimit); 2773 mutex_exit(&rxr->rxr_lock); 2774 2775 if (txmore || rxmore) { 2776 IGC_QUEUE_EVENT(iq, req, 1); 2777 igc_sched_handle_queue(sc, iq); 2778 } else { 2779 igc_enable_queue(sc, iq->eims); 2780 } 2781 2782 return 1; 2783 } 2784 2785 static int 2786 igc_intr(void *arg) 2787 { 2788 struct igc_softc *sc = arg; 2789 struct ifnet *ifp = &sc->sc_ec.ec_if; 2790 struct igc_queue *iq = &sc->queues[0]; 2791 struct rx_ring *rxr = iq->rxr; 2792 struct tx_ring *txr = iq->txr; 2793 const u_int txlimit = sc->sc_tx_intr_process_limit, 2794 rxlimit = sc->sc_rx_intr_process_limit; 2795 bool txmore, rxmore; 2796 2797 if (__predict_false(!ISSET(ifp->if_flags, IFF_RUNNING))) 2798 return 0; 2799 2800 const uint32_t reg_icr = IGC_READ_REG(&sc->hw, IGC_ICR); 2801 DPRINTF(MISC, "reg_icr=0x%x\n", reg_icr); 2802 2803 /* Definitely not our interrupt. */ 2804 if (reg_icr == 0x0) { 2805 DPRINTF(MISC, "not for me\n"); 2806 return 0; 2807 } 2808 2809 IGC_QUEUE_EVENT(iq, irqs, 1); 2810 2811 /* Hot eject? */ 2812 if (__predict_false(reg_icr == 0xffffffff)) { 2813 DPRINTF(MISC, "hot eject\n"); 2814 return 0; 2815 } 2816 2817 if (__predict_false(!(reg_icr & IGC_ICR_INT_ASSERTED))) { 2818 DPRINTF(MISC, "not set IGC_ICR_INT_ASSERTED"); 2819 return 0; 2820 } 2821 2822 /* 2823 * Only MSI-X interrupts have one-shot behavior by taking advantage 2824 * of the EIAC register. Thus, explicitly disable interrupts. This 2825 * also works around the MSI message reordering errata on certain 2826 * systems. 2827 */ 2828 igc_disable_intr(sc); 2829 2830 mutex_enter(&txr->txr_lock); 2831 txmore = igc_txeof(txr, txlimit); 2832 mutex_exit(&txr->txr_lock); 2833 mutex_enter(&rxr->rxr_lock); 2834 rxmore = igc_rxeof(rxr, rxlimit); 2835 mutex_exit(&rxr->rxr_lock); 2836 2837 /* Link status change */ 2838 // XXXX FreeBSD checks IGC_ICR_RXSEQ 2839 if (__predict_false(reg_icr & IGC_ICR_LSC)) { 2840 IGC_GLOBAL_EVENT(sc, link, 1); 2841 mutex_enter(&sc->sc_core_lock); 2842 sc->hw.mac.get_link_status = true; 2843 igc_update_link_status(sc); 2844 mutex_exit(&sc->sc_core_lock); 2845 } 2846 2847 if (txmore || rxmore) { 2848 IGC_QUEUE_EVENT(iq, req, 1); 2849 igc_sched_handle_queue(sc, iq); 2850 } else { 2851 igc_enable_intr(sc); 2852 } 2853 2854 return 1; 2855 } 2856 2857 static void 2858 igc_handle_queue(void *arg) 2859 { 2860 struct igc_queue *iq = arg; 2861 struct igc_softc *sc = iq->sc; 2862 struct tx_ring *txr = iq->txr; 2863 struct rx_ring *rxr = iq->rxr; 2864 const u_int txlimit = sc->sc_tx_process_limit, 2865 rxlimit = sc->sc_rx_process_limit; 2866 bool txmore, rxmore; 2867 2868 IGC_QUEUE_EVENT(iq, handleq, 1); 2869 2870 mutex_enter(&txr->txr_lock); 2871 txmore = igc_txeof(txr, txlimit); 2872 /* for ALTQ, dequeue from if_snd */ 2873 if (txr->me == 0) { 2874 struct ifnet *ifp = &sc->sc_ec.ec_if; 2875 2876 igc_tx_common_locked(ifp, txr, IGC_TX_START); 2877 } 2878 mutex_exit(&txr->txr_lock); 2879 2880 mutex_enter(&rxr->rxr_lock); 2881 rxmore = igc_rxeof(rxr, rxlimit); 2882 mutex_exit(&rxr->rxr_lock); 2883 2884 if (txmore || rxmore) { 2885 igc_sched_handle_queue(sc, iq); 2886 } else { 2887 if (sc->sc_intr_type == PCI_INTR_TYPE_MSIX) 2888 igc_enable_queue(sc, iq->eims); 2889 else 2890 igc_enable_intr(sc); 2891 } 2892 } 2893 2894 static void 2895 igc_handle_queue_work(struct work *wk, void *context) 2896 { 2897 struct igc_queue *iq = 2898 container_of(wk, struct igc_queue, igcq_wq_cookie); 2899 2900 igc_handle_queue(iq); 2901 } 2902 2903 static void 2904 igc_sched_handle_queue(struct igc_softc *sc, struct igc_queue *iq) 2905 { 2906 2907 if (iq->igcq_workqueue) { 2908 /* XXXRO notyet */ 2909 workqueue_enqueue(sc->sc_queue_wq, &iq->igcq_wq_cookie, 2910 curcpu()); 2911 } else { 2912 softint_schedule(iq->igcq_si); 2913 } 2914 } 2915 2916 static void 2917 igc_barrier_handle_queue(struct igc_softc *sc) 2918 { 2919 2920 if (sc->sc_txrx_workqueue) { 2921 for (int iq = 0; iq < sc->sc_nqueues; iq++) { 2922 struct igc_queue *q = &sc->queues[iq]; 2923 2924 workqueue_wait(sc->sc_queue_wq, &q->igcq_wq_cookie); 2925 } 2926 } else { 2927 xc_barrier(0); 2928 } 2929 } 2930 2931 /********************************************************************* 2932 * 2933 * Allocate memory for tx_buffer structures. The tx_buffer stores all 2934 * the information needed to transmit a packet on the wire. 2935 * 2936 **********************************************************************/ 2937 static int 2938 igc_allocate_transmit_buffers(struct tx_ring *txr) 2939 { 2940 struct igc_softc *sc = txr->sc; 2941 int error; 2942 2943 txr->tx_buffers = 2944 kmem_zalloc(sc->num_tx_desc * sizeof(struct igc_tx_buf), KM_SLEEP); 2945 txr->txtag = txr->txdma.dma_tag; 2946 2947 /* Create the descriptor buffer dma maps. */ 2948 for (int id = 0; id < sc->num_tx_desc; id++) { 2949 struct igc_tx_buf *txbuf = &txr->tx_buffers[id]; 2950 2951 error = bus_dmamap_create(txr->txdma.dma_tag, 2952 round_page(IGC_TSO_SIZE + sizeof(struct ether_vlan_header)), 2953 IGC_MAX_SCATTER, PAGE_SIZE, 0, BUS_DMA_NOWAIT, &txbuf->map); 2954 if (error != 0) { 2955 aprint_error_dev(sc->sc_dev, 2956 "unable to create TX DMA map\n"); 2957 goto fail; 2958 } 2959 2960 txbuf->eop_index = -1; 2961 } 2962 2963 return 0; 2964 fail: 2965 return error; 2966 } 2967 2968 2969 /********************************************************************* 2970 * 2971 * Allocate and initialize transmit structures. 2972 * 2973 **********************************************************************/ 2974 static int 2975 igc_setup_transmit_structures(struct igc_softc *sc) 2976 { 2977 2978 for (int iq = 0; iq < sc->sc_nqueues; iq++) { 2979 struct tx_ring *txr = &sc->tx_rings[iq]; 2980 2981 if (igc_setup_transmit_ring(txr)) 2982 goto fail; 2983 } 2984 2985 return 0; 2986 fail: 2987 igc_free_transmit_structures(sc); 2988 return ENOBUFS; 2989 } 2990 2991 /********************************************************************* 2992 * 2993 * Initialize a transmit ring. 2994 * 2995 **********************************************************************/ 2996 static int 2997 igc_setup_transmit_ring(struct tx_ring *txr) 2998 { 2999 struct igc_softc *sc = txr->sc; 3000 3001 /* Now allocate transmit buffers for the ring. */ 3002 if (igc_allocate_transmit_buffers(txr)) 3003 return ENOMEM; 3004 3005 /* Clear the old ring contents */ 3006 memset(txr->tx_base, 0, 3007 sizeof(union igc_adv_tx_desc) * sc->num_tx_desc); 3008 3009 /* Reset indices. */ 3010 txr->next_avail_desc = 0; 3011 txr->next_to_clean = 0; 3012 3013 bus_dmamap_sync(txr->txdma.dma_tag, txr->txdma.dma_map, 0, 3014 txr->txdma.dma_map->dm_mapsize, 3015 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); 3016 3017 txr->txr_interq = pcq_create(sc->num_tx_desc, KM_SLEEP); 3018 3019 mutex_init(&txr->txr_lock, MUTEX_DEFAULT, IPL_NET); 3020 3021 return 0; 3022 } 3023 3024 /********************************************************************* 3025 * 3026 * Enable transmit unit. 3027 * 3028 **********************************************************************/ 3029 static void 3030 igc_initialize_transmit_unit(struct igc_softc *sc) 3031 { 3032 struct ifnet *ifp = &sc->sc_ec.ec_if; 3033 struct igc_hw *hw = &sc->hw; 3034 3035 /* Setup the Base and Length of the TX descriptor ring. */ 3036 for (int iq = 0; iq < sc->sc_nqueues; iq++) { 3037 struct tx_ring *txr = &sc->tx_rings[iq]; 3038 const uint64_t bus_addr = 3039 txr->txdma.dma_map->dm_segs[0].ds_addr; 3040 3041 /* Base and len of TX ring */ 3042 IGC_WRITE_REG(hw, IGC_TDLEN(iq), 3043 sc->num_tx_desc * sizeof(union igc_adv_tx_desc)); 3044 IGC_WRITE_REG(hw, IGC_TDBAH(iq), (uint32_t)(bus_addr >> 32)); 3045 IGC_WRITE_REG(hw, IGC_TDBAL(iq), (uint32_t)bus_addr); 3046 3047 /* Init the HEAD/TAIL indices */ 3048 IGC_WRITE_REG(hw, IGC_TDT(iq), 0 /* XXX txr->next_avail_desc */); 3049 IGC_WRITE_REG(hw, IGC_TDH(iq), 0); 3050 3051 txr->watchdog_timer = 0; 3052 3053 uint32_t txdctl = 0; /* Clear txdctl */ 3054 txdctl |= 0x1f; /* PTHRESH */ 3055 txdctl |= 1 << 8; /* HTHRESH */ 3056 txdctl |= 1 << 16; /* WTHRESH */ 3057 txdctl |= 1 << 22; /* Reserved bit 22 must always be 1 */ 3058 txdctl |= IGC_TXDCTL_GRAN; 3059 txdctl |= 1 << 25; /* LWTHRESH */ 3060 3061 IGC_WRITE_REG(hw, IGC_TXDCTL(iq), txdctl); 3062 } 3063 ifp->if_timer = 0; 3064 3065 /* Program the Transmit Control Register */ 3066 uint32_t tctl = IGC_READ_REG(&sc->hw, IGC_TCTL); 3067 tctl &= ~IGC_TCTL_CT; 3068 tctl |= (IGC_TCTL_PSP | IGC_TCTL_RTLC | IGC_TCTL_EN | 3069 (IGC_COLLISION_THRESHOLD << IGC_CT_SHIFT)); 3070 3071 /* This write will effectively turn on the transmit unit. */ 3072 IGC_WRITE_REG(&sc->hw, IGC_TCTL, tctl); 3073 } 3074 3075 /********************************************************************* 3076 * 3077 * Free all transmit rings. 3078 * 3079 **********************************************************************/ 3080 static void 3081 igc_free_transmit_structures(struct igc_softc *sc) 3082 { 3083 3084 for (int iq = 0; iq < sc->sc_nqueues; iq++) { 3085 struct tx_ring *txr = &sc->tx_rings[iq]; 3086 3087 igc_free_transmit_buffers(txr); 3088 } 3089 } 3090 3091 /********************************************************************* 3092 * 3093 * Free transmit ring related data structures. 3094 * 3095 **********************************************************************/ 3096 static void 3097 igc_free_transmit_buffers(struct tx_ring *txr) 3098 { 3099 struct igc_softc *sc = txr->sc; 3100 3101 if (txr->tx_buffers == NULL) 3102 return; 3103 3104 igc_withdraw_transmit_packets(txr, true); 3105 3106 kmem_free(txr->tx_buffers, 3107 sc->num_tx_desc * sizeof(struct igc_tx_buf)); 3108 txr->tx_buffers = NULL; 3109 txr->txtag = NULL; 3110 3111 pcq_destroy(txr->txr_interq); 3112 mutex_destroy(&txr->txr_lock); 3113 } 3114 3115 /********************************************************************* 3116 * 3117 * Withdraw transmit packets. 3118 * 3119 **********************************************************************/ 3120 static void 3121 igc_withdraw_transmit_packets(struct tx_ring *txr, bool destroy) 3122 { 3123 struct igc_softc *sc = txr->sc; 3124 struct igc_queue *q = txr->txr_igcq; 3125 3126 mutex_enter(&txr->txr_lock); 3127 3128 for (int id = 0; id < sc->num_tx_desc; id++) { 3129 union igc_adv_tx_desc *txdesc = &txr->tx_base[id]; 3130 3131 igc_txdesc_sync(txr, id, 3132 BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE); 3133 txdesc->read.buffer_addr = 0; 3134 txdesc->read.cmd_type_len = 0; 3135 txdesc->read.olinfo_status = 0; 3136 igc_txdesc_sync(txr, id, 3137 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); 3138 3139 struct igc_tx_buf *txbuf = &txr->tx_buffers[id]; 3140 bus_dmamap_t map = txbuf->map; 3141 3142 if (map != NULL && map->dm_nsegs > 0) { 3143 bus_dmamap_sync(txr->txdma.dma_tag, map, 3144 0, map->dm_mapsize, BUS_DMASYNC_POSTWRITE); 3145 bus_dmamap_unload(txr->txdma.dma_tag, map); 3146 } 3147 m_freem(txbuf->m_head); 3148 txbuf->m_head = NULL; 3149 if (map != NULL && destroy) { 3150 bus_dmamap_destroy(txr->txdma.dma_tag, map); 3151 txbuf->map = NULL; 3152 } 3153 txbuf->eop_index = -1; 3154 3155 txr->next_avail_desc = 0; 3156 txr->next_to_clean = 0; 3157 } 3158 3159 struct mbuf *m; 3160 while ((m = pcq_get(txr->txr_interq)) != NULL) { 3161 IGC_QUEUE_EVENT(q, tx_pcq_drop, 1); 3162 m_freem(m); 3163 } 3164 3165 mutex_exit(&txr->txr_lock); 3166 } 3167 3168 3169 /********************************************************************* 3170 * 3171 * Advanced Context Descriptor setup for VLAN, CSUM or TSO 3172 * 3173 **********************************************************************/ 3174 3175 static bool 3176 igc_tx_ctx_setup(struct tx_ring *txr, struct mbuf *mp, int prod, 3177 uint32_t *cmd_type_len, uint32_t *olinfo_status) 3178 { 3179 struct ether_vlan_header *evl; 3180 struct tcphdr *th = NULL /* XXXGCC */; 3181 uint32_t vlan_macip_lens = 0; 3182 uint32_t type_tucmd_mlhl = 0; 3183 uint32_t mss_l4len_idx = 0; 3184 uint32_t ehlen, iphlen; 3185 uint16_t ehtype; 3186 3187 const int csum_flags = mp->m_pkthdr.csum_flags; 3188 const bool v4 = (csum_flags & 3189 (M_CSUM_IPv4 | M_CSUM_TCPv4 | M_CSUM_UDPv4 | M_CSUM_TSOv4)) != 0; 3190 const bool v6 = (csum_flags & 3191 (M_CSUM_UDPv6 | M_CSUM_TCPv6 | M_CSUM_TSOv6)) != 0; 3192 const bool tso = (csum_flags & (M_CSUM_TSOv4 | M_CSUM_TSOv6)) != 0; 3193 const bool tcp = tso || 3194 (csum_flags & (M_CSUM_TCPv4 | M_CSUM_TCPv6)) != 0; 3195 const bool udp = (csum_flags & (M_CSUM_UDPv4 | M_CSUM_UDPv6)) != 0; 3196 3197 /* Indicate the whole packet as payload when not doing TSO */ 3198 if (!tso) { 3199 *olinfo_status |= mp->m_pkthdr.len << IGC_ADVTXD_PAYLEN_SHIFT; 3200 } else { 3201 /* Set L4 payload length later... */ 3202 } 3203 3204 #if NVLAN > 0 3205 /* 3206 * In advanced descriptors the vlan tag must 3207 * be placed into the context descriptor. Hence 3208 * we need to make one even if not doing offloads. 3209 */ 3210 if (vlan_has_tag(mp)) { 3211 vlan_macip_lens |= (uint32_t)vlan_get_tag(mp) 3212 << IGC_ADVTXD_VLAN_SHIFT; 3213 } else 3214 #endif 3215 if (!v4 && !v6) 3216 return false; 3217 3218 KASSERT(mp->m_len >= sizeof(struct ether_header)); 3219 evl = mtod(mp, struct ether_vlan_header *); 3220 if (evl->evl_encap_proto == htons(ETHERTYPE_VLAN)) { 3221 KASSERT(mp->m_len >= sizeof(struct ether_vlan_header)); 3222 ehlen = ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN; 3223 ehtype = evl->evl_proto; 3224 } else { 3225 ehlen = ETHER_HDR_LEN; 3226 ehtype = evl->evl_encap_proto; 3227 } 3228 3229 switch (ntohs(ehtype)) { 3230 case ETHERTYPE_IP: 3231 iphlen = M_CSUM_DATA_IPv4_IPHL(mp->m_pkthdr.csum_data); 3232 type_tucmd_mlhl |= IGC_ADVTXD_TUCMD_IPV4; 3233 3234 if ((csum_flags & (M_CSUM_IPv4 | M_CSUM_TSOv4)) != 0) 3235 *olinfo_status |= IGC_TXD_POPTS_IXSM << 8; 3236 3237 if (!tso) 3238 break; 3239 3240 struct ip *ip; 3241 KASSERT(mp->m_len >= ehlen + sizeof(*ip)); 3242 ip = (void *)(mtod(mp, char *) + ehlen); 3243 ip->ip_len = 0; 3244 3245 KASSERT(mp->m_len >= ehlen + iphlen + sizeof(*th)); 3246 th = (void *)((char *)ip + iphlen); 3247 th->th_sum = in_cksum_phdr(ip->ip_src.s_addr, 3248 ip->ip_dst.s_addr, htons(IPPROTO_TCP)); 3249 break; 3250 case ETHERTYPE_IPV6: 3251 iphlen = M_CSUM_DATA_IPv6_IPHL(mp->m_pkthdr.csum_data); 3252 type_tucmd_mlhl |= IGC_ADVTXD_TUCMD_IPV6; 3253 3254 if (!tso) 3255 break; 3256 3257 struct ip6_hdr *ip6; 3258 KASSERT(mp->m_len >= ehlen + sizeof(*ip6)); 3259 ip6 = (void *)(mtod(mp, char *) + ehlen); 3260 ip6->ip6_plen = 0; 3261 3262 KASSERT(mp->m_len >= ehlen + iphlen + sizeof(*th)); 3263 th = (void *)((char *)ip6 + iphlen); 3264 th->th_sum = in6_cksum_phdr(&ip6->ip6_src, &ip6->ip6_dst, 0, 3265 htonl(IPPROTO_TCP)); 3266 break; 3267 default: 3268 /* 3269 * Unknown L3 protocol. Clear L3 header length and proceed for 3270 * LAN as done by Linux driver. 3271 */ 3272 KASSERT(!v4 && !v6); 3273 iphlen = 0; 3274 break; 3275 } 3276 3277 if (tcp) { 3278 type_tucmd_mlhl |= IGC_ADVTXD_TUCMD_L4T_TCP; 3279 *olinfo_status |= IGC_TXD_POPTS_TXSM << 8; 3280 } else if (udp) { 3281 type_tucmd_mlhl |= IGC_ADVTXD_TUCMD_L4T_UDP; 3282 *olinfo_status |= IGC_TXD_POPTS_TXSM << 8; 3283 } 3284 3285 if (tso) { 3286 const uint32_t tcphlen = th->th_off << 2; 3287 const uint32_t paylen = 3288 mp->m_pkthdr.len - ehlen - iphlen - tcphlen; 3289 3290 mss_l4len_idx |= mp->m_pkthdr.segsz << IGC_ADVTXD_MSS_SHIFT; 3291 mss_l4len_idx |= tcphlen << IGC_ADVTXD_L4LEN_SHIFT; 3292 3293 *cmd_type_len |= IGC_ADVTXD_DCMD_TSE; 3294 3295 *olinfo_status |= paylen << IGC_ADVTXD_PAYLEN_SHIFT; 3296 } 3297 3298 vlan_macip_lens |= iphlen; 3299 vlan_macip_lens |= ehlen << IGC_ADVTXD_MACLEN_SHIFT; 3300 3301 type_tucmd_mlhl |= IGC_ADVTXD_DCMD_DEXT | IGC_ADVTXD_DTYP_CTXT; 3302 3303 /* Now ready a context descriptor */ 3304 struct igc_adv_tx_context_desc *txdesc = 3305 (struct igc_adv_tx_context_desc *)&txr->tx_base[prod]; 3306 3307 igc_txdesc_sync(txr, prod, 3308 BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE); 3309 3310 /* Now copy bits into descriptor */ 3311 htolem32(&txdesc->vlan_macip_lens, vlan_macip_lens); 3312 htolem32(&txdesc->type_tucmd_mlhl, type_tucmd_mlhl); 3313 htolem32(&txdesc->seqnum_seed, 0); 3314 htolem32(&txdesc->mss_l4len_idx, mss_l4len_idx); 3315 3316 igc_txdesc_sync(txr, prod, 3317 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); 3318 3319 return 1; 3320 } 3321 3322 /********************************************************************* 3323 * 3324 * Allocate memory for rx_buffer structures. Since we use one 3325 * rx_buffer per received packet, the maximum number of rx_buffer's 3326 * that we'll need is equal to the number of receive descriptors 3327 * that we've allocated. 3328 * 3329 **********************************************************************/ 3330 static int 3331 igc_allocate_receive_buffers(struct rx_ring *rxr) 3332 { 3333 struct igc_softc *sc = rxr->sc; 3334 int error; 3335 3336 rxr->rx_buffers = 3337 kmem_zalloc(sc->num_rx_desc * sizeof(struct igc_rx_buf), KM_SLEEP); 3338 3339 for (int id = 0; id < sc->num_rx_desc; id++) { 3340 struct igc_rx_buf *rxbuf = &rxr->rx_buffers[id]; 3341 3342 error = bus_dmamap_create(rxr->rxdma.dma_tag, MCLBYTES, 1, 3343 MCLBYTES, 0, BUS_DMA_WAITOK, &rxbuf->map); 3344 if (error) { 3345 aprint_error_dev(sc->sc_dev, 3346 "unable to create RX DMA map\n"); 3347 goto fail; 3348 } 3349 } 3350 bus_dmamap_sync(rxr->rxdma.dma_tag, rxr->rxdma.dma_map, 0, 3351 rxr->rxdma.dma_map->dm_mapsize, 3352 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); 3353 3354 return 0; 3355 fail: 3356 return error; 3357 } 3358 3359 /********************************************************************* 3360 * 3361 * Allocate and initialize receive structures. 3362 * 3363 **********************************************************************/ 3364 static int 3365 igc_setup_receive_structures(struct igc_softc *sc) 3366 { 3367 3368 for (int iq = 0; iq < sc->sc_nqueues; iq++) { 3369 struct rx_ring *rxr = &sc->rx_rings[iq]; 3370 3371 if (igc_setup_receive_ring(rxr)) 3372 goto fail; 3373 } 3374 3375 return 0; 3376 fail: 3377 igc_free_receive_structures(sc); 3378 return ENOBUFS; 3379 } 3380 3381 /********************************************************************* 3382 * 3383 * Initialize a receive ring and its buffers. 3384 * 3385 **********************************************************************/ 3386 static int 3387 igc_setup_receive_ring(struct rx_ring *rxr) 3388 { 3389 struct igc_softc *sc = rxr->sc; 3390 const int rsize = roundup2( 3391 sc->num_rx_desc * sizeof(union igc_adv_rx_desc), IGC_DBA_ALIGN); 3392 3393 /* Clear the ring contents. */ 3394 memset(rxr->rx_base, 0, rsize); 3395 3396 if (igc_allocate_receive_buffers(rxr)) 3397 return ENOMEM; 3398 3399 /* Setup our descriptor indices. */ 3400 rxr->next_to_check = 0; 3401 rxr->last_desc_filled = 0; 3402 3403 mutex_init(&rxr->rxr_lock, MUTEX_DEFAULT, IPL_NET); 3404 3405 return 0; 3406 } 3407 3408 /********************************************************************* 3409 * 3410 * Enable receive unit. 3411 * 3412 **********************************************************************/ 3413 static void 3414 igc_initialize_receive_unit(struct igc_softc *sc) 3415 { 3416 struct ifnet *ifp = &sc->sc_ec.ec_if; 3417 struct igc_hw *hw = &sc->hw; 3418 uint32_t rctl, rxcsum, srrctl; 3419 3420 DPRINTF(RX, "called\n"); 3421 3422 /* 3423 * Make sure receives are disabled while setting 3424 * up the descriptor ring. 3425 */ 3426 rctl = IGC_READ_REG(hw, IGC_RCTL); 3427 IGC_WRITE_REG(hw, IGC_RCTL, rctl & ~IGC_RCTL_EN); 3428 3429 /* Setup the Receive Control Register */ 3430 rctl &= ~(3 << IGC_RCTL_MO_SHIFT); 3431 rctl |= IGC_RCTL_EN | IGC_RCTL_BAM | IGC_RCTL_LBM_NO | 3432 IGC_RCTL_RDMTS_HALF | (hw->mac.mc_filter_type << IGC_RCTL_MO_SHIFT); 3433 3434 #if 1 3435 /* Do not store bad packets */ 3436 rctl &= ~IGC_RCTL_SBP; 3437 #else 3438 /* for debug */ 3439 rctl |= IGC_RCTL_SBP; 3440 #endif 3441 3442 /* Enable Long Packet receive */ 3443 if (sc->hw.mac.max_frame_size > ETHER_MAX_LEN) 3444 rctl |= IGC_RCTL_LPE; 3445 else 3446 rctl &= ~IGC_RCTL_LPE; 3447 3448 /* Strip the CRC */ 3449 rctl |= IGC_RCTL_SECRC; 3450 3451 /* 3452 * Set the interrupt throttling rate. Value is calculated 3453 * as DEFAULT_ITR = 1/(MAX_INTS_PER_SEC * 256ns) 3454 * 3455 * XXX Sync with Linux, especially for jumbo MTU or TSO. 3456 * XXX Shouldn't be here? 3457 */ 3458 IGC_WRITE_REG(hw, IGC_ITR, DEFAULT_ITR); 3459 3460 rxcsum = IGC_READ_REG(hw, IGC_RXCSUM); 3461 rxcsum &= ~(IGC_RXCSUM_IPOFL | IGC_RXCSUM_TUOFL | IGC_RXCSUM_PCSD); 3462 if (ifp->if_capenable & IFCAP_CSUM_IPv4_Rx) 3463 rxcsum |= IGC_RXCSUM_IPOFL; 3464 if (ifp->if_capenable & (IFCAP_CSUM_TCPv4_Rx | IFCAP_CSUM_UDPv4_Rx | 3465 IFCAP_CSUM_TCPv6_Rx | IFCAP_CSUM_UDPv6_Rx)) 3466 rxcsum |= IGC_RXCSUM_TUOFL; 3467 if (sc->sc_nqueues > 1) 3468 rxcsum |= IGC_RXCSUM_PCSD; 3469 IGC_WRITE_REG(hw, IGC_RXCSUM, rxcsum); 3470 3471 if (sc->sc_nqueues > 1) 3472 igc_initialize_rss_mapping(sc); 3473 3474 srrctl = 0; 3475 #if 0 3476 srrctl |= 4096 >> IGC_SRRCTL_BSIZEPKT_SHIFT; 3477 rctl |= IGC_RCTL_SZ_4096 | IGC_RCTL_BSEX; 3478 #else 3479 srrctl |= 2048 >> IGC_SRRCTL_BSIZEPKT_SHIFT; 3480 rctl |= IGC_RCTL_SZ_2048; 3481 #endif 3482 3483 /* 3484 * If TX flow control is disabled and there's > 1 queue defined, 3485 * enable DROP. 3486 * 3487 * This drops frames rather than hanging the RX MAC for all queues. 3488 */ 3489 if (sc->sc_nqueues > 1 && 3490 (sc->fc == igc_fc_none || sc->fc == igc_fc_rx_pause)) 3491 srrctl |= IGC_SRRCTL_DROP_EN; 3492 3493 /* Setup the Base and Length of the RX descriptor rings. */ 3494 for (int iq = 0; iq < sc->sc_nqueues; iq++) { 3495 struct rx_ring *rxr = &sc->rx_rings[iq]; 3496 const uint64_t bus_addr = 3497 rxr->rxdma.dma_map->dm_segs[0].ds_addr; 3498 3499 IGC_WRITE_REG(hw, IGC_RXDCTL(iq), 0); 3500 3501 srrctl |= IGC_SRRCTL_DESCTYPE_ADV_ONEBUF; 3502 3503 IGC_WRITE_REG(hw, IGC_RDLEN(iq), 3504 sc->num_rx_desc * sizeof(union igc_adv_rx_desc)); 3505 IGC_WRITE_REG(hw, IGC_RDBAH(iq), (uint32_t)(bus_addr >> 32)); 3506 IGC_WRITE_REG(hw, IGC_RDBAL(iq), (uint32_t)bus_addr); 3507 IGC_WRITE_REG(hw, IGC_SRRCTL(iq), srrctl); 3508 3509 /* Setup the Head and Tail Descriptor Pointers */ 3510 IGC_WRITE_REG(hw, IGC_RDH(iq), 0); 3511 IGC_WRITE_REG(hw, IGC_RDT(iq), 0 /* XXX rxr->last_desc_filled */); 3512 3513 /* Enable this Queue */ 3514 uint32_t rxdctl = IGC_READ_REG(hw, IGC_RXDCTL(iq)); 3515 rxdctl |= IGC_RXDCTL_QUEUE_ENABLE; 3516 rxdctl &= 0xFFF00000; 3517 rxdctl |= IGC_RX_PTHRESH; 3518 rxdctl |= IGC_RX_HTHRESH << 8; 3519 rxdctl |= IGC_RX_WTHRESH << 16; 3520 IGC_WRITE_REG(hw, IGC_RXDCTL(iq), rxdctl); 3521 } 3522 3523 /* Make sure VLAN Filters are off */ 3524 rctl &= ~IGC_RCTL_VFE; 3525 3526 /* Write out the settings */ 3527 IGC_WRITE_REG(hw, IGC_RCTL, rctl); 3528 } 3529 3530 /********************************************************************* 3531 * 3532 * Free all receive rings. 3533 * 3534 **********************************************************************/ 3535 static void 3536 igc_free_receive_structures(struct igc_softc *sc) 3537 { 3538 3539 for (int iq = 0; iq < sc->sc_nqueues; iq++) { 3540 struct rx_ring *rxr = &sc->rx_rings[iq]; 3541 3542 igc_free_receive_buffers(rxr); 3543 } 3544 } 3545 3546 /********************************************************************* 3547 * 3548 * Free receive ring data structures 3549 * 3550 **********************************************************************/ 3551 static void 3552 igc_free_receive_buffers(struct rx_ring *rxr) 3553 { 3554 struct igc_softc *sc = rxr->sc; 3555 3556 if (rxr->rx_buffers != NULL) { 3557 for (int id = 0; id < sc->num_rx_desc; id++) { 3558 struct igc_rx_buf *rxbuf = &rxr->rx_buffers[id]; 3559 bus_dmamap_t map = rxbuf->map; 3560 3561 if (rxbuf->buf != NULL) { 3562 bus_dmamap_sync(rxr->rxdma.dma_tag, map, 3563 0, map->dm_mapsize, BUS_DMASYNC_POSTREAD); 3564 bus_dmamap_unload(rxr->rxdma.dma_tag, map); 3565 m_freem(rxbuf->buf); 3566 rxbuf->buf = NULL; 3567 } 3568 bus_dmamap_destroy(rxr->rxdma.dma_tag, map); 3569 rxbuf->map = NULL; 3570 } 3571 kmem_free(rxr->rx_buffers, 3572 sc->num_rx_desc * sizeof(struct igc_rx_buf)); 3573 rxr->rx_buffers = NULL; 3574 } 3575 3576 mutex_destroy(&rxr->rxr_lock); 3577 } 3578 3579 /********************************************************************* 3580 * 3581 * Clear status registers in all RX descriptors. 3582 * 3583 **********************************************************************/ 3584 static void 3585 igc_clear_receive_status(struct rx_ring *rxr) 3586 { 3587 struct igc_softc *sc = rxr->sc; 3588 3589 mutex_enter(&rxr->rxr_lock); 3590 3591 for (int id = 0; id < sc->num_rx_desc; id++) { 3592 union igc_adv_rx_desc *rxdesc = &rxr->rx_base[id]; 3593 3594 igc_rxdesc_sync(rxr, id, 3595 BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE); 3596 rxdesc->wb.upper.status_error = 0; 3597 igc_rxdesc_sync(rxr, id, 3598 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); 3599 } 3600 3601 mutex_exit(&rxr->rxr_lock); 3602 } 3603 3604 /* 3605 * Initialise the RSS mapping for NICs that support multiple transmit/ 3606 * receive rings. 3607 */ 3608 static void 3609 igc_initialize_rss_mapping(struct igc_softc *sc) 3610 { 3611 struct igc_hw *hw = &sc->hw; 3612 3613 /* 3614 * The redirection table controls which destination 3615 * queue each bucket redirects traffic to. 3616 * Each DWORD represents four queues, with the LSB 3617 * being the first queue in the DWORD. 3618 * 3619 * This just allocates buckets to queues using round-robin 3620 * allocation. 3621 * 3622 * NOTE: It Just Happens to line up with the default 3623 * RSS allocation method. 3624 */ 3625 3626 /* Warning FM follows */ 3627 uint32_t reta = 0; 3628 for (int i = 0; i < 128; i++) { 3629 const int shift = 0; /* XXXRO */ 3630 int queue_id = i % sc->sc_nqueues; 3631 /* Adjust if required */ 3632 queue_id <<= shift; 3633 3634 /* 3635 * The low 8 bits are for hash value (n+0); 3636 * The next 8 bits are for hash value (n+1), etc. 3637 */ 3638 reta >>= 8; 3639 reta |= ((uint32_t)queue_id) << 24; 3640 if ((i & 3) == 3) { 3641 IGC_WRITE_REG(hw, IGC_RETA(i >> 2), reta); 3642 reta = 0; 3643 } 3644 } 3645 3646 /* 3647 * MRQC: Multiple Receive Queues Command 3648 * Set queuing to RSS control, number depends on the device. 3649 */ 3650 3651 /* Set up random bits */ 3652 uint32_t rss_key[RSS_KEYSIZE / sizeof(uint32_t)]; 3653 rss_getkey((uint8_t *)rss_key); 3654 3655 /* Now fill our hash function seeds */ 3656 for (int i = 0; i < __arraycount(rss_key); i++) 3657 IGC_WRITE_REG_ARRAY(hw, IGC_RSSRK(0), i, rss_key[i]); 3658 3659 /* 3660 * Configure the RSS fields to hash upon. 3661 */ 3662 uint32_t mrqc = IGC_MRQC_ENABLE_RSS_4Q; 3663 mrqc |= IGC_MRQC_RSS_FIELD_IPV4 | IGC_MRQC_RSS_FIELD_IPV4_TCP; 3664 mrqc |= IGC_MRQC_RSS_FIELD_IPV6 | IGC_MRQC_RSS_FIELD_IPV6_TCP; 3665 mrqc |= IGC_MRQC_RSS_FIELD_IPV6_TCP_EX; 3666 3667 IGC_WRITE_REG(hw, IGC_MRQC, mrqc); 3668 } 3669 3670 /* 3671 * igc_get_hw_control sets the {CTRL_EXT|FWSM}:DRV_LOAD bit. 3672 * For ASF and Pass Through versions of f/w this means 3673 * that the driver is loaded. For AMT version type f/w 3674 * this means that the network i/f is open. 3675 */ 3676 static void 3677 igc_get_hw_control(struct igc_softc *sc) 3678 { 3679 const uint32_t ctrl_ext = IGC_READ_REG(&sc->hw, IGC_CTRL_EXT); 3680 3681 IGC_WRITE_REG(&sc->hw, IGC_CTRL_EXT, ctrl_ext | IGC_CTRL_EXT_DRV_LOAD); 3682 } 3683 3684 /* 3685 * igc_release_hw_control resets {CTRL_EXT|FWSM}:DRV_LOAD bit. 3686 * For ASF and Pass Through versions of f/w this means that 3687 * the driver is no longer loaded. For AMT versions of the 3688 * f/w this means that the network i/f is closed. 3689 */ 3690 static void 3691 igc_release_hw_control(struct igc_softc *sc) 3692 { 3693 const uint32_t ctrl_ext = IGC_READ_REG(&sc->hw, IGC_CTRL_EXT); 3694 3695 IGC_WRITE_REG(&sc->hw, IGC_CTRL_EXT, ctrl_ext & ~IGC_CTRL_EXT_DRV_LOAD); 3696 } 3697 3698 static int 3699 igc_is_valid_ether_addr(uint8_t *addr) 3700 { 3701 const char zero_addr[6] = { 0, 0, 0, 0, 0, 0 }; 3702 3703 if ((addr[0] & 1) || !bcmp(addr, zero_addr, ETHER_ADDR_LEN)) 3704 return 0; 3705 3706 return 1; 3707 } 3708 3709 static void 3710 igc_print_devinfo(struct igc_softc *sc) 3711 { 3712 device_t dev = sc->sc_dev; 3713 struct igc_hw *hw = &sc->hw; 3714 struct igc_phy_info *phy = &hw->phy; 3715 u_int oui, model, rev; 3716 uint16_t id1, id2, nvm_ver, phy_ver, etk_lo, etk_hi; 3717 char descr[MII_MAX_DESCR_LEN]; 3718 3719 /* Print PHY Info */ 3720 id1 = phy->id >> 16; 3721 /* The revision field in phy->id is cleard and it's in phy->revision */ 3722 id2 = (phy->id & 0xfff0) | phy->revision; 3723 oui = MII_OUI(id1, id2); 3724 model = MII_MODEL(id2); 3725 rev = MII_REV(id2); 3726 mii_get_descr(descr, sizeof(descr), oui, model); 3727 if (descr[0]) 3728 aprint_normal_dev(dev, "PHY: %s, rev. %d", 3729 descr, rev); 3730 else 3731 aprint_normal_dev(dev, 3732 "PHY OUI 0x%06x, model 0x%04x, rev. %d", 3733 oui, model, rev); 3734 3735 /* PHY FW version */ 3736 phy->ops.read_reg(hw, 0x1e, &phy_ver); 3737 aprint_normal(", PHY FW version 0x%04hx\n", phy_ver); 3738 3739 /* NVM version */ 3740 hw->nvm.ops.read(hw, NVM_VERSION, 1, &nvm_ver); 3741 3742 /* EtrackID */ 3743 hw->nvm.ops.read(hw, NVM_ETKID_LO, 1, &etk_lo); 3744 hw->nvm.ops.read(hw, NVM_ETKID_HI, 1, &etk_hi); 3745 3746 aprint_normal_dev(dev, 3747 "NVM image version %x.%02x, EtrackID %04hx%04hx\n", 3748 (nvm_ver & NVM_VERSION_MAJOR) >> NVM_VERSION_MAJOR_SHIFT, 3749 nvm_ver & NVM_VERSION_MINOR, etk_hi, etk_lo); 3750 } 3751
Indexes created Tue Sep 30 11:09:46 GMT 2025