if_igc.c revision 1.3.2.7
1 /* $NetBSD: if_igc.c,v 1.3.2.7 2025/06/01 12:32:06 martin Exp $ */ 2 /* $OpenBSD: if_igc.c,v 1.13 2023/04/28 10:18:57 bluhm Exp $ */ 3 /*- 4 * SPDX-License-Identifier: BSD-2-Clause 5 * 6 * Copyright (c) 2016 Nicole Graziano <nicole (at) nextbsd.org> 7 * All rights reserved. 8 * Copyright (c) 2021 Rubicon Communications, LLC (Netgate) 9 * 10 * Redistribution and use in source and binary forms, with or without 11 * modification, are permitted provided that the following conditions 12 * are met: 13 * 1. Redistributions of source code must retain the above copyright 14 * notice, this list of conditions and the following disclaimer. 15 * 2. Redistributions in binary form must reproduce the above copyright 16 * notice, this list of conditions and the following disclaimer in the 17 * documentation and/or other materials provided with the distribution. 18 * 19 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 20 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 21 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 22 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 23 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 24 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 25 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 26 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 27 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 28 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 29 * SUCH DAMAGE. 30 */ 31 32 #include <sys/cdefs.h> 33 __KERNEL_RCSID(0, "$NetBSD: if_igc.c,v 1.3.2.7 2025/06/01 12:32:06 martin Exp $"); 34 35 #ifdef _KERNEL_OPT 36 #include "opt_if_igc.h" 37 #if 0 /* notyet */ 38 #include "vlan.h" 39 #endif 40 #endif 41 42 #include <sys/param.h> 43 #include <sys/systm.h> 44 #include <sys/bus.h> 45 #include <sys/cpu.h> 46 #include <sys/device.h> 47 #include <sys/endian.h> 48 #include <sys/intr.h> 49 #include <sys/interrupt.h> 50 #include <sys/kernel.h> 51 #include <sys/kmem.h> 52 #include <sys/mbuf.h> 53 #include <sys/mutex.h> 54 #include <sys/socket.h> 55 #include <sys/workqueue.h> 56 #include <sys/xcall.h> 57 58 #include <net/bpf.h> 59 #include <net/if.h> 60 #include <net/if_ether.h> 61 #include <net/if_media.h> 62 #include <net/if_vlanvar.h> 63 #include <net/rss_config.h> 64 65 #include <netinet/in.h> 66 #include <netinet/ip.h> 67 #include <netinet/ip6.h> 68 #include <netinet/tcp.h> 69 70 #include <dev/pci/pcivar.h> 71 #include <dev/pci/pcireg.h> 72 #include <dev/pci/pcidevs.h> 73 74 #include <dev/pci/igc/if_igc.h> 75 #include <dev/pci/igc/igc_evcnt.h> 76 #include <dev/pci/igc/igc_hw.h> 77 #include <dev/mii/miivar.h> 78 79 #define IGC_WORKQUEUE_PRI PRI_SOFTNET 80 81 #ifndef IGC_RX_INTR_PROCESS_LIMIT_DEFAULT 82 #define IGC_RX_INTR_PROCESS_LIMIT_DEFAULT 0 83 #endif 84 #ifndef IGC_TX_INTR_PROCESS_LIMIT_DEFAULT 85 #define IGC_TX_INTR_PROCESS_LIMIT_DEFAULT 0 86 #endif 87 88 #ifndef IGC_RX_PROCESS_LIMIT_DEFAULT 89 #define IGC_RX_PROCESS_LIMIT_DEFAULT 256 90 #endif 91 #ifndef IGC_TX_PROCESS_LIMIT_DEFAULT 92 #define IGC_TX_PROCESS_LIMIT_DEFAULT 256 93 #endif 94 95 #define htolem32(p, x) (*((uint32_t *)(p)) = htole32(x)) 96 #define htolem64(p, x) (*((uint64_t *)(p)) = htole64(x)) 97 98 static const struct igc_product { 99 pci_vendor_id_t igcp_vendor; 100 pci_product_id_t igcp_product; 101 const char *igcp_name; 102 } igc_products[] = { 103 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_I225_IT, 104 "Intel(R) Ethernet Controller I225-IT(2)" }, 105 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_I226_LM, 106 "Intel(R) Ethernet Controller I226-LM" }, 107 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_I226_V, 108 "Intel(R) Ethernet Controller I226-V" }, 109 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_I226_IT, 110 "Intel(R) Ethernet Controller I226-IT" }, 111 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_I221_V, 112 "Intel(R) Ethernet Controller I221-V" }, 113 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_I226_BLANK_NVM, 114 "Intel(R) Ethernet Controller I226(blankNVM)" }, 115 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_I225_LM, 116 "Intel(R) Ethernet Controller I225-LM" }, 117 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_I225_V, 118 "Intel(R) Ethernet Controller I225-V" }, 119 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_I220_V, 120 "Intel(R) Ethernet Controller I220-V" }, 121 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_I225_I, 122 "Intel(R) Ethernet Controller I225-I" }, 123 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_I225_BLANK_NVM, 124 "Intel(R) Ethernet Controller I225(blankNVM)" }, 125 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_I225_K, 126 "Intel(R) Ethernet Controller I225-K" }, 127 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_I225_K2, 128 "Intel(R) Ethernet Controller I225-K(2)" }, 129 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_I226_K, 130 "Intel(R) Ethernet Controller I226-K" }, 131 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_I225_LMVP, 132 "Intel(R) Ethernet Controller I225-LMvP(2)" }, 133 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_I226_LMVP, 134 "Intel(R) Ethernet Controller I226-LMvP" }, 135 { 0, 0, NULL }, 136 }; 137 138 #define IGC_DF_CFG 0x1 139 #define IGC_DF_TX 0x2 140 #define IGC_DF_RX 0x4 141 #define IGC_DF_MISC 0x8 142 143 #ifdef IGC_DEBUG_FLAGS 144 int igc_debug_flags = IGC_DEBUG_FLAGS; 145 #else 146 int igc_debug_flags = 0; 147 #endif 148 149 #define DPRINTF(flag, fmt, args...) do { \ 150 if (igc_debug_flags & (IGC_DF_ ## flag)) \ 151 printf("%s: %d: " fmt, __func__, __LINE__, ##args); \ 152 } while (0) 153 154 /********************************************************************* 155 * Function Prototypes 156 *********************************************************************/ 157 static int igc_match(device_t, cfdata_t, void *); 158 static void igc_attach(device_t, device_t, void *); 159 static int igc_detach(device_t, int); 160 161 static void igc_identify_hardware(struct igc_softc *); 162 static int igc_adjust_nqueues(struct igc_softc *); 163 static int igc_allocate_pci_resources(struct igc_softc *); 164 static int igc_allocate_interrupts(struct igc_softc *); 165 static int igc_allocate_queues(struct igc_softc *); 166 static void igc_free_pci_resources(struct igc_softc *); 167 static void igc_free_interrupts(struct igc_softc *); 168 static void igc_free_queues(struct igc_softc *); 169 static void igc_reset(struct igc_softc *); 170 static void igc_init_dmac(struct igc_softc *, uint32_t); 171 static int igc_setup_interrupts(struct igc_softc *); 172 static void igc_attach_counters(struct igc_softc *sc); 173 static void igc_detach_counters(struct igc_softc *sc); 174 static void igc_update_counters(struct igc_softc *sc); 175 static void igc_clear_counters(struct igc_softc *sc); 176 static int igc_setup_msix(struct igc_softc *); 177 static int igc_setup_msi(struct igc_softc *); 178 static int igc_setup_intx(struct igc_softc *); 179 static int igc_dma_malloc(struct igc_softc *, bus_size_t, 180 struct igc_dma_alloc *); 181 static void igc_dma_free(struct igc_softc *, struct igc_dma_alloc *); 182 static void igc_setup_interface(struct igc_softc *); 183 184 static int igc_init(struct ifnet *); 185 static int igc_init_locked(struct igc_softc *); 186 static void igc_start(struct ifnet *); 187 static int igc_transmit(struct ifnet *, struct mbuf *); 188 static void igc_tx_common_locked(struct ifnet *, struct tx_ring *, int); 189 static bool igc_txeof(struct tx_ring *, u_int); 190 static void igc_intr_barrier(struct igc_softc *); 191 static void igc_stop(struct ifnet *, int); 192 static void igc_stop_locked(struct igc_softc *); 193 static int igc_ioctl(struct ifnet *, u_long, void *); 194 #ifdef IF_RXR 195 static int igc_rxrinfo(struct igc_softc *, struct if_rxrinfo *); 196 #endif 197 static void igc_rxfill(struct rx_ring *); 198 static void igc_rxrefill(struct rx_ring *, int); 199 static bool igc_rxeof(struct rx_ring *, u_int); 200 static int igc_rx_checksum(struct igc_queue *, uint64_t, uint32_t, 201 uint32_t); 202 static void igc_watchdog(struct ifnet *); 203 static void igc_tick(void *); 204 static void igc_media_status(struct ifnet *, struct ifmediareq *); 205 static int igc_media_change(struct ifnet *); 206 static int igc_ifflags_cb(struct ethercom *); 207 static void igc_set_filter(struct igc_softc *); 208 static void igc_update_link_status(struct igc_softc *); 209 static int igc_get_buf(struct rx_ring *, int, bool); 210 static bool igc_tx_ctx_setup(struct tx_ring *, struct mbuf *, int, 211 uint32_t *, uint32_t *); 212 213 static void igc_configure_queues(struct igc_softc *); 214 static void igc_set_queues(struct igc_softc *, uint32_t, uint32_t, int); 215 static void igc_enable_queue(struct igc_softc *, uint32_t); 216 static void igc_enable_intr(struct igc_softc *); 217 static void igc_disable_intr(struct igc_softc *); 218 static int igc_intr_link(void *); 219 static int igc_intr_queue(void *); 220 static int igc_intr(void *); 221 static void igc_handle_queue(void *); 222 static void igc_handle_queue_work(struct work *, void *); 223 static void igc_sched_handle_queue(struct igc_softc *, struct igc_queue *); 224 static void igc_barrier_handle_queue(struct igc_softc *); 225 226 static int igc_allocate_transmit_buffers(struct tx_ring *); 227 static int igc_setup_transmit_structures(struct igc_softc *); 228 static int igc_setup_transmit_ring(struct tx_ring *); 229 static void igc_initialize_transmit_unit(struct igc_softc *); 230 static void igc_free_transmit_structures(struct igc_softc *); 231 static void igc_free_transmit_buffers(struct tx_ring *); 232 static void igc_withdraw_transmit_packets(struct tx_ring *, bool); 233 static int igc_allocate_receive_buffers(struct rx_ring *); 234 static int igc_setup_receive_structures(struct igc_softc *); 235 static int igc_setup_receive_ring(struct rx_ring *); 236 static void igc_initialize_receive_unit(struct igc_softc *); 237 static void igc_free_receive_structures(struct igc_softc *); 238 static void igc_free_receive_buffers(struct rx_ring *); 239 static void igc_clear_receive_status(struct rx_ring *); 240 static void igc_initialize_rss_mapping(struct igc_softc *); 241 242 static void igc_get_hw_control(struct igc_softc *); 243 static void igc_release_hw_control(struct igc_softc *); 244 static int igc_is_valid_ether_addr(uint8_t *); 245 static void igc_print_devinfo(struct igc_softc *); 246 247 CFATTACH_DECL3_NEW(igc, sizeof(struct igc_softc), 248 igc_match, igc_attach, igc_detach, NULL, NULL, NULL, 0); 249 250 static inline int 251 igc_txdesc_incr(struct igc_softc *sc, int id) 252 { 253 254 if (++id == sc->num_tx_desc) 255 id = 0; 256 return id; 257 } 258 259 static inline int __unused 260 igc_txdesc_decr(struct igc_softc *sc, int id) 261 { 262 263 if (--id < 0) 264 id = sc->num_tx_desc - 1; 265 return id; 266 } 267 268 static inline void 269 igc_txdesc_sync(struct tx_ring *txr, int id, int ops) 270 { 271 272 bus_dmamap_sync(txr->txdma.dma_tag, txr->txdma.dma_map, 273 id * sizeof(union igc_adv_tx_desc), sizeof(union igc_adv_tx_desc), 274 ops); 275 } 276 277 static inline int 278 igc_rxdesc_incr(struct igc_softc *sc, int id) 279 { 280 281 if (++id == sc->num_rx_desc) 282 id = 0; 283 return id; 284 } 285 286 static inline int 287 igc_rxdesc_decr(struct igc_softc *sc, int id) 288 { 289 290 if (--id < 0) 291 id = sc->num_rx_desc - 1; 292 return id; 293 } 294 295 static inline void 296 igc_rxdesc_sync(struct rx_ring *rxr, int id, int ops) 297 { 298 299 bus_dmamap_sync(rxr->rxdma.dma_tag, rxr->rxdma.dma_map, 300 id * sizeof(union igc_adv_rx_desc), sizeof(union igc_adv_rx_desc), 301 ops); 302 } 303 304 static const struct igc_product * 305 igc_lookup(const struct pci_attach_args *pa) 306 { 307 const struct igc_product *igcp; 308 309 for (igcp = igc_products; igcp->igcp_name != NULL; igcp++) { 310 if (PCI_VENDOR(pa->pa_id) == igcp->igcp_vendor && 311 PCI_PRODUCT(pa->pa_id) == igcp->igcp_product) 312 return igcp; 313 } 314 return NULL; 315 } 316 317 /********************************************************************* 318 * Device identification routine 319 * 320 * igc_match determines if the driver should be loaded on 321 * adapter based on PCI vendor/device id of the adapter. 322 * 323 * return 0 on success, positive on failure 324 *********************************************************************/ 325 static int 326 igc_match(device_t parent, cfdata_t match, void *aux) 327 { 328 struct pci_attach_args *pa = aux; 329 330 if (igc_lookup(pa) != NULL) 331 return 1; 332 333 return 0; 334 } 335 336 /********************************************************************* 337 * Device initialization routine 338 * 339 * The attach entry point is called when the driver is being loaded. 340 * This routine identifies the type of hardware, allocates all resources 341 * and initializes the hardware. 342 * 343 * return 0 on success, positive on failure 344 *********************************************************************/ 345 static void 346 igc_attach(device_t parent, device_t self, void *aux) 347 { 348 struct pci_attach_args *pa = aux; 349 struct igc_softc *sc = device_private(self); 350 struct igc_hw *hw = &sc->hw; 351 352 const struct igc_product *igcp = igc_lookup(pa); 353 KASSERT(igcp != NULL); 354 355 sc->sc_dev = self; 356 callout_init(&sc->sc_tick_ch, CALLOUT_MPSAFE); 357 callout_setfunc(&sc->sc_tick_ch, igc_tick, sc); 358 sc->sc_core_stopping = false; 359 360 sc->osdep.os_sc = sc; 361 sc->osdep.os_pa = *pa; 362 #ifndef __aarch64__ 363 /* 364 * XXX PR port-arm/57643 365 * 64-bit DMA does not work at least for LX2K with 32/64GB memory. 366 * smmu(4) support may be required. 367 */ 368 if (pci_dma64_available(pa)) { 369 aprint_verbose(", 64-bit DMA"); 370 sc->osdep.os_dmat = pa->pa_dmat64; 371 } else 372 #endif 373 { 374 aprint_verbose(", 32-bit DMA"); 375 sc->osdep.os_dmat = pa->pa_dmat; 376 } 377 378 pci_aprint_devinfo_fancy(pa, "Ethernet controller", igcp->igcp_name, 1); 379 380 /* Determine hardware and mac info */ 381 igc_identify_hardware(sc); 382 383 sc->num_tx_desc = IGC_DEFAULT_TXD; 384 sc->num_rx_desc = IGC_DEFAULT_RXD; 385 386 /* Setup PCI resources */ 387 if (igc_allocate_pci_resources(sc)) { 388 aprint_error_dev(sc->sc_dev, 389 "unable to allocate PCI resources\n"); 390 goto err_pci; 391 } 392 393 if (igc_allocate_interrupts(sc)) { 394 aprint_error_dev(sc->sc_dev, "unable to allocate interrupts\n"); 395 goto err_pci; 396 } 397 398 /* Allocate TX/RX queues */ 399 if (igc_allocate_queues(sc)) { 400 aprint_error_dev(sc->sc_dev, "unable to allocate queues\n"); 401 goto err_alloc_intr; 402 } 403 404 /* Do shared code initialization */ 405 if (igc_setup_init_funcs(hw, true)) { 406 aprint_error_dev(sc->sc_dev, "unable to initialize\n"); 407 goto err_alloc_intr; 408 } 409 410 hw->mac.autoneg = DO_AUTO_NEG; 411 hw->phy.autoneg_wait_to_complete = false; 412 hw->phy.autoneg_advertised = AUTONEG_ADV_DEFAULT; 413 414 /* Copper options. */ 415 if (hw->phy.media_type == igc_media_type_copper) 416 hw->phy.mdix = AUTO_ALL_MODES; 417 418 /* Set the max frame size. */ 419 sc->hw.mac.max_frame_size = 9234; 420 421 /* Allocate multicast array memory. */ 422 sc->mta = kmem_alloc(IGC_MTA_LEN, KM_SLEEP); 423 424 /* Check SOL/IDER usage. */ 425 if (igc_check_reset_block(hw)) { 426 aprint_error_dev(sc->sc_dev, 427 "PHY reset is blocked due to SOL/IDER session\n"); 428 } 429 430 /* Disable Energy Efficient Ethernet. */ 431 sc->hw.dev_spec._i225.eee_disable = true; 432 433 igc_reset_hw(hw); 434 435 /* Make sure we have a good EEPROM before we read from it. */ 436 if (igc_validate_nvm_checksum(hw) < 0) { 437 /* 438 * Some PCI-E parts fail the first check due to 439 * the link being in sleep state, call it again, 440 * if it fails a second time its a real issue. 441 */ 442 if (igc_validate_nvm_checksum(hw) < 0) { 443 aprint_error_dev(sc->sc_dev, 444 "EEPROM checksum invalid\n"); 445 goto err_late; 446 } 447 } 448 449 /* Copy the permanent MAC address out of the EEPROM. */ 450 if (igc_read_mac_addr(hw) < 0) { 451 aprint_error_dev(sc->sc_dev, 452 "unable to read MAC address from EEPROM\n"); 453 goto err_late; 454 } 455 456 if (!igc_is_valid_ether_addr(hw->mac.addr)) { 457 aprint_error_dev(sc->sc_dev, "invalid MAC address\n"); 458 goto err_late; 459 } 460 461 if (igc_setup_interrupts(sc)) 462 goto err_late; 463 464 /* Attach counters. */ 465 igc_attach_counters(sc); 466 467 /* Setup OS specific network interface. */ 468 igc_setup_interface(sc); 469 470 igc_print_devinfo(sc); 471 472 igc_reset(sc); 473 hw->mac.get_link_status = true; 474 igc_update_link_status(sc); 475 476 /* The driver can now take control from firmware. */ 477 igc_get_hw_control(sc); 478 479 aprint_normal_dev(sc->sc_dev, "Ethernet address %s\n", 480 ether_sprintf(sc->hw.mac.addr)); 481 482 if (pmf_device_register(self, NULL, NULL)) 483 pmf_class_network_register(self, &sc->sc_ec.ec_if); 484 else 485 aprint_error_dev(self, "couldn't establish power handler\n"); 486 487 return; 488 489 err_late: 490 igc_release_hw_control(sc); 491 err_alloc_intr: 492 igc_free_interrupts(sc); 493 err_pci: 494 igc_free_pci_resources(sc); 495 kmem_free(sc->mta, IGC_MTA_LEN); 496 } 497 498 /********************************************************************* 499 * Device removal routine 500 * 501 * The detach entry point is called when the driver is being removed. 502 * This routine stops the adapter and deallocates all the resources 503 * that were allocated for driver operation. 504 * 505 * return 0 on success, positive on failure 506 *********************************************************************/ 507 static int 508 igc_detach(device_t self, int flags) 509 { 510 struct igc_softc *sc = device_private(self); 511 struct ifnet *ifp = &sc->sc_ec.ec_if; 512 513 mutex_enter(&sc->sc_core_lock); 514 igc_stop_locked(sc); 515 mutex_exit(&sc->sc_core_lock); 516 517 igc_detach_counters(sc); 518 519 igc_free_queues(sc); 520 521 igc_phy_hw_reset(&sc->hw); 522 igc_release_hw_control(sc); 523 524 ether_ifdetach(ifp); 525 if_detach(ifp); 526 ifmedia_fini(&sc->media); 527 528 igc_free_interrupts(sc); 529 igc_free_pci_resources(sc); 530 kmem_free(sc->mta, IGC_MTA_LEN); 531 532 mutex_destroy(&sc->sc_core_lock); 533 534 return 0; 535 } 536 537 static void 538 igc_identify_hardware(struct igc_softc *sc) 539 { 540 struct igc_osdep *os = &sc->osdep; 541 struct pci_attach_args *pa = &os->os_pa; 542 543 /* Save off the information about this board. */ 544 sc->hw.device_id = PCI_PRODUCT(pa->pa_id); 545 546 /* Do shared code init and setup. */ 547 if (igc_set_mac_type(&sc->hw)) { 548 aprint_error_dev(sc->sc_dev, "unable to identify hardware\n"); 549 return; 550 } 551 } 552 553 static int 554 igc_allocate_pci_resources(struct igc_softc *sc) 555 { 556 struct igc_osdep *os = &sc->osdep; 557 struct pci_attach_args *pa = &os->os_pa; 558 559 /* 560 * Enable bus mastering and memory-mapped I/O for sure. 561 */ 562 pcireg_t csr = 563 pci_conf_read(pa->pa_pc, pa->pa_tag, PCI_COMMAND_STATUS_REG); 564 csr |= PCI_COMMAND_MASTER_ENABLE | PCI_COMMAND_MEM_ENABLE; 565 pci_conf_write(pa->pa_pc, pa->pa_tag, PCI_COMMAND_STATUS_REG, csr); 566 567 const pcireg_t memtype = 568 pci_mapreg_type(pa->pa_pc, pa->pa_tag, IGC_PCIREG); 569 if (pci_mapreg_map(pa, IGC_PCIREG, memtype, 0, &os->os_memt, 570 &os->os_memh, &os->os_membase, &os->os_memsize)) { 571 aprint_error_dev(sc->sc_dev, "unable to map registers\n"); 572 return ENXIO; 573 } 574 575 sc->hw.hw_addr = os->os_membase; 576 sc->hw.back = os; 577 578 return 0; 579 } 580 581 static int __unused 582 igc_adjust_nqueues(struct igc_softc *sc) 583 { 584 struct pci_attach_args *pa = &sc->osdep.os_pa; 585 int nqueues = MIN(IGC_MAX_NQUEUES, ncpu); 586 587 const int nmsix = pci_msix_count(pa->pa_pc, pa->pa_tag); 588 if (nmsix <= 1) 589 nqueues = 1; 590 else if (nmsix < nqueues + 1) 591 nqueues = nmsix - 1; 592 593 return nqueues; 594 } 595 596 static int 597 igc_allocate_interrupts(struct igc_softc *sc) 598 { 599 struct pci_attach_args *pa = &sc->osdep.os_pa; 600 int error; 601 602 #ifndef IGC_DISABLE_MSIX 603 const int nqueues = igc_adjust_nqueues(sc); 604 if (nqueues > 1) { 605 sc->sc_nintrs = nqueues + 1; 606 error = pci_msix_alloc_exact(pa, &sc->sc_intrs, sc->sc_nintrs); 607 if (!error) { 608 sc->sc_nqueues = nqueues; 609 sc->sc_intr_type = PCI_INTR_TYPE_MSIX; 610 return 0; 611 } 612 } 613 #endif 614 615 /* fallback to MSI */ 616 sc->sc_nintrs = sc->sc_nqueues = 1; 617 618 #ifndef IGC_DISABLE_MSI 619 error = pci_msi_alloc_exact(pa, &sc->sc_intrs, sc->sc_nintrs); 620 if (!error) { 621 sc->sc_intr_type = PCI_INTR_TYPE_MSI; 622 return 0; 623 } 624 #endif 625 626 /* fallback to INTx */ 627 628 error = pci_intx_alloc(pa, &sc->sc_intrs); 629 if (!error) { 630 sc->sc_intr_type = PCI_INTR_TYPE_INTX; 631 return 0; 632 } 633 634 return error; 635 } 636 637 static int 638 igc_allocate_queues(struct igc_softc *sc) 639 { 640 device_t dev = sc->sc_dev; 641 int rxconf = 0, txconf = 0; 642 643 /* Allocate the top level queue structs. */ 644 sc->queues = 645 kmem_zalloc(sc->sc_nqueues * sizeof(struct igc_queue), KM_SLEEP); 646 647 /* Allocate the TX ring. */ 648 sc->tx_rings = 649 kmem_zalloc(sc->sc_nqueues * sizeof(struct tx_ring), KM_SLEEP); 650 651 /* Allocate the RX ring. */ 652 sc->rx_rings = 653 kmem_zalloc(sc->sc_nqueues * sizeof(struct rx_ring), KM_SLEEP); 654 655 /* Set up the TX queues. */ 656 for (int iq = 0; iq < sc->sc_nqueues; iq++, txconf++) { 657 struct tx_ring *txr = &sc->tx_rings[iq]; 658 const int tsize = roundup2( 659 sc->num_tx_desc * sizeof(union igc_adv_tx_desc), 660 IGC_DBA_ALIGN); 661 662 txr->sc = sc; 663 txr->txr_igcq = &sc->queues[iq]; 664 txr->me = iq; 665 if (igc_dma_malloc(sc, tsize, &txr->txdma)) { 666 aprint_error_dev(dev, 667 "unable to allocate TX descriptor\n"); 668 goto fail; 669 } 670 txr->tx_base = (union igc_adv_tx_desc *)txr->txdma.dma_vaddr; 671 memset(txr->tx_base, 0, tsize); 672 } 673 674 /* Prepare transmit descriptors and buffers. */ 675 if (igc_setup_transmit_structures(sc)) { 676 aprint_error_dev(dev, "unable to setup transmit structures\n"); 677 goto fail; 678 } 679 680 /* Set up the RX queues. */ 681 for (int iq = 0; iq < sc->sc_nqueues; iq++, rxconf++) { 682 struct rx_ring *rxr = &sc->rx_rings[iq]; 683 const int rsize = roundup2( 684 sc->num_rx_desc * sizeof(union igc_adv_rx_desc), 685 IGC_DBA_ALIGN); 686 687 rxr->sc = sc; 688 rxr->rxr_igcq = &sc->queues[iq]; 689 rxr->me = iq; 690 #ifdef OPENBSD 691 timeout_set(&rxr->rx_refill, igc_rxrefill, rxr); 692 #endif 693 if (igc_dma_malloc(sc, rsize, &rxr->rxdma)) { 694 aprint_error_dev(dev, 695 "unable to allocate RX descriptor\n"); 696 goto fail; 697 } 698 rxr->rx_base = (union igc_adv_rx_desc *)rxr->rxdma.dma_vaddr; 699 memset(rxr->rx_base, 0, rsize); 700 } 701 702 sc->rx_mbuf_sz = MCLBYTES; 703 /* Prepare receive descriptors and buffers. */ 704 if (igc_setup_receive_structures(sc)) { 705 aprint_error_dev(sc->sc_dev, 706 "unable to setup receive structures\n"); 707 goto fail; 708 } 709 710 /* Set up the queue holding structs. */ 711 for (int iq = 0; iq < sc->sc_nqueues; iq++) { 712 struct igc_queue *q = &sc->queues[iq]; 713 714 q->sc = sc; 715 q->txr = &sc->tx_rings[iq]; 716 q->rxr = &sc->rx_rings[iq]; 717 } 718 719 return 0; 720 721 fail: 722 for (struct rx_ring *rxr = sc->rx_rings; rxconf > 0; rxr++, rxconf--) 723 igc_dma_free(sc, &rxr->rxdma); 724 for (struct tx_ring *txr = sc->tx_rings; txconf > 0; txr++, txconf--) 725 igc_dma_free(sc, &txr->txdma); 726 727 kmem_free(sc->rx_rings, sc->sc_nqueues * sizeof(struct rx_ring)); 728 sc->rx_rings = NULL; 729 kmem_free(sc->tx_rings, sc->sc_nqueues * sizeof(struct tx_ring)); 730 sc->tx_rings = NULL; 731 kmem_free(sc->queues, sc->sc_nqueues * sizeof(struct igc_queue)); 732 sc->queues = NULL; 733 734 return ENOMEM; 735 } 736 737 static void 738 igc_free_pci_resources(struct igc_softc *sc) 739 { 740 struct igc_osdep *os = &sc->osdep; 741 742 if (os->os_membase != 0) 743 bus_space_unmap(os->os_memt, os->os_memh, os->os_memsize); 744 os->os_membase = 0; 745 } 746 747 static void 748 igc_free_interrupts(struct igc_softc *sc) 749 { 750 struct pci_attach_args *pa = &sc->osdep.os_pa; 751 pci_chipset_tag_t pc = pa->pa_pc; 752 753 for (int i = 0; i < sc->sc_nintrs; i++) { 754 if (sc->sc_ihs[i] != NULL) { 755 pci_intr_disestablish(pc, sc->sc_ihs[i]); 756 sc->sc_ihs[i] = NULL; 757 } 758 } 759 pci_intr_release(pc, sc->sc_intrs, sc->sc_nintrs); 760 } 761 762 static void 763 igc_free_queues(struct igc_softc *sc) 764 { 765 766 igc_free_receive_structures(sc); 767 for (int iq = 0; iq < sc->sc_nqueues; iq++) { 768 struct rx_ring *rxr = &sc->rx_rings[iq]; 769 770 igc_dma_free(sc, &rxr->rxdma); 771 } 772 773 igc_free_transmit_structures(sc); 774 for (int iq = 0; iq < sc->sc_nqueues; iq++) { 775 struct tx_ring *txr = &sc->tx_rings[iq]; 776 777 igc_dma_free(sc, &txr->txdma); 778 } 779 780 kmem_free(sc->rx_rings, sc->sc_nqueues * sizeof(struct rx_ring)); 781 kmem_free(sc->tx_rings, sc->sc_nqueues * sizeof(struct tx_ring)); 782 kmem_free(sc->queues, sc->sc_nqueues * sizeof(struct igc_queue)); 783 } 784 785 /********************************************************************* 786 * 787 * Initialize the hardware to a configuration as specified by the 788 * adapter structure. 789 * 790 **********************************************************************/ 791 static void 792 igc_reset(struct igc_softc *sc) 793 { 794 struct igc_hw *hw = &sc->hw; 795 796 /* Let the firmware know the OS is in control */ 797 igc_get_hw_control(sc); 798 799 /* 800 * Packet Buffer Allocation (PBA) 801 * Writing PBA sets the receive portion of the buffer 802 * the remainder is used for the transmit buffer. 803 */ 804 const uint32_t pba = IGC_PBA_34K; 805 806 /* 807 * These parameters control the automatic generation (Tx) and 808 * response (Rx) to Ethernet PAUSE frames. 809 * - High water mark should allow for at least two frames to be 810 * received after sending an XOFF. 811 * - Low water mark works best when it is very near the high water mark. 812 * This allows the receiver to restart by sending XON when it has 813 * drained a bit. Here we use an arbitrary value of 1500 which will 814 * restart after one full frame is pulled from the buffer. There 815 * could be several smaller frames in the buffer and if so they will 816 * not trigger the XON until their total number reduces the buffer 817 * by 1500. 818 * - The pause time is fairly large at 1000 x 512ns = 512 usec. 819 */ 820 const uint16_t rx_buffer_size = (pba & 0xffff) << 10; 821 822 hw->fc.high_water = rx_buffer_size - 823 roundup2(sc->hw.mac.max_frame_size, 1024); 824 /* 16-byte granularity */ 825 hw->fc.low_water = hw->fc.high_water - 16; 826 827 if (sc->fc) /* locally set flow control value? */ 828 hw->fc.requested_mode = sc->fc; 829 else 830 hw->fc.requested_mode = igc_fc_full; 831 832 hw->fc.pause_time = IGC_FC_PAUSE_TIME; 833 834 hw->fc.send_xon = true; 835 836 /* Issue a global reset */ 837 igc_reset_hw(hw); 838 IGC_WRITE_REG(hw, IGC_WUC, 0); 839 840 /* and a re-init */ 841 if (igc_init_hw(hw) < 0) { 842 aprint_error_dev(sc->sc_dev, "unable to reset hardware\n"); 843 return; 844 } 845 846 /* Setup DMA Coalescing */ 847 igc_init_dmac(sc, pba); 848 849 IGC_WRITE_REG(hw, IGC_VET, ETHERTYPE_VLAN); 850 igc_get_phy_info(hw); 851 igc_check_for_link(hw); 852 } 853 854 /********************************************************************* 855 * 856 * Initialize the DMA Coalescing feature 857 * 858 **********************************************************************/ 859 static void 860 igc_init_dmac(struct igc_softc *sc, uint32_t pba) 861 { 862 struct igc_hw *hw = &sc->hw; 863 const uint16_t max_frame_size = sc->hw.mac.max_frame_size; 864 uint32_t reg, status; 865 866 if (sc->dmac == 0) { /* Disabling it */ 867 reg = ~IGC_DMACR_DMAC_EN; /* XXXRO */ 868 IGC_WRITE_REG(hw, IGC_DMACR, reg); 869 DPRINTF(MISC, "DMA coalescing disabled\n"); 870 return; 871 } else { 872 device_printf(sc->sc_dev, "DMA coalescing enabled\n"); 873 } 874 875 /* Set starting threshold */ 876 IGC_WRITE_REG(hw, IGC_DMCTXTH, 0); 877 878 uint16_t hwm = 64 * pba - max_frame_size / 16; 879 if (hwm < 64 * (pba - 6)) 880 hwm = 64 * (pba - 6); 881 reg = IGC_READ_REG(hw, IGC_FCRTC); 882 reg &= ~IGC_FCRTC_RTH_COAL_MASK; 883 reg |= (hwm << IGC_FCRTC_RTH_COAL_SHIFT) & IGC_FCRTC_RTH_COAL_MASK; 884 IGC_WRITE_REG(hw, IGC_FCRTC, reg); 885 886 uint32_t dmac = pba - max_frame_size / 512; 887 if (dmac < pba - 10) 888 dmac = pba - 10; 889 reg = IGC_READ_REG(hw, IGC_DMACR); 890 reg &= ~IGC_DMACR_DMACTHR_MASK; 891 reg |= (dmac << IGC_DMACR_DMACTHR_SHIFT) & IGC_DMACR_DMACTHR_MASK; 892 893 /* transition to L0x or L1 if available..*/ 894 reg |= IGC_DMACR_DMAC_EN | IGC_DMACR_DMAC_LX_MASK; 895 896 /* Check if status is 2.5Gb backplane connection 897 * before configuration of watchdog timer, which is 898 * in msec values in 12.8usec intervals 899 * watchdog timer= msec values in 32usec intervals 900 * for non 2.5Gb connection 901 */ 902 status = IGC_READ_REG(hw, IGC_STATUS); 903 if ((status & IGC_STATUS_2P5_SKU) && 904 !(status & IGC_STATUS_2P5_SKU_OVER)) 905 reg |= (sc->dmac * 5) >> 6; 906 else 907 reg |= sc->dmac >> 5; 908 909 IGC_WRITE_REG(hw, IGC_DMACR, reg); 910 911 IGC_WRITE_REG(hw, IGC_DMCRTRH, 0); 912 913 /* Set the interval before transition */ 914 reg = IGC_READ_REG(hw, IGC_DMCTLX); 915 reg |= IGC_DMCTLX_DCFLUSH_DIS; 916 917 /* 918 * in 2.5Gb connection, TTLX unit is 0.4 usec 919 * which is 0x4*2 = 0xA. But delay is still 4 usec 920 */ 921 status = IGC_READ_REG(hw, IGC_STATUS); 922 if ((status & IGC_STATUS_2P5_SKU) && 923 !(status & IGC_STATUS_2P5_SKU_OVER)) 924 reg |= 0xA; 925 else 926 reg |= 0x4; 927 928 IGC_WRITE_REG(hw, IGC_DMCTLX, reg); 929 930 /* free space in tx packet buffer to wake from DMA coal */ 931 IGC_WRITE_REG(hw, IGC_DMCTXTH, 932 (IGC_TXPBSIZE - (2 * max_frame_size)) >> 6); 933 934 /* make low power state decision controlled by DMA coal */ 935 reg = IGC_READ_REG(hw, IGC_PCIEMISC); 936 reg &= ~IGC_PCIEMISC_LX_DECISION; 937 IGC_WRITE_REG(hw, IGC_PCIEMISC, reg); 938 } 939 940 static int 941 igc_setup_interrupts(struct igc_softc *sc) 942 { 943 int error; 944 945 switch (sc->sc_intr_type) { 946 case PCI_INTR_TYPE_MSIX: 947 error = igc_setup_msix(sc); 948 break; 949 case PCI_INTR_TYPE_MSI: 950 error = igc_setup_msi(sc); 951 break; 952 case PCI_INTR_TYPE_INTX: 953 error = igc_setup_intx(sc); 954 break; 955 default: 956 panic("%s: invalid interrupt type: %d", 957 device_xname(sc->sc_dev), sc->sc_intr_type); 958 } 959 960 return error; 961 } 962 963 static void 964 igc_attach_counters(struct igc_softc *sc) 965 { 966 #ifdef IGC_EVENT_COUNTERS 967 968 /* Global counters */ 969 sc->sc_global_evcnts = kmem_zalloc( 970 IGC_GLOBAL_COUNTERS * sizeof(sc->sc_global_evcnts[0]), KM_SLEEP); 971 972 for (int cnt = 0; cnt < IGC_GLOBAL_COUNTERS; cnt++) { 973 evcnt_attach_dynamic(&sc->sc_global_evcnts[cnt], 974 igc_global_counters[cnt].type, NULL, 975 device_xname(sc->sc_dev), igc_global_counters[cnt].name); 976 } 977 978 /* Driver counters */ 979 sc->sc_driver_evcnts = kmem_zalloc( 980 IGC_DRIVER_COUNTERS * sizeof(sc->sc_driver_evcnts[0]), KM_SLEEP); 981 982 for (int cnt = 0; cnt < IGC_DRIVER_COUNTERS; cnt++) { 983 evcnt_attach_dynamic(&sc->sc_driver_evcnts[cnt], 984 igc_driver_counters[cnt].type, NULL, 985 device_xname(sc->sc_dev), igc_driver_counters[cnt].name); 986 } 987 988 for (int iq = 0; iq < sc->sc_nqueues; iq++) { 989 struct igc_queue *q = &sc->queues[iq]; 990 991 q->igcq_driver_counters = kmem_zalloc( 992 IGC_DRIVER_COUNTERS * sizeof(q->igcq_driver_counters[0]), 993 KM_SLEEP); 994 } 995 996 /* Queue counters */ 997 for (int iq = 0; iq < sc->sc_nqueues; iq++) { 998 struct igc_queue *q = &sc->queues[iq]; 999 1000 snprintf(q->igcq_queue_evname, sizeof(q->igcq_queue_evname), 1001 "%s q%d", device_xname(sc->sc_dev), iq); 1002 1003 q->igcq_queue_evcnts = kmem_zalloc( 1004 IGC_QUEUE_COUNTERS * sizeof(q->igcq_queue_evcnts[0]), 1005 KM_SLEEP); 1006 1007 for (int cnt = 0; cnt < IGC_QUEUE_COUNTERS; cnt++) { 1008 evcnt_attach_dynamic(&q->igcq_queue_evcnts[cnt], 1009 igc_queue_counters[cnt].type, NULL, 1010 q->igcq_queue_evname, igc_queue_counters[cnt].name); 1011 } 1012 } 1013 1014 /* MAC counters */ 1015 snprintf(sc->sc_mac_evname, sizeof(sc->sc_mac_evname), 1016 "%s Mac Statistics", device_xname(sc->sc_dev)); 1017 1018 sc->sc_mac_evcnts = kmem_zalloc( 1019 IGC_MAC_COUNTERS * sizeof(sc->sc_mac_evcnts[0]), KM_SLEEP); 1020 1021 for (int cnt = 0; cnt < IGC_MAC_COUNTERS; cnt++) { 1022 evcnt_attach_dynamic(&sc->sc_mac_evcnts[cnt], EVCNT_TYPE_MISC, 1023 NULL, sc->sc_mac_evname, igc_mac_counters[cnt].name); 1024 } 1025 #endif 1026 } 1027 1028 static void 1029 igc_detach_counters(struct igc_softc *sc) 1030 { 1031 #ifdef IGC_EVENT_COUNTERS 1032 1033 /* Global counters */ 1034 for (int cnt = 0; cnt < IGC_GLOBAL_COUNTERS; cnt++) 1035 evcnt_detach(&sc->sc_global_evcnts[cnt]); 1036 1037 kmem_free(sc->sc_global_evcnts, 1038 IGC_GLOBAL_COUNTERS * sizeof(sc->sc_global_evcnts[0])); 1039 1040 /* Driver counters */ 1041 for (int iq = 0; iq < sc->sc_nqueues; iq++) { 1042 struct igc_queue *q = &sc->queues[iq]; 1043 1044 kmem_free(q->igcq_driver_counters, 1045 IGC_DRIVER_COUNTERS * sizeof(q->igcq_driver_counters[0])); 1046 } 1047 1048 for (int cnt = 0; cnt < IGC_DRIVER_COUNTERS; cnt++) 1049 evcnt_detach(&sc->sc_driver_evcnts[cnt]); 1050 1051 kmem_free(sc->sc_driver_evcnts, 1052 IGC_DRIVER_COUNTERS * sizeof(sc->sc_driver_evcnts[0])); 1053 1054 /* Queue counters */ 1055 for (int iq = 0; iq < sc->sc_nqueues; iq++) { 1056 struct igc_queue *q = &sc->queues[iq]; 1057 1058 for (int cnt = 0; cnt < IGC_QUEUE_COUNTERS; cnt++) 1059 evcnt_detach(&q->igcq_queue_evcnts[cnt]); 1060 1061 kmem_free(q->igcq_queue_evcnts, 1062 IGC_QUEUE_COUNTERS * sizeof(q->igcq_queue_evcnts[0])); 1063 } 1064 1065 /* MAC statistics */ 1066 for (int cnt = 0; cnt < IGC_MAC_COUNTERS; cnt++) 1067 evcnt_detach(&sc->sc_mac_evcnts[cnt]); 1068 1069 kmem_free(sc->sc_mac_evcnts, 1070 IGC_MAC_COUNTERS * sizeof(sc->sc_mac_evcnts[0])); 1071 #endif 1072 } 1073 1074 /* 1075 * XXX 1076 * FreeBSD uses 4-byte-wise read for 64-bit counters, while Linux just 1077 * drops hi words. 1078 */ 1079 static inline uint64_t __unused 1080 igc_read_mac_counter(struct igc_hw *hw, bus_size_t reg, bool is64) 1081 { 1082 uint64_t val; 1083 1084 val = IGC_READ_REG(hw, reg); 1085 if (is64) 1086 val += ((uint64_t)IGC_READ_REG(hw, reg + 4)) << 32; 1087 return val; 1088 } 1089 1090 static void 1091 igc_update_counters(struct igc_softc *sc) 1092 { 1093 #ifdef IGC_EVENT_COUNTERS 1094 1095 /* Global counters: nop */ 1096 1097 /* Driver counters */ 1098 uint64_t sum[IGC_DRIVER_COUNTERS]; 1099 1100 memset(sum, 0, sizeof(sum)); 1101 for (int iq = 0; iq < sc->sc_nqueues; iq++) { 1102 struct igc_queue *q = &sc->queues[iq]; 1103 1104 for (int cnt = 0; cnt < IGC_DRIVER_COUNTERS; cnt++) { 1105 sum[cnt] += IGC_QUEUE_DRIVER_COUNTER_VAL(q, cnt); 1106 IGC_QUEUE_DRIVER_COUNTER_STORE(q, cnt, 0); 1107 } 1108 } 1109 1110 for (int cnt = 0; cnt < IGC_DRIVER_COUNTERS; cnt++) 1111 IGC_DRIVER_COUNTER_ADD(sc, cnt, sum[cnt]); 1112 1113 /* Queue counters: nop */ 1114 1115 /* Mac statistics */ 1116 struct igc_hw *hw = &sc->hw; 1117 struct ifnet *ifp = &sc->sc_ec.ec_if; 1118 uint64_t iqdrops = 0; 1119 1120 for (int cnt = 0; cnt < IGC_MAC_COUNTERS; cnt++) { 1121 uint64_t val; 1122 bus_size_t regaddr = igc_mac_counters[cnt].reg; 1123 1124 val = igc_read_mac_counter(hw, regaddr, 1125 igc_mac_counters[cnt].is64); 1126 IGC_MAC_COUNTER_ADD(sc, cnt, val); 1127 /* XXX Count MPC to iqdrops. */ 1128 if (regaddr == IGC_MPC) 1129 iqdrops += val; 1130 } 1131 1132 for (int iq = 0; iq < sc->sc_nqueues; iq++) { 1133 uint32_t val; 1134 1135 /* XXX RQDPC should be visible via evcnt(9). */ 1136 val = IGC_READ_REG(hw, IGC_RQDPC(iq)); 1137 1138 /* RQDPC is not cleard on read. */ 1139 if (val != 0) 1140 IGC_WRITE_REG(hw, IGC_RQDPC(iq), 0); 1141 iqdrops += val; 1142 } 1143 1144 if (iqdrops != 0) 1145 if_statadd(ifp, if_iqdrops, iqdrops); 1146 #endif 1147 } 1148 1149 static void 1150 igc_clear_counters(struct igc_softc *sc) 1151 { 1152 #ifdef IGC_EVENT_COUNTERS 1153 1154 /* Global counters */ 1155 for (int cnt = 0; cnt < IGC_GLOBAL_COUNTERS; cnt++) 1156 IGC_GLOBAL_COUNTER_STORE(sc, cnt, 0); 1157 1158 /* Driver counters */ 1159 for (int iq = 0; iq < sc->sc_nqueues; iq++) { 1160 struct igc_queue *q = &sc->queues[iq]; 1161 1162 for (int cnt = 0; cnt < IGC_DRIVER_COUNTERS; cnt++) 1163 IGC_QUEUE_DRIVER_COUNTER_STORE(q, cnt, 0); 1164 } 1165 1166 for (int cnt = 0; cnt < IGC_DRIVER_COUNTERS; cnt++) 1167 IGC_DRIVER_COUNTER_STORE(sc, cnt, 0); 1168 1169 /* Queue counters */ 1170 for (int iq = 0; iq < sc->sc_nqueues; iq++) { 1171 struct igc_queue *q = &sc->queues[iq]; 1172 1173 for (int cnt = 0; cnt < IGC_QUEUE_COUNTERS; cnt++) 1174 IGC_QUEUE_COUNTER_STORE(q, cnt, 0); 1175 } 1176 1177 /* Mac statistics */ 1178 struct igc_hw *hw = &sc->hw; 1179 1180 for (int cnt = 0; cnt < IGC_MAC_COUNTERS; cnt++) { 1181 (void)igc_read_mac_counter(hw, igc_mac_counters[cnt].reg, 1182 igc_mac_counters[cnt].is64); 1183 IGC_MAC_COUNTER_STORE(sc, cnt, 0); 1184 } 1185 #endif 1186 } 1187 1188 static int 1189 igc_setup_msix(struct igc_softc *sc) 1190 { 1191 pci_chipset_tag_t pc = sc->osdep.os_pa.pa_pc; 1192 device_t dev = sc->sc_dev; 1193 pci_intr_handle_t *intrs; 1194 void **ihs; 1195 const char *intrstr; 1196 char intrbuf[PCI_INTRSTR_LEN]; 1197 char xnamebuf[MAX(32, MAXCOMLEN)]; 1198 int iq, error; 1199 1200 for (iq = 0, intrs = sc->sc_intrs, ihs = sc->sc_ihs; 1201 iq < sc->sc_nqueues; iq++, intrs++, ihs++) { 1202 struct igc_queue *q = &sc->queues[iq]; 1203 1204 snprintf(xnamebuf, sizeof(xnamebuf), "%s: txrx %d", 1205 device_xname(dev), iq); 1206 1207 intrstr = pci_intr_string(pc, *intrs, intrbuf, sizeof(intrbuf)); 1208 1209 pci_intr_setattr(pc, intrs, PCI_INTR_MPSAFE, true); 1210 *ihs = pci_intr_establish_xname(pc, *intrs, IPL_NET, 1211 igc_intr_queue, q, xnamebuf); 1212 if (*ihs == NULL) { 1213 aprint_error_dev(dev, 1214 "unable to establish txrx interrupt at %s\n", 1215 intrstr); 1216 return ENOBUFS; 1217 } 1218 aprint_normal_dev(dev, "txrx interrupting at %s\n", intrstr); 1219 1220 kcpuset_t *affinity; 1221 kcpuset_create(&affinity, true); 1222 kcpuset_set(affinity, iq % ncpu); 1223 error = interrupt_distribute(*ihs, affinity, NULL); 1224 if (error) { 1225 aprint_normal_dev(dev, 1226 "%s: unable to change affinity, use default CPU\n", 1227 intrstr); 1228 } 1229 kcpuset_destroy(affinity); 1230 1231 q->igcq_si = softint_establish(SOFTINT_NET | SOFTINT_MPSAFE, 1232 igc_handle_queue, q); 1233 if (q->igcq_si == NULL) { 1234 aprint_error_dev(dev, 1235 "%s: unable to establish softint\n", intrstr); 1236 return ENOBUFS; 1237 } 1238 1239 q->msix = iq; 1240 q->eims = 1 << iq; 1241 } 1242 1243 snprintf(xnamebuf, MAXCOMLEN, "%s_tx_rx", device_xname(dev)); 1244 error = workqueue_create(&sc->sc_queue_wq, xnamebuf, 1245 igc_handle_queue_work, sc, IGC_WORKQUEUE_PRI, IPL_NET, 1246 WQ_PERCPU | WQ_MPSAFE); 1247 if (error) { 1248 aprint_error_dev(dev, "workqueue_create failed\n"); 1249 return ENOBUFS; 1250 } 1251 sc->sc_txrx_workqueue = false; 1252 1253 intrstr = pci_intr_string(pc, *intrs, intrbuf, sizeof(intrbuf)); 1254 snprintf(xnamebuf, sizeof(xnamebuf), "%s: link", device_xname(dev)); 1255 pci_intr_setattr(pc, intrs, PCI_INTR_MPSAFE, true); 1256 *ihs = pci_intr_establish_xname(pc, *intrs, IPL_NET, 1257 igc_intr_link, sc, xnamebuf); 1258 if (*ihs == NULL) { 1259 aprint_error_dev(dev, 1260 "unable to establish link interrupt at %s\n", intrstr); 1261 return ENOBUFS; 1262 } 1263 aprint_normal_dev(dev, "link interrupting at %s\n", intrstr); 1264 /* use later in igc_configure_queues() */ 1265 sc->linkvec = iq; 1266 1267 return 0; 1268 } 1269 1270 static int 1271 igc_setup_msi(struct igc_softc *sc) 1272 { 1273 pci_chipset_tag_t pc = sc->osdep.os_pa.pa_pc; 1274 device_t dev = sc->sc_dev; 1275 pci_intr_handle_t *intr = sc->sc_intrs; 1276 void **ihs = sc->sc_ihs; 1277 const char *intrstr; 1278 char intrbuf[PCI_INTRSTR_LEN]; 1279 char xnamebuf[MAX(32, MAXCOMLEN)]; 1280 int error; 1281 1282 intrstr = pci_intr_string(pc, *intr, intrbuf, sizeof(intrbuf)); 1283 1284 snprintf(xnamebuf, sizeof(xnamebuf), "%s: msi", device_xname(dev)); 1285 pci_intr_setattr(pc, intr, PCI_INTR_MPSAFE, true); 1286 *ihs = pci_intr_establish_xname(pc, *intr, IPL_NET, 1287 igc_intr, sc, xnamebuf); 1288 if (*ihs == NULL) { 1289 aprint_error_dev(dev, 1290 "unable to establish interrupt at %s\n", intrstr); 1291 return ENOBUFS; 1292 } 1293 aprint_normal_dev(dev, "interrupting at %s\n", intrstr); 1294 1295 struct igc_queue *iq = sc->queues; 1296 iq->igcq_si = softint_establish(SOFTINT_NET | SOFTINT_MPSAFE, 1297 igc_handle_queue, iq); 1298 if (iq->igcq_si == NULL) { 1299 aprint_error_dev(dev, 1300 "%s: unable to establish softint\n", intrstr); 1301 return ENOBUFS; 1302 } 1303 1304 snprintf(xnamebuf, MAXCOMLEN, "%s_tx_rx", device_xname(dev)); 1305 error = workqueue_create(&sc->sc_queue_wq, xnamebuf, 1306 igc_handle_queue_work, sc, IGC_WORKQUEUE_PRI, IPL_NET, 1307 WQ_PERCPU | WQ_MPSAFE); 1308 if (error) { 1309 aprint_error_dev(dev, "workqueue_create failed\n"); 1310 return ENOBUFS; 1311 } 1312 sc->sc_txrx_workqueue = false; 1313 1314 sc->queues[0].msix = 0; 1315 sc->linkvec = 0; 1316 1317 return 0; 1318 } 1319 1320 static int 1321 igc_setup_intx(struct igc_softc *sc) 1322 { 1323 pci_chipset_tag_t pc = sc->osdep.os_pa.pa_pc; 1324 device_t dev = sc->sc_dev; 1325 pci_intr_handle_t *intr = sc->sc_intrs; 1326 void **ihs = sc->sc_ihs; 1327 const char *intrstr; 1328 char intrbuf[PCI_INTRSTR_LEN]; 1329 char xnamebuf[32]; 1330 1331 intrstr = pci_intr_string(pc, *intr, intrbuf, sizeof(intrbuf)); 1332 1333 snprintf(xnamebuf, sizeof(xnamebuf), "%s:intx", device_xname(dev)); 1334 pci_intr_setattr(pc, intr, PCI_INTR_MPSAFE, true); 1335 *ihs = pci_intr_establish_xname(pc, *intr, IPL_NET, 1336 igc_intr, sc, xnamebuf); 1337 if (*ihs == NULL) { 1338 aprint_error_dev(dev, 1339 "unable to establish interrupt at %s\n", intrstr); 1340 return ENOBUFS; 1341 } 1342 aprint_normal_dev(dev, "interrupting at %s\n", intrstr); 1343 1344 struct igc_queue *iq = sc->queues; 1345 iq->igcq_si = softint_establish(SOFTINT_NET | SOFTINT_MPSAFE, 1346 igc_handle_queue, iq); 1347 if (iq->igcq_si == NULL) { 1348 aprint_error_dev(dev, 1349 "%s: unable to establish softint\n", intrstr); 1350 return ENOBUFS; 1351 } 1352 1353 /* create workqueue? */ 1354 sc->sc_txrx_workqueue = false; 1355 1356 sc->queues[0].msix = 0; 1357 sc->linkvec = 0; 1358 1359 return 0; 1360 } 1361 1362 static int 1363 igc_dma_malloc(struct igc_softc *sc, bus_size_t size, struct igc_dma_alloc *dma) 1364 { 1365 struct igc_osdep *os = &sc->osdep; 1366 1367 dma->dma_tag = os->os_dmat; 1368 1369 if (bus_dmamap_create(dma->dma_tag, size, 1, size, 0, 1370 BUS_DMA_WAITOK | BUS_DMA_ALLOCNOW, &dma->dma_map)) 1371 return 1; 1372 if (bus_dmamem_alloc(dma->dma_tag, size, PAGE_SIZE, 0, &dma->dma_seg, 1373 1, &dma->dma_nseg, BUS_DMA_WAITOK)) 1374 goto destroy; 1375 /* 1376 * XXXRO 1377 * 1378 * Coherent mapping for descriptors is required for now. 1379 * 1380 * Both TX and RX descriptors are 16-byte length, which is shorter 1381 * than dcache lines on modern CPUs. Therefore, sync for a descriptor 1382 * may overwrite DMA read for descriptors in the same cache line. 1383 * 1384 * Can't we avoid this by use cache-line-aligned descriptors at once? 1385 */ 1386 if (bus_dmamem_map(dma->dma_tag, &dma->dma_seg, dma->dma_nseg, size, 1387 &dma->dma_vaddr, BUS_DMA_WAITOK | BUS_DMA_COHERENT /* XXXRO */)) 1388 goto free; 1389 if (bus_dmamap_load(dma->dma_tag, dma->dma_map, dma->dma_vaddr, size, 1390 NULL, BUS_DMA_WAITOK)) 1391 goto unmap; 1392 1393 dma->dma_size = size; 1394 1395 return 0; 1396 unmap: 1397 bus_dmamem_unmap(dma->dma_tag, dma->dma_vaddr, size); 1398 free: 1399 bus_dmamem_free(dma->dma_tag, &dma->dma_seg, dma->dma_nseg); 1400 destroy: 1401 bus_dmamap_destroy(dma->dma_tag, dma->dma_map); 1402 dma->dma_map = NULL; 1403 dma->dma_tag = NULL; 1404 return 1; 1405 } 1406 1407 static void 1408 igc_dma_free(struct igc_softc *sc, struct igc_dma_alloc *dma) 1409 { 1410 1411 if (dma->dma_tag == NULL) 1412 return; 1413 1414 if (dma->dma_map != NULL) { 1415 bus_dmamap_sync(dma->dma_tag, dma->dma_map, 0, 1416 dma->dma_map->dm_mapsize, 1417 BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE); 1418 bus_dmamap_unload(dma->dma_tag, dma->dma_map); 1419 bus_dmamem_unmap(dma->dma_tag, dma->dma_vaddr, dma->dma_size); 1420 bus_dmamem_free(dma->dma_tag, &dma->dma_seg, dma->dma_nseg); 1421 bus_dmamap_destroy(dma->dma_tag, dma->dma_map); 1422 dma->dma_map = NULL; 1423 } 1424 } 1425 1426 /********************************************************************* 1427 * 1428 * Setup networking device structure and register an interface. 1429 * 1430 **********************************************************************/ 1431 static void 1432 igc_setup_interface(struct igc_softc *sc) 1433 { 1434 struct ifnet *ifp = &sc->sc_ec.ec_if; 1435 1436 strlcpy(ifp->if_xname, device_xname(sc->sc_dev), sizeof(ifp->if_xname)); 1437 ifp->if_softc = sc; 1438 ifp->if_flags = IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST; 1439 ifp->if_extflags = IFEF_MPSAFE; 1440 ifp->if_ioctl = igc_ioctl; 1441 ifp->if_start = igc_start; 1442 if (sc->sc_nqueues > 1) 1443 ifp->if_transmit = igc_transmit; 1444 ifp->if_watchdog = igc_watchdog; 1445 ifp->if_init = igc_init; 1446 ifp->if_stop = igc_stop; 1447 1448 ifp->if_capabilities = IFCAP_TSOv4 | IFCAP_TSOv6; 1449 1450 ifp->if_capabilities |= 1451 IFCAP_CSUM_IPv4_Tx | IFCAP_CSUM_IPv4_Rx | 1452 IFCAP_CSUM_TCPv4_Tx | IFCAP_CSUM_TCPv4_Rx | 1453 IFCAP_CSUM_UDPv4_Tx | IFCAP_CSUM_UDPv4_Rx | 1454 IFCAP_CSUM_TCPv6_Tx | IFCAP_CSUM_TCPv6_Rx | 1455 IFCAP_CSUM_UDPv6_Tx | IFCAP_CSUM_UDPv6_Rx; 1456 1457 ifp->if_capenable = 0; 1458 1459 sc->sc_ec.ec_capabilities |= 1460 ETHERCAP_JUMBO_MTU | ETHERCAP_VLAN_MTU; 1461 1462 IFQ_SET_MAXLEN(&ifp->if_snd, sc->num_tx_desc - 1); 1463 IFQ_SET_READY(&ifp->if_snd); 1464 1465 #if NVLAN > 0 1466 sc->sc_ec.ec_capabilities |= ETHERCAP_VLAN_HWTAGGING; 1467 #endif 1468 1469 mutex_init(&sc->sc_core_lock, MUTEX_DEFAULT, IPL_NET); 1470 1471 /* Initialize ifmedia structures. */ 1472 sc->sc_ec.ec_ifmedia = &sc->media; 1473 ifmedia_init_with_lock(&sc->media, IFM_IMASK, igc_media_change, 1474 igc_media_status, &sc->sc_core_lock); 1475 ifmedia_add(&sc->media, IFM_ETHER | IFM_10_T, 0, NULL); 1476 ifmedia_add(&sc->media, IFM_ETHER | IFM_10_T | IFM_FDX, 0, NULL); 1477 ifmedia_add(&sc->media, IFM_ETHER | IFM_100_TX, 0, NULL); 1478 ifmedia_add(&sc->media, IFM_ETHER | IFM_100_TX | IFM_FDX, 0, NULL); 1479 ifmedia_add(&sc->media, IFM_ETHER | IFM_1000_T | IFM_FDX, 0, NULL); 1480 ifmedia_add(&sc->media, IFM_ETHER | IFM_2500_T | IFM_FDX, 0, NULL); 1481 ifmedia_add(&sc->media, IFM_ETHER | IFM_AUTO, 0, NULL); 1482 ifmedia_set(&sc->media, IFM_ETHER | IFM_AUTO); 1483 1484 sc->sc_rx_intr_process_limit = IGC_RX_INTR_PROCESS_LIMIT_DEFAULT; 1485 sc->sc_tx_intr_process_limit = IGC_TX_INTR_PROCESS_LIMIT_DEFAULT; 1486 sc->sc_rx_process_limit = IGC_RX_PROCESS_LIMIT_DEFAULT; 1487 sc->sc_tx_process_limit = IGC_TX_PROCESS_LIMIT_DEFAULT; 1488 1489 if_initialize(ifp); 1490 sc->sc_ipq = if_percpuq_create(ifp); 1491 if_deferred_start_init(ifp, NULL); 1492 ether_ifattach(ifp, sc->hw.mac.addr); 1493 ether_set_ifflags_cb(&sc->sc_ec, igc_ifflags_cb); 1494 if_register(ifp); 1495 } 1496 1497 static int 1498 igc_init(struct ifnet *ifp) 1499 { 1500 struct igc_softc *sc = ifp->if_softc; 1501 int error; 1502 1503 mutex_enter(&sc->sc_core_lock); 1504 error = igc_init_locked(sc); 1505 mutex_exit(&sc->sc_core_lock); 1506 1507 return error; 1508 } 1509 1510 static int 1511 igc_init_locked(struct igc_softc *sc) 1512 { 1513 struct ethercom *ec = &sc->sc_ec; 1514 struct ifnet *ifp = &ec->ec_if; 1515 1516 DPRINTF(CFG, "called\n"); 1517 1518 KASSERT(mutex_owned(&sc->sc_core_lock)); 1519 1520 if (ISSET(ifp->if_flags, IFF_RUNNING)) 1521 igc_stop_locked(sc); 1522 1523 /* Put the address into the receive address array. */ 1524 igc_rar_set(&sc->hw, sc->hw.mac.addr, 0); 1525 1526 /* Initialize the hardware. */ 1527 igc_reset(sc); 1528 igc_update_link_status(sc); 1529 1530 /* Setup VLAN support, basic and offload if available. */ 1531 IGC_WRITE_REG(&sc->hw, IGC_VET, ETHERTYPE_VLAN); 1532 1533 igc_initialize_transmit_unit(sc); 1534 igc_initialize_receive_unit(sc); 1535 1536 if (ec->ec_capenable & ETHERCAP_VLAN_HWTAGGING) { 1537 uint32_t ctrl = IGC_READ_REG(&sc->hw, IGC_CTRL); 1538 ctrl |= IGC_CTRL_VME; 1539 IGC_WRITE_REG(&sc->hw, IGC_CTRL, ctrl); 1540 } 1541 1542 /* Setup multicast table. */ 1543 igc_set_filter(sc); 1544 1545 igc_clear_hw_cntrs_base_generic(&sc->hw); 1546 1547 if (sc->sc_intr_type == PCI_INTR_TYPE_MSIX) 1548 igc_configure_queues(sc); 1549 1550 /* This clears any pending interrupts */ 1551 IGC_READ_REG(&sc->hw, IGC_ICR); 1552 IGC_WRITE_REG(&sc->hw, IGC_ICS, IGC_ICS_LSC); 1553 1554 /* The driver can now take control from firmware. */ 1555 igc_get_hw_control(sc); 1556 1557 /* Set Energy Efficient Ethernet. */ 1558 igc_set_eee_i225(&sc->hw, true, true, true); 1559 1560 for (int iq = 0; iq < sc->sc_nqueues; iq++) { 1561 struct rx_ring *rxr = &sc->rx_rings[iq]; 1562 1563 mutex_enter(&rxr->rxr_lock); 1564 igc_rxfill(rxr); 1565 mutex_exit(&rxr->rxr_lock); 1566 } 1567 1568 sc->sc_core_stopping = false; 1569 1570 ifp->if_flags |= IFF_RUNNING; 1571 1572 /* Save last flags for the callback */ 1573 sc->sc_if_flags = ifp->if_flags; 1574 1575 callout_schedule(&sc->sc_tick_ch, hz); 1576 1577 igc_enable_intr(sc); 1578 1579 return 0; 1580 } 1581 1582 static inline int 1583 igc_load_mbuf(struct igc_queue *q, bus_dma_tag_t dmat, bus_dmamap_t map, 1584 struct mbuf *m) 1585 { 1586 int error; 1587 1588 error = bus_dmamap_load_mbuf(dmat, map, m, 1589 BUS_DMA_WRITE | BUS_DMA_NOWAIT); 1590 1591 if (__predict_false(error == EFBIG)) { 1592 IGC_DRIVER_EVENT(q, txdma_efbig, 1); 1593 m = m_defrag(m, M_NOWAIT); 1594 if (__predict_false(m == NULL)) { 1595 IGC_DRIVER_EVENT(q, txdma_defrag, 1); 1596 return ENOBUFS; 1597 } 1598 error = bus_dmamap_load_mbuf(dmat, map, m, 1599 BUS_DMA_WRITE | BUS_DMA_NOWAIT); 1600 } 1601 1602 switch (error) { 1603 case 0: 1604 break; 1605 case ENOMEM: 1606 IGC_DRIVER_EVENT(q, txdma_enomem, 1); 1607 break; 1608 case EINVAL: 1609 IGC_DRIVER_EVENT(q, txdma_einval, 1); 1610 break; 1611 case EAGAIN: 1612 IGC_DRIVER_EVENT(q, txdma_eagain, 1); 1613 break; 1614 default: 1615 IGC_DRIVER_EVENT(q, txdma_other, 1); 1616 break; 1617 } 1618 1619 return error; 1620 } 1621 1622 #define IGC_TX_START 1 1623 #define IGC_TX_TRANSMIT 2 1624 1625 static void 1626 igc_start(struct ifnet *ifp) 1627 { 1628 struct igc_softc *sc = ifp->if_softc; 1629 1630 if (__predict_false(!sc->link_active)) { 1631 IFQ_PURGE(&ifp->if_snd); 1632 return; 1633 } 1634 1635 struct tx_ring *txr = &sc->tx_rings[0]; /* queue 0 */ 1636 mutex_enter(&txr->txr_lock); 1637 igc_tx_common_locked(ifp, txr, IGC_TX_START); 1638 mutex_exit(&txr->txr_lock); 1639 } 1640 1641 static inline u_int 1642 igc_select_txqueue(struct igc_softc *sc, struct mbuf *m __unused) 1643 { 1644 const u_int cpuid = cpu_index(curcpu()); 1645 1646 return cpuid % sc->sc_nqueues; 1647 } 1648 1649 static int 1650 igc_transmit(struct ifnet *ifp, struct mbuf *m) 1651 { 1652 struct igc_softc *sc = ifp->if_softc; 1653 const u_int qid = igc_select_txqueue(sc, m); 1654 struct tx_ring *txr = &sc->tx_rings[qid]; 1655 struct igc_queue *q = txr->txr_igcq; 1656 1657 if (__predict_false(!pcq_put(txr->txr_interq, m))) { 1658 IGC_QUEUE_EVENT(q, tx_pcq_drop, 1); 1659 m_freem(m); 1660 return ENOBUFS; 1661 } 1662 1663 mutex_enter(&txr->txr_lock); 1664 igc_tx_common_locked(ifp, txr, IGC_TX_TRANSMIT); 1665 mutex_exit(&txr->txr_lock); 1666 1667 return 0; 1668 } 1669 1670 static void 1671 igc_tx_common_locked(struct ifnet *ifp, struct tx_ring *txr, int caller) 1672 { 1673 struct igc_softc *sc = ifp->if_softc; 1674 struct igc_queue *q = txr->txr_igcq; 1675 net_stat_ref_t nsr = IF_STAT_GETREF(ifp); 1676 int prod, free, last = -1; 1677 bool post = false; 1678 1679 prod = txr->next_avail_desc; 1680 free = txr->next_to_clean; 1681 if (free <= prod) 1682 free += sc->num_tx_desc; 1683 free -= prod; 1684 1685 DPRINTF(TX, "%s: begin: msix %d prod %d n2c %d free %d\n", 1686 caller == IGC_TX_TRANSMIT ? "transmit" : "start", 1687 txr->me, prod, txr->next_to_clean, free); 1688 1689 for (;;) { 1690 struct mbuf *m; 1691 1692 if (__predict_false(free <= IGC_MAX_SCATTER)) { 1693 IGC_QUEUE_EVENT(q, tx_no_desc, 1); 1694 break; 1695 } 1696 1697 if (caller == IGC_TX_TRANSMIT) 1698 m = pcq_get(txr->txr_interq); 1699 else 1700 IFQ_DEQUEUE(&ifp->if_snd, m); 1701 if (__predict_false(m == NULL)) 1702 break; 1703 1704 struct igc_tx_buf *txbuf = &txr->tx_buffers[prod]; 1705 bus_dmamap_t map = txbuf->map; 1706 1707 if (__predict_false( 1708 igc_load_mbuf(q, txr->txdma.dma_tag, map, m))) { 1709 if (caller == IGC_TX_TRANSMIT) 1710 IGC_QUEUE_EVENT(q, tx_pcq_drop, 1); 1711 m_freem(m); 1712 if_statinc_ref(nsr, if_oerrors); 1713 continue; 1714 } 1715 1716 uint32_t ctx_cmd_type_len = 0, olinfo_status = 0; 1717 if (igc_tx_ctx_setup(txr, m, prod, &ctx_cmd_type_len, 1718 &olinfo_status)) { 1719 IGC_QUEUE_EVENT(q, tx_ctx, 1); 1720 /* Consume the first descriptor */ 1721 prod = igc_txdesc_incr(sc, prod); 1722 free--; 1723 } 1724 1725 bus_dmamap_sync(txr->txdma.dma_tag, map, 0, 1726 map->dm_mapsize, BUS_DMASYNC_PREWRITE); 1727 1728 for (int i = 0; i < map->dm_nsegs; i++) { 1729 union igc_adv_tx_desc *txdesc = &txr->tx_base[prod]; 1730 1731 uint32_t cmd_type_len = ctx_cmd_type_len | 1732 IGC_ADVTXD_DCMD_IFCS | IGC_ADVTXD_DTYP_DATA | 1733 IGC_ADVTXD_DCMD_DEXT | map->dm_segs[i].ds_len; 1734 if (i == map->dm_nsegs - 1) { 1735 cmd_type_len |= 1736 IGC_ADVTXD_DCMD_EOP | IGC_ADVTXD_DCMD_RS; 1737 } 1738 1739 igc_txdesc_sync(txr, prod, 1740 BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE); 1741 htolem64(&txdesc->read.buffer_addr, 1742 map->dm_segs[i].ds_addr); 1743 htolem32(&txdesc->read.cmd_type_len, cmd_type_len); 1744 htolem32(&txdesc->read.olinfo_status, olinfo_status); 1745 igc_txdesc_sync(txr, prod, 1746 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); 1747 1748 last = prod; 1749 prod = igc_txdesc_incr(sc, prod); 1750 } 1751 1752 txbuf->m_head = m; 1753 txbuf->eop_index = last; 1754 1755 bpf_mtap(ifp, m, BPF_D_OUT); 1756 1757 if_statadd_ref(nsr, if_obytes, m->m_pkthdr.len); 1758 if (m->m_flags & M_MCAST) 1759 if_statinc_ref(nsr, if_omcasts); 1760 IGC_QUEUE_EVENT(q, tx_packets, 1); 1761 IGC_QUEUE_EVENT(q, tx_bytes, m->m_pkthdr.len); 1762 1763 free -= map->dm_nsegs; 1764 post = true; 1765 } 1766 1767 if (post) { 1768 txr->next_avail_desc = prod; 1769 IGC_WRITE_REG(&sc->hw, IGC_TDT(txr->me), prod); 1770 } 1771 1772 DPRINTF(TX, "%s: done : msix %d prod %d n2c %d free %d\n", 1773 caller == IGC_TX_TRANSMIT ? "transmit" : "start", 1774 txr->me, prod, txr->next_to_clean, free); 1775 1776 IF_STAT_PUTREF(ifp); 1777 } 1778 1779 static bool 1780 igc_txeof(struct tx_ring *txr, u_int limit) 1781 { 1782 struct igc_softc *sc = txr->sc; 1783 struct ifnet *ifp = &sc->sc_ec.ec_if; 1784 int cons, prod; 1785 bool more = false; 1786 1787 prod = txr->next_avail_desc; 1788 cons = txr->next_to_clean; 1789 1790 if (cons == prod) { 1791 DPRINTF(TX, "false: msix %d cons %d prod %d\n", 1792 txr->me, cons, prod); 1793 return false; 1794 } 1795 1796 do { 1797 struct igc_tx_buf *txbuf = &txr->tx_buffers[cons]; 1798 const int last = txbuf->eop_index; 1799 1800 membar_consumer(); /* XXXRO necessary? */ 1801 1802 KASSERT(last != -1); 1803 union igc_adv_tx_desc *txdesc = &txr->tx_base[last]; 1804 igc_txdesc_sync(txr, last, BUS_DMASYNC_POSTREAD); 1805 const uint32_t status = le32toh(txdesc->wb.status); 1806 igc_txdesc_sync(txr, last, BUS_DMASYNC_PREREAD); 1807 1808 if (!(status & IGC_TXD_STAT_DD)) 1809 break; 1810 1811 if (limit-- == 0) { 1812 more = true; 1813 DPRINTF(TX, "pending TX " 1814 "msix %d cons %d last %d prod %d " 1815 "status 0x%08x\n", 1816 txr->me, cons, last, prod, status); 1817 break; 1818 } 1819 1820 DPRINTF(TX, "handled TX " 1821 "msix %d cons %d last %d prod %d " 1822 "status 0x%08x\n", 1823 txr->me, cons, last, prod, status); 1824 1825 if_statinc(ifp, if_opackets); 1826 1827 bus_dmamap_t map = txbuf->map; 1828 bus_dmamap_sync(txr->txdma.dma_tag, map, 0, map->dm_mapsize, 1829 BUS_DMASYNC_POSTWRITE); 1830 bus_dmamap_unload(txr->txdma.dma_tag, map); 1831 m_freem(txbuf->m_head); 1832 1833 txbuf->m_head = NULL; 1834 txbuf->eop_index = -1; 1835 1836 cons = igc_txdesc_incr(sc, last); 1837 } while (cons != prod); 1838 1839 txr->next_to_clean = cons; 1840 1841 return more; 1842 } 1843 1844 static void 1845 igc_intr_barrier(struct igc_softc *sc __unused) 1846 { 1847 1848 xc_barrier(0); 1849 } 1850 1851 static void 1852 igc_stop(struct ifnet *ifp, int disable) 1853 { 1854 struct igc_softc *sc = ifp->if_softc; 1855 1856 mutex_enter(&sc->sc_core_lock); 1857 igc_stop_locked(sc); 1858 mutex_exit(&sc->sc_core_lock); 1859 } 1860 1861 /********************************************************************* 1862 * 1863 * This routine disables all traffic on the adapter by issuing a 1864 * global reset on the MAC. 1865 * 1866 **********************************************************************/ 1867 static void 1868 igc_stop_locked(struct igc_softc *sc) 1869 { 1870 struct ifnet *ifp = &sc->sc_ec.ec_if; 1871 1872 DPRINTF(CFG, "called\n"); 1873 1874 KASSERT(mutex_owned(&sc->sc_core_lock)); 1875 1876 /* 1877 * If stopping processing has already started, do nothing. 1878 */ 1879 if ((ifp->if_flags & IFF_RUNNING) == 0) 1880 return; 1881 1882 /* Tell the stack that the interface is no longer active. */ 1883 ifp->if_flags &= ~IFF_RUNNING; 1884 1885 /* 1886 * igc_handle_queue() can enable interrupts, so wait for completion of 1887 * last igc_handle_queue() after unset IFF_RUNNING. 1888 */ 1889 mutex_exit(&sc->sc_core_lock); 1890 igc_barrier_handle_queue(sc); 1891 mutex_enter(&sc->sc_core_lock); 1892 1893 sc->sc_core_stopping = true; 1894 1895 igc_disable_intr(sc); 1896 1897 callout_halt(&sc->sc_tick_ch, &sc->sc_core_lock); 1898 1899 igc_reset_hw(&sc->hw); 1900 IGC_WRITE_REG(&sc->hw, IGC_WUC, 0); 1901 1902 /* 1903 * Wait for completion of interrupt handlers. 1904 */ 1905 mutex_exit(&sc->sc_core_lock); 1906 igc_intr_barrier(sc); 1907 mutex_enter(&sc->sc_core_lock); 1908 1909 igc_update_link_status(sc); 1910 1911 for (int iq = 0; iq < sc->sc_nqueues; iq++) { 1912 struct tx_ring *txr = &sc->tx_rings[iq]; 1913 1914 igc_withdraw_transmit_packets(txr, false); 1915 } 1916 1917 for (int iq = 0; iq < sc->sc_nqueues; iq++) { 1918 struct rx_ring *rxr = &sc->rx_rings[iq]; 1919 1920 igc_clear_receive_status(rxr); 1921 } 1922 1923 /* Save last flags for the callback */ 1924 sc->sc_if_flags = ifp->if_flags; 1925 } 1926 1927 /********************************************************************* 1928 * Ioctl entry point 1929 * 1930 * igc_ioctl is called when the user wants to configure the 1931 * interface. 1932 * 1933 * return 0 on success, positive on failure 1934 **********************************************************************/ 1935 static int 1936 igc_ioctl(struct ifnet * ifp, u_long cmd, void *data) 1937 { 1938 struct igc_softc *sc __unused = ifp->if_softc; 1939 int s; 1940 int error; 1941 1942 DPRINTF(CFG, "cmd 0x%016lx\n", cmd); 1943 1944 switch (cmd) { 1945 case SIOCADDMULTI: 1946 case SIOCDELMULTI: 1947 break; 1948 default: 1949 KASSERT(IFNET_LOCKED(ifp)); 1950 } 1951 1952 if (cmd == SIOCZIFDATA) { 1953 mutex_enter(&sc->sc_core_lock); 1954 igc_clear_counters(sc); 1955 mutex_exit(&sc->sc_core_lock); 1956 } 1957 1958 switch (cmd) { 1959 #ifdef IF_RXR 1960 case SIOCGIFRXR: 1961 s = splnet(); 1962 error = igc_rxrinfo(sc, (struct if_rxrinfo *)ifr->ifr_data); 1963 splx(s); 1964 break; 1965 #endif 1966 default: 1967 s = splnet(); 1968 error = ether_ioctl(ifp, cmd, data); 1969 splx(s); 1970 break; 1971 } 1972 1973 if (error != ENETRESET) 1974 return error; 1975 1976 error = 0; 1977 1978 if (cmd == SIOCSIFCAP) 1979 error = if_init(ifp); 1980 else if ((cmd == SIOCADDMULTI) || (cmd == SIOCDELMULTI)) { 1981 mutex_enter(&sc->sc_core_lock); 1982 if (sc->sc_if_flags & IFF_RUNNING) { 1983 /* 1984 * Multicast list has changed; set the hardware filter 1985 * accordingly. 1986 */ 1987 igc_disable_intr(sc); 1988 igc_set_filter(sc); 1989 igc_enable_intr(sc); 1990 } 1991 mutex_exit(&sc->sc_core_lock); 1992 } 1993 1994 return error; 1995 } 1996 1997 #ifdef IF_RXR 1998 static int 1999 igc_rxrinfo(struct igc_softc *sc, struct if_rxrinfo *ifri) 2000 { 2001 struct if_rxring_info *ifr, ifr1; 2002 int error; 2003 2004 if (sc->sc_nqueues > 1) { 2005 ifr = kmem_zalloc(sc->sc_nqueues * sizeof(*ifr), KM_SLEEP); 2006 } else { 2007 ifr = &ifr1; 2008 memset(ifr, 0, sizeof(*ifr)); 2009 } 2010 2011 for (int iq = 0; iq < sc->sc_nqueues; iq++) { 2012 struct rx_ring *rxr = &sc->rx_rings[iq]; 2013 2014 ifr[iq].ifr_size = MCLBYTES; 2015 snprintf(ifr[iq].ifr_name, sizeof(ifr[iq].ifr_name), "%d", iq); 2016 ifr[iq].ifr_info = rxr->rx_ring; 2017 } 2018 2019 error = if_rxr_info_ioctl(ifri, sc->sc_nqueues, ifr); 2020 if (sc->sc_nqueues > 1) 2021 kmem_free(ifr, sc->sc_nqueues * sizeof(*ifr)); 2022 2023 return error; 2024 } 2025 #endif 2026 2027 static void 2028 igc_rxfill(struct rx_ring *rxr) 2029 { 2030 struct igc_softc *sc = rxr->sc; 2031 int id; 2032 2033 for (id = 0; id < sc->num_rx_desc; id++) { 2034 if (igc_get_buf(rxr, id, false)) { 2035 panic("%s: msix=%d i=%d\n", __func__, rxr->me, id); 2036 } 2037 } 2038 2039 id = sc->num_rx_desc - 1; 2040 rxr->last_desc_filled = id; 2041 IGC_WRITE_REG(&sc->hw, IGC_RDT(rxr->me), id); 2042 rxr->next_to_check = 0; 2043 } 2044 2045 static void 2046 igc_rxrefill(struct rx_ring *rxr, int end) 2047 { 2048 struct igc_softc *sc = rxr->sc; 2049 int id; 2050 2051 for (id = rxr->next_to_check; id != end; id = igc_rxdesc_incr(sc, id)) { 2052 if (igc_get_buf(rxr, id, true)) { 2053 /* XXXRO */ 2054 panic("%s: msix=%d id=%d\n", __func__, rxr->me, id); 2055 } 2056 } 2057 2058 id = igc_rxdesc_decr(sc, id); 2059 DPRINTF(RX, "%s RDT %d id %d\n", 2060 rxr->last_desc_filled == id ? "same" : "diff", 2061 rxr->last_desc_filled, id); 2062 rxr->last_desc_filled = id; 2063 IGC_WRITE_REG(&sc->hw, IGC_RDT(rxr->me), id); 2064 } 2065 2066 /********************************************************************* 2067 * 2068 * This routine executes in interrupt context. It replenishes 2069 * the mbufs in the descriptor and sends data which has been 2070 * dma'ed into host memory to upper layer. 2071 * 2072 *********************************************************************/ 2073 static bool 2074 igc_rxeof(struct rx_ring *rxr, u_int limit) 2075 { 2076 struct igc_softc *sc = rxr->sc; 2077 struct igc_queue *q = rxr->rxr_igcq; 2078 struct ifnet *ifp = &sc->sc_ec.ec_if; 2079 int id; 2080 bool more = false; 2081 2082 id = rxr->next_to_check; 2083 for (;;) { 2084 union igc_adv_rx_desc *rxdesc = &rxr->rx_base[id]; 2085 struct igc_rx_buf *rxbuf, *nxbuf; 2086 struct mbuf *mp, *m; 2087 2088 igc_rxdesc_sync(rxr, id, 2089 BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE); 2090 2091 const uint32_t staterr = le32toh(rxdesc->wb.upper.status_error); 2092 2093 if (!ISSET(staterr, IGC_RXD_STAT_DD)) { 2094 igc_rxdesc_sync(rxr, id, 2095 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); 2096 break; 2097 } 2098 2099 if (limit-- == 0) { 2100 igc_rxdesc_sync(rxr, id, 2101 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); 2102 DPRINTF(RX, "more=true\n"); 2103 more = true; 2104 break; 2105 } 2106 2107 /* Zero out the receive descriptors status. */ 2108 rxdesc->wb.upper.status_error = 0; 2109 2110 /* Pull the mbuf off the ring. */ 2111 rxbuf = &rxr->rx_buffers[id]; 2112 bus_dmamap_t map = rxbuf->map; 2113 bus_dmamap_sync(rxr->rxdma.dma_tag, map, 2114 0, map->dm_mapsize, BUS_DMASYNC_POSTREAD); 2115 bus_dmamap_unload(rxr->rxdma.dma_tag, map); 2116 2117 mp = rxbuf->buf; 2118 rxbuf->buf = NULL; 2119 2120 const bool eop = staterr & IGC_RXD_STAT_EOP; 2121 const uint16_t len = le16toh(rxdesc->wb.upper.length); 2122 2123 #if NVLAN > 0 2124 const uint16_t vtag = le16toh(rxdesc->wb.upper.vlan); 2125 #endif 2126 2127 const uint32_t ptype = le32toh(rxdesc->wb.lower.lo_dword.data) & 2128 IGC_PKTTYPE_MASK; 2129 2130 const uint32_t hash __unused = 2131 le32toh(rxdesc->wb.lower.hi_dword.rss); 2132 const uint16_t hashtype __unused = 2133 le16toh(rxdesc->wb.lower.lo_dword.hs_rss.pkt_info) & 2134 IGC_RXDADV_RSSTYPE_MASK; 2135 2136 igc_rxdesc_sync(rxr, id, 2137 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); 2138 2139 if (__predict_false(staterr & IGC_RXDEXT_STATERR_RXE)) { 2140 if (rxbuf->fmp) { 2141 m_freem(rxbuf->fmp); 2142 rxbuf->fmp = NULL; 2143 } 2144 2145 m_freem(mp); 2146 m = NULL; 2147 2148 if_statinc(ifp, if_ierrors); 2149 IGC_QUEUE_EVENT(q, rx_discard, 1); 2150 2151 DPRINTF(RX, "ierrors++\n"); 2152 2153 goto next_desc; 2154 } 2155 2156 if (__predict_false(mp == NULL)) { 2157 panic("%s: igc_rxeof: NULL mbuf in slot %d " 2158 "(filled %d)", device_xname(sc->sc_dev), 2159 id, rxr->last_desc_filled); 2160 } 2161 2162 if (!eop) { 2163 /* 2164 * Figure out the next descriptor of this frame. 2165 */ 2166 int nextp = igc_rxdesc_incr(sc, id); 2167 2168 nxbuf = &rxr->rx_buffers[nextp]; 2169 /* 2170 * TODO prefetch(nxbuf); 2171 */ 2172 } 2173 2174 mp->m_len = len; 2175 2176 m = rxbuf->fmp; 2177 rxbuf->fmp = NULL; 2178 2179 if (m != NULL) { 2180 m->m_pkthdr.len += mp->m_len; 2181 } else { 2182 m = mp; 2183 m->m_pkthdr.len = mp->m_len; 2184 #if NVLAN > 0 2185 if (staterr & IGC_RXD_STAT_VP) 2186 vlan_set_tag(m, vtag); 2187 #endif 2188 } 2189 2190 /* Pass the head pointer on */ 2191 if (!eop) { 2192 nxbuf->fmp = m; 2193 m = NULL; 2194 mp->m_next = nxbuf->buf; 2195 } else { 2196 m_set_rcvif(m, ifp); 2197 2198 m->m_pkthdr.csum_flags = igc_rx_checksum(q, 2199 ifp->if_capenable, staterr, ptype); 2200 2201 #ifdef notyet 2202 if (hashtype != IGC_RXDADV_RSSTYPE_NONE) { 2203 m->m_pkthdr.ph_flowid = hash; 2204 SET(m->m_pkthdr.csum_flags, M_FLOWID); 2205 } 2206 ml_enqueue(&ml, m); 2207 #endif 2208 2209 if_percpuq_enqueue(sc->sc_ipq, m); 2210 2211 if_statinc(ifp, if_ipackets); 2212 IGC_QUEUE_EVENT(q, rx_packets, 1); 2213 IGC_QUEUE_EVENT(q, rx_bytes, m->m_pkthdr.len); 2214 } 2215 next_desc: 2216 /* Advance our pointers to the next descriptor. */ 2217 id = igc_rxdesc_incr(sc, id); 2218 } 2219 2220 DPRINTF(RX, "fill queue[%d]\n", rxr->me); 2221 igc_rxrefill(rxr, id); 2222 2223 DPRINTF(RX, "%s n2c %d id %d\n", 2224 rxr->next_to_check == id ? "same" : "diff", 2225 rxr->next_to_check, id); 2226 rxr->next_to_check = id; 2227 2228 #ifdef OPENBSD 2229 if (!(staterr & IGC_RXD_STAT_DD)) 2230 return 0; 2231 #endif 2232 2233 return more; 2234 } 2235 2236 /********************************************************************* 2237 * 2238 * Verify that the hardware indicated that the checksum is valid. 2239 * Inform the stack about the status of checksum so that stack 2240 * doesn't spend time verifying the checksum. 2241 * 2242 *********************************************************************/ 2243 static int 2244 igc_rx_checksum(struct igc_queue *q, uint64_t capenable, uint32_t staterr, 2245 uint32_t ptype) 2246 { 2247 const uint16_t status = (uint16_t)staterr; 2248 const uint8_t errors = (uint8_t)(staterr >> 24); 2249 int flags = 0; 2250 2251 if ((status & IGC_RXD_STAT_IPCS) != 0 && 2252 (capenable & IFCAP_CSUM_IPv4_Rx) != 0) { 2253 IGC_DRIVER_EVENT(q, rx_ipcs, 1); 2254 flags |= M_CSUM_IPv4; 2255 if (__predict_false((errors & IGC_RXD_ERR_IPE) != 0)) { 2256 IGC_DRIVER_EVENT(q, rx_ipcs_bad, 1); 2257 flags |= M_CSUM_IPv4_BAD; 2258 } 2259 } 2260 2261 if ((status & IGC_RXD_STAT_TCPCS) != 0) { 2262 IGC_DRIVER_EVENT(q, rx_tcpcs, 1); 2263 if ((capenable & IFCAP_CSUM_TCPv4_Rx) != 0) 2264 flags |= M_CSUM_TCPv4; 2265 if ((capenable & IFCAP_CSUM_TCPv6_Rx) != 0) 2266 flags |= M_CSUM_TCPv6; 2267 } 2268 2269 if ((status & IGC_RXD_STAT_UDPCS) != 0) { 2270 IGC_DRIVER_EVENT(q, rx_udpcs, 1); 2271 if ((capenable & IFCAP_CSUM_UDPv4_Rx) != 0) 2272 flags |= M_CSUM_UDPv4; 2273 if ((capenable & IFCAP_CSUM_UDPv6_Rx) != 0) 2274 flags |= M_CSUM_UDPv6; 2275 } 2276 2277 if (__predict_false((errors & IGC_RXD_ERR_TCPE) != 0)) { 2278 IGC_DRIVER_EVENT(q, rx_l4cs_bad, 1); 2279 if ((flags & ~M_CSUM_IPv4) != 0) 2280 flags |= M_CSUM_TCP_UDP_BAD; 2281 } 2282 2283 return flags; 2284 } 2285 2286 static void 2287 igc_watchdog(struct ifnet * ifp) 2288 { 2289 } 2290 2291 static void 2292 igc_tick(void *arg) 2293 { 2294 struct igc_softc *sc = arg; 2295 2296 mutex_enter(&sc->sc_core_lock); 2297 2298 if (__predict_false(sc->sc_core_stopping)) { 2299 mutex_exit(&sc->sc_core_lock); 2300 return; 2301 } 2302 2303 /* XXX watchdog */ 2304 if (0) { 2305 IGC_GLOBAL_EVENT(sc, watchdog, 1); 2306 } 2307 2308 igc_update_counters(sc); 2309 2310 mutex_exit(&sc->sc_core_lock); 2311 2312 callout_schedule(&sc->sc_tick_ch, hz); 2313 } 2314 2315 /********************************************************************* 2316 * 2317 * Media Ioctl callback 2318 * 2319 * This routine is called whenever the user queries the status of 2320 * the interface using ifconfig. 2321 * 2322 **********************************************************************/ 2323 static void 2324 igc_media_status(struct ifnet *ifp, struct ifmediareq *ifmr) 2325 { 2326 struct igc_softc *sc = ifp->if_softc; 2327 struct igc_hw *hw = &sc->hw; 2328 2329 igc_update_link_status(sc); 2330 2331 ifmr->ifm_status = IFM_AVALID; 2332 ifmr->ifm_active = IFM_ETHER; 2333 2334 if (!sc->link_active) { 2335 ifmr->ifm_active |= IFM_NONE; 2336 return; 2337 } 2338 2339 ifmr->ifm_status |= IFM_ACTIVE; 2340 2341 switch (sc->link_speed) { 2342 case 10: 2343 ifmr->ifm_active |= IFM_10_T; 2344 break; 2345 case 100: 2346 ifmr->ifm_active |= IFM_100_TX; 2347 break; 2348 case 1000: 2349 ifmr->ifm_active |= IFM_1000_T; 2350 break; 2351 case 2500: 2352 ifmr->ifm_active |= IFM_2500_T; 2353 break; 2354 } 2355 2356 if (sc->link_duplex == FULL_DUPLEX) 2357 ifmr->ifm_active |= IFM_FDX; 2358 else 2359 ifmr->ifm_active |= IFM_HDX; 2360 2361 switch (hw->fc.current_mode) { 2362 case igc_fc_tx_pause: 2363 ifmr->ifm_active |= IFM_FLOW | IFM_ETH_TXPAUSE; 2364 break; 2365 case igc_fc_rx_pause: 2366 ifmr->ifm_active |= IFM_FLOW | IFM_ETH_RXPAUSE; 2367 break; 2368 case igc_fc_full: 2369 ifmr->ifm_active |= IFM_FLOW | 2370 IFM_ETH_TXPAUSE | IFM_ETH_RXPAUSE; 2371 break; 2372 case igc_fc_none: 2373 default: 2374 break; 2375 } 2376 } 2377 2378 /********************************************************************* 2379 * 2380 * Media Ioctl callback 2381 * 2382 * This routine is called when the user changes speed/duplex using 2383 * media/mediopt option with ifconfig. 2384 * 2385 **********************************************************************/ 2386 static int 2387 igc_media_change(struct ifnet *ifp) 2388 { 2389 struct igc_softc *sc = ifp->if_softc; 2390 struct ifmedia *ifm = &sc->media; 2391 2392 if (IFM_TYPE(ifm->ifm_media) != IFM_ETHER) 2393 return EINVAL; 2394 2395 sc->hw.mac.autoneg = DO_AUTO_NEG; 2396 2397 switch (IFM_SUBTYPE(ifm->ifm_media)) { 2398 case IFM_AUTO: 2399 sc->hw.phy.autoneg_advertised = AUTONEG_ADV_DEFAULT; 2400 break; 2401 case IFM_2500_T: 2402 sc->hw.phy.autoneg_advertised = ADVERTISE_2500_FULL; 2403 break; 2404 case IFM_1000_T: 2405 sc->hw.phy.autoneg_advertised = ADVERTISE_1000_FULL; 2406 break; 2407 case IFM_100_TX: 2408 if ((ifm->ifm_media & IFM_GMASK) == IFM_FDX) 2409 sc->hw.phy.autoneg_advertised = ADVERTISE_100_FULL; 2410 else 2411 sc->hw.phy.autoneg_advertised = ADVERTISE_100_HALF; 2412 break; 2413 case IFM_10_T: 2414 if ((ifm->ifm_media & IFM_GMASK) == IFM_FDX) 2415 sc->hw.phy.autoneg_advertised = ADVERTISE_10_FULL; 2416 else 2417 sc->hw.phy.autoneg_advertised = ADVERTISE_10_HALF; 2418 break; 2419 default: 2420 return EINVAL; 2421 } 2422 2423 igc_init_locked(sc); 2424 2425 return 0; 2426 } 2427 2428 static int 2429 igc_ifflags_cb(struct ethercom *ec) 2430 { 2431 struct ifnet *ifp = &ec->ec_if; 2432 struct igc_softc *sc = ifp->if_softc; 2433 int rc = 0; 2434 u_short iffchange; 2435 bool needreset = false; 2436 2437 DPRINTF(CFG, "called\n"); 2438 2439 KASSERT(IFNET_LOCKED(ifp)); 2440 2441 mutex_enter(&sc->sc_core_lock); 2442 2443 /* 2444 * Check for if_flags. 2445 * Main usage is to prevent linkdown when opening bpf. 2446 */ 2447 iffchange = ifp->if_flags ^ sc->sc_if_flags; 2448 sc->sc_if_flags = ifp->if_flags; 2449 if ((iffchange & ~(IFF_CANTCHANGE | IFF_DEBUG)) != 0) { 2450 needreset = true; 2451 goto ec; 2452 } 2453 2454 /* iff related updates */ 2455 if ((iffchange & IFF_PROMISC) != 0) 2456 igc_set_filter(sc); 2457 2458 #ifdef notyet 2459 igc_set_vlan(sc); 2460 #endif 2461 2462 ec: 2463 #ifdef notyet 2464 /* Check for ec_capenable. */ 2465 ecchange = ec->ec_capenable ^ sc->sc_ec_capenable; 2466 sc->sc_ec_capenable = ec->ec_capenable; 2467 if ((ecchange & ~ETHERCAP_SOMETHING) != 0) { 2468 needreset = true; 2469 goto out; 2470 } 2471 #endif 2472 if (needreset) 2473 rc = ENETRESET; 2474 2475 mutex_exit(&sc->sc_core_lock); 2476 2477 return rc; 2478 } 2479 2480 static void 2481 igc_set_filter(struct igc_softc *sc) 2482 { 2483 struct ethercom *ec = &sc->sc_ec; 2484 uint32_t rctl; 2485 2486 rctl = IGC_READ_REG(&sc->hw, IGC_RCTL); 2487 rctl &= ~(IGC_RCTL_BAM |IGC_RCTL_UPE | IGC_RCTL_MPE); 2488 2489 if ((sc->sc_if_flags & IFF_BROADCAST) != 0) 2490 rctl |= IGC_RCTL_BAM; 2491 if ((sc->sc_if_flags & IFF_PROMISC) != 0) { 2492 DPRINTF(CFG, "promisc\n"); 2493 rctl |= IGC_RCTL_UPE; 2494 ETHER_LOCK(ec); 2495 allmulti: 2496 ec->ec_flags |= ETHER_F_ALLMULTI; 2497 ETHER_UNLOCK(ec); 2498 rctl |= IGC_RCTL_MPE; 2499 } else { 2500 struct ether_multistep step; 2501 struct ether_multi *enm; 2502 int mcnt = 0; 2503 2504 memset(sc->mta, 0, IGC_MTA_LEN); 2505 2506 ETHER_LOCK(ec); 2507 ETHER_FIRST_MULTI(step, ec, enm); 2508 while (enm != NULL) { 2509 if (((memcmp(enm->enm_addrlo, enm->enm_addrhi, 2510 ETHER_ADDR_LEN)) != 0) || 2511 (mcnt >= MAX_NUM_MULTICAST_ADDRESSES)) { 2512 /* 2513 * We must listen to a range of multicast 2514 * addresses. For now, just accept all 2515 * multicasts, rather than trying to set only 2516 * those filter bits needed to match the range. 2517 * (At this time, the only use of address 2518 * ranges is for IP multicast routing, for 2519 * which the range is big enough to require all 2520 * bits set.) 2521 */ 2522 goto allmulti; 2523 } 2524 DPRINTF(CFG, "%d: %s\n", mcnt, 2525 ether_sprintf(enm->enm_addrlo)); 2526 memcpy(&sc->mta[mcnt * ETHER_ADDR_LEN], 2527 enm->enm_addrlo, ETHER_ADDR_LEN); 2528 2529 mcnt++; 2530 ETHER_NEXT_MULTI(step, enm); 2531 } 2532 ec->ec_flags &= ~ETHER_F_ALLMULTI; 2533 ETHER_UNLOCK(ec); 2534 2535 DPRINTF(CFG, "hw filter\n"); 2536 igc_update_mc_addr_list(&sc->hw, sc->mta, mcnt); 2537 } 2538 2539 IGC_WRITE_REG(&sc->hw, IGC_RCTL, rctl); 2540 } 2541 2542 static void 2543 igc_update_link_status(struct igc_softc *sc) 2544 { 2545 struct ifnet *ifp = &sc->sc_ec.ec_if; 2546 struct igc_hw *hw = &sc->hw; 2547 2548 if (hw->mac.get_link_status == true) 2549 igc_check_for_link(hw); 2550 2551 if (IGC_READ_REG(&sc->hw, IGC_STATUS) & IGC_STATUS_LU) { 2552 if (sc->link_active == 0) { 2553 igc_get_speed_and_duplex(hw, &sc->link_speed, 2554 &sc->link_duplex); 2555 sc->link_active = 1; 2556 ifp->if_baudrate = IF_Mbps(sc->link_speed); 2557 if_link_state_change(ifp, LINK_STATE_UP); 2558 } 2559 } else { 2560 if (sc->link_active == 1) { 2561 ifp->if_baudrate = sc->link_speed = 0; 2562 sc->link_duplex = 0; 2563 sc->link_active = 0; 2564 if_link_state_change(ifp, LINK_STATE_DOWN); 2565 } 2566 } 2567 } 2568 2569 /********************************************************************* 2570 * 2571 * Get a buffer from system mbuf buffer pool. 2572 * 2573 **********************************************************************/ 2574 static int 2575 igc_get_buf(struct rx_ring *rxr, int id, bool strict) 2576 { 2577 struct igc_softc *sc = rxr->sc; 2578 struct igc_queue *q = rxr->rxr_igcq; 2579 struct igc_rx_buf *rxbuf = &rxr->rx_buffers[id]; 2580 bus_dmamap_t map = rxbuf->map; 2581 struct mbuf *m; 2582 int error; 2583 2584 if (__predict_false(rxbuf->buf)) { 2585 if (strict) { 2586 DPRINTF(RX, "slot %d already has an mbuf\n", id); 2587 return EINVAL; 2588 } 2589 return 0; 2590 } 2591 2592 MGETHDR(m, M_DONTWAIT, MT_DATA); 2593 if (__predict_false(m == NULL)) { 2594 enobuf: 2595 IGC_QUEUE_EVENT(q, rx_no_mbuf, 1); 2596 return ENOBUFS; 2597 } 2598 2599 MCLGET(m, M_DONTWAIT); 2600 if (__predict_false(!(m->m_flags & M_EXT))) { 2601 m_freem(m); 2602 goto enobuf; 2603 } 2604 2605 m->m_len = m->m_pkthdr.len = sc->rx_mbuf_sz; 2606 2607 error = bus_dmamap_load_mbuf(rxr->rxdma.dma_tag, map, m, 2608 BUS_DMA_READ | BUS_DMA_NOWAIT); 2609 if (error) { 2610 m_freem(m); 2611 return error; 2612 } 2613 2614 bus_dmamap_sync(rxr->rxdma.dma_tag, map, 0, 2615 map->dm_mapsize, BUS_DMASYNC_PREREAD); 2616 rxbuf->buf = m; 2617 2618 union igc_adv_rx_desc *rxdesc = &rxr->rx_base[id]; 2619 igc_rxdesc_sync(rxr, id, BUS_DMASYNC_POSTWRITE | BUS_DMASYNC_POSTREAD); 2620 rxdesc->read.pkt_addr = htole64(map->dm_segs[0].ds_addr); 2621 igc_rxdesc_sync(rxr, id, BUS_DMASYNC_PREWRITE | BUS_DMASYNC_PREREAD); 2622 2623 return 0; 2624 } 2625 2626 static void 2627 igc_configure_queues(struct igc_softc *sc) 2628 { 2629 struct igc_hw *hw = &sc->hw; 2630 uint32_t ivar; 2631 2632 /* First turn on RSS capability */ 2633 IGC_WRITE_REG(hw, IGC_GPIE, IGC_GPIE_MSIX_MODE | IGC_GPIE_EIAME | 2634 IGC_GPIE_PBA | IGC_GPIE_NSICR); 2635 2636 /* Set the starting interrupt rate */ 2637 uint32_t newitr = (4000000 / MAX_INTS_PER_SEC) & 0x7FFC; 2638 newitr |= IGC_EITR_CNT_IGNR; 2639 2640 /* Turn on MSI-X */ 2641 uint32_t newmask = 0; 2642 for (int iq = 0; iq < sc->sc_nqueues; iq++) { 2643 struct igc_queue *q = &sc->queues[iq]; 2644 2645 /* RX entries */ 2646 igc_set_queues(sc, iq, q->msix, 0); 2647 /* TX entries */ 2648 igc_set_queues(sc, iq, q->msix, 1); 2649 newmask |= q->eims; 2650 IGC_WRITE_REG(hw, IGC_EITR(q->msix), newitr); 2651 } 2652 sc->msix_queuesmask = newmask; 2653 2654 #if 1 2655 ivar = IGC_READ_REG_ARRAY(hw, IGC_IVAR0, 0); 2656 DPRINTF(CFG, "ivar(0)=0x%x\n", ivar); 2657 ivar = IGC_READ_REG_ARRAY(hw, IGC_IVAR0, 1); 2658 DPRINTF(CFG, "ivar(1)=0x%x\n", ivar); 2659 #endif 2660 2661 /* And for the link interrupt */ 2662 ivar = (sc->linkvec | IGC_IVAR_VALID) << 8; 2663 sc->msix_linkmask = 1 << sc->linkvec; 2664 IGC_WRITE_REG(hw, IGC_IVAR_MISC, ivar); 2665 } 2666 2667 static void 2668 igc_set_queues(struct igc_softc *sc, uint32_t entry, uint32_t vector, int type) 2669 { 2670 struct igc_hw *hw = &sc->hw; 2671 const uint32_t index = entry >> 1; 2672 uint32_t ivar = IGC_READ_REG_ARRAY(hw, IGC_IVAR0, index); 2673 2674 if (type) { 2675 if (entry & 1) { 2676 ivar &= 0x00FFFFFF; 2677 ivar |= (vector | IGC_IVAR_VALID) << 24; 2678 } else { 2679 ivar &= 0xFFFF00FF; 2680 ivar |= (vector | IGC_IVAR_VALID) << 8; 2681 } 2682 } else { 2683 if (entry & 1) { 2684 ivar &= 0xFF00FFFF; 2685 ivar |= (vector | IGC_IVAR_VALID) << 16; 2686 } else { 2687 ivar &= 0xFFFFFF00; 2688 ivar |= vector | IGC_IVAR_VALID; 2689 } 2690 } 2691 IGC_WRITE_REG_ARRAY(hw, IGC_IVAR0, index, ivar); 2692 } 2693 2694 static void 2695 igc_enable_queue(struct igc_softc *sc, uint32_t eims) 2696 { 2697 IGC_WRITE_REG(&sc->hw, IGC_EIMS, eims); 2698 } 2699 2700 static void 2701 igc_enable_intr(struct igc_softc *sc) 2702 { 2703 struct igc_hw *hw = &sc->hw; 2704 2705 if (sc->sc_intr_type == PCI_INTR_TYPE_MSIX) { 2706 const uint32_t mask = sc->msix_queuesmask | sc->msix_linkmask; 2707 2708 IGC_WRITE_REG(hw, IGC_EIAC, mask); 2709 IGC_WRITE_REG(hw, IGC_EIAM, mask); 2710 IGC_WRITE_REG(hw, IGC_EIMS, mask); 2711 IGC_WRITE_REG(hw, IGC_IMS, IGC_IMS_LSC); 2712 } else { 2713 IGC_WRITE_REG(hw, IGC_IMS, IMS_ENABLE_MASK); 2714 } 2715 IGC_WRITE_FLUSH(hw); 2716 } 2717 2718 static void 2719 igc_disable_intr(struct igc_softc *sc) 2720 { 2721 struct igc_hw *hw = &sc->hw; 2722 2723 if (sc->sc_intr_type == PCI_INTR_TYPE_MSIX) { 2724 IGC_WRITE_REG(hw, IGC_EIMC, 0xffffffff); 2725 IGC_WRITE_REG(hw, IGC_EIAC, 0); 2726 } 2727 IGC_WRITE_REG(hw, IGC_IMC, 0xffffffff); 2728 IGC_WRITE_FLUSH(hw); 2729 } 2730 2731 static int 2732 igc_intr_link(void *arg) 2733 { 2734 struct igc_softc *sc = (struct igc_softc *)arg; 2735 const uint32_t reg_icr = IGC_READ_REG(&sc->hw, IGC_ICR); 2736 2737 IGC_GLOBAL_EVENT(sc, link, 1); 2738 2739 if (reg_icr & IGC_ICR_LSC) { 2740 mutex_enter(&sc->sc_core_lock); 2741 sc->hw.mac.get_link_status = true; 2742 igc_update_link_status(sc); 2743 mutex_exit(&sc->sc_core_lock); 2744 } 2745 2746 IGC_WRITE_REG(&sc->hw, IGC_IMS, IGC_IMS_LSC); 2747 IGC_WRITE_REG(&sc->hw, IGC_EIMS, sc->msix_linkmask); 2748 2749 return 1; 2750 } 2751 2752 static int 2753 igc_intr_queue(void *arg) 2754 { 2755 struct igc_queue *iq = arg; 2756 struct igc_softc *sc = iq->sc; 2757 struct ifnet *ifp = &sc->sc_ec.ec_if; 2758 struct rx_ring *rxr = iq->rxr; 2759 struct tx_ring *txr = iq->txr; 2760 const u_int txlimit = sc->sc_tx_intr_process_limit, 2761 rxlimit = sc->sc_rx_intr_process_limit; 2762 bool txmore, rxmore; 2763 2764 IGC_QUEUE_EVENT(iq, irqs, 1); 2765 2766 if (__predict_false(!ISSET(ifp->if_flags, IFF_RUNNING))) 2767 return 0; 2768 2769 mutex_enter(&txr->txr_lock); 2770 txmore = igc_txeof(txr, txlimit); 2771 mutex_exit(&txr->txr_lock); 2772 mutex_enter(&rxr->rxr_lock); 2773 rxmore = igc_rxeof(rxr, rxlimit); 2774 mutex_exit(&rxr->rxr_lock); 2775 2776 if (txmore || rxmore) { 2777 IGC_QUEUE_EVENT(iq, req, 1); 2778 igc_sched_handle_queue(sc, iq); 2779 } else { 2780 igc_enable_queue(sc, iq->eims); 2781 } 2782 2783 return 1; 2784 } 2785 2786 static int 2787 igc_intr(void *arg) 2788 { 2789 struct igc_softc *sc = arg; 2790 struct ifnet *ifp = &sc->sc_ec.ec_if; 2791 struct igc_queue *iq = &sc->queues[0]; 2792 struct rx_ring *rxr = iq->rxr; 2793 struct tx_ring *txr = iq->txr; 2794 const u_int txlimit = sc->sc_tx_intr_process_limit, 2795 rxlimit = sc->sc_rx_intr_process_limit; 2796 bool txmore, rxmore; 2797 2798 if (__predict_false(!ISSET(ifp->if_flags, IFF_RUNNING))) 2799 return 0; 2800 2801 const uint32_t reg_icr = IGC_READ_REG(&sc->hw, IGC_ICR); 2802 DPRINTF(MISC, "reg_icr=0x%x\n", reg_icr); 2803 2804 /* Definitely not our interrupt. */ 2805 if (reg_icr == 0x0) { 2806 DPRINTF(MISC, "not for me\n"); 2807 return 0; 2808 } 2809 2810 IGC_QUEUE_EVENT(iq, irqs, 1); 2811 2812 /* Hot eject? */ 2813 if (__predict_false(reg_icr == 0xffffffff)) { 2814 DPRINTF(MISC, "hot eject\n"); 2815 return 0; 2816 } 2817 2818 if (__predict_false(!(reg_icr & IGC_ICR_INT_ASSERTED))) { 2819 DPRINTF(MISC, "not set IGC_ICR_INT_ASSERTED"); 2820 return 0; 2821 } 2822 2823 /* 2824 * Only MSI-X interrupts have one-shot behavior by taking advantage 2825 * of the EIAC register. Thus, explicitly disable interrupts. This 2826 * also works around the MSI message reordering errata on certain 2827 * systems. 2828 */ 2829 igc_disable_intr(sc); 2830 2831 mutex_enter(&txr->txr_lock); 2832 txmore = igc_txeof(txr, txlimit); 2833 mutex_exit(&txr->txr_lock); 2834 mutex_enter(&rxr->rxr_lock); 2835 rxmore = igc_rxeof(rxr, rxlimit); 2836 mutex_exit(&rxr->rxr_lock); 2837 2838 /* Link status change */ 2839 // XXXX FreeBSD checks IGC_ICR_RXSEQ 2840 if (__predict_false(reg_icr & IGC_ICR_LSC)) { 2841 IGC_GLOBAL_EVENT(sc, link, 1); 2842 mutex_enter(&sc->sc_core_lock); 2843 sc->hw.mac.get_link_status = true; 2844 igc_update_link_status(sc); 2845 mutex_exit(&sc->sc_core_lock); 2846 } 2847 2848 if (txmore || rxmore) { 2849 IGC_QUEUE_EVENT(iq, req, 1); 2850 igc_sched_handle_queue(sc, iq); 2851 } else { 2852 igc_enable_intr(sc); 2853 } 2854 2855 return 1; 2856 } 2857 2858 static void 2859 igc_handle_queue(void *arg) 2860 { 2861 struct igc_queue *iq = arg; 2862 struct igc_softc *sc = iq->sc; 2863 struct tx_ring *txr = iq->txr; 2864 struct rx_ring *rxr = iq->rxr; 2865 const u_int txlimit = sc->sc_tx_process_limit, 2866 rxlimit = sc->sc_rx_process_limit; 2867 bool txmore, rxmore; 2868 2869 IGC_QUEUE_EVENT(iq, handleq, 1); 2870 2871 mutex_enter(&txr->txr_lock); 2872 txmore = igc_txeof(txr, txlimit); 2873 /* for ALTQ, dequeue from if_snd */ 2874 if (txr->me == 0) { 2875 struct ifnet *ifp = &sc->sc_ec.ec_if; 2876 2877 igc_tx_common_locked(ifp, txr, IGC_TX_START); 2878 } 2879 mutex_exit(&txr->txr_lock); 2880 2881 mutex_enter(&rxr->rxr_lock); 2882 rxmore = igc_rxeof(rxr, rxlimit); 2883 mutex_exit(&rxr->rxr_lock); 2884 2885 if (txmore || rxmore) { 2886 igc_sched_handle_queue(sc, iq); 2887 } else { 2888 if (sc->sc_intr_type == PCI_INTR_TYPE_MSIX) 2889 igc_enable_queue(sc, iq->eims); 2890 else 2891 igc_enable_intr(sc); 2892 } 2893 } 2894 2895 static void 2896 igc_handle_queue_work(struct work *wk, void *context) 2897 { 2898 struct igc_queue *iq = 2899 container_of(wk, struct igc_queue, igcq_wq_cookie); 2900 2901 igc_handle_queue(iq); 2902 } 2903 2904 static void 2905 igc_sched_handle_queue(struct igc_softc *sc, struct igc_queue *iq) 2906 { 2907 2908 if (iq->igcq_workqueue) { 2909 /* XXXRO notyet */ 2910 workqueue_enqueue(sc->sc_queue_wq, &iq->igcq_wq_cookie, 2911 curcpu()); 2912 } else { 2913 softint_schedule(iq->igcq_si); 2914 } 2915 } 2916 2917 static void 2918 igc_barrier_handle_queue(struct igc_softc *sc) 2919 { 2920 2921 if (sc->sc_txrx_workqueue) { 2922 for (int iq = 0; iq < sc->sc_nqueues; iq++) { 2923 struct igc_queue *q = &sc->queues[iq]; 2924 2925 workqueue_wait(sc->sc_queue_wq, &q->igcq_wq_cookie); 2926 } 2927 } else { 2928 xc_barrier(0); 2929 } 2930 } 2931 2932 /********************************************************************* 2933 * 2934 * Allocate memory for tx_buffer structures. The tx_buffer stores all 2935 * the information needed to transmit a packet on the wire. 2936 * 2937 **********************************************************************/ 2938 static int 2939 igc_allocate_transmit_buffers(struct tx_ring *txr) 2940 { 2941 struct igc_softc *sc = txr->sc; 2942 int error; 2943 2944 txr->tx_buffers = 2945 kmem_zalloc(sc->num_tx_desc * sizeof(struct igc_tx_buf), KM_SLEEP); 2946 txr->txtag = txr->txdma.dma_tag; 2947 2948 /* Create the descriptor buffer dma maps. */ 2949 for (int id = 0; id < sc->num_tx_desc; id++) { 2950 struct igc_tx_buf *txbuf = &txr->tx_buffers[id]; 2951 2952 error = bus_dmamap_create(txr->txdma.dma_tag, 2953 round_page(IGC_TSO_SIZE + sizeof(struct ether_vlan_header)), 2954 IGC_MAX_SCATTER, PAGE_SIZE, 0, BUS_DMA_NOWAIT, &txbuf->map); 2955 if (error != 0) { 2956 aprint_error_dev(sc->sc_dev, 2957 "unable to create TX DMA map\n"); 2958 goto fail; 2959 } 2960 2961 txbuf->eop_index = -1; 2962 } 2963 2964 return 0; 2965 fail: 2966 return error; 2967 } 2968 2969 2970 /********************************************************************* 2971 * 2972 * Allocate and initialize transmit structures. 2973 * 2974 **********************************************************************/ 2975 static int 2976 igc_setup_transmit_structures(struct igc_softc *sc) 2977 { 2978 2979 for (int iq = 0; iq < sc->sc_nqueues; iq++) { 2980 struct tx_ring *txr = &sc->tx_rings[iq]; 2981 2982 if (igc_setup_transmit_ring(txr)) 2983 goto fail; 2984 } 2985 2986 return 0; 2987 fail: 2988 igc_free_transmit_structures(sc); 2989 return ENOBUFS; 2990 } 2991 2992 /********************************************************************* 2993 * 2994 * Initialize a transmit ring. 2995 * 2996 **********************************************************************/ 2997 static int 2998 igc_setup_transmit_ring(struct tx_ring *txr) 2999 { 3000 struct igc_softc *sc = txr->sc; 3001 3002 /* Now allocate transmit buffers for the ring. */ 3003 if (igc_allocate_transmit_buffers(txr)) 3004 return ENOMEM; 3005 3006 /* Clear the old ring contents */ 3007 memset(txr->tx_base, 0, 3008 sizeof(union igc_adv_tx_desc) * sc->num_tx_desc); 3009 3010 /* Reset indices. */ 3011 txr->next_avail_desc = 0; 3012 txr->next_to_clean = 0; 3013 3014 bus_dmamap_sync(txr->txdma.dma_tag, txr->txdma.dma_map, 0, 3015 txr->txdma.dma_map->dm_mapsize, 3016 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); 3017 3018 txr->txr_interq = pcq_create(sc->num_tx_desc, KM_SLEEP); 3019 3020 mutex_init(&txr->txr_lock, MUTEX_DEFAULT, IPL_NET); 3021 3022 return 0; 3023 } 3024 3025 /********************************************************************* 3026 * 3027 * Enable transmit unit. 3028 * 3029 **********************************************************************/ 3030 static void 3031 igc_initialize_transmit_unit(struct igc_softc *sc) 3032 { 3033 struct ifnet *ifp = &sc->sc_ec.ec_if; 3034 struct igc_hw *hw = &sc->hw; 3035 3036 /* Setup the Base and Length of the TX descriptor ring. */ 3037 for (int iq = 0; iq < sc->sc_nqueues; iq++) { 3038 struct tx_ring *txr = &sc->tx_rings[iq]; 3039 const uint64_t bus_addr = 3040 txr->txdma.dma_map->dm_segs[0].ds_addr; 3041 3042 /* Base and len of TX ring */ 3043 IGC_WRITE_REG(hw, IGC_TDLEN(iq), 3044 sc->num_tx_desc * sizeof(union igc_adv_tx_desc)); 3045 IGC_WRITE_REG(hw, IGC_TDBAH(iq), (uint32_t)(bus_addr >> 32)); 3046 IGC_WRITE_REG(hw, IGC_TDBAL(iq), (uint32_t)bus_addr); 3047 3048 /* Init the HEAD/TAIL indices */ 3049 IGC_WRITE_REG(hw, IGC_TDT(iq), 0 /* XXX txr->next_avail_desc */); 3050 IGC_WRITE_REG(hw, IGC_TDH(iq), 0); 3051 3052 txr->watchdog_timer = 0; 3053 3054 uint32_t txdctl = 0; /* Clear txdctl */ 3055 txdctl |= 0x1f; /* PTHRESH */ 3056 txdctl |= 1 << 8; /* HTHRESH */ 3057 txdctl |= 1 << 16; /* WTHRESH */ 3058 txdctl |= 1 << 22; /* Reserved bit 22 must always be 1 */ 3059 txdctl |= IGC_TXDCTL_GRAN; 3060 txdctl |= 1 << 25; /* LWTHRESH */ 3061 3062 IGC_WRITE_REG(hw, IGC_TXDCTL(iq), txdctl); 3063 } 3064 ifp->if_timer = 0; 3065 3066 /* Program the Transmit Control Register */ 3067 uint32_t tctl = IGC_READ_REG(&sc->hw, IGC_TCTL); 3068 tctl &= ~IGC_TCTL_CT; 3069 tctl |= (IGC_TCTL_PSP | IGC_TCTL_RTLC | IGC_TCTL_EN | 3070 (IGC_COLLISION_THRESHOLD << IGC_CT_SHIFT)); 3071 3072 /* This write will effectively turn on the transmit unit. */ 3073 IGC_WRITE_REG(&sc->hw, IGC_TCTL, tctl); 3074 } 3075 3076 /********************************************************************* 3077 * 3078 * Free all transmit rings. 3079 * 3080 **********************************************************************/ 3081 static void 3082 igc_free_transmit_structures(struct igc_softc *sc) 3083 { 3084 3085 for (int iq = 0; iq < sc->sc_nqueues; iq++) { 3086 struct tx_ring *txr = &sc->tx_rings[iq]; 3087 3088 igc_free_transmit_buffers(txr); 3089 } 3090 } 3091 3092 /********************************************************************* 3093 * 3094 * Free transmit ring related data structures. 3095 * 3096 **********************************************************************/ 3097 static void 3098 igc_free_transmit_buffers(struct tx_ring *txr) 3099 { 3100 struct igc_softc *sc = txr->sc; 3101 3102 if (txr->tx_buffers == NULL) 3103 return; 3104 3105 igc_withdraw_transmit_packets(txr, true); 3106 3107 kmem_free(txr->tx_buffers, 3108 sc->num_tx_desc * sizeof(struct igc_tx_buf)); 3109 txr->tx_buffers = NULL; 3110 txr->txtag = NULL; 3111 3112 pcq_destroy(txr->txr_interq); 3113 mutex_destroy(&txr->txr_lock); 3114 } 3115 3116 /********************************************************************* 3117 * 3118 * Withdraw transmit packets. 3119 * 3120 **********************************************************************/ 3121 static void 3122 igc_withdraw_transmit_packets(struct tx_ring *txr, bool destroy) 3123 { 3124 struct igc_softc *sc = txr->sc; 3125 struct igc_queue *q = txr->txr_igcq; 3126 3127 mutex_enter(&txr->txr_lock); 3128 3129 for (int id = 0; id < sc->num_tx_desc; id++) { 3130 union igc_adv_tx_desc *txdesc = &txr->tx_base[id]; 3131 3132 igc_txdesc_sync(txr, id, 3133 BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE); 3134 txdesc->read.buffer_addr = 0; 3135 txdesc->read.cmd_type_len = 0; 3136 txdesc->read.olinfo_status = 0; 3137 igc_txdesc_sync(txr, id, 3138 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); 3139 3140 struct igc_tx_buf *txbuf = &txr->tx_buffers[id]; 3141 bus_dmamap_t map = txbuf->map; 3142 3143 if (map != NULL && map->dm_nsegs > 0) { 3144 bus_dmamap_sync(txr->txdma.dma_tag, map, 3145 0, map->dm_mapsize, BUS_DMASYNC_POSTWRITE); 3146 bus_dmamap_unload(txr->txdma.dma_tag, map); 3147 } 3148 if (txbuf->m_head != NULL) { 3149 m_freem(txbuf->m_head); 3150 txbuf->m_head = NULL; 3151 } 3152 if (map != NULL && destroy) { 3153 bus_dmamap_destroy(txr->txdma.dma_tag, map); 3154 txbuf->map = NULL; 3155 } 3156 txbuf->eop_index = -1; 3157 3158 txr->next_avail_desc = 0; 3159 txr->next_to_clean = 0; 3160 } 3161 3162 struct mbuf *m; 3163 while ((m = pcq_get(txr->txr_interq)) != NULL) { 3164 IGC_QUEUE_EVENT(q, tx_pcq_drop, 1); 3165 m_freem(m); 3166 } 3167 3168 mutex_exit(&txr->txr_lock); 3169 } 3170 3171 3172 /********************************************************************* 3173 * 3174 * Advanced Context Descriptor setup for VLAN, CSUM or TSO 3175 * 3176 **********************************************************************/ 3177 3178 static bool 3179 igc_tx_ctx_setup(struct tx_ring *txr, struct mbuf *mp, int prod, 3180 uint32_t *cmd_type_len, uint32_t *olinfo_status) 3181 { 3182 struct ether_vlan_header *evl; 3183 struct tcphdr *th = NULL /* XXXGCC */; 3184 uint32_t vlan_macip_lens = 0; 3185 uint32_t type_tucmd_mlhl = 0; 3186 uint32_t mss_l4len_idx = 0; 3187 uint32_t ehlen, iphlen; 3188 uint16_t ehtype; 3189 3190 const int csum_flags = mp->m_pkthdr.csum_flags; 3191 const bool v4 = (csum_flags & 3192 (M_CSUM_IPv4 | M_CSUM_TCPv4 | M_CSUM_UDPv4 | M_CSUM_TSOv4)) != 0; 3193 const bool v6 = (csum_flags & 3194 (M_CSUM_UDPv6 | M_CSUM_TCPv6 | M_CSUM_TSOv6)) != 0; 3195 const bool tso = (csum_flags & (M_CSUM_TSOv4 | M_CSUM_TSOv6)) != 0; 3196 const bool tcp = tso || 3197 (csum_flags & (M_CSUM_TCPv4 | M_CSUM_TCPv6)) != 0; 3198 const bool udp = (csum_flags & (M_CSUM_UDPv4 | M_CSUM_UDPv6)) != 0; 3199 3200 /* Indicate the whole packet as payload when not doing TSO */ 3201 if (!tso) { 3202 *olinfo_status |= mp->m_pkthdr.len << IGC_ADVTXD_PAYLEN_SHIFT; 3203 } else { 3204 /* Set L4 payload length later... */ 3205 } 3206 3207 #if NVLAN > 0 3208 /* 3209 * In advanced descriptors the vlan tag must 3210 * be placed into the context descriptor. Hence 3211 * we need to make one even if not doing offloads. 3212 */ 3213 if (vlan_has_tag(mp)) { 3214 vlan_macip_lens |= (uint32_t)vlan_get_tag(mp) 3215 << IGC_ADVTXD_VLAN_SHIFT; 3216 } else 3217 #endif 3218 if (!v4 && !v6) 3219 return false; 3220 3221 KASSERT(mp->m_len >= sizeof(struct ether_header)); 3222 evl = mtod(mp, struct ether_vlan_header *); 3223 if (evl->evl_encap_proto == htons(ETHERTYPE_VLAN)) { 3224 KASSERT(mp->m_len >= sizeof(struct ether_vlan_header)); 3225 ehlen = ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN; 3226 ehtype = evl->evl_proto; 3227 } else { 3228 ehlen = ETHER_HDR_LEN; 3229 ehtype = evl->evl_encap_proto; 3230 } 3231 3232 switch (ntohs(ehtype)) { 3233 case ETHERTYPE_IP: 3234 iphlen = M_CSUM_DATA_IPv4_IPHL(mp->m_pkthdr.csum_data); 3235 type_tucmd_mlhl |= IGC_ADVTXD_TUCMD_IPV4; 3236 3237 if ((csum_flags & (M_CSUM_IPv4 | M_CSUM_TSOv4)) != 0) 3238 *olinfo_status |= IGC_TXD_POPTS_IXSM << 8; 3239 3240 if (!tso) 3241 break; 3242 3243 struct ip *ip; 3244 KASSERT(mp->m_len >= ehlen + sizeof(*ip)); 3245 ip = (void *)(mtod(mp, char *) + ehlen); 3246 ip->ip_len = 0; 3247 3248 KASSERT(mp->m_len >= ehlen + iphlen + sizeof(*th)); 3249 th = (void *)((char *)ip + iphlen); 3250 th->th_sum = in_cksum_phdr(ip->ip_src.s_addr, 3251 ip->ip_dst.s_addr, htons(IPPROTO_TCP)); 3252 break; 3253 case ETHERTYPE_IPV6: 3254 iphlen = M_CSUM_DATA_IPv6_IPHL(mp->m_pkthdr.csum_data); 3255 type_tucmd_mlhl |= IGC_ADVTXD_TUCMD_IPV6; 3256 3257 if (!tso) 3258 break; 3259 3260 struct ip6_hdr *ip6; 3261 KASSERT(mp->m_len >= ehlen + sizeof(*ip6)); 3262 ip6 = (void *)(mtod(mp, char *) + ehlen); 3263 ip6->ip6_plen = 0; 3264 3265 KASSERT(mp->m_len >= ehlen + iphlen + sizeof(*th)); 3266 th = (void *)((char *)ip6 + iphlen); 3267 th->th_sum = in6_cksum_phdr(&ip6->ip6_src, &ip6->ip6_dst, 0, 3268 htonl(IPPROTO_TCP)); 3269 break; 3270 default: 3271 /* 3272 * Unknown L3 protocol. Clear L3 header length and proceed for 3273 * LAN as done by Linux driver. 3274 */ 3275 KASSERT(!v4 && !v6); 3276 iphlen = 0; 3277 break; 3278 } 3279 3280 if (tcp) { 3281 type_tucmd_mlhl |= IGC_ADVTXD_TUCMD_L4T_TCP; 3282 *olinfo_status |= IGC_TXD_POPTS_TXSM << 8; 3283 } else if (udp) { 3284 type_tucmd_mlhl |= IGC_ADVTXD_TUCMD_L4T_UDP; 3285 *olinfo_status |= IGC_TXD_POPTS_TXSM << 8; 3286 } 3287 3288 if (tso) { 3289 const uint32_t tcphlen = th->th_off << 2; 3290 const uint32_t paylen = 3291 mp->m_pkthdr.len - ehlen - iphlen - tcphlen; 3292 3293 mss_l4len_idx |= mp->m_pkthdr.segsz << IGC_ADVTXD_MSS_SHIFT; 3294 mss_l4len_idx |= tcphlen << IGC_ADVTXD_L4LEN_SHIFT; 3295 3296 *cmd_type_len |= IGC_ADVTXD_DCMD_TSE; 3297 3298 *olinfo_status |= paylen << IGC_ADVTXD_PAYLEN_SHIFT; 3299 } 3300 3301 vlan_macip_lens |= iphlen; 3302 vlan_macip_lens |= ehlen << IGC_ADVTXD_MACLEN_SHIFT; 3303 3304 type_tucmd_mlhl |= IGC_ADVTXD_DCMD_DEXT | IGC_ADVTXD_DTYP_CTXT; 3305 3306 /* Now ready a context descriptor */ 3307 struct igc_adv_tx_context_desc *txdesc = 3308 (struct igc_adv_tx_context_desc *)&txr->tx_base[prod]; 3309 3310 igc_txdesc_sync(txr, prod, 3311 BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE); 3312 3313 /* Now copy bits into descriptor */ 3314 htolem32(&txdesc->vlan_macip_lens, vlan_macip_lens); 3315 htolem32(&txdesc->type_tucmd_mlhl, type_tucmd_mlhl); 3316 htolem32(&txdesc->seqnum_seed, 0); 3317 htolem32(&txdesc->mss_l4len_idx, mss_l4len_idx); 3318 3319 igc_txdesc_sync(txr, prod, 3320 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); 3321 3322 return 1; 3323 } 3324 3325 /********************************************************************* 3326 * 3327 * Allocate memory for rx_buffer structures. Since we use one 3328 * rx_buffer per received packet, the maximum number of rx_buffer's 3329 * that we'll need is equal to the number of receive descriptors 3330 * that we've allocated. 3331 * 3332 **********************************************************************/ 3333 static int 3334 igc_allocate_receive_buffers(struct rx_ring *rxr) 3335 { 3336 struct igc_softc *sc = rxr->sc; 3337 int error; 3338 3339 rxr->rx_buffers = 3340 kmem_zalloc(sc->num_rx_desc * sizeof(struct igc_rx_buf), KM_SLEEP); 3341 3342 for (int id = 0; id < sc->num_rx_desc; id++) { 3343 struct igc_rx_buf *rxbuf = &rxr->rx_buffers[id]; 3344 3345 error = bus_dmamap_create(rxr->rxdma.dma_tag, MCLBYTES, 1, 3346 MCLBYTES, 0, BUS_DMA_WAITOK, &rxbuf->map); 3347 if (error) { 3348 aprint_error_dev(sc->sc_dev, 3349 "unable to create RX DMA map\n"); 3350 goto fail; 3351 } 3352 } 3353 bus_dmamap_sync(rxr->rxdma.dma_tag, rxr->rxdma.dma_map, 0, 3354 rxr->rxdma.dma_map->dm_mapsize, 3355 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); 3356 3357 return 0; 3358 fail: 3359 return error; 3360 } 3361 3362 /********************************************************************* 3363 * 3364 * Allocate and initialize receive structures. 3365 * 3366 **********************************************************************/ 3367 static int 3368 igc_setup_receive_structures(struct igc_softc *sc) 3369 { 3370 3371 for (int iq = 0; iq < sc->sc_nqueues; iq++) { 3372 struct rx_ring *rxr = &sc->rx_rings[iq]; 3373 3374 if (igc_setup_receive_ring(rxr)) 3375 goto fail; 3376 } 3377 3378 return 0; 3379 fail: 3380 igc_free_receive_structures(sc); 3381 return ENOBUFS; 3382 } 3383 3384 /********************************************************************* 3385 * 3386 * Initialize a receive ring and its buffers. 3387 * 3388 **********************************************************************/ 3389 static int 3390 igc_setup_receive_ring(struct rx_ring *rxr) 3391 { 3392 struct igc_softc *sc = rxr->sc; 3393 const int rsize = roundup2( 3394 sc->num_rx_desc * sizeof(union igc_adv_rx_desc), IGC_DBA_ALIGN); 3395 3396 /* Clear the ring contents. */ 3397 memset(rxr->rx_base, 0, rsize); 3398 3399 if (igc_allocate_receive_buffers(rxr)) 3400 return ENOMEM; 3401 3402 /* Setup our descriptor indices. */ 3403 rxr->next_to_check = 0; 3404 rxr->last_desc_filled = 0; 3405 3406 mutex_init(&rxr->rxr_lock, MUTEX_DEFAULT, IPL_NET); 3407 3408 return 0; 3409 } 3410 3411 /********************************************************************* 3412 * 3413 * Enable receive unit. 3414 * 3415 **********************************************************************/ 3416 static void 3417 igc_initialize_receive_unit(struct igc_softc *sc) 3418 { 3419 struct ifnet *ifp = &sc->sc_ec.ec_if; 3420 struct igc_hw *hw = &sc->hw; 3421 uint32_t rctl, rxcsum, srrctl; 3422 3423 DPRINTF(RX, "called\n"); 3424 3425 /* 3426 * Make sure receives are disabled while setting 3427 * up the descriptor ring. 3428 */ 3429 rctl = IGC_READ_REG(hw, IGC_RCTL); 3430 IGC_WRITE_REG(hw, IGC_RCTL, rctl & ~IGC_RCTL_EN); 3431 3432 /* Setup the Receive Control Register */ 3433 rctl &= ~(3 << IGC_RCTL_MO_SHIFT); 3434 rctl |= IGC_RCTL_EN | IGC_RCTL_BAM | IGC_RCTL_LBM_NO | 3435 IGC_RCTL_RDMTS_HALF | (hw->mac.mc_filter_type << IGC_RCTL_MO_SHIFT); 3436 3437 #if 1 3438 /* Do not store bad packets */ 3439 rctl &= ~IGC_RCTL_SBP; 3440 #else 3441 /* for debug */ 3442 rctl |= IGC_RCTL_SBP; 3443 #endif 3444 3445 /* Enable Long Packet receive */ 3446 if (sc->hw.mac.max_frame_size > ETHER_MAX_LEN) 3447 rctl |= IGC_RCTL_LPE; 3448 else 3449 rctl &= ~IGC_RCTL_LPE; 3450 3451 /* Strip the CRC */ 3452 rctl |= IGC_RCTL_SECRC; 3453 3454 /* 3455 * Set the interrupt throttling rate. Value is calculated 3456 * as DEFAULT_ITR = 1/(MAX_INTS_PER_SEC * 256ns) 3457 * 3458 * XXX Sync with Linux, especially for jumbo MTU or TSO. 3459 * XXX Shouldn't be here? 3460 */ 3461 IGC_WRITE_REG(hw, IGC_ITR, DEFAULT_ITR); 3462 3463 rxcsum = IGC_READ_REG(hw, IGC_RXCSUM); 3464 rxcsum &= ~(IGC_RXCSUM_IPOFL | IGC_RXCSUM_TUOFL | IGC_RXCSUM_PCSD); 3465 if (ifp->if_capenable & IFCAP_CSUM_IPv4_Rx) 3466 rxcsum |= IGC_RXCSUM_IPOFL; 3467 if (ifp->if_capenable & (IFCAP_CSUM_TCPv4_Rx | IFCAP_CSUM_UDPv4_Rx | 3468 IFCAP_CSUM_TCPv6_Rx | IFCAP_CSUM_UDPv6_Rx)) 3469 rxcsum |= IGC_RXCSUM_TUOFL; 3470 if (sc->sc_nqueues > 1) 3471 rxcsum |= IGC_RXCSUM_PCSD; 3472 IGC_WRITE_REG(hw, IGC_RXCSUM, rxcsum); 3473 3474 if (sc->sc_nqueues > 1) 3475 igc_initialize_rss_mapping(sc); 3476 3477 srrctl = 0; 3478 #if 0 3479 srrctl |= 4096 >> IGC_SRRCTL_BSIZEPKT_SHIFT; 3480 rctl |= IGC_RCTL_SZ_4096 | IGC_RCTL_BSEX; 3481 #else 3482 srrctl |= 2048 >> IGC_SRRCTL_BSIZEPKT_SHIFT; 3483 rctl |= IGC_RCTL_SZ_2048; 3484 #endif 3485 3486 /* 3487 * If TX flow control is disabled and there's > 1 queue defined, 3488 * enable DROP. 3489 * 3490 * This drops frames rather than hanging the RX MAC for all queues. 3491 */ 3492 if (sc->sc_nqueues > 1 && 3493 (sc->fc == igc_fc_none || sc->fc == igc_fc_rx_pause)) 3494 srrctl |= IGC_SRRCTL_DROP_EN; 3495 3496 /* Setup the Base and Length of the RX descriptor rings. */ 3497 for (int iq = 0; iq < sc->sc_nqueues; iq++) { 3498 struct rx_ring *rxr = &sc->rx_rings[iq]; 3499 const uint64_t bus_addr = 3500 rxr->rxdma.dma_map->dm_segs[0].ds_addr; 3501 3502 IGC_WRITE_REG(hw, IGC_RXDCTL(iq), 0); 3503 3504 srrctl |= IGC_SRRCTL_DESCTYPE_ADV_ONEBUF; 3505 3506 IGC_WRITE_REG(hw, IGC_RDLEN(iq), 3507 sc->num_rx_desc * sizeof(union igc_adv_rx_desc)); 3508 IGC_WRITE_REG(hw, IGC_RDBAH(iq), (uint32_t)(bus_addr >> 32)); 3509 IGC_WRITE_REG(hw, IGC_RDBAL(iq), (uint32_t)bus_addr); 3510 IGC_WRITE_REG(hw, IGC_SRRCTL(iq), srrctl); 3511 3512 /* Setup the Head and Tail Descriptor Pointers */ 3513 IGC_WRITE_REG(hw, IGC_RDH(iq), 0); 3514 IGC_WRITE_REG(hw, IGC_RDT(iq), 0 /* XXX rxr->last_desc_filled */); 3515 3516 /* Enable this Queue */ 3517 uint32_t rxdctl = IGC_READ_REG(hw, IGC_RXDCTL(iq)); 3518 rxdctl |= IGC_RXDCTL_QUEUE_ENABLE; 3519 rxdctl &= 0xFFF00000; 3520 rxdctl |= IGC_RX_PTHRESH; 3521 rxdctl |= IGC_RX_HTHRESH << 8; 3522 rxdctl |= IGC_RX_WTHRESH << 16; 3523 IGC_WRITE_REG(hw, IGC_RXDCTL(iq), rxdctl); 3524 } 3525 3526 /* Make sure VLAN Filters are off */ 3527 rctl &= ~IGC_RCTL_VFE; 3528 3529 /* Write out the settings */ 3530 IGC_WRITE_REG(hw, IGC_RCTL, rctl); 3531 } 3532 3533 /********************************************************************* 3534 * 3535 * Free all receive rings. 3536 * 3537 **********************************************************************/ 3538 static void 3539 igc_free_receive_structures(struct igc_softc *sc) 3540 { 3541 3542 for (int iq = 0; iq < sc->sc_nqueues; iq++) { 3543 struct rx_ring *rxr = &sc->rx_rings[iq]; 3544 3545 igc_free_receive_buffers(rxr); 3546 } 3547 } 3548 3549 /********************************************************************* 3550 * 3551 * Free receive ring data structures 3552 * 3553 **********************************************************************/ 3554 static void 3555 igc_free_receive_buffers(struct rx_ring *rxr) 3556 { 3557 struct igc_softc *sc = rxr->sc; 3558 3559 if (rxr->rx_buffers != NULL) { 3560 for (int id = 0; id < sc->num_rx_desc; id++) { 3561 struct igc_rx_buf *rxbuf = &rxr->rx_buffers[id]; 3562 bus_dmamap_t map = rxbuf->map; 3563 3564 if (rxbuf->buf != NULL) { 3565 bus_dmamap_sync(rxr->rxdma.dma_tag, map, 3566 0, map->dm_mapsize, BUS_DMASYNC_POSTREAD); 3567 bus_dmamap_unload(rxr->rxdma.dma_tag, map); 3568 m_freem(rxbuf->buf); 3569 rxbuf->buf = NULL; 3570 } 3571 bus_dmamap_destroy(rxr->rxdma.dma_tag, map); 3572 rxbuf->map = NULL; 3573 } 3574 kmem_free(rxr->rx_buffers, 3575 sc->num_rx_desc * sizeof(struct igc_rx_buf)); 3576 rxr->rx_buffers = NULL; 3577 } 3578 3579 mutex_destroy(&rxr->rxr_lock); 3580 } 3581 3582 /********************************************************************* 3583 * 3584 * Clear status registers in all RX descriptors. 3585 * 3586 **********************************************************************/ 3587 static void 3588 igc_clear_receive_status(struct rx_ring *rxr) 3589 { 3590 struct igc_softc *sc = rxr->sc; 3591 3592 mutex_enter(&rxr->rxr_lock); 3593 3594 for (int id = 0; id < sc->num_rx_desc; id++) { 3595 union igc_adv_rx_desc *rxdesc = &rxr->rx_base[id]; 3596 3597 igc_rxdesc_sync(rxr, id, 3598 BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE); 3599 rxdesc->wb.upper.status_error = 0; 3600 igc_rxdesc_sync(rxr, id, 3601 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); 3602 } 3603 3604 mutex_exit(&rxr->rxr_lock); 3605 } 3606 3607 /* 3608 * Initialise the RSS mapping for NICs that support multiple transmit/ 3609 * receive rings. 3610 */ 3611 static void 3612 igc_initialize_rss_mapping(struct igc_softc *sc) 3613 { 3614 struct igc_hw *hw = &sc->hw; 3615 3616 /* 3617 * The redirection table controls which destination 3618 * queue each bucket redirects traffic to. 3619 * Each DWORD represents four queues, with the LSB 3620 * being the first queue in the DWORD. 3621 * 3622 * This just allocates buckets to queues using round-robin 3623 * allocation. 3624 * 3625 * NOTE: It Just Happens to line up with the default 3626 * RSS allocation method. 3627 */ 3628 3629 /* Warning FM follows */ 3630 uint32_t reta = 0; 3631 for (int i = 0; i < 128; i++) { 3632 const int shift = 0; /* XXXRO */ 3633 int queue_id = i % sc->sc_nqueues; 3634 /* Adjust if required */ 3635 queue_id <<= shift; 3636 3637 /* 3638 * The low 8 bits are for hash value (n+0); 3639 * The next 8 bits are for hash value (n+1), etc. 3640 */ 3641 reta >>= 8; 3642 reta |= ((uint32_t)queue_id) << 24; 3643 if ((i & 3) == 3) { 3644 IGC_WRITE_REG(hw, IGC_RETA(i >> 2), reta); 3645 reta = 0; 3646 } 3647 } 3648 3649 /* 3650 * MRQC: Multiple Receive Queues Command 3651 * Set queuing to RSS control, number depends on the device. 3652 */ 3653 3654 /* Set up random bits */ 3655 uint32_t rss_key[RSS_KEYSIZE / sizeof(uint32_t)]; 3656 rss_getkey((uint8_t *)rss_key); 3657 3658 /* Now fill our hash function seeds */ 3659 for (int i = 0; i < __arraycount(rss_key); i++) 3660 IGC_WRITE_REG_ARRAY(hw, IGC_RSSRK(0), i, rss_key[i]); 3661 3662 /* 3663 * Configure the RSS fields to hash upon. 3664 */ 3665 uint32_t mrqc = IGC_MRQC_ENABLE_RSS_4Q; 3666 mrqc |= IGC_MRQC_RSS_FIELD_IPV4 | IGC_MRQC_RSS_FIELD_IPV4_TCP; 3667 mrqc |= IGC_MRQC_RSS_FIELD_IPV6 | IGC_MRQC_RSS_FIELD_IPV6_TCP; 3668 mrqc |= IGC_MRQC_RSS_FIELD_IPV6_TCP_EX; 3669 3670 IGC_WRITE_REG(hw, IGC_MRQC, mrqc); 3671 } 3672 3673 /* 3674 * igc_get_hw_control sets the {CTRL_EXT|FWSM}:DRV_LOAD bit. 3675 * For ASF and Pass Through versions of f/w this means 3676 * that the driver is loaded. For AMT version type f/w 3677 * this means that the network i/f is open. 3678 */ 3679 static void 3680 igc_get_hw_control(struct igc_softc *sc) 3681 { 3682 const uint32_t ctrl_ext = IGC_READ_REG(&sc->hw, IGC_CTRL_EXT); 3683 3684 IGC_WRITE_REG(&sc->hw, IGC_CTRL_EXT, ctrl_ext | IGC_CTRL_EXT_DRV_LOAD); 3685 } 3686 3687 /* 3688 * igc_release_hw_control resets {CTRL_EXT|FWSM}:DRV_LOAD bit. 3689 * For ASF and Pass Through versions of f/w this means that 3690 * the driver is no longer loaded. For AMT versions of the 3691 * f/w this means that the network i/f is closed. 3692 */ 3693 static void 3694 igc_release_hw_control(struct igc_softc *sc) 3695 { 3696 const uint32_t ctrl_ext = IGC_READ_REG(&sc->hw, IGC_CTRL_EXT); 3697 3698 IGC_WRITE_REG(&sc->hw, IGC_CTRL_EXT, ctrl_ext & ~IGC_CTRL_EXT_DRV_LOAD); 3699 } 3700 3701 static int 3702 igc_is_valid_ether_addr(uint8_t *addr) 3703 { 3704 const char zero_addr[6] = { 0, 0, 0, 0, 0, 0 }; 3705 3706 if ((addr[0] & 1) || !bcmp(addr, zero_addr, ETHER_ADDR_LEN)) 3707 return 0; 3708 3709 return 1; 3710 } 3711 3712 static void 3713 igc_print_devinfo(struct igc_softc *sc) 3714 { 3715 device_t dev = sc->sc_dev; 3716 struct igc_hw *hw = &sc->hw; 3717 struct igc_phy_info *phy = &hw->phy; 3718 u_int oui, model, rev; 3719 uint16_t id1, id2, nvm_ver, phy_ver, etk_lo, etk_hi; 3720 char descr[MII_MAX_DESCR_LEN]; 3721 3722 /* Print PHY Info */ 3723 id1 = phy->id >> 16; 3724 /* The revision field in phy->id is cleard and it's in phy->revision */ 3725 id2 = (phy->id & 0xfff0) | phy->revision; 3726 oui = MII_OUI(id1, id2); 3727 model = MII_MODEL(id2); 3728 rev = MII_REV(id2); 3729 mii_get_descr(descr, sizeof(descr), oui, model); 3730 if (descr[0]) 3731 aprint_normal_dev(dev, "PHY: %s, rev. %d", 3732 descr, rev); 3733 else 3734 aprint_normal_dev(dev, 3735 "PHY OUI 0x%06x, model 0x%04x, rev. %d", 3736 oui, model, rev); 3737 3738 /* PHY FW version */ 3739 phy->ops.read_reg(hw, 0x1e, &phy_ver); 3740 aprint_normal(", PHY FW version 0x%04hx\n", phy_ver); 3741 3742 /* NVM version */ 3743 hw->nvm.ops.read(hw, NVM_VERSION, 1, &nvm_ver); 3744 3745 /* EtrackID */ 3746 hw->nvm.ops.read(hw, NVM_ETKID_LO, 1, &etk_lo); 3747 hw->nvm.ops.read(hw, NVM_ETKID_HI, 1, &etk_hi); 3748 3749 aprint_normal_dev(dev, 3750 "NVM image version %x.%02x, EtrackID %04hx%04hx\n", 3751 (nvm_ver & NVM_VERSION_MAJOR) >> NVM_VERSION_MAJOR_SHIFT, 3752 nvm_ver & NVM_VERSION_MINOR, etk_hi, etk_lo); 3753 } 3754
Indexes created Tue Sep 30 11:09:46 GMT 2025