1 /* $NetBSD: if_igc.c,v 1.23 2025/10/21 15:20:34 pgoyette Exp $ */ 2 /* $OpenBSD: if_igc.c,v 1.13 2023/04/28 10:18:57 bluhm Exp $ */ 3 /*- 4 * SPDX-License-Identifier: BSD-2-Clause 5 * 6 * Copyright (c) 2016 Nicole Graziano <nicole (at) nextbsd.org> 7 * All rights reserved. 8 * Copyright (c) 2021 Rubicon Communications, LLC (Netgate) 9 * 10 * Redistribution and use in source and binary forms, with or without 11 * modification, are permitted provided that the following conditions 12 * are met: 13 * 1. Redistributions of source code must retain the above copyright 14 * notice, this list of conditions and the following disclaimer. 15 * 2. Redistributions in binary form must reproduce the above copyright 16 * notice, this list of conditions and the following disclaimer in the 17 * documentation and/or other materials provided with the distribution. 18 * 19 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 20 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 21 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 22 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 23 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 24 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 25 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 26 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 27 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 28 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 29 * SUCH DAMAGE. 30 */ 31 32 #include <sys/cdefs.h> 33 __KERNEL_RCSID(0, "$NetBSD: if_igc.c,v 1.23 2025/10/21 15:20:34 pgoyette Exp $"); 34 35 #ifdef _KERNEL_OPT 36 #include "opt_if_igc.h" 37 #if 0 /* notyet */ 38 #include "vlan.h" 39 #endif 40 #endif 41 42 #include <sys/param.h> 43 #include <sys/systm.h> 44 #include <sys/bus.h> 45 #include <sys/cpu.h> 46 #include <sys/device.h> 47 #include <sys/endian.h> 48 #include <sys/intr.h> 49 #include <sys/interrupt.h> 50 #include <sys/kernel.h> 51 #include <sys/kmem.h> 52 #include <sys/mbuf.h> 53 #include <sys/mutex.h> 54 #include <sys/socket.h> 55 #include <sys/workqueue.h> 56 #include <sys/xcall.h> 57 #include <sys/module.h> 58 59 #include <net/bpf.h> 60 #include <net/if.h> 61 #include <net/if_ether.h> 62 #include <net/if_media.h> 63 #include <net/if_vlanvar.h> 64 #include <net/rss_config.h> 65 66 #include <netinet/in.h> 67 #include <netinet/ip.h> 68 #include <netinet/ip6.h> 69 #include <netinet/tcp.h> 70 71 #include <dev/pci/pcivar.h> 72 #include <dev/pci/pcireg.h> 73 #include <dev/pci/pcidevs.h> 74 75 #include <dev/pci/igc/if_igc.h> 76 #include <dev/pci/igc/igc_evcnt.h> 77 #include <dev/pci/igc/igc_hw.h> 78 #include <dev/mii/miivar.h> 79 80 #define IGC_WORKQUEUE_PRI PRI_SOFTNET 81 82 #ifndef IGC_RX_INTR_PROCESS_LIMIT_DEFAULT 83 #define IGC_RX_INTR_PROCESS_LIMIT_DEFAULT 0 84 #endif 85 #ifndef IGC_TX_INTR_PROCESS_LIMIT_DEFAULT 86 #define IGC_TX_INTR_PROCESS_LIMIT_DEFAULT 0 87 #endif 88 89 #ifndef IGC_RX_PROCESS_LIMIT_DEFAULT 90 #define IGC_RX_PROCESS_LIMIT_DEFAULT 256 91 #endif 92 #ifndef IGC_TX_PROCESS_LIMIT_DEFAULT 93 #define IGC_TX_PROCESS_LIMIT_DEFAULT 256 94 #endif 95 96 #define htolem32(p, x) (*((uint32_t *)(p)) = htole32(x)) 97 #define htolem64(p, x) (*((uint64_t *)(p)) = htole64(x)) 98 99 static const struct igc_product { 100 pci_vendor_id_t igcp_vendor; 101 pci_product_id_t igcp_product; 102 const char *igcp_name; 103 } igc_products[] = { 104 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_I225_IT, 105 "Intel(R) Ethernet Controller I225-IT(2)" }, 106 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_I226_LM, 107 "Intel(R) Ethernet Controller I226-LM" }, 108 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_I226_V, 109 "Intel(R) Ethernet Controller I226-V" }, 110 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_I226_IT, 111 "Intel(R) Ethernet Controller I226-IT" }, 112 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_I221_V, 113 "Intel(R) Ethernet Controller I221-V" }, 114 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_I226_BLANK_NVM, 115 "Intel(R) Ethernet Controller I226(blankNVM)" }, 116 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_I225_LM, 117 "Intel(R) Ethernet Controller I225-LM" }, 118 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_I225_V, 119 "Intel(R) Ethernet Controller I225-V" }, 120 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_I220_V, 121 "Intel(R) Ethernet Controller I220-V" }, 122 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_I225_I, 123 "Intel(R) Ethernet Controller I225-I" }, 124 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_I225_BLANK_NVM, 125 "Intel(R) Ethernet Controller I225(blankNVM)" }, 126 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_I225_K, 127 "Intel(R) Ethernet Controller I225-K" }, 128 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_I225_K2, 129 "Intel(R) Ethernet Controller I225-K(2)" }, 130 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_I226_K, 131 "Intel(R) Ethernet Controller I226-K" }, 132 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_I225_LMVP, 133 "Intel(R) Ethernet Controller I225-LMvP(2)" }, 134 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_I226_LMVP, 135 "Intel(R) Ethernet Controller I226-LMvP" }, 136 { 0, 0, NULL }, 137 }; 138 139 #define IGC_DF_CFG 0x1 140 #define IGC_DF_TX 0x2 141 #define IGC_DF_RX 0x4 142 #define IGC_DF_MISC 0x8 143 144 #ifdef IGC_DEBUG_FLAGS 145 int igc_debug_flags = IGC_DEBUG_FLAGS; 146 #else 147 int igc_debug_flags = 0; 148 #endif 149 150 #define DPRINTF(flag, fmt, args...) do { \ 151 if (igc_debug_flags & (IGC_DF_ ## flag)) \ 152 printf("%s: %d: " fmt, __func__, __LINE__, ##args); \ 153 } while (0) 154 155 /********************************************************************* 156 * Function Prototypes 157 *********************************************************************/ 158 static int igc_match(device_t, cfdata_t, void *); 159 static void igc_attach(device_t, device_t, void *); 160 static int igc_detach(device_t, int); 161 162 static void igc_identify_hardware(struct igc_softc *); 163 static int igc_adjust_nqueues(struct igc_softc *); 164 static int igc_allocate_pci_resources(struct igc_softc *); 165 static int igc_allocate_interrupts(struct igc_softc *); 166 static int igc_allocate_queues(struct igc_softc *); 167 static void igc_free_pci_resources(struct igc_softc *); 168 static void igc_free_interrupts(struct igc_softc *); 169 static void igc_free_queues(struct igc_softc *); 170 static void igc_reset(struct igc_softc *); 171 static void igc_init_dmac(struct igc_softc *, uint32_t); 172 static int igc_setup_interrupts(struct igc_softc *); 173 static void igc_attach_counters(struct igc_softc *sc); 174 static void igc_detach_counters(struct igc_softc *sc); 175 static void igc_update_counters(struct igc_softc *sc); 176 static void igc_clear_counters(struct igc_softc *sc); 177 static int igc_setup_msix(struct igc_softc *); 178 static int igc_setup_msi(struct igc_softc *); 179 static int igc_setup_intx(struct igc_softc *); 180 static int igc_dma_malloc(struct igc_softc *, bus_size_t, 181 struct igc_dma_alloc *); 182 static void igc_dma_free(struct igc_softc *, struct igc_dma_alloc *); 183 static void igc_setup_interface(struct igc_softc *); 184 185 static int igc_init(struct ifnet *); 186 static int igc_init_locked(struct igc_softc *); 187 static void igc_start(struct ifnet *); 188 static int igc_transmit(struct ifnet *, struct mbuf *); 189 static void igc_tx_common_locked(struct ifnet *, struct tx_ring *, int); 190 static bool igc_txeof(struct tx_ring *, u_int); 191 static void igc_intr_barrier(struct igc_softc *); 192 static void igc_stop(struct ifnet *, int); 193 static void igc_stop_locked(struct igc_softc *); 194 static int igc_ioctl(struct ifnet *, u_long, void *); 195 #ifdef IF_RXR 196 static int igc_rxrinfo(struct igc_softc *, struct if_rxrinfo *); 197 #endif 198 static void igc_rxfill(struct rx_ring *); 199 static void igc_rxrefill(struct rx_ring *, int); 200 static bool igc_rxeof(struct rx_ring *, u_int); 201 static int igc_rx_checksum(struct igc_queue *, uint64_t, uint32_t, 202 uint32_t); 203 static void igc_watchdog(struct ifnet *); 204 static void igc_tick(void *); 205 static void igc_media_status(struct ifnet *, struct ifmediareq *); 206 static int igc_media_change(struct ifnet *); 207 static int igc_ifflags_cb(struct ethercom *); 208 static void igc_set_filter(struct igc_softc *); 209 static void igc_update_link_status(struct igc_softc *); 210 static int igc_get_buf(struct rx_ring *, int, bool); 211 static bool igc_tx_ctx_setup(struct tx_ring *, struct mbuf *, int, 212 uint32_t *, uint32_t *); 213 214 static void igc_configure_queues(struct igc_softc *); 215 static void igc_set_queues(struct igc_softc *, uint32_t, uint32_t, int); 216 static void igc_enable_queue(struct igc_softc *, uint32_t); 217 static void igc_enable_intr(struct igc_softc *); 218 static void igc_disable_intr(struct igc_softc *); 219 static int igc_intr_link(void *); 220 static int igc_intr_queue(void *); 221 static int igc_intr(void *); 222 static void igc_handle_queue(void *); 223 static void igc_handle_queue_work(struct work *, void *); 224 static void igc_sched_handle_queue(struct igc_softc *, struct igc_queue *); 225 static void igc_barrier_handle_queue(struct igc_softc *); 226 227 static int igc_allocate_transmit_buffers(struct tx_ring *); 228 static int igc_setup_transmit_structures(struct igc_softc *); 229 static int igc_setup_transmit_ring(struct tx_ring *); 230 static void igc_initialize_transmit_unit(struct igc_softc *); 231 static void igc_free_transmit_structures(struct igc_softc *); 232 static void igc_free_transmit_buffers(struct tx_ring *); 233 static void igc_withdraw_transmit_packets(struct tx_ring *, bool); 234 static int igc_allocate_receive_buffers(struct rx_ring *); 235 static int igc_setup_receive_structures(struct igc_softc *); 236 static int igc_setup_receive_ring(struct rx_ring *); 237 static void igc_initialize_receive_unit(struct igc_softc *); 238 static void igc_free_receive_structures(struct igc_softc *); 239 static void igc_free_receive_buffers(struct rx_ring *); 240 static void igc_clear_receive_status(struct rx_ring *); 241 static void igc_initialize_rss_mapping(struct igc_softc *); 242 243 static void igc_get_hw_control(struct igc_softc *); 244 static void igc_release_hw_control(struct igc_softc *); 245 static int igc_is_valid_ether_addr(uint8_t *); 246 static void igc_print_devinfo(struct igc_softc *); 247 248 CFATTACH_DECL3_NEW(igc, sizeof(struct igc_softc), 249 igc_match, igc_attach, igc_detach, NULL, NULL, NULL, 0); 250 251 static inline int 252 igc_txdesc_incr(struct igc_softc *sc, int id) 253 { 254 255 if (++id == sc->num_tx_desc) 256 id = 0; 257 return id; 258 } 259 260 static inline int __unused 261 igc_txdesc_decr(struct igc_softc *sc, int id) 262 { 263 264 if (--id < 0) 265 id = sc->num_tx_desc - 1; 266 return id; 267 } 268 269 static inline void 270 igc_txdesc_sync(struct tx_ring *txr, int id, int ops) 271 { 272 273 bus_dmamap_sync(txr->txdma.dma_tag, txr->txdma.dma_map, 274 id * sizeof(union igc_adv_tx_desc), sizeof(union igc_adv_tx_desc), 275 ops); 276 } 277 278 static inline int 279 igc_rxdesc_incr(struct igc_softc *sc, int id) 280 { 281 282 if (++id == sc->num_rx_desc) 283 id = 0; 284 return id; 285 } 286 287 static inline int 288 igc_rxdesc_decr(struct igc_softc *sc, int id) 289 { 290 291 if (--id < 0) 292 id = sc->num_rx_desc - 1; 293 return id; 294 } 295 296 static inline void 297 igc_rxdesc_sync(struct rx_ring *rxr, int id, int ops) 298 { 299 300 bus_dmamap_sync(rxr->rxdma.dma_tag, rxr->rxdma.dma_map, 301 id * sizeof(union igc_adv_rx_desc), sizeof(union igc_adv_rx_desc), 302 ops); 303 } 304 305 static const struct igc_product * 306 igc_lookup(const struct pci_attach_args *pa) 307 { 308 const struct igc_product *igcp; 309 310 for (igcp = igc_products; igcp->igcp_name != NULL; igcp++) { 311 if (PCI_VENDOR(pa->pa_id) == igcp->igcp_vendor && 312 PCI_PRODUCT(pa->pa_id) == igcp->igcp_product) 313 return igcp; 314 } 315 return NULL; 316 } 317 318 /********************************************************************* 319 * Device identification routine 320 * 321 * igc_match determines if the driver should be loaded on 322 * adapter based on PCI vendor/device id of the adapter. 323 * 324 * return 0 on success, positive on failure 325 *********************************************************************/ 326 static int 327 igc_match(device_t parent, cfdata_t match, void *aux) 328 { 329 struct pci_attach_args *pa = aux; 330 331 if (igc_lookup(pa) != NULL) 332 return 1; 333 334 return 0; 335 } 336 337 /********************************************************************* 338 * Device initialization routine 339 * 340 * The attach entry point is called when the driver is being loaded. 341 * This routine identifies the type of hardware, allocates all resources 342 * and initializes the hardware. 343 * 344 * return 0 on success, positive on failure 345 *********************************************************************/ 346 static void 347 igc_attach(device_t parent, device_t self, void *aux) 348 { 349 struct pci_attach_args *pa = aux; 350 struct igc_softc *sc = device_private(self); 351 struct igc_hw *hw = &sc->hw; 352 353 const struct igc_product *igcp = igc_lookup(pa); 354 KASSERT(igcp != NULL); 355 356 sc->sc_dev = self; 357 callout_init(&sc->sc_tick_ch, CALLOUT_MPSAFE); 358 callout_setfunc(&sc->sc_tick_ch, igc_tick, sc); 359 sc->sc_core_stopping = false; 360 361 sc->osdep.os_sc = sc; 362 sc->osdep.os_pa = *pa; 363 #ifndef __aarch64__ 364 /* 365 * XXX PR port-arm/57643 366 * 64-bit DMA does not work at least for LX2K with 32/64GB memory. 367 * smmu(4) support may be required. 368 */ 369 if (pci_dma64_available(pa)) { 370 aprint_verbose(", 64-bit DMA"); 371 sc->osdep.os_dmat = pa->pa_dmat64; 372 } else 373 #endif 374 { 375 aprint_verbose(", 32-bit DMA"); 376 sc->osdep.os_dmat = pa->pa_dmat; 377 } 378 379 pci_aprint_devinfo_fancy(pa, "Ethernet controller", igcp->igcp_name, 1); 380 381 /* Determine hardware and mac info */ 382 igc_identify_hardware(sc); 383 384 sc->num_tx_desc = IGC_DEFAULT_TXD; 385 sc->num_rx_desc = IGC_DEFAULT_RXD; 386 387 /* Setup PCI resources */ 388 if (igc_allocate_pci_resources(sc)) { 389 aprint_error_dev(sc->sc_dev, 390 "unable to allocate PCI resources\n"); 391 goto err_pci; 392 } 393 394 if (igc_allocate_interrupts(sc)) { 395 aprint_error_dev(sc->sc_dev, "unable to allocate interrupts\n"); 396 goto err_pci; 397 } 398 399 /* Allocate TX/RX queues */ 400 if (igc_allocate_queues(sc)) { 401 aprint_error_dev(sc->sc_dev, "unable to allocate queues\n"); 402 goto err_alloc_intr; 403 } 404 405 /* Do shared code initialization */ 406 if (igc_setup_init_funcs(hw, true)) { 407 aprint_error_dev(sc->sc_dev, "unable to initialize\n"); 408 goto err_alloc_intr; 409 } 410 411 hw->mac.autoneg = DO_AUTO_NEG; 412 hw->phy.autoneg_wait_to_complete = false; 413 hw->phy.autoneg_advertised = AUTONEG_ADV_DEFAULT; 414 415 /* Copper options. */ 416 if (hw->phy.media_type == igc_media_type_copper) 417 hw->phy.mdix = AUTO_ALL_MODES; 418 419 /* Set the max frame size. */ 420 sc->hw.mac.max_frame_size = 9234; 421 422 /* Allocate multicast array memory. */ 423 sc->mta = kmem_alloc(IGC_MTA_LEN, KM_SLEEP); 424 425 /* Check SOL/IDER usage. */ 426 if (igc_check_reset_block(hw)) { 427 aprint_error_dev(sc->sc_dev, 428 "PHY reset is blocked due to SOL/IDER session\n"); 429 } 430 431 /* Disable Energy Efficient Ethernet. */ 432 sc->hw.dev_spec._i225.eee_disable = true; 433 434 igc_reset_hw(hw); 435 436 /* Make sure we have a good EEPROM before we read from it. */ 437 if (igc_validate_nvm_checksum(hw) < 0) { 438 /* 439 * Some PCI-E parts fail the first check due to 440 * the link being in sleep state, call it again, 441 * if it fails a second time its a real issue. 442 */ 443 if (igc_validate_nvm_checksum(hw) < 0) { 444 aprint_error_dev(sc->sc_dev, 445 "EEPROM checksum invalid\n"); 446 goto err_late; 447 } 448 } 449 450 /* Copy the permanent MAC address out of the EEPROM. */ 451 if (igc_read_mac_addr(hw) < 0) { 452 aprint_error_dev(sc->sc_dev, 453 "unable to read MAC address from EEPROM\n"); 454 goto err_late; 455 } 456 457 if (!igc_is_valid_ether_addr(hw->mac.addr)) { 458 aprint_error_dev(sc->sc_dev, "invalid MAC address\n"); 459 goto err_late; 460 } 461 462 if (igc_setup_interrupts(sc)) 463 goto err_late; 464 465 /* Attach counters. */ 466 igc_attach_counters(sc); 467 468 /* Setup OS specific network interface. */ 469 igc_setup_interface(sc); 470 471 igc_print_devinfo(sc); 472 473 igc_reset(sc); 474 hw->mac.get_link_status = true; 475 igc_update_link_status(sc); 476 477 /* The driver can now take control from firmware. */ 478 igc_get_hw_control(sc); 479 480 aprint_normal_dev(sc->sc_dev, "Ethernet address %s\n", 481 ether_sprintf(sc->hw.mac.addr)); 482 483 if (pmf_device_register(self, NULL, NULL)) 484 pmf_class_network_register(self, &sc->sc_ec.ec_if); 485 else 486 aprint_error_dev(self, "couldn't establish power handler\n"); 487 488 return; 489 490 err_late: 491 igc_release_hw_control(sc); 492 err_alloc_intr: 493 igc_free_interrupts(sc); 494 err_pci: 495 igc_free_pci_resources(sc); 496 kmem_free(sc->mta, IGC_MTA_LEN); 497 } 498 499 /********************************************************************* 500 * Device removal routine 501 * 502 * The detach entry point is called when the driver is being removed. 503 * This routine stops the adapter and deallocates all the resources 504 * that were allocated for driver operation. 505 * 506 * return 0 on success, positive on failure 507 *********************************************************************/ 508 static int 509 igc_detach(device_t self, int flags) 510 { 511 struct igc_softc *sc = device_private(self); 512 struct ifnet *ifp = &sc->sc_ec.ec_if; 513 514 mutex_enter(&sc->sc_core_lock); 515 igc_stop_locked(sc); 516 mutex_exit(&sc->sc_core_lock); 517 518 igc_detach_counters(sc); 519 520 igc_free_queues(sc); 521 522 igc_phy_hw_reset(&sc->hw); 523 igc_release_hw_control(sc); 524 525 ether_ifdetach(ifp); 526 if_detach(ifp); 527 ifmedia_fini(&sc->media); 528 529 igc_free_interrupts(sc); 530 igc_free_pci_resources(sc); 531 kmem_free(sc->mta, IGC_MTA_LEN); 532 533 mutex_destroy(&sc->sc_core_lock); 534 535 return 0; 536 } 537 538 static void 539 igc_identify_hardware(struct igc_softc *sc) 540 { 541 struct igc_osdep *os = &sc->osdep; 542 struct pci_attach_args *pa = &os->os_pa; 543 544 /* Save off the information about this board. */ 545 sc->hw.device_id = PCI_PRODUCT(pa->pa_id); 546 547 /* Do shared code init and setup. */ 548 if (igc_set_mac_type(&sc->hw)) { 549 aprint_error_dev(sc->sc_dev, "unable to identify hardware\n"); 550 return; 551 } 552 } 553 554 static int 555 igc_allocate_pci_resources(struct igc_softc *sc) 556 { 557 struct igc_osdep *os = &sc->osdep; 558 struct pci_attach_args *pa = &os->os_pa; 559 560 /* 561 * Enable bus mastering and memory-mapped I/O for sure. 562 */ 563 pcireg_t csr = 564 pci_conf_read(pa->pa_pc, pa->pa_tag, PCI_COMMAND_STATUS_REG); 565 csr |= PCI_COMMAND_MASTER_ENABLE | PCI_COMMAND_MEM_ENABLE; 566 pci_conf_write(pa->pa_pc, pa->pa_tag, PCI_COMMAND_STATUS_REG, csr); 567 568 const pcireg_t memtype = 569 pci_mapreg_type(pa->pa_pc, pa->pa_tag, IGC_PCIREG); 570 if (pci_mapreg_map(pa, IGC_PCIREG, memtype, 0, &os->os_memt, 571 &os->os_memh, &os->os_membase, &os->os_memsize)) { 572 aprint_error_dev(sc->sc_dev, "unable to map registers\n"); 573 return ENXIO; 574 } 575 576 sc->hw.hw_addr = os->os_membase; 577 sc->hw.back = os; 578 579 return 0; 580 } 581 582 static int __unused 583 igc_adjust_nqueues(struct igc_softc *sc) 584 { 585 struct pci_attach_args *pa = &sc->osdep.os_pa; 586 int nqueues = MIN(IGC_MAX_NQUEUES, ncpu); 587 588 const int nmsix = pci_msix_count(pa->pa_pc, pa->pa_tag); 589 if (nmsix <= 1) 590 nqueues = 1; 591 else if (nmsix < nqueues + 1) 592 nqueues = nmsix - 1; 593 594 return nqueues; 595 } 596 597 static int 598 igc_allocate_interrupts(struct igc_softc *sc) 599 { 600 struct pci_attach_args *pa = &sc->osdep.os_pa; 601 int error; 602 603 #ifndef IGC_DISABLE_MSIX 604 const int nqueues = igc_adjust_nqueues(sc); 605 if (nqueues > 1) { 606 sc->sc_nintrs = nqueues + 1; 607 error = pci_msix_alloc_exact(pa, &sc->sc_intrs, sc->sc_nintrs); 608 if (!error) { 609 sc->sc_nqueues = nqueues; 610 sc->sc_intr_type = PCI_INTR_TYPE_MSIX; 611 return 0; 612 } 613 } 614 #endif 615 616 /* fallback to MSI */ 617 sc->sc_nintrs = sc->sc_nqueues = 1; 618 619 #ifndef IGC_DISABLE_MSI 620 error = pci_msi_alloc_exact(pa, &sc->sc_intrs, sc->sc_nintrs); 621 if (!error) { 622 sc->sc_intr_type = PCI_INTR_TYPE_MSI; 623 return 0; 624 } 625 #endif 626 627 /* fallback to INTx */ 628 629 error = pci_intx_alloc(pa, &sc->sc_intrs); 630 if (!error) { 631 sc->sc_intr_type = PCI_INTR_TYPE_INTX; 632 return 0; 633 } 634 635 return error; 636 } 637 638 static int 639 igc_allocate_queues(struct igc_softc *sc) 640 { 641 device_t dev = sc->sc_dev; 642 int rxconf = 0, txconf = 0; 643 644 /* Allocate the top level queue structs. */ 645 sc->queues = 646 kmem_zalloc(sc->sc_nqueues * sizeof(struct igc_queue), KM_SLEEP); 647 648 /* Allocate the TX ring. */ 649 sc->tx_rings = 650 kmem_zalloc(sc->sc_nqueues * sizeof(struct tx_ring), KM_SLEEP); 651 652 /* Allocate the RX ring. */ 653 sc->rx_rings = 654 kmem_zalloc(sc->sc_nqueues * sizeof(struct rx_ring), KM_SLEEP); 655 656 /* Set up the TX queues. */ 657 for (int iq = 0; iq < sc->sc_nqueues; iq++, txconf++) { 658 struct tx_ring *txr = &sc->tx_rings[iq]; 659 const int tsize = roundup2( 660 sc->num_tx_desc * sizeof(union igc_adv_tx_desc), 661 IGC_DBA_ALIGN); 662 663 txr->sc = sc; 664 txr->txr_igcq = &sc->queues[iq]; 665 txr->me = iq; 666 if (igc_dma_malloc(sc, tsize, &txr->txdma)) { 667 aprint_error_dev(dev, 668 "unable to allocate TX descriptor\n"); 669 goto fail; 670 } 671 txr->tx_base = (union igc_adv_tx_desc *)txr->txdma.dma_vaddr; 672 memset(txr->tx_base, 0, tsize); 673 } 674 675 /* Prepare transmit descriptors and buffers. */ 676 if (igc_setup_transmit_structures(sc)) { 677 aprint_error_dev(dev, "unable to setup transmit structures\n"); 678 goto fail; 679 } 680 681 /* Set up the RX queues. */ 682 for (int iq = 0; iq < sc->sc_nqueues; iq++, rxconf++) { 683 struct rx_ring *rxr = &sc->rx_rings[iq]; 684 const int rsize = roundup2( 685 sc->num_rx_desc * sizeof(union igc_adv_rx_desc), 686 IGC_DBA_ALIGN); 687 688 rxr->sc = sc; 689 rxr->rxr_igcq = &sc->queues[iq]; 690 rxr->me = iq; 691 #ifdef OPENBSD 692 timeout_set(&rxr->rx_refill, igc_rxrefill, rxr); 693 #endif 694 if (igc_dma_malloc(sc, rsize, &rxr->rxdma)) { 695 aprint_error_dev(dev, 696 "unable to allocate RX descriptor\n"); 697 goto fail; 698 } 699 rxr->rx_base = (union igc_adv_rx_desc *)rxr->rxdma.dma_vaddr; 700 memset(rxr->rx_base, 0, rsize); 701 } 702 703 sc->rx_mbuf_sz = MCLBYTES; 704 /* Prepare receive descriptors and buffers. */ 705 if (igc_setup_receive_structures(sc)) { 706 aprint_error_dev(sc->sc_dev, 707 "unable to setup receive structures\n"); 708 goto fail; 709 } 710 711 /* Set up the queue holding structs. */ 712 for (int iq = 0; iq < sc->sc_nqueues; iq++) { 713 struct igc_queue *q = &sc->queues[iq]; 714 715 q->sc = sc; 716 q->txr = &sc->tx_rings[iq]; 717 q->rxr = &sc->rx_rings[iq]; 718 } 719 720 return 0; 721 722 fail: 723 for (struct rx_ring *rxr = sc->rx_rings; rxconf > 0; rxr++, rxconf--) 724 igc_dma_free(sc, &rxr->rxdma); 725 for (struct tx_ring *txr = sc->tx_rings; txconf > 0; txr++, txconf--) 726 igc_dma_free(sc, &txr->txdma); 727 728 kmem_free(sc->rx_rings, sc->sc_nqueues * sizeof(struct rx_ring)); 729 sc->rx_rings = NULL; 730 kmem_free(sc->tx_rings, sc->sc_nqueues * sizeof(struct tx_ring)); 731 sc->tx_rings = NULL; 732 kmem_free(sc->queues, sc->sc_nqueues * sizeof(struct igc_queue)); 733 sc->queues = NULL; 734 735 return ENOMEM; 736 } 737 738 static void 739 igc_free_pci_resources(struct igc_softc *sc) 740 { 741 struct igc_osdep *os = &sc->osdep; 742 743 if (os->os_membase != 0) 744 bus_space_unmap(os->os_memt, os->os_memh, os->os_memsize); 745 os->os_membase = 0; 746 } 747 748 static void 749 igc_free_interrupts(struct igc_softc *sc) 750 { 751 struct pci_attach_args *pa = &sc->osdep.os_pa; 752 pci_chipset_tag_t pc = pa->pa_pc; 753 754 for (int i = 0; i < sc->sc_nintrs; i++) { 755 if (sc->sc_ihs[i] != NULL) { 756 pci_intr_disestablish(pc, sc->sc_ihs[i]); 757 sc->sc_ihs[i] = NULL; 758 } 759 } 760 pci_intr_release(pc, sc->sc_intrs, sc->sc_nintrs); 761 } 762 763 static void 764 igc_free_queues(struct igc_softc *sc) 765 { 766 767 igc_free_receive_structures(sc); 768 for (int iq = 0; iq < sc->sc_nqueues; iq++) { 769 struct rx_ring *rxr = &sc->rx_rings[iq]; 770 771 igc_dma_free(sc, &rxr->rxdma); 772 } 773 774 igc_free_transmit_structures(sc); 775 for (int iq = 0; iq < sc->sc_nqueues; iq++) { 776 struct tx_ring *txr = &sc->tx_rings[iq]; 777 778 igc_dma_free(sc, &txr->txdma); 779 } 780 781 kmem_free(sc->rx_rings, sc->sc_nqueues * sizeof(struct rx_ring)); 782 kmem_free(sc->tx_rings, sc->sc_nqueues * sizeof(struct tx_ring)); 783 kmem_free(sc->queues, sc->sc_nqueues * sizeof(struct igc_queue)); 784 } 785 786 /********************************************************************* 787 * 788 * Initialize the hardware to a configuration as specified by the 789 * adapter structure. 790 * 791 **********************************************************************/ 792 static void 793 igc_reset(struct igc_softc *sc) 794 { 795 struct igc_hw *hw = &sc->hw; 796 797 /* Let the firmware know the OS is in control */ 798 igc_get_hw_control(sc); 799 800 /* 801 * Packet Buffer Allocation (PBA) 802 * Writing PBA sets the receive portion of the buffer 803 * the remainder is used for the transmit buffer. 804 */ 805 const uint32_t pba = IGC_PBA_34K; 806 807 /* 808 * These parameters control the automatic generation (Tx) and 809 * response (Rx) to Ethernet PAUSE frames. 810 * - High water mark should allow for at least two frames to be 811 * received after sending an XOFF. 812 * - Low water mark works best when it is very near the high water mark. 813 * This allows the receiver to restart by sending XON when it has 814 * drained a bit. Here we use an arbitrary value of 1500 which will 815 * restart after one full frame is pulled from the buffer. There 816 * could be several smaller frames in the buffer and if so they will 817 * not trigger the XON until their total number reduces the buffer 818 * by 1500. 819 * - The pause time is fairly large at 1000 x 512ns = 512 usec. 820 */ 821 const uint16_t rx_buffer_size = (pba & 0xffff) << 10; 822 823 hw->fc.high_water = rx_buffer_size - 824 roundup2(sc->hw.mac.max_frame_size, 1024); 825 /* 16-byte granularity */ 826 hw->fc.low_water = hw->fc.high_water - 16; 827 828 if (sc->fc) /* locally set flow control value? */ 829 hw->fc.requested_mode = sc->fc; 830 else 831 hw->fc.requested_mode = igc_fc_full; 832 833 hw->fc.pause_time = IGC_FC_PAUSE_TIME; 834 835 hw->fc.send_xon = true; 836 837 /* Issue a global reset */ 838 igc_reset_hw(hw); 839 IGC_WRITE_REG(hw, IGC_WUC, 0); 840 841 /* and a re-init */ 842 if (igc_init_hw(hw) < 0) { 843 aprint_error_dev(sc->sc_dev, "unable to reset hardware\n"); 844 return; 845 } 846 847 /* Setup DMA Coalescing */ 848 igc_init_dmac(sc, pba); 849 850 IGC_WRITE_REG(hw, IGC_VET, ETHERTYPE_VLAN); 851 igc_get_phy_info(hw); 852 igc_check_for_link(hw); 853 } 854 855 /********************************************************************* 856 * 857 * Initialize the DMA Coalescing feature 858 * 859 **********************************************************************/ 860 static void 861 igc_init_dmac(struct igc_softc *sc, uint32_t pba) 862 { 863 struct igc_hw *hw = &sc->hw; 864 const uint16_t max_frame_size = sc->hw.mac.max_frame_size; 865 uint32_t reg, status; 866 867 if (sc->dmac == 0) { /* Disabling it */ 868 reg = ~IGC_DMACR_DMAC_EN; /* XXXRO */ 869 IGC_WRITE_REG(hw, IGC_DMACR, reg); 870 DPRINTF(MISC, "DMA coalescing disabled\n"); 871 return; 872 } else { 873 device_printf(sc->sc_dev, "DMA coalescing enabled\n"); 874 } 875 876 /* Set starting threshold */ 877 IGC_WRITE_REG(hw, IGC_DMCTXTH, 0); 878 879 uint16_t hwm = 64 * pba - max_frame_size / 16; 880 if (hwm < 64 * (pba - 6)) 881 hwm = 64 * (pba - 6); 882 reg = IGC_READ_REG(hw, IGC_FCRTC); 883 reg &= ~IGC_FCRTC_RTH_COAL_MASK; 884 reg |= (hwm << IGC_FCRTC_RTH_COAL_SHIFT) & IGC_FCRTC_RTH_COAL_MASK; 885 IGC_WRITE_REG(hw, IGC_FCRTC, reg); 886 887 uint32_t dmac = pba - max_frame_size / 512; 888 if (dmac < pba - 10) 889 dmac = pba - 10; 890 reg = IGC_READ_REG(hw, IGC_DMACR); 891 reg &= ~IGC_DMACR_DMACTHR_MASK; 892 reg |= (dmac << IGC_DMACR_DMACTHR_SHIFT) & IGC_DMACR_DMACTHR_MASK; 893 894 /* transition to L0x or L1 if available..*/ 895 reg |= IGC_DMACR_DMAC_EN | IGC_DMACR_DMAC_LX_MASK; 896 897 /* Check if status is 2.5Gb backplane connection 898 * before configuration of watchdog timer, which is 899 * in msec values in 12.8usec intervals 900 * watchdog timer= msec values in 32usec intervals 901 * for non 2.5Gb connection 902 */ 903 status = IGC_READ_REG(hw, IGC_STATUS); 904 if ((status & IGC_STATUS_2P5_SKU) && 905 !(status & IGC_STATUS_2P5_SKU_OVER)) 906 reg |= (sc->dmac * 5) >> 6; 907 else 908 reg |= sc->dmac >> 5; 909 910 IGC_WRITE_REG(hw, IGC_DMACR, reg); 911 912 IGC_WRITE_REG(hw, IGC_DMCRTRH, 0); 913 914 /* Set the interval before transition */ 915 reg = IGC_READ_REG(hw, IGC_DMCTLX); 916 reg |= IGC_DMCTLX_DCFLUSH_DIS; 917 918 /* 919 * in 2.5Gb connection, TTLX unit is 0.4 usec 920 * which is 0x4*2 = 0xA. But delay is still 4 usec 921 */ 922 status = IGC_READ_REG(hw, IGC_STATUS); 923 if ((status & IGC_STATUS_2P5_SKU) && 924 !(status & IGC_STATUS_2P5_SKU_OVER)) 925 reg |= 0xA; 926 else 927 reg |= 0x4; 928 929 IGC_WRITE_REG(hw, IGC_DMCTLX, reg); 930 931 /* free space in tx packet buffer to wake from DMA coal */ 932 IGC_WRITE_REG(hw, IGC_DMCTXTH, 933 (IGC_TXPBSIZE - (2 * max_frame_size)) >> 6); 934 935 /* make low power state decision controlled by DMA coal */ 936 reg = IGC_READ_REG(hw, IGC_PCIEMISC); 937 reg &= ~IGC_PCIEMISC_LX_DECISION; 938 IGC_WRITE_REG(hw, IGC_PCIEMISC, reg); 939 } 940 941 static int 942 igc_setup_interrupts(struct igc_softc *sc) 943 { 944 int error; 945 946 switch (sc->sc_intr_type) { 947 case PCI_INTR_TYPE_MSIX: 948 error = igc_setup_msix(sc); 949 break; 950 case PCI_INTR_TYPE_MSI: 951 error = igc_setup_msi(sc); 952 break; 953 case PCI_INTR_TYPE_INTX: 954 error = igc_setup_intx(sc); 955 break; 956 default: 957 panic("%s: invalid interrupt type: %d", 958 device_xname(sc->sc_dev), sc->sc_intr_type); 959 } 960 961 return error; 962 } 963 964 static void 965 igc_attach_counters(struct igc_softc *sc) 966 { 967 #ifdef IGC_EVENT_COUNTERS 968 969 /* Global counters */ 970 sc->sc_global_evcnts = kmem_zalloc( 971 IGC_GLOBAL_COUNTERS * sizeof(sc->sc_global_evcnts[0]), KM_SLEEP); 972 973 for (int cnt = 0; cnt < IGC_GLOBAL_COUNTERS; cnt++) { 974 evcnt_attach_dynamic(&sc->sc_global_evcnts[cnt], 975 igc_global_counters[cnt].type, NULL, 976 device_xname(sc->sc_dev), igc_global_counters[cnt].name); 977 } 978 979 /* Driver counters */ 980 sc->sc_driver_evcnts = kmem_zalloc( 981 IGC_DRIVER_COUNTERS * sizeof(sc->sc_driver_evcnts[0]), KM_SLEEP); 982 983 for (int cnt = 0; cnt < IGC_DRIVER_COUNTERS; cnt++) { 984 evcnt_attach_dynamic(&sc->sc_driver_evcnts[cnt], 985 igc_driver_counters[cnt].type, NULL, 986 device_xname(sc->sc_dev), igc_driver_counters[cnt].name); 987 } 988 989 for (int iq = 0; iq < sc->sc_nqueues; iq++) { 990 struct igc_queue *q = &sc->queues[iq]; 991 992 q->igcq_driver_counters = kmem_zalloc( 993 IGC_DRIVER_COUNTERS * sizeof(q->igcq_driver_counters[0]), 994 KM_SLEEP); 995 } 996 997 /* Queue counters */ 998 for (int iq = 0; iq < sc->sc_nqueues; iq++) { 999 struct igc_queue *q = &sc->queues[iq]; 1000 1001 snprintf(q->igcq_queue_evname, sizeof(q->igcq_queue_evname), 1002 "%s q%d", device_xname(sc->sc_dev), iq); 1003 1004 q->igcq_queue_evcnts = kmem_zalloc( 1005 IGC_QUEUE_COUNTERS * sizeof(q->igcq_queue_evcnts[0]), 1006 KM_SLEEP); 1007 1008 for (int cnt = 0; cnt < IGC_QUEUE_COUNTERS; cnt++) { 1009 evcnt_attach_dynamic(&q->igcq_queue_evcnts[cnt], 1010 igc_queue_counters[cnt].type, NULL, 1011 q->igcq_queue_evname, igc_queue_counters[cnt].name); 1012 } 1013 } 1014 1015 /* MAC counters */ 1016 snprintf(sc->sc_mac_evname, sizeof(sc->sc_mac_evname), 1017 "%s Mac Statistics", device_xname(sc->sc_dev)); 1018 1019 sc->sc_mac_evcnts = kmem_zalloc( 1020 IGC_MAC_COUNTERS * sizeof(sc->sc_mac_evcnts[0]), KM_SLEEP); 1021 1022 for (int cnt = 0; cnt < IGC_MAC_COUNTERS; cnt++) { 1023 evcnt_attach_dynamic(&sc->sc_mac_evcnts[cnt], EVCNT_TYPE_MISC, 1024 NULL, sc->sc_mac_evname, igc_mac_counters[cnt].name); 1025 } 1026 #endif 1027 } 1028 1029 static void 1030 igc_detach_counters(struct igc_softc *sc) 1031 { 1032 #ifdef IGC_EVENT_COUNTERS 1033 1034 /* Global counters */ 1035 for (int cnt = 0; cnt < IGC_GLOBAL_COUNTERS; cnt++) 1036 evcnt_detach(&sc->sc_global_evcnts[cnt]); 1037 1038 kmem_free(sc->sc_global_evcnts, 1039 IGC_GLOBAL_COUNTERS * sizeof(sc->sc_global_evcnts[0])); 1040 1041 /* Driver counters */ 1042 for (int iq = 0; iq < sc->sc_nqueues; iq++) { 1043 struct igc_queue *q = &sc->queues[iq]; 1044 1045 kmem_free(q->igcq_driver_counters, 1046 IGC_DRIVER_COUNTERS * sizeof(q->igcq_driver_counters[0])); 1047 } 1048 1049 for (int cnt = 0; cnt < IGC_DRIVER_COUNTERS; cnt++) 1050 evcnt_detach(&sc->sc_driver_evcnts[cnt]); 1051 1052 kmem_free(sc->sc_driver_evcnts, 1053 IGC_DRIVER_COUNTERS * sizeof(sc->sc_driver_evcnts[0])); 1054 1055 /* Queue counters */ 1056 for (int iq = 0; iq < sc->sc_nqueues; iq++) { 1057 struct igc_queue *q = &sc->queues[iq]; 1058 1059 for (int cnt = 0; cnt < IGC_QUEUE_COUNTERS; cnt++) 1060 evcnt_detach(&q->igcq_queue_evcnts[cnt]); 1061 1062 kmem_free(q->igcq_queue_evcnts, 1063 IGC_QUEUE_COUNTERS * sizeof(q->igcq_queue_evcnts[0])); 1064 } 1065 1066 /* MAC statistics */ 1067 for (int cnt = 0; cnt < IGC_MAC_COUNTERS; cnt++) 1068 evcnt_detach(&sc->sc_mac_evcnts[cnt]); 1069 1070 kmem_free(sc->sc_mac_evcnts, 1071 IGC_MAC_COUNTERS * sizeof(sc->sc_mac_evcnts[0])); 1072 #endif 1073 } 1074 1075 /* 1076 * XXX 1077 * FreeBSD uses 4-byte-wise read for 64-bit counters, while Linux just 1078 * drops hi words. 1079 */ 1080 static inline uint64_t __unused 1081 igc_read_mac_counter(struct igc_hw *hw, bus_size_t reg, bool is64) 1082 { 1083 uint64_t val; 1084 1085 val = IGC_READ_REG(hw, reg); 1086 if (is64) 1087 val += ((uint64_t)IGC_READ_REG(hw, reg + 4)) << 32; 1088 return val; 1089 } 1090 1091 static void 1092 igc_update_counters(struct igc_softc *sc) 1093 { 1094 #ifdef IGC_EVENT_COUNTERS 1095 1096 /* Global counters: nop */ 1097 1098 /* Driver counters */ 1099 uint64_t sum[IGC_DRIVER_COUNTERS]; 1100 1101 memset(sum, 0, sizeof(sum)); 1102 for (int iq = 0; iq < sc->sc_nqueues; iq++) { 1103 struct igc_queue *q = &sc->queues[iq]; 1104 1105 for (int cnt = 0; cnt < IGC_DRIVER_COUNTERS; cnt++) { 1106 sum[cnt] += IGC_QUEUE_DRIVER_COUNTER_VAL(q, cnt); 1107 IGC_QUEUE_DRIVER_COUNTER_STORE(q, cnt, 0); 1108 } 1109 } 1110 1111 for (int cnt = 0; cnt < IGC_DRIVER_COUNTERS; cnt++) 1112 IGC_DRIVER_COUNTER_ADD(sc, cnt, sum[cnt]); 1113 1114 /* Queue counters: nop */ 1115 1116 /* Mac statistics */ 1117 struct igc_hw *hw = &sc->hw; 1118 struct ifnet *ifp = &sc->sc_ec.ec_if; 1119 uint64_t iqdrops = 0; 1120 1121 for (int cnt = 0; cnt < IGC_MAC_COUNTERS; cnt++) { 1122 uint64_t val; 1123 bus_size_t regaddr = igc_mac_counters[cnt].reg; 1124 1125 val = igc_read_mac_counter(hw, regaddr, 1126 igc_mac_counters[cnt].is64); 1127 IGC_MAC_COUNTER_ADD(sc, cnt, val); 1128 /* XXX Count MPC to iqdrops. */ 1129 if (regaddr == IGC_MPC) 1130 iqdrops += val; 1131 } 1132 1133 for (int iq = 0; iq < sc->sc_nqueues; iq++) { 1134 uint32_t val; 1135 1136 /* XXX RQDPC should be visible via evcnt(9). */ 1137 val = IGC_READ_REG(hw, IGC_RQDPC(iq)); 1138 1139 /* RQDPC is not cleard on read. */ 1140 if (val != 0) 1141 IGC_WRITE_REG(hw, IGC_RQDPC(iq), 0); 1142 iqdrops += val; 1143 } 1144 1145 if (iqdrops != 0) 1146 if_statadd(ifp, if_iqdrops, iqdrops); 1147 #endif 1148 } 1149 1150 static void 1151 igc_clear_counters(struct igc_softc *sc) 1152 { 1153 #ifdef IGC_EVENT_COUNTERS 1154 1155 /* Global counters */ 1156 for (int cnt = 0; cnt < IGC_GLOBAL_COUNTERS; cnt++) 1157 IGC_GLOBAL_COUNTER_STORE(sc, cnt, 0); 1158 1159 /* Driver counters */ 1160 for (int iq = 0; iq < sc->sc_nqueues; iq++) { 1161 struct igc_queue *q = &sc->queues[iq]; 1162 1163 for (int cnt = 0; cnt < IGC_DRIVER_COUNTERS; cnt++) 1164 IGC_QUEUE_DRIVER_COUNTER_STORE(q, cnt, 0); 1165 } 1166 1167 for (int cnt = 0; cnt < IGC_DRIVER_COUNTERS; cnt++) 1168 IGC_DRIVER_COUNTER_STORE(sc, cnt, 0); 1169 1170 /* Queue counters */ 1171 for (int iq = 0; iq < sc->sc_nqueues; iq++) { 1172 struct igc_queue *q = &sc->queues[iq]; 1173 1174 for (int cnt = 0; cnt < IGC_QUEUE_COUNTERS; cnt++) 1175 IGC_QUEUE_COUNTER_STORE(q, cnt, 0); 1176 } 1177 1178 /* Mac statistics */ 1179 struct igc_hw *hw = &sc->hw; 1180 1181 for (int cnt = 0; cnt < IGC_MAC_COUNTERS; cnt++) { 1182 (void)igc_read_mac_counter(hw, igc_mac_counters[cnt].reg, 1183 igc_mac_counters[cnt].is64); 1184 IGC_MAC_COUNTER_STORE(sc, cnt, 0); 1185 } 1186 #endif 1187 } 1188 1189 static int 1190 igc_setup_msix(struct igc_softc *sc) 1191 { 1192 pci_chipset_tag_t pc = sc->osdep.os_pa.pa_pc; 1193 device_t dev = sc->sc_dev; 1194 pci_intr_handle_t *intrs; 1195 void **ihs; 1196 const char *intrstr; 1197 char intrbuf[PCI_INTRSTR_LEN]; 1198 char xnamebuf[MAX(32, MAXCOMLEN)]; 1199 int iq, error; 1200 1201 for (iq = 0, intrs = sc->sc_intrs, ihs = sc->sc_ihs; 1202 iq < sc->sc_nqueues; iq++, intrs++, ihs++) { 1203 struct igc_queue *q = &sc->queues[iq]; 1204 1205 snprintf(xnamebuf, sizeof(xnamebuf), "%s: txrx %d", 1206 device_xname(dev), iq); 1207 1208 intrstr = pci_intr_string(pc, *intrs, intrbuf, sizeof(intrbuf)); 1209 1210 pci_intr_setattr(pc, intrs, PCI_INTR_MPSAFE, true); 1211 *ihs = pci_intr_establish_xname(pc, *intrs, IPL_NET, 1212 igc_intr_queue, q, xnamebuf); 1213 if (*ihs == NULL) { 1214 aprint_error_dev(dev, 1215 "unable to establish txrx interrupt at %s\n", 1216 intrstr); 1217 return ENOBUFS; 1218 } 1219 aprint_normal_dev(dev, "txrx interrupting at %s\n", intrstr); 1220 1221 kcpuset_t *affinity; 1222 kcpuset_create(&affinity, true); 1223 kcpuset_set(affinity, iq % ncpu); 1224 error = interrupt_distribute(*ihs, affinity, NULL); 1225 if (error) { 1226 aprint_normal_dev(dev, 1227 "%s: unable to change affinity, use default CPU\n", 1228 intrstr); 1229 } 1230 kcpuset_destroy(affinity); 1231 1232 q->igcq_si = softint_establish(SOFTINT_NET | SOFTINT_MPSAFE, 1233 igc_handle_queue, q); 1234 if (q->igcq_si == NULL) { 1235 aprint_error_dev(dev, 1236 "%s: unable to establish softint\n", intrstr); 1237 return ENOBUFS; 1238 } 1239 1240 q->msix = iq; 1241 q->eims = 1 << iq; 1242 } 1243 1244 snprintf(xnamebuf, MAXCOMLEN, "%s_tx_rx", device_xname(dev)); 1245 error = workqueue_create(&sc->sc_queue_wq, xnamebuf, 1246 igc_handle_queue_work, sc, IGC_WORKQUEUE_PRI, IPL_NET, 1247 WQ_PERCPU | WQ_MPSAFE); 1248 if (error) { 1249 aprint_error_dev(dev, "workqueue_create failed\n"); 1250 return ENOBUFS; 1251 } 1252 sc->sc_txrx_workqueue = false; 1253 1254 intrstr = pci_intr_string(pc, *intrs, intrbuf, sizeof(intrbuf)); 1255 snprintf(xnamebuf, sizeof(xnamebuf), "%s: link", device_xname(dev)); 1256 pci_intr_setattr(pc, intrs, PCI_INTR_MPSAFE, true); 1257 *ihs = pci_intr_establish_xname(pc, *intrs, IPL_NET, 1258 igc_intr_link, sc, xnamebuf); 1259 if (*ihs == NULL) { 1260 aprint_error_dev(dev, 1261 "unable to establish link interrupt at %s\n", intrstr); 1262 return ENOBUFS; 1263 } 1264 aprint_normal_dev(dev, "link interrupting at %s\n", intrstr); 1265 /* use later in igc_configure_queues() */ 1266 sc->linkvec = iq; 1267 1268 return 0; 1269 } 1270 1271 static int 1272 igc_setup_msi(struct igc_softc *sc) 1273 { 1274 pci_chipset_tag_t pc = sc->osdep.os_pa.pa_pc; 1275 device_t dev = sc->sc_dev; 1276 pci_intr_handle_t *intr = sc->sc_intrs; 1277 void **ihs = sc->sc_ihs; 1278 const char *intrstr; 1279 char intrbuf[PCI_INTRSTR_LEN]; 1280 char xnamebuf[MAX(32, MAXCOMLEN)]; 1281 int error; 1282 1283 intrstr = pci_intr_string(pc, *intr, intrbuf, sizeof(intrbuf)); 1284 1285 snprintf(xnamebuf, sizeof(xnamebuf), "%s: msi", device_xname(dev)); 1286 pci_intr_setattr(pc, intr, PCI_INTR_MPSAFE, true); 1287 *ihs = pci_intr_establish_xname(pc, *intr, IPL_NET, 1288 igc_intr, sc, xnamebuf); 1289 if (*ihs == NULL) { 1290 aprint_error_dev(dev, 1291 "unable to establish interrupt at %s\n", intrstr); 1292 return ENOBUFS; 1293 } 1294 aprint_normal_dev(dev, "interrupting at %s\n", intrstr); 1295 1296 struct igc_queue *iq = sc->queues; 1297 iq->igcq_si = softint_establish(SOFTINT_NET | SOFTINT_MPSAFE, 1298 igc_handle_queue, iq); 1299 if (iq->igcq_si == NULL) { 1300 aprint_error_dev(dev, 1301 "%s: unable to establish softint\n", intrstr); 1302 return ENOBUFS; 1303 } 1304 1305 snprintf(xnamebuf, MAXCOMLEN, "%s_tx_rx", device_xname(dev)); 1306 error = workqueue_create(&sc->sc_queue_wq, xnamebuf, 1307 igc_handle_queue_work, sc, IGC_WORKQUEUE_PRI, IPL_NET, 1308 WQ_PERCPU | WQ_MPSAFE); 1309 if (error) { 1310 aprint_error_dev(dev, "workqueue_create failed\n"); 1311 return ENOBUFS; 1312 } 1313 sc->sc_txrx_workqueue = false; 1314 1315 sc->queues[0].msix = 0; 1316 sc->linkvec = 0; 1317 1318 return 0; 1319 } 1320 1321 static int 1322 igc_setup_intx(struct igc_softc *sc) 1323 { 1324 pci_chipset_tag_t pc = sc->osdep.os_pa.pa_pc; 1325 device_t dev = sc->sc_dev; 1326 pci_intr_handle_t *intr = sc->sc_intrs; 1327 void **ihs = sc->sc_ihs; 1328 const char *intrstr; 1329 char intrbuf[PCI_INTRSTR_LEN]; 1330 char xnamebuf[32]; 1331 1332 intrstr = pci_intr_string(pc, *intr, intrbuf, sizeof(intrbuf)); 1333 1334 snprintf(xnamebuf, sizeof(xnamebuf), "%s:intx", device_xname(dev)); 1335 pci_intr_setattr(pc, intr, PCI_INTR_MPSAFE, true); 1336 *ihs = pci_intr_establish_xname(pc, *intr, IPL_NET, 1337 igc_intr, sc, xnamebuf); 1338 if (*ihs == NULL) { 1339 aprint_error_dev(dev, 1340 "unable to establish interrupt at %s\n", intrstr); 1341 return ENOBUFS; 1342 } 1343 aprint_normal_dev(dev, "interrupting at %s\n", intrstr); 1344 1345 struct igc_queue *iq = sc->queues; 1346 iq->igcq_si = softint_establish(SOFTINT_NET | SOFTINT_MPSAFE, 1347 igc_handle_queue, iq); 1348 if (iq->igcq_si == NULL) { 1349 aprint_error_dev(dev, 1350 "%s: unable to establish softint\n", intrstr); 1351 return ENOBUFS; 1352 } 1353 1354 /* create workqueue? */ 1355 sc->sc_txrx_workqueue = false; 1356 1357 sc->queues[0].msix = 0; 1358 sc->linkvec = 0; 1359 1360 return 0; 1361 } 1362 1363 static int 1364 igc_dma_malloc(struct igc_softc *sc, bus_size_t size, struct igc_dma_alloc *dma) 1365 { 1366 struct igc_osdep *os = &sc->osdep; 1367 1368 dma->dma_tag = os->os_dmat; 1369 1370 if (bus_dmamap_create(dma->dma_tag, size, 1, size, 0, 1371 BUS_DMA_WAITOK | BUS_DMA_ALLOCNOW, &dma->dma_map)) 1372 return 1; 1373 if (bus_dmamem_alloc(dma->dma_tag, size, PAGE_SIZE, 0, &dma->dma_seg, 1374 1, &dma->dma_nseg, BUS_DMA_WAITOK)) 1375 goto destroy; 1376 /* 1377 * XXXRO 1378 * 1379 * Coherent mapping for descriptors is required for now. 1380 * 1381 * Both TX and RX descriptors are 16-byte length, which is shorter 1382 * than dcache lines on modern CPUs. Therefore, sync for a descriptor 1383 * may overwrite DMA read for descriptors in the same cache line. 1384 * 1385 * Can't we avoid this by use cache-line-aligned descriptors at once? 1386 */ 1387 if (bus_dmamem_map(dma->dma_tag, &dma->dma_seg, dma->dma_nseg, size, 1388 &dma->dma_vaddr, BUS_DMA_WAITOK | BUS_DMA_COHERENT /* XXXRO */)) 1389 goto free; 1390 if (bus_dmamap_load(dma->dma_tag, dma->dma_map, dma->dma_vaddr, size, 1391 NULL, BUS_DMA_WAITOK)) 1392 goto unmap; 1393 1394 dma->dma_size = size; 1395 1396 return 0; 1397 unmap: 1398 bus_dmamem_unmap(dma->dma_tag, dma->dma_vaddr, size); 1399 free: 1400 bus_dmamem_free(dma->dma_tag, &dma->dma_seg, dma->dma_nseg); 1401 destroy: 1402 bus_dmamap_destroy(dma->dma_tag, dma->dma_map); 1403 dma->dma_map = NULL; 1404 dma->dma_tag = NULL; 1405 return 1; 1406 } 1407 1408 static void 1409 igc_dma_free(struct igc_softc *sc, struct igc_dma_alloc *dma) 1410 { 1411 1412 if (dma->dma_tag == NULL) 1413 return; 1414 1415 if (dma->dma_map != NULL) { 1416 bus_dmamap_sync(dma->dma_tag, dma->dma_map, 0, 1417 dma->dma_map->dm_mapsize, 1418 BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE); 1419 bus_dmamap_unload(dma->dma_tag, dma->dma_map); 1420 bus_dmamem_unmap(dma->dma_tag, dma->dma_vaddr, dma->dma_size); 1421 bus_dmamem_free(dma->dma_tag, &dma->dma_seg, dma->dma_nseg); 1422 bus_dmamap_destroy(dma->dma_tag, dma->dma_map); 1423 dma->dma_map = NULL; 1424 } 1425 } 1426 1427 /********************************************************************* 1428 * 1429 * Setup networking device structure and register an interface. 1430 * 1431 **********************************************************************/ 1432 static void 1433 igc_setup_interface(struct igc_softc *sc) 1434 { 1435 struct ifnet *ifp = &sc->sc_ec.ec_if; 1436 1437 strlcpy(ifp->if_xname, device_xname(sc->sc_dev), sizeof(ifp->if_xname)); 1438 ifp->if_softc = sc; 1439 ifp->if_flags = IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST; 1440 ifp->if_extflags = IFEF_MPSAFE; 1441 ifp->if_ioctl = igc_ioctl; 1442 ifp->if_start = igc_start; 1443 if (sc->sc_nqueues > 1) 1444 ifp->if_transmit = igc_transmit; 1445 ifp->if_watchdog = igc_watchdog; 1446 ifp->if_init = igc_init; 1447 ifp->if_stop = igc_stop; 1448 1449 ifp->if_capabilities = IFCAP_TSOv4 | IFCAP_TSOv6; 1450 1451 ifp->if_capabilities |= 1452 IFCAP_CSUM_IPv4_Tx | IFCAP_CSUM_IPv4_Rx | 1453 IFCAP_CSUM_TCPv4_Tx | IFCAP_CSUM_TCPv4_Rx | 1454 IFCAP_CSUM_UDPv4_Tx | IFCAP_CSUM_UDPv4_Rx | 1455 IFCAP_CSUM_TCPv6_Tx | IFCAP_CSUM_TCPv6_Rx | 1456 IFCAP_CSUM_UDPv6_Tx | IFCAP_CSUM_UDPv6_Rx; 1457 1458 ifp->if_capenable = 0; 1459 1460 sc->sc_ec.ec_capabilities |= 1461 ETHERCAP_JUMBO_MTU | ETHERCAP_VLAN_MTU; 1462 1463 IFQ_SET_MAXLEN(&ifp->if_snd, sc->num_tx_desc - 1); 1464 IFQ_SET_READY(&ifp->if_snd); 1465 1466 #if NVLAN > 0 1467 sc->sc_ec.ec_capabilities |= ETHERCAP_VLAN_HWTAGGING; 1468 #endif 1469 1470 mutex_init(&sc->sc_core_lock, MUTEX_DEFAULT, IPL_NET); 1471 1472 /* Initialize ifmedia structures. */ 1473 sc->sc_ec.ec_ifmedia = &sc->media; 1474 ifmedia_init_with_lock(&sc->media, IFM_IMASK, igc_media_change, 1475 igc_media_status, &sc->sc_core_lock); 1476 ifmedia_add(&sc->media, IFM_ETHER | IFM_10_T, 0, NULL); 1477 ifmedia_add(&sc->media, IFM_ETHER | IFM_10_T | IFM_FDX, 0, NULL); 1478 ifmedia_add(&sc->media, IFM_ETHER | IFM_100_TX, 0, NULL); 1479 ifmedia_add(&sc->media, IFM_ETHER | IFM_100_TX | IFM_FDX, 0, NULL); 1480 ifmedia_add(&sc->media, IFM_ETHER | IFM_1000_T | IFM_FDX, 0, NULL); 1481 ifmedia_add(&sc->media, IFM_ETHER | IFM_2500_T | IFM_FDX, 0, NULL); 1482 ifmedia_add(&sc->media, IFM_ETHER | IFM_AUTO, 0, NULL); 1483 ifmedia_set(&sc->media, IFM_ETHER | IFM_AUTO); 1484 1485 sc->sc_rx_intr_process_limit = IGC_RX_INTR_PROCESS_LIMIT_DEFAULT; 1486 sc->sc_tx_intr_process_limit = IGC_TX_INTR_PROCESS_LIMIT_DEFAULT; 1487 sc->sc_rx_process_limit = IGC_RX_PROCESS_LIMIT_DEFAULT; 1488 sc->sc_tx_process_limit = IGC_TX_PROCESS_LIMIT_DEFAULT; 1489 1490 if_initialize(ifp); 1491 sc->sc_ipq = if_percpuq_create(ifp); 1492 if_deferred_start_init(ifp, NULL); 1493 ether_ifattach(ifp, sc->hw.mac.addr); 1494 ether_set_ifflags_cb(&sc->sc_ec, igc_ifflags_cb); 1495 if_register(ifp); 1496 } 1497 1498 static int 1499 igc_init(struct ifnet *ifp) 1500 { 1501 struct igc_softc *sc = ifp->if_softc; 1502 int error; 1503 1504 mutex_enter(&sc->sc_core_lock); 1505 error = igc_init_locked(sc); 1506 mutex_exit(&sc->sc_core_lock); 1507 1508 return error; 1509 } 1510 1511 static int 1512 igc_init_locked(struct igc_softc *sc) 1513 { 1514 struct ethercom *ec = &sc->sc_ec; 1515 struct ifnet *ifp = &ec->ec_if; 1516 1517 DPRINTF(CFG, "called\n"); 1518 1519 KASSERT(mutex_owned(&sc->sc_core_lock)); 1520 1521 if (ISSET(ifp->if_flags, IFF_RUNNING)) 1522 igc_stop_locked(sc); 1523 1524 /* Put the address into the receive address array. */ 1525 igc_rar_set(&sc->hw, sc->hw.mac.addr, 0); 1526 1527 /* Initialize the hardware. */ 1528 igc_reset(sc); 1529 igc_update_link_status(sc); 1530 1531 /* Setup VLAN support, basic and offload if available. */ 1532 IGC_WRITE_REG(&sc->hw, IGC_VET, ETHERTYPE_VLAN); 1533 1534 igc_initialize_transmit_unit(sc); 1535 igc_initialize_receive_unit(sc); 1536 1537 if (ec->ec_capenable & ETHERCAP_VLAN_HWTAGGING) { 1538 uint32_t ctrl = IGC_READ_REG(&sc->hw, IGC_CTRL); 1539 ctrl |= IGC_CTRL_VME; 1540 IGC_WRITE_REG(&sc->hw, IGC_CTRL, ctrl); 1541 } 1542 1543 /* Setup multicast table. */ 1544 igc_set_filter(sc); 1545 1546 igc_clear_hw_cntrs_base_generic(&sc->hw); 1547 1548 if (sc->sc_intr_type == PCI_INTR_TYPE_MSIX) 1549 igc_configure_queues(sc); 1550 1551 /* This clears any pending interrupts */ 1552 IGC_READ_REG(&sc->hw, IGC_ICR); 1553 IGC_WRITE_REG(&sc->hw, IGC_ICS, IGC_ICS_LSC); 1554 1555 /* The driver can now take control from firmware. */ 1556 igc_get_hw_control(sc); 1557 1558 /* Set Energy Efficient Ethernet. */ 1559 igc_set_eee_i225(&sc->hw, true, true, true); 1560 1561 for (int iq = 0; iq < sc->sc_nqueues; iq++) { 1562 struct rx_ring *rxr = &sc->rx_rings[iq]; 1563 1564 mutex_enter(&rxr->rxr_lock); 1565 igc_rxfill(rxr); 1566 mutex_exit(&rxr->rxr_lock); 1567 } 1568 1569 sc->sc_core_stopping = false; 1570 1571 ifp->if_flags |= IFF_RUNNING; 1572 1573 /* Save last flags for the callback */ 1574 sc->sc_if_flags = ifp->if_flags; 1575 1576 callout_schedule(&sc->sc_tick_ch, hz); 1577 1578 igc_enable_intr(sc); 1579 1580 return 0; 1581 } 1582 1583 static inline int 1584 igc_load_mbuf(struct igc_queue *q, bus_dma_tag_t dmat, bus_dmamap_t map, 1585 struct mbuf *m) 1586 { 1587 int error; 1588 1589 error = bus_dmamap_load_mbuf(dmat, map, m, 1590 BUS_DMA_WRITE | BUS_DMA_NOWAIT); 1591 1592 if (__predict_false(error == EFBIG)) { 1593 IGC_DRIVER_EVENT(q, txdma_efbig, 1); 1594 m = m_defrag(m, M_NOWAIT); 1595 if (__predict_false(m == NULL)) { 1596 IGC_DRIVER_EVENT(q, txdma_defrag, 1); 1597 return ENOBUFS; 1598 } 1599 error = bus_dmamap_load_mbuf(dmat, map, m, 1600 BUS_DMA_WRITE | BUS_DMA_NOWAIT); 1601 } 1602 1603 switch (error) { 1604 case 0: 1605 break; 1606 case ENOMEM: 1607 IGC_DRIVER_EVENT(q, txdma_enomem, 1); 1608 break; 1609 case EINVAL: 1610 IGC_DRIVER_EVENT(q, txdma_einval, 1); 1611 break; 1612 case EAGAIN: 1613 IGC_DRIVER_EVENT(q, txdma_eagain, 1); 1614 break; 1615 default: 1616 IGC_DRIVER_EVENT(q, txdma_other, 1); 1617 break; 1618 } 1619 1620 return error; 1621 } 1622 1623 #define IGC_TX_START 1 1624 #define IGC_TX_TRANSMIT 2 1625 1626 static void 1627 igc_start(struct ifnet *ifp) 1628 { 1629 struct igc_softc *sc = ifp->if_softc; 1630 1631 if (__predict_false(!sc->link_active)) { 1632 IFQ_PURGE(&ifp->if_snd); 1633 return; 1634 } 1635 1636 struct tx_ring *txr = &sc->tx_rings[0]; /* queue 0 */ 1637 mutex_enter(&txr->txr_lock); 1638 igc_tx_common_locked(ifp, txr, IGC_TX_START); 1639 mutex_exit(&txr->txr_lock); 1640 } 1641 1642 static inline u_int 1643 igc_select_txqueue(struct igc_softc *sc, struct mbuf *m __unused) 1644 { 1645 const u_int cpuid = cpu_index(curcpu()); 1646 1647 return cpuid % sc->sc_nqueues; 1648 } 1649 1650 static int 1651 igc_transmit(struct ifnet *ifp, struct mbuf *m) 1652 { 1653 struct igc_softc *sc = ifp->if_softc; 1654 const u_int qid = igc_select_txqueue(sc, m); 1655 struct tx_ring *txr = &sc->tx_rings[qid]; 1656 struct igc_queue *q = txr->txr_igcq; 1657 1658 if (__predict_false(!pcq_put(txr->txr_interq, m))) { 1659 IGC_QUEUE_EVENT(q, tx_pcq_drop, 1); 1660 m_freem(m); 1661 return ENOBUFS; 1662 } 1663 1664 mutex_enter(&txr->txr_lock); 1665 igc_tx_common_locked(ifp, txr, IGC_TX_TRANSMIT); 1666 mutex_exit(&txr->txr_lock); 1667 1668 return 0; 1669 } 1670 1671 static void 1672 igc_tx_common_locked(struct ifnet *ifp, struct tx_ring *txr, int caller) 1673 { 1674 struct igc_softc *sc = ifp->if_softc; 1675 struct igc_queue *q = txr->txr_igcq; 1676 net_stat_ref_t nsr = IF_STAT_GETREF(ifp); 1677 int prod, free, last = -1; 1678 bool post = false; 1679 1680 prod = txr->next_avail_desc; 1681 free = txr->next_to_clean; 1682 if (free <= prod) 1683 free += sc->num_tx_desc; 1684 free -= prod; 1685 1686 DPRINTF(TX, "%s: begin: msix %d prod %d n2c %d free %d\n", 1687 caller == IGC_TX_TRANSMIT ? "transmit" : "start", 1688 txr->me, prod, txr->next_to_clean, free); 1689 1690 for (;;) { 1691 struct mbuf *m; 1692 1693 if (__predict_false(free <= IGC_MAX_SCATTER)) { 1694 IGC_QUEUE_EVENT(q, tx_no_desc, 1); 1695 break; 1696 } 1697 1698 if (caller == IGC_TX_TRANSMIT) 1699 m = pcq_get(txr->txr_interq); 1700 else 1701 IFQ_DEQUEUE(&ifp->if_snd, m); 1702 if (__predict_false(m == NULL)) 1703 break; 1704 1705 struct igc_tx_buf *txbuf = &txr->tx_buffers[prod]; 1706 bus_dmamap_t map = txbuf->map; 1707 1708 if (__predict_false( 1709 igc_load_mbuf(q, txr->txdma.dma_tag, map, m))) { 1710 if (caller == IGC_TX_TRANSMIT) 1711 IGC_QUEUE_EVENT(q, tx_pcq_drop, 1); 1712 m_freem(m); 1713 if_statinc_ref(ifp, nsr, if_oerrors); 1714 continue; 1715 } 1716 1717 uint32_t ctx_cmd_type_len = 0, olinfo_status = 0; 1718 if (igc_tx_ctx_setup(txr, m, prod, &ctx_cmd_type_len, 1719 &olinfo_status)) { 1720 IGC_QUEUE_EVENT(q, tx_ctx, 1); 1721 /* Consume the first descriptor */ 1722 prod = igc_txdesc_incr(sc, prod); 1723 free--; 1724 } 1725 1726 bus_dmamap_sync(txr->txdma.dma_tag, map, 0, 1727 map->dm_mapsize, BUS_DMASYNC_PREWRITE); 1728 1729 for (int i = 0; i < map->dm_nsegs; i++) { 1730 union igc_adv_tx_desc *txdesc = &txr->tx_base[prod]; 1731 1732 uint32_t cmd_type_len = ctx_cmd_type_len | 1733 IGC_ADVTXD_DCMD_IFCS | IGC_ADVTXD_DTYP_DATA | 1734 IGC_ADVTXD_DCMD_DEXT | map->dm_segs[i].ds_len; 1735 if (i == map->dm_nsegs - 1) { 1736 cmd_type_len |= 1737 IGC_ADVTXD_DCMD_EOP | IGC_ADVTXD_DCMD_RS; 1738 } 1739 1740 igc_txdesc_sync(txr, prod, 1741 BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE); 1742 htolem64(&txdesc->read.buffer_addr, 1743 map->dm_segs[i].ds_addr); 1744 htolem32(&txdesc->read.cmd_type_len, cmd_type_len); 1745 htolem32(&txdesc->read.olinfo_status, olinfo_status); 1746 igc_txdesc_sync(txr, prod, 1747 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); 1748 1749 last = prod; 1750 prod = igc_txdesc_incr(sc, prod); 1751 } 1752 1753 txbuf->m_head = m; 1754 txbuf->eop_index = last; 1755 1756 bpf_mtap(ifp, m, BPF_D_OUT); 1757 1758 if_statadd_ref(ifp, nsr, if_obytes, m->m_pkthdr.len); 1759 if (m->m_flags & M_MCAST) 1760 if_statinc_ref(ifp, nsr, if_omcasts); 1761 IGC_QUEUE_EVENT(q, tx_packets, 1); 1762 IGC_QUEUE_EVENT(q, tx_bytes, m->m_pkthdr.len); 1763 1764 free -= map->dm_nsegs; 1765 post = true; 1766 } 1767 1768 if (post) { 1769 txr->next_avail_desc = prod; 1770 IGC_WRITE_REG(&sc->hw, IGC_TDT(txr->me), prod); 1771 } 1772 1773 DPRINTF(TX, "%s: done : msix %d prod %d n2c %d free %d\n", 1774 caller == IGC_TX_TRANSMIT ? "transmit" : "start", 1775 txr->me, prod, txr->next_to_clean, free); 1776 1777 IF_STAT_PUTREF(ifp); 1778 } 1779 1780 static bool 1781 igc_txeof(struct tx_ring *txr, u_int limit) 1782 { 1783 struct igc_softc *sc = txr->sc; 1784 struct ifnet *ifp = &sc->sc_ec.ec_if; 1785 int cons, prod; 1786 bool more = false; 1787 1788 prod = txr->next_avail_desc; 1789 cons = txr->next_to_clean; 1790 1791 if (cons == prod) { 1792 DPRINTF(TX, "false: msix %d cons %d prod %d\n", 1793 txr->me, cons, prod); 1794 return false; 1795 } 1796 1797 do { 1798 struct igc_tx_buf *txbuf = &txr->tx_buffers[cons]; 1799 const int last = txbuf->eop_index; 1800 1801 membar_consumer(); /* XXXRO necessary? */ 1802 1803 KASSERT(last != -1); 1804 union igc_adv_tx_desc *txdesc = &txr->tx_base[last]; 1805 igc_txdesc_sync(txr, last, BUS_DMASYNC_POSTREAD); 1806 const uint32_t status = le32toh(txdesc->wb.status); 1807 igc_txdesc_sync(txr, last, BUS_DMASYNC_PREREAD); 1808 1809 if (!(status & IGC_TXD_STAT_DD)) 1810 break; 1811 1812 if (limit-- == 0) { 1813 more = true; 1814 DPRINTF(TX, "pending TX " 1815 "msix %d cons %d last %d prod %d " 1816 "status 0x%08x\n", 1817 txr->me, cons, last, prod, status); 1818 break; 1819 } 1820 1821 DPRINTF(TX, "handled TX " 1822 "msix %d cons %d last %d prod %d " 1823 "status 0x%08x\n", 1824 txr->me, cons, last, prod, status); 1825 1826 if_statinc(ifp, if_opackets); 1827 1828 bus_dmamap_t map = txbuf->map; 1829 bus_dmamap_sync(txr->txdma.dma_tag, map, 0, map->dm_mapsize, 1830 BUS_DMASYNC_POSTWRITE); 1831 bus_dmamap_unload(txr->txdma.dma_tag, map); 1832 m_freem(txbuf->m_head); 1833 1834 txbuf->m_head = NULL; 1835 txbuf->eop_index = -1; 1836 1837 cons = igc_txdesc_incr(sc, last); 1838 } while (cons != prod); 1839 1840 txr->next_to_clean = cons; 1841 1842 return more; 1843 } 1844 1845 static void 1846 igc_intr_barrier(struct igc_softc *sc __unused) 1847 { 1848 1849 xc_barrier(0); 1850 } 1851 1852 static void 1853 igc_stop(struct ifnet *ifp, int disable) 1854 { 1855 struct igc_softc *sc = ifp->if_softc; 1856 1857 mutex_enter(&sc->sc_core_lock); 1858 igc_stop_locked(sc); 1859 mutex_exit(&sc->sc_core_lock); 1860 } 1861 1862 /********************************************************************* 1863 * 1864 * This routine disables all traffic on the adapter by issuing a 1865 * global reset on the MAC. 1866 * 1867 **********************************************************************/ 1868 static void 1869 igc_stop_locked(struct igc_softc *sc) 1870 { 1871 struct ifnet *ifp = &sc->sc_ec.ec_if; 1872 1873 DPRINTF(CFG, "called\n"); 1874 1875 KASSERT(mutex_owned(&sc->sc_core_lock)); 1876 1877 /* 1878 * If stopping processing has already started, do nothing. 1879 */ 1880 if ((ifp->if_flags & IFF_RUNNING) == 0) 1881 return; 1882 1883 /* Tell the stack that the interface is no longer active. */ 1884 ifp->if_flags &= ~IFF_RUNNING; 1885 1886 /* 1887 * igc_handle_queue() can enable interrupts, so wait for completion of 1888 * last igc_handle_queue() after unset IFF_RUNNING. 1889 */ 1890 mutex_exit(&sc->sc_core_lock); 1891 igc_barrier_handle_queue(sc); 1892 mutex_enter(&sc->sc_core_lock); 1893 1894 sc->sc_core_stopping = true; 1895 1896 igc_disable_intr(sc); 1897 1898 callout_halt(&sc->sc_tick_ch, &sc->sc_core_lock); 1899 1900 igc_reset_hw(&sc->hw); 1901 IGC_WRITE_REG(&sc->hw, IGC_WUC, 0); 1902 1903 /* 1904 * Wait for completion of interrupt handlers. 1905 */ 1906 mutex_exit(&sc->sc_core_lock); 1907 igc_intr_barrier(sc); 1908 mutex_enter(&sc->sc_core_lock); 1909 1910 igc_update_link_status(sc); 1911 1912 for (int iq = 0; iq < sc->sc_nqueues; iq++) { 1913 struct tx_ring *txr = &sc->tx_rings[iq]; 1914 1915 igc_withdraw_transmit_packets(txr, false); 1916 } 1917 1918 for (int iq = 0; iq < sc->sc_nqueues; iq++) { 1919 struct rx_ring *rxr = &sc->rx_rings[iq]; 1920 1921 igc_clear_receive_status(rxr); 1922 } 1923 1924 /* Save last flags for the callback */ 1925 sc->sc_if_flags = ifp->if_flags; 1926 } 1927 1928 /********************************************************************* 1929 * Ioctl entry point 1930 * 1931 * igc_ioctl is called when the user wants to configure the 1932 * interface. 1933 * 1934 * return 0 on success, positive on failure 1935 **********************************************************************/ 1936 static int 1937 igc_ioctl(struct ifnet * ifp, u_long cmd, void *data) 1938 { 1939 struct igc_softc *sc __unused = ifp->if_softc; 1940 int s; 1941 int error; 1942 1943 DPRINTF(CFG, "cmd 0x%016lx\n", cmd); 1944 1945 switch (cmd) { 1946 case SIOCADDMULTI: 1947 case SIOCDELMULTI: 1948 break; 1949 default: 1950 KASSERT(IFNET_LOCKED(ifp)); 1951 } 1952 1953 if (cmd == SIOCZIFDATA) { 1954 mutex_enter(&sc->sc_core_lock); 1955 igc_clear_counters(sc); 1956 mutex_exit(&sc->sc_core_lock); 1957 } 1958 1959 switch (cmd) { 1960 #ifdef IF_RXR 1961 case SIOCGIFRXR: 1962 s = splnet(); 1963 error = igc_rxrinfo(sc, (struct if_rxrinfo *)ifr->ifr_data); 1964 splx(s); 1965 break; 1966 #endif 1967 default: 1968 s = splnet(); 1969 error = ether_ioctl(ifp, cmd, data); 1970 splx(s); 1971 break; 1972 } 1973 1974 if (error != ENETRESET) 1975 return error; 1976 1977 error = 0; 1978 1979 if (cmd == SIOCSIFCAP) 1980 error = if_init(ifp); 1981 else if ((cmd == SIOCADDMULTI) || (cmd == SIOCDELMULTI)) { 1982 mutex_enter(&sc->sc_core_lock); 1983 if (sc->sc_if_flags & IFF_RUNNING) { 1984 /* 1985 * Multicast list has changed; set the hardware filter 1986 * accordingly. 1987 */ 1988 igc_disable_intr(sc); 1989 igc_set_filter(sc); 1990 igc_enable_intr(sc); 1991 } 1992 mutex_exit(&sc->sc_core_lock); 1993 } 1994 1995 return error; 1996 } 1997 1998 #ifdef IF_RXR 1999 static int 2000 igc_rxrinfo(struct igc_softc *sc, struct if_rxrinfo *ifri) 2001 { 2002 struct if_rxring_info *ifr, ifr1; 2003 int error; 2004 2005 if (sc->sc_nqueues > 1) { 2006 ifr = kmem_zalloc(sc->sc_nqueues * sizeof(*ifr), KM_SLEEP); 2007 } else { 2008 ifr = &ifr1; 2009 memset(ifr, 0, sizeof(*ifr)); 2010 } 2011 2012 for (int iq = 0; iq < sc->sc_nqueues; iq++) { 2013 struct rx_ring *rxr = &sc->rx_rings[iq]; 2014 2015 ifr[iq].ifr_size = MCLBYTES; 2016 snprintf(ifr[iq].ifr_name, sizeof(ifr[iq].ifr_name), "%d", iq); 2017 ifr[iq].ifr_info = rxr->rx_ring; 2018 } 2019 2020 error = if_rxr_info_ioctl(ifri, sc->sc_nqueues, ifr); 2021 if (sc->sc_nqueues > 1) 2022 kmem_free(ifr, sc->sc_nqueues * sizeof(*ifr)); 2023 2024 return error; 2025 } 2026 #endif 2027 2028 static void 2029 igc_rxfill(struct rx_ring *rxr) 2030 { 2031 struct igc_softc *sc = rxr->sc; 2032 int id; 2033 2034 for (id = 0; id < sc->num_rx_desc; id++) { 2035 if (igc_get_buf(rxr, id, false)) { 2036 panic("%s: msix=%d i=%d\n", __func__, rxr->me, id); 2037 } 2038 } 2039 2040 id = sc->num_rx_desc - 1; 2041 rxr->last_desc_filled = id; 2042 IGC_WRITE_REG(&sc->hw, IGC_RDT(rxr->me), id); 2043 rxr->next_to_check = 0; 2044 } 2045 2046 static void 2047 igc_rxrefill(struct rx_ring *rxr, int end) 2048 { 2049 struct igc_softc *sc = rxr->sc; 2050 int id; 2051 2052 for (id = rxr->next_to_check; id != end; id = igc_rxdesc_incr(sc, id)) { 2053 if (igc_get_buf(rxr, id, true)) { 2054 /* XXXRO */ 2055 panic("%s: msix=%d id=%d\n", __func__, rxr->me, id); 2056 } 2057 } 2058 2059 id = igc_rxdesc_decr(sc, id); 2060 DPRINTF(RX, "%s RDT %d id %d\n", 2061 rxr->last_desc_filled == id ? "same" : "diff", 2062 rxr->last_desc_filled, id); 2063 rxr->last_desc_filled = id; 2064 IGC_WRITE_REG(&sc->hw, IGC_RDT(rxr->me), id); 2065 } 2066 2067 /********************************************************************* 2068 * 2069 * This routine executes in interrupt context. It replenishes 2070 * the mbufs in the descriptor and sends data which has been 2071 * dma'ed into host memory to upper layer. 2072 * 2073 *********************************************************************/ 2074 static bool 2075 igc_rxeof(struct rx_ring *rxr, u_int limit) 2076 { 2077 struct igc_softc *sc = rxr->sc; 2078 struct igc_queue *q = rxr->rxr_igcq; 2079 struct ifnet *ifp = &sc->sc_ec.ec_if; 2080 int id; 2081 bool more = false; 2082 2083 id = rxr->next_to_check; 2084 for (;;) { 2085 union igc_adv_rx_desc *rxdesc = &rxr->rx_base[id]; 2086 struct igc_rx_buf *rxbuf, *nxbuf; 2087 struct mbuf *mp, *m; 2088 2089 igc_rxdesc_sync(rxr, id, 2090 BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE); 2091 2092 const uint32_t staterr = le32toh(rxdesc->wb.upper.status_error); 2093 2094 if (!ISSET(staterr, IGC_RXD_STAT_DD)) { 2095 igc_rxdesc_sync(rxr, id, 2096 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); 2097 break; 2098 } 2099 2100 if (limit-- == 0) { 2101 igc_rxdesc_sync(rxr, id, 2102 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); 2103 DPRINTF(RX, "more=true\n"); 2104 more = true; 2105 break; 2106 } 2107 2108 /* Zero out the receive descriptors status. */ 2109 rxdesc->wb.upper.status_error = 0; 2110 2111 /* Pull the mbuf off the ring. */ 2112 rxbuf = &rxr->rx_buffers[id]; 2113 bus_dmamap_t map = rxbuf->map; 2114 bus_dmamap_sync(rxr->rxdma.dma_tag, map, 2115 0, map->dm_mapsize, BUS_DMASYNC_POSTREAD); 2116 bus_dmamap_unload(rxr->rxdma.dma_tag, map); 2117 2118 mp = rxbuf->buf; 2119 rxbuf->buf = NULL; 2120 2121 const bool eop = staterr & IGC_RXD_STAT_EOP; 2122 const uint16_t len = le16toh(rxdesc->wb.upper.length); 2123 2124 #if NVLAN > 0 2125 const uint16_t vtag = le16toh(rxdesc->wb.upper.vlan); 2126 #endif 2127 2128 const uint32_t ptype = le32toh(rxdesc->wb.lower.lo_dword.data) & 2129 IGC_PKTTYPE_MASK; 2130 2131 const uint32_t hash __unused = 2132 le32toh(rxdesc->wb.lower.hi_dword.rss); 2133 const uint16_t hashtype __unused = 2134 le16toh(rxdesc->wb.lower.lo_dword.hs_rss.pkt_info) & 2135 IGC_RXDADV_RSSTYPE_MASK; 2136 2137 igc_rxdesc_sync(rxr, id, 2138 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); 2139 2140 if (__predict_false(staterr & IGC_RXDEXT_STATERR_RXE)) { 2141 m_freem(rxbuf->fmp); 2142 rxbuf->fmp = NULL; 2143 2144 m_freem(mp); 2145 m = NULL; 2146 2147 if_statinc(ifp, if_ierrors); 2148 IGC_QUEUE_EVENT(q, rx_discard, 1); 2149 2150 DPRINTF(RX, "ierrors++\n"); 2151 2152 goto next_desc; 2153 } 2154 2155 if (__predict_false(mp == NULL)) { 2156 panic("%s: igc_rxeof: NULL mbuf in slot %d " 2157 "(filled %d)", device_xname(sc->sc_dev), 2158 id, rxr->last_desc_filled); 2159 } 2160 2161 if (!eop) { 2162 /* 2163 * Figure out the next descriptor of this frame. 2164 */ 2165 int nextp = igc_rxdesc_incr(sc, id); 2166 2167 nxbuf = &rxr->rx_buffers[nextp]; 2168 /* 2169 * TODO prefetch(nxbuf); 2170 */ 2171 } 2172 2173 mp->m_len = len; 2174 2175 m = rxbuf->fmp; 2176 rxbuf->fmp = NULL; 2177 2178 if (m != NULL) { 2179 m->m_pkthdr.len += mp->m_len; 2180 } else { 2181 m = mp; 2182 m->m_pkthdr.len = mp->m_len; 2183 #if NVLAN > 0 2184 if (staterr & IGC_RXD_STAT_VP) 2185 vlan_set_tag(m, vtag); 2186 #endif 2187 } 2188 2189 /* Pass the head pointer on */ 2190 if (!eop) { 2191 nxbuf->fmp = m; 2192 m = NULL; 2193 mp->m_next = nxbuf->buf; 2194 } else { 2195 m_set_rcvif(m, ifp); 2196 2197 m->m_pkthdr.csum_flags = igc_rx_checksum(q, 2198 ifp->if_capenable, staterr, ptype); 2199 2200 #ifdef notyet 2201 if (hashtype != IGC_RXDADV_RSSTYPE_NONE) { 2202 m->m_pkthdr.ph_flowid = hash; 2203 SET(m->m_pkthdr.csum_flags, M_FLOWID); 2204 } 2205 ml_enqueue(&ml, m); 2206 #endif 2207 2208 if_percpuq_enqueue(sc->sc_ipq, m); 2209 2210 if_statinc(ifp, if_ipackets); 2211 IGC_QUEUE_EVENT(q, rx_packets, 1); 2212 IGC_QUEUE_EVENT(q, rx_bytes, m->m_pkthdr.len); 2213 } 2214 next_desc: 2215 /* Advance our pointers to the next descriptor. */ 2216 id = igc_rxdesc_incr(sc, id); 2217 } 2218 2219 DPRINTF(RX, "fill queue[%d]\n", rxr->me); 2220 igc_rxrefill(rxr, id); 2221 2222 DPRINTF(RX, "%s n2c %d id %d\n", 2223 rxr->next_to_check == id ? "same" : "diff", 2224 rxr->next_to_check, id); 2225 rxr->next_to_check = id; 2226 2227 #ifdef OPENBSD 2228 if (!(staterr & IGC_RXD_STAT_DD)) 2229 return 0; 2230 #endif 2231 2232 return more; 2233 } 2234 2235 /********************************************************************* 2236 * 2237 * Verify that the hardware indicated that the checksum is valid. 2238 * Inform the stack about the status of checksum so that stack 2239 * doesn't spend time verifying the checksum. 2240 * 2241 *********************************************************************/ 2242 static int 2243 igc_rx_checksum(struct igc_queue *q, uint64_t capenable, uint32_t staterr, 2244 uint32_t ptype) 2245 { 2246 const uint16_t status = (uint16_t)staterr; 2247 const uint8_t errors = (uint8_t)(staterr >> 24); 2248 int flags = 0; 2249 2250 if ((status & IGC_RXD_STAT_IPCS) != 0 && 2251 (capenable & IFCAP_CSUM_IPv4_Rx) != 0) { 2252 IGC_DRIVER_EVENT(q, rx_ipcs, 1); 2253 flags |= M_CSUM_IPv4; 2254 if (__predict_false((errors & IGC_RXD_ERR_IPE) != 0)) { 2255 IGC_DRIVER_EVENT(q, rx_ipcs_bad, 1); 2256 flags |= M_CSUM_IPv4_BAD; 2257 } 2258 } 2259 2260 if ((status & IGC_RXD_STAT_TCPCS) != 0) { 2261 IGC_DRIVER_EVENT(q, rx_tcpcs, 1); 2262 if ((capenable & IFCAP_CSUM_TCPv4_Rx) != 0) 2263 flags |= M_CSUM_TCPv4; 2264 if ((capenable & IFCAP_CSUM_TCPv6_Rx) != 0) 2265 flags |= M_CSUM_TCPv6; 2266 } 2267 2268 if ((status & IGC_RXD_STAT_UDPCS) != 0) { 2269 IGC_DRIVER_EVENT(q, rx_udpcs, 1); 2270 if ((capenable & IFCAP_CSUM_UDPv4_Rx) != 0) 2271 flags |= M_CSUM_UDPv4; 2272 if ((capenable & IFCAP_CSUM_UDPv6_Rx) != 0) 2273 flags |= M_CSUM_UDPv6; 2274 } 2275 2276 if (__predict_false((errors & IGC_RXD_ERR_TCPE) != 0)) { 2277 IGC_DRIVER_EVENT(q, rx_l4cs_bad, 1); 2278 if ((flags & ~M_CSUM_IPv4) != 0) 2279 flags |= M_CSUM_TCP_UDP_BAD; 2280 } 2281 2282 return flags; 2283 } 2284 2285 static void 2286 igc_watchdog(struct ifnet * ifp) 2287 { 2288 } 2289 2290 static void 2291 igc_tick(void *arg) 2292 { 2293 struct igc_softc *sc = arg; 2294 2295 mutex_enter(&sc->sc_core_lock); 2296 2297 if (__predict_false(sc->sc_core_stopping)) { 2298 mutex_exit(&sc->sc_core_lock); 2299 return; 2300 } 2301 2302 /* XXX watchdog */ 2303 if (0) { 2304 IGC_GLOBAL_EVENT(sc, watchdog, 1); 2305 } 2306 2307 igc_update_counters(sc); 2308 2309 mutex_exit(&sc->sc_core_lock); 2310 2311 callout_schedule(&sc->sc_tick_ch, hz); 2312 } 2313 2314 /********************************************************************* 2315 * 2316 * Media Ioctl callback 2317 * 2318 * This routine is called whenever the user queries the status of 2319 * the interface using ifconfig. 2320 * 2321 **********************************************************************/ 2322 static void 2323 igc_media_status(struct ifnet *ifp, struct ifmediareq *ifmr) 2324 { 2325 struct igc_softc *sc = ifp->if_softc; 2326 struct igc_hw *hw = &sc->hw; 2327 2328 igc_update_link_status(sc); 2329 2330 ifmr->ifm_status = IFM_AVALID; 2331 ifmr->ifm_active = IFM_ETHER; 2332 2333 if (!sc->link_active) { 2334 ifmr->ifm_active |= IFM_NONE; 2335 return; 2336 } 2337 2338 ifmr->ifm_status |= IFM_ACTIVE; 2339 2340 switch (sc->link_speed) { 2341 case 10: 2342 ifmr->ifm_active |= IFM_10_T; 2343 break; 2344 case 100: 2345 ifmr->ifm_active |= IFM_100_TX; 2346 break; 2347 case 1000: 2348 ifmr->ifm_active |= IFM_1000_T; 2349 break; 2350 case 2500: 2351 ifmr->ifm_active |= IFM_2500_T; 2352 break; 2353 } 2354 2355 if (sc->link_duplex == FULL_DUPLEX) 2356 ifmr->ifm_active |= IFM_FDX; 2357 else 2358 ifmr->ifm_active |= IFM_HDX; 2359 2360 switch (hw->fc.current_mode) { 2361 case igc_fc_tx_pause: 2362 ifmr->ifm_active |= IFM_FLOW | IFM_ETH_TXPAUSE; 2363 break; 2364 case igc_fc_rx_pause: 2365 ifmr->ifm_active |= IFM_FLOW | IFM_ETH_RXPAUSE; 2366 break; 2367 case igc_fc_full: 2368 ifmr->ifm_active |= IFM_FLOW | 2369 IFM_ETH_TXPAUSE | IFM_ETH_RXPAUSE; 2370 break; 2371 case igc_fc_none: 2372 default: 2373 break; 2374 } 2375 } 2376 2377 /********************************************************************* 2378 * 2379 * Media Ioctl callback 2380 * 2381 * This routine is called when the user changes speed/duplex using 2382 * media/mediopt option with ifconfig. 2383 * 2384 **********************************************************************/ 2385 static int 2386 igc_media_change(struct ifnet *ifp) 2387 { 2388 struct igc_softc *sc = ifp->if_softc; 2389 struct ifmedia *ifm = &sc->media; 2390 2391 if (IFM_TYPE(ifm->ifm_media) != IFM_ETHER) 2392 return EINVAL; 2393 2394 sc->hw.mac.autoneg = DO_AUTO_NEG; 2395 2396 switch (IFM_SUBTYPE(ifm->ifm_media)) { 2397 case IFM_AUTO: 2398 sc->hw.phy.autoneg_advertised = AUTONEG_ADV_DEFAULT; 2399 break; 2400 case IFM_2500_T: 2401 sc->hw.phy.autoneg_advertised = ADVERTISE_2500_FULL; 2402 break; 2403 case IFM_1000_T: 2404 sc->hw.phy.autoneg_advertised = ADVERTISE_1000_FULL; 2405 break; 2406 case IFM_100_TX: 2407 if ((ifm->ifm_media & IFM_GMASK) == IFM_FDX) 2408 sc->hw.phy.autoneg_advertised = ADVERTISE_100_FULL; 2409 else 2410 sc->hw.phy.autoneg_advertised = ADVERTISE_100_HALF; 2411 break; 2412 case IFM_10_T: 2413 if ((ifm->ifm_media & IFM_GMASK) == IFM_FDX) 2414 sc->hw.phy.autoneg_advertised = ADVERTISE_10_FULL; 2415 else 2416 sc->hw.phy.autoneg_advertised = ADVERTISE_10_HALF; 2417 break; 2418 default: 2419 return EINVAL; 2420 } 2421 2422 igc_init_locked(sc); 2423 2424 return 0; 2425 } 2426 2427 static int 2428 igc_ifflags_cb(struct ethercom *ec) 2429 { 2430 struct ifnet *ifp = &ec->ec_if; 2431 struct igc_softc *sc = ifp->if_softc; 2432 int rc = 0; 2433 u_short iffchange; 2434 bool needreset = false; 2435 2436 DPRINTF(CFG, "called\n"); 2437 2438 KASSERT(IFNET_LOCKED(ifp)); 2439 2440 mutex_enter(&sc->sc_core_lock); 2441 2442 /* 2443 * Check for if_flags. 2444 * Main usage is to prevent linkdown when opening bpf. 2445 */ 2446 iffchange = ifp->if_flags ^ sc->sc_if_flags; 2447 sc->sc_if_flags = ifp->if_flags; 2448 if ((iffchange & ~(IFF_CANTCHANGE | IFF_DEBUG)) != 0) { 2449 needreset = true; 2450 goto ec; 2451 } 2452 2453 /* iff related updates */ 2454 if ((iffchange & IFF_PROMISC) != 0) 2455 igc_set_filter(sc); 2456 2457 #ifdef notyet 2458 igc_set_vlan(sc); 2459 #endif 2460 2461 ec: 2462 #ifdef notyet 2463 /* Check for ec_capenable. */ 2464 ecchange = ec->ec_capenable ^ sc->sc_ec_capenable; 2465 sc->sc_ec_capenable = ec->ec_capenable; 2466 if ((ecchange & ~ETHERCAP_SOMETHING) != 0) { 2467 needreset = true; 2468 goto out; 2469 } 2470 #endif 2471 if (needreset) 2472 rc = ENETRESET; 2473 2474 mutex_exit(&sc->sc_core_lock); 2475 2476 return rc; 2477 } 2478 2479 static void 2480 igc_set_filter(struct igc_softc *sc) 2481 { 2482 struct ethercom *ec = &sc->sc_ec; 2483 uint32_t rctl; 2484 2485 rctl = IGC_READ_REG(&sc->hw, IGC_RCTL); 2486 rctl &= ~(IGC_RCTL_BAM |IGC_RCTL_UPE | IGC_RCTL_MPE); 2487 2488 if ((sc->sc_if_flags & IFF_BROADCAST) != 0) 2489 rctl |= IGC_RCTL_BAM; 2490 if ((sc->sc_if_flags & IFF_PROMISC) != 0) { 2491 DPRINTF(CFG, "promisc\n"); 2492 rctl |= IGC_RCTL_UPE; 2493 ETHER_LOCK(ec); 2494 allmulti: 2495 ec->ec_flags |= ETHER_F_ALLMULTI; 2496 ETHER_UNLOCK(ec); 2497 rctl |= IGC_RCTL_MPE; 2498 } else { 2499 struct ether_multistep step; 2500 struct ether_multi *enm; 2501 int mcnt = 0; 2502 2503 memset(sc->mta, 0, IGC_MTA_LEN); 2504 2505 ETHER_LOCK(ec); 2506 ETHER_FIRST_MULTI(step, ec, enm); 2507 while (enm != NULL) { 2508 if (((memcmp(enm->enm_addrlo, enm->enm_addrhi, 2509 ETHER_ADDR_LEN)) != 0) || 2510 (mcnt >= MAX_NUM_MULTICAST_ADDRESSES)) { 2511 /* 2512 * We must listen to a range of multicast 2513 * addresses. For now, just accept all 2514 * multicasts, rather than trying to set only 2515 * those filter bits needed to match the range. 2516 * (At this time, the only use of address 2517 * ranges is for IP multicast routing, for 2518 * which the range is big enough to require all 2519 * bits set.) 2520 */ 2521 goto allmulti; 2522 } 2523 DPRINTF(CFG, "%d: %s\n", mcnt, 2524 ether_sprintf(enm->enm_addrlo)); 2525 memcpy(&sc->mta[mcnt * ETHER_ADDR_LEN], 2526 enm->enm_addrlo, ETHER_ADDR_LEN); 2527 2528 mcnt++; 2529 ETHER_NEXT_MULTI(step, enm); 2530 } 2531 ec->ec_flags &= ~ETHER_F_ALLMULTI; 2532 ETHER_UNLOCK(ec); 2533 2534 DPRINTF(CFG, "hw filter\n"); 2535 igc_update_mc_addr_list(&sc->hw, sc->mta, mcnt); 2536 } 2537 2538 IGC_WRITE_REG(&sc->hw, IGC_RCTL, rctl); 2539 } 2540 2541 static void 2542 igc_update_link_status(struct igc_softc *sc) 2543 { 2544 struct ifnet *ifp = &sc->sc_ec.ec_if; 2545 struct igc_hw *hw = &sc->hw; 2546 2547 if (hw->mac.get_link_status == true) 2548 igc_check_for_link(hw); 2549 2550 if (IGC_READ_REG(&sc->hw, IGC_STATUS) & IGC_STATUS_LU) { 2551 if (sc->link_active == 0) { 2552 igc_get_speed_and_duplex(hw, &sc->link_speed, 2553 &sc->link_duplex); 2554 sc->link_active = 1; 2555 ifp->if_baudrate = IF_Mbps(sc->link_speed); 2556 if_link_state_change(ifp, LINK_STATE_UP); 2557 } 2558 } else { 2559 if (sc->link_active == 1) { 2560 ifp->if_baudrate = sc->link_speed = 0; 2561 sc->link_duplex = 0; 2562 sc->link_active = 0; 2563 if_link_state_change(ifp, LINK_STATE_DOWN); 2564 } 2565 } 2566 } 2567 2568 /********************************************************************* 2569 * 2570 * Get a buffer from system mbuf buffer pool. 2571 * 2572 **********************************************************************/ 2573 static int 2574 igc_get_buf(struct rx_ring *rxr, int id, bool strict) 2575 { 2576 struct igc_softc *sc = rxr->sc; 2577 struct igc_queue *q = rxr->rxr_igcq; 2578 struct igc_rx_buf *rxbuf = &rxr->rx_buffers[id]; 2579 bus_dmamap_t map = rxbuf->map; 2580 struct mbuf *m; 2581 int error; 2582 2583 if (__predict_false(rxbuf->buf)) { 2584 if (strict) { 2585 DPRINTF(RX, "slot %d already has an mbuf\n", id); 2586 return EINVAL; 2587 } 2588 return 0; 2589 } 2590 2591 MGETHDR(m, M_DONTWAIT, MT_DATA); 2592 if (__predict_false(m == NULL)) { 2593 enobuf: 2594 IGC_QUEUE_EVENT(q, rx_no_mbuf, 1); 2595 return ENOBUFS; 2596 } 2597 MCLAIM(m, &sc->sc_ec.ec_rx_mowner); 2598 2599 MCLGET(m, M_DONTWAIT); 2600 if (__predict_false(!(m->m_flags & M_EXT))) { 2601 m_freem(m); 2602 goto enobuf; 2603 } 2604 2605 m->m_len = m->m_pkthdr.len = sc->rx_mbuf_sz; 2606 2607 error = bus_dmamap_load_mbuf(rxr->rxdma.dma_tag, map, m, 2608 BUS_DMA_READ | BUS_DMA_NOWAIT); 2609 if (error) { 2610 m_freem(m); 2611 return error; 2612 } 2613 2614 bus_dmamap_sync(rxr->rxdma.dma_tag, map, 0, 2615 map->dm_mapsize, BUS_DMASYNC_PREREAD); 2616 rxbuf->buf = m; 2617 2618 union igc_adv_rx_desc *rxdesc = &rxr->rx_base[id]; 2619 igc_rxdesc_sync(rxr, id, BUS_DMASYNC_POSTWRITE | BUS_DMASYNC_POSTREAD); 2620 rxdesc->read.pkt_addr = htole64(map->dm_segs[0].ds_addr); 2621 igc_rxdesc_sync(rxr, id, BUS_DMASYNC_PREWRITE | BUS_DMASYNC_PREREAD); 2622 2623 return 0; 2624 } 2625 2626 static void 2627 igc_configure_queues(struct igc_softc *sc) 2628 { 2629 struct igc_hw *hw = &sc->hw; 2630 uint32_t ivar; 2631 2632 /* First turn on RSS capability */ 2633 IGC_WRITE_REG(hw, IGC_GPIE, IGC_GPIE_MSIX_MODE | IGC_GPIE_EIAME | 2634 IGC_GPIE_PBA | IGC_GPIE_NSICR); 2635 2636 /* Set the starting interrupt rate */ 2637 uint32_t newitr = (4000000 / MAX_INTS_PER_SEC) & 0x7FFC; 2638 newitr |= IGC_EITR_CNT_IGNR; 2639 2640 /* Turn on MSI-X */ 2641 uint32_t newmask = 0; 2642 for (int iq = 0; iq < sc->sc_nqueues; iq++) { 2643 struct igc_queue *q = &sc->queues[iq]; 2644 2645 /* RX entries */ 2646 igc_set_queues(sc, iq, q->msix, 0); 2647 /* TX entries */ 2648 igc_set_queues(sc, iq, q->msix, 1); 2649 newmask |= q->eims; 2650 IGC_WRITE_REG(hw, IGC_EITR(q->msix), newitr); 2651 } 2652 sc->msix_queuesmask = newmask; 2653 2654 #if 1 2655 ivar = IGC_READ_REG_ARRAY(hw, IGC_IVAR0, 0); 2656 DPRINTF(CFG, "ivar(0)=0x%x\n", ivar); 2657 ivar = IGC_READ_REG_ARRAY(hw, IGC_IVAR0, 1); 2658 DPRINTF(CFG, "ivar(1)=0x%x\n", ivar); 2659 #endif 2660 2661 /* And for the link interrupt */ 2662 ivar = (sc->linkvec | IGC_IVAR_VALID) << 8; 2663 sc->msix_linkmask = 1 << sc->linkvec; 2664 IGC_WRITE_REG(hw, IGC_IVAR_MISC, ivar); 2665 } 2666 2667 static void 2668 igc_set_queues(struct igc_softc *sc, uint32_t entry, uint32_t vector, int type) 2669 { 2670 struct igc_hw *hw = &sc->hw; 2671 const uint32_t index = entry >> 1; 2672 uint32_t ivar = IGC_READ_REG_ARRAY(hw, IGC_IVAR0, index); 2673 2674 if (type) { 2675 if (entry & 1) { 2676 ivar &= 0x00FFFFFF; 2677 ivar |= (vector | IGC_IVAR_VALID) << 24; 2678 } else { 2679 ivar &= 0xFFFF00FF; 2680 ivar |= (vector | IGC_IVAR_VALID) << 8; 2681 } 2682 } else { 2683 if (entry & 1) { 2684 ivar &= 0xFF00FFFF; 2685 ivar |= (vector | IGC_IVAR_VALID) << 16; 2686 } else { 2687 ivar &= 0xFFFFFF00; 2688 ivar |= vector | IGC_IVAR_VALID; 2689 } 2690 } 2691 IGC_WRITE_REG_ARRAY(hw, IGC_IVAR0, index, ivar); 2692 } 2693 2694 static void 2695 igc_enable_queue(struct igc_softc *sc, uint32_t eims) 2696 { 2697 IGC_WRITE_REG(&sc->hw, IGC_EIMS, eims); 2698 } 2699 2700 static void 2701 igc_enable_intr(struct igc_softc *sc) 2702 { 2703 struct igc_hw *hw = &sc->hw; 2704 2705 if (sc->sc_intr_type == PCI_INTR_TYPE_MSIX) { 2706 const uint32_t mask = sc->msix_queuesmask | sc->msix_linkmask; 2707 2708 IGC_WRITE_REG(hw, IGC_EIAC, mask); 2709 IGC_WRITE_REG(hw, IGC_EIAM, mask); 2710 IGC_WRITE_REG(hw, IGC_EIMS, mask); 2711 IGC_WRITE_REG(hw, IGC_IMS, IGC_IMS_LSC); 2712 } else { 2713 IGC_WRITE_REG(hw, IGC_IMS, IMS_ENABLE_MASK); 2714 } 2715 IGC_WRITE_FLUSH(hw); 2716 } 2717 2718 static void 2719 igc_disable_intr(struct igc_softc *sc) 2720 { 2721 struct igc_hw *hw = &sc->hw; 2722 2723 if (sc->sc_intr_type == PCI_INTR_TYPE_MSIX) { 2724 IGC_WRITE_REG(hw, IGC_EIMC, 0xffffffff); 2725 IGC_WRITE_REG(hw, IGC_EIAC, 0); 2726 } 2727 IGC_WRITE_REG(hw, IGC_IMC, 0xffffffff); 2728 IGC_WRITE_FLUSH(hw); 2729 } 2730 2731 static int 2732 igc_intr_link(void *arg) 2733 { 2734 struct igc_softc *sc = (struct igc_softc *)arg; 2735 const uint32_t reg_icr = IGC_READ_REG(&sc->hw, IGC_ICR); 2736 2737 IGC_GLOBAL_EVENT(sc, link, 1); 2738 2739 if (reg_icr & IGC_ICR_LSC) { 2740 mutex_enter(&sc->sc_core_lock); 2741 sc->hw.mac.get_link_status = true; 2742 igc_update_link_status(sc); 2743 mutex_exit(&sc->sc_core_lock); 2744 } 2745 2746 IGC_WRITE_REG(&sc->hw, IGC_IMS, IGC_IMS_LSC); 2747 IGC_WRITE_REG(&sc->hw, IGC_EIMS, sc->msix_linkmask); 2748 2749 return 1; 2750 } 2751 2752 static int 2753 igc_intr_queue(void *arg) 2754 { 2755 struct igc_queue *iq = arg; 2756 struct igc_softc *sc = iq->sc; 2757 struct ifnet *ifp = &sc->sc_ec.ec_if; 2758 struct rx_ring *rxr = iq->rxr; 2759 struct tx_ring *txr = iq->txr; 2760 const u_int txlimit = sc->sc_tx_intr_process_limit, 2761 rxlimit = sc->sc_rx_intr_process_limit; 2762 bool txmore, rxmore; 2763 2764 IGC_QUEUE_EVENT(iq, irqs, 1); 2765 2766 if (__predict_false(!ISSET(ifp->if_flags, IFF_RUNNING))) 2767 return 0; 2768 2769 mutex_enter(&txr->txr_lock); 2770 txmore = igc_txeof(txr, txlimit); 2771 mutex_exit(&txr->txr_lock); 2772 mutex_enter(&rxr->rxr_lock); 2773 rxmore = igc_rxeof(rxr, rxlimit); 2774 mutex_exit(&rxr->rxr_lock); 2775 2776 if (txmore || rxmore) { 2777 IGC_QUEUE_EVENT(iq, req, 1); 2778 igc_sched_handle_queue(sc, iq); 2779 } else { 2780 igc_enable_queue(sc, iq->eims); 2781 } 2782 2783 return 1; 2784 } 2785 2786 static int 2787 igc_intr(void *arg) 2788 { 2789 struct igc_softc *sc = arg; 2790 struct ifnet *ifp = &sc->sc_ec.ec_if; 2791 struct igc_queue *iq = &sc->queues[0]; 2792 struct rx_ring *rxr = iq->rxr; 2793 struct tx_ring *txr = iq->txr; 2794 const u_int txlimit = sc->sc_tx_intr_process_limit, 2795 rxlimit = sc->sc_rx_intr_process_limit; 2796 bool txmore, rxmore; 2797 2798 if (__predict_false(!ISSET(ifp->if_flags, IFF_RUNNING))) 2799 return 0; 2800 2801 const uint32_t reg_icr = IGC_READ_REG(&sc->hw, IGC_ICR); 2802 DPRINTF(MISC, "reg_icr=0x%x\n", reg_icr); 2803 2804 /* Definitely not our interrupt. */ 2805 if (reg_icr == 0x0) { 2806 DPRINTF(MISC, "not for me\n"); 2807 return 0; 2808 } 2809 2810 IGC_QUEUE_EVENT(iq, irqs, 1); 2811 2812 /* Hot eject? */ 2813 if (__predict_false(reg_icr == 0xffffffff)) { 2814 DPRINTF(MISC, "hot eject\n"); 2815 return 0; 2816 } 2817 2818 if (__predict_false(!(reg_icr & IGC_ICR_INT_ASSERTED))) { 2819 DPRINTF(MISC, "not set IGC_ICR_INT_ASSERTED"); 2820 return 0; 2821 } 2822 2823 /* 2824 * Only MSI-X interrupts have one-shot behavior by taking advantage 2825 * of the EIAC register. Thus, explicitly disable interrupts. This 2826 * also works around the MSI message reordering errata on certain 2827 * systems. 2828 */ 2829 igc_disable_intr(sc); 2830 2831 mutex_enter(&txr->txr_lock); 2832 txmore = igc_txeof(txr, txlimit); 2833 mutex_exit(&txr->txr_lock); 2834 mutex_enter(&rxr->rxr_lock); 2835 rxmore = igc_rxeof(rxr, rxlimit); 2836 mutex_exit(&rxr->rxr_lock); 2837 2838 /* Link status change */ 2839 // XXXX FreeBSD checks IGC_ICR_RXSEQ 2840 if (__predict_false(reg_icr & IGC_ICR_LSC)) { 2841 IGC_GLOBAL_EVENT(sc, link, 1); 2842 mutex_enter(&sc->sc_core_lock); 2843 sc->hw.mac.get_link_status = true; 2844 igc_update_link_status(sc); 2845 mutex_exit(&sc->sc_core_lock); 2846 } 2847 2848 if (txmore || rxmore) { 2849 IGC_QUEUE_EVENT(iq, req, 1); 2850 igc_sched_handle_queue(sc, iq); 2851 } else { 2852 igc_enable_intr(sc); 2853 } 2854 2855 return 1; 2856 } 2857 2858 static void 2859 igc_handle_queue(void *arg) 2860 { 2861 struct igc_queue *iq = arg; 2862 struct igc_softc *sc = iq->sc; 2863 struct tx_ring *txr = iq->txr; 2864 struct rx_ring *rxr = iq->rxr; 2865 const u_int txlimit = sc->sc_tx_process_limit, 2866 rxlimit = sc->sc_rx_process_limit; 2867 bool txmore, rxmore; 2868 2869 IGC_QUEUE_EVENT(iq, handleq, 1); 2870 2871 mutex_enter(&txr->txr_lock); 2872 txmore = igc_txeof(txr, txlimit); 2873 /* for ALTQ, dequeue from if_snd */ 2874 if (txr->me == 0) { 2875 struct ifnet *ifp = &sc->sc_ec.ec_if; 2876 2877 igc_tx_common_locked(ifp, txr, IGC_TX_START); 2878 } 2879 mutex_exit(&txr->txr_lock); 2880 2881 mutex_enter(&rxr->rxr_lock); 2882 rxmore = igc_rxeof(rxr, rxlimit); 2883 mutex_exit(&rxr->rxr_lock); 2884 2885 if (txmore || rxmore) { 2886 igc_sched_handle_queue(sc, iq); 2887 } else { 2888 if (sc->sc_intr_type == PCI_INTR_TYPE_MSIX) 2889 igc_enable_queue(sc, iq->eims); 2890 else 2891 igc_enable_intr(sc); 2892 } 2893 } 2894 2895 static void 2896 igc_handle_queue_work(struct work *wk, void *context) 2897 { 2898 struct igc_queue *iq = 2899 container_of(wk, struct igc_queue, igcq_wq_cookie); 2900 2901 igc_handle_queue(iq); 2902 } 2903 2904 static void 2905 igc_sched_handle_queue(struct igc_softc *sc, struct igc_queue *iq) 2906 { 2907 2908 if (iq->igcq_workqueue) { 2909 /* XXXRO notyet */ 2910 workqueue_enqueue(sc->sc_queue_wq, &iq->igcq_wq_cookie, 2911 curcpu()); 2912 } else { 2913 softint_schedule(iq->igcq_si); 2914 } 2915 } 2916 2917 static void 2918 igc_barrier_handle_queue(struct igc_softc *sc) 2919 { 2920 2921 if (sc->sc_txrx_workqueue) { 2922 for (int iq = 0; iq < sc->sc_nqueues; iq++) { 2923 struct igc_queue *q = &sc->queues[iq]; 2924 2925 workqueue_wait(sc->sc_queue_wq, &q->igcq_wq_cookie); 2926 } 2927 } else { 2928 xc_barrier(0); 2929 } 2930 } 2931 2932 /********************************************************************* 2933 * 2934 * Allocate memory for tx_buffer structures. The tx_buffer stores all 2935 * the information needed to transmit a packet on the wire. 2936 * 2937 **********************************************************************/ 2938 static int 2939 igc_allocate_transmit_buffers(struct tx_ring *txr) 2940 { 2941 struct igc_softc *sc = txr->sc; 2942 int error; 2943 2944 txr->tx_buffers = 2945 kmem_zalloc(sc->num_tx_desc * sizeof(struct igc_tx_buf), KM_SLEEP); 2946 txr->txtag = txr->txdma.dma_tag; 2947 2948 /* Create the descriptor buffer dma maps. */ 2949 for (int id = 0; id < sc->num_tx_desc; id++) { 2950 struct igc_tx_buf *txbuf = &txr->tx_buffers[id]; 2951 2952 error = bus_dmamap_create(txr->txdma.dma_tag, 2953 round_page(IGC_TSO_SIZE + sizeof(struct ether_vlan_header)), 2954 IGC_MAX_SCATTER, PAGE_SIZE, 0, BUS_DMA_NOWAIT, &txbuf->map); 2955 if (error != 0) { 2956 aprint_error_dev(sc->sc_dev, 2957 "unable to create TX DMA map\n"); 2958 goto fail; 2959 } 2960 2961 txbuf->eop_index = -1; 2962 } 2963 2964 return 0; 2965 fail: 2966 return error; 2967 } 2968 2969 2970 /********************************************************************* 2971 * 2972 * Allocate and initialize transmit structures. 2973 * 2974 **********************************************************************/ 2975 static int 2976 igc_setup_transmit_structures(struct igc_softc *sc) 2977 { 2978 2979 for (int iq = 0; iq < sc->sc_nqueues; iq++) { 2980 struct tx_ring *txr = &sc->tx_rings[iq]; 2981 2982 if (igc_setup_transmit_ring(txr)) 2983 goto fail; 2984 } 2985 2986 return 0; 2987 fail: 2988 igc_free_transmit_structures(sc); 2989 return ENOBUFS; 2990 } 2991 2992 /********************************************************************* 2993 * 2994 * Initialize a transmit ring. 2995 * 2996 **********************************************************************/ 2997 static int 2998 igc_setup_transmit_ring(struct tx_ring *txr) 2999 { 3000 struct igc_softc *sc = txr->sc; 3001 3002 /* Now allocate transmit buffers for the ring. */ 3003 if (igc_allocate_transmit_buffers(txr)) 3004 return ENOMEM; 3005 3006 /* Clear the old ring contents */ 3007 memset(txr->tx_base, 0, 3008 sizeof(union igc_adv_tx_desc) * sc->num_tx_desc); 3009 3010 /* Reset indices. */ 3011 txr->next_avail_desc = 0; 3012 txr->next_to_clean = 0; 3013 3014 bus_dmamap_sync(txr->txdma.dma_tag, txr->txdma.dma_map, 0, 3015 txr->txdma.dma_map->dm_mapsize, 3016 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); 3017 3018 txr->txr_interq = pcq_create(sc->num_tx_desc, KM_SLEEP); 3019 3020 mutex_init(&txr->txr_lock, MUTEX_DEFAULT, IPL_NET); 3021 3022 return 0; 3023 } 3024 3025 /********************************************************************* 3026 * 3027 * Enable transmit unit. 3028 * 3029 **********************************************************************/ 3030 static void 3031 igc_initialize_transmit_unit(struct igc_softc *sc) 3032 { 3033 struct ifnet *ifp = &sc->sc_ec.ec_if; 3034 struct igc_hw *hw = &sc->hw; 3035 3036 /* Setup the Base and Length of the TX descriptor ring. */ 3037 for (int iq = 0; iq < sc->sc_nqueues; iq++) { 3038 struct tx_ring *txr = &sc->tx_rings[iq]; 3039 const uint64_t bus_addr = 3040 txr->txdma.dma_map->dm_segs[0].ds_addr; 3041 3042 /* Base and len of TX ring */ 3043 IGC_WRITE_REG(hw, IGC_TDLEN(iq), 3044 sc->num_tx_desc * sizeof(union igc_adv_tx_desc)); 3045 IGC_WRITE_REG(hw, IGC_TDBAH(iq), (uint32_t)(bus_addr >> 32)); 3046 IGC_WRITE_REG(hw, IGC_TDBAL(iq), (uint32_t)bus_addr); 3047 3048 /* Init the HEAD/TAIL indices */ 3049 IGC_WRITE_REG(hw, IGC_TDT(iq), 0 /* XXX txr->next_avail_desc */); 3050 IGC_WRITE_REG(hw, IGC_TDH(iq), 0); 3051 3052 txr->watchdog_timer = 0; 3053 3054 uint32_t txdctl = 0; /* Clear txdctl */ 3055 txdctl |= 0x1f; /* PTHRESH */ 3056 txdctl |= 1 << 8; /* HTHRESH */ 3057 txdctl |= 1 << 16; /* WTHRESH */ 3058 txdctl |= 1 << 22; /* Reserved bit 22 must always be 1 */ 3059 txdctl |= IGC_TXDCTL_GRAN; 3060 txdctl |= 1 << 25; /* LWTHRESH */ 3061 3062 IGC_WRITE_REG(hw, IGC_TXDCTL(iq), txdctl); 3063 } 3064 ifp->if_timer = 0; 3065 3066 /* Program the Transmit Control Register */ 3067 uint32_t tctl = IGC_READ_REG(&sc->hw, IGC_TCTL); 3068 tctl &= ~IGC_TCTL_CT; 3069 tctl |= (IGC_TCTL_PSP | IGC_TCTL_RTLC | IGC_TCTL_EN | 3070 (IGC_COLLISION_THRESHOLD << IGC_CT_SHIFT)); 3071 3072 /* This write will effectively turn on the transmit unit. */ 3073 IGC_WRITE_REG(&sc->hw, IGC_TCTL, tctl); 3074 } 3075 3076 /********************************************************************* 3077 * 3078 * Free all transmit rings. 3079 * 3080 **********************************************************************/ 3081 static void 3082 igc_free_transmit_structures(struct igc_softc *sc) 3083 { 3084 3085 for (int iq = 0; iq < sc->sc_nqueues; iq++) { 3086 struct tx_ring *txr = &sc->tx_rings[iq]; 3087 3088 igc_free_transmit_buffers(txr); 3089 } 3090 } 3091 3092 /********************************************************************* 3093 * 3094 * Free transmit ring related data structures. 3095 * 3096 **********************************************************************/ 3097 static void 3098 igc_free_transmit_buffers(struct tx_ring *txr) 3099 { 3100 struct igc_softc *sc = txr->sc; 3101 3102 if (txr->tx_buffers == NULL) 3103 return; 3104 3105 igc_withdraw_transmit_packets(txr, true); 3106 3107 kmem_free(txr->tx_buffers, 3108 sc->num_tx_desc * sizeof(struct igc_tx_buf)); 3109 txr->tx_buffers = NULL; 3110 txr->txtag = NULL; 3111 3112 pcq_destroy(txr->txr_interq); 3113 mutex_destroy(&txr->txr_lock); 3114 } 3115 3116 /********************************************************************* 3117 * 3118 * Withdraw transmit packets. 3119 * 3120 **********************************************************************/ 3121 static void 3122 igc_withdraw_transmit_packets(struct tx_ring *txr, bool destroy) 3123 { 3124 struct igc_softc *sc = txr->sc; 3125 struct igc_queue *q = txr->txr_igcq; 3126 3127 mutex_enter(&txr->txr_lock); 3128 3129 for (int id = 0; id < sc->num_tx_desc; id++) { 3130 union igc_adv_tx_desc *txdesc = &txr->tx_base[id]; 3131 3132 igc_txdesc_sync(txr, id, 3133 BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE); 3134 txdesc->read.buffer_addr = 0; 3135 txdesc->read.cmd_type_len = 0; 3136 txdesc->read.olinfo_status = 0; 3137 igc_txdesc_sync(txr, id, 3138 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); 3139 3140 struct igc_tx_buf *txbuf = &txr->tx_buffers[id]; 3141 bus_dmamap_t map = txbuf->map; 3142 3143 if (map != NULL && map->dm_nsegs > 0) { 3144 bus_dmamap_sync(txr->txdma.dma_tag, map, 3145 0, map->dm_mapsize, BUS_DMASYNC_POSTWRITE); 3146 bus_dmamap_unload(txr->txdma.dma_tag, map); 3147 } 3148 m_freem(txbuf->m_head); 3149 txbuf->m_head = NULL; 3150 if (map != NULL && destroy) { 3151 bus_dmamap_destroy(txr->txdma.dma_tag, map); 3152 txbuf->map = NULL; 3153 } 3154 txbuf->eop_index = -1; 3155 3156 txr->next_avail_desc = 0; 3157 txr->next_to_clean = 0; 3158 } 3159 3160 struct mbuf *m; 3161 while ((m = pcq_get(txr->txr_interq)) != NULL) { 3162 IGC_QUEUE_EVENT(q, tx_pcq_drop, 1); 3163 m_freem(m); 3164 } 3165 3166 mutex_exit(&txr->txr_lock); 3167 } 3168 3169 3170 /********************************************************************* 3171 * 3172 * Advanced Context Descriptor setup for VLAN, CSUM or TSO 3173 * 3174 **********************************************************************/ 3175 3176 static bool 3177 igc_tx_ctx_setup(struct tx_ring *txr, struct mbuf *mp, int prod, 3178 uint32_t *cmd_type_len, uint32_t *olinfo_status) 3179 { 3180 struct ether_vlan_header *evl; 3181 struct tcphdr *th = NULL /* XXXGCC */; 3182 uint32_t vlan_macip_lens = 0; 3183 uint32_t type_tucmd_mlhl = 0; 3184 uint32_t mss_l4len_idx = 0; 3185 uint32_t ehlen, iphlen; 3186 uint16_t ehtype; 3187 3188 const int csum_flags = mp->m_pkthdr.csum_flags; 3189 const bool v4 = (csum_flags & 3190 (M_CSUM_IPv4 | M_CSUM_TCPv4 | M_CSUM_UDPv4 | M_CSUM_TSOv4)) != 0; 3191 const bool v6 = (csum_flags & 3192 (M_CSUM_UDPv6 | M_CSUM_TCPv6 | M_CSUM_TSOv6)) != 0; 3193 const bool tso = (csum_flags & (M_CSUM_TSOv4 | M_CSUM_TSOv6)) != 0; 3194 const bool tcp = tso || 3195 (csum_flags & (M_CSUM_TCPv4 | M_CSUM_TCPv6)) != 0; 3196 const bool udp = (csum_flags & (M_CSUM_UDPv4 | M_CSUM_UDPv6)) != 0; 3197 3198 /* Indicate the whole packet as payload when not doing TSO */ 3199 if (!tso) { 3200 *olinfo_status |= mp->m_pkthdr.len << IGC_ADVTXD_PAYLEN_SHIFT; 3201 } else { 3202 /* Set L4 payload length later... */ 3203 } 3204 3205 #if NVLAN > 0 3206 /* 3207 * In advanced descriptors the vlan tag must 3208 * be placed into the context descriptor. Hence 3209 * we need to make one even if not doing offloads. 3210 */ 3211 if (vlan_has_tag(mp)) { 3212 vlan_macip_lens |= (uint32_t)vlan_get_tag(mp) 3213 << IGC_ADVTXD_VLAN_SHIFT; 3214 } else 3215 #endif 3216 if (!v4 && !v6) 3217 return false; 3218 3219 KASSERT(mp->m_len >= sizeof(struct ether_header)); 3220 evl = mtod(mp, struct ether_vlan_header *); 3221 if (evl->evl_encap_proto == htons(ETHERTYPE_VLAN)) { 3222 KASSERT(mp->m_len >= sizeof(struct ether_vlan_header)); 3223 ehlen = ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN; 3224 ehtype = evl->evl_proto; 3225 } else { 3226 ehlen = ETHER_HDR_LEN; 3227 ehtype = evl->evl_encap_proto; 3228 } 3229 3230 switch (ntohs(ehtype)) { 3231 case ETHERTYPE_IP: 3232 iphlen = M_CSUM_DATA_IPv4_IPHL(mp->m_pkthdr.csum_data); 3233 type_tucmd_mlhl |= IGC_ADVTXD_TUCMD_IPV4; 3234 3235 if ((csum_flags & (M_CSUM_IPv4 | M_CSUM_TSOv4)) != 0) 3236 *olinfo_status |= IGC_TXD_POPTS_IXSM << 8; 3237 3238 if (!tso) 3239 break; 3240 3241 struct ip *ip; 3242 KASSERT(mp->m_len >= ehlen + sizeof(*ip)); 3243 ip = (void *)(mtod(mp, char *) + ehlen); 3244 ip->ip_len = 0; 3245 3246 KASSERT(mp->m_len >= ehlen + iphlen + sizeof(*th)); 3247 th = (void *)((char *)ip + iphlen); 3248 th->th_sum = in_cksum_phdr(ip->ip_src.s_addr, 3249 ip->ip_dst.s_addr, htons(IPPROTO_TCP)); 3250 break; 3251 case ETHERTYPE_IPV6: 3252 iphlen = M_CSUM_DATA_IPv6_IPHL(mp->m_pkthdr.csum_data); 3253 type_tucmd_mlhl |= IGC_ADVTXD_TUCMD_IPV6; 3254 3255 if (!tso) 3256 break; 3257 3258 struct ip6_hdr *ip6; 3259 KASSERT(mp->m_len >= ehlen + sizeof(*ip6)); 3260 ip6 = (void *)(mtod(mp, char *) + ehlen); 3261 ip6->ip6_plen = 0; 3262 3263 KASSERT(mp->m_len >= ehlen + iphlen + sizeof(*th)); 3264 th = (void *)((char *)ip6 + iphlen); 3265 th->th_sum = in6_cksum_phdr(&ip6->ip6_src, &ip6->ip6_dst, 0, 3266 htonl(IPPROTO_TCP)); 3267 break; 3268 default: 3269 /* 3270 * Unknown L3 protocol. Clear L3 header length and proceed for 3271 * LAN as done by Linux driver. 3272 */ 3273 KASSERT(!v4 && !v6); 3274 iphlen = 0; 3275 break; 3276 } 3277 3278 if (tcp) { 3279 type_tucmd_mlhl |= IGC_ADVTXD_TUCMD_L4T_TCP; 3280 *olinfo_status |= IGC_TXD_POPTS_TXSM << 8; 3281 } else if (udp) { 3282 type_tucmd_mlhl |= IGC_ADVTXD_TUCMD_L4T_UDP; 3283 *olinfo_status |= IGC_TXD_POPTS_TXSM << 8; 3284 } 3285 3286 if (tso) { 3287 const uint32_t tcphlen = th->th_off << 2; 3288 const uint32_t paylen = 3289 mp->m_pkthdr.len - ehlen - iphlen - tcphlen; 3290 3291 mss_l4len_idx |= mp->m_pkthdr.segsz << IGC_ADVTXD_MSS_SHIFT; 3292 mss_l4len_idx |= tcphlen << IGC_ADVTXD_L4LEN_SHIFT; 3293 3294 *cmd_type_len |= IGC_ADVTXD_DCMD_TSE; 3295 3296 *olinfo_status |= paylen << IGC_ADVTXD_PAYLEN_SHIFT; 3297 } 3298 3299 vlan_macip_lens |= iphlen; 3300 vlan_macip_lens |= ehlen << IGC_ADVTXD_MACLEN_SHIFT; 3301 3302 type_tucmd_mlhl |= IGC_ADVTXD_DCMD_DEXT | IGC_ADVTXD_DTYP_CTXT; 3303 3304 /* Now ready a context descriptor */ 3305 struct igc_adv_tx_context_desc *txdesc = 3306 (struct igc_adv_tx_context_desc *)&txr->tx_base[prod]; 3307 3308 igc_txdesc_sync(txr, prod, 3309 BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE); 3310 3311 /* Now copy bits into descriptor */ 3312 htolem32(&txdesc->vlan_macip_lens, vlan_macip_lens); 3313 htolem32(&txdesc->type_tucmd_mlhl, type_tucmd_mlhl); 3314 htolem32(&txdesc->seqnum_seed, 0); 3315 htolem32(&txdesc->mss_l4len_idx, mss_l4len_idx); 3316 3317 igc_txdesc_sync(txr, prod, 3318 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); 3319 3320 return 1; 3321 } 3322 3323 /********************************************************************* 3324 * 3325 * Allocate memory for rx_buffer structures. Since we use one 3326 * rx_buffer per received packet, the maximum number of rx_buffer's 3327 * that we'll need is equal to the number of receive descriptors 3328 * that we've allocated. 3329 * 3330 **********************************************************************/ 3331 static int 3332 igc_allocate_receive_buffers(struct rx_ring *rxr) 3333 { 3334 struct igc_softc *sc = rxr->sc; 3335 int error; 3336 3337 rxr->rx_buffers = 3338 kmem_zalloc(sc->num_rx_desc * sizeof(struct igc_rx_buf), KM_SLEEP); 3339 3340 for (int id = 0; id < sc->num_rx_desc; id++) { 3341 struct igc_rx_buf *rxbuf = &rxr->rx_buffers[id]; 3342 3343 error = bus_dmamap_create(rxr->rxdma.dma_tag, MCLBYTES, 1, 3344 MCLBYTES, 0, BUS_DMA_WAITOK, &rxbuf->map); 3345 if (error) { 3346 aprint_error_dev(sc->sc_dev, 3347 "unable to create RX DMA map\n"); 3348 goto fail; 3349 } 3350 } 3351 bus_dmamap_sync(rxr->rxdma.dma_tag, rxr->rxdma.dma_map, 0, 3352 rxr->rxdma.dma_map->dm_mapsize, 3353 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); 3354 3355 return 0; 3356 fail: 3357 return error; 3358 } 3359 3360 /********************************************************************* 3361 * 3362 * Allocate and initialize receive structures. 3363 * 3364 **********************************************************************/ 3365 static int 3366 igc_setup_receive_structures(struct igc_softc *sc) 3367 { 3368 3369 for (int iq = 0; iq < sc->sc_nqueues; iq++) { 3370 struct rx_ring *rxr = &sc->rx_rings[iq]; 3371 3372 if (igc_setup_receive_ring(rxr)) 3373 goto fail; 3374 } 3375 3376 return 0; 3377 fail: 3378 igc_free_receive_structures(sc); 3379 return ENOBUFS; 3380 } 3381 3382 /********************************************************************* 3383 * 3384 * Initialize a receive ring and its buffers. 3385 * 3386 **********************************************************************/ 3387 static int 3388 igc_setup_receive_ring(struct rx_ring *rxr) 3389 { 3390 struct igc_softc *sc = rxr->sc; 3391 const int rsize = roundup2( 3392 sc->num_rx_desc * sizeof(union igc_adv_rx_desc), IGC_DBA_ALIGN); 3393 3394 /* Clear the ring contents. */ 3395 memset(rxr->rx_base, 0, rsize); 3396 3397 if (igc_allocate_receive_buffers(rxr)) 3398 return ENOMEM; 3399 3400 /* Setup our descriptor indices. */ 3401 rxr->next_to_check = 0; 3402 rxr->last_desc_filled = 0; 3403 3404 mutex_init(&rxr->rxr_lock, MUTEX_DEFAULT, IPL_NET); 3405 3406 return 0; 3407 } 3408 3409 /********************************************************************* 3410 * 3411 * Enable receive unit. 3412 * 3413 **********************************************************************/ 3414 static void 3415 igc_initialize_receive_unit(struct igc_softc *sc) 3416 { 3417 struct ifnet *ifp = &sc->sc_ec.ec_if; 3418 struct igc_hw *hw = &sc->hw; 3419 uint32_t rctl, rxcsum, srrctl; 3420 3421 DPRINTF(RX, "called\n"); 3422 3423 /* 3424 * Make sure receives are disabled while setting 3425 * up the descriptor ring. 3426 */ 3427 rctl = IGC_READ_REG(hw, IGC_RCTL); 3428 IGC_WRITE_REG(hw, IGC_RCTL, rctl & ~IGC_RCTL_EN); 3429 3430 /* Setup the Receive Control Register */ 3431 rctl &= ~(3 << IGC_RCTL_MO_SHIFT); 3432 rctl |= IGC_RCTL_EN | IGC_RCTL_BAM | IGC_RCTL_LBM_NO | 3433 IGC_RCTL_RDMTS_HALF | (hw->mac.mc_filter_type << IGC_RCTL_MO_SHIFT); 3434 3435 #if 1 3436 /* Do not store bad packets */ 3437 rctl &= ~IGC_RCTL_SBP; 3438 #else 3439 /* for debug */ 3440 rctl |= IGC_RCTL_SBP; 3441 #endif 3442 3443 /* Enable Long Packet receive */ 3444 if (sc->hw.mac.max_frame_size > ETHER_MAX_LEN) 3445 rctl |= IGC_RCTL_LPE; 3446 else 3447 rctl &= ~IGC_RCTL_LPE; 3448 3449 /* Strip the CRC */ 3450 rctl |= IGC_RCTL_SECRC; 3451 3452 /* 3453 * Set the interrupt throttling rate. Value is calculated 3454 * as DEFAULT_ITR = 1/(MAX_INTS_PER_SEC * 256ns) 3455 * 3456 * XXX Sync with Linux, especially for jumbo MTU or TSO. 3457 * XXX Shouldn't be here? 3458 */ 3459 IGC_WRITE_REG(hw, IGC_ITR, DEFAULT_ITR); 3460 3461 rxcsum = IGC_READ_REG(hw, IGC_RXCSUM); 3462 rxcsum &= ~(IGC_RXCSUM_IPOFL | IGC_RXCSUM_TUOFL | IGC_RXCSUM_PCSD); 3463 if (ifp->if_capenable & IFCAP_CSUM_IPv4_Rx) 3464 rxcsum |= IGC_RXCSUM_IPOFL; 3465 if (ifp->if_capenable & (IFCAP_CSUM_TCPv4_Rx | IFCAP_CSUM_UDPv4_Rx | 3466 IFCAP_CSUM_TCPv6_Rx | IFCAP_CSUM_UDPv6_Rx)) 3467 rxcsum |= IGC_RXCSUM_TUOFL; 3468 if (sc->sc_nqueues > 1) 3469 rxcsum |= IGC_RXCSUM_PCSD; 3470 IGC_WRITE_REG(hw, IGC_RXCSUM, rxcsum); 3471 3472 if (sc->sc_nqueues > 1) 3473 igc_initialize_rss_mapping(sc); 3474 3475 srrctl = 0; 3476 #if 0 3477 srrctl |= 4096 >> IGC_SRRCTL_BSIZEPKT_SHIFT; 3478 rctl |= IGC_RCTL_SZ_4096 | IGC_RCTL_BSEX; 3479 #else 3480 srrctl |= 2048 >> IGC_SRRCTL_BSIZEPKT_SHIFT; 3481 rctl |= IGC_RCTL_SZ_2048; 3482 #endif 3483 3484 /* 3485 * If TX flow control is disabled and there's > 1 queue defined, 3486 * enable DROP. 3487 * 3488 * This drops frames rather than hanging the RX MAC for all queues. 3489 */ 3490 if (sc->sc_nqueues > 1 && 3491 (sc->fc == igc_fc_none || sc->fc == igc_fc_rx_pause)) 3492 srrctl |= IGC_SRRCTL_DROP_EN; 3493 3494 /* Setup the Base and Length of the RX descriptor rings. */ 3495 for (int iq = 0; iq < sc->sc_nqueues; iq++) { 3496 struct rx_ring *rxr = &sc->rx_rings[iq]; 3497 const uint64_t bus_addr = 3498 rxr->rxdma.dma_map->dm_segs[0].ds_addr; 3499 3500 IGC_WRITE_REG(hw, IGC_RXDCTL(iq), 0); 3501 3502 srrctl |= IGC_SRRCTL_DESCTYPE_ADV_ONEBUF; 3503 3504 IGC_WRITE_REG(hw, IGC_RDLEN(iq), 3505 sc->num_rx_desc * sizeof(union igc_adv_rx_desc)); 3506 IGC_WRITE_REG(hw, IGC_RDBAH(iq), (uint32_t)(bus_addr >> 32)); 3507 IGC_WRITE_REG(hw, IGC_RDBAL(iq), (uint32_t)bus_addr); 3508 IGC_WRITE_REG(hw, IGC_SRRCTL(iq), srrctl); 3509 3510 /* Setup the Head and Tail Descriptor Pointers */ 3511 IGC_WRITE_REG(hw, IGC_RDH(iq), 0); 3512 IGC_WRITE_REG(hw, IGC_RDT(iq), 0 /* XXX rxr->last_desc_filled */); 3513 3514 /* Enable this Queue */ 3515 uint32_t rxdctl = IGC_READ_REG(hw, IGC_RXDCTL(iq)); 3516 rxdctl |= IGC_RXDCTL_QUEUE_ENABLE; 3517 rxdctl &= 0xFFF00000; 3518 rxdctl |= IGC_RX_PTHRESH; 3519 rxdctl |= IGC_RX_HTHRESH << 8; 3520 rxdctl |= IGC_RX_WTHRESH << 16; 3521 IGC_WRITE_REG(hw, IGC_RXDCTL(iq), rxdctl); 3522 } 3523 3524 /* Make sure VLAN Filters are off */ 3525 rctl &= ~IGC_RCTL_VFE; 3526 3527 /* Write out the settings */ 3528 IGC_WRITE_REG(hw, IGC_RCTL, rctl); 3529 } 3530 3531 /********************************************************************* 3532 * 3533 * Free all receive rings. 3534 * 3535 **********************************************************************/ 3536 static void 3537 igc_free_receive_structures(struct igc_softc *sc) 3538 { 3539 3540 for (int iq = 0; iq < sc->sc_nqueues; iq++) { 3541 struct rx_ring *rxr = &sc->rx_rings[iq]; 3542 3543 igc_free_receive_buffers(rxr); 3544 } 3545 } 3546 3547 /********************************************************************* 3548 * 3549 * Free receive ring data structures 3550 * 3551 **********************************************************************/ 3552 static void 3553 igc_free_receive_buffers(struct rx_ring *rxr) 3554 { 3555 struct igc_softc *sc = rxr->sc; 3556 3557 if (rxr->rx_buffers != NULL) { 3558 for (int id = 0; id < sc->num_rx_desc; id++) { 3559 struct igc_rx_buf *rxbuf = &rxr->rx_buffers[id]; 3560 bus_dmamap_t map = rxbuf->map; 3561 3562 if (rxbuf->buf != NULL) { 3563 bus_dmamap_sync(rxr->rxdma.dma_tag, map, 3564 0, map->dm_mapsize, BUS_DMASYNC_POSTREAD); 3565 bus_dmamap_unload(rxr->rxdma.dma_tag, map); 3566 m_freem(rxbuf->buf); 3567 rxbuf->buf = NULL; 3568 } 3569 bus_dmamap_destroy(rxr->rxdma.dma_tag, map); 3570 rxbuf->map = NULL; 3571 } 3572 kmem_free(rxr->rx_buffers, 3573 sc->num_rx_desc * sizeof(struct igc_rx_buf)); 3574 rxr->rx_buffers = NULL; 3575 } 3576 3577 mutex_destroy(&rxr->rxr_lock); 3578 } 3579 3580 /********************************************************************* 3581 * 3582 * Clear status registers in all RX descriptors. 3583 * 3584 **********************************************************************/ 3585 static void 3586 igc_clear_receive_status(struct rx_ring *rxr) 3587 { 3588 struct igc_softc *sc = rxr->sc; 3589 3590 mutex_enter(&rxr->rxr_lock); 3591 3592 for (int id = 0; id < sc->num_rx_desc; id++) { 3593 union igc_adv_rx_desc *rxdesc = &rxr->rx_base[id]; 3594 3595 igc_rxdesc_sync(rxr, id, 3596 BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE); 3597 rxdesc->wb.upper.status_error = 0; 3598 igc_rxdesc_sync(rxr, id, 3599 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); 3600 } 3601 3602 mutex_exit(&rxr->rxr_lock); 3603 } 3604 3605 /* 3606 * Initialise the RSS mapping for NICs that support multiple transmit/ 3607 * receive rings. 3608 */ 3609 static void 3610 igc_initialize_rss_mapping(struct igc_softc *sc) 3611 { 3612 struct igc_hw *hw = &sc->hw; 3613 3614 /* 3615 * The redirection table controls which destination 3616 * queue each bucket redirects traffic to. 3617 * Each DWORD represents four queues, with the LSB 3618 * being the first queue in the DWORD. 3619 * 3620 * This just allocates buckets to queues using round-robin 3621 * allocation. 3622 * 3623 * NOTE: It Just Happens to line up with the default 3624 * RSS allocation method. 3625 */ 3626 3627 /* Warning FM follows */ 3628 uint32_t reta = 0; 3629 for (int i = 0; i < 128; i++) { 3630 const int shift = 0; /* XXXRO */ 3631 int queue_id = i % sc->sc_nqueues; 3632 /* Adjust if required */ 3633 queue_id <<= shift; 3634 3635 /* 3636 * The low 8 bits are for hash value (n+0); 3637 * The next 8 bits are for hash value (n+1), etc. 3638 */ 3639 reta >>= 8; 3640 reta |= ((uint32_t)queue_id) << 24; 3641 if ((i & 3) == 3) { 3642 IGC_WRITE_REG(hw, IGC_RETA(i >> 2), reta); 3643 reta = 0; 3644 } 3645 } 3646 3647 /* 3648 * MRQC: Multiple Receive Queues Command 3649 * Set queuing to RSS control, number depends on the device. 3650 */ 3651 3652 /* Set up random bits */ 3653 uint32_t rss_key[RSS_KEYSIZE / sizeof(uint32_t)]; 3654 rss_getkey((uint8_t *)rss_key); 3655 3656 /* Now fill our hash function seeds */ 3657 for (int i = 0; i < __arraycount(rss_key); i++) 3658 IGC_WRITE_REG_ARRAY(hw, IGC_RSSRK(0), i, rss_key[i]); 3659 3660 /* 3661 * Configure the RSS fields to hash upon. 3662 */ 3663 uint32_t mrqc = IGC_MRQC_ENABLE_RSS_4Q; 3664 mrqc |= IGC_MRQC_RSS_FIELD_IPV4 | IGC_MRQC_RSS_FIELD_IPV4_TCP; 3665 mrqc |= IGC_MRQC_RSS_FIELD_IPV6 | IGC_MRQC_RSS_FIELD_IPV6_TCP; 3666 mrqc |= IGC_MRQC_RSS_FIELD_IPV6_TCP_EX; 3667 3668 IGC_WRITE_REG(hw, IGC_MRQC, mrqc); 3669 } 3670 3671 /* 3672 * igc_get_hw_control sets the {CTRL_EXT|FWSM}:DRV_LOAD bit. 3673 * For ASF and Pass Through versions of f/w this means 3674 * that the driver is loaded. For AMT version type f/w 3675 * this means that the network i/f is open. 3676 */ 3677 static void 3678 igc_get_hw_control(struct igc_softc *sc) 3679 { 3680 const uint32_t ctrl_ext = IGC_READ_REG(&sc->hw, IGC_CTRL_EXT); 3681 3682 IGC_WRITE_REG(&sc->hw, IGC_CTRL_EXT, ctrl_ext | IGC_CTRL_EXT_DRV_LOAD); 3683 } 3684 3685 /* 3686 * igc_release_hw_control resets {CTRL_EXT|FWSM}:DRV_LOAD bit. 3687 * For ASF and Pass Through versions of f/w this means that 3688 * the driver is no longer loaded. For AMT versions of the 3689 * f/w this means that the network i/f is closed. 3690 */ 3691 static void 3692 igc_release_hw_control(struct igc_softc *sc) 3693 { 3694 const uint32_t ctrl_ext = IGC_READ_REG(&sc->hw, IGC_CTRL_EXT); 3695 3696 IGC_WRITE_REG(&sc->hw, IGC_CTRL_EXT, ctrl_ext & ~IGC_CTRL_EXT_DRV_LOAD); 3697 } 3698 3699 static int 3700 igc_is_valid_ether_addr(uint8_t *addr) 3701 { 3702 const char zero_addr[6] = { 0, 0, 0, 0, 0, 0 }; 3703 3704 if ((addr[0] & 1) || !bcmp(addr, zero_addr, ETHER_ADDR_LEN)) 3705 return 0; 3706 3707 return 1; 3708 } 3709 3710 static void 3711 igc_print_devinfo(struct igc_softc *sc) 3712 { 3713 device_t dev = sc->sc_dev; 3714 struct igc_hw *hw = &sc->hw; 3715 struct igc_phy_info *phy = &hw->phy; 3716 u_int oui, model, rev; 3717 uint16_t id1, id2, nvm_ver, phy_ver, etk_lo, etk_hi; 3718 char descr[MII_MAX_DESCR_LEN]; 3719 3720 /* Print PHY Info */ 3721 id1 = phy->id >> 16; 3722 /* The revision field in phy->id is cleard and it's in phy->revision */ 3723 id2 = (phy->id & 0xfff0) | phy->revision; 3724 oui = MII_OUI(id1, id2); 3725 model = MII_MODEL(id2); 3726 rev = MII_REV(id2); 3727 mii_get_descr(descr, sizeof(descr), oui, model); 3728 if (descr[0]) 3729 aprint_normal_dev(dev, "PHY: %s, rev. %d", 3730 descr, rev); 3731 else 3732 aprint_normal_dev(dev, 3733 "PHY OUI 0x%06x, model 0x%04x, rev. %d", 3734 oui, model, rev); 3735 3736 /* PHY FW version */ 3737 phy->ops.read_reg(hw, 0x1e, &phy_ver); 3738 aprint_normal(", PHY FW version 0x%04hx\n", phy_ver); 3739 3740 /* NVM version */ 3741 hw->nvm.ops.read(hw, NVM_VERSION, 1, &nvm_ver); 3742 3743 /* EtrackID */ 3744 hw->nvm.ops.read(hw, NVM_ETKID_LO, 1, &etk_lo); 3745 hw->nvm.ops.read(hw, NVM_ETKID_HI, 1, &etk_hi); 3746 3747 aprint_normal_dev(dev, 3748 "NVM image version %x.%02x, EtrackID %04hx%04hx\n", 3749 (nvm_ver & NVM_VERSION_MAJOR) >> NVM_VERSION_MAJOR_SHIFT, 3750 nvm_ver & NVM_VERSION_MINOR, etk_hi, etk_lo); 3751 } 3752 3753 /* Module interface */ 3754 3755 MODULE(MODULE_CLASS_DRIVER, if_igc, "pci"); 3756 3757 #ifdef _MODULE 3758 #include "ioconf.c" 3759 #endif 3760 3761 static int 3762 if_igc_modcmd(modcmd_t cmd, void *opaque) 3763 { 3764 int error = 0; 3765 3766 switch (cmd) { 3767 case MODULE_CMD_INIT: 3768 #ifdef _MODULE 3769 error = config_init_component(cfdriver_ioconf_igc, 3770 cfattach_ioconf_igc, cfdata_ioconf_igc); 3771 #endif 3772 return error; 3773 case MODULE_CMD_FINI: 3774 #ifdef _MODULE 3775 error = config_fini_component(cfdriver_ioconf_igc, 3776 cfattach_ioconf_igc, cfdata_ioconf_igc); 3777 #endif 3778 return error; 3779 default: 3780 return ENOTTY; 3781 } 3782 } 3783
Indexes created Wed Oct 22 06:10:02 GMT 2025