1 /* $NetBSD: if_igc.c,v 1.26 2026/08/12 01:42:16 rin Exp $ */ 2 /* $OpenBSD: if_igc.c,v 1.13 2023/04/28 10:18:57 bluhm Exp $ */ 3 /*- 4 * SPDX-License-Identifier: BSD-2-Clause 5 * 6 * Copyright (c) 2016 Nicole Graziano <nicole (at) nextbsd.org> 7 * All rights reserved. 8 * Copyright (c) 2021 Rubicon Communications, LLC (Netgate) 9 * 10 * Redistribution and use in source and binary forms, with or without 11 * modification, are permitted provided that the following conditions 12 * are met: 13 * 1. Redistributions of source code must retain the above copyright 14 * notice, this list of conditions and the following disclaimer. 15 * 2. Redistributions in binary form must reproduce the above copyright 16 * notice, this list of conditions and the following disclaimer in the 17 * documentation and/or other materials provided with the distribution. 18 * 19 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 20 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 21 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 22 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 23 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 24 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 25 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 26 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 27 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 28 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 29 * SUCH DAMAGE. 30 */ 31 32 #include <sys/cdefs.h> 33 __KERNEL_RCSID(0, "$NetBSD: if_igc.c,v 1.26 2026/08/12 01:42:16 rin Exp $"); 34 35 #ifdef _KERNEL_OPT 36 #include "opt_if_igc.h" 37 #if 0 /* notyet */ 38 #include "vlan.h" 39 #endif 40 #endif 41 42 #include <sys/param.h> 43 #include <sys/systm.h> 44 #include <sys/bus.h> 45 #include <sys/cpu.h> 46 #include <sys/device.h> 47 #include <sys/endian.h> 48 #include <sys/intr.h> 49 #include <sys/interrupt.h> 50 #include <sys/kernel.h> 51 #include <sys/kmem.h> 52 #include <sys/mbuf.h> 53 #include <sys/mutex.h> 54 #include <sys/socket.h> 55 #include <sys/workqueue.h> 56 #include <sys/xcall.h> 57 #include <sys/module.h> 58 59 #include <net/bpf.h> 60 #include <net/if.h> 61 #include <net/if_ether.h> 62 #include <net/if_media.h> 63 #include <net/if_vlanvar.h> 64 #include <net/rss_config.h> 65 66 #include <netinet/in.h> 67 #include <netinet/ip.h> 68 #include <netinet/ip6.h> 69 #include <netinet/tcp.h> 70 71 #include <dev/pci/pcivar.h> 72 #include <dev/pci/pcireg.h> 73 #include <dev/pci/pcidevs.h> 74 75 #include <dev/pci/igc/if_igc.h> 76 #include <dev/pci/igc/igc_evcnt.h> 77 #include <dev/pci/igc/igc_hw.h> 78 #include <dev/mii/miivar.h> 79 80 #define IGC_WORKQUEUE_PRI PRI_SOFTNET 81 82 #ifndef IGC_RX_INTR_PROCESS_LIMIT_DEFAULT 83 #define IGC_RX_INTR_PROCESS_LIMIT_DEFAULT 0 84 #endif 85 #ifndef IGC_TX_INTR_PROCESS_LIMIT_DEFAULT 86 #define IGC_TX_INTR_PROCESS_LIMIT_DEFAULT 0 87 #endif 88 89 #ifndef IGC_RX_PROCESS_LIMIT_DEFAULT 90 #define IGC_RX_PROCESS_LIMIT_DEFAULT 256 91 #endif 92 #ifndef IGC_TX_PROCESS_LIMIT_DEFAULT 93 #define IGC_TX_PROCESS_LIMIT_DEFAULT 256 94 #endif 95 96 #define htolem32(p, x) (*((uint32_t *)(p)) = htole32(x)) 97 #define htolem64(p, x) (*((uint64_t *)(p)) = htole64(x)) 98 99 static const struct igc_product { 100 pci_vendor_id_t igcp_vendor; 101 pci_product_id_t igcp_product; 102 const char *igcp_name; 103 } igc_products[] = { 104 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_I225_IT, 105 "Intel(R) Ethernet Controller I225-IT(2)" }, 106 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_I226_LM, 107 "Intel(R) Ethernet Controller I226-LM" }, 108 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_I226_V, 109 "Intel(R) Ethernet Controller I226-V" }, 110 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_I226_IT, 111 "Intel(R) Ethernet Controller I226-IT" }, 112 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_I221_V, 113 "Intel(R) Ethernet Controller I221-V" }, 114 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_I226_BLANK_NVM, 115 "Intel(R) Ethernet Controller I226(blankNVM)" }, 116 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_I225_LM, 117 "Intel(R) Ethernet Controller I225-LM" }, 118 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_I225_V, 119 "Intel(R) Ethernet Controller I225-V" }, 120 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_I220_V, 121 "Intel(R) Ethernet Controller I220-V" }, 122 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_I225_I, 123 "Intel(R) Ethernet Controller I225-I" }, 124 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_I225_BLANK_NVM, 125 "Intel(R) Ethernet Controller I225(blankNVM)" }, 126 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_I225_K, 127 "Intel(R) Ethernet Controller I225-K" }, 128 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_I225_K2, 129 "Intel(R) Ethernet Controller I225-K(2)" }, 130 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_I226_K, 131 "Intel(R) Ethernet Controller I226-K" }, 132 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_I225_LMVP, 133 "Intel(R) Ethernet Controller I225-LMvP(2)" }, 134 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_I226_LMVP, 135 "Intel(R) Ethernet Controller I226-LMvP" }, 136 { 0, 0, NULL }, 137 }; 138 139 #define IGC_DF_CFG 0x1 140 #define IGC_DF_TX 0x2 141 #define IGC_DF_RX 0x4 142 #define IGC_DF_MISC 0x8 143 144 #ifdef IGC_DEBUG_FLAGS 145 int igc_debug_flags = IGC_DEBUG_FLAGS; 146 #else 147 int igc_debug_flags = 0; 148 #endif 149 150 #define DPRINTF(flag, fmt, args...) do { \ 151 if (igc_debug_flags & (IGC_DF_ ## flag)) \ 152 printf("%s: %d: " fmt, __func__, __LINE__, ##args); \ 153 } while (0) 154 155 /********************************************************************* 156 * Function Prototypes 157 *********************************************************************/ 158 static int igc_match(device_t, cfdata_t, void *); 159 static void igc_attach(device_t, device_t, void *); 160 static int igc_detach(device_t, int); 161 162 static void igc_identify_hardware(struct igc_softc *); 163 static int igc_adjust_nqueues(struct igc_softc *); 164 static int igc_allocate_pci_resources(struct igc_softc *); 165 static int igc_allocate_interrupts(struct igc_softc *); 166 static int igc_allocate_queues(struct igc_softc *); 167 static void igc_free_pci_resources(struct igc_softc *); 168 static void igc_free_interrupts(struct igc_softc *); 169 static void igc_free_queues(struct igc_softc *); 170 static void igc_reset(struct igc_softc *); 171 static void igc_init_dmac(struct igc_softc *, uint32_t); 172 static int igc_setup_interrupts(struct igc_softc *); 173 static void igc_attach_counters(struct igc_softc *sc); 174 static void igc_detach_counters(struct igc_softc *sc); 175 static void igc_update_counters(struct igc_softc *sc); 176 static void igc_clear_counters(struct igc_softc *sc); 177 static int igc_setup_msix(struct igc_softc *); 178 static int igc_setup_msi(struct igc_softc *); 179 static int igc_setup_intx(struct igc_softc *); 180 static int igc_dma_malloc(struct igc_softc *, bus_size_t, 181 struct igc_dma_alloc *); 182 static void igc_dma_free(struct igc_softc *, struct igc_dma_alloc *); 183 static void igc_setup_interface(struct igc_softc *); 184 185 static int igc_init(struct ifnet *); 186 static int igc_init_locked(struct igc_softc *); 187 static void igc_start(struct ifnet *); 188 static int igc_transmit(struct ifnet *, struct mbuf *); 189 static void igc_tx_common_locked(struct ifnet *, struct tx_ring *, int); 190 static bool igc_txeof(struct tx_ring *, u_int); 191 static void igc_intr_barrier(struct igc_softc *); 192 static void igc_stop(struct ifnet *, int); 193 static void igc_stop_locked(struct igc_softc *); 194 static int igc_ioctl(struct ifnet *, u_long, void *); 195 #ifdef IF_RXR 196 static int igc_rxrinfo(struct igc_softc *, struct if_rxrinfo *); 197 #endif 198 static void igc_rxfill(struct rx_ring *); 199 static bool igc_rxrefill(struct rx_ring *); 200 static void igc_rxrefill_callout(void *); 201 static bool igc_rxeof(struct rx_ring *, u_int); 202 static int igc_rx_checksum(struct igc_queue *, uint64_t, uint32_t, 203 uint32_t); 204 static void igc_watchdog(struct ifnet *); 205 static void igc_tick(void *); 206 static void igc_media_status(struct ifnet *, struct ifmediareq *); 207 static int igc_media_change(struct ifnet *); 208 static int igc_ifflags_cb(struct ethercom *); 209 static void igc_set_filter(struct igc_softc *); 210 static void igc_update_link_status(struct igc_softc *); 211 static int igc_get_buf(struct rx_ring *, int, bool); 212 static bool igc_tx_ctx_setup(struct tx_ring *, struct mbuf *, int, 213 uint32_t *, uint32_t *); 214 215 static void igc_configure_queues(struct igc_softc *); 216 static void igc_set_queues(struct igc_softc *, uint32_t, uint32_t, int); 217 static void igc_enable_queue(struct igc_softc *, uint32_t); 218 static void igc_enable_intr(struct igc_softc *); 219 static void igc_disable_intr(struct igc_softc *); 220 static int igc_intr_link(void *); 221 static int igc_intr_queue(void *); 222 static int igc_intr(void *); 223 static void igc_handle_queue(void *); 224 static void igc_handle_queue_work(struct work *, void *); 225 static void igc_sched_handle_queue(struct igc_softc *, struct igc_queue *); 226 static void igc_barrier_handle_queue(struct igc_softc *); 227 228 static int igc_allocate_transmit_buffers(struct tx_ring *); 229 static int igc_setup_transmit_structures(struct igc_softc *); 230 static int igc_setup_transmit_ring(struct tx_ring *); 231 static void igc_initialize_transmit_unit(struct igc_softc *); 232 static void igc_free_transmit_structures(struct igc_softc *); 233 static void igc_free_transmit_buffers(struct tx_ring *); 234 static void igc_withdraw_transmit_packets(struct tx_ring *, bool); 235 static int igc_allocate_receive_buffers(struct rx_ring *); 236 static int igc_setup_receive_structures(struct igc_softc *); 237 static int igc_setup_receive_ring(struct rx_ring *); 238 static void igc_initialize_receive_unit(struct igc_softc *); 239 static void igc_free_receive_structures(struct igc_softc *); 240 static void igc_free_receive_buffers(struct rx_ring *); 241 static void igc_clear_receive_status(struct rx_ring *); 242 static void igc_initialize_rss_mapping(struct igc_softc *); 243 244 static void igc_get_hw_control(struct igc_softc *); 245 static void igc_release_hw_control(struct igc_softc *); 246 static int igc_is_valid_ether_addr(uint8_t *); 247 static void igc_print_devinfo(struct igc_softc *); 248 249 CFATTACH_DECL3_NEW(igc, sizeof(struct igc_softc), 250 igc_match, igc_attach, igc_detach, NULL, NULL, NULL, 0); 251 252 static inline int 253 igc_txdesc_incr(struct igc_softc *sc, int id) 254 { 255 256 if (++id == sc->num_tx_desc) 257 id = 0; 258 return id; 259 } 260 261 static inline int __unused 262 igc_txdesc_decr(struct igc_softc *sc, int id) 263 { 264 265 if (--id < 0) 266 id = sc->num_tx_desc - 1; 267 return id; 268 } 269 270 static inline void 271 igc_txdesc_sync(struct tx_ring *txr, int id, int ops) 272 { 273 274 bus_dmamap_sync(txr->txdma.dma_tag, txr->txdma.dma_map, 275 id * sizeof(union igc_adv_tx_desc), sizeof(union igc_adv_tx_desc), 276 ops); 277 } 278 279 static inline int 280 igc_rxdesc_incr(struct igc_softc *sc, int id) 281 { 282 283 if (++id == sc->num_rx_desc) 284 id = 0; 285 return id; 286 } 287 288 static inline int 289 igc_rxdesc_decr(struct igc_softc *sc, int id) 290 { 291 292 if (--id < 0) 293 id = sc->num_rx_desc - 1; 294 return id; 295 } 296 297 static inline void 298 igc_rxdesc_sync(struct rx_ring *rxr, int id, int ops) 299 { 300 301 bus_dmamap_sync(rxr->rxdma.dma_tag, rxr->rxdma.dma_map, 302 id * sizeof(union igc_adv_rx_desc), sizeof(union igc_adv_rx_desc), 303 ops); 304 } 305 306 static const struct igc_product * 307 igc_lookup(const struct pci_attach_args *pa) 308 { 309 const struct igc_product *igcp; 310 311 for (igcp = igc_products; igcp->igcp_name != NULL; igcp++) { 312 if (PCI_VENDOR(pa->pa_id) == igcp->igcp_vendor && 313 PCI_PRODUCT(pa->pa_id) == igcp->igcp_product) 314 return igcp; 315 } 316 return NULL; 317 } 318 319 /********************************************************************* 320 * Device identification routine 321 * 322 * igc_match determines if the driver should be loaded on 323 * adapter based on PCI vendor/device id of the adapter. 324 * 325 * return 0 on success, positive on failure 326 *********************************************************************/ 327 static int 328 igc_match(device_t parent, cfdata_t match, void *aux) 329 { 330 struct pci_attach_args *pa = aux; 331 332 if (igc_lookup(pa) != NULL) 333 return 1; 334 335 return 0; 336 } 337 338 /********************************************************************* 339 * Device initialization routine 340 * 341 * The attach entry point is called when the driver is being loaded. 342 * This routine identifies the type of hardware, allocates all resources 343 * and initializes the hardware. 344 * 345 * return 0 on success, positive on failure 346 *********************************************************************/ 347 static void 348 igc_attach(device_t parent, device_t self, void *aux) 349 { 350 struct pci_attach_args *pa = aux; 351 struct igc_softc *sc = device_private(self); 352 struct igc_hw *hw = &sc->hw; 353 354 const struct igc_product *igcp = igc_lookup(pa); 355 KASSERT(igcp != NULL); 356 357 sc->sc_dev = self; 358 callout_init(&sc->sc_tick_ch, CALLOUT_MPSAFE); 359 callout_setfunc(&sc->sc_tick_ch, igc_tick, sc); 360 sc->sc_core_stopping = false; 361 362 sc->osdep.os_sc = sc; 363 sc->osdep.os_pa = *pa; 364 #ifndef __aarch64__ 365 /* 366 * XXX PR port-arm/57643 367 * 64-bit DMA does not work at least for LX2K with 32/64GB memory. 368 * smmu(4) support may be required. 369 */ 370 if (pci_dma64_available(pa)) { 371 aprint_verbose(", 64-bit DMA"); 372 sc->osdep.os_dmat = pa->pa_dmat64; 373 } else 374 #endif 375 { 376 aprint_verbose(", 32-bit DMA"); 377 sc->osdep.os_dmat = pa->pa_dmat; 378 } 379 380 pci_aprint_devinfo_fancy(pa, "Ethernet controller", igcp->igcp_name, 1); 381 382 /* Determine hardware and mac info */ 383 igc_identify_hardware(sc); 384 385 sc->num_tx_desc = IGC_DEFAULT_TXD; 386 sc->num_rx_desc = IGC_DEFAULT_RXD; 387 388 /* Setup PCI resources */ 389 if (igc_allocate_pci_resources(sc)) { 390 aprint_error_dev(sc->sc_dev, 391 "unable to allocate PCI resources\n"); 392 goto err; 393 } 394 395 if (igc_allocate_interrupts(sc)) { 396 aprint_error_dev(sc->sc_dev, "unable to allocate interrupts\n"); 397 goto err_pci; 398 } 399 400 /* Allocate TX/RX queues */ 401 if (igc_allocate_queues(sc)) { 402 aprint_error_dev(sc->sc_dev, "unable to allocate queues\n"); 403 goto err_alloc_intr; 404 } 405 406 /* Do shared code initialization */ 407 if (igc_setup_init_funcs(hw, true)) { 408 aprint_error_dev(sc->sc_dev, "unable to initialize\n"); 409 goto err_alloc_intr; 410 } 411 412 hw->mac.autoneg = DO_AUTO_NEG; 413 hw->phy.autoneg_wait_to_complete = false; 414 hw->phy.autoneg_advertised = AUTONEG_ADV_DEFAULT; 415 416 /* Copper options. */ 417 if (hw->phy.media_type == igc_media_type_copper) 418 hw->phy.mdix = AUTO_ALL_MODES; 419 420 /* Set the max frame size. */ 421 sc->hw.mac.max_frame_size = 9234; 422 423 /* Allocate multicast array memory. */ 424 sc->mta = kmem_alloc(IGC_MTA_LEN, KM_SLEEP); 425 426 /* Check SOL/IDER usage. */ 427 if (igc_check_reset_block(hw)) { 428 aprint_error_dev(sc->sc_dev, 429 "PHY reset is blocked due to SOL/IDER session\n"); 430 } 431 432 /* Disable Energy Efficient Ethernet. */ 433 sc->hw.dev_spec._i225.eee_disable = true; 434 435 igc_reset_hw(hw); 436 437 /* Make sure we have a good EEPROM before we read from it. */ 438 if (igc_validate_nvm_checksum(hw) < 0) { 439 /* 440 * Some PCI-E parts fail the first check due to 441 * the link being in sleep state, call it again, 442 * if it fails a second time its a real issue. 443 */ 444 if (igc_validate_nvm_checksum(hw) < 0) { 445 aprint_error_dev(sc->sc_dev, 446 "EEPROM checksum invalid\n"); 447 goto err_late; 448 } 449 } 450 451 /* Copy the permanent MAC address out of the EEPROM. */ 452 if (igc_read_mac_addr(hw) < 0) { 453 aprint_error_dev(sc->sc_dev, 454 "unable to read MAC address from EEPROM\n"); 455 goto err_late; 456 } 457 458 if (!igc_is_valid_ether_addr(hw->mac.addr)) { 459 aprint_error_dev(sc->sc_dev, "invalid MAC address\n"); 460 goto err_late; 461 } 462 463 if (igc_setup_interrupts(sc)) 464 goto err_late; 465 466 /* Attach counters. */ 467 igc_attach_counters(sc); 468 469 /* Setup OS specific network interface. */ 470 igc_setup_interface(sc); 471 472 igc_print_devinfo(sc); 473 474 igc_reset(sc); 475 hw->mac.get_link_status = true; 476 igc_update_link_status(sc); 477 478 /* The driver can now take control from firmware. */ 479 igc_get_hw_control(sc); 480 481 aprint_normal_dev(sc->sc_dev, "Ethernet address %s\n", 482 ether_sprintf(sc->hw.mac.addr)); 483 484 if (pmf_device_register(self, NULL, NULL)) 485 pmf_class_network_register(self, &sc->sc_ec.ec_if); 486 else 487 aprint_error_dev(self, "couldn't establish power handler\n"); 488 489 return; 490 491 err_late: 492 igc_release_hw_control(sc); 493 kmem_free(sc->mta, IGC_MTA_LEN); 494 err_alloc_intr: 495 igc_free_interrupts(sc); 496 err_pci: 497 igc_free_pci_resources(sc); 498 err: return; 499 } 500 501 /********************************************************************* 502 * Device removal routine 503 * 504 * The detach entry point is called when the driver is being removed. 505 * This routine stops the adapter and deallocates all the resources 506 * that were allocated for driver operation. 507 * 508 * return 0 on success, positive on failure 509 *********************************************************************/ 510 static int 511 igc_detach(device_t self, int flags) 512 { 513 struct igc_softc *sc = device_private(self); 514 struct ifnet *ifp = &sc->sc_ec.ec_if; 515 516 mutex_enter(&sc->sc_core_lock); 517 igc_stop_locked(sc); 518 mutex_exit(&sc->sc_core_lock); 519 520 igc_detach_counters(sc); 521 522 igc_free_queues(sc); 523 524 igc_phy_hw_reset(&sc->hw); 525 igc_release_hw_control(sc); 526 527 ether_ifdetach(ifp); 528 if_detach(ifp); 529 ifmedia_fini(&sc->media); 530 531 igc_free_interrupts(sc); 532 igc_free_pci_resources(sc); 533 kmem_free(sc->mta, IGC_MTA_LEN); 534 535 mutex_destroy(&sc->sc_core_lock); 536 537 return 0; 538 } 539 540 static void 541 igc_identify_hardware(struct igc_softc *sc) 542 { 543 struct igc_osdep *os = &sc->osdep; 544 struct pci_attach_args *pa = &os->os_pa; 545 546 /* Save off the information about this board. */ 547 sc->hw.device_id = PCI_PRODUCT(pa->pa_id); 548 549 /* Do shared code init and setup. */ 550 if (igc_set_mac_type(&sc->hw)) { 551 aprint_error_dev(sc->sc_dev, "unable to identify hardware\n"); 552 return; 553 } 554 } 555 556 static int 557 igc_allocate_pci_resources(struct igc_softc *sc) 558 { 559 struct igc_osdep *os = &sc->osdep; 560 struct pci_attach_args *pa = &os->os_pa; 561 562 /* 563 * Enable bus mastering and memory-mapped I/O for sure. 564 */ 565 pcireg_t csr = 566 pci_conf_read(pa->pa_pc, pa->pa_tag, PCI_COMMAND_STATUS_REG); 567 csr |= PCI_COMMAND_MASTER_ENABLE | PCI_COMMAND_MEM_ENABLE; 568 pci_conf_write(pa->pa_pc, pa->pa_tag, PCI_COMMAND_STATUS_REG, csr); 569 570 const pcireg_t memtype = 571 pci_mapreg_type(pa->pa_pc, pa->pa_tag, IGC_PCIREG); 572 if (pci_mapreg_map(pa, IGC_PCIREG, memtype, 0, &os->os_memt, 573 &os->os_memh, &os->os_membase, &os->os_memsize)) { 574 aprint_error_dev(sc->sc_dev, "unable to map registers\n"); 575 return ENXIO; 576 } 577 578 sc->hw.hw_addr = os->os_membase; 579 sc->hw.back = os; 580 581 return 0; 582 } 583 584 static int __unused 585 igc_adjust_nqueues(struct igc_softc *sc) 586 { 587 struct pci_attach_args *pa = &sc->osdep.os_pa; 588 int nqueues = MIN(IGC_MAX_NQUEUES, ncpu); 589 590 const int nmsix = pci_msix_count(pa->pa_pc, pa->pa_tag); 591 if (nmsix <= 1) 592 nqueues = 1; 593 else if (nmsix < nqueues + 1) 594 nqueues = nmsix - 1; 595 596 return nqueues; 597 } 598 599 static int 600 igc_allocate_interrupts(struct igc_softc *sc) 601 { 602 struct pci_attach_args *pa = &sc->osdep.os_pa; 603 int error; 604 605 #ifndef IGC_DISABLE_MSIX 606 const int nqueues = igc_adjust_nqueues(sc); 607 if (nqueues > 1) { 608 sc->sc_nintrs = nqueues + 1; 609 error = pci_msix_alloc_exact(pa, &sc->sc_intrs, sc->sc_nintrs); 610 if (!error) { 611 sc->sc_nqueues = nqueues; 612 sc->sc_intr_type = PCI_INTR_TYPE_MSIX; 613 return 0; 614 } 615 } 616 #endif 617 618 /* fallback to MSI */ 619 sc->sc_nintrs = sc->sc_nqueues = 1; 620 621 #ifndef IGC_DISABLE_MSI 622 error = pci_msi_alloc_exact(pa, &sc->sc_intrs, sc->sc_nintrs); 623 if (!error) { 624 sc->sc_intr_type = PCI_INTR_TYPE_MSI; 625 return 0; 626 } 627 #endif 628 629 /* fallback to INTx */ 630 631 error = pci_intx_alloc(pa, &sc->sc_intrs); 632 if (!error) { 633 sc->sc_intr_type = PCI_INTR_TYPE_INTX; 634 return 0; 635 } 636 637 return error; 638 } 639 640 static int 641 igc_allocate_queues(struct igc_softc *sc) 642 { 643 device_t dev = sc->sc_dev; 644 int rxconf = 0, txconf = 0; 645 646 /* Allocate the top level queue structs. */ 647 sc->queues = 648 kmem_zalloc(sc->sc_nqueues * sizeof(struct igc_queue), KM_SLEEP); 649 650 /* Allocate the TX ring. */ 651 sc->tx_rings = 652 kmem_zalloc(sc->sc_nqueues * sizeof(struct tx_ring), KM_SLEEP); 653 654 /* Allocate the RX ring. */ 655 sc->rx_rings = 656 kmem_zalloc(sc->sc_nqueues * sizeof(struct rx_ring), KM_SLEEP); 657 658 /* Set up the TX queues. */ 659 for (int iq = 0; iq < sc->sc_nqueues; iq++, txconf++) { 660 struct tx_ring *txr = &sc->tx_rings[iq]; 661 const int tsize = roundup2( 662 sc->num_tx_desc * sizeof(union igc_adv_tx_desc), 663 IGC_DBA_ALIGN); 664 665 txr->sc = sc; 666 txr->txr_igcq = &sc->queues[iq]; 667 txr->me = iq; 668 if (igc_dma_malloc(sc, tsize, &txr->txdma)) { 669 aprint_error_dev(dev, 670 "unable to allocate TX descriptor\n"); 671 goto fail; 672 } 673 txr->tx_base = (union igc_adv_tx_desc *)txr->txdma.dma_vaddr; 674 memset(txr->tx_base, 0, tsize); 675 } 676 677 /* Prepare transmit descriptors and buffers. */ 678 if (igc_setup_transmit_structures(sc)) { 679 aprint_error_dev(dev, "unable to setup transmit structures\n"); 680 goto fail; 681 } 682 683 /* Set up the RX queues. */ 684 for (int iq = 0; iq < sc->sc_nqueues; iq++, rxconf++) { 685 struct rx_ring *rxr = &sc->rx_rings[iq]; 686 const int rsize = roundup2( 687 sc->num_rx_desc * sizeof(union igc_adv_rx_desc), 688 IGC_DBA_ALIGN); 689 690 rxr->sc = sc; 691 rxr->rxr_igcq = &sc->queues[iq]; 692 rxr->me = iq; 693 #ifdef OPENBSD 694 timeout_set(&rxr->rx_refill, igc_rxrefill, rxr); 695 #endif 696 callout_init(&rxr->rx_refill, CALLOUT_MPSAFE); 697 callout_setfunc(&rxr->rx_refill, igc_rxrefill_callout, rxr); 698 699 if (igc_dma_malloc(sc, rsize, &rxr->rxdma)) { 700 aprint_error_dev(dev, 701 "unable to allocate RX descriptor\n"); 702 goto fail; 703 } 704 rxr->rx_base = (union igc_adv_rx_desc *)rxr->rxdma.dma_vaddr; 705 memset(rxr->rx_base, 0, rsize); 706 } 707 708 sc->rx_mbuf_sz = MCLBYTES; 709 /* Prepare receive descriptors and buffers. */ 710 if (igc_setup_receive_structures(sc)) { 711 aprint_error_dev(sc->sc_dev, 712 "unable to setup receive structures\n"); 713 goto fail; 714 } 715 716 /* Set up the queue holding structs. */ 717 for (int iq = 0; iq < sc->sc_nqueues; iq++) { 718 struct igc_queue *q = &sc->queues[iq]; 719 720 q->sc = sc; 721 q->txr = &sc->tx_rings[iq]; 722 q->rxr = &sc->rx_rings[iq]; 723 } 724 725 return 0; 726 727 fail: 728 for (struct rx_ring *rxr = sc->rx_rings; rxconf > 0; rxr++, rxconf--) 729 igc_dma_free(sc, &rxr->rxdma); 730 for (struct tx_ring *txr = sc->tx_rings; txconf > 0; txr++, txconf--) 731 igc_dma_free(sc, &txr->txdma); 732 733 kmem_free(sc->rx_rings, sc->sc_nqueues * sizeof(struct rx_ring)); 734 sc->rx_rings = NULL; 735 kmem_free(sc->tx_rings, sc->sc_nqueues * sizeof(struct tx_ring)); 736 sc->tx_rings = NULL; 737 kmem_free(sc->queues, sc->sc_nqueues * sizeof(struct igc_queue)); 738 sc->queues = NULL; 739 740 return ENOMEM; 741 } 742 743 static void 744 igc_free_pci_resources(struct igc_softc *sc) 745 { 746 struct igc_osdep *os = &sc->osdep; 747 748 if (os->os_membase != 0) 749 bus_space_unmap(os->os_memt, os->os_memh, os->os_memsize); 750 os->os_membase = 0; 751 } 752 753 static void 754 igc_free_interrupts(struct igc_softc *sc) 755 { 756 struct pci_attach_args *pa = &sc->osdep.os_pa; 757 pci_chipset_tag_t pc = pa->pa_pc; 758 759 for (int i = 0; i < sc->sc_nintrs; i++) { 760 if (sc->sc_ihs[i] != NULL) { 761 pci_intr_disestablish(pc, sc->sc_ihs[i]); 762 sc->sc_ihs[i] = NULL; 763 } 764 } 765 pci_intr_release(pc, sc->sc_intrs, sc->sc_nintrs); 766 } 767 768 static void 769 igc_free_queues(struct igc_softc *sc) 770 { 771 772 igc_free_receive_structures(sc); 773 for (int iq = 0; iq < sc->sc_nqueues; iq++) { 774 struct rx_ring *rxr = &sc->rx_rings[iq]; 775 776 igc_dma_free(sc, &rxr->rxdma); 777 } 778 779 igc_free_transmit_structures(sc); 780 for (int iq = 0; iq < sc->sc_nqueues; iq++) { 781 struct tx_ring *txr = &sc->tx_rings[iq]; 782 783 igc_dma_free(sc, &txr->txdma); 784 } 785 786 kmem_free(sc->rx_rings, sc->sc_nqueues * sizeof(struct rx_ring)); 787 kmem_free(sc->tx_rings, sc->sc_nqueues * sizeof(struct tx_ring)); 788 kmem_free(sc->queues, sc->sc_nqueues * sizeof(struct igc_queue)); 789 } 790 791 /********************************************************************* 792 * 793 * Initialize the hardware to a configuration as specified by the 794 * adapter structure. 795 * 796 **********************************************************************/ 797 static void 798 igc_reset(struct igc_softc *sc) 799 { 800 struct igc_hw *hw = &sc->hw; 801 802 /* Let the firmware know the OS is in control */ 803 igc_get_hw_control(sc); 804 805 /* 806 * Packet Buffer Allocation (PBA) 807 * Writing PBA sets the receive portion of the buffer 808 * the remainder is used for the transmit buffer. 809 */ 810 const uint32_t pba = IGC_PBA_34K; 811 812 /* 813 * These parameters control the automatic generation (Tx) and 814 * response (Rx) to Ethernet PAUSE frames. 815 * - High water mark should allow for at least two frames to be 816 * received after sending an XOFF. 817 * - Low water mark works best when it is very near the high water mark. 818 * This allows the receiver to restart by sending XON when it has 819 * drained a bit. Here we use an arbitrary value of 1500 which will 820 * restart after one full frame is pulled from the buffer. There 821 * could be several smaller frames in the buffer and if so they will 822 * not trigger the XON until their total number reduces the buffer 823 * by 1500. 824 * - The pause time is fairly large at 1000 x 512ns = 512 usec. 825 */ 826 const uint16_t rx_buffer_size = (pba & 0xffff) << 10; 827 828 hw->fc.high_water = rx_buffer_size - 829 roundup2(sc->hw.mac.max_frame_size, 1024); 830 /* 16-byte granularity */ 831 hw->fc.low_water = hw->fc.high_water - 16; 832 833 if (sc->fc) /* locally set flow control value? */ 834 hw->fc.requested_mode = sc->fc; 835 else 836 hw->fc.requested_mode = igc_fc_full; 837 838 hw->fc.pause_time = IGC_FC_PAUSE_TIME; 839 840 hw->fc.send_xon = true; 841 842 /* Issue a global reset */ 843 igc_reset_hw(hw); 844 IGC_WRITE_REG(hw, IGC_WUC, 0); 845 846 /* and a re-init */ 847 if (igc_init_hw(hw) < 0) { 848 aprint_error_dev(sc->sc_dev, "unable to reset hardware\n"); 849 return; 850 } 851 852 /* Setup DMA Coalescing */ 853 igc_init_dmac(sc, pba); 854 855 IGC_WRITE_REG(hw, IGC_VET, ETHERTYPE_VLAN); 856 igc_get_phy_info(hw); 857 igc_check_for_link(hw); 858 } 859 860 /********************************************************************* 861 * 862 * Initialize the DMA Coalescing feature 863 * 864 **********************************************************************/ 865 static void 866 igc_init_dmac(struct igc_softc *sc, uint32_t pba) 867 { 868 struct igc_hw *hw = &sc->hw; 869 const uint16_t max_frame_size = sc->hw.mac.max_frame_size; 870 uint32_t reg, status; 871 872 if (sc->dmac == 0) { /* Disabling it */ 873 reg = ~IGC_DMACR_DMAC_EN; /* XXXRO */ 874 IGC_WRITE_REG(hw, IGC_DMACR, reg); 875 DPRINTF(MISC, "DMA coalescing disabled\n"); 876 return; 877 } else { 878 device_printf(sc->sc_dev, "DMA coalescing enabled\n"); 879 } 880 881 /* Set starting threshold */ 882 IGC_WRITE_REG(hw, IGC_DMCTXTH, 0); 883 884 uint16_t hwm = 64 * pba - max_frame_size / 16; 885 if (hwm < 64 * (pba - 6)) 886 hwm = 64 * (pba - 6); 887 reg = IGC_READ_REG(hw, IGC_FCRTC); 888 reg &= ~IGC_FCRTC_RTH_COAL_MASK; 889 reg |= (hwm << IGC_FCRTC_RTH_COAL_SHIFT) & IGC_FCRTC_RTH_COAL_MASK; 890 IGC_WRITE_REG(hw, IGC_FCRTC, reg); 891 892 uint32_t dmac = pba - max_frame_size / 512; 893 if (dmac < pba - 10) 894 dmac = pba - 10; 895 reg = IGC_READ_REG(hw, IGC_DMACR); 896 reg &= ~IGC_DMACR_DMACTHR_MASK; 897 reg |= (dmac << IGC_DMACR_DMACTHR_SHIFT) & IGC_DMACR_DMACTHR_MASK; 898 899 /* transition to L0x or L1 if available..*/ 900 reg |= IGC_DMACR_DMAC_EN | IGC_DMACR_DMAC_LX_MASK; 901 902 /* Check if status is 2.5Gb backplane connection 903 * before configuration of watchdog timer, which is 904 * in msec values in 12.8usec intervals 905 * watchdog timer= msec values in 32usec intervals 906 * for non 2.5Gb connection 907 */ 908 status = IGC_READ_REG(hw, IGC_STATUS); 909 if ((status & IGC_STATUS_2P5_SKU) && 910 !(status & IGC_STATUS_2P5_SKU_OVER)) 911 reg |= (sc->dmac * 5) >> 6; 912 else 913 reg |= sc->dmac >> 5; 914 915 IGC_WRITE_REG(hw, IGC_DMACR, reg); 916 917 IGC_WRITE_REG(hw, IGC_DMCRTRH, 0); 918 919 /* Set the interval before transition */ 920 reg = IGC_READ_REG(hw, IGC_DMCTLX); 921 reg |= IGC_DMCTLX_DCFLUSH_DIS; 922 923 /* 924 * in 2.5Gb connection, TTLX unit is 0.4 usec 925 * which is 0x4*2 = 0xA. But delay is still 4 usec 926 */ 927 status = IGC_READ_REG(hw, IGC_STATUS); 928 if ((status & IGC_STATUS_2P5_SKU) && 929 !(status & IGC_STATUS_2P5_SKU_OVER)) 930 reg |= 0xA; 931 else 932 reg |= 0x4; 933 934 IGC_WRITE_REG(hw, IGC_DMCTLX, reg); 935 936 /* free space in tx packet buffer to wake from DMA coal */ 937 IGC_WRITE_REG(hw, IGC_DMCTXTH, 938 (IGC_TXPBSIZE - (2 * max_frame_size)) >> 6); 939 940 /* make low power state decision controlled by DMA coal */ 941 reg = IGC_READ_REG(hw, IGC_PCIEMISC); 942 reg &= ~IGC_PCIEMISC_LX_DECISION; 943 IGC_WRITE_REG(hw, IGC_PCIEMISC, reg); 944 } 945 946 static int 947 igc_setup_interrupts(struct igc_softc *sc) 948 { 949 int error; 950 951 switch (sc->sc_intr_type) { 952 case PCI_INTR_TYPE_MSIX: 953 error = igc_setup_msix(sc); 954 break; 955 case PCI_INTR_TYPE_MSI: 956 error = igc_setup_msi(sc); 957 break; 958 case PCI_INTR_TYPE_INTX: 959 error = igc_setup_intx(sc); 960 break; 961 default: 962 panic("%s: invalid interrupt type: %d", 963 device_xname(sc->sc_dev), sc->sc_intr_type); 964 } 965 966 return error; 967 } 968 969 static void 970 igc_attach_counters(struct igc_softc *sc) 971 { 972 #ifdef IGC_EVENT_COUNTERS 973 974 /* Global counters */ 975 sc->sc_global_evcnts = kmem_zalloc( 976 IGC_GLOBAL_COUNTERS * sizeof(sc->sc_global_evcnts[0]), KM_SLEEP); 977 978 for (int cnt = 0; cnt < IGC_GLOBAL_COUNTERS; cnt++) { 979 evcnt_attach_dynamic(&sc->sc_global_evcnts[cnt], 980 igc_global_counters[cnt].type, NULL, 981 device_xname(sc->sc_dev), igc_global_counters[cnt].name); 982 } 983 984 /* Driver counters */ 985 sc->sc_driver_evcnts = kmem_zalloc( 986 IGC_DRIVER_COUNTERS * sizeof(sc->sc_driver_evcnts[0]), KM_SLEEP); 987 988 for (int cnt = 0; cnt < IGC_DRIVER_COUNTERS; cnt++) { 989 evcnt_attach_dynamic(&sc->sc_driver_evcnts[cnt], 990 igc_driver_counters[cnt].type, NULL, 991 device_xname(sc->sc_dev), igc_driver_counters[cnt].name); 992 } 993 994 for (int iq = 0; iq < sc->sc_nqueues; iq++) { 995 struct igc_queue *q = &sc->queues[iq]; 996 997 q->igcq_driver_counters = kmem_zalloc( 998 IGC_DRIVER_COUNTERS * sizeof(q->igcq_driver_counters[0]), 999 KM_SLEEP); 1000 } 1001 1002 /* Queue counters */ 1003 for (int iq = 0; iq < sc->sc_nqueues; iq++) { 1004 struct igc_queue *q = &sc->queues[iq]; 1005 1006 snprintf(q->igcq_queue_evname, sizeof(q->igcq_queue_evname), 1007 "%s q%d", device_xname(sc->sc_dev), iq); 1008 1009 q->igcq_queue_evcnts = kmem_zalloc( 1010 IGC_QUEUE_COUNTERS * sizeof(q->igcq_queue_evcnts[0]), 1011 KM_SLEEP); 1012 1013 for (int cnt = 0; cnt < IGC_QUEUE_COUNTERS; cnt++) { 1014 evcnt_attach_dynamic(&q->igcq_queue_evcnts[cnt], 1015 igc_queue_counters[cnt].type, NULL, 1016 q->igcq_queue_evname, igc_queue_counters[cnt].name); 1017 } 1018 } 1019 1020 /* MAC counters */ 1021 snprintf(sc->sc_mac_evname, sizeof(sc->sc_mac_evname), 1022 "%s Mac Statistics", device_xname(sc->sc_dev)); 1023 1024 sc->sc_mac_evcnts = kmem_zalloc( 1025 IGC_MAC_COUNTERS * sizeof(sc->sc_mac_evcnts[0]), KM_SLEEP); 1026 1027 for (int cnt = 0; cnt < IGC_MAC_COUNTERS; cnt++) { 1028 evcnt_attach_dynamic(&sc->sc_mac_evcnts[cnt], EVCNT_TYPE_MISC, 1029 NULL, sc->sc_mac_evname, igc_mac_counters[cnt].name); 1030 } 1031 #endif 1032 } 1033 1034 static void 1035 igc_detach_counters(struct igc_softc *sc) 1036 { 1037 #ifdef IGC_EVENT_COUNTERS 1038 1039 /* Global counters */ 1040 for (int cnt = 0; cnt < IGC_GLOBAL_COUNTERS; cnt++) 1041 evcnt_detach(&sc->sc_global_evcnts[cnt]); 1042 1043 kmem_free(sc->sc_global_evcnts, 1044 IGC_GLOBAL_COUNTERS * sizeof(sc->sc_global_evcnts[0])); 1045 1046 /* Driver counters */ 1047 for (int iq = 0; iq < sc->sc_nqueues; iq++) { 1048 struct igc_queue *q = &sc->queues[iq]; 1049 1050 kmem_free(q->igcq_driver_counters, 1051 IGC_DRIVER_COUNTERS * sizeof(q->igcq_driver_counters[0])); 1052 } 1053 1054 for (int cnt = 0; cnt < IGC_DRIVER_COUNTERS; cnt++) 1055 evcnt_detach(&sc->sc_driver_evcnts[cnt]); 1056 1057 kmem_free(sc->sc_driver_evcnts, 1058 IGC_DRIVER_COUNTERS * sizeof(sc->sc_driver_evcnts[0])); 1059 1060 /* Queue counters */ 1061 for (int iq = 0; iq < sc->sc_nqueues; iq++) { 1062 struct igc_queue *q = &sc->queues[iq]; 1063 1064 for (int cnt = 0; cnt < IGC_QUEUE_COUNTERS; cnt++) 1065 evcnt_detach(&q->igcq_queue_evcnts[cnt]); 1066 1067 kmem_free(q->igcq_queue_evcnts, 1068 IGC_QUEUE_COUNTERS * sizeof(q->igcq_queue_evcnts[0])); 1069 } 1070 1071 /* MAC statistics */ 1072 for (int cnt = 0; cnt < IGC_MAC_COUNTERS; cnt++) 1073 evcnt_detach(&sc->sc_mac_evcnts[cnt]); 1074 1075 kmem_free(sc->sc_mac_evcnts, 1076 IGC_MAC_COUNTERS * sizeof(sc->sc_mac_evcnts[0])); 1077 #endif 1078 } 1079 1080 /* 1081 * XXX 1082 * FreeBSD uses 4-byte-wise read for 64-bit counters, while Linux just 1083 * drops hi words. 1084 */ 1085 static inline uint64_t __unused 1086 igc_read_mac_counter(struct igc_hw *hw, bus_size_t reg, bool is64) 1087 { 1088 uint64_t val; 1089 1090 val = IGC_READ_REG(hw, reg); 1091 if (is64) 1092 val += ((uint64_t)IGC_READ_REG(hw, reg + 4)) << 32; 1093 return val; 1094 } 1095 1096 static void 1097 igc_update_counters(struct igc_softc *sc) 1098 { 1099 #ifdef IGC_EVENT_COUNTERS 1100 1101 /* Global counters: nop */ 1102 1103 /* Driver counters */ 1104 uint64_t sum[IGC_DRIVER_COUNTERS]; 1105 1106 memset(sum, 0, sizeof(sum)); 1107 for (int iq = 0; iq < sc->sc_nqueues; iq++) { 1108 struct igc_queue *q = &sc->queues[iq]; 1109 1110 for (int cnt = 0; cnt < IGC_DRIVER_COUNTERS; cnt++) { 1111 sum[cnt] += IGC_QUEUE_DRIVER_COUNTER_VAL(q, cnt); 1112 IGC_QUEUE_DRIVER_COUNTER_STORE(q, cnt, 0); 1113 } 1114 } 1115 1116 for (int cnt = 0; cnt < IGC_DRIVER_COUNTERS; cnt++) 1117 IGC_DRIVER_COUNTER_ADD(sc, cnt, sum[cnt]); 1118 1119 /* Queue counters: nop */ 1120 1121 /* Mac statistics */ 1122 struct igc_hw *hw = &sc->hw; 1123 struct ifnet *ifp = &sc->sc_ec.ec_if; 1124 uint64_t iqdrops = 0; 1125 1126 for (int cnt = 0; cnt < IGC_MAC_COUNTERS; cnt++) { 1127 uint64_t val; 1128 bus_size_t regaddr = igc_mac_counters[cnt].reg; 1129 1130 val = igc_read_mac_counter(hw, regaddr, 1131 igc_mac_counters[cnt].is64); 1132 IGC_MAC_COUNTER_ADD(sc, cnt, val); 1133 /* XXX Count MPC to iqdrops. */ 1134 if (regaddr == IGC_MPC) 1135 iqdrops += val; 1136 } 1137 1138 for (int iq = 0; iq < sc->sc_nqueues; iq++) { 1139 uint32_t val; 1140 1141 /* XXX RQDPC should be visible via evcnt(9). */ 1142 val = IGC_READ_REG(hw, IGC_RQDPC(iq)); 1143 1144 /* RQDPC is not cleard on read. */ 1145 if (val != 0) 1146 IGC_WRITE_REG(hw, IGC_RQDPC(iq), 0); 1147 iqdrops += val; 1148 } 1149 1150 if (iqdrops != 0) 1151 if_statadd(ifp, if_iqdrops, iqdrops); 1152 #endif 1153 } 1154 1155 static void 1156 igc_clear_counters(struct igc_softc *sc) 1157 { 1158 #ifdef IGC_EVENT_COUNTERS 1159 1160 /* Global counters */ 1161 for (int cnt = 0; cnt < IGC_GLOBAL_COUNTERS; cnt++) 1162 IGC_GLOBAL_COUNTER_STORE(sc, cnt, 0); 1163 1164 /* Driver counters */ 1165 for (int iq = 0; iq < sc->sc_nqueues; iq++) { 1166 struct igc_queue *q = &sc->queues[iq]; 1167 1168 for (int cnt = 0; cnt < IGC_DRIVER_COUNTERS; cnt++) 1169 IGC_QUEUE_DRIVER_COUNTER_STORE(q, cnt, 0); 1170 } 1171 1172 for (int cnt = 0; cnt < IGC_DRIVER_COUNTERS; cnt++) 1173 IGC_DRIVER_COUNTER_STORE(sc, cnt, 0); 1174 1175 /* Queue counters */ 1176 for (int iq = 0; iq < sc->sc_nqueues; iq++) { 1177 struct igc_queue *q = &sc->queues[iq]; 1178 1179 for (int cnt = 0; cnt < IGC_QUEUE_COUNTERS; cnt++) 1180 IGC_QUEUE_COUNTER_STORE(q, cnt, 0); 1181 } 1182 1183 /* Mac statistics */ 1184 struct igc_hw *hw = &sc->hw; 1185 1186 for (int cnt = 0; cnt < IGC_MAC_COUNTERS; cnt++) { 1187 (void)igc_read_mac_counter(hw, igc_mac_counters[cnt].reg, 1188 igc_mac_counters[cnt].is64); 1189 IGC_MAC_COUNTER_STORE(sc, cnt, 0); 1190 } 1191 #endif 1192 } 1193 1194 static int 1195 igc_setup_msix(struct igc_softc *sc) 1196 { 1197 pci_chipset_tag_t pc = sc->osdep.os_pa.pa_pc; 1198 device_t dev = sc->sc_dev; 1199 pci_intr_handle_t *intrs; 1200 void **ihs; 1201 const char *intrstr; 1202 char intrbuf[PCI_INTRSTR_LEN]; 1203 char xnamebuf[MAX(32, MAXCOMLEN)]; 1204 int iq, error; 1205 1206 for (iq = 0, intrs = sc->sc_intrs, ihs = sc->sc_ihs; 1207 iq < sc->sc_nqueues; iq++, intrs++, ihs++) { 1208 struct igc_queue *q = &sc->queues[iq]; 1209 1210 snprintf(xnamebuf, sizeof(xnamebuf), "%s: txrx %d", 1211 device_xname(dev), iq); 1212 1213 intrstr = pci_intr_string(pc, *intrs, intrbuf, sizeof(intrbuf)); 1214 1215 pci_intr_setattr(pc, intrs, PCI_INTR_MPSAFE, true); 1216 *ihs = pci_intr_establish_xname(pc, *intrs, IPL_NET, 1217 igc_intr_queue, q, xnamebuf); 1218 if (*ihs == NULL) { 1219 aprint_error_dev(dev, 1220 "unable to establish txrx interrupt at %s\n", 1221 intrstr); 1222 return ENOBUFS; 1223 } 1224 aprint_normal_dev(dev, "txrx interrupting at %s\n", intrstr); 1225 1226 kcpuset_t *affinity; 1227 kcpuset_create(&affinity, true); 1228 kcpuset_set(affinity, iq % ncpu); 1229 error = interrupt_distribute(*ihs, affinity, NULL); 1230 if (error) { 1231 aprint_normal_dev(dev, 1232 "%s: unable to change affinity, use default CPU\n", 1233 intrstr); 1234 } 1235 kcpuset_destroy(affinity); 1236 1237 q->igcq_si = softint_establish(SOFTINT_NET | SOFTINT_MPSAFE, 1238 igc_handle_queue, q); 1239 if (q->igcq_si == NULL) { 1240 aprint_error_dev(dev, 1241 "%s: unable to establish softint\n", intrstr); 1242 return ENOBUFS; 1243 } 1244 1245 q->msix = iq; 1246 q->eims = 1 << iq; 1247 } 1248 1249 snprintf(xnamebuf, MAXCOMLEN, "%s_tx_rx", device_xname(dev)); 1250 error = workqueue_create(&sc->sc_queue_wq, xnamebuf, 1251 igc_handle_queue_work, sc, IGC_WORKQUEUE_PRI, IPL_NET, 1252 WQ_PERCPU | WQ_MPSAFE); 1253 if (error) { 1254 aprint_error_dev(dev, "workqueue_create failed\n"); 1255 return ENOBUFS; 1256 } 1257 sc->sc_txrx_workqueue = false; 1258 1259 intrstr = pci_intr_string(pc, *intrs, intrbuf, sizeof(intrbuf)); 1260 snprintf(xnamebuf, sizeof(xnamebuf), "%s: link", device_xname(dev)); 1261 pci_intr_setattr(pc, intrs, PCI_INTR_MPSAFE, true); 1262 *ihs = pci_intr_establish_xname(pc, *intrs, IPL_NET, 1263 igc_intr_link, sc, xnamebuf); 1264 if (*ihs == NULL) { 1265 aprint_error_dev(dev, 1266 "unable to establish link interrupt at %s\n", intrstr); 1267 return ENOBUFS; 1268 } 1269 aprint_normal_dev(dev, "link interrupting at %s\n", intrstr); 1270 /* use later in igc_configure_queues() */ 1271 sc->linkvec = iq; 1272 1273 return 0; 1274 } 1275 1276 static int 1277 igc_setup_msi(struct igc_softc *sc) 1278 { 1279 pci_chipset_tag_t pc = sc->osdep.os_pa.pa_pc; 1280 device_t dev = sc->sc_dev; 1281 pci_intr_handle_t *intr = sc->sc_intrs; 1282 void **ihs = sc->sc_ihs; 1283 const char *intrstr; 1284 char intrbuf[PCI_INTRSTR_LEN]; 1285 char xnamebuf[MAX(32, MAXCOMLEN)]; 1286 int error; 1287 1288 intrstr = pci_intr_string(pc, *intr, intrbuf, sizeof(intrbuf)); 1289 1290 snprintf(xnamebuf, sizeof(xnamebuf), "%s: msi", device_xname(dev)); 1291 pci_intr_setattr(pc, intr, PCI_INTR_MPSAFE, true); 1292 *ihs = pci_intr_establish_xname(pc, *intr, IPL_NET, 1293 igc_intr, sc, xnamebuf); 1294 if (*ihs == NULL) { 1295 aprint_error_dev(dev, 1296 "unable to establish interrupt at %s\n", intrstr); 1297 return ENOBUFS; 1298 } 1299 aprint_normal_dev(dev, "interrupting at %s\n", intrstr); 1300 1301 struct igc_queue *iq = sc->queues; 1302 iq->igcq_si = softint_establish(SOFTINT_NET | SOFTINT_MPSAFE, 1303 igc_handle_queue, iq); 1304 if (iq->igcq_si == NULL) { 1305 aprint_error_dev(dev, 1306 "%s: unable to establish softint\n", intrstr); 1307 return ENOBUFS; 1308 } 1309 1310 snprintf(xnamebuf, MAXCOMLEN, "%s_tx_rx", device_xname(dev)); 1311 error = workqueue_create(&sc->sc_queue_wq, xnamebuf, 1312 igc_handle_queue_work, sc, IGC_WORKQUEUE_PRI, IPL_NET, 1313 WQ_PERCPU | WQ_MPSAFE); 1314 if (error) { 1315 aprint_error_dev(dev, "workqueue_create failed\n"); 1316 return ENOBUFS; 1317 } 1318 sc->sc_txrx_workqueue = false; 1319 1320 sc->queues[0].msix = 0; 1321 sc->linkvec = 0; 1322 1323 return 0; 1324 } 1325 1326 static int 1327 igc_setup_intx(struct igc_softc *sc) 1328 { 1329 pci_chipset_tag_t pc = sc->osdep.os_pa.pa_pc; 1330 device_t dev = sc->sc_dev; 1331 pci_intr_handle_t *intr = sc->sc_intrs; 1332 void **ihs = sc->sc_ihs; 1333 const char *intrstr; 1334 char intrbuf[PCI_INTRSTR_LEN]; 1335 char xnamebuf[32]; 1336 1337 intrstr = pci_intr_string(pc, *intr, intrbuf, sizeof(intrbuf)); 1338 1339 snprintf(xnamebuf, sizeof(xnamebuf), "%s:intx", device_xname(dev)); 1340 pci_intr_setattr(pc, intr, PCI_INTR_MPSAFE, true); 1341 *ihs = pci_intr_establish_xname(pc, *intr, IPL_NET, 1342 igc_intr, sc, xnamebuf); 1343 if (*ihs == NULL) { 1344 aprint_error_dev(dev, 1345 "unable to establish interrupt at %s\n", intrstr); 1346 return ENOBUFS; 1347 } 1348 aprint_normal_dev(dev, "interrupting at %s\n", intrstr); 1349 1350 struct igc_queue *iq = sc->queues; 1351 iq->igcq_si = softint_establish(SOFTINT_NET | SOFTINT_MPSAFE, 1352 igc_handle_queue, iq); 1353 if (iq->igcq_si == NULL) { 1354 aprint_error_dev(dev, 1355 "%s: unable to establish softint\n", intrstr); 1356 return ENOBUFS; 1357 } 1358 1359 /* create workqueue? */ 1360 sc->sc_txrx_workqueue = false; 1361 1362 sc->queues[0].msix = 0; 1363 sc->linkvec = 0; 1364 1365 return 0; 1366 } 1367 1368 static int 1369 igc_dma_malloc(struct igc_softc *sc, bus_size_t size, struct igc_dma_alloc *dma) 1370 { 1371 struct igc_osdep *os = &sc->osdep; 1372 1373 dma->dma_tag = os->os_dmat; 1374 1375 if (bus_dmamap_create(dma->dma_tag, size, 1, size, 0, 1376 BUS_DMA_WAITOK | BUS_DMA_ALLOCNOW, &dma->dma_map)) 1377 return 1; 1378 if (bus_dmamem_alloc(dma->dma_tag, size, PAGE_SIZE, 0, &dma->dma_seg, 1379 1, &dma->dma_nseg, BUS_DMA_WAITOK)) 1380 goto destroy; 1381 /* 1382 * XXXRO 1383 * 1384 * Coherent mapping for descriptors is required for now. 1385 * 1386 * Both TX and RX descriptors are 16-byte length, which is shorter 1387 * than dcache lines on modern CPUs. Therefore, sync for a descriptor 1388 * may overwrite DMA read for descriptors in the same cache line. 1389 * 1390 * Can't we avoid this by use cache-line-aligned descriptors at once? 1391 */ 1392 if (bus_dmamem_map(dma->dma_tag, &dma->dma_seg, dma->dma_nseg, size, 1393 &dma->dma_vaddr, BUS_DMA_WAITOK | BUS_DMA_COHERENT /* XXXRO */)) 1394 goto free; 1395 if (bus_dmamap_load(dma->dma_tag, dma->dma_map, dma->dma_vaddr, size, 1396 NULL, BUS_DMA_WAITOK)) 1397 goto unmap; 1398 1399 dma->dma_size = size; 1400 1401 return 0; 1402 unmap: 1403 bus_dmamem_unmap(dma->dma_tag, dma->dma_vaddr, size); 1404 free: 1405 bus_dmamem_free(dma->dma_tag, &dma->dma_seg, dma->dma_nseg); 1406 destroy: 1407 bus_dmamap_destroy(dma->dma_tag, dma->dma_map); 1408 dma->dma_map = NULL; 1409 dma->dma_tag = NULL; 1410 return 1; 1411 } 1412 1413 static void 1414 igc_dma_free(struct igc_softc *sc, struct igc_dma_alloc *dma) 1415 { 1416 1417 if (dma->dma_tag == NULL) 1418 return; 1419 1420 if (dma->dma_map != NULL) { 1421 bus_dmamap_sync(dma->dma_tag, dma->dma_map, 0, 1422 dma->dma_map->dm_mapsize, 1423 BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE); 1424 bus_dmamap_unload(dma->dma_tag, dma->dma_map); 1425 bus_dmamem_unmap(dma->dma_tag, dma->dma_vaddr, dma->dma_size); 1426 bus_dmamem_free(dma->dma_tag, &dma->dma_seg, dma->dma_nseg); 1427 bus_dmamap_destroy(dma->dma_tag, dma->dma_map); 1428 dma->dma_map = NULL; 1429 } 1430 } 1431 1432 /********************************************************************* 1433 * 1434 * Setup networking device structure and register an interface. 1435 * 1436 **********************************************************************/ 1437 static void 1438 igc_setup_interface(struct igc_softc *sc) 1439 { 1440 struct ifnet *ifp = &sc->sc_ec.ec_if; 1441 1442 strlcpy(ifp->if_xname, device_xname(sc->sc_dev), sizeof(ifp->if_xname)); 1443 ifp->if_softc = sc; 1444 ifp->if_flags = IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST; 1445 ifp->if_extflags = IFEF_MPSAFE; 1446 ifp->if_ioctl = igc_ioctl; 1447 ifp->if_start = igc_start; 1448 if (sc->sc_nqueues > 1) 1449 ifp->if_transmit = igc_transmit; 1450 ifp->if_watchdog = igc_watchdog; 1451 ifp->if_init = igc_init; 1452 ifp->if_stop = igc_stop; 1453 1454 ifp->if_capabilities = IFCAP_TSOv4 | IFCAP_TSOv6; 1455 1456 ifp->if_capabilities |= 1457 IFCAP_CSUM_IPv4_Tx | IFCAP_CSUM_IPv4_Rx | 1458 IFCAP_CSUM_TCPv4_Tx | IFCAP_CSUM_TCPv4_Rx | 1459 IFCAP_CSUM_UDPv4_Tx | IFCAP_CSUM_UDPv4_Rx | 1460 IFCAP_CSUM_TCPv6_Tx | IFCAP_CSUM_TCPv6_Rx | 1461 IFCAP_CSUM_UDPv6_Tx | IFCAP_CSUM_UDPv6_Rx; 1462 1463 ifp->if_capenable = 0; 1464 1465 sc->sc_ec.ec_capabilities |= 1466 ETHERCAP_JUMBO_MTU | ETHERCAP_VLAN_MTU; 1467 1468 IFQ_SET_MAXLEN(&ifp->if_snd, sc->num_tx_desc - 1); 1469 IFQ_SET_READY(&ifp->if_snd); 1470 1471 #if NVLAN > 0 1472 sc->sc_ec.ec_capabilities |= ETHERCAP_VLAN_HWTAGGING; 1473 #endif 1474 1475 mutex_init(&sc->sc_core_lock, MUTEX_DEFAULT, IPL_NET); 1476 1477 /* Initialize ifmedia structures. */ 1478 sc->sc_ec.ec_ifmedia = &sc->media; 1479 ifmedia_init_with_lock(&sc->media, IFM_IMASK, igc_media_change, 1480 igc_media_status, &sc->sc_core_lock); 1481 ifmedia_add(&sc->media, IFM_ETHER | IFM_10_T, 0, NULL); 1482 ifmedia_add(&sc->media, IFM_ETHER | IFM_10_T | IFM_FDX, 0, NULL); 1483 ifmedia_add(&sc->media, IFM_ETHER | IFM_100_TX, 0, NULL); 1484 ifmedia_add(&sc->media, IFM_ETHER | IFM_100_TX | IFM_FDX, 0, NULL); 1485 ifmedia_add(&sc->media, IFM_ETHER | IFM_1000_T | IFM_FDX, 0, NULL); 1486 ifmedia_add(&sc->media, IFM_ETHER | IFM_2500_T | IFM_FDX, 0, NULL); 1487 ifmedia_add(&sc->media, IFM_ETHER | IFM_AUTO, 0, NULL); 1488 ifmedia_set(&sc->media, IFM_ETHER | IFM_AUTO); 1489 1490 sc->sc_rx_intr_process_limit = IGC_RX_INTR_PROCESS_LIMIT_DEFAULT; 1491 sc->sc_tx_intr_process_limit = IGC_TX_INTR_PROCESS_LIMIT_DEFAULT; 1492 sc->sc_rx_process_limit = IGC_RX_PROCESS_LIMIT_DEFAULT; 1493 sc->sc_tx_process_limit = IGC_TX_PROCESS_LIMIT_DEFAULT; 1494 1495 if_initialize(ifp); 1496 sc->sc_ipq = if_percpuq_create(ifp); 1497 if_deferred_start_init(ifp, NULL); 1498 ether_ifattach(ifp, sc->hw.mac.addr); 1499 ether_set_ifflags_cb(&sc->sc_ec, igc_ifflags_cb); 1500 if_register(ifp); 1501 } 1502 1503 static int 1504 igc_init(struct ifnet *ifp) 1505 { 1506 struct igc_softc *sc = ifp->if_softc; 1507 int error; 1508 1509 mutex_enter(&sc->sc_core_lock); 1510 error = igc_init_locked(sc); 1511 mutex_exit(&sc->sc_core_lock); 1512 1513 return error; 1514 } 1515 1516 static int 1517 igc_init_locked(struct igc_softc *sc) 1518 { 1519 struct ethercom *ec = &sc->sc_ec; 1520 struct ifnet *ifp = &ec->ec_if; 1521 1522 DPRINTF(CFG, "called\n"); 1523 1524 KASSERT(mutex_owned(&sc->sc_core_lock)); 1525 1526 if (ISSET(ifp->if_flags, IFF_RUNNING)) 1527 igc_stop_locked(sc); 1528 1529 /* Put the address into the receive address array. */ 1530 igc_rar_set(&sc->hw, sc->hw.mac.addr, 0); 1531 1532 /* Initialize the hardware. */ 1533 igc_reset(sc); 1534 igc_update_link_status(sc); 1535 1536 /* Setup VLAN support, basic and offload if available. */ 1537 IGC_WRITE_REG(&sc->hw, IGC_VET, ETHERTYPE_VLAN); 1538 1539 igc_initialize_transmit_unit(sc); 1540 igc_initialize_receive_unit(sc); 1541 1542 if (ec->ec_capenable & ETHERCAP_VLAN_HWTAGGING) { 1543 uint32_t ctrl = IGC_READ_REG(&sc->hw, IGC_CTRL); 1544 ctrl |= IGC_CTRL_VME; 1545 IGC_WRITE_REG(&sc->hw, IGC_CTRL, ctrl); 1546 } 1547 1548 /* Setup multicast table. */ 1549 igc_set_filter(sc); 1550 1551 igc_clear_hw_cntrs_base_generic(&sc->hw); 1552 1553 if (sc->sc_intr_type == PCI_INTR_TYPE_MSIX) 1554 igc_configure_queues(sc); 1555 1556 /* This clears any pending interrupts */ 1557 IGC_READ_REG(&sc->hw, IGC_ICR); 1558 IGC_WRITE_REG(&sc->hw, IGC_ICS, IGC_ICS_LSC); 1559 1560 /* The driver can now take control from firmware. */ 1561 igc_get_hw_control(sc); 1562 1563 /* Set Energy Efficient Ethernet. */ 1564 igc_set_eee_i225(&sc->hw, true, true, true); 1565 1566 for (int iq = 0; iq < sc->sc_nqueues; iq++) { 1567 struct rx_ring *rxr = &sc->rx_rings[iq]; 1568 1569 mutex_enter(&rxr->rxr_lock); 1570 igc_rxfill(rxr); 1571 mutex_exit(&rxr->rxr_lock); 1572 } 1573 1574 sc->sc_core_stopping = false; 1575 1576 ifp->if_flags |= IFF_RUNNING; 1577 1578 /* Save last flags for the callback */ 1579 sc->sc_if_flags = ifp->if_flags; 1580 1581 callout_schedule(&sc->sc_tick_ch, hz); 1582 1583 igc_enable_intr(sc); 1584 1585 return 0; 1586 } 1587 1588 static inline int 1589 igc_load_mbuf(struct igc_queue *q, bus_dma_tag_t dmat, bus_dmamap_t map, 1590 struct mbuf *m) 1591 { 1592 int error; 1593 1594 error = bus_dmamap_load_mbuf(dmat, map, m, 1595 BUS_DMA_WRITE | BUS_DMA_NOWAIT); 1596 1597 if (__predict_false(error == EFBIG)) { 1598 IGC_DRIVER_EVENT(q, txdma_efbig, 1); 1599 m = m_defrag(m, M_NOWAIT); 1600 if (__predict_false(m == NULL)) { 1601 IGC_DRIVER_EVENT(q, txdma_defrag, 1); 1602 return ENOBUFS; 1603 } 1604 error = bus_dmamap_load_mbuf(dmat, map, m, 1605 BUS_DMA_WRITE | BUS_DMA_NOWAIT); 1606 } 1607 1608 switch (error) { 1609 case 0: 1610 break; 1611 case ENOMEM: 1612 IGC_DRIVER_EVENT(q, txdma_enomem, 1); 1613 break; 1614 case EINVAL: 1615 IGC_DRIVER_EVENT(q, txdma_einval, 1); 1616 break; 1617 case EAGAIN: 1618 IGC_DRIVER_EVENT(q, txdma_eagain, 1); 1619 break; 1620 default: 1621 IGC_DRIVER_EVENT(q, txdma_other, 1); 1622 break; 1623 } 1624 1625 return error; 1626 } 1627 1628 #define IGC_TX_START 1 1629 #define IGC_TX_TRANSMIT 2 1630 1631 static void 1632 igc_start(struct ifnet *ifp) 1633 { 1634 struct igc_softc *sc = ifp->if_softc; 1635 1636 if (__predict_false(!sc->link_active)) { 1637 IFQ_PURGE(&ifp->if_snd); 1638 return; 1639 } 1640 1641 struct tx_ring *txr = &sc->tx_rings[0]; /* queue 0 */ 1642 mutex_enter(&txr->txr_lock); 1643 igc_tx_common_locked(ifp, txr, IGC_TX_START); 1644 mutex_exit(&txr->txr_lock); 1645 } 1646 1647 static inline u_int 1648 igc_select_txqueue(struct igc_softc *sc, struct mbuf *m __unused) 1649 { 1650 const u_int cpuid = cpu_index(curcpu()); 1651 1652 return cpuid % sc->sc_nqueues; 1653 } 1654 1655 static int 1656 igc_transmit(struct ifnet *ifp, struct mbuf *m) 1657 { 1658 struct igc_softc *sc = ifp->if_softc; 1659 const u_int qid = igc_select_txqueue(sc, m); 1660 struct tx_ring *txr = &sc->tx_rings[qid]; 1661 struct igc_queue *q = txr->txr_igcq; 1662 1663 if (__predict_false(!pcq_put(txr->txr_interq, m))) { 1664 IGC_QUEUE_EVENT(q, tx_pcq_drop, 1); 1665 m_freem(m); 1666 return ENOBUFS; 1667 } 1668 1669 mutex_enter(&txr->txr_lock); 1670 igc_tx_common_locked(ifp, txr, IGC_TX_TRANSMIT); 1671 mutex_exit(&txr->txr_lock); 1672 1673 return 0; 1674 } 1675 1676 static void 1677 igc_tx_common_locked(struct ifnet *ifp, struct tx_ring *txr, int caller) 1678 { 1679 struct igc_softc *sc = ifp->if_softc; 1680 struct igc_queue *q = txr->txr_igcq; 1681 net_stat_ref_t nsr = IF_STAT_GETREF(ifp); 1682 int prod, free, last = -1; 1683 bool post = false; 1684 1685 prod = txr->next_avail_desc; 1686 free = txr->next_to_clean; 1687 if (free <= prod) 1688 free += sc->num_tx_desc; 1689 free -= prod; 1690 1691 DPRINTF(TX, "%s: begin: msix %d prod %d n2c %d free %d\n", 1692 caller == IGC_TX_TRANSMIT ? "transmit" : "start", 1693 txr->me, prod, txr->next_to_clean, free); 1694 1695 for (;;) { 1696 struct mbuf *m; 1697 1698 if (__predict_false(free <= IGC_MAX_SCATTER)) { 1699 IGC_QUEUE_EVENT(q, tx_no_desc, 1); 1700 break; 1701 } 1702 1703 if (caller == IGC_TX_TRANSMIT) 1704 m = pcq_get(txr->txr_interq); 1705 else 1706 IFQ_DEQUEUE(&ifp->if_snd, m); 1707 if (__predict_false(m == NULL)) 1708 break; 1709 1710 struct igc_tx_buf *txbuf = &txr->tx_buffers[prod]; 1711 bus_dmamap_t map = txbuf->map; 1712 1713 if (__predict_false( 1714 igc_load_mbuf(q, txr->txdma.dma_tag, map, m))) { 1715 if (caller == IGC_TX_TRANSMIT) 1716 IGC_QUEUE_EVENT(q, tx_pcq_drop, 1); 1717 m_freem(m); 1718 if_statinc_ref(ifp, nsr, if_oerrors); 1719 continue; 1720 } 1721 1722 uint32_t ctx_cmd_type_len = 0, olinfo_status = 0; 1723 if (igc_tx_ctx_setup(txr, m, prod, &ctx_cmd_type_len, 1724 &olinfo_status)) { 1725 IGC_QUEUE_EVENT(q, tx_ctx, 1); 1726 /* Consume the first descriptor */ 1727 prod = igc_txdesc_incr(sc, prod); 1728 free--; 1729 } 1730 1731 bus_dmamap_sync(txr->txdma.dma_tag, map, 0, 1732 map->dm_mapsize, BUS_DMASYNC_PREWRITE); 1733 1734 for (int i = 0; i < map->dm_nsegs; i++) { 1735 union igc_adv_tx_desc *txdesc = &txr->tx_base[prod]; 1736 1737 uint32_t cmd_type_len = ctx_cmd_type_len | 1738 IGC_ADVTXD_DCMD_IFCS | IGC_ADVTXD_DTYP_DATA | 1739 IGC_ADVTXD_DCMD_DEXT | map->dm_segs[i].ds_len; 1740 if (i == map->dm_nsegs - 1) { 1741 cmd_type_len |= 1742 IGC_ADVTXD_DCMD_EOP | IGC_ADVTXD_DCMD_RS; 1743 } 1744 1745 igc_txdesc_sync(txr, prod, 1746 BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE); 1747 htolem64(&txdesc->read.buffer_addr, 1748 map->dm_segs[i].ds_addr); 1749 htolem32(&txdesc->read.cmd_type_len, cmd_type_len); 1750 htolem32(&txdesc->read.olinfo_status, olinfo_status); 1751 igc_txdesc_sync(txr, prod, 1752 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); 1753 1754 last = prod; 1755 prod = igc_txdesc_incr(sc, prod); 1756 } 1757 1758 txbuf->m_head = m; 1759 txbuf->eop_index = last; 1760 1761 bpf_mtap(ifp, m, BPF_D_OUT); 1762 1763 if_statadd_ref(ifp, nsr, if_obytes, m->m_pkthdr.len); 1764 if (m->m_flags & M_MCAST) 1765 if_statinc_ref(ifp, nsr, if_omcasts); 1766 IGC_QUEUE_EVENT(q, tx_packets, 1); 1767 IGC_QUEUE_EVENT(q, tx_bytes, m->m_pkthdr.len); 1768 1769 free -= map->dm_nsegs; 1770 post = true; 1771 } 1772 1773 if (post) { 1774 txr->next_avail_desc = prod; 1775 IGC_WRITE_REG(&sc->hw, IGC_TDT(txr->me), prod); 1776 } 1777 1778 DPRINTF(TX, "%s: done : msix %d prod %d n2c %d free %d\n", 1779 caller == IGC_TX_TRANSMIT ? "transmit" : "start", 1780 txr->me, prod, txr->next_to_clean, free); 1781 1782 IF_STAT_PUTREF(ifp); 1783 } 1784 1785 static bool 1786 igc_txeof(struct tx_ring *txr, u_int limit) 1787 { 1788 struct igc_softc *sc = txr->sc; 1789 struct ifnet *ifp = &sc->sc_ec.ec_if; 1790 int cons, prod; 1791 bool more = false; 1792 1793 prod = txr->next_avail_desc; 1794 cons = txr->next_to_clean; 1795 1796 if (cons == prod) { 1797 DPRINTF(TX, "false: msix %d cons %d prod %d\n", 1798 txr->me, cons, prod); 1799 return false; 1800 } 1801 1802 do { 1803 struct igc_tx_buf *txbuf = &txr->tx_buffers[cons]; 1804 const int last = txbuf->eop_index; 1805 1806 membar_consumer(); /* XXXRO necessary? */ 1807 1808 KASSERT(last != -1); 1809 union igc_adv_tx_desc *txdesc = &txr->tx_base[last]; 1810 igc_txdesc_sync(txr, last, BUS_DMASYNC_POSTREAD); 1811 const uint32_t status = le32toh(txdesc->wb.status); 1812 igc_txdesc_sync(txr, last, BUS_DMASYNC_PREREAD); 1813 1814 if (!(status & IGC_TXD_STAT_DD)) 1815 break; 1816 1817 if (limit-- == 0) { 1818 more = true; 1819 DPRINTF(TX, "pending TX " 1820 "msix %d cons %d last %d prod %d " 1821 "status 0x%08x\n", 1822 txr->me, cons, last, prod, status); 1823 break; 1824 } 1825 1826 DPRINTF(TX, "handled TX " 1827 "msix %d cons %d last %d prod %d " 1828 "status 0x%08x\n", 1829 txr->me, cons, last, prod, status); 1830 1831 if_statinc(ifp, if_opackets); 1832 1833 bus_dmamap_t map = txbuf->map; 1834 bus_dmamap_sync(txr->txdma.dma_tag, map, 0, map->dm_mapsize, 1835 BUS_DMASYNC_POSTWRITE); 1836 bus_dmamap_unload(txr->txdma.dma_tag, map); 1837 m_freem(txbuf->m_head); 1838 1839 txbuf->m_head = NULL; 1840 txbuf->eop_index = -1; 1841 1842 cons = igc_txdesc_incr(sc, last); 1843 } while (cons != prod); 1844 1845 txr->next_to_clean = cons; 1846 1847 return more; 1848 } 1849 1850 static void 1851 igc_intr_barrier(struct igc_softc *sc __unused) 1852 { 1853 1854 xc_barrier(0); 1855 } 1856 1857 static void 1858 igc_stop(struct ifnet *ifp, int disable) 1859 { 1860 struct igc_softc *sc = ifp->if_softc; 1861 1862 mutex_enter(&sc->sc_core_lock); 1863 igc_stop_locked(sc); 1864 mutex_exit(&sc->sc_core_lock); 1865 } 1866 1867 /********************************************************************* 1868 * 1869 * This routine disables all traffic on the adapter by issuing a 1870 * global reset on the MAC. 1871 * 1872 **********************************************************************/ 1873 static void 1874 igc_stop_locked(struct igc_softc *sc) 1875 { 1876 struct ifnet *ifp = &sc->sc_ec.ec_if; 1877 1878 DPRINTF(CFG, "called\n"); 1879 1880 KASSERT(mutex_owned(&sc->sc_core_lock)); 1881 1882 /* 1883 * If stopping processing has already started, do nothing. 1884 */ 1885 if ((ifp->if_flags & IFF_RUNNING) == 0) 1886 return; 1887 1888 /* Tell the stack that the interface is no longer active. */ 1889 ifp->if_flags &= ~IFF_RUNNING; 1890 1891 /* 1892 * igc_handle_queue() can enable interrupts, so wait for completion of 1893 * last igc_handle_queue() after unset IFF_RUNNING. 1894 */ 1895 mutex_exit(&sc->sc_core_lock); 1896 igc_barrier_handle_queue(sc); 1897 mutex_enter(&sc->sc_core_lock); 1898 1899 sc->sc_core_stopping = true; 1900 1901 igc_disable_intr(sc); 1902 1903 callout_halt(&sc->sc_tick_ch, &sc->sc_core_lock); 1904 1905 igc_reset_hw(&sc->hw); 1906 IGC_WRITE_REG(&sc->hw, IGC_WUC, 0); 1907 1908 /* 1909 * Wait for completion of interrupt handlers. 1910 */ 1911 mutex_exit(&sc->sc_core_lock); 1912 igc_intr_barrier(sc); 1913 mutex_enter(&sc->sc_core_lock); 1914 1915 igc_update_link_status(sc); 1916 1917 for (int iq = 0; iq < sc->sc_nqueues; iq++) { 1918 struct tx_ring *txr = &sc->tx_rings[iq]; 1919 1920 igc_withdraw_transmit_packets(txr, false); 1921 } 1922 1923 for (int iq = 0; iq < sc->sc_nqueues; iq++) { 1924 struct rx_ring *rxr = &sc->rx_rings[iq]; 1925 1926 callout_halt(&rxr->rx_refill, NULL); 1927 igc_clear_receive_status(rxr); 1928 } 1929 1930 /* Save last flags for the callback */ 1931 sc->sc_if_flags = ifp->if_flags; 1932 } 1933 1934 /********************************************************************* 1935 * Ioctl entry point 1936 * 1937 * igc_ioctl is called when the user wants to configure the 1938 * interface. 1939 * 1940 * return 0 on success, positive on failure 1941 **********************************************************************/ 1942 static int 1943 igc_ioctl(struct ifnet * ifp, u_long cmd, void *data) 1944 { 1945 struct igc_softc *sc __unused = ifp->if_softc; 1946 int s; 1947 int error; 1948 1949 DPRINTF(CFG, "cmd 0x%016lx\n", cmd); 1950 1951 switch (cmd) { 1952 case SIOCADDMULTI: 1953 case SIOCDELMULTI: 1954 break; 1955 default: 1956 KASSERT(IFNET_LOCKED(ifp)); 1957 } 1958 1959 if (cmd == SIOCZIFDATA) { 1960 mutex_enter(&sc->sc_core_lock); 1961 igc_clear_counters(sc); 1962 mutex_exit(&sc->sc_core_lock); 1963 } 1964 1965 switch (cmd) { 1966 #ifdef IF_RXR 1967 case SIOCGIFRXR: 1968 s = splnet(); 1969 error = igc_rxrinfo(sc, (struct if_rxrinfo *)ifr->ifr_data); 1970 splx(s); 1971 break; 1972 #endif 1973 default: 1974 s = splnet(); 1975 error = ether_ioctl(ifp, cmd, data); 1976 splx(s); 1977 break; 1978 } 1979 1980 if (error != ENETRESET) 1981 return error; 1982 1983 error = 0; 1984 1985 if (cmd == SIOCSIFCAP) 1986 error = if_init(ifp); 1987 else if ((cmd == SIOCADDMULTI) || (cmd == SIOCDELMULTI)) { 1988 mutex_enter(&sc->sc_core_lock); 1989 if (sc->sc_if_flags & IFF_RUNNING) { 1990 /* 1991 * Multicast list has changed; set the hardware filter 1992 * accordingly. 1993 */ 1994 igc_disable_intr(sc); 1995 igc_set_filter(sc); 1996 igc_enable_intr(sc); 1997 } 1998 mutex_exit(&sc->sc_core_lock); 1999 } 2000 2001 return error; 2002 } 2003 2004 #ifdef IF_RXR 2005 static int 2006 igc_rxrinfo(struct igc_softc *sc, struct if_rxrinfo *ifri) 2007 { 2008 struct if_rxring_info *ifr, ifr1; 2009 int error; 2010 2011 if (sc->sc_nqueues > 1) { 2012 ifr = kmem_zalloc(sc->sc_nqueues * sizeof(*ifr), KM_SLEEP); 2013 } else { 2014 ifr = &ifr1; 2015 memset(ifr, 0, sizeof(*ifr)); 2016 } 2017 2018 for (int iq = 0; iq < sc->sc_nqueues; iq++) { 2019 struct rx_ring *rxr = &sc->rx_rings[iq]; 2020 2021 ifr[iq].ifr_size = MCLBYTES; 2022 snprintf(ifr[iq].ifr_name, sizeof(ifr[iq].ifr_name), "%d", iq); 2023 ifr[iq].ifr_info = rxr->rx_ring; 2024 } 2025 2026 error = if_rxr_info_ioctl(ifri, sc->sc_nqueues, ifr); 2027 if (sc->sc_nqueues > 1) 2028 kmem_free(ifr, sc->sc_nqueues * sizeof(*ifr)); 2029 2030 return error; 2031 } 2032 #endif 2033 2034 static void 2035 igc_rxfill(struct rx_ring *rxr) 2036 { 2037 struct igc_softc *sc = rxr->sc; 2038 int id; 2039 2040 for (id = 0; id < sc->num_rx_desc; id++) { 2041 if (igc_get_buf(rxr, id, false)) { 2042 panic("%s: msix=%d i=%d\n", __func__, rxr->me, id); 2043 } 2044 } 2045 2046 id = sc->num_rx_desc - 1; 2047 rxr->last_desc_filled = id; 2048 IGC_WRITE_REG(&sc->hw, IGC_RDT(rxr->me), id); 2049 rxr->next_to_check = 0; 2050 } 2051 2052 static bool 2053 igc_rxrefill(struct rx_ring *rxr) 2054 { 2055 struct igc_softc *sc = rxr->sc; 2056 int id; 2057 const int start = igc_rxdesc_incr(sc, rxr->last_desc_filled); 2058 bool progress = false; 2059 2060 for (id = start; id != rxr->next_to_check; id = igc_rxdesc_incr(sc, id)) { 2061 if (igc_get_buf(rxr, id, true)) 2062 break; 2063 progress = true; 2064 } 2065 2066 if (progress) { 2067 int last = igc_rxdesc_decr(sc, id); 2068 DPRINTF(RX, "%s RDT %d last%d\n", 2069 rxr->last_desc_filled == last ? "same" : "diff", 2070 rxr->last_desc_filled, last); 2071 rxr->last_desc_filled = last; 2072 IGC_WRITE_REG(&sc->hw, IGC_RDT(rxr->me), last); 2073 } 2074 2075 return id != rxr->next_to_check; 2076 } 2077 2078 static void 2079 igc_rxrefill_callout(void *callout_arg) 2080 { 2081 struct rx_ring *rxr = callout_arg; 2082 2083 mutex_enter(&rxr->rxr_lock); 2084 if (igc_rxrefill(rxr)) 2085 callout_schedule(&rxr->rx_refill, 1); 2086 mutex_exit(&rxr->rxr_lock); 2087 } 2088 2089 /********************************************************************* 2090 * 2091 * This routine executes in interrupt context. It replenishes 2092 * the mbufs in the descriptor and sends data which has been 2093 * dma'ed into host memory to upper layer. 2094 * 2095 *********************************************************************/ 2096 static bool 2097 igc_rxeof(struct rx_ring *rxr, u_int limit) 2098 { 2099 struct igc_softc *sc = rxr->sc; 2100 struct igc_queue *q = rxr->rxr_igcq; 2101 struct ifnet *ifp = &sc->sc_ec.ec_if; 2102 int id; 2103 bool more = false; 2104 2105 id = rxr->next_to_check; 2106 for (;;) { 2107 union igc_adv_rx_desc *rxdesc = &rxr->rx_base[id]; 2108 struct igc_rx_buf *rxbuf, *nxbuf; 2109 struct mbuf *mp, *m; 2110 2111 igc_rxdesc_sync(rxr, id, 2112 BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE); 2113 2114 const uint32_t staterr = le32toh(rxdesc->wb.upper.status_error); 2115 2116 if (!ISSET(staterr, IGC_RXD_STAT_DD)) { 2117 igc_rxdesc_sync(rxr, id, 2118 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); 2119 break; 2120 } 2121 2122 if (limit-- == 0) { 2123 igc_rxdesc_sync(rxr, id, 2124 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); 2125 DPRINTF(RX, "more=true\n"); 2126 more = true; 2127 break; 2128 } 2129 2130 /* Zero out the receive descriptors status. */ 2131 rxdesc->wb.upper.status_error = 0; 2132 2133 /* Pull the mbuf off the ring. */ 2134 rxbuf = &rxr->rx_buffers[id]; 2135 bus_dmamap_t map = rxbuf->map; 2136 bus_dmamap_sync(rxr->rxdma.dma_tag, map, 2137 0, map->dm_mapsize, BUS_DMASYNC_POSTREAD); 2138 bus_dmamap_unload(rxr->rxdma.dma_tag, map); 2139 2140 mp = rxbuf->buf; 2141 rxbuf->buf = NULL; 2142 2143 const bool eop = staterr & IGC_RXD_STAT_EOP; 2144 const uint16_t len = le16toh(rxdesc->wb.upper.length); 2145 2146 #if NVLAN > 0 2147 const uint16_t vtag = le16toh(rxdesc->wb.upper.vlan); 2148 #endif 2149 2150 const uint32_t ptype = le32toh(rxdesc->wb.lower.lo_dword.data) & 2151 IGC_PKTTYPE_MASK; 2152 2153 const uint32_t hash __unused = 2154 le32toh(rxdesc->wb.lower.hi_dword.rss); 2155 const uint16_t hashtype __unused = 2156 le16toh(rxdesc->wb.lower.lo_dword.hs_rss.pkt_info) & 2157 IGC_RXDADV_RSSTYPE_MASK; 2158 2159 igc_rxdesc_sync(rxr, id, 2160 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); 2161 2162 if (__predict_false(staterr & IGC_RXDEXT_STATERR_RXE)) { 2163 m_freem(rxbuf->fmp); 2164 rxbuf->fmp = NULL; 2165 2166 m_freem(mp); 2167 m = NULL; 2168 2169 if_statinc(ifp, if_ierrors); 2170 IGC_QUEUE_EVENT(q, rx_discard, 1); 2171 2172 DPRINTF(RX, "ierrors++\n"); 2173 2174 goto next_desc; 2175 } 2176 2177 if (__predict_false(mp == NULL)) { 2178 panic("%s: igc_rxeof: NULL mbuf in slot %d " 2179 "(filled %d)", device_xname(sc->sc_dev), 2180 id, rxr->last_desc_filled); 2181 } 2182 2183 if (!eop) { 2184 /* 2185 * Figure out the next descriptor of this frame. 2186 */ 2187 int nextp = igc_rxdesc_incr(sc, id); 2188 2189 nxbuf = &rxr->rx_buffers[nextp]; 2190 /* 2191 * TODO prefetch(nxbuf); 2192 */ 2193 } 2194 2195 mp->m_len = len; 2196 2197 m = rxbuf->fmp; 2198 rxbuf->fmp = NULL; 2199 2200 if (m != NULL) { 2201 m->m_pkthdr.len += mp->m_len; 2202 } else { 2203 m = mp; 2204 m->m_pkthdr.len = mp->m_len; 2205 #if NVLAN > 0 2206 if (staterr & IGC_RXD_STAT_VP) 2207 vlan_set_tag(m, vtag); 2208 #endif 2209 } 2210 2211 /* Pass the head pointer on */ 2212 if (!eop) { 2213 nxbuf->fmp = m; 2214 m = NULL; 2215 mp->m_next = nxbuf->buf; 2216 } else { 2217 m_set_rcvif(m, ifp); 2218 2219 m->m_pkthdr.csum_flags = igc_rx_checksum(q, 2220 ifp->if_capenable, staterr, ptype); 2221 2222 #ifdef notyet 2223 if (hashtype != IGC_RXDADV_RSSTYPE_NONE) { 2224 m->m_pkthdr.ph_flowid = hash; 2225 SET(m->m_pkthdr.csum_flags, M_FLOWID); 2226 } 2227 ml_enqueue(&ml, m); 2228 #endif 2229 2230 if_percpuq_enqueue(sc->sc_ipq, m); 2231 2232 if_statinc(ifp, if_ipackets); 2233 IGC_QUEUE_EVENT(q, rx_packets, 1); 2234 IGC_QUEUE_EVENT(q, rx_bytes, m->m_pkthdr.len); 2235 } 2236 next_desc: 2237 /* Advance our pointers to the next descriptor. */ 2238 id = igc_rxdesc_incr(sc, id); 2239 } 2240 2241 DPRINTF(RX, "%s n2c %d id %d\n", 2242 rxr->next_to_check == id ? "same" : "diff", 2243 rxr->next_to_check, id); 2244 rxr->next_to_check = id; 2245 2246 DPRINTF(RX, "fill queue[%d]\n", rxr->me); 2247 if (igc_rxrefill(rxr)) 2248 callout_schedule(&rxr->rx_refill, 1); 2249 2250 #ifdef OPENBSD 2251 if (!(staterr & IGC_RXD_STAT_DD)) 2252 return 0; 2253 #endif 2254 2255 return more; 2256 } 2257 2258 /********************************************************************* 2259 * 2260 * Verify that the hardware indicated that the checksum is valid. 2261 * Inform the stack about the status of checksum so that stack 2262 * doesn't spend time verifying the checksum. 2263 * 2264 *********************************************************************/ 2265 static int 2266 igc_rx_checksum(struct igc_queue *q, uint64_t capenable, uint32_t staterr, 2267 uint32_t ptype) 2268 { 2269 const uint16_t status = (uint16_t)staterr; 2270 const uint8_t errors = (uint8_t)(staterr >> 24); 2271 int flags = 0; 2272 2273 if ((status & IGC_RXD_STAT_IPCS) != 0 && 2274 (capenable & IFCAP_CSUM_IPv4_Rx) != 0) { 2275 IGC_DRIVER_EVENT(q, rx_ipcs, 1); 2276 flags |= M_CSUM_IPv4; 2277 if (__predict_false((errors & IGC_RXD_ERR_IPE) != 0)) { 2278 IGC_DRIVER_EVENT(q, rx_ipcs_bad, 1); 2279 flags |= M_CSUM_IPv4_BAD; 2280 } 2281 } 2282 2283 if ((status & IGC_RXD_STAT_TCPCS) != 0) { 2284 IGC_DRIVER_EVENT(q, rx_tcpcs, 1); 2285 if ((capenable & IFCAP_CSUM_TCPv4_Rx) != 0) 2286 flags |= M_CSUM_TCPv4; 2287 if ((capenable & IFCAP_CSUM_TCPv6_Rx) != 0) 2288 flags |= M_CSUM_TCPv6; 2289 } 2290 2291 if ((status & IGC_RXD_STAT_UDPCS) != 0) { 2292 IGC_DRIVER_EVENT(q, rx_udpcs, 1); 2293 if ((capenable & IFCAP_CSUM_UDPv4_Rx) != 0) 2294 flags |= M_CSUM_UDPv4; 2295 if ((capenable & IFCAP_CSUM_UDPv6_Rx) != 0) 2296 flags |= M_CSUM_UDPv6; 2297 } 2298 2299 if (__predict_false((errors & IGC_RXD_ERR_TCPE) != 0)) { 2300 IGC_DRIVER_EVENT(q, rx_l4cs_bad, 1); 2301 if ((flags & ~M_CSUM_IPv4) != 0) 2302 flags |= M_CSUM_TCP_UDP_BAD; 2303 } 2304 2305 return flags; 2306 } 2307 2308 static void 2309 igc_watchdog(struct ifnet * ifp) 2310 { 2311 } 2312 2313 static void 2314 igc_tick(void *arg) 2315 { 2316 struct igc_softc *sc = arg; 2317 2318 mutex_enter(&sc->sc_core_lock); 2319 2320 if (__predict_false(sc->sc_core_stopping)) { 2321 mutex_exit(&sc->sc_core_lock); 2322 return; 2323 } 2324 2325 /* XXX watchdog */ 2326 if (0) { 2327 IGC_GLOBAL_EVENT(sc, watchdog, 1); 2328 } 2329 2330 igc_update_counters(sc); 2331 2332 mutex_exit(&sc->sc_core_lock); 2333 2334 callout_schedule(&sc->sc_tick_ch, hz); 2335 } 2336 2337 /********************************************************************* 2338 * 2339 * Media Ioctl callback 2340 * 2341 * This routine is called whenever the user queries the status of 2342 * the interface using ifconfig. 2343 * 2344 **********************************************************************/ 2345 static void 2346 igc_media_status(struct ifnet *ifp, struct ifmediareq *ifmr) 2347 { 2348 struct igc_softc *sc = ifp->if_softc; 2349 struct igc_hw *hw = &sc->hw; 2350 2351 igc_update_link_status(sc); 2352 2353 ifmr->ifm_status = IFM_AVALID; 2354 ifmr->ifm_active = IFM_ETHER; 2355 2356 if (!sc->link_active) { 2357 ifmr->ifm_active |= IFM_NONE; 2358 return; 2359 } 2360 2361 ifmr->ifm_status |= IFM_ACTIVE; 2362 2363 switch (sc->link_speed) { 2364 case 10: 2365 ifmr->ifm_active |= IFM_10_T; 2366 break; 2367 case 100: 2368 ifmr->ifm_active |= IFM_100_TX; 2369 break; 2370 case 1000: 2371 ifmr->ifm_active |= IFM_1000_T; 2372 break; 2373 case 2500: 2374 ifmr->ifm_active |= IFM_2500_T; 2375 break; 2376 } 2377 2378 if (sc->link_duplex == FULL_DUPLEX) 2379 ifmr->ifm_active |= IFM_FDX; 2380 else 2381 ifmr->ifm_active |= IFM_HDX; 2382 2383 switch (hw->fc.current_mode) { 2384 case igc_fc_tx_pause: 2385 ifmr->ifm_active |= IFM_FLOW | IFM_ETH_TXPAUSE; 2386 break; 2387 case igc_fc_rx_pause: 2388 ifmr->ifm_active |= IFM_FLOW | IFM_ETH_RXPAUSE; 2389 break; 2390 case igc_fc_full: 2391 ifmr->ifm_active |= IFM_FLOW | 2392 IFM_ETH_TXPAUSE | IFM_ETH_RXPAUSE; 2393 break; 2394 case igc_fc_none: 2395 default: 2396 break; 2397 } 2398 } 2399 2400 /********************************************************************* 2401 * 2402 * Media Ioctl callback 2403 * 2404 * This routine is called when the user changes speed/duplex using 2405 * media/mediopt option with ifconfig. 2406 * 2407 **********************************************************************/ 2408 static int 2409 igc_media_change(struct ifnet *ifp) 2410 { 2411 struct igc_softc *sc = ifp->if_softc; 2412 struct ifmedia *ifm = &sc->media; 2413 2414 if (IFM_TYPE(ifm->ifm_media) != IFM_ETHER) 2415 return EINVAL; 2416 2417 sc->hw.mac.autoneg = DO_AUTO_NEG; 2418 2419 switch (IFM_SUBTYPE(ifm->ifm_media)) { 2420 case IFM_AUTO: 2421 sc->hw.phy.autoneg_advertised = AUTONEG_ADV_DEFAULT; 2422 break; 2423 case IFM_2500_T: 2424 sc->hw.phy.autoneg_advertised = ADVERTISE_2500_FULL; 2425 break; 2426 case IFM_1000_T: 2427 sc->hw.phy.autoneg_advertised = ADVERTISE_1000_FULL; 2428 break; 2429 case IFM_100_TX: 2430 if ((ifm->ifm_media & IFM_GMASK) == IFM_FDX) 2431 sc->hw.phy.autoneg_advertised = ADVERTISE_100_FULL; 2432 else 2433 sc->hw.phy.autoneg_advertised = ADVERTISE_100_HALF; 2434 break; 2435 case IFM_10_T: 2436 if ((ifm->ifm_media & IFM_GMASK) == IFM_FDX) 2437 sc->hw.phy.autoneg_advertised = ADVERTISE_10_FULL; 2438 else 2439 sc->hw.phy.autoneg_advertised = ADVERTISE_10_HALF; 2440 break; 2441 default: 2442 return EINVAL; 2443 } 2444 2445 igc_init_locked(sc); 2446 2447 return 0; 2448 } 2449 2450 static int 2451 igc_ifflags_cb(struct ethercom *ec) 2452 { 2453 struct ifnet *ifp = &ec->ec_if; 2454 struct igc_softc *sc = ifp->if_softc; 2455 int rc = 0; 2456 u_short iffchange; 2457 bool needreset = false; 2458 2459 DPRINTF(CFG, "called\n"); 2460 2461 KASSERT(IFNET_LOCKED(ifp)); 2462 2463 mutex_enter(&sc->sc_core_lock); 2464 2465 /* 2466 * Check for if_flags. 2467 * Main usage is to prevent linkdown when opening bpf. 2468 */ 2469 iffchange = ifp->if_flags ^ sc->sc_if_flags; 2470 sc->sc_if_flags = ifp->if_flags; 2471 if ((iffchange & ~(IFF_CANTCHANGE | IFF_DEBUG)) != 0) { 2472 needreset = true; 2473 goto ec; 2474 } 2475 2476 /* iff related updates */ 2477 if ((iffchange & IFF_PROMISC) != 0) 2478 igc_set_filter(sc); 2479 2480 #ifdef notyet 2481 igc_set_vlan(sc); 2482 #endif 2483 2484 ec: 2485 #ifdef notyet 2486 /* Check for ec_capenable. */ 2487 ecchange = ec->ec_capenable ^ sc->sc_ec_capenable; 2488 sc->sc_ec_capenable = ec->ec_capenable; 2489 if ((ecchange & ~ETHERCAP_SOMETHING) != 0) { 2490 needreset = true; 2491 goto out; 2492 } 2493 #endif 2494 if (needreset) 2495 rc = ENETRESET; 2496 2497 mutex_exit(&sc->sc_core_lock); 2498 2499 return rc; 2500 } 2501 2502 static void 2503 igc_set_filter(struct igc_softc *sc) 2504 { 2505 struct ethercom *ec = &sc->sc_ec; 2506 uint32_t rctl; 2507 2508 rctl = IGC_READ_REG(&sc->hw, IGC_RCTL); 2509 rctl &= ~(IGC_RCTL_BAM |IGC_RCTL_UPE | IGC_RCTL_MPE); 2510 2511 if ((sc->sc_if_flags & IFF_BROADCAST) != 0) 2512 rctl |= IGC_RCTL_BAM; 2513 if ((sc->sc_if_flags & IFF_PROMISC) != 0) { 2514 DPRINTF(CFG, "promisc\n"); 2515 rctl |= IGC_RCTL_UPE; 2516 ETHER_LOCK(ec); 2517 allmulti: 2518 ec->ec_flags |= ETHER_F_ALLMULTI; 2519 ETHER_UNLOCK(ec); 2520 rctl |= IGC_RCTL_MPE; 2521 } else { 2522 struct ether_multistep step; 2523 struct ether_multi *enm; 2524 int mcnt = 0; 2525 2526 memset(sc->mta, 0, IGC_MTA_LEN); 2527 2528 ETHER_LOCK(ec); 2529 ETHER_FIRST_MULTI(step, ec, enm); 2530 while (enm != NULL) { 2531 if (((memcmp(enm->enm_addrlo, enm->enm_addrhi, 2532 ETHER_ADDR_LEN)) != 0) || 2533 (mcnt >= MAX_NUM_MULTICAST_ADDRESSES)) { 2534 /* 2535 * We must listen to a range of multicast 2536 * addresses. For now, just accept all 2537 * multicasts, rather than trying to set only 2538 * those filter bits needed to match the range. 2539 * (At this time, the only use of address 2540 * ranges is for IP multicast routing, for 2541 * which the range is big enough to require all 2542 * bits set.) 2543 */ 2544 goto allmulti; 2545 } 2546 DPRINTF(CFG, "%d: %s\n", mcnt, 2547 ether_sprintf(enm->enm_addrlo)); 2548 memcpy(&sc->mta[mcnt * ETHER_ADDR_LEN], 2549 enm->enm_addrlo, ETHER_ADDR_LEN); 2550 2551 mcnt++; 2552 ETHER_NEXT_MULTI(step, enm); 2553 } 2554 ec->ec_flags &= ~ETHER_F_ALLMULTI; 2555 ETHER_UNLOCK(ec); 2556 2557 DPRINTF(CFG, "hw filter\n"); 2558 igc_update_mc_addr_list(&sc->hw, sc->mta, mcnt); 2559 } 2560 2561 IGC_WRITE_REG(&sc->hw, IGC_RCTL, rctl); 2562 } 2563 2564 static void 2565 igc_update_link_status(struct igc_softc *sc) 2566 { 2567 struct ifnet *ifp = &sc->sc_ec.ec_if; 2568 struct igc_hw *hw = &sc->hw; 2569 2570 if (hw->mac.get_link_status == true) 2571 igc_check_for_link(hw); 2572 2573 if (IGC_READ_REG(&sc->hw, IGC_STATUS) & IGC_STATUS_LU) { 2574 if (sc->link_active == 0) { 2575 igc_get_speed_and_duplex(hw, &sc->link_speed, 2576 &sc->link_duplex); 2577 sc->link_active = 1; 2578 ifp->if_baudrate = IF_Mbps(sc->link_speed); 2579 if_link_state_change(ifp, LINK_STATE_UP); 2580 } 2581 } else { 2582 if (sc->link_active == 1) { 2583 ifp->if_baudrate = sc->link_speed = 0; 2584 sc->link_duplex = 0; 2585 sc->link_active = 0; 2586 if_link_state_change(ifp, LINK_STATE_DOWN); 2587 } 2588 } 2589 } 2590 2591 /********************************************************************* 2592 * 2593 * Get a buffer from system mbuf buffer pool. 2594 * 2595 **********************************************************************/ 2596 static int 2597 igc_get_buf(struct rx_ring *rxr, int id, bool strict) 2598 { 2599 struct igc_softc *sc = rxr->sc; 2600 struct igc_queue *q = rxr->rxr_igcq; 2601 struct igc_rx_buf *rxbuf = &rxr->rx_buffers[id]; 2602 bus_dmamap_t map = rxbuf->map; 2603 struct mbuf *m; 2604 int error; 2605 2606 if (__predict_false(rxbuf->buf)) { 2607 if (strict) { 2608 DPRINTF(RX, "slot %d already has an mbuf\n", id); 2609 return EINVAL; 2610 } 2611 return 0; 2612 } 2613 2614 MGETHDR(m, M_DONTWAIT, MT_DATA); 2615 if (__predict_false(m == NULL)) { 2616 enobuf: 2617 IGC_QUEUE_EVENT(q, rx_no_mbuf, 1); 2618 return ENOBUFS; 2619 } 2620 MCLAIM(m, &sc->sc_ec.ec_rx_mowner); 2621 2622 MCLGET(m, M_DONTWAIT); 2623 if (__predict_false(!(m->m_flags & M_EXT))) { 2624 m_freem(m); 2625 goto enobuf; 2626 } 2627 2628 m->m_len = m->m_pkthdr.len = sc->rx_mbuf_sz; 2629 2630 error = bus_dmamap_load_mbuf(rxr->rxdma.dma_tag, map, m, 2631 BUS_DMA_READ | BUS_DMA_NOWAIT); 2632 if (error) { 2633 m_freem(m); 2634 return error; 2635 } 2636 2637 bus_dmamap_sync(rxr->rxdma.dma_tag, map, 0, 2638 map->dm_mapsize, BUS_DMASYNC_PREREAD); 2639 rxbuf->buf = m; 2640 2641 union igc_adv_rx_desc *rxdesc = &rxr->rx_base[id]; 2642 igc_rxdesc_sync(rxr, id, BUS_DMASYNC_POSTWRITE | BUS_DMASYNC_POSTREAD); 2643 rxdesc->read.pkt_addr = htole64(map->dm_segs[0].ds_addr); 2644 igc_rxdesc_sync(rxr, id, BUS_DMASYNC_PREWRITE | BUS_DMASYNC_PREREAD); 2645 2646 return 0; 2647 } 2648 2649 static void 2650 igc_configure_queues(struct igc_softc *sc) 2651 { 2652 struct igc_hw *hw = &sc->hw; 2653 uint32_t ivar; 2654 2655 /* First turn on RSS capability */ 2656 IGC_WRITE_REG(hw, IGC_GPIE, IGC_GPIE_MSIX_MODE | IGC_GPIE_EIAME | 2657 IGC_GPIE_PBA | IGC_GPIE_NSICR); 2658 2659 /* Set the starting interrupt rate */ 2660 uint32_t newitr = (4000000 / MAX_INTS_PER_SEC) & 0x7FFC; 2661 newitr |= IGC_EITR_CNT_IGNR; 2662 2663 /* Turn on MSI-X */ 2664 uint32_t newmask = 0; 2665 for (int iq = 0; iq < sc->sc_nqueues; iq++) { 2666 struct igc_queue *q = &sc->queues[iq]; 2667 2668 /* RX entries */ 2669 igc_set_queues(sc, iq, q->msix, 0); 2670 /* TX entries */ 2671 igc_set_queues(sc, iq, q->msix, 1); 2672 newmask |= q->eims; 2673 IGC_WRITE_REG(hw, IGC_EITR(q->msix), newitr); 2674 } 2675 sc->msix_queuesmask = newmask; 2676 2677 #if 1 2678 ivar = IGC_READ_REG_ARRAY(hw, IGC_IVAR0, 0); 2679 DPRINTF(CFG, "ivar(0)=0x%x\n", ivar); 2680 ivar = IGC_READ_REG_ARRAY(hw, IGC_IVAR0, 1); 2681 DPRINTF(CFG, "ivar(1)=0x%x\n", ivar); 2682 #endif 2683 2684 /* And for the link interrupt */ 2685 ivar = (sc->linkvec | IGC_IVAR_VALID) << 8; 2686 sc->msix_linkmask = 1 << sc->linkvec; 2687 IGC_WRITE_REG(hw, IGC_IVAR_MISC, ivar); 2688 } 2689 2690 static void 2691 igc_set_queues(struct igc_softc *sc, uint32_t entry, uint32_t vector, int type) 2692 { 2693 struct igc_hw *hw = &sc->hw; 2694 const uint32_t index = entry >> 1; 2695 uint32_t ivar = IGC_READ_REG_ARRAY(hw, IGC_IVAR0, index); 2696 2697 if (type) { 2698 if (entry & 1) { 2699 ivar &= 0x00FFFFFF; 2700 ivar |= (vector | IGC_IVAR_VALID) << 24; 2701 } else { 2702 ivar &= 0xFFFF00FF; 2703 ivar |= (vector | IGC_IVAR_VALID) << 8; 2704 } 2705 } else { 2706 if (entry & 1) { 2707 ivar &= 0xFF00FFFF; 2708 ivar |= (vector | IGC_IVAR_VALID) << 16; 2709 } else { 2710 ivar &= 0xFFFFFF00; 2711 ivar |= vector | IGC_IVAR_VALID; 2712 } 2713 } 2714 IGC_WRITE_REG_ARRAY(hw, IGC_IVAR0, index, ivar); 2715 } 2716 2717 static void 2718 igc_enable_queue(struct igc_softc *sc, uint32_t eims) 2719 { 2720 IGC_WRITE_REG(&sc->hw, IGC_EIMS, eims); 2721 } 2722 2723 static void 2724 igc_enable_intr(struct igc_softc *sc) 2725 { 2726 struct igc_hw *hw = &sc->hw; 2727 2728 if (sc->sc_intr_type == PCI_INTR_TYPE_MSIX) { 2729 const uint32_t mask = sc->msix_queuesmask | sc->msix_linkmask; 2730 2731 IGC_WRITE_REG(hw, IGC_EIAC, mask); 2732 IGC_WRITE_REG(hw, IGC_EIAM, mask); 2733 IGC_WRITE_REG(hw, IGC_EIMS, mask); 2734 IGC_WRITE_REG(hw, IGC_IMS, IGC_IMS_LSC); 2735 } else { 2736 IGC_WRITE_REG(hw, IGC_IMS, IMS_ENABLE_MASK); 2737 } 2738 IGC_WRITE_FLUSH(hw); 2739 } 2740 2741 static void 2742 igc_disable_intr(struct igc_softc *sc) 2743 { 2744 struct igc_hw *hw = &sc->hw; 2745 2746 if (sc->sc_intr_type == PCI_INTR_TYPE_MSIX) { 2747 IGC_WRITE_REG(hw, IGC_EIMC, 0xffffffff); 2748 IGC_WRITE_REG(hw, IGC_EIAC, 0); 2749 } 2750 IGC_WRITE_REG(hw, IGC_IMC, 0xffffffff); 2751 IGC_WRITE_FLUSH(hw); 2752 } 2753 2754 static int 2755 igc_intr_link(void *arg) 2756 { 2757 struct igc_softc *sc = (struct igc_softc *)arg; 2758 const uint32_t reg_icr = IGC_READ_REG(&sc->hw, IGC_ICR); 2759 2760 IGC_GLOBAL_EVENT(sc, link, 1); 2761 2762 if (reg_icr & IGC_ICR_LSC) { 2763 mutex_enter(&sc->sc_core_lock); 2764 sc->hw.mac.get_link_status = true; 2765 igc_update_link_status(sc); 2766 mutex_exit(&sc->sc_core_lock); 2767 } 2768 2769 IGC_WRITE_REG(&sc->hw, IGC_IMS, IGC_IMS_LSC); 2770 IGC_WRITE_REG(&sc->hw, IGC_EIMS, sc->msix_linkmask); 2771 2772 return 1; 2773 } 2774 2775 static int 2776 igc_intr_queue(void *arg) 2777 { 2778 struct igc_queue *iq = arg; 2779 struct igc_softc *sc = iq->sc; 2780 struct ifnet *ifp = &sc->sc_ec.ec_if; 2781 struct rx_ring *rxr = iq->rxr; 2782 struct tx_ring *txr = iq->txr; 2783 const u_int txlimit = sc->sc_tx_intr_process_limit, 2784 rxlimit = sc->sc_rx_intr_process_limit; 2785 bool txmore, rxmore; 2786 2787 IGC_QUEUE_EVENT(iq, irqs, 1); 2788 2789 if (__predict_false(!ISSET(ifp->if_flags, IFF_RUNNING))) 2790 return 0; 2791 2792 mutex_enter(&txr->txr_lock); 2793 txmore = igc_txeof(txr, txlimit); 2794 mutex_exit(&txr->txr_lock); 2795 mutex_enter(&rxr->rxr_lock); 2796 rxmore = igc_rxeof(rxr, rxlimit); 2797 mutex_exit(&rxr->rxr_lock); 2798 2799 if (txmore || rxmore) { 2800 IGC_QUEUE_EVENT(iq, req, 1); 2801 igc_sched_handle_queue(sc, iq); 2802 } else { 2803 igc_enable_queue(sc, iq->eims); 2804 } 2805 2806 return 1; 2807 } 2808 2809 static int 2810 igc_intr(void *arg) 2811 { 2812 struct igc_softc *sc = arg; 2813 struct ifnet *ifp = &sc->sc_ec.ec_if; 2814 struct igc_queue *iq = &sc->queues[0]; 2815 struct rx_ring *rxr = iq->rxr; 2816 struct tx_ring *txr = iq->txr; 2817 const u_int txlimit = sc->sc_tx_intr_process_limit, 2818 rxlimit = sc->sc_rx_intr_process_limit; 2819 bool txmore, rxmore; 2820 2821 if (__predict_false(!ISSET(ifp->if_flags, IFF_RUNNING))) 2822 return 0; 2823 2824 const uint32_t reg_icr = IGC_READ_REG(&sc->hw, IGC_ICR); 2825 DPRINTF(MISC, "reg_icr=0x%x\n", reg_icr); 2826 2827 /* Definitely not our interrupt. */ 2828 if (reg_icr == 0x0) { 2829 DPRINTF(MISC, "not for me\n"); 2830 return 0; 2831 } 2832 2833 IGC_QUEUE_EVENT(iq, irqs, 1); 2834 2835 /* Hot eject? */ 2836 if (__predict_false(reg_icr == 0xffffffff)) { 2837 DPRINTF(MISC, "hot eject\n"); 2838 return 0; 2839 } 2840 2841 if (__predict_false(!(reg_icr & IGC_ICR_INT_ASSERTED))) { 2842 DPRINTF(MISC, "not set IGC_ICR_INT_ASSERTED"); 2843 return 0; 2844 } 2845 2846 /* 2847 * Only MSI-X interrupts have one-shot behavior by taking advantage 2848 * of the EIAC register. Thus, explicitly disable interrupts. This 2849 * also works around the MSI message reordering errata on certain 2850 * systems. 2851 */ 2852 igc_disable_intr(sc); 2853 2854 mutex_enter(&txr->txr_lock); 2855 txmore = igc_txeof(txr, txlimit); 2856 mutex_exit(&txr->txr_lock); 2857 mutex_enter(&rxr->rxr_lock); 2858 rxmore = igc_rxeof(rxr, rxlimit); 2859 mutex_exit(&rxr->rxr_lock); 2860 2861 /* Link status change */ 2862 // XXXX FreeBSD checks IGC_ICR_RXSEQ 2863 if (__predict_false(reg_icr & IGC_ICR_LSC)) { 2864 IGC_GLOBAL_EVENT(sc, link, 1); 2865 mutex_enter(&sc->sc_core_lock); 2866 sc->hw.mac.get_link_status = true; 2867 igc_update_link_status(sc); 2868 mutex_exit(&sc->sc_core_lock); 2869 } 2870 2871 if (txmore || rxmore) { 2872 IGC_QUEUE_EVENT(iq, req, 1); 2873 igc_sched_handle_queue(sc, iq); 2874 } else { 2875 igc_enable_intr(sc); 2876 } 2877 2878 return 1; 2879 } 2880 2881 static void 2882 igc_handle_queue(void *arg) 2883 { 2884 struct igc_queue *iq = arg; 2885 struct igc_softc *sc = iq->sc; 2886 struct tx_ring *txr = iq->txr; 2887 struct rx_ring *rxr = iq->rxr; 2888 const u_int txlimit = sc->sc_tx_process_limit, 2889 rxlimit = sc->sc_rx_process_limit; 2890 bool txmore, rxmore; 2891 2892 IGC_QUEUE_EVENT(iq, handleq, 1); 2893 2894 mutex_enter(&txr->txr_lock); 2895 txmore = igc_txeof(txr, txlimit); 2896 /* for ALTQ, dequeue from if_snd */ 2897 if (txr->me == 0) { 2898 struct ifnet *ifp = &sc->sc_ec.ec_if; 2899 2900 igc_tx_common_locked(ifp, txr, IGC_TX_START); 2901 } 2902 mutex_exit(&txr->txr_lock); 2903 2904 mutex_enter(&rxr->rxr_lock); 2905 rxmore = igc_rxeof(rxr, rxlimit); 2906 mutex_exit(&rxr->rxr_lock); 2907 2908 if (txmore || rxmore) { 2909 igc_sched_handle_queue(sc, iq); 2910 } else { 2911 if (sc->sc_intr_type == PCI_INTR_TYPE_MSIX) 2912 igc_enable_queue(sc, iq->eims); 2913 else 2914 igc_enable_intr(sc); 2915 } 2916 } 2917 2918 static void 2919 igc_handle_queue_work(struct work *wk, void *context) 2920 { 2921 struct igc_queue *iq = 2922 container_of(wk, struct igc_queue, igcq_wq_cookie); 2923 2924 igc_handle_queue(iq); 2925 } 2926 2927 static void 2928 igc_sched_handle_queue(struct igc_softc *sc, struct igc_queue *iq) 2929 { 2930 2931 if (iq->igcq_workqueue) { 2932 /* XXXRO notyet */ 2933 workqueue_enqueue(sc->sc_queue_wq, &iq->igcq_wq_cookie, 2934 curcpu()); 2935 } else { 2936 softint_schedule(iq->igcq_si); 2937 } 2938 } 2939 2940 static void 2941 igc_barrier_handle_queue(struct igc_softc *sc) 2942 { 2943 2944 if (sc->sc_txrx_workqueue) { 2945 for (int iq = 0; iq < sc->sc_nqueues; iq++) { 2946 struct igc_queue *q = &sc->queues[iq]; 2947 2948 workqueue_wait(sc->sc_queue_wq, &q->igcq_wq_cookie); 2949 } 2950 } else { 2951 xc_barrier(0); 2952 } 2953 } 2954 2955 /********************************************************************* 2956 * 2957 * Allocate memory for tx_buffer structures. The tx_buffer stores all 2958 * the information needed to transmit a packet on the wire. 2959 * 2960 **********************************************************************/ 2961 static int 2962 igc_allocate_transmit_buffers(struct tx_ring *txr) 2963 { 2964 struct igc_softc *sc = txr->sc; 2965 int error; 2966 2967 txr->tx_buffers = 2968 kmem_zalloc(sc->num_tx_desc * sizeof(struct igc_tx_buf), KM_SLEEP); 2969 txr->txtag = txr->txdma.dma_tag; 2970 2971 /* Create the descriptor buffer dma maps. */ 2972 for (int id = 0; id < sc->num_tx_desc; id++) { 2973 struct igc_tx_buf *txbuf = &txr->tx_buffers[id]; 2974 2975 error = bus_dmamap_create(txr->txdma.dma_tag, 2976 round_page(IGC_TSO_SIZE + sizeof(struct ether_vlan_header)), 2977 IGC_MAX_SCATTER, PAGE_SIZE, 0, BUS_DMA_NOWAIT, &txbuf->map); 2978 if (error != 0) { 2979 aprint_error_dev(sc->sc_dev, 2980 "unable to create TX DMA map\n"); 2981 goto fail; 2982 } 2983 2984 txbuf->eop_index = -1; 2985 } 2986 2987 return 0; 2988 fail: 2989 return error; 2990 } 2991 2992 2993 /********************************************************************* 2994 * 2995 * Allocate and initialize transmit structures. 2996 * 2997 **********************************************************************/ 2998 static int 2999 igc_setup_transmit_structures(struct igc_softc *sc) 3000 { 3001 3002 for (int iq = 0; iq < sc->sc_nqueues; iq++) { 3003 struct tx_ring *txr = &sc->tx_rings[iq]; 3004 3005 if (igc_setup_transmit_ring(txr)) 3006 goto fail; 3007 } 3008 3009 return 0; 3010 fail: 3011 igc_free_transmit_structures(sc); 3012 return ENOBUFS; 3013 } 3014 3015 /********************************************************************* 3016 * 3017 * Initialize a transmit ring. 3018 * 3019 **********************************************************************/ 3020 static int 3021 igc_setup_transmit_ring(struct tx_ring *txr) 3022 { 3023 struct igc_softc *sc = txr->sc; 3024 3025 /* Now allocate transmit buffers for the ring. */ 3026 if (igc_allocate_transmit_buffers(txr)) 3027 return ENOMEM; 3028 3029 /* Clear the old ring contents */ 3030 memset(txr->tx_base, 0, 3031 sizeof(union igc_adv_tx_desc) * sc->num_tx_desc); 3032 3033 /* Reset indices. */ 3034 txr->next_avail_desc = 0; 3035 txr->next_to_clean = 0; 3036 3037 bus_dmamap_sync(txr->txdma.dma_tag, txr->txdma.dma_map, 0, 3038 txr->txdma.dma_map->dm_mapsize, 3039 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); 3040 3041 txr->txr_interq = pcq_create(sc->num_tx_desc, KM_SLEEP); 3042 3043 mutex_init(&txr->txr_lock, MUTEX_DEFAULT, IPL_NET); 3044 3045 return 0; 3046 } 3047 3048 /********************************************************************* 3049 * 3050 * Enable transmit unit. 3051 * 3052 **********************************************************************/ 3053 static void 3054 igc_initialize_transmit_unit(struct igc_softc *sc) 3055 { 3056 struct ifnet *ifp = &sc->sc_ec.ec_if; 3057 struct igc_hw *hw = &sc->hw; 3058 3059 /* Setup the Base and Length of the TX descriptor ring. */ 3060 for (int iq = 0; iq < sc->sc_nqueues; iq++) { 3061 struct tx_ring *txr = &sc->tx_rings[iq]; 3062 const uint64_t bus_addr = 3063 txr->txdma.dma_map->dm_segs[0].ds_addr; 3064 3065 /* Base and len of TX ring */ 3066 IGC_WRITE_REG(hw, IGC_TDLEN(iq), 3067 sc->num_tx_desc * sizeof(union igc_adv_tx_desc)); 3068 IGC_WRITE_REG(hw, IGC_TDBAH(iq), (uint32_t)(bus_addr >> 32)); 3069 IGC_WRITE_REG(hw, IGC_TDBAL(iq), (uint32_t)bus_addr); 3070 3071 /* Init the HEAD/TAIL indices */ 3072 IGC_WRITE_REG(hw, IGC_TDT(iq), 0 /* XXX txr->next_avail_desc */); 3073 IGC_WRITE_REG(hw, IGC_TDH(iq), 0); 3074 3075 txr->watchdog_timer = 0; 3076 3077 uint32_t txdctl = 0; /* Clear txdctl */ 3078 txdctl |= 0x1f; /* PTHRESH */ 3079 txdctl |= 1 << 8; /* HTHRESH */ 3080 txdctl |= 1 << 16; /* WTHRESH */ 3081 txdctl |= 1 << 22; /* Reserved bit 22 must always be 1 */ 3082 txdctl |= IGC_TXDCTL_GRAN; 3083 txdctl |= 1 << 25; /* LWTHRESH */ 3084 3085 IGC_WRITE_REG(hw, IGC_TXDCTL(iq), txdctl); 3086 } 3087 ifp->if_timer = 0; 3088 3089 /* Program the Transmit Control Register */ 3090 uint32_t tctl = IGC_READ_REG(&sc->hw, IGC_TCTL); 3091 tctl &= ~IGC_TCTL_CT; 3092 tctl |= (IGC_TCTL_PSP | IGC_TCTL_RTLC | IGC_TCTL_EN | 3093 (IGC_COLLISION_THRESHOLD << IGC_CT_SHIFT)); 3094 3095 /* This write will effectively turn on the transmit unit. */ 3096 IGC_WRITE_REG(&sc->hw, IGC_TCTL, tctl); 3097 } 3098 3099 /********************************************************************* 3100 * 3101 * Free all transmit rings. 3102 * 3103 **********************************************************************/ 3104 static void 3105 igc_free_transmit_structures(struct igc_softc *sc) 3106 { 3107 3108 for (int iq = 0; iq < sc->sc_nqueues; iq++) { 3109 struct tx_ring *txr = &sc->tx_rings[iq]; 3110 3111 igc_free_transmit_buffers(txr); 3112 } 3113 } 3114 3115 /********************************************************************* 3116 * 3117 * Free transmit ring related data structures. 3118 * 3119 **********************************************************************/ 3120 static void 3121 igc_free_transmit_buffers(struct tx_ring *txr) 3122 { 3123 struct igc_softc *sc = txr->sc; 3124 3125 if (txr->tx_buffers == NULL) 3126 return; 3127 3128 igc_withdraw_transmit_packets(txr, true); 3129 3130 kmem_free(txr->tx_buffers, 3131 sc->num_tx_desc * sizeof(struct igc_tx_buf)); 3132 txr->tx_buffers = NULL; 3133 txr->txtag = NULL; 3134 3135 pcq_destroy(txr->txr_interq); 3136 mutex_destroy(&txr->txr_lock); 3137 } 3138 3139 /********************************************************************* 3140 * 3141 * Withdraw transmit packets. 3142 * 3143 **********************************************************************/ 3144 static void 3145 igc_withdraw_transmit_packets(struct tx_ring *txr, bool destroy) 3146 { 3147 struct igc_softc *sc = txr->sc; 3148 struct igc_queue *q = txr->txr_igcq; 3149 3150 mutex_enter(&txr->txr_lock); 3151 3152 for (int id = 0; id < sc->num_tx_desc; id++) { 3153 union igc_adv_tx_desc *txdesc = &txr->tx_base[id]; 3154 3155 igc_txdesc_sync(txr, id, 3156 BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE); 3157 txdesc->read.buffer_addr = 0; 3158 txdesc->read.cmd_type_len = 0; 3159 txdesc->read.olinfo_status = 0; 3160 igc_txdesc_sync(txr, id, 3161 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); 3162 3163 struct igc_tx_buf *txbuf = &txr->tx_buffers[id]; 3164 bus_dmamap_t map = txbuf->map; 3165 3166 if (map != NULL && map->dm_nsegs > 0) { 3167 bus_dmamap_sync(txr->txdma.dma_tag, map, 3168 0, map->dm_mapsize, BUS_DMASYNC_POSTWRITE); 3169 bus_dmamap_unload(txr->txdma.dma_tag, map); 3170 } 3171 m_freem(txbuf->m_head); 3172 txbuf->m_head = NULL; 3173 if (map != NULL && destroy) { 3174 bus_dmamap_destroy(txr->txdma.dma_tag, map); 3175 txbuf->map = NULL; 3176 } 3177 txbuf->eop_index = -1; 3178 3179 txr->next_avail_desc = 0; 3180 txr->next_to_clean = 0; 3181 } 3182 3183 struct mbuf *m; 3184 while ((m = pcq_get(txr->txr_interq)) != NULL) { 3185 IGC_QUEUE_EVENT(q, tx_pcq_drop, 1); 3186 m_freem(m); 3187 } 3188 3189 mutex_exit(&txr->txr_lock); 3190 } 3191 3192 3193 /********************************************************************* 3194 * 3195 * Advanced Context Descriptor setup for VLAN, CSUM or TSO 3196 * 3197 **********************************************************************/ 3198 3199 static bool 3200 igc_tx_ctx_setup(struct tx_ring *txr, struct mbuf *mp, int prod, 3201 uint32_t *cmd_type_len, uint32_t *olinfo_status) 3202 { 3203 struct ether_vlan_header *evl; 3204 struct tcphdr *th = NULL /* XXXGCC */; 3205 uint32_t vlan_macip_lens = 0; 3206 uint32_t type_tucmd_mlhl = 0; 3207 uint32_t mss_l4len_idx = 0; 3208 uint32_t ehlen, iphlen; 3209 uint16_t ehtype; 3210 3211 const int csum_flags = mp->m_pkthdr.csum_flags; 3212 const bool v4 = (csum_flags & 3213 (M_CSUM_IPv4 | M_CSUM_TCPv4 | M_CSUM_UDPv4 | M_CSUM_TSOv4)) != 0; 3214 const bool v6 = (csum_flags & 3215 (M_CSUM_UDPv6 | M_CSUM_TCPv6 | M_CSUM_TSOv6)) != 0; 3216 const bool tso = (csum_flags & (M_CSUM_TSOv4 | M_CSUM_TSOv6)) != 0; 3217 const bool tcp = tso || 3218 (csum_flags & (M_CSUM_TCPv4 | M_CSUM_TCPv6)) != 0; 3219 const bool udp = (csum_flags & (M_CSUM_UDPv4 | M_CSUM_UDPv6)) != 0; 3220 3221 /* Indicate the whole packet as payload when not doing TSO */ 3222 if (!tso) { 3223 *olinfo_status |= mp->m_pkthdr.len << IGC_ADVTXD_PAYLEN_SHIFT; 3224 } else { 3225 /* Set L4 payload length later... */ 3226 } 3227 3228 #if NVLAN > 0 3229 /* 3230 * In advanced descriptors the vlan tag must 3231 * be placed into the context descriptor. Hence 3232 * we need to make one even if not doing offloads. 3233 */ 3234 if (vlan_has_tag(mp)) { 3235 vlan_macip_lens |= (uint32_t)vlan_get_tag(mp) 3236 << IGC_ADVTXD_VLAN_SHIFT; 3237 } else 3238 #endif 3239 if (!v4 && !v6) 3240 return false; 3241 3242 KASSERT(mp->m_len >= sizeof(struct ether_header)); 3243 evl = mtod(mp, struct ether_vlan_header *); 3244 if (evl->evl_encap_proto == htons(ETHERTYPE_VLAN)) { 3245 KASSERT(mp->m_len >= sizeof(struct ether_vlan_header)); 3246 ehlen = ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN; 3247 ehtype = evl->evl_proto; 3248 } else { 3249 ehlen = ETHER_HDR_LEN; 3250 ehtype = evl->evl_encap_proto; 3251 } 3252 3253 switch (ntohs(ehtype)) { 3254 case ETHERTYPE_IP: 3255 iphlen = M_CSUM_DATA_IPv4_IPHL(mp->m_pkthdr.csum_data); 3256 type_tucmd_mlhl |= IGC_ADVTXD_TUCMD_IPV4; 3257 3258 if ((csum_flags & (M_CSUM_IPv4 | M_CSUM_TSOv4)) != 0) 3259 *olinfo_status |= IGC_TXD_POPTS_IXSM << 8; 3260 3261 if (!tso) 3262 break; 3263 3264 struct ip *ip; 3265 KASSERT(mp->m_len >= ehlen + sizeof(*ip)); 3266 ip = (void *)(mtod(mp, char *) + ehlen); 3267 ip->ip_len = 0; 3268 3269 KASSERT(mp->m_len >= ehlen + iphlen + sizeof(*th)); 3270 th = (void *)((char *)ip + iphlen); 3271 th->th_sum = in_cksum_phdr(ip->ip_src.s_addr, 3272 ip->ip_dst.s_addr, htons(IPPROTO_TCP)); 3273 break; 3274 case ETHERTYPE_IPV6: 3275 iphlen = M_CSUM_DATA_IPv6_IPHL(mp->m_pkthdr.csum_data); 3276 type_tucmd_mlhl |= IGC_ADVTXD_TUCMD_IPV6; 3277 3278 if (!tso) 3279 break; 3280 3281 struct ip6_hdr *ip6; 3282 KASSERT(mp->m_len >= ehlen + sizeof(*ip6)); 3283 ip6 = (void *)(mtod(mp, char *) + ehlen); 3284 ip6->ip6_plen = 0; 3285 3286 KASSERT(mp->m_len >= ehlen + iphlen + sizeof(*th)); 3287 th = (void *)((char *)ip6 + iphlen); 3288 th->th_sum = in6_cksum_phdr(&ip6->ip6_src, &ip6->ip6_dst, 0, 3289 htonl(IPPROTO_TCP)); 3290 break; 3291 default: 3292 /* 3293 * Unknown L3 protocol. Clear L3 header length and proceed for 3294 * LAN as done by Linux driver. 3295 */ 3296 KASSERT(!v4 && !v6); 3297 iphlen = 0; 3298 break; 3299 } 3300 3301 if (tcp) { 3302 type_tucmd_mlhl |= IGC_ADVTXD_TUCMD_L4T_TCP; 3303 *olinfo_status |= IGC_TXD_POPTS_TXSM << 8; 3304 } else if (udp) { 3305 type_tucmd_mlhl |= IGC_ADVTXD_TUCMD_L4T_UDP; 3306 *olinfo_status |= IGC_TXD_POPTS_TXSM << 8; 3307 } 3308 3309 if (tso) { 3310 const uint32_t tcphlen = th->th_off << 2; 3311 const uint32_t paylen = 3312 mp->m_pkthdr.len - ehlen - iphlen - tcphlen; 3313 3314 mss_l4len_idx |= mp->m_pkthdr.segsz << IGC_ADVTXD_MSS_SHIFT; 3315 mss_l4len_idx |= tcphlen << IGC_ADVTXD_L4LEN_SHIFT; 3316 3317 *cmd_type_len |= IGC_ADVTXD_DCMD_TSE; 3318 3319 *olinfo_status |= paylen << IGC_ADVTXD_PAYLEN_SHIFT; 3320 } 3321 3322 vlan_macip_lens |= iphlen; 3323 vlan_macip_lens |= ehlen << IGC_ADVTXD_MACLEN_SHIFT; 3324 3325 type_tucmd_mlhl |= IGC_ADVTXD_DCMD_DEXT | IGC_ADVTXD_DTYP_CTXT; 3326 3327 /* Now ready a context descriptor */ 3328 struct igc_adv_tx_context_desc *txdesc = 3329 (struct igc_adv_tx_context_desc *)&txr->tx_base[prod]; 3330 3331 igc_txdesc_sync(txr, prod, 3332 BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE); 3333 3334 /* Now copy bits into descriptor */ 3335 htolem32(&txdesc->vlan_macip_lens, vlan_macip_lens); 3336 htolem32(&txdesc->type_tucmd_mlhl, type_tucmd_mlhl); 3337 htolem32(&txdesc->seqnum_seed, 0); 3338 htolem32(&txdesc->mss_l4len_idx, mss_l4len_idx); 3339 3340 igc_txdesc_sync(txr, prod, 3341 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); 3342 3343 return 1; 3344 } 3345 3346 /********************************************************************* 3347 * 3348 * Allocate memory for rx_buffer structures. Since we use one 3349 * rx_buffer per received packet, the maximum number of rx_buffer's 3350 * that we'll need is equal to the number of receive descriptors 3351 * that we've allocated. 3352 * 3353 **********************************************************************/ 3354 static int 3355 igc_allocate_receive_buffers(struct rx_ring *rxr) 3356 { 3357 struct igc_softc *sc = rxr->sc; 3358 int error; 3359 3360 rxr->rx_buffers = 3361 kmem_zalloc(sc->num_rx_desc * sizeof(struct igc_rx_buf), KM_SLEEP); 3362 3363 for (int id = 0; id < sc->num_rx_desc; id++) { 3364 struct igc_rx_buf *rxbuf = &rxr->rx_buffers[id]; 3365 3366 error = bus_dmamap_create(rxr->rxdma.dma_tag, MCLBYTES, 1, 3367 MCLBYTES, 0, BUS_DMA_WAITOK, &rxbuf->map); 3368 if (error) { 3369 aprint_error_dev(sc->sc_dev, 3370 "unable to create RX DMA map\n"); 3371 goto fail; 3372 } 3373 } 3374 bus_dmamap_sync(rxr->rxdma.dma_tag, rxr->rxdma.dma_map, 0, 3375 rxr->rxdma.dma_map->dm_mapsize, 3376 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); 3377 3378 return 0; 3379 fail: 3380 return error; 3381 } 3382 3383 /********************************************************************* 3384 * 3385 * Allocate and initialize receive structures. 3386 * 3387 **********************************************************************/ 3388 static int 3389 igc_setup_receive_structures(struct igc_softc *sc) 3390 { 3391 3392 for (int iq = 0; iq < sc->sc_nqueues; iq++) { 3393 struct rx_ring *rxr = &sc->rx_rings[iq]; 3394 3395 if (igc_setup_receive_ring(rxr)) 3396 goto fail; 3397 } 3398 3399 return 0; 3400 fail: 3401 igc_free_receive_structures(sc); 3402 return ENOBUFS; 3403 } 3404 3405 /********************************************************************* 3406 * 3407 * Initialize a receive ring and its buffers. 3408 * 3409 **********************************************************************/ 3410 static int 3411 igc_setup_receive_ring(struct rx_ring *rxr) 3412 { 3413 struct igc_softc *sc = rxr->sc; 3414 const int rsize = roundup2( 3415 sc->num_rx_desc * sizeof(union igc_adv_rx_desc), IGC_DBA_ALIGN); 3416 3417 /* Clear the ring contents. */ 3418 memset(rxr->rx_base, 0, rsize); 3419 3420 if (igc_allocate_receive_buffers(rxr)) 3421 return ENOMEM; 3422 3423 /* Setup our descriptor indices. */ 3424 rxr->next_to_check = 0; 3425 rxr->last_desc_filled = 0; 3426 3427 mutex_init(&rxr->rxr_lock, MUTEX_DEFAULT, IPL_NET); 3428 3429 return 0; 3430 } 3431 3432 /********************************************************************* 3433 * 3434 * Enable receive unit. 3435 * 3436 **********************************************************************/ 3437 static void 3438 igc_initialize_receive_unit(struct igc_softc *sc) 3439 { 3440 struct ifnet *ifp = &sc->sc_ec.ec_if; 3441 struct igc_hw *hw = &sc->hw; 3442 uint32_t rctl, rxcsum, srrctl; 3443 3444 DPRINTF(RX, "called\n"); 3445 3446 /* 3447 * Make sure receives are disabled while setting 3448 * up the descriptor ring. 3449 */ 3450 rctl = IGC_READ_REG(hw, IGC_RCTL); 3451 IGC_WRITE_REG(hw, IGC_RCTL, rctl & ~IGC_RCTL_EN); 3452 3453 /* Setup the Receive Control Register */ 3454 rctl &= ~(3 << IGC_RCTL_MO_SHIFT); 3455 rctl |= IGC_RCTL_EN | IGC_RCTL_BAM | IGC_RCTL_LBM_NO | 3456 IGC_RCTL_RDMTS_HALF | (hw->mac.mc_filter_type << IGC_RCTL_MO_SHIFT); 3457 3458 #if 1 3459 /* Do not store bad packets */ 3460 rctl &= ~IGC_RCTL_SBP; 3461 #else 3462 /* for debug */ 3463 rctl |= IGC_RCTL_SBP; 3464 #endif 3465 3466 /* Enable Long Packet receive */ 3467 if (sc->hw.mac.max_frame_size > ETHER_MAX_LEN) 3468 rctl |= IGC_RCTL_LPE; 3469 else 3470 rctl &= ~IGC_RCTL_LPE; 3471 3472 /* Strip the CRC */ 3473 rctl |= IGC_RCTL_SECRC; 3474 3475 /* 3476 * Set the interrupt throttling rate. Value is calculated 3477 * as DEFAULT_ITR = 1/(MAX_INTS_PER_SEC * 256ns) 3478 * 3479 * XXX Sync with Linux, especially for jumbo MTU or TSO. 3480 * XXX Shouldn't be here? 3481 */ 3482 IGC_WRITE_REG(hw, IGC_ITR, DEFAULT_ITR); 3483 3484 rxcsum = IGC_READ_REG(hw, IGC_RXCSUM); 3485 rxcsum &= ~(IGC_RXCSUM_IPOFL | IGC_RXCSUM_TUOFL | IGC_RXCSUM_PCSD); 3486 if (ifp->if_capenable & IFCAP_CSUM_IPv4_Rx) 3487 rxcsum |= IGC_RXCSUM_IPOFL; 3488 if (ifp->if_capenable & (IFCAP_CSUM_TCPv4_Rx | IFCAP_CSUM_UDPv4_Rx | 3489 IFCAP_CSUM_TCPv6_Rx | IFCAP_CSUM_UDPv6_Rx)) 3490 rxcsum |= IGC_RXCSUM_TUOFL; 3491 if (sc->sc_nqueues > 1) 3492 rxcsum |= IGC_RXCSUM_PCSD; 3493 IGC_WRITE_REG(hw, IGC_RXCSUM, rxcsum); 3494 3495 if (sc->sc_nqueues > 1) 3496 igc_initialize_rss_mapping(sc); 3497 3498 srrctl = 0; 3499 #if 0 3500 srrctl |= 4096 >> IGC_SRRCTL_BSIZEPKT_SHIFT; 3501 rctl |= IGC_RCTL_SZ_4096 | IGC_RCTL_BSEX; 3502 #else 3503 srrctl |= 2048 >> IGC_SRRCTL_BSIZEPKT_SHIFT; 3504 rctl |= IGC_RCTL_SZ_2048; 3505 #endif 3506 3507 /* 3508 * If TX flow control is disabled and there's > 1 queue defined, 3509 * enable DROP. 3510 * 3511 * This drops frames rather than hanging the RX MAC for all queues. 3512 */ 3513 if (sc->sc_nqueues > 1 && 3514 (sc->fc == igc_fc_none || sc->fc == igc_fc_rx_pause)) 3515 srrctl |= IGC_SRRCTL_DROP_EN; 3516 3517 /* Setup the Base and Length of the RX descriptor rings. */ 3518 for (int iq = 0; iq < sc->sc_nqueues; iq++) { 3519 struct rx_ring *rxr = &sc->rx_rings[iq]; 3520 const uint64_t bus_addr = 3521 rxr->rxdma.dma_map->dm_segs[0].ds_addr; 3522 3523 IGC_WRITE_REG(hw, IGC_RXDCTL(iq), 0); 3524 3525 srrctl |= IGC_SRRCTL_DESCTYPE_ADV_ONEBUF; 3526 3527 IGC_WRITE_REG(hw, IGC_RDLEN(iq), 3528 sc->num_rx_desc * sizeof(union igc_adv_rx_desc)); 3529 IGC_WRITE_REG(hw, IGC_RDBAH(iq), (uint32_t)(bus_addr >> 32)); 3530 IGC_WRITE_REG(hw, IGC_RDBAL(iq), (uint32_t)bus_addr); 3531 IGC_WRITE_REG(hw, IGC_SRRCTL(iq), srrctl); 3532 3533 /* Setup the Head and Tail Descriptor Pointers */ 3534 IGC_WRITE_REG(hw, IGC_RDH(iq), 0); 3535 IGC_WRITE_REG(hw, IGC_RDT(iq), 0 /* XXX rxr->last_desc_filled */); 3536 3537 /* Enable this Queue */ 3538 uint32_t rxdctl = IGC_READ_REG(hw, IGC_RXDCTL(iq)); 3539 rxdctl |= IGC_RXDCTL_QUEUE_ENABLE; 3540 rxdctl &= 0xFFF00000; 3541 rxdctl |= IGC_RX_PTHRESH; 3542 rxdctl |= IGC_RX_HTHRESH << 8; 3543 rxdctl |= IGC_RX_WTHRESH << 16; 3544 IGC_WRITE_REG(hw, IGC_RXDCTL(iq), rxdctl); 3545 } 3546 3547 /* Make sure VLAN Filters are off */ 3548 rctl &= ~IGC_RCTL_VFE; 3549 3550 /* Write out the settings */ 3551 IGC_WRITE_REG(hw, IGC_RCTL, rctl); 3552 } 3553 3554 /********************************************************************* 3555 * 3556 * Free all receive rings. 3557 * 3558 **********************************************************************/ 3559 static void 3560 igc_free_receive_structures(struct igc_softc *sc) 3561 { 3562 3563 for (int iq = 0; iq < sc->sc_nqueues; iq++) { 3564 struct rx_ring *rxr = &sc->rx_rings[iq]; 3565 3566 igc_free_receive_buffers(rxr); 3567 } 3568 } 3569 3570 /********************************************************************* 3571 * 3572 * Free receive ring data structures 3573 * 3574 **********************************************************************/ 3575 static void 3576 igc_free_receive_buffers(struct rx_ring *rxr) 3577 { 3578 struct igc_softc *sc = rxr->sc; 3579 3580 callout_halt(&rxr->rx_refill, NULL); 3581 callout_destroy(&rxr->rx_refill); 3582 3583 if (rxr->rx_buffers != NULL) { 3584 for (int id = 0; id < sc->num_rx_desc; id++) { 3585 struct igc_rx_buf *rxbuf = &rxr->rx_buffers[id]; 3586 bus_dmamap_t map = rxbuf->map; 3587 3588 if (rxbuf->buf != NULL) { 3589 bus_dmamap_sync(rxr->rxdma.dma_tag, map, 3590 0, map->dm_mapsize, BUS_DMASYNC_POSTREAD); 3591 bus_dmamap_unload(rxr->rxdma.dma_tag, map); 3592 m_freem(rxbuf->buf); 3593 rxbuf->buf = NULL; 3594 } 3595 bus_dmamap_destroy(rxr->rxdma.dma_tag, map); 3596 rxbuf->map = NULL; 3597 } 3598 kmem_free(rxr->rx_buffers, 3599 sc->num_rx_desc * sizeof(struct igc_rx_buf)); 3600 rxr->rx_buffers = NULL; 3601 } 3602 3603 mutex_destroy(&rxr->rxr_lock); 3604 } 3605 3606 /********************************************************************* 3607 * 3608 * Clear status registers in all RX descriptors. 3609 * 3610 **********************************************************************/ 3611 static void 3612 igc_clear_receive_status(struct rx_ring *rxr) 3613 { 3614 struct igc_softc *sc = rxr->sc; 3615 3616 mutex_enter(&rxr->rxr_lock); 3617 3618 for (int id = 0; id < sc->num_rx_desc; id++) { 3619 union igc_adv_rx_desc *rxdesc = &rxr->rx_base[id]; 3620 3621 igc_rxdesc_sync(rxr, id, 3622 BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE); 3623 rxdesc->wb.upper.status_error = 0; 3624 igc_rxdesc_sync(rxr, id, 3625 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); 3626 } 3627 3628 mutex_exit(&rxr->rxr_lock); 3629 } 3630 3631 /* 3632 * Initialise the RSS mapping for NICs that support multiple transmit/ 3633 * receive rings. 3634 */ 3635 static void 3636 igc_initialize_rss_mapping(struct igc_softc *sc) 3637 { 3638 struct igc_hw *hw = &sc->hw; 3639 3640 /* 3641 * The redirection table controls which destination 3642 * queue each bucket redirects traffic to. 3643 * Each DWORD represents four queues, with the LSB 3644 * being the first queue in the DWORD. 3645 * 3646 * This just allocates buckets to queues using round-robin 3647 * allocation. 3648 * 3649 * NOTE: It Just Happens to line up with the default 3650 * RSS allocation method. 3651 */ 3652 3653 /* Warning FM follows */ 3654 uint32_t reta = 0; 3655 for (int i = 0; i < 128; i++) { 3656 const int shift = 0; /* XXXRO */ 3657 int queue_id = i % sc->sc_nqueues; 3658 /* Adjust if required */ 3659 queue_id <<= shift; 3660 3661 /* 3662 * The low 8 bits are for hash value (n+0); 3663 * The next 8 bits are for hash value (n+1), etc. 3664 */ 3665 reta >>= 8; 3666 reta |= ((uint32_t)queue_id) << 24; 3667 if ((i & 3) == 3) { 3668 IGC_WRITE_REG(hw, IGC_RETA(i >> 2), reta); 3669 reta = 0; 3670 } 3671 } 3672 3673 /* 3674 * MRQC: Multiple Receive Queues Command 3675 * Set queuing to RSS control, number depends on the device. 3676 */ 3677 3678 /* Set up random bits */ 3679 uint32_t rss_key[RSS_KEYSIZE / sizeof(uint32_t)]; 3680 rss_getkey((uint8_t *)rss_key); 3681 3682 /* Now fill our hash function seeds */ 3683 for (int i = 0; i < __arraycount(rss_key); i++) 3684 IGC_WRITE_REG_ARRAY(hw, IGC_RSSRK(0), i, rss_key[i]); 3685 3686 /* 3687 * Configure the RSS fields to hash upon. 3688 */ 3689 uint32_t mrqc = IGC_MRQC_ENABLE_RSS_4Q; 3690 mrqc |= IGC_MRQC_RSS_FIELD_IPV4 | IGC_MRQC_RSS_FIELD_IPV4_TCP; 3691 mrqc |= IGC_MRQC_RSS_FIELD_IPV6 | IGC_MRQC_RSS_FIELD_IPV6_TCP; 3692 mrqc |= IGC_MRQC_RSS_FIELD_IPV6_TCP_EX; 3693 3694 IGC_WRITE_REG(hw, IGC_MRQC, mrqc); 3695 } 3696 3697 /* 3698 * igc_get_hw_control sets the {CTRL_EXT|FWSM}:DRV_LOAD bit. 3699 * For ASF and Pass Through versions of f/w this means 3700 * that the driver is loaded. For AMT version type f/w 3701 * this means that the network i/f is open. 3702 */ 3703 static void 3704 igc_get_hw_control(struct igc_softc *sc) 3705 { 3706 const uint32_t ctrl_ext = IGC_READ_REG(&sc->hw, IGC_CTRL_EXT); 3707 3708 IGC_WRITE_REG(&sc->hw, IGC_CTRL_EXT, ctrl_ext | IGC_CTRL_EXT_DRV_LOAD); 3709 } 3710 3711 /* 3712 * igc_release_hw_control resets {CTRL_EXT|FWSM}:DRV_LOAD bit. 3713 * For ASF and Pass Through versions of f/w this means that 3714 * the driver is no longer loaded. For AMT versions of the 3715 * f/w this means that the network i/f is closed. 3716 */ 3717 static void 3718 igc_release_hw_control(struct igc_softc *sc) 3719 { 3720 const uint32_t ctrl_ext = IGC_READ_REG(&sc->hw, IGC_CTRL_EXT); 3721 3722 IGC_WRITE_REG(&sc->hw, IGC_CTRL_EXT, ctrl_ext & ~IGC_CTRL_EXT_DRV_LOAD); 3723 } 3724 3725 static int 3726 igc_is_valid_ether_addr(uint8_t *addr) 3727 { 3728 const char zero_addr[6] = { 0, 0, 0, 0, 0, 0 }; 3729 3730 if ((addr[0] & 1) || !bcmp(addr, zero_addr, ETHER_ADDR_LEN)) 3731 return 0; 3732 3733 return 1; 3734 } 3735 3736 static void 3737 igc_print_devinfo(struct igc_softc *sc) 3738 { 3739 device_t dev = sc->sc_dev; 3740 struct igc_hw *hw = &sc->hw; 3741 struct igc_phy_info *phy = &hw->phy; 3742 u_int oui, model, rev; 3743 uint16_t id1, id2, nvm_ver, phy_ver, etk_lo, etk_hi; 3744 char descr[MII_MAX_DESCR_LEN]; 3745 3746 /* Print PHY Info */ 3747 id1 = phy->id >> 16; 3748 /* The revision field in phy->id is cleard and it's in phy->revision */ 3749 id2 = (phy->id & 0xfff0) | phy->revision; 3750 oui = MII_OUI(id1, id2); 3751 model = MII_MODEL(id2); 3752 rev = MII_REV(id2); 3753 mii_get_descr(descr, sizeof(descr), oui, model); 3754 if (descr[0]) 3755 aprint_normal_dev(dev, "PHY: %s, rev. %d", 3756 descr, rev); 3757 else 3758 aprint_normal_dev(dev, 3759 "PHY OUI 0x%06x, model 0x%04x, rev. %d", 3760 oui, model, rev); 3761 3762 /* PHY FW version */ 3763 phy->ops.read_reg(hw, 0x1e, &phy_ver); 3764 aprint_normal(", PHY FW version 0x%04hx\n", phy_ver); 3765 3766 /* NVM version */ 3767 hw->nvm.ops.read(hw, NVM_VERSION, 1, &nvm_ver); 3768 3769 /* EtrackID */ 3770 hw->nvm.ops.read(hw, NVM_ETKID_LO, 1, &etk_lo); 3771 hw->nvm.ops.read(hw, NVM_ETKID_HI, 1, &etk_hi); 3772 3773 aprint_normal_dev(dev, 3774 "NVM image version %x.%02x, EtrackID %04hx%04hx\n", 3775 (nvm_ver & NVM_VERSION_MAJOR) >> NVM_VERSION_MAJOR_SHIFT, 3776 nvm_ver & NVM_VERSION_MINOR, etk_hi, etk_lo); 3777 } 3778 3779 /* Module interface */ 3780 3781 MODULE(MODULE_CLASS_DRIVER, if_igc, "pci"); 3782 3783 #ifdef _MODULE 3784 #include "ioconf.c" 3785 #endif 3786 3787 static int 3788 if_igc_modcmd(modcmd_t cmd, void *opaque) 3789 { 3790 int error = 0; 3791 3792 switch (cmd) { 3793 case MODULE_CMD_INIT: 3794 #ifdef _MODULE 3795 error = config_init_component(cfdriver_ioconf_igc, 3796 cfattach_ioconf_igc, cfdata_ioconf_igc); 3797 #endif 3798 return error; 3799 case MODULE_CMD_FINI: 3800 #ifdef _MODULE 3801 error = config_fini_component(cfdriver_ioconf_igc, 3802 cfattach_ioconf_igc, cfdata_ioconf_igc); 3803 #endif 3804 return error; 3805 default: 3806 return ENOTTY; 3807 } 3808 } 3809
Indexes created Sun Oct 25 00:26:45 UTC 2026