if_igc.c revision 1.9
1 /* $NetBSD: if_igc.c,v 1.9 2023/12/20 18:09:19 skrll Exp $ */ 2 /* $OpenBSD: if_igc.c,v 1.13 2023/04/28 10:18:57 bluhm Exp $ */ 3 /*- 4 * SPDX-License-Identifier: BSD-2-Clause 5 * 6 * Copyright (c) 2016 Nicole Graziano <nicole (at) nextbsd.org> 7 * All rights reserved. 8 * Copyright (c) 2021 Rubicon Communications, LLC (Netgate) 9 * 10 * Redistribution and use in source and binary forms, with or without 11 * modification, are permitted provided that the following conditions 12 * are met: 13 * 1. Redistributions of source code must retain the above copyright 14 * notice, this list of conditions and the following disclaimer. 15 * 2. Redistributions in binary form must reproduce the above copyright 16 * notice, this list of conditions and the following disclaimer in the 17 * documentation and/or other materials provided with the distribution. 18 * 19 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 20 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 21 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 22 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 23 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 24 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 25 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 26 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 27 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 28 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 29 * SUCH DAMAGE. 30 */ 31 32 #include <sys/cdefs.h> 33 __KERNEL_RCSID(0, "$NetBSD: if_igc.c,v 1.9 2023/12/20 18:09:19 skrll Exp $"); 34 35 #ifdef _KERNEL_OPT 36 #include "opt_if_igc.h" 37 #if 0 /* notyet */ 38 #include "vlan.h" 39 #endif 40 #endif 41 42 #include <sys/param.h> 43 #include <sys/systm.h> 44 #include <sys/bus.h> 45 #include <sys/cpu.h> 46 #include <sys/device.h> 47 #include <sys/endian.h> 48 #include <sys/intr.h> 49 #include <sys/interrupt.h> 50 #include <sys/kernel.h> 51 #include <sys/kmem.h> 52 #include <sys/mbuf.h> 53 #include <sys/mutex.h> 54 #include <sys/socket.h> 55 #include <sys/workqueue.h> 56 #include <sys/xcall.h> 57 58 #include <net/bpf.h> 59 #include <net/if.h> 60 #include <net/if_ether.h> 61 #include <net/if_media.h> 62 #include <net/if_vlanvar.h> 63 #include <net/rss_config.h> 64 65 #include <netinet/in.h> 66 #include <netinet/ip.h> 67 #include <netinet/ip6.h> 68 #include <netinet/tcp.h> 69 70 #include <dev/pci/pcivar.h> 71 #include <dev/pci/pcireg.h> 72 #include <dev/pci/pcidevs.h> 73 74 #include <dev/pci/igc/if_igc.h> 75 #include <dev/pci/igc/igc_evcnt.h> 76 #include <dev/pci/igc/igc_hw.h> 77 #include <dev/mii/miivar.h> 78 79 #define IGC_WORKQUEUE_PRI PRI_SOFTNET 80 81 #ifndef IGC_RX_INTR_PROCESS_LIMIT_DEFAULT 82 #define IGC_RX_INTR_PROCESS_LIMIT_DEFAULT 0 83 #endif 84 #ifndef IGC_TX_INTR_PROCESS_LIMIT_DEFAULT 85 #define IGC_TX_INTR_PROCESS_LIMIT_DEFAULT 0 86 #endif 87 88 #ifndef IGC_RX_PROCESS_LIMIT_DEFAULT 89 #define IGC_RX_PROCESS_LIMIT_DEFAULT 256 90 #endif 91 #ifndef IGC_TX_PROCESS_LIMIT_DEFAULT 92 #define IGC_TX_PROCESS_LIMIT_DEFAULT 256 93 #endif 94 95 #define htolem32(p, x) (*((uint32_t *)(p)) = htole32(x)) 96 #define htolem64(p, x) (*((uint64_t *)(p)) = htole64(x)) 97 98 static const struct igc_product { 99 pci_vendor_id_t igcp_vendor; 100 pci_product_id_t igcp_product; 101 const char *igcp_name; 102 } igc_products[] = { 103 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_I225_IT, 104 "Intel(R) Ethernet Controller I225-IT(2)" }, 105 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_I226_LM, 106 "Intel(R) Ethernet Controller I226-LM" }, 107 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_I226_V, 108 "Intel(R) Ethernet Controller I226-V" }, 109 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_I226_IT, 110 "Intel(R) Ethernet Controller I226-IT" }, 111 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_I221_V, 112 "Intel(R) Ethernet Controller I221-V" }, 113 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_I226_BLANK_NVM, 114 "Intel(R) Ethernet Controller I226(blankNVM)" }, 115 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_I225_LM, 116 "Intel(R) Ethernet Controller I225-LM" }, 117 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_I225_V, 118 "Intel(R) Ethernet Controller I225-V" }, 119 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_I220_V, 120 "Intel(R) Ethernet Controller I220-V" }, 121 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_I225_I, 122 "Intel(R) Ethernet Controller I225-I" }, 123 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_I225_BLANK_NVM, 124 "Intel(R) Ethernet Controller I225(blankNVM)" }, 125 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_I225_K, 126 "Intel(R) Ethernet Controller I225-K" }, 127 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_I225_K2, 128 "Intel(R) Ethernet Controller I225-K(2)" }, 129 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_I226_K, 130 "Intel(R) Ethernet Controller I226-K" }, 131 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_I225_LMVP, 132 "Intel(R) Ethernet Controller I225-LMvP(2)" }, 133 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_I226_LMVP, 134 "Intel(R) Ethernet Controller I226-LMvP" }, 135 { 0, 0, NULL }, 136 }; 137 138 #define IGC_DF_CFG 0x1 139 #define IGC_DF_TX 0x2 140 #define IGC_DF_RX 0x4 141 #define IGC_DF_MISC 0x8 142 143 #ifdef IGC_DEBUG_FLAGS 144 int igc_debug_flags = IGC_DEBUG_FLAGS; 145 #else 146 int igc_debug_flags = 0; 147 #endif 148 149 #define DPRINTF(flag, fmt, args...) do { \ 150 if (igc_debug_flags & (IGC_DF_ ## flag)) \ 151 printf("%s: %d: " fmt, __func__, __LINE__, ##args); \ 152 } while (0) 153 154 /********************************************************************* 155 * Function Prototypes 156 *********************************************************************/ 157 static int igc_match(device_t, cfdata_t, void *); 158 static void igc_attach(device_t, device_t, void *); 159 static int igc_detach(device_t, int); 160 161 static void igc_identify_hardware(struct igc_softc *); 162 static int igc_adjust_nqueues(struct igc_softc *); 163 static int igc_allocate_pci_resources(struct igc_softc *); 164 static int igc_allocate_interrupts(struct igc_softc *); 165 static int igc_allocate_queues(struct igc_softc *); 166 static void igc_free_pci_resources(struct igc_softc *); 167 static void igc_free_interrupts(struct igc_softc *); 168 static void igc_free_queues(struct igc_softc *); 169 static void igc_reset(struct igc_softc *); 170 static void igc_init_dmac(struct igc_softc *, uint32_t); 171 static int igc_setup_interrupts(struct igc_softc *); 172 static void igc_attach_counters(struct igc_softc *sc); 173 static void igc_detach_counters(struct igc_softc *sc); 174 static void igc_update_counters(struct igc_softc *sc); 175 static void igc_clear_counters(struct igc_softc *sc); 176 static int igc_setup_msix(struct igc_softc *); 177 static int igc_setup_msi(struct igc_softc *); 178 static int igc_setup_intx(struct igc_softc *); 179 static int igc_dma_malloc(struct igc_softc *, bus_size_t, 180 struct igc_dma_alloc *); 181 static void igc_dma_free(struct igc_softc *, struct igc_dma_alloc *); 182 static void igc_setup_interface(struct igc_softc *); 183 184 static int igc_init(struct ifnet *); 185 static int igc_init_locked(struct igc_softc *); 186 static void igc_start(struct ifnet *); 187 static int igc_transmit(struct ifnet *, struct mbuf *); 188 static void igc_tx_common_locked(struct ifnet *, struct tx_ring *, int); 189 static bool igc_txeof(struct tx_ring *, u_int); 190 static void igc_intr_barrier(struct igc_softc *); 191 static void igc_stop(struct ifnet *, int); 192 static void igc_stop_locked(struct igc_softc *); 193 static int igc_ioctl(struct ifnet *, u_long, void *); 194 #ifdef IF_RXR 195 static int igc_rxrinfo(struct igc_softc *, struct if_rxrinfo *); 196 #endif 197 static void igc_rxfill(struct rx_ring *); 198 static void igc_rxrefill(struct rx_ring *, int); 199 static bool igc_rxeof(struct rx_ring *, u_int); 200 static int igc_rx_checksum(struct igc_queue *, uint64_t, uint32_t, 201 uint32_t); 202 static void igc_watchdog(struct ifnet *); 203 static void igc_tick(void *); 204 static void igc_media_status(struct ifnet *, struct ifmediareq *); 205 static int igc_media_change(struct ifnet *); 206 static int igc_ifflags_cb(struct ethercom *); 207 static void igc_set_filter(struct igc_softc *); 208 static void igc_update_link_status(struct igc_softc *); 209 static int igc_get_buf(struct rx_ring *, int, bool); 210 static int igc_tx_ctx_setup(struct tx_ring *, struct mbuf *, int, 211 uint32_t *, uint32_t *); 212 static int igc_tso_setup(struct tx_ring *, struct mbuf *, int, 213 uint32_t *, uint32_t *); 214 215 static void igc_configure_queues(struct igc_softc *); 216 static void igc_set_queues(struct igc_softc *, uint32_t, uint32_t, int); 217 static void igc_enable_queue(struct igc_softc *, uint32_t); 218 static void igc_enable_intr(struct igc_softc *); 219 static void igc_disable_intr(struct igc_softc *); 220 static int igc_intr_link(void *); 221 static int igc_intr_queue(void *); 222 static int igc_intr(void *); 223 static void igc_handle_queue(void *); 224 static void igc_handle_queue_work(struct work *, void *); 225 static void igc_sched_handle_queue(struct igc_softc *, struct igc_queue *); 226 static void igc_barrier_handle_queue(struct igc_softc *); 227 228 static int igc_allocate_transmit_buffers(struct tx_ring *); 229 static int igc_setup_transmit_structures(struct igc_softc *); 230 static int igc_setup_transmit_ring(struct tx_ring *); 231 static void igc_initialize_transmit_unit(struct igc_softc *); 232 static void igc_free_transmit_structures(struct igc_softc *); 233 static void igc_free_transmit_buffers(struct tx_ring *); 234 static void igc_withdraw_transmit_packets(struct tx_ring *, bool); 235 static int igc_allocate_receive_buffers(struct rx_ring *); 236 static int igc_setup_receive_structures(struct igc_softc *); 237 static int igc_setup_receive_ring(struct rx_ring *); 238 static void igc_initialize_receive_unit(struct igc_softc *); 239 static void igc_free_receive_structures(struct igc_softc *); 240 static void igc_free_receive_buffers(struct rx_ring *); 241 static void igc_clear_receive_status(struct rx_ring *); 242 static void igc_initialize_rss_mapping(struct igc_softc *); 243 244 static void igc_get_hw_control(struct igc_softc *); 245 static void igc_release_hw_control(struct igc_softc *); 246 static int igc_is_valid_ether_addr(uint8_t *); 247 static void igc_print_devinfo(struct igc_softc *); 248 249 CFATTACH_DECL3_NEW(igc, sizeof(struct igc_softc), 250 igc_match, igc_attach, igc_detach, NULL, NULL, NULL, 0); 251 252 static inline int 253 igc_txdesc_incr(struct igc_softc *sc, int id) 254 { 255 256 if (++id == sc->num_tx_desc) 257 id = 0; 258 return id; 259 } 260 261 static inline int __unused 262 igc_txdesc_decr(struct igc_softc *sc, int id) 263 { 264 265 if (--id < 0) 266 id = sc->num_tx_desc - 1; 267 return id; 268 } 269 270 static inline void 271 igc_txdesc_sync(struct tx_ring *txr, int id, int ops) 272 { 273 274 bus_dmamap_sync(txr->txdma.dma_tag, txr->txdma.dma_map, 275 id * sizeof(union igc_adv_tx_desc), sizeof(union igc_adv_tx_desc), 276 ops); 277 } 278 279 static inline int 280 igc_rxdesc_incr(struct igc_softc *sc, int id) 281 { 282 283 if (++id == sc->num_rx_desc) 284 id = 0; 285 return id; 286 } 287 288 static inline int 289 igc_rxdesc_decr(struct igc_softc *sc, int id) 290 { 291 292 if (--id < 0) 293 id = sc->num_rx_desc - 1; 294 return id; 295 } 296 297 static inline void 298 igc_rxdesc_sync(struct rx_ring *rxr, int id, int ops) 299 { 300 301 bus_dmamap_sync(rxr->rxdma.dma_tag, rxr->rxdma.dma_map, 302 id * sizeof(union igc_adv_rx_desc), sizeof(union igc_adv_rx_desc), 303 ops); 304 } 305 306 static const struct igc_product * 307 igc_lookup(const struct pci_attach_args *pa) 308 { 309 const struct igc_product *igcp; 310 311 for (igcp = igc_products; igcp->igcp_name != NULL; igcp++) { 312 if (PCI_VENDOR(pa->pa_id) == igcp->igcp_vendor && 313 PCI_PRODUCT(pa->pa_id) == igcp->igcp_product) 314 return igcp; 315 } 316 return NULL; 317 } 318 319 /********************************************************************* 320 * Device identification routine 321 * 322 * igc_match determines if the driver should be loaded on 323 * adapter based on PCI vendor/device id of the adapter. 324 * 325 * return 0 on success, positive on failure 326 *********************************************************************/ 327 static int 328 igc_match(device_t parent, cfdata_t match, void *aux) 329 { 330 struct pci_attach_args *pa = aux; 331 332 if (igc_lookup(pa) != NULL) 333 return 1; 334 335 return 0; 336 } 337 338 /********************************************************************* 339 * Device initialization routine 340 * 341 * The attach entry point is called when the driver is being loaded. 342 * This routine identifies the type of hardware, allocates all resources 343 * and initializes the hardware. 344 * 345 * return 0 on success, positive on failure 346 *********************************************************************/ 347 static void 348 igc_attach(device_t parent, device_t self, void *aux) 349 { 350 struct pci_attach_args *pa = aux; 351 struct igc_softc *sc = device_private(self); 352 struct igc_hw *hw = &sc->hw; 353 354 const struct igc_product *igcp = igc_lookup(pa); 355 KASSERT(igcp != NULL); 356 357 sc->sc_dev = self; 358 callout_init(&sc->sc_tick_ch, CALLOUT_MPSAFE); 359 callout_setfunc(&sc->sc_tick_ch, igc_tick, sc); 360 sc->sc_core_stopping = false; 361 362 sc->osdep.os_sc = sc; 363 sc->osdep.os_pa = *pa; 364 #ifndef __aarch64__ 365 /* 366 * XXX PR port-arm/57643 367 * 64-bit DMA does not work at least for LX2K with 32/64GB memory. 368 * smmu(4) support may be required. 369 */ 370 if (pci_dma64_available(pa)) { 371 aprint_verbose(", 64-bit DMA"); 372 sc->osdep.os_dmat = pa->pa_dmat64; 373 } else 374 #endif 375 { 376 aprint_verbose(", 32-bit DMA"); 377 sc->osdep.os_dmat = pa->pa_dmat; 378 } 379 380 pci_aprint_devinfo_fancy(pa, "Ethernet controller", igcp->igcp_name, 1); 381 382 /* Determine hardware and mac info */ 383 igc_identify_hardware(sc); 384 385 sc->num_tx_desc = IGC_DEFAULT_TXD; 386 sc->num_rx_desc = IGC_DEFAULT_RXD; 387 388 /* Setup PCI resources */ 389 if (igc_allocate_pci_resources(sc)) { 390 aprint_error_dev(sc->sc_dev, 391 "unable to allocate PCI resources\n"); 392 goto err_pci; 393 } 394 395 if (igc_allocate_interrupts(sc)) { 396 aprint_error_dev(sc->sc_dev, "unable to allocate interrupts\n"); 397 goto err_pci; 398 } 399 400 /* Allocate TX/RX queues */ 401 if (igc_allocate_queues(sc)) { 402 aprint_error_dev(sc->sc_dev, "unable to allocate queues\n"); 403 goto err_alloc_intr; 404 } 405 406 /* Do shared code initialization */ 407 if (igc_setup_init_funcs(hw, true)) { 408 aprint_error_dev(sc->sc_dev, "unable to initialize\n"); 409 goto err_alloc_intr; 410 } 411 412 hw->mac.autoneg = DO_AUTO_NEG; 413 hw->phy.autoneg_wait_to_complete = false; 414 hw->phy.autoneg_advertised = AUTONEG_ADV_DEFAULT; 415 416 /* Copper options. */ 417 if (hw->phy.media_type == igc_media_type_copper) 418 hw->phy.mdix = AUTO_ALL_MODES; 419 420 /* Set the max frame size. */ 421 sc->hw.mac.max_frame_size = 9234; 422 423 /* Allocate multicast array memory. */ 424 sc->mta = kmem_alloc(IGC_MTA_LEN, KM_SLEEP); 425 426 /* Check SOL/IDER usage. */ 427 if (igc_check_reset_block(hw)) { 428 aprint_error_dev(sc->sc_dev, 429 "PHY reset is blocked due to SOL/IDER session\n"); 430 } 431 432 /* Disable Energy Efficient Ethernet. */ 433 sc->hw.dev_spec._i225.eee_disable = true; 434 435 igc_reset_hw(hw); 436 437 /* Make sure we have a good EEPROM before we read from it. */ 438 if (igc_validate_nvm_checksum(hw) < 0) { 439 /* 440 * Some PCI-E parts fail the first check due to 441 * the link being in sleep state, call it again, 442 * if it fails a second time its a real issue. 443 */ 444 if (igc_validate_nvm_checksum(hw) < 0) { 445 aprint_error_dev(sc->sc_dev, 446 "EEPROM checksum invalid\n"); 447 goto err_late; 448 } 449 } 450 451 /* Copy the permanent MAC address out of the EEPROM. */ 452 if (igc_read_mac_addr(hw) < 0) { 453 aprint_error_dev(sc->sc_dev, 454 "unable to read MAC address from EEPROM\n"); 455 goto err_late; 456 } 457 458 if (!igc_is_valid_ether_addr(hw->mac.addr)) { 459 aprint_error_dev(sc->sc_dev, "invalid MAC address\n"); 460 goto err_late; 461 } 462 463 if (igc_setup_interrupts(sc)) 464 goto err_late; 465 466 /* Attach counters. */ 467 igc_attach_counters(sc); 468 469 /* Setup OS specific network interface. */ 470 igc_setup_interface(sc); 471 472 igc_print_devinfo(sc); 473 474 igc_reset(sc); 475 hw->mac.get_link_status = true; 476 igc_update_link_status(sc); 477 478 /* The driver can now take control from firmware. */ 479 igc_get_hw_control(sc); 480 481 aprint_normal_dev(sc->sc_dev, "Ethernet address %s\n", 482 ether_sprintf(sc->hw.mac.addr)); 483 484 if (pmf_device_register(self, NULL, NULL)) 485 pmf_class_network_register(self, &sc->sc_ec.ec_if); 486 else 487 aprint_error_dev(self, "couldn't establish power handler\n"); 488 489 return; 490 491 err_late: 492 igc_release_hw_control(sc); 493 err_alloc_intr: 494 igc_free_interrupts(sc); 495 err_pci: 496 igc_free_pci_resources(sc); 497 kmem_free(sc->mta, IGC_MTA_LEN); 498 } 499 500 /********************************************************************* 501 * Device removal routine 502 * 503 * The detach entry point is called when the driver is being removed. 504 * This routine stops the adapter and deallocates all the resources 505 * that were allocated for driver operation. 506 * 507 * return 0 on success, positive on failure 508 *********************************************************************/ 509 static int 510 igc_detach(device_t self, int flags) 511 { 512 struct igc_softc *sc = device_private(self); 513 struct ifnet *ifp = &sc->sc_ec.ec_if; 514 515 mutex_enter(&sc->sc_core_lock); 516 igc_stop_locked(sc); 517 mutex_exit(&sc->sc_core_lock); 518 519 igc_detach_counters(sc); 520 521 igc_free_queues(sc); 522 523 igc_phy_hw_reset(&sc->hw); 524 igc_release_hw_control(sc); 525 526 ether_ifdetach(ifp); 527 if_detach(ifp); 528 ifmedia_fini(&sc->media); 529 530 igc_free_interrupts(sc); 531 igc_free_pci_resources(sc); 532 kmem_free(sc->mta, IGC_MTA_LEN); 533 534 mutex_destroy(&sc->sc_core_lock); 535 536 return 0; 537 } 538 539 static void 540 igc_identify_hardware(struct igc_softc *sc) 541 { 542 struct igc_osdep *os = &sc->osdep; 543 struct pci_attach_args *pa = &os->os_pa; 544 545 /* Save off the information about this board. */ 546 sc->hw.device_id = PCI_PRODUCT(pa->pa_id); 547 548 /* Do shared code init and setup. */ 549 if (igc_set_mac_type(&sc->hw)) { 550 aprint_error_dev(sc->sc_dev, "unable to identify hardware\n"); 551 return; 552 } 553 } 554 555 static int 556 igc_allocate_pci_resources(struct igc_softc *sc) 557 { 558 struct igc_osdep *os = &sc->osdep; 559 struct pci_attach_args *pa = &os->os_pa; 560 561 /* 562 * Enable bus mastering and memory-mapped I/O for sure. 563 */ 564 pcireg_t csr = 565 pci_conf_read(pa->pa_pc, pa->pa_tag, PCI_COMMAND_STATUS_REG); 566 csr |= PCI_COMMAND_MASTER_ENABLE | PCI_COMMAND_MEM_ENABLE; 567 pci_conf_write(pa->pa_pc, pa->pa_tag, PCI_COMMAND_STATUS_REG, csr); 568 569 const pcireg_t memtype = 570 pci_mapreg_type(pa->pa_pc, pa->pa_tag, IGC_PCIREG); 571 if (pci_mapreg_map(pa, IGC_PCIREG, memtype, 0, &os->os_memt, 572 &os->os_memh, &os->os_membase, &os->os_memsize)) { 573 aprint_error_dev(sc->sc_dev, "unable to map registers\n"); 574 return ENXIO; 575 } 576 577 sc->hw.hw_addr = os->os_membase; 578 sc->hw.back = os; 579 580 return 0; 581 } 582 583 static int __unused 584 igc_adjust_nqueues(struct igc_softc *sc) 585 { 586 struct pci_attach_args *pa = &sc->osdep.os_pa; 587 int nqueues = MIN(IGC_MAX_NQUEUES, ncpu); 588 589 const int nmsix = pci_msix_count(pa->pa_pc, pa->pa_tag); 590 if (nmsix <= 1) 591 nqueues = 1; 592 else if (nmsix < nqueues + 1) 593 nqueues = nmsix - 1; 594 595 return nqueues; 596 } 597 598 static int 599 igc_allocate_interrupts(struct igc_softc *sc) 600 { 601 struct pci_attach_args *pa = &sc->osdep.os_pa; 602 int error; 603 604 #ifndef IGC_DISABLE_MSIX 605 const int nqueues = igc_adjust_nqueues(sc); 606 if (nqueues > 1) { 607 sc->sc_nintrs = nqueues + 1; 608 error = pci_msix_alloc_exact(pa, &sc->sc_intrs, sc->sc_nintrs); 609 if (!error) { 610 sc->sc_nqueues = nqueues; 611 sc->sc_intr_type = PCI_INTR_TYPE_MSIX; 612 return 0; 613 } 614 } 615 #endif 616 617 /* fallback to MSI */ 618 sc->sc_nintrs = sc->sc_nqueues = 1; 619 620 #ifndef IGC_DISABLE_MSI 621 error = pci_msi_alloc_exact(pa, &sc->sc_intrs, sc->sc_nintrs); 622 if (!error) { 623 sc->sc_intr_type = PCI_INTR_TYPE_MSI; 624 return 0; 625 } 626 #endif 627 628 /* fallback to INTx */ 629 630 error = pci_intx_alloc(pa, &sc->sc_intrs); 631 if (!error) { 632 sc->sc_intr_type = PCI_INTR_TYPE_INTX; 633 return 0; 634 } 635 636 return error; 637 } 638 639 static int 640 igc_allocate_queues(struct igc_softc *sc) 641 { 642 device_t dev = sc->sc_dev; 643 int rxconf = 0, txconf = 0; 644 645 /* Allocate the top level queue structs. */ 646 sc->queues = 647 kmem_zalloc(sc->sc_nqueues * sizeof(struct igc_queue), KM_SLEEP); 648 649 /* Allocate the TX ring. */ 650 sc->tx_rings = 651 kmem_zalloc(sc->sc_nqueues * sizeof(struct tx_ring), KM_SLEEP); 652 653 /* Allocate the RX ring. */ 654 sc->rx_rings = 655 kmem_zalloc(sc->sc_nqueues * sizeof(struct rx_ring), KM_SLEEP); 656 657 /* Set up the TX queues. */ 658 for (int iq = 0; iq < sc->sc_nqueues; iq++, txconf++) { 659 struct tx_ring *txr = &sc->tx_rings[iq]; 660 const int tsize = roundup2( 661 sc->num_tx_desc * sizeof(union igc_adv_tx_desc), 662 IGC_DBA_ALIGN); 663 664 txr->sc = sc; 665 txr->txr_igcq = &sc->queues[iq]; 666 txr->me = iq; 667 if (igc_dma_malloc(sc, tsize, &txr->txdma)) { 668 aprint_error_dev(dev, 669 "unable to allocate TX descriptor\n"); 670 goto fail; 671 } 672 txr->tx_base = (union igc_adv_tx_desc *)txr->txdma.dma_vaddr; 673 memset(txr->tx_base, 0, tsize); 674 } 675 676 /* Prepare transmit descriptors and buffers. */ 677 if (igc_setup_transmit_structures(sc)) { 678 aprint_error_dev(dev, "unable to setup transmit structures\n"); 679 goto fail; 680 } 681 682 /* Set up the RX queues. */ 683 for (int iq = 0; iq < sc->sc_nqueues; iq++, rxconf++) { 684 struct rx_ring *rxr = &sc->rx_rings[iq]; 685 const int rsize = roundup2( 686 sc->num_rx_desc * sizeof(union igc_adv_rx_desc), 687 IGC_DBA_ALIGN); 688 689 rxr->sc = sc; 690 rxr->rxr_igcq = &sc->queues[iq]; 691 rxr->me = iq; 692 #ifdef OPENBSD 693 timeout_set(&rxr->rx_refill, igc_rxrefill, rxr); 694 #endif 695 if (igc_dma_malloc(sc, rsize, &rxr->rxdma)) { 696 aprint_error_dev(dev, 697 "unable to allocate RX descriptor\n"); 698 goto fail; 699 } 700 rxr->rx_base = (union igc_adv_rx_desc *)rxr->rxdma.dma_vaddr; 701 memset(rxr->rx_base, 0, rsize); 702 } 703 704 sc->rx_mbuf_sz = MCLBYTES; 705 /* Prepare receive descriptors and buffers. */ 706 if (igc_setup_receive_structures(sc)) { 707 aprint_error_dev(sc->sc_dev, 708 "unable to setup receive structures\n"); 709 goto fail; 710 } 711 712 /* Set up the queue holding structs. */ 713 for (int iq = 0; iq < sc->sc_nqueues; iq++) { 714 struct igc_queue *q = &sc->queues[iq]; 715 716 q->sc = sc; 717 q->txr = &sc->tx_rings[iq]; 718 q->rxr = &sc->rx_rings[iq]; 719 } 720 721 return 0; 722 723 fail: 724 for (struct rx_ring *rxr = sc->rx_rings; rxconf > 0; rxr++, rxconf--) 725 igc_dma_free(sc, &rxr->rxdma); 726 for (struct tx_ring *txr = sc->tx_rings; txconf > 0; txr++, txconf--) 727 igc_dma_free(sc, &txr->txdma); 728 729 kmem_free(sc->rx_rings, sc->sc_nqueues * sizeof(struct rx_ring)); 730 sc->rx_rings = NULL; 731 kmem_free(sc->tx_rings, sc->sc_nqueues * sizeof(struct tx_ring)); 732 sc->tx_rings = NULL; 733 kmem_free(sc->queues, sc->sc_nqueues * sizeof(struct igc_queue)); 734 sc->queues = NULL; 735 736 return ENOMEM; 737 } 738 739 static void 740 igc_free_pci_resources(struct igc_softc *sc) 741 { 742 struct igc_osdep *os = &sc->osdep; 743 744 if (os->os_membase != 0) 745 bus_space_unmap(os->os_memt, os->os_memh, os->os_memsize); 746 os->os_membase = 0; 747 } 748 749 static void 750 igc_free_interrupts(struct igc_softc *sc) 751 { 752 struct pci_attach_args *pa = &sc->osdep.os_pa; 753 pci_chipset_tag_t pc = pa->pa_pc; 754 755 for (int i = 0; i < sc->sc_nintrs; i++) { 756 if (sc->sc_ihs[i] != NULL) { 757 pci_intr_disestablish(pc, sc->sc_ihs[i]); 758 sc->sc_ihs[i] = NULL; 759 } 760 } 761 pci_intr_release(pc, sc->sc_intrs, sc->sc_nintrs); 762 } 763 764 static void 765 igc_free_queues(struct igc_softc *sc) 766 { 767 768 igc_free_receive_structures(sc); 769 for (int iq = 0; iq < sc->sc_nqueues; iq++) { 770 struct rx_ring *rxr = &sc->rx_rings[iq]; 771 772 igc_dma_free(sc, &rxr->rxdma); 773 } 774 775 igc_free_transmit_structures(sc); 776 for (int iq = 0; iq < sc->sc_nqueues; iq++) { 777 struct tx_ring *txr = &sc->tx_rings[iq]; 778 779 igc_dma_free(sc, &txr->txdma); 780 } 781 782 kmem_free(sc->rx_rings, sc->sc_nqueues * sizeof(struct rx_ring)); 783 kmem_free(sc->tx_rings, sc->sc_nqueues * sizeof(struct tx_ring)); 784 kmem_free(sc->queues, sc->sc_nqueues * sizeof(struct igc_queue)); 785 } 786 787 /********************************************************************* 788 * 789 * Initialize the hardware to a configuration as specified by the 790 * adapter structure. 791 * 792 **********************************************************************/ 793 static void 794 igc_reset(struct igc_softc *sc) 795 { 796 struct igc_hw *hw = &sc->hw; 797 798 /* Let the firmware know the OS is in control */ 799 igc_get_hw_control(sc); 800 801 /* 802 * Packet Buffer Allocation (PBA) 803 * Writing PBA sets the receive portion of the buffer 804 * the remainder is used for the transmit buffer. 805 */ 806 const uint32_t pba = IGC_PBA_34K; 807 808 /* 809 * These parameters control the automatic generation (Tx) and 810 * response (Rx) to Ethernet PAUSE frames. 811 * - High water mark should allow for at least two frames to be 812 * received after sending an XOFF. 813 * - Low water mark works best when it is very near the high water mark. 814 * This allows the receiver to restart by sending XON when it has 815 * drained a bit. Here we use an arbitrary value of 1500 which will 816 * restart after one full frame is pulled from the buffer. There 817 * could be several smaller frames in the buffer and if so they will 818 * not trigger the XON until their total number reduces the buffer 819 * by 1500. 820 * - The pause time is fairly large at 1000 x 512ns = 512 usec. 821 */ 822 const uint16_t rx_buffer_size = (pba & 0xffff) << 10; 823 824 hw->fc.high_water = rx_buffer_size - 825 roundup2(sc->hw.mac.max_frame_size, 1024); 826 /* 16-byte granularity */ 827 hw->fc.low_water = hw->fc.high_water - 16; 828 829 if (sc->fc) /* locally set flow control value? */ 830 hw->fc.requested_mode = sc->fc; 831 else 832 hw->fc.requested_mode = igc_fc_full; 833 834 hw->fc.pause_time = IGC_FC_PAUSE_TIME; 835 836 hw->fc.send_xon = true; 837 838 /* Issue a global reset */ 839 igc_reset_hw(hw); 840 IGC_WRITE_REG(hw, IGC_WUC, 0); 841 842 /* and a re-init */ 843 if (igc_init_hw(hw) < 0) { 844 aprint_error_dev(sc->sc_dev, "unable to reset hardware\n"); 845 return; 846 } 847 848 /* Setup DMA Coalescing */ 849 igc_init_dmac(sc, pba); 850 851 IGC_WRITE_REG(hw, IGC_VET, ETHERTYPE_VLAN); 852 igc_get_phy_info(hw); 853 igc_check_for_link(hw); 854 } 855 856 /********************************************************************* 857 * 858 * Initialize the DMA Coalescing feature 859 * 860 **********************************************************************/ 861 static void 862 igc_init_dmac(struct igc_softc *sc, uint32_t pba) 863 { 864 struct igc_hw *hw = &sc->hw; 865 const uint16_t max_frame_size = sc->hw.mac.max_frame_size; 866 uint32_t reg, status; 867 868 if (sc->dmac == 0) { /* Disabling it */ 869 reg = ~IGC_DMACR_DMAC_EN; /* XXXRO */ 870 IGC_WRITE_REG(hw, IGC_DMACR, reg); 871 DPRINTF(MISC, "DMA coalescing disabled\n"); 872 return; 873 } else { 874 device_printf(sc->sc_dev, "DMA coalescing enabled\n"); 875 } 876 877 /* Set starting threshold */ 878 IGC_WRITE_REG(hw, IGC_DMCTXTH, 0); 879 880 uint16_t hwm = 64 * pba - max_frame_size / 16; 881 if (hwm < 64 * (pba - 6)) 882 hwm = 64 * (pba - 6); 883 reg = IGC_READ_REG(hw, IGC_FCRTC); 884 reg &= ~IGC_FCRTC_RTH_COAL_MASK; 885 reg |= (hwm << IGC_FCRTC_RTH_COAL_SHIFT) & IGC_FCRTC_RTH_COAL_MASK; 886 IGC_WRITE_REG(hw, IGC_FCRTC, reg); 887 888 uint32_t dmac = pba - max_frame_size / 512; 889 if (dmac < pba - 10) 890 dmac = pba - 10; 891 reg = IGC_READ_REG(hw, IGC_DMACR); 892 reg &= ~IGC_DMACR_DMACTHR_MASK; 893 reg |= (dmac << IGC_DMACR_DMACTHR_SHIFT) & IGC_DMACR_DMACTHR_MASK; 894 895 /* transition to L0x or L1 if available..*/ 896 reg |= IGC_DMACR_DMAC_EN | IGC_DMACR_DMAC_LX_MASK; 897 898 /* Check if status is 2.5Gb backplane connection 899 * before configuration of watchdog timer, which is 900 * in msec values in 12.8usec intervals 901 * watchdog timer= msec values in 32usec intervals 902 * for non 2.5Gb connection 903 */ 904 status = IGC_READ_REG(hw, IGC_STATUS); 905 if ((status & IGC_STATUS_2P5_SKU) && 906 !(status & IGC_STATUS_2P5_SKU_OVER)) 907 reg |= (sc->dmac * 5) >> 6; 908 else 909 reg |= sc->dmac >> 5; 910 911 IGC_WRITE_REG(hw, IGC_DMACR, reg); 912 913 IGC_WRITE_REG(hw, IGC_DMCRTRH, 0); 914 915 /* Set the interval before transition */ 916 reg = IGC_READ_REG(hw, IGC_DMCTLX); 917 reg |= IGC_DMCTLX_DCFLUSH_DIS; 918 919 /* 920 * in 2.5Gb connection, TTLX unit is 0.4 usec 921 * which is 0x4*2 = 0xA. But delay is still 4 usec 922 */ 923 status = IGC_READ_REG(hw, IGC_STATUS); 924 if ((status & IGC_STATUS_2P5_SKU) && 925 !(status & IGC_STATUS_2P5_SKU_OVER)) 926 reg |= 0xA; 927 else 928 reg |= 0x4; 929 930 IGC_WRITE_REG(hw, IGC_DMCTLX, reg); 931 932 /* free space in tx packet buffer to wake from DMA coal */ 933 IGC_WRITE_REG(hw, IGC_DMCTXTH, 934 (IGC_TXPBSIZE - (2 * max_frame_size)) >> 6); 935 936 /* make low power state decision controlled by DMA coal */ 937 reg = IGC_READ_REG(hw, IGC_PCIEMISC); 938 reg &= ~IGC_PCIEMISC_LX_DECISION; 939 IGC_WRITE_REG(hw, IGC_PCIEMISC, reg); 940 } 941 942 static int 943 igc_setup_interrupts(struct igc_softc *sc) 944 { 945 int error; 946 947 switch (sc->sc_intr_type) { 948 case PCI_INTR_TYPE_MSIX: 949 error = igc_setup_msix(sc); 950 break; 951 case PCI_INTR_TYPE_MSI: 952 error = igc_setup_msi(sc); 953 break; 954 case PCI_INTR_TYPE_INTX: 955 error = igc_setup_intx(sc); 956 break; 957 default: 958 panic("%s: invalid interrupt type: %d", 959 device_xname(sc->sc_dev), sc->sc_intr_type); 960 } 961 962 return error; 963 } 964 965 static void 966 igc_attach_counters(struct igc_softc *sc) 967 { 968 #ifdef IGC_EVENT_COUNTERS 969 970 /* Global counters */ 971 sc->sc_global_evcnts = kmem_zalloc( 972 IGC_GLOBAL_COUNTERS * sizeof(sc->sc_global_evcnts[0]), KM_SLEEP); 973 974 for (int cnt = 0; cnt < IGC_GLOBAL_COUNTERS; cnt++) { 975 evcnt_attach_dynamic(&sc->sc_global_evcnts[cnt], 976 igc_global_counters[cnt].type, NULL, 977 device_xname(sc->sc_dev), igc_global_counters[cnt].name); 978 } 979 980 /* Driver counters */ 981 sc->sc_driver_evcnts = kmem_zalloc( 982 IGC_DRIVER_COUNTERS * sizeof(sc->sc_driver_evcnts[0]), KM_SLEEP); 983 984 for (int cnt = 0; cnt < IGC_DRIVER_COUNTERS; cnt++) { 985 evcnt_attach_dynamic(&sc->sc_driver_evcnts[cnt], 986 igc_driver_counters[cnt].type, NULL, 987 device_xname(sc->sc_dev), igc_driver_counters[cnt].name); 988 } 989 990 for (int iq = 0; iq < sc->sc_nqueues; iq++) { 991 struct igc_queue *q = &sc->queues[iq]; 992 993 q->igcq_driver_counters = kmem_zalloc( 994 IGC_DRIVER_COUNTERS * sizeof(q->igcq_driver_counters[0]), 995 KM_SLEEP); 996 } 997 998 /* Queue counters */ 999 for (int iq = 0; iq < sc->sc_nqueues; iq++) { 1000 struct igc_queue *q = &sc->queues[iq]; 1001 1002 snprintf(q->igcq_queue_evname, sizeof(q->igcq_queue_evname), 1003 "%s q%d", device_xname(sc->sc_dev), iq); 1004 1005 q->igcq_queue_evcnts = kmem_zalloc( 1006 IGC_QUEUE_COUNTERS * sizeof(q->igcq_queue_evcnts[0]), 1007 KM_SLEEP); 1008 1009 for (int cnt = 0; cnt < IGC_QUEUE_COUNTERS; cnt++) { 1010 evcnt_attach_dynamic(&q->igcq_queue_evcnts[cnt], 1011 igc_queue_counters[cnt].type, NULL, 1012 q->igcq_queue_evname, igc_queue_counters[cnt].name); 1013 } 1014 } 1015 1016 /* MAC counters */ 1017 snprintf(sc->sc_mac_evname, sizeof(sc->sc_mac_evname), 1018 "%s Mac Statistics", device_xname(sc->sc_dev)); 1019 1020 sc->sc_mac_evcnts = kmem_zalloc( 1021 IGC_MAC_COUNTERS * sizeof(sc->sc_mac_evcnts[0]), KM_SLEEP); 1022 1023 for (int cnt = 0; cnt < IGC_MAC_COUNTERS; cnt++) { 1024 evcnt_attach_dynamic(&sc->sc_mac_evcnts[cnt], EVCNT_TYPE_MISC, 1025 NULL, sc->sc_mac_evname, igc_mac_counters[cnt].name); 1026 } 1027 #endif 1028 } 1029 1030 static void 1031 igc_detach_counters(struct igc_softc *sc) 1032 { 1033 #ifdef IGC_EVENT_COUNTERS 1034 1035 /* Global counters */ 1036 for (int cnt = 0; cnt < IGC_GLOBAL_COUNTERS; cnt++) 1037 evcnt_detach(&sc->sc_global_evcnts[cnt]); 1038 1039 kmem_free(sc->sc_global_evcnts, 1040 IGC_GLOBAL_COUNTERS * sizeof(sc->sc_global_evcnts)); 1041 1042 /* Driver counters */ 1043 for (int iq = 0; iq < sc->sc_nqueues; iq++) { 1044 struct igc_queue *q = &sc->queues[iq]; 1045 1046 kmem_free(q->igcq_driver_counters, 1047 IGC_DRIVER_COUNTERS * sizeof(q->igcq_driver_counters[0])); 1048 } 1049 1050 for (int cnt = 0; cnt < IGC_DRIVER_COUNTERS; cnt++) 1051 evcnt_detach(&sc->sc_driver_evcnts[cnt]); 1052 1053 kmem_free(sc->sc_driver_evcnts, 1054 IGC_DRIVER_COUNTERS * sizeof(sc->sc_driver_evcnts)); 1055 1056 /* Queue counters */ 1057 for (int iq = 0; iq < sc->sc_nqueues; iq++) { 1058 struct igc_queue *q = &sc->queues[iq]; 1059 1060 for (int cnt = 0; cnt < IGC_QUEUE_COUNTERS; cnt++) 1061 evcnt_detach(&q->igcq_queue_evcnts[cnt]); 1062 1063 kmem_free(q->igcq_queue_evcnts, 1064 IGC_QUEUE_COUNTERS * sizeof(q->igcq_queue_evcnts[0])); 1065 } 1066 1067 /* MAC statistics */ 1068 for (int cnt = 0; cnt < IGC_MAC_COUNTERS; cnt++) 1069 evcnt_detach(&sc->sc_mac_evcnts[cnt]); 1070 1071 kmem_free(sc->sc_mac_evcnts, 1072 IGC_MAC_COUNTERS * sizeof(sc->sc_mac_evcnts[0])); 1073 #endif 1074 } 1075 1076 /* 1077 * XXX 1078 * FreeBSD uses 4-byte-wise read for 64-bit counters, while Linux just 1079 * drops hi words. 1080 */ 1081 static inline uint64_t __unused 1082 igc_read_mac_counter(struct igc_hw *hw, bus_size_t reg, bool is64) 1083 { 1084 uint64_t val; 1085 1086 val = IGC_READ_REG(hw, reg); 1087 if (is64) 1088 val += ((uint64_t)IGC_READ_REG(hw, reg + 4)) << 32; 1089 return val; 1090 } 1091 1092 static void 1093 igc_update_counters(struct igc_softc *sc) 1094 { 1095 #ifdef IGC_EVENT_COUNTERS 1096 1097 /* Global counters: nop */ 1098 1099 /* Driver counters */ 1100 uint64_t sum[IGC_DRIVER_COUNTERS]; 1101 1102 memset(sum, 0, sizeof(sum)); 1103 for (int iq = 0; iq < sc->sc_nqueues; iq++) { 1104 struct igc_queue *q = &sc->queues[iq]; 1105 1106 for (int cnt = 0; cnt < IGC_DRIVER_COUNTERS; cnt++) { 1107 sum[cnt] += IGC_QUEUE_DRIVER_COUNTER_VAL(q, cnt); 1108 IGC_QUEUE_DRIVER_COUNTER_STORE(q, cnt, 0); 1109 } 1110 } 1111 1112 for (int cnt = 0; cnt < IGC_DRIVER_COUNTERS; cnt++) 1113 IGC_DRIVER_COUNTER_ADD(sc, cnt, sum[cnt]); 1114 1115 /* Queue counters: nop */ 1116 1117 /* Mac statistics */ 1118 struct igc_hw *hw = &sc->hw; 1119 1120 for (int cnt = 0; cnt < IGC_MAC_COUNTERS; cnt++) { 1121 IGC_MAC_COUNTER_ADD(sc, cnt, igc_read_mac_counter(hw, 1122 igc_mac_counters[cnt].reg, igc_mac_counters[cnt].is64)); 1123 } 1124 #endif 1125 } 1126 1127 static void 1128 igc_clear_counters(struct igc_softc *sc) 1129 { 1130 #ifdef IGC_EVENT_COUNTERS 1131 1132 /* Global counters */ 1133 for (int cnt = 0; cnt < IGC_GLOBAL_COUNTERS; cnt++) 1134 IGC_GLOBAL_COUNTER_STORE(sc, cnt, 0); 1135 1136 /* Driver counters */ 1137 for (int iq = 0; iq < sc->sc_nqueues; iq++) { 1138 struct igc_queue *q = &sc->queues[iq]; 1139 1140 for (int cnt = 0; cnt < IGC_DRIVER_COUNTERS; cnt++) 1141 IGC_QUEUE_DRIVER_COUNTER_STORE(q, cnt, 0); 1142 } 1143 1144 for (int cnt = 0; cnt < IGC_DRIVER_COUNTERS; cnt++) 1145 IGC_DRIVER_COUNTER_STORE(sc, cnt, 0); 1146 1147 /* Queue counters */ 1148 for (int iq = 0; iq < sc->sc_nqueues; iq++) { 1149 struct igc_queue *q = &sc->queues[iq]; 1150 1151 for (int cnt = 0; cnt < IGC_QUEUE_COUNTERS; cnt++) 1152 IGC_QUEUE_COUNTER_STORE(q, cnt, 0); 1153 } 1154 1155 /* Mac statistics */ 1156 struct igc_hw *hw = &sc->hw; 1157 1158 for (int cnt = 0; cnt < IGC_MAC_COUNTERS; cnt++) { 1159 (void)igc_read_mac_counter(hw, igc_mac_counters[cnt].reg, 1160 igc_mac_counters[cnt].is64); 1161 IGC_MAC_COUNTER_STORE(sc, cnt, 0); 1162 } 1163 #endif 1164 } 1165 1166 static int 1167 igc_setup_msix(struct igc_softc *sc) 1168 { 1169 pci_chipset_tag_t pc = sc->osdep.os_pa.pa_pc; 1170 device_t dev = sc->sc_dev; 1171 pci_intr_handle_t *intrs; 1172 void **ihs; 1173 const char *intrstr; 1174 char intrbuf[PCI_INTRSTR_LEN]; 1175 char xnamebuf[MAX(32, MAXCOMLEN)]; 1176 int iq, error; 1177 1178 for (iq = 0, intrs = sc->sc_intrs, ihs = sc->sc_ihs; 1179 iq < sc->sc_nqueues; iq++, intrs++, ihs++) { 1180 struct igc_queue *q = &sc->queues[iq]; 1181 1182 snprintf(xnamebuf, sizeof(xnamebuf), "%s: txrx %d", 1183 device_xname(dev), iq); 1184 1185 intrstr = pci_intr_string(pc, *intrs, intrbuf, sizeof(intrbuf)); 1186 1187 pci_intr_setattr(pc, intrs, PCI_INTR_MPSAFE, true); 1188 *ihs = pci_intr_establish_xname(pc, *intrs, IPL_NET, 1189 igc_intr_queue, q, xnamebuf); 1190 if (*ihs == NULL) { 1191 aprint_error_dev(dev, 1192 "unable to establish txrx interrupt at %s\n", 1193 intrstr); 1194 return ENOBUFS; 1195 } 1196 aprint_normal_dev(dev, "txrx interrupting at %s\n", intrstr); 1197 1198 kcpuset_t *affinity; 1199 kcpuset_create(&affinity, true); 1200 kcpuset_set(affinity, iq % ncpu); 1201 error = interrupt_distribute(*ihs, affinity, NULL); 1202 if (error) { 1203 aprint_normal_dev(dev, 1204 "%s: unable to change affinity, use default CPU\n", 1205 intrstr); 1206 } 1207 kcpuset_destroy(affinity); 1208 1209 q->igcq_si = softint_establish(SOFTINT_NET | SOFTINT_MPSAFE, 1210 igc_handle_queue, q); 1211 if (q->igcq_si == NULL) { 1212 aprint_error_dev(dev, 1213 "%s: unable to establish softint\n", intrstr); 1214 return ENOBUFS; 1215 } 1216 1217 q->msix = iq; 1218 q->eims = 1 << iq; 1219 } 1220 1221 snprintf(xnamebuf, MAXCOMLEN, "%s_tx_rx", device_xname(dev)); 1222 error = workqueue_create(&sc->sc_queue_wq, xnamebuf, 1223 igc_handle_queue_work, sc, IGC_WORKQUEUE_PRI, IPL_NET, 1224 WQ_PERCPU | WQ_MPSAFE); 1225 if (error) { 1226 aprint_error_dev(dev, "workqueue_create failed\n"); 1227 return ENOBUFS; 1228 } 1229 sc->sc_txrx_workqueue = false; 1230 1231 intrstr = pci_intr_string(pc, *intrs, intrbuf, sizeof(intrbuf)); 1232 snprintf(xnamebuf, sizeof(xnamebuf), "%s: link", device_xname(dev)); 1233 pci_intr_setattr(pc, intrs, PCI_INTR_MPSAFE, true); 1234 *ihs = pci_intr_establish_xname(pc, *intrs, IPL_NET, 1235 igc_intr_link, sc, xnamebuf); 1236 if (*ihs == NULL) { 1237 aprint_error_dev(dev, 1238 "unable to establish link interrupt at %s\n", intrstr); 1239 return ENOBUFS; 1240 } 1241 aprint_normal_dev(dev, "link interrupting at %s\n", intrstr); 1242 /* use later in igc_configure_queues() */ 1243 sc->linkvec = iq; 1244 1245 return 0; 1246 } 1247 1248 static int 1249 igc_setup_msi(struct igc_softc *sc) 1250 { 1251 pci_chipset_tag_t pc = sc->osdep.os_pa.pa_pc; 1252 device_t dev = sc->sc_dev; 1253 pci_intr_handle_t *intr = sc->sc_intrs; 1254 void **ihs = sc->sc_ihs; 1255 const char *intrstr; 1256 char intrbuf[PCI_INTRSTR_LEN]; 1257 char xnamebuf[MAX(32, MAXCOMLEN)]; 1258 int error; 1259 1260 intrstr = pci_intr_string(pc, *intr, intrbuf, sizeof(intrbuf)); 1261 1262 snprintf(xnamebuf, sizeof(xnamebuf), "%s: msi", device_xname(dev)); 1263 pci_intr_setattr(pc, intr, PCI_INTR_MPSAFE, true); 1264 *ihs = pci_intr_establish_xname(pc, *intr, IPL_NET, 1265 igc_intr, sc, xnamebuf); 1266 if (*ihs == NULL) { 1267 aprint_error_dev(dev, 1268 "unable to establish interrupt at %s\n", intrstr); 1269 return ENOBUFS; 1270 } 1271 aprint_normal_dev(dev, "interrupting at %s\n", intrstr); 1272 1273 struct igc_queue *iq = sc->queues; 1274 iq->igcq_si = softint_establish(SOFTINT_NET | SOFTINT_MPSAFE, 1275 igc_handle_queue, iq); 1276 if (iq->igcq_si == NULL) { 1277 aprint_error_dev(dev, 1278 "%s: unable to establish softint\n", intrstr); 1279 return ENOBUFS; 1280 } 1281 1282 snprintf(xnamebuf, MAXCOMLEN, "%s_tx_rx", device_xname(dev)); 1283 error = workqueue_create(&sc->sc_queue_wq, xnamebuf, 1284 igc_handle_queue_work, sc, IGC_WORKQUEUE_PRI, IPL_NET, 1285 WQ_PERCPU | WQ_MPSAFE); 1286 if (error) { 1287 aprint_error_dev(dev, "workqueue_create failed\n"); 1288 return ENOBUFS; 1289 } 1290 sc->sc_txrx_workqueue = false; 1291 1292 sc->queues[0].msix = 0; 1293 sc->linkvec = 0; 1294 1295 return 0; 1296 } 1297 1298 static int 1299 igc_setup_intx(struct igc_softc *sc) 1300 { 1301 pci_chipset_tag_t pc = sc->osdep.os_pa.pa_pc; 1302 device_t dev = sc->sc_dev; 1303 pci_intr_handle_t *intr = sc->sc_intrs; 1304 void **ihs = sc->sc_ihs; 1305 const char *intrstr; 1306 char intrbuf[PCI_INTRSTR_LEN]; 1307 char xnamebuf[32]; 1308 1309 intrstr = pci_intr_string(pc, *intr, intrbuf, sizeof(intrbuf)); 1310 1311 snprintf(xnamebuf, sizeof(xnamebuf), "%s:intx", device_xname(dev)); 1312 pci_intr_setattr(pc, intr, PCI_INTR_MPSAFE, true); 1313 *ihs = pci_intr_establish_xname(pc, *intr, IPL_NET, 1314 igc_intr, sc, xnamebuf); 1315 if (*ihs == NULL) { 1316 aprint_error_dev(dev, 1317 "unable to establish interrupt at %s\n", intrstr); 1318 return ENOBUFS; 1319 } 1320 aprint_normal_dev(dev, "interrupting at %s\n", intrstr); 1321 1322 struct igc_queue *iq = sc->queues; 1323 iq->igcq_si = softint_establish(SOFTINT_NET | SOFTINT_MPSAFE, 1324 igc_handle_queue, iq); 1325 if (iq->igcq_si == NULL) { 1326 aprint_error_dev(dev, 1327 "%s: unable to establish softint\n", intrstr); 1328 return ENOBUFS; 1329 } 1330 1331 /* create workqueue? */ 1332 sc->sc_txrx_workqueue = false; 1333 1334 sc->queues[0].msix = 0; 1335 sc->linkvec = 0; 1336 1337 return 0; 1338 } 1339 1340 static int 1341 igc_dma_malloc(struct igc_softc *sc, bus_size_t size, struct igc_dma_alloc *dma) 1342 { 1343 struct igc_osdep *os = &sc->osdep; 1344 1345 dma->dma_tag = os->os_dmat; 1346 1347 if (bus_dmamap_create(dma->dma_tag, size, 1, size, 0, 1348 BUS_DMA_WAITOK | BUS_DMA_ALLOCNOW, &dma->dma_map)) 1349 return 1; 1350 if (bus_dmamem_alloc(dma->dma_tag, size, PAGE_SIZE, 0, &dma->dma_seg, 1351 1, &dma->dma_nseg, BUS_DMA_WAITOK)) 1352 goto destroy; 1353 /* 1354 * XXXRO 1355 * 1356 * Coherent mapping for descriptors is required for now. 1357 * 1358 * Both TX and RX descriptors are 16-byte length, which is shorter 1359 * than dcache lines on modern CPUs. Therefore, sync for a descriptor 1360 * may overwrite DMA read for descriptors in the same cache line. 1361 * 1362 * Can't we avoid this by use cache-line-aligned descriptors at once? 1363 */ 1364 if (bus_dmamem_map(dma->dma_tag, &dma->dma_seg, dma->dma_nseg, size, 1365 &dma->dma_vaddr, BUS_DMA_WAITOK | BUS_DMA_COHERENT /* XXXRO */)) 1366 goto free; 1367 if (bus_dmamap_load(dma->dma_tag, dma->dma_map, dma->dma_vaddr, size, 1368 NULL, BUS_DMA_WAITOK)) 1369 goto unmap; 1370 1371 dma->dma_size = size; 1372 1373 return 0; 1374 unmap: 1375 bus_dmamem_unmap(dma->dma_tag, dma->dma_vaddr, size); 1376 free: 1377 bus_dmamem_free(dma->dma_tag, &dma->dma_seg, dma->dma_nseg); 1378 destroy: 1379 bus_dmamap_destroy(dma->dma_tag, dma->dma_map); 1380 dma->dma_map = NULL; 1381 dma->dma_tag = NULL; 1382 return 1; 1383 } 1384 1385 static void 1386 igc_dma_free(struct igc_softc *sc, struct igc_dma_alloc *dma) 1387 { 1388 1389 if (dma->dma_tag == NULL) 1390 return; 1391 1392 if (dma->dma_map != NULL) { 1393 bus_dmamap_sync(dma->dma_tag, dma->dma_map, 0, 1394 dma->dma_map->dm_mapsize, 1395 BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE); 1396 bus_dmamap_unload(dma->dma_tag, dma->dma_map); 1397 bus_dmamem_unmap(dma->dma_tag, dma->dma_vaddr, dma->dma_size); 1398 bus_dmamem_free(dma->dma_tag, &dma->dma_seg, dma->dma_nseg); 1399 bus_dmamap_destroy(dma->dma_tag, dma->dma_map); 1400 dma->dma_map = NULL; 1401 } 1402 } 1403 1404 /********************************************************************* 1405 * 1406 * Setup networking device structure and register an interface. 1407 * 1408 **********************************************************************/ 1409 static void 1410 igc_setup_interface(struct igc_softc *sc) 1411 { 1412 struct ifnet *ifp = &sc->sc_ec.ec_if; 1413 1414 strlcpy(ifp->if_xname, device_xname(sc->sc_dev), sizeof(ifp->if_xname)); 1415 ifp->if_softc = sc; 1416 ifp->if_flags = IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST; 1417 ifp->if_extflags = IFEF_MPSAFE; 1418 ifp->if_ioctl = igc_ioctl; 1419 ifp->if_start = igc_start; 1420 if (sc->sc_nqueues > 1) 1421 ifp->if_transmit = igc_transmit; 1422 ifp->if_watchdog = igc_watchdog; 1423 ifp->if_init = igc_init; 1424 ifp->if_stop = igc_stop; 1425 1426 #if 0 /* notyet */ 1427 ifp->if_capabilities = IFCAP_TSOv4 | IFCAP_TSOv6; 1428 #endif 1429 1430 ifp->if_capabilities |= 1431 IFCAP_CSUM_IPv4_Tx | IFCAP_CSUM_IPv4_Rx | 1432 IFCAP_CSUM_TCPv4_Tx | IFCAP_CSUM_TCPv4_Rx | 1433 IFCAP_CSUM_UDPv4_Tx | IFCAP_CSUM_UDPv4_Rx | 1434 IFCAP_CSUM_TCPv6_Tx | IFCAP_CSUM_TCPv6_Rx | 1435 IFCAP_CSUM_UDPv6_Tx | IFCAP_CSUM_UDPv6_Rx; 1436 1437 ifp->if_capenable = 0; 1438 1439 sc->sc_ec.ec_capabilities |= 1440 ETHERCAP_JUMBO_MTU | ETHERCAP_VLAN_MTU; 1441 1442 IFQ_SET_MAXLEN(&ifp->if_snd, sc->num_tx_desc - 1); 1443 IFQ_SET_READY(&ifp->if_snd); 1444 1445 #if NVLAN > 0 1446 sc->sc_ec.ec_capabilities |= ETHERCAP_VLAN_HWTAGGING; 1447 #endif 1448 1449 mutex_init(&sc->sc_core_lock, MUTEX_DEFAULT, IPL_NET); 1450 1451 /* Initialize ifmedia structures. */ 1452 sc->sc_ec.ec_ifmedia = &sc->media; 1453 ifmedia_init_with_lock(&sc->media, IFM_IMASK, igc_media_change, 1454 igc_media_status, &sc->sc_core_lock); 1455 ifmedia_add(&sc->media, IFM_ETHER | IFM_10_T, 0, NULL); 1456 ifmedia_add(&sc->media, IFM_ETHER | IFM_10_T | IFM_FDX, 0, NULL); 1457 ifmedia_add(&sc->media, IFM_ETHER | IFM_100_TX, 0, NULL); 1458 ifmedia_add(&sc->media, IFM_ETHER | IFM_100_TX | IFM_FDX, 0, NULL); 1459 ifmedia_add(&sc->media, IFM_ETHER | IFM_1000_T | IFM_FDX, 0, NULL); 1460 ifmedia_add(&sc->media, IFM_ETHER | IFM_2500_T | IFM_FDX, 0, NULL); 1461 ifmedia_add(&sc->media, IFM_ETHER | IFM_AUTO, 0, NULL); 1462 ifmedia_set(&sc->media, IFM_ETHER | IFM_AUTO); 1463 1464 sc->sc_rx_intr_process_limit = IGC_RX_INTR_PROCESS_LIMIT_DEFAULT; 1465 sc->sc_tx_intr_process_limit = IGC_TX_INTR_PROCESS_LIMIT_DEFAULT; 1466 sc->sc_rx_process_limit = IGC_RX_PROCESS_LIMIT_DEFAULT; 1467 sc->sc_tx_process_limit = IGC_TX_PROCESS_LIMIT_DEFAULT; 1468 1469 if_initialize(ifp); 1470 sc->sc_ipq = if_percpuq_create(ifp); 1471 if_deferred_start_init(ifp, NULL); 1472 ether_ifattach(ifp, sc->hw.mac.addr); 1473 ether_set_ifflags_cb(&sc->sc_ec, igc_ifflags_cb); 1474 if_register(ifp); 1475 } 1476 1477 static int 1478 igc_init(struct ifnet *ifp) 1479 { 1480 struct igc_softc *sc = ifp->if_softc; 1481 int error; 1482 1483 mutex_enter(&sc->sc_core_lock); 1484 error = igc_init_locked(sc); 1485 mutex_exit(&sc->sc_core_lock); 1486 1487 return error; 1488 } 1489 1490 static int 1491 igc_init_locked(struct igc_softc *sc) 1492 { 1493 struct ethercom *ec = &sc->sc_ec; 1494 struct ifnet *ifp = &ec->ec_if; 1495 1496 DPRINTF(CFG, "called\n"); 1497 1498 KASSERT(mutex_owned(&sc->sc_core_lock)); 1499 1500 if (ISSET(ifp->if_flags, IFF_RUNNING)) 1501 igc_stop_locked(sc); 1502 1503 /* Put the address into the receive address array. */ 1504 igc_rar_set(&sc->hw, sc->hw.mac.addr, 0); 1505 1506 /* Initialize the hardware. */ 1507 igc_reset(sc); 1508 igc_update_link_status(sc); 1509 1510 /* Setup VLAN support, basic and offload if available. */ 1511 IGC_WRITE_REG(&sc->hw, IGC_VET, ETHERTYPE_VLAN); 1512 1513 igc_initialize_transmit_unit(sc); 1514 igc_initialize_receive_unit(sc); 1515 1516 if (ec->ec_capenable & ETHERCAP_VLAN_HWTAGGING) { 1517 uint32_t ctrl = IGC_READ_REG(&sc->hw, IGC_CTRL); 1518 ctrl |= IGC_CTRL_VME; 1519 IGC_WRITE_REG(&sc->hw, IGC_CTRL, ctrl); 1520 } 1521 1522 /* Setup multicast table. */ 1523 igc_set_filter(sc); 1524 1525 igc_clear_hw_cntrs_base_generic(&sc->hw); 1526 1527 if (sc->sc_intr_type == PCI_INTR_TYPE_MSIX) 1528 igc_configure_queues(sc); 1529 1530 /* This clears any pending interrupts */ 1531 IGC_READ_REG(&sc->hw, IGC_ICR); 1532 IGC_WRITE_REG(&sc->hw, IGC_ICS, IGC_ICS_LSC); 1533 1534 /* The driver can now take control from firmware. */ 1535 igc_get_hw_control(sc); 1536 1537 /* Set Energy Efficient Ethernet. */ 1538 igc_set_eee_i225(&sc->hw, true, true, true); 1539 1540 for (int iq = 0; iq < sc->sc_nqueues; iq++) { 1541 struct rx_ring *rxr = &sc->rx_rings[iq]; 1542 1543 mutex_enter(&rxr->rxr_lock); 1544 igc_rxfill(rxr); 1545 mutex_exit(&rxr->rxr_lock); 1546 } 1547 1548 sc->sc_core_stopping = false; 1549 1550 ifp->if_flags |= IFF_RUNNING; 1551 1552 /* Save last flags for the callback */ 1553 sc->sc_if_flags = ifp->if_flags; 1554 1555 callout_schedule(&sc->sc_tick_ch, hz); 1556 1557 igc_enable_intr(sc); 1558 1559 return 0; 1560 } 1561 1562 static inline int 1563 igc_load_mbuf(struct igc_queue *q, bus_dma_tag_t dmat, bus_dmamap_t map, 1564 struct mbuf *m) 1565 { 1566 int error; 1567 1568 error = bus_dmamap_load_mbuf(dmat, map, m, 1569 BUS_DMA_WRITE | BUS_DMA_NOWAIT); 1570 1571 if (__predict_false(error == EFBIG)) { 1572 IGC_DRIVER_EVENT(q, txdma_efbig, 1); 1573 m = m_defrag(m, M_NOWAIT); 1574 if (__predict_false(m == NULL)) { 1575 IGC_DRIVER_EVENT(q, txdma_defrag, 1); 1576 return ENOBUFS; 1577 } 1578 error = bus_dmamap_load_mbuf(dmat, map, m, 1579 BUS_DMA_WRITE | BUS_DMA_NOWAIT); 1580 } 1581 1582 switch (error) { 1583 case 0: 1584 break; 1585 case ENOMEM: 1586 IGC_DRIVER_EVENT(q, txdma_enomem, 1); 1587 break; 1588 case EINVAL: 1589 IGC_DRIVER_EVENT(q, txdma_einval, 1); 1590 break; 1591 case EAGAIN: 1592 IGC_DRIVER_EVENT(q, txdma_eagain, 1); 1593 break; 1594 default: 1595 IGC_DRIVER_EVENT(q, txdma_other, 1); 1596 break; 1597 } 1598 1599 return error; 1600 } 1601 1602 #define IGC_TX_START 1 1603 #define IGC_TX_TRANSMIT 2 1604 1605 static void 1606 igc_start(struct ifnet *ifp) 1607 { 1608 struct igc_softc *sc = ifp->if_softc; 1609 1610 if (__predict_false(!sc->link_active)) { 1611 IFQ_PURGE(&ifp->if_snd); 1612 return; 1613 } 1614 1615 struct tx_ring *txr = &sc->tx_rings[0]; /* queue 0 */ 1616 mutex_enter(&txr->txr_lock); 1617 igc_tx_common_locked(ifp, txr, IGC_TX_START); 1618 mutex_exit(&txr->txr_lock); 1619 } 1620 1621 static inline u_int 1622 igc_select_txqueue(struct igc_softc *sc, struct mbuf *m __unused) 1623 { 1624 const u_int cpuid = cpu_index(curcpu()); 1625 1626 return cpuid % sc->sc_nqueues; 1627 } 1628 1629 static int 1630 igc_transmit(struct ifnet *ifp, struct mbuf *m) 1631 { 1632 struct igc_softc *sc = ifp->if_softc; 1633 const u_int qid = igc_select_txqueue(sc, m); 1634 struct tx_ring *txr = &sc->tx_rings[qid]; 1635 struct igc_queue *q = txr->txr_igcq; 1636 1637 if (__predict_false(!pcq_put(txr->txr_interq, m))) { 1638 IGC_QUEUE_EVENT(q, tx_pcq_drop, 1); 1639 m_freem(m); 1640 return ENOBUFS; 1641 } 1642 1643 mutex_enter(&txr->txr_lock); 1644 igc_tx_common_locked(ifp, txr, IGC_TX_TRANSMIT); 1645 mutex_exit(&txr->txr_lock); 1646 1647 return 0; 1648 } 1649 1650 static void 1651 igc_tx_common_locked(struct ifnet *ifp, struct tx_ring *txr, int caller) 1652 { 1653 struct igc_softc *sc = ifp->if_softc; 1654 struct igc_queue *q = txr->txr_igcq; 1655 net_stat_ref_t nsr = IF_STAT_GETREF(ifp); 1656 int prod, free, last = -1; 1657 bool post = false; 1658 1659 prod = txr->next_avail_desc; 1660 free = txr->next_to_clean; 1661 if (free <= prod) 1662 free += sc->num_tx_desc; 1663 free -= prod; 1664 1665 DPRINTF(TX, "%s: begin: msix %d prod %d n2c %d free %d\n", 1666 caller == IGC_TX_TRANSMIT ? "transmit" : "start", 1667 txr->me, prod, txr->next_to_clean, free); 1668 1669 for (;;) { 1670 struct mbuf *m; 1671 1672 if (__predict_false(free <= IGC_MAX_SCATTER)) { 1673 IGC_QUEUE_EVENT(q, tx_no_desc, 1); 1674 break; 1675 } 1676 1677 if (caller == IGC_TX_TRANSMIT) 1678 m = pcq_get(txr->txr_interq); 1679 else 1680 IFQ_DEQUEUE(&ifp->if_snd, m); 1681 if (__predict_false(m == NULL)) 1682 break; 1683 1684 struct igc_tx_buf *txbuf = &txr->tx_buffers[prod]; 1685 bus_dmamap_t map = txbuf->map; 1686 1687 if (__predict_false( 1688 igc_load_mbuf(q, txr->txdma.dma_tag, map, m))) { 1689 if (caller == IGC_TX_TRANSMIT) 1690 IGC_QUEUE_EVENT(q, tx_pcq_drop, 1); 1691 m_freem(m); 1692 if_statinc_ref(nsr, if_oerrors); 1693 continue; 1694 } 1695 1696 bus_dmamap_sync(txr->txdma.dma_tag, map, 0, 1697 map->dm_mapsize, BUS_DMASYNC_PREWRITE); 1698 1699 uint32_t ctx_cmd_type_len = 0, olinfo_status = 0; 1700 if (igc_tx_ctx_setup(txr, m, prod, &ctx_cmd_type_len, 1701 &olinfo_status)) { 1702 IGC_QUEUE_EVENT(q, tx_ctx, 1); 1703 /* Consume the first descriptor */ 1704 prod = igc_txdesc_incr(sc, prod); 1705 free--; 1706 } 1707 for (int i = 0; i < map->dm_nsegs; i++) { 1708 union igc_adv_tx_desc *txdesc = &txr->tx_base[prod]; 1709 1710 uint32_t cmd_type_len = ctx_cmd_type_len | 1711 IGC_ADVTXD_DCMD_IFCS | IGC_ADVTXD_DTYP_DATA | 1712 IGC_ADVTXD_DCMD_DEXT | map->dm_segs[i].ds_len; 1713 if (i == map->dm_nsegs - 1) { 1714 cmd_type_len |= 1715 IGC_ADVTXD_DCMD_EOP | IGC_ADVTXD_DCMD_RS; 1716 } 1717 1718 igc_txdesc_sync(txr, prod, 1719 BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE); 1720 htolem64(&txdesc->read.buffer_addr, 1721 map->dm_segs[i].ds_addr); 1722 htolem32(&txdesc->read.cmd_type_len, cmd_type_len); 1723 htolem32(&txdesc->read.olinfo_status, olinfo_status); 1724 igc_txdesc_sync(txr, prod, 1725 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); 1726 1727 last = prod; 1728 prod = igc_txdesc_incr(sc, prod); 1729 } 1730 1731 txbuf->m_head = m; 1732 txbuf->eop_index = last; 1733 1734 bpf_mtap(ifp, m, BPF_D_OUT); 1735 1736 if_statadd_ref(nsr, if_obytes, m->m_pkthdr.len); 1737 if (m->m_flags & M_MCAST) 1738 if_statinc_ref(nsr, if_omcasts); 1739 IGC_QUEUE_EVENT(q, tx_packets, 1); 1740 IGC_QUEUE_EVENT(q, tx_bytes, m->m_pkthdr.len); 1741 1742 free -= map->dm_nsegs; 1743 post = true; 1744 } 1745 1746 if (post) { 1747 txr->next_avail_desc = prod; 1748 IGC_WRITE_REG(&sc->hw, IGC_TDT(txr->me), prod); 1749 } 1750 1751 DPRINTF(TX, "%s: done : msix %d prod %d n2c %d free %d\n", 1752 caller == IGC_TX_TRANSMIT ? "transmit" : "start", 1753 txr->me, prod, txr->next_to_clean, free); 1754 1755 IF_STAT_PUTREF(ifp); 1756 } 1757 1758 static bool 1759 igc_txeof(struct tx_ring *txr, u_int limit) 1760 { 1761 struct igc_softc *sc = txr->sc; 1762 struct ifnet *ifp = &sc->sc_ec.ec_if; 1763 int cons, prod; 1764 bool more = false; 1765 1766 prod = txr->next_avail_desc; 1767 cons = txr->next_to_clean; 1768 1769 if (cons == prod) { 1770 DPRINTF(TX, "false: msix %d cons %d prod %d\n", 1771 txr->me, cons, prod); 1772 return false; 1773 } 1774 1775 do { 1776 struct igc_tx_buf *txbuf = &txr->tx_buffers[cons]; 1777 const int last = txbuf->eop_index; 1778 1779 membar_consumer(); /* XXXRO necessary? */ 1780 1781 KASSERT(last != -1); 1782 union igc_adv_tx_desc *txdesc = &txr->tx_base[last]; 1783 igc_txdesc_sync(txr, last, BUS_DMASYNC_POSTREAD); 1784 const uint32_t status = le32toh(txdesc->wb.status); 1785 igc_txdesc_sync(txr, last, BUS_DMASYNC_PREREAD); 1786 1787 if (!(status & IGC_TXD_STAT_DD)) 1788 break; 1789 1790 if (limit-- == 0) { 1791 more = true; 1792 DPRINTF(TX, "pending TX " 1793 "msix %d cons %d last %d prod %d " 1794 "status 0x%08x\n", 1795 txr->me, cons, last, prod, status); 1796 break; 1797 } 1798 1799 DPRINTF(TX, "handled TX " 1800 "msix %d cons %d last %d prod %d " 1801 "status 0x%08x\n", 1802 txr->me, cons, last, prod, status); 1803 1804 if_statinc(ifp, if_opackets); 1805 1806 bus_dmamap_t map = txbuf->map; 1807 bus_dmamap_sync(txr->txdma.dma_tag, map, 0, map->dm_mapsize, 1808 BUS_DMASYNC_POSTWRITE); 1809 bus_dmamap_unload(txr->txdma.dma_tag, map); 1810 m_freem(txbuf->m_head); 1811 1812 txbuf->m_head = NULL; 1813 txbuf->eop_index = -1; 1814 1815 cons = igc_txdesc_incr(sc, last); 1816 } while (cons != prod); 1817 1818 txr->next_to_clean = cons; 1819 1820 return more; 1821 } 1822 1823 static void 1824 igc_intr_barrier(struct igc_softc *sc __unused) 1825 { 1826 1827 xc_barrier(0); 1828 } 1829 1830 static void 1831 igc_stop(struct ifnet *ifp, int disable) 1832 { 1833 struct igc_softc *sc = ifp->if_softc; 1834 1835 mutex_enter(&sc->sc_core_lock); 1836 igc_stop_locked(sc); 1837 mutex_exit(&sc->sc_core_lock); 1838 } 1839 1840 /********************************************************************* 1841 * 1842 * This routine disables all traffic on the adapter by issuing a 1843 * global reset on the MAC. 1844 * 1845 **********************************************************************/ 1846 static void 1847 igc_stop_locked(struct igc_softc *sc) 1848 { 1849 struct ifnet *ifp = &sc->sc_ec.ec_if; 1850 1851 DPRINTF(CFG, "called\n"); 1852 1853 KASSERT(mutex_owned(&sc->sc_core_lock)); 1854 1855 /* 1856 * If stopping processing has already started, do nothing. 1857 */ 1858 if ((ifp->if_flags & IFF_RUNNING) == 0) 1859 return; 1860 1861 /* Tell the stack that the interface is no longer active. */ 1862 ifp->if_flags &= ~IFF_RUNNING; 1863 1864 /* 1865 * igc_handle_queue() can enable interrupts, so wait for completion of 1866 * last igc_handle_queue() after unset IFF_RUNNING. 1867 */ 1868 mutex_exit(&sc->sc_core_lock); 1869 igc_barrier_handle_queue(sc); 1870 mutex_enter(&sc->sc_core_lock); 1871 1872 sc->sc_core_stopping = true; 1873 1874 igc_disable_intr(sc); 1875 1876 callout_halt(&sc->sc_tick_ch, &sc->sc_core_lock); 1877 1878 igc_reset_hw(&sc->hw); 1879 IGC_WRITE_REG(&sc->hw, IGC_WUC, 0); 1880 1881 /* 1882 * Wait for completion of interrupt handlers. 1883 */ 1884 mutex_exit(&sc->sc_core_lock); 1885 igc_intr_barrier(sc); 1886 mutex_enter(&sc->sc_core_lock); 1887 1888 igc_update_link_status(sc); 1889 1890 for (int iq = 0; iq < sc->sc_nqueues; iq++) { 1891 struct tx_ring *txr = &sc->tx_rings[iq]; 1892 1893 igc_withdraw_transmit_packets(txr, false); 1894 } 1895 1896 for (int iq = 0; iq < sc->sc_nqueues; iq++) { 1897 struct rx_ring *rxr = &sc->rx_rings[iq]; 1898 1899 igc_clear_receive_status(rxr); 1900 } 1901 1902 /* Save last flags for the callback */ 1903 sc->sc_if_flags = ifp->if_flags; 1904 } 1905 1906 /********************************************************************* 1907 * Ioctl entry point 1908 * 1909 * igc_ioctl is called when the user wants to configure the 1910 * interface. 1911 * 1912 * return 0 on success, positive on failure 1913 **********************************************************************/ 1914 static int 1915 igc_ioctl(struct ifnet * ifp, u_long cmd, void *data) 1916 { 1917 struct igc_softc *sc __unused = ifp->if_softc; 1918 int s; 1919 int error; 1920 1921 DPRINTF(CFG, "cmd 0x%016lx\n", cmd); 1922 1923 switch (cmd) { 1924 case SIOCADDMULTI: 1925 case SIOCDELMULTI: 1926 break; 1927 default: 1928 KASSERT(IFNET_LOCKED(ifp)); 1929 } 1930 1931 if (cmd == SIOCZIFDATA) { 1932 mutex_enter(&sc->sc_core_lock); 1933 igc_clear_counters(sc); 1934 mutex_exit(&sc->sc_core_lock); 1935 } 1936 1937 switch (cmd) { 1938 #ifdef IF_RXR 1939 case SIOCGIFRXR: 1940 s = splnet(); 1941 error = igc_rxrinfo(sc, (struct if_rxrinfo *)ifr->ifr_data); 1942 splx(s); 1943 break; 1944 #endif 1945 default: 1946 s = splnet(); 1947 error = ether_ioctl(ifp, cmd, data); 1948 splx(s); 1949 break; 1950 } 1951 1952 if (error != ENETRESET) 1953 return error; 1954 1955 error = 0; 1956 1957 if (cmd == SIOCSIFCAP) 1958 error = if_init(ifp); 1959 else if ((cmd == SIOCADDMULTI) || (cmd == SIOCDELMULTI)) { 1960 mutex_enter(&sc->sc_core_lock); 1961 if (sc->sc_if_flags & IFF_RUNNING) { 1962 /* 1963 * Multicast list has changed; set the hardware filter 1964 * accordingly. 1965 */ 1966 igc_disable_intr(sc); 1967 igc_set_filter(sc); 1968 igc_enable_intr(sc); 1969 } 1970 mutex_exit(&sc->sc_core_lock); 1971 } 1972 1973 return error; 1974 } 1975 1976 #ifdef IF_RXR 1977 static int 1978 igc_rxrinfo(struct igc_softc *sc, struct if_rxrinfo *ifri) 1979 { 1980 struct if_rxring_info *ifr, ifr1; 1981 int error; 1982 1983 if (sc->sc_nqueues > 1) { 1984 ifr = kmem_zalloc(sc->sc_nqueues * sizeof(*ifr), KM_SLEEP); 1985 } else { 1986 ifr = &ifr1; 1987 memset(ifr, 0, sizeof(*ifr)); 1988 } 1989 1990 for (int iq = 0; iq < sc->sc_nqueues; iq++) { 1991 struct rx_ring *rxr = &sc->rx_rings[iq]; 1992 1993 ifr[iq].ifr_size = MCLBYTES; 1994 snprintf(ifr[iq].ifr_name, sizeof(ifr[iq].ifr_name), "%d", iq); 1995 ifr[iq].ifr_info = rxr->rx_ring; 1996 } 1997 1998 error = if_rxr_info_ioctl(ifri, sc->sc_nqueues, ifr); 1999 if (sc->sc_nqueues > 1) 2000 kmem_free(ifr, sc->sc_nqueues * sizeof(*ifr)); 2001 2002 return error; 2003 } 2004 #endif 2005 2006 static void 2007 igc_rxfill(struct rx_ring *rxr) 2008 { 2009 struct igc_softc *sc = rxr->sc; 2010 int id; 2011 2012 for (id = 0; id < sc->num_rx_desc; id++) { 2013 if (igc_get_buf(rxr, id, false)) { 2014 panic("%s: msix=%d i=%d\n", __func__, rxr->me, id); 2015 } 2016 } 2017 2018 id = sc->num_rx_desc - 1; 2019 rxr->last_desc_filled = id; 2020 IGC_WRITE_REG(&sc->hw, IGC_RDT(rxr->me), id); 2021 rxr->next_to_check = 0; 2022 } 2023 2024 static void 2025 igc_rxrefill(struct rx_ring *rxr, int end) 2026 { 2027 struct igc_softc *sc = rxr->sc; 2028 int id; 2029 2030 for (id = rxr->next_to_check; id != end; id = igc_rxdesc_incr(sc, id)) { 2031 if (igc_get_buf(rxr, id, true)) { 2032 /* XXXRO */ 2033 panic("%s: msix=%d id=%d\n", __func__, rxr->me, id); 2034 } 2035 } 2036 2037 id = igc_rxdesc_decr(sc, id); 2038 DPRINTF(RX, "%s RDT %d id %d\n", 2039 rxr->last_desc_filled == id ? "same" : "diff", 2040 rxr->last_desc_filled, id); 2041 rxr->last_desc_filled = id; 2042 IGC_WRITE_REG(&sc->hw, IGC_RDT(rxr->me), id); 2043 } 2044 2045 /********************************************************************* 2046 * 2047 * This routine executes in interrupt context. It replenishes 2048 * the mbufs in the descriptor and sends data which has been 2049 * dma'ed into host memory to upper layer. 2050 * 2051 *********************************************************************/ 2052 static bool 2053 igc_rxeof(struct rx_ring *rxr, u_int limit) 2054 { 2055 struct igc_softc *sc = rxr->sc; 2056 struct igc_queue *q = rxr->rxr_igcq; 2057 struct ifnet *ifp = &sc->sc_ec.ec_if; 2058 int id; 2059 bool more = false; 2060 2061 id = rxr->next_to_check; 2062 for (;;) { 2063 union igc_adv_rx_desc *rxdesc = &rxr->rx_base[id]; 2064 struct igc_rx_buf *rxbuf, *nxbuf; 2065 struct mbuf *mp, *m; 2066 2067 igc_rxdesc_sync(rxr, id, 2068 BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE); 2069 2070 const uint32_t staterr = le32toh(rxdesc->wb.upper.status_error); 2071 2072 if (!ISSET(staterr, IGC_RXD_STAT_DD)) { 2073 igc_rxdesc_sync(rxr, id, 2074 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); 2075 break; 2076 } 2077 2078 if (limit-- == 0) { 2079 igc_rxdesc_sync(rxr, id, 2080 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); 2081 DPRINTF(RX, "more=true\n"); 2082 more = true; 2083 break; 2084 } 2085 2086 /* Zero out the receive descriptors status. */ 2087 rxdesc->wb.upper.status_error = 0; 2088 2089 /* Pull the mbuf off the ring. */ 2090 rxbuf = &rxr->rx_buffers[id]; 2091 bus_dmamap_t map = rxbuf->map; 2092 bus_dmamap_sync(rxr->rxdma.dma_tag, map, 2093 0, map->dm_mapsize, BUS_DMASYNC_POSTREAD); 2094 bus_dmamap_unload(rxr->rxdma.dma_tag, map); 2095 2096 mp = rxbuf->buf; 2097 rxbuf->buf = NULL; 2098 2099 const bool eop = staterr & IGC_RXD_STAT_EOP; 2100 const uint16_t len = le16toh(rxdesc->wb.upper.length); 2101 2102 #if NVLAN > 0 2103 const uint16_t vtag = le16toh(rxdesc->wb.upper.vlan); 2104 #endif 2105 2106 const uint32_t ptype = le32toh(rxdesc->wb.lower.lo_dword.data) & 2107 IGC_PKTTYPE_MASK; 2108 2109 const uint32_t hash __unused = 2110 le32toh(rxdesc->wb.lower.hi_dword.rss); 2111 const uint16_t hashtype __unused = 2112 le16toh(rxdesc->wb.lower.lo_dword.hs_rss.pkt_info) & 2113 IGC_RXDADV_RSSTYPE_MASK; 2114 2115 igc_rxdesc_sync(rxr, id, 2116 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); 2117 2118 if (__predict_false(staterr & IGC_RXDEXT_STATERR_RXE)) { 2119 if (rxbuf->fmp) { 2120 m_freem(rxbuf->fmp); 2121 rxbuf->fmp = NULL; 2122 } 2123 2124 m_freem(mp); 2125 m = NULL; 2126 2127 if_statinc(ifp, if_ierrors); 2128 IGC_QUEUE_EVENT(q, rx_discard, 1); 2129 2130 DPRINTF(RX, "ierrors++\n"); 2131 2132 goto next_desc; 2133 } 2134 2135 if (__predict_false(mp == NULL)) { 2136 panic("%s: igc_rxeof: NULL mbuf in slot %d " 2137 "(filled %d)", device_xname(sc->sc_dev), 2138 id, rxr->last_desc_filled); 2139 } 2140 2141 if (!eop) { 2142 /* 2143 * Figure out the next descriptor of this frame. 2144 */ 2145 int nextp = igc_rxdesc_incr(sc, id); 2146 2147 nxbuf = &rxr->rx_buffers[nextp]; 2148 /* 2149 * TODO prefetch(nxbuf); 2150 */ 2151 } 2152 2153 mp->m_len = len; 2154 2155 m = rxbuf->fmp; 2156 rxbuf->fmp = NULL; 2157 2158 if (m != NULL) { 2159 m->m_pkthdr.len += mp->m_len; 2160 } else { 2161 m = mp; 2162 m->m_pkthdr.len = mp->m_len; 2163 #if NVLAN > 0 2164 if (staterr & IGC_RXD_STAT_VP) 2165 vlan_set_tag(m, vtag); 2166 #endif 2167 } 2168 2169 /* Pass the head pointer on */ 2170 if (!eop) { 2171 nxbuf->fmp = m; 2172 m = NULL; 2173 mp->m_next = nxbuf->buf; 2174 } else { 2175 m_set_rcvif(m, ifp); 2176 2177 m->m_pkthdr.csum_flags = igc_rx_checksum(q, 2178 ifp->if_capenable, staterr, ptype); 2179 2180 #ifdef notyet 2181 if (hashtype != IGC_RXDADV_RSSTYPE_NONE) { 2182 m->m_pkthdr.ph_flowid = hash; 2183 SET(m->m_pkthdr.csum_flags, M_FLOWID); 2184 } 2185 ml_enqueue(&ml, m); 2186 #endif 2187 2188 if_percpuq_enqueue(sc->sc_ipq, m); 2189 2190 if_statinc(ifp, if_ipackets); 2191 IGC_QUEUE_EVENT(q, rx_packets, 1); 2192 IGC_QUEUE_EVENT(q, rx_bytes, m->m_pkthdr.len); 2193 } 2194 next_desc: 2195 /* Advance our pointers to the next descriptor. */ 2196 id = igc_rxdesc_incr(sc, id); 2197 } 2198 2199 DPRINTF(RX, "fill queue[%d]\n", rxr->me); 2200 igc_rxrefill(rxr, id); 2201 2202 DPRINTF(RX, "%s n2c %d id %d\n", 2203 rxr->next_to_check == id ? "same" : "diff", 2204 rxr->next_to_check, id); 2205 rxr->next_to_check = id; 2206 2207 #ifdef OPENBSD 2208 if (!(staterr & IGC_RXD_STAT_DD)) 2209 return 0; 2210 #endif 2211 2212 return more; 2213 } 2214 2215 /********************************************************************* 2216 * 2217 * Verify that the hardware indicated that the checksum is valid. 2218 * Inform the stack about the status of checksum so that stack 2219 * doesn't spend time verifying the checksum. 2220 * 2221 *********************************************************************/ 2222 static int 2223 igc_rx_checksum(struct igc_queue *q, uint64_t capenable, uint32_t staterr, 2224 uint32_t ptype) 2225 { 2226 const uint16_t status = (uint16_t)staterr; 2227 const uint8_t errors = (uint8_t)(staterr >> 24); 2228 int flags = 0; 2229 2230 if ((status & IGC_RXD_STAT_IPCS) != 0 && 2231 (capenable & IFCAP_CSUM_IPv4_Rx) != 0) { 2232 IGC_DRIVER_EVENT(q, rx_ipcs, 1); 2233 flags |= M_CSUM_IPv4; 2234 if (__predict_false((errors & IGC_RXD_ERR_IPE) != 0)) { 2235 IGC_DRIVER_EVENT(q, rx_ipcs_bad, 1); 2236 flags |= M_CSUM_IPv4_BAD; 2237 } 2238 } 2239 2240 if ((status & IGC_RXD_STAT_TCPCS) != 0) { 2241 IGC_DRIVER_EVENT(q, rx_tcpcs, 1); 2242 if ((capenable & IFCAP_CSUM_TCPv4_Rx) != 0) 2243 flags |= M_CSUM_TCPv4; 2244 if ((capenable & IFCAP_CSUM_TCPv6_Rx) != 0) 2245 flags |= M_CSUM_TCPv6; 2246 } 2247 2248 if ((status & IGC_RXD_STAT_UDPCS) != 0) { 2249 IGC_DRIVER_EVENT(q, rx_udpcs, 1); 2250 if ((capenable & IFCAP_CSUM_UDPv4_Rx) != 0) 2251 flags |= M_CSUM_UDPv4; 2252 if ((capenable & IFCAP_CSUM_UDPv6_Rx) != 0) 2253 flags |= M_CSUM_UDPv6; 2254 } 2255 2256 if (__predict_false((errors & IGC_RXD_ERR_TCPE) != 0)) { 2257 IGC_DRIVER_EVENT(q, rx_l4cs_bad, 1); 2258 if ((flags & ~M_CSUM_IPv4) != 0) 2259 flags |= M_CSUM_TCP_UDP_BAD; 2260 } 2261 2262 return flags; 2263 } 2264 2265 static void 2266 igc_watchdog(struct ifnet * ifp) 2267 { 2268 } 2269 2270 static void 2271 igc_tick(void *arg) 2272 { 2273 struct igc_softc *sc = arg; 2274 2275 mutex_enter(&sc->sc_core_lock); 2276 2277 if (__predict_false(sc->sc_core_stopping)) { 2278 mutex_exit(&sc->sc_core_lock); 2279 return; 2280 } 2281 2282 /* XXX watchdog */ 2283 if (0) { 2284 IGC_GLOBAL_EVENT(sc, watchdog, 1); 2285 } 2286 2287 igc_update_counters(sc); 2288 2289 mutex_exit(&sc->sc_core_lock); 2290 2291 callout_schedule(&sc->sc_tick_ch, hz); 2292 } 2293 2294 /********************************************************************* 2295 * 2296 * Media Ioctl callback 2297 * 2298 * This routine is called whenever the user queries the status of 2299 * the interface using ifconfig. 2300 * 2301 **********************************************************************/ 2302 static void 2303 igc_media_status(struct ifnet *ifp, struct ifmediareq *ifmr) 2304 { 2305 struct igc_softc *sc = ifp->if_softc; 2306 struct igc_hw *hw = &sc->hw; 2307 2308 igc_update_link_status(sc); 2309 2310 ifmr->ifm_status = IFM_AVALID; 2311 ifmr->ifm_active = IFM_ETHER; 2312 2313 if (!sc->link_active) { 2314 ifmr->ifm_active |= IFM_NONE; 2315 return; 2316 } 2317 2318 ifmr->ifm_status |= IFM_ACTIVE; 2319 2320 switch (sc->link_speed) { 2321 case 10: 2322 ifmr->ifm_active |= IFM_10_T; 2323 break; 2324 case 100: 2325 ifmr->ifm_active |= IFM_100_TX; 2326 break; 2327 case 1000: 2328 ifmr->ifm_active |= IFM_1000_T; 2329 break; 2330 case 2500: 2331 ifmr->ifm_active |= IFM_2500_T; 2332 break; 2333 } 2334 2335 if (sc->link_duplex == FULL_DUPLEX) 2336 ifmr->ifm_active |= IFM_FDX; 2337 else 2338 ifmr->ifm_active |= IFM_HDX; 2339 2340 switch (hw->fc.current_mode) { 2341 case igc_fc_tx_pause: 2342 ifmr->ifm_active |= IFM_FLOW | IFM_ETH_TXPAUSE; 2343 break; 2344 case igc_fc_rx_pause: 2345 ifmr->ifm_active |= IFM_FLOW | IFM_ETH_RXPAUSE; 2346 break; 2347 case igc_fc_full: 2348 ifmr->ifm_active |= IFM_FLOW | 2349 IFM_ETH_TXPAUSE | IFM_ETH_RXPAUSE; 2350 break; 2351 case igc_fc_none: 2352 default: 2353 break; 2354 } 2355 } 2356 2357 /********************************************************************* 2358 * 2359 * Media Ioctl callback 2360 * 2361 * This routine is called when the user changes speed/duplex using 2362 * media/mediopt option with ifconfig. 2363 * 2364 **********************************************************************/ 2365 static int 2366 igc_media_change(struct ifnet *ifp) 2367 { 2368 struct igc_softc *sc = ifp->if_softc; 2369 struct ifmedia *ifm = &sc->media; 2370 2371 if (IFM_TYPE(ifm->ifm_media) != IFM_ETHER) 2372 return EINVAL; 2373 2374 sc->hw.mac.autoneg = DO_AUTO_NEG; 2375 2376 switch (IFM_SUBTYPE(ifm->ifm_media)) { 2377 case IFM_AUTO: 2378 sc->hw.phy.autoneg_advertised = AUTONEG_ADV_DEFAULT; 2379 break; 2380 case IFM_2500_T: 2381 sc->hw.phy.autoneg_advertised = ADVERTISE_2500_FULL; 2382 break; 2383 case IFM_1000_T: 2384 sc->hw.phy.autoneg_advertised = ADVERTISE_1000_FULL; 2385 break; 2386 case IFM_100_TX: 2387 if ((ifm->ifm_media & IFM_GMASK) == IFM_FDX) 2388 sc->hw.phy.autoneg_advertised = ADVERTISE_100_FULL; 2389 else 2390 sc->hw.phy.autoneg_advertised = ADVERTISE_100_HALF; 2391 break; 2392 case IFM_10_T: 2393 if ((ifm->ifm_media & IFM_GMASK) == IFM_FDX) 2394 sc->hw.phy.autoneg_advertised = ADVERTISE_10_FULL; 2395 else 2396 sc->hw.phy.autoneg_advertised = ADVERTISE_10_HALF; 2397 break; 2398 default: 2399 return EINVAL; 2400 } 2401 2402 igc_init_locked(sc); 2403 2404 return 0; 2405 } 2406 2407 static int 2408 igc_ifflags_cb(struct ethercom *ec) 2409 { 2410 struct ifnet *ifp = &ec->ec_if; 2411 struct igc_softc *sc = ifp->if_softc; 2412 int rc = 0; 2413 u_short iffchange; 2414 bool needreset = false; 2415 2416 DPRINTF(CFG, "called\n"); 2417 2418 KASSERT(IFNET_LOCKED(ifp)); 2419 2420 mutex_enter(&sc->sc_core_lock); 2421 2422 /* 2423 * Check for if_flags. 2424 * Main usage is to prevent linkdown when opening bpf. 2425 */ 2426 iffchange = ifp->if_flags ^ sc->sc_if_flags; 2427 sc->sc_if_flags = ifp->if_flags; 2428 if ((iffchange & ~(IFF_CANTCHANGE | IFF_DEBUG)) != 0) { 2429 needreset = true; 2430 goto ec; 2431 } 2432 2433 /* iff related updates */ 2434 if ((iffchange & IFF_PROMISC) != 0) 2435 igc_set_filter(sc); 2436 2437 #ifdef notyet 2438 igc_set_vlan(sc); 2439 #endif 2440 2441 ec: 2442 #ifdef notyet 2443 /* Check for ec_capenable. */ 2444 ecchange = ec->ec_capenable ^ sc->sc_ec_capenable; 2445 sc->sc_ec_capenable = ec->ec_capenable; 2446 if ((ecchange & ~ETHERCAP_SOMETHING) != 0) { 2447 needreset = true; 2448 goto out; 2449 } 2450 #endif 2451 if (needreset) 2452 rc = ENETRESET; 2453 2454 mutex_exit(&sc->sc_core_lock); 2455 2456 return rc; 2457 } 2458 2459 static void 2460 igc_set_filter(struct igc_softc *sc) 2461 { 2462 struct ethercom *ec = &sc->sc_ec; 2463 uint32_t rctl; 2464 2465 rctl = IGC_READ_REG(&sc->hw, IGC_RCTL); 2466 rctl &= ~(IGC_RCTL_BAM |IGC_RCTL_UPE | IGC_RCTL_MPE); 2467 2468 if ((sc->sc_if_flags & IFF_BROADCAST) != 0) 2469 rctl |= IGC_RCTL_BAM; 2470 if ((sc->sc_if_flags & IFF_PROMISC) != 0) { 2471 DPRINTF(CFG, "promisc\n"); 2472 rctl |= IGC_RCTL_UPE; 2473 ETHER_LOCK(ec); 2474 allmulti: 2475 ec->ec_flags |= ETHER_F_ALLMULTI; 2476 ETHER_UNLOCK(ec); 2477 rctl |= IGC_RCTL_MPE; 2478 } else { 2479 struct ether_multistep step; 2480 struct ether_multi *enm; 2481 int mcnt = 0; 2482 2483 memset(sc->mta, 0, IGC_MTA_LEN); 2484 2485 ETHER_LOCK(ec); 2486 ETHER_FIRST_MULTI(step, ec, enm); 2487 while (enm != NULL) { 2488 if (((memcmp(enm->enm_addrlo, enm->enm_addrhi, 2489 ETHER_ADDR_LEN)) != 0) || 2490 (mcnt >= MAX_NUM_MULTICAST_ADDRESSES)) { 2491 /* 2492 * We must listen to a range of multicast 2493 * addresses. For now, just accept all 2494 * multicasts, rather than trying to set only 2495 * those filter bits needed to match the range. 2496 * (At this time, the only use of address 2497 * ranges is for IP multicast routing, for 2498 * which the range is big enough to require all 2499 * bits set.) 2500 */ 2501 goto allmulti; 2502 } 2503 DPRINTF(CFG, "%d: %s\n", mcnt, 2504 ether_sprintf(enm->enm_addrlo)); 2505 memcpy(&sc->mta[mcnt * ETHER_ADDR_LEN], 2506 enm->enm_addrlo, ETHER_ADDR_LEN); 2507 2508 mcnt++; 2509 ETHER_NEXT_MULTI(step, enm); 2510 } 2511 ec->ec_flags &= ~ETHER_F_ALLMULTI; 2512 ETHER_UNLOCK(ec); 2513 2514 DPRINTF(CFG, "hw filter\n"); 2515 igc_update_mc_addr_list(&sc->hw, sc->mta, mcnt); 2516 } 2517 2518 IGC_WRITE_REG(&sc->hw, IGC_RCTL, rctl); 2519 } 2520 2521 static void 2522 igc_update_link_status(struct igc_softc *sc) 2523 { 2524 struct ifnet *ifp = &sc->sc_ec.ec_if; 2525 struct igc_hw *hw = &sc->hw; 2526 2527 if (IGC_READ_REG(&sc->hw, IGC_STATUS) & IGC_STATUS_LU) { 2528 if (sc->link_active == 0) { 2529 igc_get_speed_and_duplex(hw, &sc->link_speed, 2530 &sc->link_duplex); 2531 sc->link_active = 1; 2532 ifp->if_baudrate = IF_Mbps(sc->link_speed); 2533 if_link_state_change(ifp, LINK_STATE_UP); 2534 } 2535 } else { 2536 if (sc->link_active == 1) { 2537 ifp->if_baudrate = sc->link_speed = 0; 2538 sc->link_duplex = 0; 2539 sc->link_active = 0; 2540 if_link_state_change(ifp, LINK_STATE_DOWN); 2541 } 2542 } 2543 } 2544 2545 /********************************************************************* 2546 * 2547 * Get a buffer from system mbuf buffer pool. 2548 * 2549 **********************************************************************/ 2550 static int 2551 igc_get_buf(struct rx_ring *rxr, int id, bool strict) 2552 { 2553 struct igc_softc *sc = rxr->sc; 2554 struct igc_queue *q = rxr->rxr_igcq; 2555 struct igc_rx_buf *rxbuf = &rxr->rx_buffers[id]; 2556 bus_dmamap_t map = rxbuf->map; 2557 struct mbuf *m; 2558 int error; 2559 2560 if (__predict_false(rxbuf->buf)) { 2561 if (strict) { 2562 DPRINTF(RX, "slot %d already has an mbuf\n", id); 2563 return EINVAL; 2564 } 2565 return 0; 2566 } 2567 2568 MGETHDR(m, M_DONTWAIT, MT_DATA); 2569 if (__predict_false(m == NULL)) { 2570 enobuf: 2571 IGC_QUEUE_EVENT(q, rx_no_mbuf, 1); 2572 return ENOBUFS; 2573 } 2574 2575 MCLGET(m, M_DONTWAIT); 2576 if (__predict_false(!(m->m_flags & M_EXT))) { 2577 m_freem(m); 2578 goto enobuf; 2579 } 2580 2581 m->m_len = m->m_pkthdr.len = sc->rx_mbuf_sz; 2582 2583 error = bus_dmamap_load_mbuf(rxr->rxdma.dma_tag, map, m, 2584 BUS_DMA_READ | BUS_DMA_NOWAIT); 2585 if (error) { 2586 m_freem(m); 2587 return error; 2588 } 2589 2590 bus_dmamap_sync(rxr->rxdma.dma_tag, map, 0, 2591 map->dm_mapsize, BUS_DMASYNC_PREREAD); 2592 rxbuf->buf = m; 2593 2594 union igc_adv_rx_desc *rxdesc = &rxr->rx_base[id]; 2595 igc_rxdesc_sync(rxr, id, BUS_DMASYNC_POSTWRITE | BUS_DMASYNC_POSTREAD); 2596 rxdesc->read.pkt_addr = htole64(map->dm_segs[0].ds_addr); 2597 igc_rxdesc_sync(rxr, id, BUS_DMASYNC_PREWRITE | BUS_DMASYNC_PREREAD); 2598 2599 return 0; 2600 } 2601 2602 static void 2603 igc_configure_queues(struct igc_softc *sc) 2604 { 2605 struct igc_hw *hw = &sc->hw; 2606 uint32_t ivar; 2607 2608 /* First turn on RSS capability */ 2609 IGC_WRITE_REG(hw, IGC_GPIE, IGC_GPIE_MSIX_MODE | IGC_GPIE_EIAME | 2610 IGC_GPIE_PBA | IGC_GPIE_NSICR); 2611 2612 /* Set the starting interrupt rate */ 2613 uint32_t newitr = (4000000 / MAX_INTS_PER_SEC) & 0x7FFC; 2614 newitr |= IGC_EITR_CNT_IGNR; 2615 2616 /* Turn on MSI-X */ 2617 uint32_t newmask = 0; 2618 for (int iq = 0; iq < sc->sc_nqueues; iq++) { 2619 struct igc_queue *q = &sc->queues[iq]; 2620 2621 /* RX entries */ 2622 igc_set_queues(sc, iq, q->msix, 0); 2623 /* TX entries */ 2624 igc_set_queues(sc, iq, q->msix, 1); 2625 newmask |= q->eims; 2626 IGC_WRITE_REG(hw, IGC_EITR(q->msix), newitr); 2627 } 2628 sc->msix_queuesmask = newmask; 2629 2630 #if 1 2631 ivar = IGC_READ_REG_ARRAY(hw, IGC_IVAR0, 0); 2632 DPRINTF(CFG, "ivar(0)=0x%x\n", ivar); 2633 ivar = IGC_READ_REG_ARRAY(hw, IGC_IVAR0, 1); 2634 DPRINTF(CFG, "ivar(1)=0x%x\n", ivar); 2635 #endif 2636 2637 /* And for the link interrupt */ 2638 ivar = (sc->linkvec | IGC_IVAR_VALID) << 8; 2639 sc->msix_linkmask = 1 << sc->linkvec; 2640 IGC_WRITE_REG(hw, IGC_IVAR_MISC, ivar); 2641 } 2642 2643 static void 2644 igc_set_queues(struct igc_softc *sc, uint32_t entry, uint32_t vector, int type) 2645 { 2646 struct igc_hw *hw = &sc->hw; 2647 const uint32_t index = entry >> 1; 2648 uint32_t ivar = IGC_READ_REG_ARRAY(hw, IGC_IVAR0, index); 2649 2650 if (type) { 2651 if (entry & 1) { 2652 ivar &= 0x00FFFFFF; 2653 ivar |= (vector | IGC_IVAR_VALID) << 24; 2654 } else { 2655 ivar &= 0xFFFF00FF; 2656 ivar |= (vector | IGC_IVAR_VALID) << 8; 2657 } 2658 } else { 2659 if (entry & 1) { 2660 ivar &= 0xFF00FFFF; 2661 ivar |= (vector | IGC_IVAR_VALID) << 16; 2662 } else { 2663 ivar &= 0xFFFFFF00; 2664 ivar |= vector | IGC_IVAR_VALID; 2665 } 2666 } 2667 IGC_WRITE_REG_ARRAY(hw, IGC_IVAR0, index, ivar); 2668 } 2669 2670 static void 2671 igc_enable_queue(struct igc_softc *sc, uint32_t eims) 2672 { 2673 IGC_WRITE_REG(&sc->hw, IGC_EIMS, eims); 2674 } 2675 2676 static void 2677 igc_enable_intr(struct igc_softc *sc) 2678 { 2679 struct igc_hw *hw = &sc->hw; 2680 2681 if (sc->sc_intr_type == PCI_INTR_TYPE_MSIX) { 2682 const uint32_t mask = sc->msix_queuesmask | sc->msix_linkmask; 2683 2684 IGC_WRITE_REG(hw, IGC_EIAC, mask); 2685 IGC_WRITE_REG(hw, IGC_EIAM, mask); 2686 IGC_WRITE_REG(hw, IGC_EIMS, mask); 2687 IGC_WRITE_REG(hw, IGC_IMS, IGC_IMS_LSC); 2688 } else { 2689 IGC_WRITE_REG(hw, IGC_IMS, IMS_ENABLE_MASK); 2690 } 2691 IGC_WRITE_FLUSH(hw); 2692 } 2693 2694 static void 2695 igc_disable_intr(struct igc_softc *sc) 2696 { 2697 struct igc_hw *hw = &sc->hw; 2698 2699 if (sc->sc_intr_type == PCI_INTR_TYPE_MSIX) { 2700 IGC_WRITE_REG(hw, IGC_EIMC, 0xffffffff); 2701 IGC_WRITE_REG(hw, IGC_EIAC, 0); 2702 } 2703 IGC_WRITE_REG(hw, IGC_IMC, 0xffffffff); 2704 IGC_WRITE_FLUSH(hw); 2705 } 2706 2707 static int 2708 igc_intr_link(void *arg) 2709 { 2710 struct igc_softc *sc = (struct igc_softc *)arg; 2711 const uint32_t reg_icr = IGC_READ_REG(&sc->hw, IGC_ICR); 2712 2713 IGC_GLOBAL_EVENT(sc, link, 1); 2714 2715 if (reg_icr & IGC_ICR_LSC) { 2716 mutex_enter(&sc->sc_core_lock); 2717 sc->hw.mac.get_link_status = true; 2718 igc_update_link_status(sc); 2719 mutex_exit(&sc->sc_core_lock); 2720 } 2721 2722 IGC_WRITE_REG(&sc->hw, IGC_IMS, IGC_IMS_LSC); 2723 IGC_WRITE_REG(&sc->hw, IGC_EIMS, sc->msix_linkmask); 2724 2725 return 1; 2726 } 2727 2728 static int 2729 igc_intr_queue(void *arg) 2730 { 2731 struct igc_queue *iq = arg; 2732 struct igc_softc *sc = iq->sc; 2733 struct ifnet *ifp = &sc->sc_ec.ec_if; 2734 struct rx_ring *rxr = iq->rxr; 2735 struct tx_ring *txr = iq->txr; 2736 const u_int txlimit = sc->sc_tx_intr_process_limit, 2737 rxlimit = sc->sc_rx_intr_process_limit; 2738 bool txmore, rxmore; 2739 2740 IGC_QUEUE_EVENT(iq, irqs, 1); 2741 2742 if (__predict_false(!ISSET(ifp->if_flags, IFF_RUNNING))) 2743 return 0; 2744 2745 mutex_enter(&txr->txr_lock); 2746 txmore = igc_txeof(txr, txlimit); 2747 mutex_exit(&txr->txr_lock); 2748 mutex_enter(&rxr->rxr_lock); 2749 rxmore = igc_rxeof(rxr, rxlimit); 2750 mutex_exit(&rxr->rxr_lock); 2751 2752 if (txmore || rxmore) { 2753 IGC_QUEUE_EVENT(iq, req, 1); 2754 igc_sched_handle_queue(sc, iq); 2755 } else { 2756 igc_enable_queue(sc, iq->eims); 2757 } 2758 2759 return 1; 2760 } 2761 2762 static int 2763 igc_intr(void *arg) 2764 { 2765 struct igc_softc *sc = arg; 2766 struct ifnet *ifp = &sc->sc_ec.ec_if; 2767 struct igc_queue *iq = &sc->queues[0]; 2768 struct rx_ring *rxr = iq->rxr; 2769 struct tx_ring *txr = iq->txr; 2770 const u_int txlimit = sc->sc_tx_intr_process_limit, 2771 rxlimit = sc->sc_rx_intr_process_limit; 2772 bool txmore, rxmore; 2773 2774 if (__predict_false(!ISSET(ifp->if_flags, IFF_RUNNING))) 2775 return 0; 2776 2777 const uint32_t reg_icr = IGC_READ_REG(&sc->hw, IGC_ICR); 2778 DPRINTF(MISC, "reg_icr=0x%x\n", reg_icr); 2779 2780 /* Definitely not our interrupt. */ 2781 if (reg_icr == 0x0) { 2782 DPRINTF(MISC, "not for me"); 2783 return 0; 2784 } 2785 2786 IGC_QUEUE_EVENT(iq, irqs, 1); 2787 2788 /* Hot eject? */ 2789 if (__predict_false(reg_icr == 0xffffffff)) { 2790 DPRINTF(MISC, "hot eject\n"); 2791 return 0; 2792 } 2793 2794 if (__predict_false(!(reg_icr & IGC_ICR_INT_ASSERTED))) { 2795 DPRINTF(MISC, "not set IGC_ICR_INT_ASSERTED"); 2796 return 0; 2797 } 2798 2799 /* 2800 * Only MSI-X interrupts have one-shot behavior by taking advantage 2801 * of the EIAC register. Thus, explicitly disable interrupts. This 2802 * also works around the MSI message reordering errata on certain 2803 * systems. 2804 */ 2805 igc_disable_intr(sc); 2806 2807 mutex_enter(&txr->txr_lock); 2808 txmore = igc_txeof(txr, txlimit); 2809 mutex_exit(&txr->txr_lock); 2810 mutex_enter(&rxr->rxr_lock); 2811 rxmore = igc_rxeof(rxr, rxlimit); 2812 mutex_exit(&rxr->rxr_lock); 2813 2814 /* Link status change */ 2815 // XXXX FreeBSD checks IGC_ICR_RXSEQ 2816 if (__predict_false(reg_icr & IGC_ICR_LSC)) { 2817 IGC_GLOBAL_EVENT(sc, link, 1); 2818 mutex_enter(&sc->sc_core_lock); 2819 sc->hw.mac.get_link_status = true; 2820 igc_update_link_status(sc); 2821 mutex_exit(&sc->sc_core_lock); 2822 } 2823 2824 if (txmore || rxmore) { 2825 IGC_QUEUE_EVENT(iq, req, 1); 2826 igc_sched_handle_queue(sc, iq); 2827 } else { 2828 igc_enable_intr(sc); 2829 } 2830 2831 return 1; 2832 } 2833 2834 static void 2835 igc_handle_queue(void *arg) 2836 { 2837 struct igc_queue *iq = arg; 2838 struct igc_softc *sc = iq->sc; 2839 struct tx_ring *txr = iq->txr; 2840 struct rx_ring *rxr = iq->rxr; 2841 const u_int txlimit = sc->sc_tx_process_limit, 2842 rxlimit = sc->sc_rx_process_limit; 2843 bool txmore, rxmore; 2844 2845 IGC_QUEUE_EVENT(iq, handleq, 1); 2846 2847 mutex_enter(&txr->txr_lock); 2848 txmore = igc_txeof(txr, txlimit); 2849 /* for ALTQ, dequeue from if_snd */ 2850 if (txr->me == 0) { 2851 struct ifnet *ifp = &sc->sc_ec.ec_if; 2852 2853 igc_tx_common_locked(ifp, txr, IGC_TX_START); 2854 } 2855 mutex_exit(&txr->txr_lock); 2856 2857 mutex_enter(&rxr->rxr_lock); 2858 rxmore = igc_rxeof(rxr, rxlimit); 2859 mutex_exit(&rxr->rxr_lock); 2860 2861 if (txmore || rxmore) { 2862 igc_sched_handle_queue(sc, iq); 2863 } else { 2864 if (sc->sc_intr_type == PCI_INTR_TYPE_MSIX) 2865 igc_enable_queue(sc, iq->eims); 2866 else 2867 igc_enable_intr(sc); 2868 } 2869 } 2870 2871 static void 2872 igc_handle_queue_work(struct work *wk, void *context) 2873 { 2874 struct igc_queue *iq = 2875 container_of(wk, struct igc_queue, igcq_wq_cookie); 2876 2877 igc_handle_queue(iq); 2878 } 2879 2880 static void 2881 igc_sched_handle_queue(struct igc_softc *sc, struct igc_queue *iq) 2882 { 2883 2884 if (iq->igcq_workqueue) { 2885 /* XXXRO notyet */ 2886 workqueue_enqueue(sc->sc_queue_wq, &iq->igcq_wq_cookie, 2887 curcpu()); 2888 } else { 2889 softint_schedule(iq->igcq_si); 2890 } 2891 } 2892 2893 static void 2894 igc_barrier_handle_queue(struct igc_softc *sc) 2895 { 2896 2897 if (sc->sc_txrx_workqueue) { 2898 for (int iq = 0; iq < sc->sc_nqueues; iq++) { 2899 struct igc_queue *q = &sc->queues[iq]; 2900 2901 workqueue_wait(sc->sc_queue_wq, &q->igcq_wq_cookie); 2902 } 2903 } else { 2904 xc_barrier(0); 2905 } 2906 } 2907 2908 /********************************************************************* 2909 * 2910 * Allocate memory for tx_buffer structures. The tx_buffer stores all 2911 * the information needed to transmit a packet on the wire. 2912 * 2913 **********************************************************************/ 2914 static int 2915 igc_allocate_transmit_buffers(struct tx_ring *txr) 2916 { 2917 struct igc_softc *sc = txr->sc; 2918 int error; 2919 2920 txr->tx_buffers = 2921 kmem_zalloc(sc->num_tx_desc * sizeof(struct igc_tx_buf), KM_SLEEP); 2922 txr->txtag = txr->txdma.dma_tag; 2923 2924 /* Create the descriptor buffer dma maps. */ 2925 for (int id = 0; id < sc->num_tx_desc; id++) { 2926 struct igc_tx_buf *txbuf = &txr->tx_buffers[id]; 2927 2928 error = bus_dmamap_create(txr->txdma.dma_tag, 2929 round_page(IGC_TSO_SIZE + sizeof(struct ether_vlan_header)), 2930 IGC_MAX_SCATTER, PAGE_SIZE, 0, BUS_DMA_NOWAIT, &txbuf->map); 2931 if (error != 0) { 2932 aprint_error_dev(sc->sc_dev, 2933 "unable to create TX DMA map\n"); 2934 goto fail; 2935 } 2936 2937 txbuf->eop_index = -1; 2938 } 2939 2940 return 0; 2941 fail: 2942 return error; 2943 } 2944 2945 2946 /********************************************************************* 2947 * 2948 * Allocate and initialize transmit structures. 2949 * 2950 **********************************************************************/ 2951 static int 2952 igc_setup_transmit_structures(struct igc_softc *sc) 2953 { 2954 2955 for (int iq = 0; iq < sc->sc_nqueues; iq++) { 2956 struct tx_ring *txr = &sc->tx_rings[iq]; 2957 2958 if (igc_setup_transmit_ring(txr)) 2959 goto fail; 2960 } 2961 2962 return 0; 2963 fail: 2964 igc_free_transmit_structures(sc); 2965 return ENOBUFS; 2966 } 2967 2968 /********************************************************************* 2969 * 2970 * Initialize a transmit ring. 2971 * 2972 **********************************************************************/ 2973 static int 2974 igc_setup_transmit_ring(struct tx_ring *txr) 2975 { 2976 struct igc_softc *sc = txr->sc; 2977 2978 /* Now allocate transmit buffers for the ring. */ 2979 if (igc_allocate_transmit_buffers(txr)) 2980 return ENOMEM; 2981 2982 /* Clear the old ring contents */ 2983 memset(txr->tx_base, 0, 2984 sizeof(union igc_adv_tx_desc) * sc->num_tx_desc); 2985 2986 /* Reset indices. */ 2987 txr->next_avail_desc = 0; 2988 txr->next_to_clean = 0; 2989 2990 bus_dmamap_sync(txr->txdma.dma_tag, txr->txdma.dma_map, 0, 2991 txr->txdma.dma_map->dm_mapsize, 2992 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); 2993 2994 txr->txr_interq = pcq_create(sc->num_tx_desc, KM_SLEEP); 2995 2996 mutex_init(&txr->txr_lock, MUTEX_DEFAULT, IPL_NET); 2997 2998 return 0; 2999 } 3000 3001 /********************************************************************* 3002 * 3003 * Enable transmit unit. 3004 * 3005 **********************************************************************/ 3006 static void 3007 igc_initialize_transmit_unit(struct igc_softc *sc) 3008 { 3009 struct ifnet *ifp = &sc->sc_ec.ec_if; 3010 struct igc_hw *hw = &sc->hw; 3011 3012 /* Setup the Base and Length of the TX descriptor ring. */ 3013 for (int iq = 0; iq < sc->sc_nqueues; iq++) { 3014 struct tx_ring *txr = &sc->tx_rings[iq]; 3015 const uint64_t bus_addr = 3016 txr->txdma.dma_map->dm_segs[0].ds_addr; 3017 3018 /* Base and len of TX ring */ 3019 IGC_WRITE_REG(hw, IGC_TDLEN(iq), 3020 sc->num_tx_desc * sizeof(union igc_adv_tx_desc)); 3021 IGC_WRITE_REG(hw, IGC_TDBAH(iq), (uint32_t)(bus_addr >> 32)); 3022 IGC_WRITE_REG(hw, IGC_TDBAL(iq), (uint32_t)bus_addr); 3023 3024 /* Init the HEAD/TAIL indices */ 3025 IGC_WRITE_REG(hw, IGC_TDT(iq), 0 /* XXX txr->next_avail_desc */); 3026 IGC_WRITE_REG(hw, IGC_TDH(iq), 0); 3027 3028 txr->watchdog_timer = 0; 3029 3030 uint32_t txdctl = 0; /* Clear txdctl */ 3031 txdctl |= 0x1f; /* PTHRESH */ 3032 txdctl |= 1 << 8; /* HTHRESH */ 3033 txdctl |= 1 << 16; /* WTHRESH */ 3034 txdctl |= 1 << 22; /* Reserved bit 22 must always be 1 */ 3035 txdctl |= IGC_TXDCTL_GRAN; 3036 txdctl |= 1 << 25; /* LWTHRESH */ 3037 3038 IGC_WRITE_REG(hw, IGC_TXDCTL(iq), txdctl); 3039 } 3040 ifp->if_timer = 0; 3041 3042 /* Program the Transmit Control Register */ 3043 uint32_t tctl = IGC_READ_REG(&sc->hw, IGC_TCTL); 3044 tctl &= ~IGC_TCTL_CT; 3045 tctl |= (IGC_TCTL_PSP | IGC_TCTL_RTLC | IGC_TCTL_EN | 3046 (IGC_COLLISION_THRESHOLD << IGC_CT_SHIFT)); 3047 3048 /* This write will effectively turn on the transmit unit. */ 3049 IGC_WRITE_REG(&sc->hw, IGC_TCTL, tctl); 3050 } 3051 3052 /********************************************************************* 3053 * 3054 * Free all transmit rings. 3055 * 3056 **********************************************************************/ 3057 static void 3058 igc_free_transmit_structures(struct igc_softc *sc) 3059 { 3060 3061 for (int iq = 0; iq < sc->sc_nqueues; iq++) { 3062 struct tx_ring *txr = &sc->tx_rings[iq]; 3063 3064 igc_free_transmit_buffers(txr); 3065 } 3066 } 3067 3068 /********************************************************************* 3069 * 3070 * Free transmit ring related data structures. 3071 * 3072 **********************************************************************/ 3073 static void 3074 igc_free_transmit_buffers(struct tx_ring *txr) 3075 { 3076 struct igc_softc *sc = txr->sc; 3077 3078 if (txr->tx_buffers == NULL) 3079 return; 3080 3081 igc_withdraw_transmit_packets(txr, true); 3082 3083 kmem_free(txr->tx_buffers, 3084 sc->num_tx_desc * sizeof(struct igc_tx_buf)); 3085 txr->tx_buffers = NULL; 3086 txr->txtag = NULL; 3087 3088 pcq_destroy(txr->txr_interq); 3089 mutex_destroy(&txr->txr_lock); 3090 } 3091 3092 /********************************************************************* 3093 * 3094 * Withdraw transmit packets. 3095 * 3096 **********************************************************************/ 3097 static void 3098 igc_withdraw_transmit_packets(struct tx_ring *txr, bool destroy) 3099 { 3100 struct igc_softc *sc = txr->sc; 3101 struct igc_queue *q = txr->txr_igcq; 3102 3103 mutex_enter(&txr->txr_lock); 3104 3105 for (int id = 0; id < sc->num_tx_desc; id++) { 3106 union igc_adv_tx_desc *txdesc = &txr->tx_base[id]; 3107 3108 igc_txdesc_sync(txr, id, 3109 BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE); 3110 txdesc->read.buffer_addr = 0; 3111 txdesc->read.cmd_type_len = 0; 3112 txdesc->read.olinfo_status = 0; 3113 igc_txdesc_sync(txr, id, 3114 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); 3115 3116 struct igc_tx_buf *txbuf = &txr->tx_buffers[id]; 3117 bus_dmamap_t map = txbuf->map; 3118 3119 if (map != NULL && map->dm_nsegs > 0) { 3120 bus_dmamap_sync(txr->txdma.dma_tag, map, 3121 0, map->dm_mapsize, BUS_DMASYNC_POSTWRITE); 3122 bus_dmamap_unload(txr->txdma.dma_tag, map); 3123 } 3124 if (txbuf->m_head != NULL) { 3125 m_freem(txbuf->m_head); 3126 txbuf->m_head = NULL; 3127 } 3128 if (map != NULL && destroy) { 3129 bus_dmamap_destroy(txr->txdma.dma_tag, map); 3130 txbuf->map = NULL; 3131 } 3132 txbuf->eop_index = -1; 3133 3134 txr->next_avail_desc = 0; 3135 txr->next_to_clean = 0; 3136 } 3137 3138 struct mbuf *m; 3139 while ((m = pcq_get(txr->txr_interq)) != NULL) { 3140 IGC_QUEUE_EVENT(q, tx_pcq_drop, 1); 3141 m_freem(m); 3142 } 3143 3144 mutex_exit(&txr->txr_lock); 3145 } 3146 3147 3148 /********************************************************************* 3149 * 3150 * Advanced Context Descriptor setup for VLAN, CSUM or TSO 3151 * 3152 **********************************************************************/ 3153 3154 static int 3155 igc_tx_ctx_setup(struct tx_ring *txr, struct mbuf *mp, int prod, 3156 uint32_t *cmd_type_len, uint32_t *olinfo_status) 3157 { 3158 struct ether_vlan_header *evl; 3159 uint32_t type_tucmd_mlhl = 0; 3160 uint32_t vlan_macip_lens = 0; 3161 uint32_t ehlen, iphlen; 3162 uint16_t ehtype; 3163 int off = 0; 3164 3165 const int csum_flags = mp->m_pkthdr.csum_flags; 3166 3167 /* First check if TSO is to be used */ 3168 if ((csum_flags & (M_CSUM_TSOv4 | M_CSUM_TSOv6)) != 0) { 3169 return igc_tso_setup(txr, mp, prod, cmd_type_len, 3170 olinfo_status); 3171 } 3172 3173 const bool v4 = (csum_flags & 3174 (M_CSUM_IPv4 | M_CSUM_TCPv4 | M_CSUM_UDPv4)) != 0; 3175 const bool v6 = (csum_flags & (M_CSUM_UDPv6 | M_CSUM_TCPv6)) != 0; 3176 3177 /* Indicate the whole packet as payload when not doing TSO */ 3178 *olinfo_status |= mp->m_pkthdr.len << IGC_ADVTXD_PAYLEN_SHIFT; 3179 3180 /* 3181 * In advanced descriptors the vlan tag must 3182 * be placed into the context descriptor. Hence 3183 * we need to make one even if not doing offloads. 3184 */ 3185 #if NVLAN > 0 3186 if (vlan_has_tag(mp)) { 3187 vlan_macip_lens |= (uint32_t)vlan_get_tag(mp) 3188 << IGC_ADVTXD_VLAN_SHIFT; 3189 off = 1; 3190 } else 3191 #endif 3192 if (!v4 && !v6) 3193 return 0; 3194 3195 KASSERT(mp->m_len >= sizeof(struct ether_header)); 3196 evl = mtod(mp, struct ether_vlan_header *); 3197 if (evl->evl_encap_proto == htons(ETHERTYPE_VLAN)) { 3198 KASSERT(mp->m_len >= sizeof(struct ether_vlan_header)); 3199 ehlen = ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN; 3200 ehtype = evl->evl_proto; 3201 } else { 3202 ehlen = ETHER_HDR_LEN; 3203 ehtype = evl->evl_encap_proto; 3204 } 3205 3206 vlan_macip_lens |= ehlen << IGC_ADVTXD_MACLEN_SHIFT; 3207 3208 #ifdef IGC_DEBUG 3209 /* 3210 * For checksum offloading, L3 headers are not mandatory. 3211 * We use these only for consistency checks. 3212 */ 3213 struct ip *ip; 3214 struct ip6_hdr *ip6; 3215 uint8_t ipproto; 3216 char *l3d; 3217 3218 if (mp->m_len == ehlen && mp->m_next != NULL) 3219 l3d = mtod(mp->m_next, char *); 3220 else 3221 l3d = mtod(mp, char *) + ehlen; 3222 #endif 3223 3224 switch (ntohs(ehtype)) { 3225 case ETHERTYPE_IP: 3226 iphlen = M_CSUM_DATA_IPv4_IPHL(mp->m_pkthdr.csum_data); 3227 type_tucmd_mlhl |= IGC_ADVTXD_TUCMD_IPV4; 3228 3229 if ((csum_flags & M_CSUM_IPv4) != 0) { 3230 *olinfo_status |= IGC_TXD_POPTS_IXSM << 8; 3231 off = 1; 3232 } 3233 #ifdef IGC_DEBUG 3234 KASSERT(!v6); 3235 ip = (void *)l3d; 3236 ipproto = ip->ip_p; 3237 KASSERT(iphlen == ip->ip_hl << 2); 3238 KASSERT((mp->m_pkthdr.csum_flags & M_CSUM_IPv4) == 0 || 3239 ip->ip_sum == 0); 3240 #endif 3241 break; 3242 case ETHERTYPE_IPV6: 3243 iphlen = M_CSUM_DATA_IPv6_IPHL(mp->m_pkthdr.csum_data); 3244 type_tucmd_mlhl |= IGC_ADVTXD_TUCMD_IPV6; 3245 #ifdef IGC_DEBUG 3246 KASSERT(!v4); 3247 ip6 = (void *)l3d; 3248 ipproto = ip6->ip6_nxt; /* XXX */ 3249 KASSERT(iphlen == sizeof(struct ip6_hdr)); 3250 #endif 3251 break; 3252 default: 3253 /* 3254 * Unknown L3 protocol. Clear L3 header length and proceed for 3255 * LAN as done by Linux driver. 3256 */ 3257 iphlen = 0; 3258 #ifdef IGC_DEBUG 3259 KASSERT(!v4 && !v6); 3260 ipproto = 0; 3261 #endif 3262 break; 3263 } 3264 3265 vlan_macip_lens |= iphlen; 3266 3267 const bool tcp = (csum_flags & (M_CSUM_TCPv4 | M_CSUM_TCPv6)) != 0; 3268 const bool udp = (csum_flags & (M_CSUM_UDPv4 | M_CSUM_UDPv6)) != 0; 3269 3270 if (tcp) { 3271 #ifdef IGC_DEBUG 3272 KASSERTMSG(ipproto == IPPROTO_TCP, "ipproto = %d", ipproto); 3273 #endif 3274 type_tucmd_mlhl |= IGC_ADVTXD_TUCMD_L4T_TCP; 3275 *olinfo_status |= IGC_TXD_POPTS_TXSM << 8; 3276 off = 1; 3277 } else if (udp) { 3278 #ifdef IGC_DEBUG 3279 KASSERTMSG(ipproto == IPPROTO_UDP, "ipproto = %d", ipproto); 3280 #endif 3281 type_tucmd_mlhl |= IGC_ADVTXD_TUCMD_L4T_UDP; 3282 *olinfo_status |= IGC_TXD_POPTS_TXSM << 8; 3283 off = 1; 3284 } 3285 3286 if (off == 0) 3287 return 0; 3288 3289 type_tucmd_mlhl |= IGC_ADVTXD_DCMD_DEXT | IGC_ADVTXD_DTYP_CTXT; 3290 3291 /* Now ready a context descriptor */ 3292 struct igc_adv_tx_context_desc *txdesc = 3293 (struct igc_adv_tx_context_desc *)&txr->tx_base[prod]; 3294 3295 /* Now copy bits into descriptor */ 3296 igc_txdesc_sync(txr, prod, 3297 BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE); 3298 htolem32(&txdesc->vlan_macip_lens, vlan_macip_lens); 3299 htolem32(&txdesc->type_tucmd_mlhl, type_tucmd_mlhl); 3300 htolem32(&txdesc->seqnum_seed, 0); 3301 htolem32(&txdesc->mss_l4len_idx, 0); 3302 igc_txdesc_sync(txr, prod, 3303 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); 3304 3305 return 1; 3306 } 3307 3308 /********************************************************************* 3309 * 3310 * Advanced Context Descriptor setup for TSO 3311 * 3312 * XXX XXXRO 3313 * Not working. Some packets are sent with correct csums, but 3314 * others aren't. th->th_sum may be adjusted. 3315 * 3316 **********************************************************************/ 3317 3318 static int 3319 igc_tso_setup(struct tx_ring *txr, struct mbuf *mp, int prod, 3320 uint32_t *cmd_type_len, uint32_t *olinfo_status) 3321 { 3322 #if 1 /* notyet */ 3323 return 0; 3324 #else 3325 struct ether_vlan_header *evl; 3326 struct ip *ip; 3327 struct ip6_hdr *ip6; 3328 struct tcphdr *th; 3329 uint32_t type_tucmd_mlhl = 0; 3330 uint32_t vlan_macip_lens = 0; 3331 uint32_t mss_l4len_idx = 0; 3332 uint32_t ehlen, iphlen, tcphlen, paylen; 3333 uint16_t ehtype; 3334 3335 /* 3336 * In advanced descriptors the vlan tag must 3337 * be placed into the context descriptor. Hence 3338 * we need to make one even if not doing offloads. 3339 */ 3340 #if NVLAN > 0 3341 if (vlan_has_tag(mp)) { 3342 vlan_macip_lens |= (uint32_t)vlan_get_tag(mp) 3343 << IGC_ADVTXD_VLAN_SHIFT; 3344 } 3345 #endif 3346 3347 KASSERT(mp->m_len >= sizeof(struct ether_header)); 3348 evl = mtod(mp, struct ether_vlan_header *); 3349 if (evl->evl_encap_proto == htons(ETHERTYPE_VLAN)) { 3350 KASSERT(mp->m_len >= sizeof(struct ether_vlan_header)); 3351 ehlen = ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN; 3352 ehtype = evl->evl_proto; 3353 } else { 3354 ehlen = ETHER_HDR_LEN; 3355 ehtype = evl->evl_encap_proto; 3356 } 3357 3358 vlan_macip_lens |= ehlen << IGC_ADVTXD_MACLEN_SHIFT; 3359 3360 switch (ntohs(ehtype)) { 3361 case ETHERTYPE_IP: 3362 iphlen = M_CSUM_DATA_IPv4_IPHL(mp->m_pkthdr.csum_data); 3363 type_tucmd_mlhl |= IGC_ADVTXD_TUCMD_IPV4; 3364 *olinfo_status |= IGC_TXD_POPTS_IXSM << 8; 3365 3366 KASSERT(mp->m_len >= ehlen + sizeof(*ip)); 3367 ip = (void *)(mtod(mp, char *) + ehlen); 3368 ip->ip_len = 0; 3369 KASSERT(iphlen == ip->ip_hl << 2); 3370 KASSERT(ip->ip_sum == 0); 3371 KASSERT(ip->ip_p == IPPROTO_TCP); 3372 3373 KASSERT(mp->m_len >= ehlen + iphlen + sizeof(*th)); 3374 th = (void *)((char *)ip + iphlen); 3375 th->th_sum = in_cksum_phdr(ip->ip_src.s_addr, ip->ip_dst.s_addr, 3376 htons(IPPROTO_TCP)); 3377 break; 3378 case ETHERTYPE_IPV6: 3379 iphlen = M_CSUM_DATA_IPv6_IPHL(mp->m_pkthdr.csum_data); 3380 type_tucmd_mlhl |= IGC_ADVTXD_TUCMD_IPV6; 3381 3382 KASSERT(mp->m_len >= ehlen + sizeof(*ip6)); 3383 ip6 = (void *)(mtod(mp, char *) + ehlen); 3384 ip6->ip6_plen = 0; 3385 KASSERT(iphlen == sizeof(struct ip6_hdr)); 3386 KASSERT(ip6->ip6_nxt == IPPROTO_TCP); 3387 3388 KASSERT(mp->m_len >= ehlen + iphlen + sizeof(*th)); 3389 th = (void *)((char *)ip6 + iphlen); 3390 tcphlen = th->th_off << 2; 3391 paylen = mp->m_pkthdr.len - ehlen - iphlen - tcphlen; 3392 th->th_sum = in6_cksum_phdr(&ip6->ip6_src, &ip6->ip6_dst, 0, 3393 htonl(IPPROTO_TCP)); 3394 break; 3395 default: 3396 panic("%s", __func__); 3397 } 3398 3399 tcphlen = th->th_off << 2; 3400 paylen = mp->m_pkthdr.len - ehlen - iphlen - tcphlen; 3401 3402 vlan_macip_lens |= iphlen; 3403 3404 type_tucmd_mlhl |= IGC_ADVTXD_DCMD_DEXT | IGC_ADVTXD_DTYP_CTXT; 3405 type_tucmd_mlhl |= IGC_ADVTXD_TUCMD_L4T_TCP; 3406 3407 mss_l4len_idx |= mp->m_pkthdr.segsz << IGC_ADVTXD_MSS_SHIFT; 3408 mss_l4len_idx |= tcphlen << IGC_ADVTXD_L4LEN_SHIFT; 3409 3410 /* Now ready a context descriptor */ 3411 struct igc_adv_tx_context_desc *txdesc = 3412 (struct igc_adv_tx_context_desc *)&txr->tx_base[prod]; 3413 3414 /* Now copy bits into descriptor */ 3415 igc_txdesc_sync(txr, prod, 3416 BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE); 3417 htolem32(&txdesc->vlan_macip_lens, vlan_macip_lens); 3418 htolem32(&txdesc->type_tucmd_mlhl, type_tucmd_mlhl); 3419 htolem32(&txdesc->seqnum_seed, 0); 3420 htolem32(&txdesc->mss_l4len_idx, mss_l4len_idx); 3421 igc_txdesc_sync(txr, prod, 3422 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); 3423 3424 *cmd_type_len |= IGC_ADVTXD_DCMD_TSE; 3425 *olinfo_status |= IGC_TXD_POPTS_TXSM << 8; 3426 *olinfo_status |= paylen << IGC_ADVTXD_PAYLEN_SHIFT; 3427 3428 return 1; 3429 #endif /* notyet */ 3430 } 3431 3432 /********************************************************************* 3433 * 3434 * Allocate memory for rx_buffer structures. Since we use one 3435 * rx_buffer per received packet, the maximum number of rx_buffer's 3436 * that we'll need is equal to the number of receive descriptors 3437 * that we've allocated. 3438 * 3439 **********************************************************************/ 3440 static int 3441 igc_allocate_receive_buffers(struct rx_ring *rxr) 3442 { 3443 struct igc_softc *sc = rxr->sc; 3444 int error; 3445 3446 rxr->rx_buffers = 3447 kmem_zalloc(sc->num_rx_desc * sizeof(struct igc_rx_buf), KM_SLEEP); 3448 3449 for (int id = 0; id < sc->num_rx_desc; id++) { 3450 struct igc_rx_buf *rxbuf = &rxr->rx_buffers[id]; 3451 3452 error = bus_dmamap_create(rxr->rxdma.dma_tag, MCLBYTES, 1, 3453 MCLBYTES, 0, BUS_DMA_WAITOK, &rxbuf->map); 3454 if (error) { 3455 aprint_error_dev(sc->sc_dev, 3456 "unable to create RX DMA map\n"); 3457 goto fail; 3458 } 3459 } 3460 bus_dmamap_sync(rxr->rxdma.dma_tag, rxr->rxdma.dma_map, 0, 3461 rxr->rxdma.dma_map->dm_mapsize, 3462 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); 3463 3464 return 0; 3465 fail: 3466 return error; 3467 } 3468 3469 /********************************************************************* 3470 * 3471 * Allocate and initialize receive structures. 3472 * 3473 **********************************************************************/ 3474 static int 3475 igc_setup_receive_structures(struct igc_softc *sc) 3476 { 3477 3478 for (int iq = 0; iq < sc->sc_nqueues; iq++) { 3479 struct rx_ring *rxr = &sc->rx_rings[iq]; 3480 3481 if (igc_setup_receive_ring(rxr)) 3482 goto fail; 3483 } 3484 3485 return 0; 3486 fail: 3487 igc_free_receive_structures(sc); 3488 return ENOBUFS; 3489 } 3490 3491 /********************************************************************* 3492 * 3493 * Initialize a receive ring and its buffers. 3494 * 3495 **********************************************************************/ 3496 static int 3497 igc_setup_receive_ring(struct rx_ring *rxr) 3498 { 3499 struct igc_softc *sc = rxr->sc; 3500 const int rsize = roundup2( 3501 sc->num_rx_desc * sizeof(union igc_adv_rx_desc), IGC_DBA_ALIGN); 3502 3503 /* Clear the ring contents. */ 3504 memset(rxr->rx_base, 0, rsize); 3505 3506 if (igc_allocate_receive_buffers(rxr)) 3507 return ENOMEM; 3508 3509 /* Setup our descriptor indices. */ 3510 rxr->next_to_check = 0; 3511 rxr->last_desc_filled = 0; 3512 3513 mutex_init(&rxr->rxr_lock, MUTEX_DEFAULT, IPL_NET); 3514 3515 return 0; 3516 } 3517 3518 /********************************************************************* 3519 * 3520 * Enable receive unit. 3521 * 3522 **********************************************************************/ 3523 static void 3524 igc_initialize_receive_unit(struct igc_softc *sc) 3525 { 3526 struct ifnet *ifp = &sc->sc_ec.ec_if; 3527 struct igc_hw *hw = &sc->hw; 3528 uint32_t rctl, rxcsum, srrctl; 3529 3530 DPRINTF(RX, "called\n"); 3531 3532 /* 3533 * Make sure receives are disabled while setting 3534 * up the descriptor ring. 3535 */ 3536 rctl = IGC_READ_REG(hw, IGC_RCTL); 3537 IGC_WRITE_REG(hw, IGC_RCTL, rctl & ~IGC_RCTL_EN); 3538 3539 /* Setup the Receive Control Register */ 3540 rctl &= ~(3 << IGC_RCTL_MO_SHIFT); 3541 rctl |= IGC_RCTL_EN | IGC_RCTL_BAM | IGC_RCTL_LBM_NO | 3542 IGC_RCTL_RDMTS_HALF | (hw->mac.mc_filter_type << IGC_RCTL_MO_SHIFT); 3543 3544 #if 1 3545 /* Do not store bad packets */ 3546 rctl &= ~IGC_RCTL_SBP; 3547 #else 3548 /* for debug */ 3549 rctl |= IGC_RCTL_SBP; 3550 #endif 3551 3552 /* Enable Long Packet receive */ 3553 if (sc->hw.mac.max_frame_size > ETHER_MAX_LEN) 3554 rctl |= IGC_RCTL_LPE; 3555 else 3556 rctl &= ~IGC_RCTL_LPE; 3557 3558 /* Strip the CRC */ 3559 rctl |= IGC_RCTL_SECRC; 3560 3561 /* 3562 * Set the interrupt throttling rate. Value is calculated 3563 * as DEFAULT_ITR = 1/(MAX_INTS_PER_SEC * 256ns) 3564 * 3565 * XXX Sync with Linux, especially for jumbo MTU or TSO. 3566 * XXX Shouldn't be here? 3567 */ 3568 IGC_WRITE_REG(hw, IGC_ITR, DEFAULT_ITR); 3569 3570 rxcsum = IGC_READ_REG(hw, IGC_RXCSUM); 3571 rxcsum &= ~(IGC_RXCSUM_IPOFL | IGC_RXCSUM_TUOFL | IGC_RXCSUM_PCSD); 3572 if (ifp->if_capenable & IFCAP_CSUM_IPv4_Rx) 3573 rxcsum |= IGC_RXCSUM_IPOFL; 3574 if (ifp->if_capenable & (IFCAP_CSUM_TCPv4_Rx | IFCAP_CSUM_UDPv4_Rx | 3575 IFCAP_CSUM_TCPv6_Rx | IFCAP_CSUM_UDPv6_Rx)) 3576 rxcsum |= IGC_RXCSUM_TUOFL; 3577 if (sc->sc_nqueues > 1) 3578 rxcsum |= IGC_RXCSUM_PCSD; 3579 IGC_WRITE_REG(hw, IGC_RXCSUM, rxcsum); 3580 3581 if (sc->sc_nqueues > 1) 3582 igc_initialize_rss_mapping(sc); 3583 3584 srrctl = 0; 3585 #if 0 3586 srrctl |= 4096 >> IGC_SRRCTL_BSIZEPKT_SHIFT; 3587 rctl |= IGC_RCTL_SZ_4096 | IGC_RCTL_BSEX; 3588 #else 3589 srrctl |= 2048 >> IGC_SRRCTL_BSIZEPKT_SHIFT; 3590 rctl |= IGC_RCTL_SZ_2048; 3591 #endif 3592 3593 /* 3594 * If TX flow control is disabled and there's > 1 queue defined, 3595 * enable DROP. 3596 * 3597 * This drops frames rather than hanging the RX MAC for all queues. 3598 */ 3599 if (sc->sc_nqueues > 1 && 3600 (sc->fc == igc_fc_none || sc->fc == igc_fc_rx_pause)) 3601 srrctl |= IGC_SRRCTL_DROP_EN; 3602 3603 /* Setup the Base and Length of the RX descriptor rings. */ 3604 for (int iq = 0; iq < sc->sc_nqueues; iq++) { 3605 struct rx_ring *rxr = &sc->rx_rings[iq]; 3606 const uint64_t bus_addr = 3607 rxr->rxdma.dma_map->dm_segs[0].ds_addr; 3608 3609 IGC_WRITE_REG(hw, IGC_RXDCTL(iq), 0); 3610 3611 srrctl |= IGC_SRRCTL_DESCTYPE_ADV_ONEBUF; 3612 3613 IGC_WRITE_REG(hw, IGC_RDLEN(iq), 3614 sc->num_rx_desc * sizeof(union igc_adv_rx_desc)); 3615 IGC_WRITE_REG(hw, IGC_RDBAH(iq), (uint32_t)(bus_addr >> 32)); 3616 IGC_WRITE_REG(hw, IGC_RDBAL(iq), (uint32_t)bus_addr); 3617 IGC_WRITE_REG(hw, IGC_SRRCTL(iq), srrctl); 3618 3619 /* Setup the Head and Tail Descriptor Pointers */ 3620 IGC_WRITE_REG(hw, IGC_RDH(iq), 0); 3621 IGC_WRITE_REG(hw, IGC_RDT(iq), 0 /* XXX rxr->last_desc_filled */); 3622 3623 /* Enable this Queue */ 3624 uint32_t rxdctl = IGC_READ_REG(hw, IGC_RXDCTL(iq)); 3625 rxdctl |= IGC_RXDCTL_QUEUE_ENABLE; 3626 rxdctl &= 0xFFF00000; 3627 rxdctl |= IGC_RX_PTHRESH; 3628 rxdctl |= IGC_RX_HTHRESH << 8; 3629 rxdctl |= IGC_RX_WTHRESH << 16; 3630 IGC_WRITE_REG(hw, IGC_RXDCTL(iq), rxdctl); 3631 } 3632 3633 /* Make sure VLAN Filters are off */ 3634 rctl &= ~IGC_RCTL_VFE; 3635 3636 /* Write out the settings */ 3637 IGC_WRITE_REG(hw, IGC_RCTL, rctl); 3638 } 3639 3640 /********************************************************************* 3641 * 3642 * Free all receive rings. 3643 * 3644 **********************************************************************/ 3645 static void 3646 igc_free_receive_structures(struct igc_softc *sc) 3647 { 3648 3649 for (int iq = 0; iq < sc->sc_nqueues; iq++) { 3650 struct rx_ring *rxr = &sc->rx_rings[iq]; 3651 3652 igc_free_receive_buffers(rxr); 3653 } 3654 } 3655 3656 /********************************************************************* 3657 * 3658 * Free receive ring data structures 3659 * 3660 **********************************************************************/ 3661 static void 3662 igc_free_receive_buffers(struct rx_ring *rxr) 3663 { 3664 struct igc_softc *sc = rxr->sc; 3665 3666 if (rxr->rx_buffers != NULL) { 3667 for (int id = 0; id < sc->num_rx_desc; id++) { 3668 struct igc_rx_buf *rxbuf = &rxr->rx_buffers[id]; 3669 bus_dmamap_t map = rxbuf->map; 3670 3671 if (rxbuf->buf != NULL) { 3672 bus_dmamap_sync(rxr->rxdma.dma_tag, map, 3673 0, map->dm_mapsize, BUS_DMASYNC_POSTREAD); 3674 bus_dmamap_unload(rxr->rxdma.dma_tag, map); 3675 m_freem(rxbuf->buf); 3676 rxbuf->buf = NULL; 3677 } 3678 bus_dmamap_destroy(rxr->rxdma.dma_tag, map); 3679 rxbuf->map = NULL; 3680 } 3681 kmem_free(rxr->rx_buffers, 3682 sc->num_rx_desc * sizeof(struct igc_rx_buf)); 3683 rxr->rx_buffers = NULL; 3684 } 3685 3686 mutex_destroy(&rxr->rxr_lock); 3687 } 3688 3689 /********************************************************************* 3690 * 3691 * Clear status registers in all RX descriptors. 3692 * 3693 **********************************************************************/ 3694 static void 3695 igc_clear_receive_status(struct rx_ring *rxr) 3696 { 3697 struct igc_softc *sc = rxr->sc; 3698 3699 mutex_enter(&rxr->rxr_lock); 3700 3701 for (int id = 0; id < sc->num_rx_desc; id++) { 3702 union igc_adv_rx_desc *rxdesc = &rxr->rx_base[id]; 3703 3704 igc_rxdesc_sync(rxr, id, 3705 BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE); 3706 rxdesc->wb.upper.status_error = 0; 3707 igc_rxdesc_sync(rxr, id, 3708 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); 3709 } 3710 3711 mutex_exit(&rxr->rxr_lock); 3712 } 3713 3714 /* 3715 * Initialise the RSS mapping for NICs that support multiple transmit/ 3716 * receive rings. 3717 */ 3718 static void 3719 igc_initialize_rss_mapping(struct igc_softc *sc) 3720 { 3721 struct igc_hw *hw = &sc->hw; 3722 3723 /* 3724 * The redirection table controls which destination 3725 * queue each bucket redirects traffic to. 3726 * Each DWORD represents four queues, with the LSB 3727 * being the first queue in the DWORD. 3728 * 3729 * This just allocates buckets to queues using round-robin 3730 * allocation. 3731 * 3732 * NOTE: It Just Happens to line up with the default 3733 * RSS allocation method. 3734 */ 3735 3736 /* Warning FM follows */ 3737 uint32_t reta = 0; 3738 for (int i = 0; i < 128; i++) { 3739 const int shift = 0; /* XXXRO */ 3740 int queue_id = i % sc->sc_nqueues; 3741 /* Adjust if required */ 3742 queue_id <<= shift; 3743 3744 /* 3745 * The low 8 bits are for hash value (n+0); 3746 * The next 8 bits are for hash value (n+1), etc. 3747 */ 3748 reta >>= 8; 3749 reta |= ((uint32_t)queue_id) << 24; 3750 if ((i & 3) == 3) { 3751 IGC_WRITE_REG(hw, IGC_RETA(i >> 2), reta); 3752 reta = 0; 3753 } 3754 } 3755 3756 /* 3757 * MRQC: Multiple Receive Queues Command 3758 * Set queuing to RSS control, number depends on the device. 3759 */ 3760 3761 /* Set up random bits */ 3762 uint32_t rss_key[RSS_KEYSIZE / sizeof(uint32_t)]; 3763 rss_getkey((uint8_t *)rss_key); 3764 3765 /* Now fill our hash function seeds */ 3766 for (int i = 0; i < __arraycount(rss_key); i++) 3767 IGC_WRITE_REG_ARRAY(hw, IGC_RSSRK(0), i, rss_key[i]); 3768 3769 /* 3770 * Configure the RSS fields to hash upon. 3771 */ 3772 uint32_t mrqc = IGC_MRQC_ENABLE_RSS_4Q; 3773 mrqc |= IGC_MRQC_RSS_FIELD_IPV4 | IGC_MRQC_RSS_FIELD_IPV4_TCP; 3774 mrqc |= IGC_MRQC_RSS_FIELD_IPV6 | IGC_MRQC_RSS_FIELD_IPV6_TCP; 3775 mrqc |= IGC_MRQC_RSS_FIELD_IPV6_TCP_EX; 3776 3777 IGC_WRITE_REG(hw, IGC_MRQC, mrqc); 3778 } 3779 3780 /* 3781 * igc_get_hw_control sets the {CTRL_EXT|FWSM}:DRV_LOAD bit. 3782 * For ASF and Pass Through versions of f/w this means 3783 * that the driver is loaded. For AMT version type f/w 3784 * this means that the network i/f is open. 3785 */ 3786 static void 3787 igc_get_hw_control(struct igc_softc *sc) 3788 { 3789 const uint32_t ctrl_ext = IGC_READ_REG(&sc->hw, IGC_CTRL_EXT); 3790 3791 IGC_WRITE_REG(&sc->hw, IGC_CTRL_EXT, ctrl_ext | IGC_CTRL_EXT_DRV_LOAD); 3792 } 3793 3794 /* 3795 * igc_release_hw_control resets {CTRL_EXT|FWSM}:DRV_LOAD bit. 3796 * For ASF and Pass Through versions of f/w this means that 3797 * the driver is no longer loaded. For AMT versions of the 3798 * f/w this means that the network i/f is closed. 3799 */ 3800 static void 3801 igc_release_hw_control(struct igc_softc *sc) 3802 { 3803 const uint32_t ctrl_ext = IGC_READ_REG(&sc->hw, IGC_CTRL_EXT); 3804 3805 IGC_WRITE_REG(&sc->hw, IGC_CTRL_EXT, ctrl_ext & ~IGC_CTRL_EXT_DRV_LOAD); 3806 } 3807 3808 static int 3809 igc_is_valid_ether_addr(uint8_t *addr) 3810 { 3811 const char zero_addr[6] = { 0, 0, 0, 0, 0, 0 }; 3812 3813 if ((addr[0] & 1) || !bcmp(addr, zero_addr, ETHER_ADDR_LEN)) 3814 return 0; 3815 3816 return 1; 3817 } 3818 3819 static void 3820 igc_print_devinfo(struct igc_softc *sc) 3821 { 3822 device_t dev = sc->sc_dev; 3823 struct igc_hw *hw = &sc->hw; 3824 struct igc_phy_info *phy = &hw->phy; 3825 u_int oui, model, rev; 3826 uint16_t id1, id2, nvm_ver, phy_ver; 3827 char descr[MII_MAX_DESCR_LEN]; 3828 3829 /* Print PHY Info */ 3830 id1 = phy->id >> 16; 3831 /* The revision field in phy->id is cleard and it's in phy->revision */ 3832 id2 = (phy->id & 0xfff0) | phy->revision; 3833 oui = MII_OUI(id1, id2); 3834 model = MII_MODEL(id2); 3835 rev = MII_REV(id2); 3836 mii_get_descr(descr, sizeof(descr), oui, model); 3837 if (descr[0]) 3838 aprint_normal_dev(dev, "PHY: %s, rev. %d\n", 3839 descr, rev); 3840 else 3841 aprint_normal_dev(dev, 3842 "PHY OUI 0x%06x, model 0x%04x, rev. %d\n", 3843 oui, model, rev); 3844 3845 /* Get NVM version */ 3846 hw->nvm.ops.read(hw, NVM_VERSION, 1, &nvm_ver); 3847 3848 /* Get PHY FW version */ 3849 phy->ops.read_reg(hw, 0x1e, &phy_ver); 3850 3851 aprint_normal_dev(dev, "ROM image version %x.%02x", 3852 (nvm_ver & NVM_VERSION_MAJOR) >> NVM_VERSION_MAJOR_SHIFT, 3853 (nvm_ver & NVM_VERSION_MINOR)); 3854 aprint_debug("(0x%04hx)", nvm_ver); 3855 3856 aprint_normal(", PHY FW version 0x%04hx\n", phy_ver); 3857 } 3858
Indexes created Tue Sep 30 11:09:46 GMT 2025