if_igc.c revision 1.3.2.4
1 /* $NetBSD: if_igc.c,v 1.3.2.4 2024/02/23 18:41:02 martin Exp $ */ 2 /* $OpenBSD: if_igc.c,v 1.13 2023/04/28 10:18:57 bluhm Exp $ */ 3 /*- 4 * SPDX-License-Identifier: BSD-2-Clause 5 * 6 * Copyright (c) 2016 Nicole Graziano <nicole (at) nextbsd.org> 7 * All rights reserved. 8 * Copyright (c) 2021 Rubicon Communications, LLC (Netgate) 9 * 10 * Redistribution and use in source and binary forms, with or without 11 * modification, are permitted provided that the following conditions 12 * are met: 13 * 1. Redistributions of source code must retain the above copyright 14 * notice, this list of conditions and the following disclaimer. 15 * 2. Redistributions in binary form must reproduce the above copyright 16 * notice, this list of conditions and the following disclaimer in the 17 * documentation and/or other materials provided with the distribution. 18 * 19 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 20 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 21 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 22 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 23 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 24 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 25 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 26 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 27 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 28 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 29 * SUCH DAMAGE. 30 */ 31 32 #include <sys/cdefs.h> 33 __KERNEL_RCSID(0, "$NetBSD: if_igc.c,v 1.3.2.4 2024/02/23 18:41:02 martin Exp $"); 34 35 #ifdef _KERNEL_OPT 36 #include "opt_net_mpsafe.h" 37 #include "opt_if_igc.h" 38 #if 0 /* notyet */ 39 #include "vlan.h" 40 #endif 41 #endif 42 43 #include <sys/param.h> 44 #include <sys/systm.h> 45 #include <sys/bus.h> 46 #include <sys/cpu.h> 47 #include <sys/device.h> 48 #include <sys/endian.h> 49 #include <sys/intr.h> 50 #include <sys/interrupt.h> 51 #include <sys/kernel.h> 52 #include <sys/kmem.h> 53 #include <sys/mbuf.h> 54 #include <sys/mutex.h> 55 #include <sys/socket.h> 56 #include <sys/workqueue.h> 57 #include <sys/xcall.h> 58 59 #include <net/bpf.h> 60 #include <net/if.h> 61 #include <net/if_ether.h> 62 #include <net/if_media.h> 63 #include <net/if_vlanvar.h> 64 #include <net/rss_config.h> 65 66 #include <netinet/in.h> 67 #include <netinet/ip.h> 68 #include <netinet/ip6.h> 69 #include <netinet/tcp.h> 70 71 #include <dev/pci/pcivar.h> 72 #include <dev/pci/pcireg.h> 73 #include <dev/pci/pcidevs.h> 74 75 #include <dev/pci/igc/if_igc.h> 76 #include <dev/pci/igc/igc_evcnt.h> 77 #include <dev/pci/igc/igc_hw.h> 78 #include <dev/mii/miivar.h> 79 80 #define IGC_WORKQUEUE_PRI PRI_SOFTNET 81 82 #ifndef IGC_RX_INTR_PROCESS_LIMIT_DEFAULT 83 #define IGC_RX_INTR_PROCESS_LIMIT_DEFAULT 0 84 #endif 85 #ifndef IGC_TX_INTR_PROCESS_LIMIT_DEFAULT 86 #define IGC_TX_INTR_PROCESS_LIMIT_DEFAULT 0 87 #endif 88 89 #ifndef IGC_RX_PROCESS_LIMIT_DEFAULT 90 #define IGC_RX_PROCESS_LIMIT_DEFAULT 256 91 #endif 92 #ifndef IGC_TX_PROCESS_LIMIT_DEFAULT 93 #define IGC_TX_PROCESS_LIMIT_DEFAULT 256 94 #endif 95 96 #define htolem32(p, x) (*((uint32_t *)(p)) = htole32(x)) 97 #define htolem64(p, x) (*((uint64_t *)(p)) = htole64(x)) 98 99 static const struct igc_product { 100 pci_vendor_id_t igcp_vendor; 101 pci_product_id_t igcp_product; 102 const char *igcp_name; 103 } igc_products[] = { 104 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_I225_IT, 105 "Intel(R) Ethernet Controller I225-IT(2)" }, 106 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_I226_LM, 107 "Intel(R) Ethernet Controller I226-LM" }, 108 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_I226_V, 109 "Intel(R) Ethernet Controller I226-V" }, 110 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_I226_IT, 111 "Intel(R) Ethernet Controller I226-IT" }, 112 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_I221_V, 113 "Intel(R) Ethernet Controller I221-V" }, 114 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_I226_BLANK_NVM, 115 "Intel(R) Ethernet Controller I226(blankNVM)" }, 116 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_I225_LM, 117 "Intel(R) Ethernet Controller I225-LM" }, 118 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_I225_V, 119 "Intel(R) Ethernet Controller I225-V" }, 120 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_I220_V, 121 "Intel(R) Ethernet Controller I220-V" }, 122 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_I225_I, 123 "Intel(R) Ethernet Controller I225-I" }, 124 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_I225_BLANK_NVM, 125 "Intel(R) Ethernet Controller I225(blankNVM)" }, 126 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_I225_K, 127 "Intel(R) Ethernet Controller I225-K" }, 128 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_I225_K2, 129 "Intel(R) Ethernet Controller I225-K(2)" }, 130 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_I226_K, 131 "Intel(R) Ethernet Controller I226-K" }, 132 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_I225_LMVP, 133 "Intel(R) Ethernet Controller I225-LMvP(2)" }, 134 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_I226_LMVP, 135 "Intel(R) Ethernet Controller I226-LMvP" }, 136 { 0, 0, NULL }, 137 }; 138 139 #define IGC_DF_CFG 0x1 140 #define IGC_DF_TX 0x2 141 #define IGC_DF_RX 0x4 142 #define IGC_DF_MISC 0x8 143 144 #ifdef IGC_DEBUG_FLAGS 145 int igc_debug_flags = IGC_DEBUG_FLAGS; 146 #else 147 int igc_debug_flags = 0; 148 #endif 149 150 #define DPRINTF(flag, fmt, args...) do { \ 151 if (igc_debug_flags & (IGC_DF_ ## flag)) \ 152 printf("%s: %d: " fmt, __func__, __LINE__, ##args); \ 153 } while (0) 154 155 /********************************************************************* 156 * Function Prototypes 157 *********************************************************************/ 158 static int igc_match(device_t, cfdata_t, void *); 159 static void igc_attach(device_t, device_t, void *); 160 static int igc_detach(device_t, int); 161 162 static void igc_identify_hardware(struct igc_softc *); 163 static int igc_adjust_nqueues(struct igc_softc *); 164 static int igc_allocate_pci_resources(struct igc_softc *); 165 static int igc_allocate_interrupts(struct igc_softc *); 166 static int igc_allocate_queues(struct igc_softc *); 167 static void igc_free_pci_resources(struct igc_softc *); 168 static void igc_free_interrupts(struct igc_softc *); 169 static void igc_free_queues(struct igc_softc *); 170 static void igc_reset(struct igc_softc *); 171 static void igc_init_dmac(struct igc_softc *, uint32_t); 172 static int igc_setup_interrupts(struct igc_softc *); 173 static void igc_attach_counters(struct igc_softc *sc); 174 static void igc_detach_counters(struct igc_softc *sc); 175 static void igc_update_counters(struct igc_softc *sc); 176 static void igc_clear_counters(struct igc_softc *sc); 177 static int igc_setup_msix(struct igc_softc *); 178 static int igc_setup_msi(struct igc_softc *); 179 static int igc_setup_intx(struct igc_softc *); 180 static int igc_dma_malloc(struct igc_softc *, bus_size_t, 181 struct igc_dma_alloc *); 182 static void igc_dma_free(struct igc_softc *, struct igc_dma_alloc *); 183 static void igc_setup_interface(struct igc_softc *); 184 185 static int igc_init(struct ifnet *); 186 static int igc_init_locked(struct igc_softc *); 187 static void igc_start(struct ifnet *); 188 static int igc_transmit(struct ifnet *, struct mbuf *); 189 static void igc_tx_common_locked(struct ifnet *, struct tx_ring *, int); 190 static bool igc_txeof(struct tx_ring *, u_int); 191 static void igc_intr_barrier(struct igc_softc *); 192 static void igc_stop(struct ifnet *, int); 193 static void igc_stop_locked(struct igc_softc *); 194 static int igc_ioctl(struct ifnet *, u_long, void *); 195 #ifdef IF_RXR 196 static int igc_rxrinfo(struct igc_softc *, struct if_rxrinfo *); 197 #endif 198 static void igc_rxfill(struct rx_ring *); 199 static void igc_rxrefill(struct rx_ring *, int); 200 static bool igc_rxeof(struct rx_ring *, u_int); 201 static int igc_rx_checksum(struct igc_queue *, uint64_t, uint32_t, 202 uint32_t); 203 static void igc_watchdog(struct ifnet *); 204 static void igc_tick(void *); 205 static void igc_media_status(struct ifnet *, struct ifmediareq *); 206 static int igc_media_change(struct ifnet *); 207 static int igc_ifflags_cb(struct ethercom *); 208 static void igc_set_filter(struct igc_softc *); 209 static void igc_update_link_status(struct igc_softc *); 210 static int igc_get_buf(struct rx_ring *, int, bool); 211 static int igc_tx_ctx_setup(struct tx_ring *, struct mbuf *, int, 212 uint32_t *, uint32_t *); 213 static int igc_tso_setup(struct tx_ring *, struct mbuf *, int, 214 uint32_t *, uint32_t *); 215 216 static void igc_configure_queues(struct igc_softc *); 217 static void igc_set_queues(struct igc_softc *, uint32_t, uint32_t, int); 218 static void igc_enable_queue(struct igc_softc *, uint32_t); 219 static void igc_enable_intr(struct igc_softc *); 220 static void igc_disable_intr(struct igc_softc *); 221 static int igc_intr_link(void *); 222 static int igc_intr_queue(void *); 223 static int igc_intr(void *); 224 static void igc_handle_queue(void *); 225 static void igc_handle_queue_work(struct work *, void *); 226 static void igc_sched_handle_queue(struct igc_softc *, struct igc_queue *); 227 static void igc_barrier_handle_queue(struct igc_softc *); 228 229 static int igc_allocate_transmit_buffers(struct tx_ring *); 230 static int igc_setup_transmit_structures(struct igc_softc *); 231 static int igc_setup_transmit_ring(struct tx_ring *); 232 static void igc_initialize_transmit_unit(struct igc_softc *); 233 static void igc_free_transmit_structures(struct igc_softc *); 234 static void igc_free_transmit_buffers(struct tx_ring *); 235 static void igc_withdraw_transmit_packets(struct tx_ring *, bool); 236 static int igc_allocate_receive_buffers(struct rx_ring *); 237 static int igc_setup_receive_structures(struct igc_softc *); 238 static int igc_setup_receive_ring(struct rx_ring *); 239 static void igc_initialize_receive_unit(struct igc_softc *); 240 static void igc_free_receive_structures(struct igc_softc *); 241 static void igc_free_receive_buffers(struct rx_ring *); 242 static void igc_clear_receive_status(struct rx_ring *); 243 static void igc_initialize_rss_mapping(struct igc_softc *); 244 245 static void igc_get_hw_control(struct igc_softc *); 246 static void igc_release_hw_control(struct igc_softc *); 247 static int igc_is_valid_ether_addr(uint8_t *); 248 static void igc_print_devinfo(struct igc_softc *); 249 250 CFATTACH_DECL3_NEW(igc, sizeof(struct igc_softc), 251 igc_match, igc_attach, igc_detach, NULL, NULL, NULL, 0); 252 253 static inline int 254 igc_txdesc_incr(struct igc_softc *sc, int id) 255 { 256 257 if (++id == sc->num_tx_desc) 258 id = 0; 259 return id; 260 } 261 262 static inline int __unused 263 igc_txdesc_decr(struct igc_softc *sc, int id) 264 { 265 266 if (--id < 0) 267 id = sc->num_tx_desc - 1; 268 return id; 269 } 270 271 static inline void 272 igc_txdesc_sync(struct tx_ring *txr, int id, int ops) 273 { 274 275 bus_dmamap_sync(txr->txdma.dma_tag, txr->txdma.dma_map, 276 id * sizeof(union igc_adv_tx_desc), sizeof(union igc_adv_tx_desc), 277 ops); 278 } 279 280 static inline int 281 igc_rxdesc_incr(struct igc_softc *sc, int id) 282 { 283 284 if (++id == sc->num_rx_desc) 285 id = 0; 286 return id; 287 } 288 289 static inline int 290 igc_rxdesc_decr(struct igc_softc *sc, int id) 291 { 292 293 if (--id < 0) 294 id = sc->num_rx_desc - 1; 295 return id; 296 } 297 298 static inline void 299 igc_rxdesc_sync(struct rx_ring *rxr, int id, int ops) 300 { 301 302 bus_dmamap_sync(rxr->rxdma.dma_tag, rxr->rxdma.dma_map, 303 id * sizeof(union igc_adv_rx_desc), sizeof(union igc_adv_rx_desc), 304 ops); 305 } 306 307 static const struct igc_product * 308 igc_lookup(const struct pci_attach_args *pa) 309 { 310 const struct igc_product *igcp; 311 312 for (igcp = igc_products; igcp->igcp_name != NULL; igcp++) { 313 if (PCI_VENDOR(pa->pa_id) == igcp->igcp_vendor && 314 PCI_PRODUCT(pa->pa_id) == igcp->igcp_product) 315 return igcp; 316 } 317 return NULL; 318 } 319 320 /********************************************************************* 321 * Device identification routine 322 * 323 * igc_match determines if the driver should be loaded on 324 * adapter based on PCI vendor/device id of the adapter. 325 * 326 * return 0 on success, positive on failure 327 *********************************************************************/ 328 static int 329 igc_match(device_t parent, cfdata_t match, void *aux) 330 { 331 struct pci_attach_args *pa = aux; 332 333 if (igc_lookup(pa) != NULL) 334 return 1; 335 336 return 0; 337 } 338 339 /********************************************************************* 340 * Device initialization routine 341 * 342 * The attach entry point is called when the driver is being loaded. 343 * This routine identifies the type of hardware, allocates all resources 344 * and initializes the hardware. 345 * 346 * return 0 on success, positive on failure 347 *********************************************************************/ 348 static void 349 igc_attach(device_t parent, device_t self, void *aux) 350 { 351 struct pci_attach_args *pa = aux; 352 struct igc_softc *sc = device_private(self); 353 struct igc_hw *hw = &sc->hw; 354 355 const struct igc_product *igcp = igc_lookup(pa); 356 KASSERT(igcp != NULL); 357 pci_aprint_devinfo_fancy(pa, "Ethernet controller", igcp->igcp_name, 1); 358 359 sc->sc_dev = self; 360 callout_init(&sc->sc_tick_ch, CALLOUT_MPSAFE); 361 callout_setfunc(&sc->sc_tick_ch, igc_tick, sc); 362 sc->sc_core_stopping = false; 363 364 sc->osdep.os_sc = sc; 365 sc->osdep.os_pa = *pa; 366 #ifdef __aarch64__ 367 /* 368 * XXX PR port-arm/57643 369 * 64-bit DMA does not work at least for LX2K with 32/64GB memory. 370 * smmu(4) support may be required. 371 */ 372 sc->osdep.os_dmat = pa->pa_dmat; 373 #else 374 sc->osdep.os_dmat = pci_dma64_available(pa) ? 375 pa->pa_dmat64 : pa->pa_dmat; 376 #endif 377 378 /* Determine hardware and mac info */ 379 igc_identify_hardware(sc); 380 381 sc->num_tx_desc = IGC_DEFAULT_TXD; 382 sc->num_rx_desc = IGC_DEFAULT_RXD; 383 384 /* Setup PCI resources */ 385 if (igc_allocate_pci_resources(sc)) { 386 aprint_error_dev(sc->sc_dev, 387 "unable to allocate PCI resources\n"); 388 goto err_pci; 389 } 390 391 if (igc_allocate_interrupts(sc)) { 392 aprint_error_dev(sc->sc_dev, "unable to allocate interrupts\n"); 393 goto err_pci; 394 } 395 396 /* Allocate TX/RX queues */ 397 if (igc_allocate_queues(sc)) { 398 aprint_error_dev(sc->sc_dev, "unable to allocate queues\n"); 399 goto err_alloc_intr; 400 } 401 402 /* Do shared code initialization */ 403 if (igc_setup_init_funcs(hw, true)) { 404 aprint_error_dev(sc->sc_dev, "unable to initialize\n"); 405 goto err_alloc_intr; 406 } 407 408 hw->mac.autoneg = DO_AUTO_NEG; 409 hw->phy.autoneg_wait_to_complete = false; 410 hw->phy.autoneg_advertised = AUTONEG_ADV_DEFAULT; 411 412 /* Copper options. */ 413 if (hw->phy.media_type == igc_media_type_copper) 414 hw->phy.mdix = AUTO_ALL_MODES; 415 416 /* Set the max frame size. */ 417 sc->hw.mac.max_frame_size = 9234; 418 419 /* Allocate multicast array memory. */ 420 sc->mta = kmem_alloc(IGC_MTA_LEN, KM_SLEEP); 421 422 /* Check SOL/IDER usage. */ 423 if (igc_check_reset_block(hw)) { 424 aprint_error_dev(sc->sc_dev, 425 "PHY reset is blocked due to SOL/IDER session\n"); 426 } 427 428 /* Disable Energy Efficient Ethernet. */ 429 sc->hw.dev_spec._i225.eee_disable = true; 430 431 igc_reset_hw(hw); 432 433 /* Make sure we have a good EEPROM before we read from it. */ 434 if (igc_validate_nvm_checksum(hw) < 0) { 435 /* 436 * Some PCI-E parts fail the first check due to 437 * the link being in sleep state, call it again, 438 * if it fails a second time its a real issue. 439 */ 440 if (igc_validate_nvm_checksum(hw) < 0) { 441 aprint_error_dev(sc->sc_dev, 442 "EEPROM checksum invalid\n"); 443 goto err_late; 444 } 445 } 446 447 /* Copy the permanent MAC address out of the EEPROM. */ 448 if (igc_read_mac_addr(hw) < 0) { 449 aprint_error_dev(sc->sc_dev, 450 "unable to read MAC address from EEPROM\n"); 451 goto err_late; 452 } 453 454 if (!igc_is_valid_ether_addr(hw->mac.addr)) { 455 aprint_error_dev(sc->sc_dev, "invalid MAC address\n"); 456 goto err_late; 457 } 458 459 if (igc_setup_interrupts(sc)) 460 goto err_late; 461 462 /* Attach counters. */ 463 igc_attach_counters(sc); 464 465 /* Setup OS specific network interface. */ 466 igc_setup_interface(sc); 467 468 igc_print_devinfo(sc); 469 470 igc_reset(sc); 471 hw->mac.get_link_status = true; 472 igc_update_link_status(sc); 473 474 /* The driver can now take control from firmware. */ 475 igc_get_hw_control(sc); 476 477 aprint_normal_dev(sc->sc_dev, "Ethernet address %s\n", 478 ether_sprintf(sc->hw.mac.addr)); 479 480 if (pmf_device_register(self, NULL, NULL)) 481 pmf_class_network_register(self, &sc->sc_ec.ec_if); 482 else 483 aprint_error_dev(self, "couldn't establish power handler\n"); 484 485 return; 486 487 err_late: 488 igc_release_hw_control(sc); 489 err_alloc_intr: 490 igc_free_interrupts(sc); 491 err_pci: 492 igc_free_pci_resources(sc); 493 kmem_free(sc->mta, IGC_MTA_LEN); 494 } 495 496 /********************************************************************* 497 * Device removal routine 498 * 499 * The detach entry point is called when the driver is being removed. 500 * This routine stops the adapter and deallocates all the resources 501 * that were allocated for driver operation. 502 * 503 * return 0 on success, positive on failure 504 *********************************************************************/ 505 static int 506 igc_detach(device_t self, int flags) 507 { 508 struct igc_softc *sc = device_private(self); 509 struct ifnet *ifp = &sc->sc_ec.ec_if; 510 511 mutex_enter(&sc->sc_core_lock); 512 igc_stop_locked(sc); 513 mutex_exit(&sc->sc_core_lock); 514 515 igc_detach_counters(sc); 516 517 igc_free_queues(sc); 518 519 igc_phy_hw_reset(&sc->hw); 520 igc_release_hw_control(sc); 521 522 ether_ifdetach(ifp); 523 if_detach(ifp); 524 ifmedia_fini(&sc->media); 525 526 igc_free_interrupts(sc); 527 igc_free_pci_resources(sc); 528 kmem_free(sc->mta, IGC_MTA_LEN); 529 530 mutex_destroy(&sc->sc_core_lock); 531 532 return 0; 533 } 534 535 static void 536 igc_identify_hardware(struct igc_softc *sc) 537 { 538 struct igc_osdep *os = &sc->osdep; 539 struct pci_attach_args *pa = &os->os_pa; 540 541 /* Save off the information about this board. */ 542 sc->hw.device_id = PCI_PRODUCT(pa->pa_id); 543 544 /* Do shared code init and setup. */ 545 if (igc_set_mac_type(&sc->hw)) { 546 aprint_error_dev(sc->sc_dev, "unable to identify hardware\n"); 547 return; 548 } 549 } 550 551 static int 552 igc_allocate_pci_resources(struct igc_softc *sc) 553 { 554 struct igc_osdep *os = &sc->osdep; 555 struct pci_attach_args *pa = &os->os_pa; 556 557 /* 558 * Enable bus mastering and memory-mapped I/O for sure. 559 */ 560 pcireg_t csr = 561 pci_conf_read(pa->pa_pc, pa->pa_tag, PCI_COMMAND_STATUS_REG); 562 csr |= PCI_COMMAND_MASTER_ENABLE | PCI_COMMAND_MEM_ENABLE; 563 pci_conf_write(pa->pa_pc, pa->pa_tag, PCI_COMMAND_STATUS_REG, csr); 564 565 const pcireg_t memtype = 566 pci_mapreg_type(pa->pa_pc, pa->pa_tag, IGC_PCIREG); 567 if (pci_mapreg_map(pa, IGC_PCIREG, memtype, 0, &os->os_memt, 568 &os->os_memh, &os->os_membase, &os->os_memsize)) { 569 aprint_error_dev(sc->sc_dev, "unable to map registers\n"); 570 return ENXIO; 571 } 572 573 sc->hw.hw_addr = os->os_membase; 574 sc->hw.back = os; 575 576 return 0; 577 } 578 579 static int __unused 580 igc_adjust_nqueues(struct igc_softc *sc) 581 { 582 struct pci_attach_args *pa = &sc->osdep.os_pa; 583 int nqueues = MIN(IGC_MAX_NQUEUES, ncpu); 584 585 const int nmsix = pci_msix_count(pa->pa_pc, pa->pa_tag); 586 if (nmsix <= 1) 587 nqueues = 1; 588 else if (nmsix < nqueues + 1) 589 nqueues = nmsix - 1; 590 591 return nqueues; 592 } 593 594 static int 595 igc_allocate_interrupts(struct igc_softc *sc) 596 { 597 struct pci_attach_args *pa = &sc->osdep.os_pa; 598 int error; 599 600 #ifndef IGC_DISABLE_MSIX 601 const int nqueues = igc_adjust_nqueues(sc); 602 if (nqueues > 1) { 603 sc->sc_nintrs = nqueues + 1; 604 error = pci_msix_alloc_exact(pa, &sc->sc_intrs, sc->sc_nintrs); 605 if (!error) { 606 sc->sc_nqueues = nqueues; 607 sc->sc_intr_type = PCI_INTR_TYPE_MSIX; 608 return 0; 609 } 610 } 611 #endif 612 613 /* fallback to MSI */ 614 sc->sc_nintrs = sc->sc_nqueues = 1; 615 616 #ifndef IGC_DISABLE_MSI 617 error = pci_msi_alloc_exact(pa, &sc->sc_intrs, sc->sc_nintrs); 618 if (!error) { 619 sc->sc_intr_type = PCI_INTR_TYPE_MSI; 620 return 0; 621 } 622 #endif 623 624 /* fallback to INTx */ 625 626 error = pci_intx_alloc(pa, &sc->sc_intrs); 627 if (!error) { 628 sc->sc_intr_type = PCI_INTR_TYPE_INTX; 629 return 0; 630 } 631 632 return error; 633 } 634 635 static int 636 igc_allocate_queues(struct igc_softc *sc) 637 { 638 device_t dev = sc->sc_dev; 639 int rxconf = 0, txconf = 0; 640 641 /* Allocate the top level queue structs. */ 642 sc->queues = 643 kmem_zalloc(sc->sc_nqueues * sizeof(struct igc_queue), KM_SLEEP); 644 645 /* Allocate the TX ring. */ 646 sc->tx_rings = 647 kmem_zalloc(sc->sc_nqueues * sizeof(struct tx_ring), KM_SLEEP); 648 649 /* Allocate the RX ring. */ 650 sc->rx_rings = 651 kmem_zalloc(sc->sc_nqueues * sizeof(struct rx_ring), KM_SLEEP); 652 653 /* Set up the TX queues. */ 654 for (int iq = 0; iq < sc->sc_nqueues; iq++, txconf++) { 655 struct tx_ring *txr = &sc->tx_rings[iq]; 656 const int tsize = roundup2( 657 sc->num_tx_desc * sizeof(union igc_adv_tx_desc), 658 IGC_DBA_ALIGN); 659 660 txr->sc = sc; 661 txr->txr_igcq = &sc->queues[iq]; 662 txr->me = iq; 663 if (igc_dma_malloc(sc, tsize, &txr->txdma)) { 664 aprint_error_dev(dev, 665 "unable to allocate TX descriptor\n"); 666 goto fail; 667 } 668 txr->tx_base = (union igc_adv_tx_desc *)txr->txdma.dma_vaddr; 669 memset(txr->tx_base, 0, tsize); 670 } 671 672 /* Prepare transmit descriptors and buffers. */ 673 if (igc_setup_transmit_structures(sc)) { 674 aprint_error_dev(dev, "unable to setup transmit structures\n"); 675 goto fail; 676 } 677 678 /* Set up the RX queues. */ 679 for (int iq = 0; iq < sc->sc_nqueues; iq++, rxconf++) { 680 struct rx_ring *rxr = &sc->rx_rings[iq]; 681 const int rsize = roundup2( 682 sc->num_rx_desc * sizeof(union igc_adv_rx_desc), 683 IGC_DBA_ALIGN); 684 685 rxr->sc = sc; 686 rxr->rxr_igcq = &sc->queues[iq]; 687 rxr->me = iq; 688 #ifdef OPENBSD 689 timeout_set(&rxr->rx_refill, igc_rxrefill, rxr); 690 #endif 691 if (igc_dma_malloc(sc, rsize, &rxr->rxdma)) { 692 aprint_error_dev(dev, 693 "unable to allocate RX descriptor\n"); 694 goto fail; 695 } 696 rxr->rx_base = (union igc_adv_rx_desc *)rxr->rxdma.dma_vaddr; 697 memset(rxr->rx_base, 0, rsize); 698 } 699 700 sc->rx_mbuf_sz = MCLBYTES; 701 /* Prepare receive descriptors and buffers. */ 702 if (igc_setup_receive_structures(sc)) { 703 aprint_error_dev(sc->sc_dev, 704 "unable to setup receive structures\n"); 705 goto fail; 706 } 707 708 /* Set up the queue holding structs. */ 709 for (int iq = 0; iq < sc->sc_nqueues; iq++) { 710 struct igc_queue *q = &sc->queues[iq]; 711 712 q->sc = sc; 713 q->txr = &sc->tx_rings[iq]; 714 q->rxr = &sc->rx_rings[iq]; 715 } 716 717 return 0; 718 719 fail: 720 for (struct rx_ring *rxr = sc->rx_rings; rxconf > 0; rxr++, rxconf--) 721 igc_dma_free(sc, &rxr->rxdma); 722 for (struct tx_ring *txr = sc->tx_rings; txconf > 0; txr++, txconf--) 723 igc_dma_free(sc, &txr->txdma); 724 725 kmem_free(sc->rx_rings, sc->sc_nqueues * sizeof(struct rx_ring)); 726 sc->rx_rings = NULL; 727 kmem_free(sc->tx_rings, sc->sc_nqueues * sizeof(struct tx_ring)); 728 sc->tx_rings = NULL; 729 kmem_free(sc->queues, sc->sc_nqueues * sizeof(struct igc_queue)); 730 sc->queues = NULL; 731 732 return ENOMEM; 733 } 734 735 static void 736 igc_free_pci_resources(struct igc_softc *sc) 737 { 738 struct igc_osdep *os = &sc->osdep; 739 740 if (os->os_membase != 0) 741 bus_space_unmap(os->os_memt, os->os_memh, os->os_memsize); 742 os->os_membase = 0; 743 } 744 745 static void 746 igc_free_interrupts(struct igc_softc *sc) 747 { 748 struct pci_attach_args *pa = &sc->osdep.os_pa; 749 pci_chipset_tag_t pc = pa->pa_pc; 750 751 for (int i = 0; i < sc->sc_nintrs; i++) { 752 if (sc->sc_ihs[i] != NULL) { 753 pci_intr_disestablish(pc, sc->sc_ihs[i]); 754 sc->sc_ihs[i] = NULL; 755 } 756 } 757 pci_intr_release(pc, sc->sc_intrs, sc->sc_nintrs); 758 } 759 760 static void 761 igc_free_queues(struct igc_softc *sc) 762 { 763 764 igc_free_receive_structures(sc); 765 for (int iq = 0; iq < sc->sc_nqueues; iq++) { 766 struct rx_ring *rxr = &sc->rx_rings[iq]; 767 768 igc_dma_free(sc, &rxr->rxdma); 769 } 770 771 igc_free_transmit_structures(sc); 772 for (int iq = 0; iq < sc->sc_nqueues; iq++) { 773 struct tx_ring *txr = &sc->tx_rings[iq]; 774 775 igc_dma_free(sc, &txr->txdma); 776 } 777 778 kmem_free(sc->rx_rings, sc->sc_nqueues * sizeof(struct rx_ring)); 779 kmem_free(sc->tx_rings, sc->sc_nqueues * sizeof(struct tx_ring)); 780 kmem_free(sc->queues, sc->sc_nqueues * sizeof(struct igc_queue)); 781 } 782 783 /********************************************************************* 784 * 785 * Initialize the hardware to a configuration as specified by the 786 * adapter structure. 787 * 788 **********************************************************************/ 789 static void 790 igc_reset(struct igc_softc *sc) 791 { 792 struct igc_hw *hw = &sc->hw; 793 794 /* Let the firmware know the OS is in control */ 795 igc_get_hw_control(sc); 796 797 /* 798 * Packet Buffer Allocation (PBA) 799 * Writing PBA sets the receive portion of the buffer 800 * the remainder is used for the transmit buffer. 801 */ 802 const uint32_t pba = IGC_PBA_34K; 803 804 /* 805 * These parameters control the automatic generation (Tx) and 806 * response (Rx) to Ethernet PAUSE frames. 807 * - High water mark should allow for at least two frames to be 808 * received after sending an XOFF. 809 * - Low water mark works best when it is very near the high water mark. 810 * This allows the receiver to restart by sending XON when it has 811 * drained a bit. Here we use an arbitrary value of 1500 which will 812 * restart after one full frame is pulled from the buffer. There 813 * could be several smaller frames in the buffer and if so they will 814 * not trigger the XON until their total number reduces the buffer 815 * by 1500. 816 * - The pause time is fairly large at 1000 x 512ns = 512 usec. 817 */ 818 const uint16_t rx_buffer_size = (pba & 0xffff) << 10; 819 820 hw->fc.high_water = rx_buffer_size - 821 roundup2(sc->hw.mac.max_frame_size, 1024); 822 /* 16-byte granularity */ 823 hw->fc.low_water = hw->fc.high_water - 16; 824 825 if (sc->fc) /* locally set flow control value? */ 826 hw->fc.requested_mode = sc->fc; 827 else 828 hw->fc.requested_mode = igc_fc_full; 829 830 hw->fc.pause_time = IGC_FC_PAUSE_TIME; 831 832 hw->fc.send_xon = true; 833 834 /* Issue a global reset */ 835 igc_reset_hw(hw); 836 IGC_WRITE_REG(hw, IGC_WUC, 0); 837 838 /* and a re-init */ 839 if (igc_init_hw(hw) < 0) { 840 aprint_error_dev(sc->sc_dev, "unable to reset hardware\n"); 841 return; 842 } 843 844 /* Setup DMA Coalescing */ 845 igc_init_dmac(sc, pba); 846 847 IGC_WRITE_REG(hw, IGC_VET, ETHERTYPE_VLAN); 848 igc_get_phy_info(hw); 849 igc_check_for_link(hw); 850 } 851 852 /********************************************************************* 853 * 854 * Initialize the DMA Coalescing feature 855 * 856 **********************************************************************/ 857 static void 858 igc_init_dmac(struct igc_softc *sc, uint32_t pba) 859 { 860 struct igc_hw *hw = &sc->hw; 861 const uint16_t max_frame_size = sc->hw.mac.max_frame_size; 862 uint32_t reg, status; 863 864 if (sc->dmac == 0) { /* Disabling it */ 865 reg = ~IGC_DMACR_DMAC_EN; /* XXXRO */ 866 IGC_WRITE_REG(hw, IGC_DMACR, reg); 867 DPRINTF(MISC, "DMA coalescing disabled\n"); 868 return; 869 } else { 870 device_printf(sc->sc_dev, "DMA coalescing enabled\n"); 871 } 872 873 /* Set starting threshold */ 874 IGC_WRITE_REG(hw, IGC_DMCTXTH, 0); 875 876 uint16_t hwm = 64 * pba - max_frame_size / 16; 877 if (hwm < 64 * (pba - 6)) 878 hwm = 64 * (pba - 6); 879 reg = IGC_READ_REG(hw, IGC_FCRTC); 880 reg &= ~IGC_FCRTC_RTH_COAL_MASK; 881 reg |= (hwm << IGC_FCRTC_RTH_COAL_SHIFT) & IGC_FCRTC_RTH_COAL_MASK; 882 IGC_WRITE_REG(hw, IGC_FCRTC, reg); 883 884 uint32_t dmac = pba - max_frame_size / 512; 885 if (dmac < pba - 10) 886 dmac = pba - 10; 887 reg = IGC_READ_REG(hw, IGC_DMACR); 888 reg &= ~IGC_DMACR_DMACTHR_MASK; 889 reg |= (dmac << IGC_DMACR_DMACTHR_SHIFT) & IGC_DMACR_DMACTHR_MASK; 890 891 /* transition to L0x or L1 if available..*/ 892 reg |= IGC_DMACR_DMAC_EN | IGC_DMACR_DMAC_LX_MASK; 893 894 /* Check if status is 2.5Gb backplane connection 895 * before configuration of watchdog timer, which is 896 * in msec values in 12.8usec intervals 897 * watchdog timer= msec values in 32usec intervals 898 * for non 2.5Gb connection 899 */ 900 status = IGC_READ_REG(hw, IGC_STATUS); 901 if ((status & IGC_STATUS_2P5_SKU) && 902 !(status & IGC_STATUS_2P5_SKU_OVER)) 903 reg |= (sc->dmac * 5) >> 6; 904 else 905 reg |= sc->dmac >> 5; 906 907 IGC_WRITE_REG(hw, IGC_DMACR, reg); 908 909 IGC_WRITE_REG(hw, IGC_DMCRTRH, 0); 910 911 /* Set the interval before transition */ 912 reg = IGC_READ_REG(hw, IGC_DMCTLX); 913 reg |= IGC_DMCTLX_DCFLUSH_DIS; 914 915 /* 916 * in 2.5Gb connection, TTLX unit is 0.4 usec 917 * which is 0x4*2 = 0xA. But delay is still 4 usec 918 */ 919 status = IGC_READ_REG(hw, IGC_STATUS); 920 if ((status & IGC_STATUS_2P5_SKU) && 921 !(status & IGC_STATUS_2P5_SKU_OVER)) 922 reg |= 0xA; 923 else 924 reg |= 0x4; 925 926 IGC_WRITE_REG(hw, IGC_DMCTLX, reg); 927 928 /* free space in tx packet buffer to wake from DMA coal */ 929 IGC_WRITE_REG(hw, IGC_DMCTXTH, 930 (IGC_TXPBSIZE - (2 * max_frame_size)) >> 6); 931 932 /* make low power state decision controlled by DMA coal */ 933 reg = IGC_READ_REG(hw, IGC_PCIEMISC); 934 reg &= ~IGC_PCIEMISC_LX_DECISION; 935 IGC_WRITE_REG(hw, IGC_PCIEMISC, reg); 936 } 937 938 static int 939 igc_setup_interrupts(struct igc_softc *sc) 940 { 941 int error; 942 943 switch (sc->sc_intr_type) { 944 case PCI_INTR_TYPE_MSIX: 945 error = igc_setup_msix(sc); 946 break; 947 case PCI_INTR_TYPE_MSI: 948 error = igc_setup_msi(sc); 949 break; 950 case PCI_INTR_TYPE_INTX: 951 error = igc_setup_intx(sc); 952 break; 953 default: 954 panic("%s: invalid interrupt type: %d", 955 device_xname(sc->sc_dev), sc->sc_intr_type); 956 } 957 958 return error; 959 } 960 961 static void 962 igc_attach_counters(struct igc_softc *sc) 963 { 964 #ifdef IGC_EVENT_COUNTERS 965 966 /* Global counters */ 967 sc->sc_global_evcnts = kmem_zalloc( 968 IGC_GLOBAL_COUNTERS * sizeof(sc->sc_global_evcnts[0]), KM_SLEEP); 969 970 for (int cnt = 0; cnt < IGC_GLOBAL_COUNTERS; cnt++) { 971 evcnt_attach_dynamic(&sc->sc_global_evcnts[cnt], 972 igc_global_counters[cnt].type, NULL, 973 device_xname(sc->sc_dev), igc_global_counters[cnt].name); 974 } 975 976 /* Driver counters */ 977 sc->sc_driver_evcnts = kmem_zalloc( 978 IGC_DRIVER_COUNTERS * sizeof(sc->sc_driver_evcnts[0]), KM_SLEEP); 979 980 for (int cnt = 0; cnt < IGC_DRIVER_COUNTERS; cnt++) { 981 evcnt_attach_dynamic(&sc->sc_driver_evcnts[cnt], 982 igc_driver_counters[cnt].type, NULL, 983 device_xname(sc->sc_dev), igc_driver_counters[cnt].name); 984 } 985 986 for (int iq = 0; iq < sc->sc_nqueues; iq++) { 987 struct igc_queue *q = &sc->queues[iq]; 988 989 q->igcq_driver_counters = kmem_zalloc( 990 IGC_DRIVER_COUNTERS * sizeof(q->igcq_driver_counters[0]), 991 KM_SLEEP); 992 } 993 994 /* Queue counters */ 995 for (int iq = 0; iq < sc->sc_nqueues; iq++) { 996 struct igc_queue *q = &sc->queues[iq]; 997 998 snprintf(q->igcq_queue_evname, sizeof(q->igcq_queue_evname), 999 "%s q%d", device_xname(sc->sc_dev), iq); 1000 1001 q->igcq_queue_evcnts = kmem_zalloc( 1002 IGC_QUEUE_COUNTERS * sizeof(q->igcq_queue_evcnts[0]), 1003 KM_SLEEP); 1004 1005 for (int cnt = 0; cnt < IGC_QUEUE_COUNTERS; cnt++) { 1006 evcnt_attach_dynamic(&q->igcq_queue_evcnts[cnt], 1007 igc_queue_counters[cnt].type, NULL, 1008 q->igcq_queue_evname, igc_queue_counters[cnt].name); 1009 } 1010 } 1011 1012 /* MAC counters */ 1013 snprintf(sc->sc_mac_evname, sizeof(sc->sc_mac_evname), 1014 "%s Mac Statistics", device_xname(sc->sc_dev)); 1015 1016 sc->sc_mac_evcnts = kmem_zalloc( 1017 IGC_MAC_COUNTERS * sizeof(sc->sc_mac_evcnts[0]), KM_SLEEP); 1018 1019 for (int cnt = 0; cnt < IGC_MAC_COUNTERS; cnt++) { 1020 evcnt_attach_dynamic(&sc->sc_mac_evcnts[cnt], EVCNT_TYPE_MISC, 1021 NULL, sc->sc_mac_evname, igc_mac_counters[cnt].name); 1022 } 1023 #endif 1024 } 1025 1026 static void 1027 igc_detach_counters(struct igc_softc *sc) 1028 { 1029 #ifdef IGC_EVENT_COUNTERS 1030 1031 /* Global counters */ 1032 for (int cnt = 0; cnt < IGC_GLOBAL_COUNTERS; cnt++) 1033 evcnt_detach(&sc->sc_global_evcnts[cnt]); 1034 1035 kmem_free(sc->sc_global_evcnts, 1036 IGC_GLOBAL_COUNTERS * sizeof(sc->sc_global_evcnts)); 1037 1038 /* Driver counters */ 1039 for (int iq = 0; iq < sc->sc_nqueues; iq++) { 1040 struct igc_queue *q = &sc->queues[iq]; 1041 1042 kmem_free(q->igcq_driver_counters, 1043 IGC_DRIVER_COUNTERS * sizeof(q->igcq_driver_counters[0])); 1044 } 1045 1046 for (int cnt = 0; cnt < IGC_DRIVER_COUNTERS; cnt++) 1047 evcnt_detach(&sc->sc_driver_evcnts[cnt]); 1048 1049 kmem_free(sc->sc_driver_evcnts, 1050 IGC_DRIVER_COUNTERS * sizeof(sc->sc_driver_evcnts)); 1051 1052 /* Queue counters */ 1053 for (int iq = 0; iq < sc->sc_nqueues; iq++) { 1054 struct igc_queue *q = &sc->queues[iq]; 1055 1056 for (int cnt = 0; cnt < IGC_QUEUE_COUNTERS; cnt++) 1057 evcnt_detach(&q->igcq_queue_evcnts[cnt]); 1058 1059 kmem_free(q->igcq_queue_evcnts, 1060 IGC_QUEUE_COUNTERS * sizeof(q->igcq_queue_evcnts[0])); 1061 } 1062 1063 /* MAC statistics */ 1064 for (int cnt = 0; cnt < IGC_MAC_COUNTERS; cnt++) 1065 evcnt_detach(&sc->sc_mac_evcnts[cnt]); 1066 1067 kmem_free(sc->sc_mac_evcnts, 1068 IGC_MAC_COUNTERS * sizeof(sc->sc_mac_evcnts[0])); 1069 #endif 1070 } 1071 1072 /* 1073 * XXX 1074 * FreeBSD uses 4-byte-wise read for 64-bit counters, while Linux just 1075 * drops hi words. 1076 */ 1077 static inline uint64_t __unused 1078 igc_read_mac_counter(struct igc_hw *hw, bus_size_t reg, bool is64) 1079 { 1080 uint64_t val; 1081 1082 val = IGC_READ_REG(hw, reg); 1083 if (is64) 1084 val += ((uint64_t)IGC_READ_REG(hw, reg + 4)) << 32; 1085 return val; 1086 } 1087 1088 static void 1089 igc_update_counters(struct igc_softc *sc) 1090 { 1091 #ifdef IGC_EVENT_COUNTERS 1092 1093 /* Global counters: nop */ 1094 1095 /* Driver counters */ 1096 uint64_t sum[IGC_DRIVER_COUNTERS]; 1097 1098 memset(sum, 0, sizeof(sum)); 1099 for (int iq = 0; iq < sc->sc_nqueues; iq++) { 1100 struct igc_queue *q = &sc->queues[iq]; 1101 1102 for (int cnt = 0; cnt < IGC_DRIVER_COUNTERS; cnt++) { 1103 sum[cnt] += IGC_QUEUE_DRIVER_COUNTER_VAL(q, cnt); 1104 IGC_QUEUE_DRIVER_COUNTER_STORE(q, cnt, 0); 1105 } 1106 } 1107 1108 for (int cnt = 0; cnt < IGC_DRIVER_COUNTERS; cnt++) 1109 IGC_DRIVER_COUNTER_ADD(sc, cnt, sum[cnt]); 1110 1111 /* Queue counters: nop */ 1112 1113 /* Mac statistics */ 1114 struct igc_hw *hw = &sc->hw; 1115 struct ifnet *ifp = &sc->sc_ec.ec_if; 1116 uint64_t iqdrops = 0; 1117 1118 for (int cnt = 0; cnt < IGC_MAC_COUNTERS; cnt++) { 1119 uint64_t val; 1120 bus_size_t regaddr = igc_mac_counters[cnt].reg; 1121 1122 val = igc_read_mac_counter(hw, regaddr, 1123 igc_mac_counters[cnt].is64); 1124 IGC_MAC_COUNTER_ADD(sc, cnt, val); 1125 /* XXX Count MPC to iqdrops. */ 1126 if (regaddr == IGC_MPC) 1127 iqdrops += val; 1128 } 1129 1130 for (int iq = 0; iq < sc->sc_nqueues; iq++) { 1131 uint32_t val; 1132 1133 /* XXX RQDPC should be visible via evcnt(9). */ 1134 val = IGC_READ_REG(hw, IGC_RQDPC(iq)); 1135 1136 /* RQDPC is not cleard on read. */ 1137 if (val != 0) 1138 IGC_WRITE_REG(hw, IGC_RQDPC(iq), 0); 1139 iqdrops += val; 1140 } 1141 1142 if (iqdrops != 0) 1143 if_statadd(ifp, if_iqdrops, iqdrops); 1144 #endif 1145 } 1146 1147 static void 1148 igc_clear_counters(struct igc_softc *sc) 1149 { 1150 #ifdef IGC_EVENT_COUNTERS 1151 1152 /* Global counters */ 1153 for (int cnt = 0; cnt < IGC_GLOBAL_COUNTERS; cnt++) 1154 IGC_GLOBAL_COUNTER_STORE(sc, cnt, 0); 1155 1156 /* Driver counters */ 1157 for (int iq = 0; iq < sc->sc_nqueues; iq++) { 1158 struct igc_queue *q = &sc->queues[iq]; 1159 1160 for (int cnt = 0; cnt < IGC_DRIVER_COUNTERS; cnt++) 1161 IGC_QUEUE_DRIVER_COUNTER_STORE(q, cnt, 0); 1162 } 1163 1164 for (int cnt = 0; cnt < IGC_DRIVER_COUNTERS; cnt++) 1165 IGC_DRIVER_COUNTER_STORE(sc, cnt, 0); 1166 1167 /* Queue counters */ 1168 for (int iq = 0; iq < sc->sc_nqueues; iq++) { 1169 struct igc_queue *q = &sc->queues[iq]; 1170 1171 for (int cnt = 0; cnt < IGC_QUEUE_COUNTERS; cnt++) 1172 IGC_QUEUE_COUNTER_STORE(q, cnt, 0); 1173 } 1174 1175 /* Mac statistics */ 1176 struct igc_hw *hw = &sc->hw; 1177 1178 for (int cnt = 0; cnt < IGC_MAC_COUNTERS; cnt++) { 1179 (void)igc_read_mac_counter(hw, igc_mac_counters[cnt].reg, 1180 igc_mac_counters[cnt].is64); 1181 IGC_MAC_COUNTER_STORE(sc, cnt, 0); 1182 } 1183 #endif 1184 } 1185 1186 static int 1187 igc_setup_msix(struct igc_softc *sc) 1188 { 1189 pci_chipset_tag_t pc = sc->osdep.os_pa.pa_pc; 1190 device_t dev = sc->sc_dev; 1191 pci_intr_handle_t *intrs; 1192 void **ihs; 1193 const char *intrstr; 1194 char intrbuf[PCI_INTRSTR_LEN]; 1195 char xnamebuf[MAX(32, MAXCOMLEN)]; 1196 int iq, error; 1197 1198 for (iq = 0, intrs = sc->sc_intrs, ihs = sc->sc_ihs; 1199 iq < sc->sc_nqueues; iq++, intrs++, ihs++) { 1200 struct igc_queue *q = &sc->queues[iq]; 1201 1202 snprintf(xnamebuf, sizeof(xnamebuf), "%s: txrx %d", 1203 device_xname(dev), iq); 1204 1205 intrstr = pci_intr_string(pc, *intrs, intrbuf, sizeof(intrbuf)); 1206 1207 pci_intr_setattr(pc, intrs, PCI_INTR_MPSAFE, true); 1208 *ihs = pci_intr_establish_xname(pc, *intrs, IPL_NET, 1209 igc_intr_queue, q, xnamebuf); 1210 if (*ihs == NULL) { 1211 aprint_error_dev(dev, 1212 "unable to establish txrx interrupt at %s\n", 1213 intrstr); 1214 return ENOBUFS; 1215 } 1216 aprint_normal_dev(dev, "txrx interrupting at %s\n", intrstr); 1217 1218 kcpuset_t *affinity; 1219 kcpuset_create(&affinity, true); 1220 kcpuset_set(affinity, iq % ncpu); 1221 error = interrupt_distribute(*ihs, affinity, NULL); 1222 if (error) { 1223 aprint_normal_dev(dev, 1224 "%s: unable to change affinity, use default CPU\n", 1225 intrstr); 1226 } 1227 kcpuset_destroy(affinity); 1228 1229 q->igcq_si = softint_establish(SOFTINT_NET | SOFTINT_MPSAFE, 1230 igc_handle_queue, q); 1231 if (q->igcq_si == NULL) { 1232 aprint_error_dev(dev, 1233 "%s: unable to establish softint\n", intrstr); 1234 return ENOBUFS; 1235 } 1236 1237 q->msix = iq; 1238 q->eims = 1 << iq; 1239 } 1240 1241 snprintf(xnamebuf, MAXCOMLEN, "%s_tx_rx", device_xname(dev)); 1242 error = workqueue_create(&sc->sc_queue_wq, xnamebuf, 1243 igc_handle_queue_work, sc, IGC_WORKQUEUE_PRI, IPL_NET, 1244 WQ_PERCPU | WQ_MPSAFE); 1245 if (error) { 1246 aprint_error_dev(dev, "workqueue_create failed\n"); 1247 return ENOBUFS; 1248 } 1249 sc->sc_txrx_workqueue = false; 1250 1251 intrstr = pci_intr_string(pc, *intrs, intrbuf, sizeof(intrbuf)); 1252 snprintf(xnamebuf, sizeof(xnamebuf), "%s: link", device_xname(dev)); 1253 pci_intr_setattr(pc, intrs, PCI_INTR_MPSAFE, true); 1254 *ihs = pci_intr_establish_xname(pc, *intrs, IPL_NET, 1255 igc_intr_link, sc, xnamebuf); 1256 if (*ihs == NULL) { 1257 aprint_error_dev(dev, 1258 "unable to establish link interrupt at %s\n", intrstr); 1259 return ENOBUFS; 1260 } 1261 aprint_normal_dev(dev, "link interrupting at %s\n", intrstr); 1262 /* use later in igc_configure_queues() */ 1263 sc->linkvec = iq; 1264 1265 return 0; 1266 } 1267 1268 static int 1269 igc_setup_msi(struct igc_softc *sc) 1270 { 1271 pci_chipset_tag_t pc = sc->osdep.os_pa.pa_pc; 1272 device_t dev = sc->sc_dev; 1273 pci_intr_handle_t *intr = sc->sc_intrs; 1274 void **ihs = sc->sc_ihs; 1275 const char *intrstr; 1276 char intrbuf[PCI_INTRSTR_LEN]; 1277 char xnamebuf[MAX(32, MAXCOMLEN)]; 1278 int error; 1279 1280 intrstr = pci_intr_string(pc, *intr, intrbuf, sizeof(intrbuf)); 1281 1282 snprintf(xnamebuf, sizeof(xnamebuf), "%s: msi", device_xname(dev)); 1283 pci_intr_setattr(pc, intr, PCI_INTR_MPSAFE, true); 1284 *ihs = pci_intr_establish_xname(pc, *intr, IPL_NET, 1285 igc_intr, sc, xnamebuf); 1286 if (*ihs == NULL) { 1287 aprint_error_dev(dev, 1288 "unable to establish interrupt at %s\n", intrstr); 1289 return ENOBUFS; 1290 } 1291 aprint_normal_dev(dev, "interrupting at %s\n", intrstr); 1292 1293 struct igc_queue *iq = sc->queues; 1294 iq->igcq_si = softint_establish(SOFTINT_NET | SOFTINT_MPSAFE, 1295 igc_handle_queue, iq); 1296 if (iq->igcq_si == NULL) { 1297 aprint_error_dev(dev, 1298 "%s: unable to establish softint\n", intrstr); 1299 return ENOBUFS; 1300 } 1301 1302 snprintf(xnamebuf, MAXCOMLEN, "%s_tx_rx", device_xname(dev)); 1303 error = workqueue_create(&sc->sc_queue_wq, xnamebuf, 1304 igc_handle_queue_work, sc, IGC_WORKQUEUE_PRI, IPL_NET, 1305 WQ_PERCPU | WQ_MPSAFE); 1306 if (error) { 1307 aprint_error_dev(dev, "workqueue_create failed\n"); 1308 return ENOBUFS; 1309 } 1310 sc->sc_txrx_workqueue = false; 1311 1312 sc->queues[0].msix = 0; 1313 sc->linkvec = 0; 1314 1315 return 0; 1316 } 1317 1318 static int 1319 igc_setup_intx(struct igc_softc *sc) 1320 { 1321 pci_chipset_tag_t pc = sc->osdep.os_pa.pa_pc; 1322 device_t dev = sc->sc_dev; 1323 pci_intr_handle_t *intr = sc->sc_intrs; 1324 void **ihs = sc->sc_ihs; 1325 const char *intrstr; 1326 char intrbuf[PCI_INTRSTR_LEN]; 1327 char xnamebuf[32]; 1328 1329 intrstr = pci_intr_string(pc, *intr, intrbuf, sizeof(intrbuf)); 1330 1331 snprintf(xnamebuf, sizeof(xnamebuf), "%s:intx", device_xname(dev)); 1332 pci_intr_setattr(pc, intr, PCI_INTR_MPSAFE, true); 1333 *ihs = pci_intr_establish_xname(pc, *intr, IPL_NET, 1334 igc_intr, sc, xnamebuf); 1335 if (*ihs == NULL) { 1336 aprint_error_dev(dev, 1337 "unable to establish interrupt at %s\n", intrstr); 1338 return ENOBUFS; 1339 } 1340 aprint_normal_dev(dev, "interrupting at %s\n", intrstr); 1341 1342 struct igc_queue *iq = sc->queues; 1343 iq->igcq_si = softint_establish(SOFTINT_NET | SOFTINT_MPSAFE, 1344 igc_handle_queue, iq); 1345 if (iq->igcq_si == NULL) { 1346 aprint_error_dev(dev, 1347 "%s: unable to establish softint\n", intrstr); 1348 return ENOBUFS; 1349 } 1350 1351 /* create workqueue? */ 1352 sc->sc_txrx_workqueue = false; 1353 1354 sc->queues[0].msix = 0; 1355 sc->linkvec = 0; 1356 1357 return 0; 1358 } 1359 1360 static int 1361 igc_dma_malloc(struct igc_softc *sc, bus_size_t size, struct igc_dma_alloc *dma) 1362 { 1363 struct igc_osdep *os = &sc->osdep; 1364 1365 dma->dma_tag = os->os_dmat; 1366 1367 if (bus_dmamap_create(dma->dma_tag, size, 1, size, 0, 1368 BUS_DMA_WAITOK | BUS_DMA_ALLOCNOW, &dma->dma_map)) 1369 return 1; 1370 if (bus_dmamem_alloc(dma->dma_tag, size, PAGE_SIZE, 0, &dma->dma_seg, 1371 1, &dma->dma_nseg, BUS_DMA_WAITOK)) 1372 goto destroy; 1373 /* 1374 * XXXRO 1375 * 1376 * Coherent mapping for descriptors is required for now. 1377 * 1378 * Both TX and RX descriptors are 16-byte length, which is shorter 1379 * than dcache lines on modern CPUs. Therefore, sync for a descriptor 1380 * may overwrite DMA read for descriptors in the same cache line. 1381 * 1382 * Can't we avoid this by use cache-line-aligned descriptors at once? 1383 */ 1384 if (bus_dmamem_map(dma->dma_tag, &dma->dma_seg, dma->dma_nseg, size, 1385 &dma->dma_vaddr, BUS_DMA_WAITOK | BUS_DMA_COHERENT /* XXXRO */)) 1386 goto free; 1387 if (bus_dmamap_load(dma->dma_tag, dma->dma_map, dma->dma_vaddr, size, 1388 NULL, BUS_DMA_WAITOK)) 1389 goto unmap; 1390 1391 dma->dma_size = size; 1392 1393 return 0; 1394 unmap: 1395 bus_dmamem_unmap(dma->dma_tag, dma->dma_vaddr, size); 1396 free: 1397 bus_dmamem_free(dma->dma_tag, &dma->dma_seg, dma->dma_nseg); 1398 destroy: 1399 bus_dmamap_destroy(dma->dma_tag, dma->dma_map); 1400 dma->dma_map = NULL; 1401 dma->dma_tag = NULL; 1402 return 1; 1403 } 1404 1405 static void 1406 igc_dma_free(struct igc_softc *sc, struct igc_dma_alloc *dma) 1407 { 1408 1409 if (dma->dma_tag == NULL) 1410 return; 1411 1412 if (dma->dma_map != NULL) { 1413 bus_dmamap_sync(dma->dma_tag, dma->dma_map, 0, 1414 dma->dma_map->dm_mapsize, 1415 BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE); 1416 bus_dmamap_unload(dma->dma_tag, dma->dma_map); 1417 bus_dmamem_unmap(dma->dma_tag, dma->dma_vaddr, dma->dma_size); 1418 bus_dmamem_free(dma->dma_tag, &dma->dma_seg, dma->dma_nseg); 1419 bus_dmamap_destroy(dma->dma_tag, dma->dma_map); 1420 dma->dma_map = NULL; 1421 } 1422 } 1423 1424 /********************************************************************* 1425 * 1426 * Setup networking device structure and register an interface. 1427 * 1428 **********************************************************************/ 1429 static void 1430 igc_setup_interface(struct igc_softc *sc) 1431 { 1432 struct ifnet *ifp = &sc->sc_ec.ec_if; 1433 1434 strlcpy(ifp->if_xname, device_xname(sc->sc_dev), sizeof(ifp->if_xname)); 1435 ifp->if_softc = sc; 1436 ifp->if_flags = IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST; 1437 ifp->if_extflags = IFEF_MPSAFE; 1438 ifp->if_ioctl = igc_ioctl; 1439 ifp->if_start = igc_start; 1440 if (sc->sc_nqueues > 1) 1441 ifp->if_transmit = igc_transmit; 1442 ifp->if_watchdog = igc_watchdog; 1443 ifp->if_init = igc_init; 1444 ifp->if_stop = igc_stop; 1445 1446 #if 0 /* notyet */ 1447 ifp->if_capabilities = IFCAP_TSOv4 | IFCAP_TSOv6; 1448 #endif 1449 1450 ifp->if_capabilities |= 1451 IFCAP_CSUM_IPv4_Tx | IFCAP_CSUM_IPv4_Rx | 1452 IFCAP_CSUM_TCPv4_Tx | IFCAP_CSUM_TCPv4_Rx | 1453 IFCAP_CSUM_UDPv4_Tx | IFCAP_CSUM_UDPv4_Rx | 1454 IFCAP_CSUM_TCPv6_Tx | IFCAP_CSUM_TCPv6_Rx | 1455 IFCAP_CSUM_UDPv6_Tx | IFCAP_CSUM_UDPv6_Rx; 1456 1457 ifp->if_capenable = 0; 1458 1459 sc->sc_ec.ec_capabilities |= 1460 ETHERCAP_JUMBO_MTU | ETHERCAP_VLAN_MTU; 1461 1462 IFQ_SET_MAXLEN(&ifp->if_snd, sc->num_tx_desc - 1); 1463 IFQ_SET_READY(&ifp->if_snd); 1464 1465 #if NVLAN > 0 1466 sc->sc_ec.ec_capabilities |= ETHERCAP_VLAN_HWTAGGING; 1467 #endif 1468 1469 mutex_init(&sc->sc_core_lock, MUTEX_DEFAULT, IPL_NET); 1470 1471 /* Initialize ifmedia structures. */ 1472 sc->sc_ec.ec_ifmedia = &sc->media; 1473 ifmedia_init_with_lock(&sc->media, IFM_IMASK, igc_media_change, 1474 igc_media_status, &sc->sc_core_lock); 1475 ifmedia_add(&sc->media, IFM_ETHER | IFM_10_T, 0, NULL); 1476 ifmedia_add(&sc->media, IFM_ETHER | IFM_10_T | IFM_FDX, 0, NULL); 1477 ifmedia_add(&sc->media, IFM_ETHER | IFM_100_TX, 0, NULL); 1478 ifmedia_add(&sc->media, IFM_ETHER | IFM_100_TX | IFM_FDX, 0, NULL); 1479 ifmedia_add(&sc->media, IFM_ETHER | IFM_1000_T | IFM_FDX, 0, NULL); 1480 ifmedia_add(&sc->media, IFM_ETHER | IFM_2500_T | IFM_FDX, 0, NULL); 1481 ifmedia_add(&sc->media, IFM_ETHER | IFM_AUTO, 0, NULL); 1482 ifmedia_set(&sc->media, IFM_ETHER | IFM_AUTO); 1483 1484 sc->sc_rx_intr_process_limit = IGC_RX_INTR_PROCESS_LIMIT_DEFAULT; 1485 sc->sc_tx_intr_process_limit = IGC_TX_INTR_PROCESS_LIMIT_DEFAULT; 1486 sc->sc_rx_process_limit = IGC_RX_PROCESS_LIMIT_DEFAULT; 1487 sc->sc_tx_process_limit = IGC_TX_PROCESS_LIMIT_DEFAULT; 1488 1489 if_initialize(ifp); 1490 sc->sc_ipq = if_percpuq_create(ifp); 1491 if_deferred_start_init(ifp, NULL); 1492 ether_ifattach(ifp, sc->hw.mac.addr); 1493 ether_set_ifflags_cb(&sc->sc_ec, igc_ifflags_cb); 1494 if_register(ifp); 1495 } 1496 1497 static int 1498 igc_init(struct ifnet *ifp) 1499 { 1500 struct igc_softc *sc = ifp->if_softc; 1501 int error; 1502 1503 mutex_enter(&sc->sc_core_lock); 1504 error = igc_init_locked(sc); 1505 mutex_exit(&sc->sc_core_lock); 1506 1507 return error; 1508 } 1509 1510 static int 1511 igc_init_locked(struct igc_softc *sc) 1512 { 1513 struct ethercom *ec = &sc->sc_ec; 1514 struct ifnet *ifp = &ec->ec_if; 1515 1516 DPRINTF(CFG, "called\n"); 1517 1518 KASSERT(mutex_owned(&sc->sc_core_lock)); 1519 1520 if (ISSET(ifp->if_flags, IFF_RUNNING)) 1521 igc_stop_locked(sc); 1522 1523 /* Put the address into the receive address array. */ 1524 igc_rar_set(&sc->hw, sc->hw.mac.addr, 0); 1525 1526 /* Initialize the hardware. */ 1527 igc_reset(sc); 1528 igc_update_link_status(sc); 1529 1530 /* Setup VLAN support, basic and offload if available. */ 1531 IGC_WRITE_REG(&sc->hw, IGC_VET, ETHERTYPE_VLAN); 1532 1533 igc_initialize_transmit_unit(sc); 1534 igc_initialize_receive_unit(sc); 1535 1536 if (ec->ec_capenable & ETHERCAP_VLAN_HWTAGGING) { 1537 uint32_t ctrl = IGC_READ_REG(&sc->hw, IGC_CTRL); 1538 ctrl |= IGC_CTRL_VME; 1539 IGC_WRITE_REG(&sc->hw, IGC_CTRL, ctrl); 1540 } 1541 1542 /* Setup multicast table. */ 1543 igc_set_filter(sc); 1544 1545 igc_clear_hw_cntrs_base_generic(&sc->hw); 1546 1547 if (sc->sc_intr_type == PCI_INTR_TYPE_MSIX) 1548 igc_configure_queues(sc); 1549 1550 /* This clears any pending interrupts */ 1551 IGC_READ_REG(&sc->hw, IGC_ICR); 1552 IGC_WRITE_REG(&sc->hw, IGC_ICS, IGC_ICS_LSC); 1553 1554 /* The driver can now take control from firmware. */ 1555 igc_get_hw_control(sc); 1556 1557 /* Set Energy Efficient Ethernet. */ 1558 igc_set_eee_i225(&sc->hw, true, true, true); 1559 1560 for (int iq = 0; iq < sc->sc_nqueues; iq++) { 1561 struct rx_ring *rxr = &sc->rx_rings[iq]; 1562 1563 mutex_enter(&rxr->rxr_lock); 1564 igc_rxfill(rxr); 1565 mutex_exit(&rxr->rxr_lock); 1566 } 1567 1568 sc->sc_core_stopping = false; 1569 1570 ifp->if_flags |= IFF_RUNNING; 1571 1572 /* Save last flags for the callback */ 1573 sc->sc_if_flags = ifp->if_flags; 1574 1575 callout_schedule(&sc->sc_tick_ch, hz); 1576 1577 igc_enable_intr(sc); 1578 1579 return 0; 1580 } 1581 1582 static inline int 1583 igc_load_mbuf(struct igc_queue *q, bus_dma_tag_t dmat, bus_dmamap_t map, 1584 struct mbuf *m) 1585 { 1586 int error; 1587 1588 error = bus_dmamap_load_mbuf(dmat, map, m, 1589 BUS_DMA_WRITE | BUS_DMA_NOWAIT); 1590 1591 if (__predict_false(error == EFBIG)) { 1592 IGC_DRIVER_EVENT(q, txdma_efbig, 1); 1593 m = m_defrag(m, M_NOWAIT); 1594 if (__predict_false(m == NULL)) { 1595 IGC_DRIVER_EVENT(q, txdma_defrag, 1); 1596 return ENOBUFS; 1597 } 1598 error = bus_dmamap_load_mbuf(dmat, map, m, 1599 BUS_DMA_WRITE | BUS_DMA_NOWAIT); 1600 } 1601 1602 switch (error) { 1603 case 0: 1604 break; 1605 case ENOMEM: 1606 IGC_DRIVER_EVENT(q, txdma_enomem, 1); 1607 break; 1608 case EINVAL: 1609 IGC_DRIVER_EVENT(q, txdma_einval, 1); 1610 break; 1611 case EAGAIN: 1612 IGC_DRIVER_EVENT(q, txdma_eagain, 1); 1613 break; 1614 default: 1615 IGC_DRIVER_EVENT(q, txdma_other, 1); 1616 break; 1617 } 1618 1619 return error; 1620 } 1621 1622 #define IGC_TX_START 1 1623 #define IGC_TX_TRANSMIT 2 1624 1625 static void 1626 igc_start(struct ifnet *ifp) 1627 { 1628 struct igc_softc *sc = ifp->if_softc; 1629 1630 if (__predict_false(!sc->link_active)) { 1631 IFQ_PURGE(&ifp->if_snd); 1632 return; 1633 } 1634 1635 struct tx_ring *txr = &sc->tx_rings[0]; /* queue 0 */ 1636 mutex_enter(&txr->txr_lock); 1637 igc_tx_common_locked(ifp, txr, IGC_TX_START); 1638 mutex_exit(&txr->txr_lock); 1639 } 1640 1641 static inline u_int 1642 igc_select_txqueue(struct igc_softc *sc, struct mbuf *m __unused) 1643 { 1644 const u_int cpuid = cpu_index(curcpu()); 1645 1646 return cpuid % sc->sc_nqueues; 1647 } 1648 1649 static int 1650 igc_transmit(struct ifnet *ifp, struct mbuf *m) 1651 { 1652 struct igc_softc *sc = ifp->if_softc; 1653 const u_int qid = igc_select_txqueue(sc, m); 1654 struct tx_ring *txr = &sc->tx_rings[qid]; 1655 struct igc_queue *q = txr->txr_igcq; 1656 1657 if (__predict_false(!pcq_put(txr->txr_interq, m))) { 1658 IGC_QUEUE_EVENT(q, tx_pcq_drop, 1); 1659 m_freem(m); 1660 return ENOBUFS; 1661 } 1662 1663 mutex_enter(&txr->txr_lock); 1664 igc_tx_common_locked(ifp, txr, IGC_TX_TRANSMIT); 1665 mutex_exit(&txr->txr_lock); 1666 1667 return 0; 1668 } 1669 1670 static void 1671 igc_tx_common_locked(struct ifnet *ifp, struct tx_ring *txr, int caller) 1672 { 1673 struct igc_softc *sc = ifp->if_softc; 1674 struct igc_queue *q = txr->txr_igcq; 1675 net_stat_ref_t nsr = IF_STAT_GETREF(ifp); 1676 int prod, free, last = -1; 1677 bool post = false; 1678 1679 prod = txr->next_avail_desc; 1680 free = txr->next_to_clean; 1681 if (free <= prod) 1682 free += sc->num_tx_desc; 1683 free -= prod; 1684 1685 DPRINTF(TX, "%s: begin: msix %d prod %d n2c %d free %d\n", 1686 caller == IGC_TX_TRANSMIT ? "transmit" : "start", 1687 txr->me, prod, txr->next_to_clean, free); 1688 1689 for (;;) { 1690 struct mbuf *m; 1691 1692 if (__predict_false(free <= IGC_MAX_SCATTER)) { 1693 IGC_QUEUE_EVENT(q, tx_no_desc, 1); 1694 break; 1695 } 1696 1697 if (caller == IGC_TX_TRANSMIT) 1698 m = pcq_get(txr->txr_interq); 1699 else 1700 IFQ_DEQUEUE(&ifp->if_snd, m); 1701 if (__predict_false(m == NULL)) 1702 break; 1703 1704 struct igc_tx_buf *txbuf = &txr->tx_buffers[prod]; 1705 bus_dmamap_t map = txbuf->map; 1706 1707 if (__predict_false( 1708 igc_load_mbuf(q, txr->txdma.dma_tag, map, m))) { 1709 if (caller == IGC_TX_TRANSMIT) 1710 IGC_QUEUE_EVENT(q, tx_pcq_drop, 1); 1711 m_freem(m); 1712 if_statinc_ref(nsr, if_oerrors); 1713 continue; 1714 } 1715 1716 bus_dmamap_sync(txr->txdma.dma_tag, map, 0, 1717 map->dm_mapsize, BUS_DMASYNC_PREWRITE); 1718 1719 uint32_t ctx_cmd_type_len = 0, olinfo_status = 0; 1720 if (igc_tx_ctx_setup(txr, m, prod, &ctx_cmd_type_len, 1721 &olinfo_status)) { 1722 IGC_QUEUE_EVENT(q, tx_ctx, 1); 1723 /* Consume the first descriptor */ 1724 prod = igc_txdesc_incr(sc, prod); 1725 free--; 1726 } 1727 for (int i = 0; i < map->dm_nsegs; i++) { 1728 union igc_adv_tx_desc *txdesc = &txr->tx_base[prod]; 1729 1730 uint32_t cmd_type_len = ctx_cmd_type_len | 1731 IGC_ADVTXD_DCMD_IFCS | IGC_ADVTXD_DTYP_DATA | 1732 IGC_ADVTXD_DCMD_DEXT | map->dm_segs[i].ds_len; 1733 if (i == map->dm_nsegs - 1) { 1734 cmd_type_len |= 1735 IGC_ADVTXD_DCMD_EOP | IGC_ADVTXD_DCMD_RS; 1736 } 1737 1738 igc_txdesc_sync(txr, prod, 1739 BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE); 1740 htolem64(&txdesc->read.buffer_addr, 1741 map->dm_segs[i].ds_addr); 1742 htolem32(&txdesc->read.cmd_type_len, cmd_type_len); 1743 htolem32(&txdesc->read.olinfo_status, olinfo_status); 1744 igc_txdesc_sync(txr, prod, 1745 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); 1746 1747 last = prod; 1748 prod = igc_txdesc_incr(sc, prod); 1749 } 1750 1751 txbuf->m_head = m; 1752 txbuf->eop_index = last; 1753 1754 bpf_mtap(ifp, m, BPF_D_OUT); 1755 1756 if_statadd_ref(nsr, if_obytes, m->m_pkthdr.len); 1757 if (m->m_flags & M_MCAST) 1758 if_statinc_ref(nsr, if_omcasts); 1759 IGC_QUEUE_EVENT(q, tx_packets, 1); 1760 IGC_QUEUE_EVENT(q, tx_bytes, m->m_pkthdr.len); 1761 1762 free -= map->dm_nsegs; 1763 post = true; 1764 } 1765 1766 if (post) { 1767 txr->next_avail_desc = prod; 1768 IGC_WRITE_REG(&sc->hw, IGC_TDT(txr->me), prod); 1769 } 1770 1771 DPRINTF(TX, "%s: done : msix %d prod %d n2c %d free %d\n", 1772 caller == IGC_TX_TRANSMIT ? "transmit" : "start", 1773 txr->me, prod, txr->next_to_clean, free); 1774 1775 IF_STAT_PUTREF(ifp); 1776 } 1777 1778 static bool 1779 igc_txeof(struct tx_ring *txr, u_int limit) 1780 { 1781 struct igc_softc *sc = txr->sc; 1782 struct ifnet *ifp = &sc->sc_ec.ec_if; 1783 int cons, prod; 1784 bool more = false; 1785 1786 prod = txr->next_avail_desc; 1787 cons = txr->next_to_clean; 1788 1789 if (cons == prod) { 1790 DPRINTF(TX, "false: msix %d cons %d prod %d\n", 1791 txr->me, cons, prod); 1792 return false; 1793 } 1794 1795 do { 1796 struct igc_tx_buf *txbuf = &txr->tx_buffers[cons]; 1797 const int last = txbuf->eop_index; 1798 1799 membar_consumer(); /* XXXRO necessary? */ 1800 1801 KASSERT(last != -1); 1802 union igc_adv_tx_desc *txdesc = &txr->tx_base[last]; 1803 igc_txdesc_sync(txr, last, BUS_DMASYNC_POSTREAD); 1804 const uint32_t status = le32toh(txdesc->wb.status); 1805 igc_txdesc_sync(txr, last, BUS_DMASYNC_PREREAD); 1806 1807 if (!(status & IGC_TXD_STAT_DD)) 1808 break; 1809 1810 if (limit-- == 0) { 1811 more = true; 1812 DPRINTF(TX, "pending TX " 1813 "msix %d cons %d last %d prod %d " 1814 "status 0x%08x\n", 1815 txr->me, cons, last, prod, status); 1816 break; 1817 } 1818 1819 DPRINTF(TX, "handled TX " 1820 "msix %d cons %d last %d prod %d " 1821 "status 0x%08x\n", 1822 txr->me, cons, last, prod, status); 1823 1824 if_statinc(ifp, if_opackets); 1825 1826 bus_dmamap_t map = txbuf->map; 1827 bus_dmamap_sync(txr->txdma.dma_tag, map, 0, map->dm_mapsize, 1828 BUS_DMASYNC_POSTWRITE); 1829 bus_dmamap_unload(txr->txdma.dma_tag, map); 1830 m_freem(txbuf->m_head); 1831 1832 txbuf->m_head = NULL; 1833 txbuf->eop_index = -1; 1834 1835 cons = igc_txdesc_incr(sc, last); 1836 } while (cons != prod); 1837 1838 txr->next_to_clean = cons; 1839 1840 return more; 1841 } 1842 1843 static void 1844 igc_intr_barrier(struct igc_softc *sc __unused) 1845 { 1846 1847 xc_barrier(0); 1848 } 1849 1850 static void 1851 igc_stop(struct ifnet *ifp, int disable) 1852 { 1853 struct igc_softc *sc = ifp->if_softc; 1854 1855 mutex_enter(&sc->sc_core_lock); 1856 igc_stop_locked(sc); 1857 mutex_exit(&sc->sc_core_lock); 1858 } 1859 1860 /********************************************************************* 1861 * 1862 * This routine disables all traffic on the adapter by issuing a 1863 * global reset on the MAC. 1864 * 1865 **********************************************************************/ 1866 static void 1867 igc_stop_locked(struct igc_softc *sc) 1868 { 1869 struct ifnet *ifp = &sc->sc_ec.ec_if; 1870 1871 DPRINTF(CFG, "called\n"); 1872 1873 KASSERT(mutex_owned(&sc->sc_core_lock)); 1874 1875 /* 1876 * If stopping processing has already started, do nothing. 1877 */ 1878 if ((ifp->if_flags & IFF_RUNNING) == 0) 1879 return; 1880 1881 /* Tell the stack that the interface is no longer active. */ 1882 ifp->if_flags &= ~IFF_RUNNING; 1883 1884 /* 1885 * igc_handle_queue() can enable interrupts, so wait for completion of 1886 * last igc_handle_queue() after unset IFF_RUNNING. 1887 */ 1888 mutex_exit(&sc->sc_core_lock); 1889 igc_barrier_handle_queue(sc); 1890 mutex_enter(&sc->sc_core_lock); 1891 1892 sc->sc_core_stopping = true; 1893 1894 igc_disable_intr(sc); 1895 1896 callout_halt(&sc->sc_tick_ch, &sc->sc_core_lock); 1897 1898 igc_reset_hw(&sc->hw); 1899 IGC_WRITE_REG(&sc->hw, IGC_WUC, 0); 1900 1901 /* 1902 * Wait for completion of interrupt handlers. 1903 */ 1904 mutex_exit(&sc->sc_core_lock); 1905 igc_intr_barrier(sc); 1906 mutex_enter(&sc->sc_core_lock); 1907 1908 igc_update_link_status(sc); 1909 1910 for (int iq = 0; iq < sc->sc_nqueues; iq++) { 1911 struct tx_ring *txr = &sc->tx_rings[iq]; 1912 1913 igc_withdraw_transmit_packets(txr, false); 1914 } 1915 1916 for (int iq = 0; iq < sc->sc_nqueues; iq++) { 1917 struct rx_ring *rxr = &sc->rx_rings[iq]; 1918 1919 igc_clear_receive_status(rxr); 1920 } 1921 1922 /* Save last flags for the callback */ 1923 sc->sc_if_flags = ifp->if_flags; 1924 } 1925 1926 /********************************************************************* 1927 * Ioctl entry point 1928 * 1929 * igc_ioctl is called when the user wants to configure the 1930 * interface. 1931 * 1932 * return 0 on success, positive on failure 1933 **********************************************************************/ 1934 static int 1935 igc_ioctl(struct ifnet * ifp, u_long cmd, void *data) 1936 { 1937 struct igc_softc *sc __unused = ifp->if_softc; 1938 int s; 1939 int error; 1940 1941 DPRINTF(CFG, "cmd 0x%016lx\n", cmd); 1942 1943 switch (cmd) { 1944 case SIOCADDMULTI: 1945 case SIOCDELMULTI: 1946 break; 1947 default: 1948 KASSERT(IFNET_LOCKED(ifp)); 1949 } 1950 1951 if (cmd == SIOCZIFDATA) { 1952 mutex_enter(&sc->sc_core_lock); 1953 igc_clear_counters(sc); 1954 mutex_exit(&sc->sc_core_lock); 1955 } 1956 1957 switch (cmd) { 1958 #ifdef IF_RXR 1959 case SIOCGIFRXR: 1960 s = splnet(); 1961 error = igc_rxrinfo(sc, (struct if_rxrinfo *)ifr->ifr_data); 1962 splx(s); 1963 break; 1964 #endif 1965 default: 1966 s = splnet(); 1967 error = ether_ioctl(ifp, cmd, data); 1968 splx(s); 1969 break; 1970 } 1971 1972 if (error != ENETRESET) 1973 return error; 1974 1975 error = 0; 1976 1977 if (cmd == SIOCSIFCAP) 1978 error = if_init(ifp); 1979 else if ((cmd == SIOCADDMULTI) || (cmd == SIOCDELMULTI)) { 1980 mutex_enter(&sc->sc_core_lock); 1981 if (sc->sc_if_flags & IFF_RUNNING) { 1982 /* 1983 * Multicast list has changed; set the hardware filter 1984 * accordingly. 1985 */ 1986 igc_disable_intr(sc); 1987 igc_set_filter(sc); 1988 igc_enable_intr(sc); 1989 } 1990 mutex_exit(&sc->sc_core_lock); 1991 } 1992 1993 return error; 1994 } 1995 1996 #ifdef IF_RXR 1997 static int 1998 igc_rxrinfo(struct igc_softc *sc, struct if_rxrinfo *ifri) 1999 { 2000 struct if_rxring_info *ifr, ifr1; 2001 int error; 2002 2003 if (sc->sc_nqueues > 1) { 2004 ifr = kmem_zalloc(sc->sc_nqueues * sizeof(*ifr), KM_SLEEP); 2005 } else { 2006 ifr = &ifr1; 2007 memset(ifr, 0, sizeof(*ifr)); 2008 } 2009 2010 for (int iq = 0; iq < sc->sc_nqueues; iq++) { 2011 struct rx_ring *rxr = &sc->rx_rings[iq]; 2012 2013 ifr[iq].ifr_size = MCLBYTES; 2014 snprintf(ifr[iq].ifr_name, sizeof(ifr[iq].ifr_name), "%d", iq); 2015 ifr[iq].ifr_info = rxr->rx_ring; 2016 } 2017 2018 error = if_rxr_info_ioctl(ifri, sc->sc_nqueues, ifr); 2019 if (sc->sc_nqueues > 1) 2020 kmem_free(ifr, sc->sc_nqueues * sizeof(*ifr)); 2021 2022 return error; 2023 } 2024 #endif 2025 2026 static void 2027 igc_rxfill(struct rx_ring *rxr) 2028 { 2029 struct igc_softc *sc = rxr->sc; 2030 int id; 2031 2032 for (id = 0; id < sc->num_rx_desc; id++) { 2033 if (igc_get_buf(rxr, id, false)) { 2034 panic("%s: msix=%d i=%d\n", __func__, rxr->me, id); 2035 } 2036 } 2037 2038 id = sc->num_rx_desc - 1; 2039 rxr->last_desc_filled = id; 2040 IGC_WRITE_REG(&sc->hw, IGC_RDT(rxr->me), id); 2041 rxr->next_to_check = 0; 2042 } 2043 2044 static void 2045 igc_rxrefill(struct rx_ring *rxr, int end) 2046 { 2047 struct igc_softc *sc = rxr->sc; 2048 int id; 2049 2050 for (id = rxr->next_to_check; id != end; id = igc_rxdesc_incr(sc, id)) { 2051 if (igc_get_buf(rxr, id, true)) { 2052 /* XXXRO */ 2053 panic("%s: msix=%d id=%d\n", __func__, rxr->me, id); 2054 } 2055 } 2056 2057 id = igc_rxdesc_decr(sc, id); 2058 DPRINTF(RX, "%s RDT %d id %d\n", 2059 rxr->last_desc_filled == id ? "same" : "diff", 2060 rxr->last_desc_filled, id); 2061 rxr->last_desc_filled = id; 2062 IGC_WRITE_REG(&sc->hw, IGC_RDT(rxr->me), id); 2063 } 2064 2065 /********************************************************************* 2066 * 2067 * This routine executes in interrupt context. It replenishes 2068 * the mbufs in the descriptor and sends data which has been 2069 * dma'ed into host memory to upper layer. 2070 * 2071 *********************************************************************/ 2072 static bool 2073 igc_rxeof(struct rx_ring *rxr, u_int limit) 2074 { 2075 struct igc_softc *sc = rxr->sc; 2076 struct igc_queue *q = rxr->rxr_igcq; 2077 struct ifnet *ifp = &sc->sc_ec.ec_if; 2078 int id; 2079 bool more = false; 2080 2081 id = rxr->next_to_check; 2082 for (;;) { 2083 union igc_adv_rx_desc *rxdesc = &rxr->rx_base[id]; 2084 struct igc_rx_buf *rxbuf, *nxbuf; 2085 struct mbuf *mp, *m; 2086 2087 igc_rxdesc_sync(rxr, id, 2088 BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE); 2089 2090 const uint32_t staterr = le32toh(rxdesc->wb.upper.status_error); 2091 2092 if (!ISSET(staterr, IGC_RXD_STAT_DD)) { 2093 igc_rxdesc_sync(rxr, id, 2094 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); 2095 break; 2096 } 2097 2098 if (limit-- == 0) { 2099 igc_rxdesc_sync(rxr, id, 2100 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); 2101 DPRINTF(RX, "more=true\n"); 2102 more = true; 2103 break; 2104 } 2105 2106 /* Zero out the receive descriptors status. */ 2107 rxdesc->wb.upper.status_error = 0; 2108 2109 /* Pull the mbuf off the ring. */ 2110 rxbuf = &rxr->rx_buffers[id]; 2111 bus_dmamap_t map = rxbuf->map; 2112 bus_dmamap_sync(rxr->rxdma.dma_tag, map, 2113 0, map->dm_mapsize, BUS_DMASYNC_POSTREAD); 2114 bus_dmamap_unload(rxr->rxdma.dma_tag, map); 2115 2116 mp = rxbuf->buf; 2117 rxbuf->buf = NULL; 2118 2119 const bool eop = staterr & IGC_RXD_STAT_EOP; 2120 const uint16_t len = le16toh(rxdesc->wb.upper.length); 2121 2122 const uint16_t vtag = le16toh(rxdesc->wb.upper.vlan); 2123 2124 const uint32_t ptype = le32toh(rxdesc->wb.lower.lo_dword.data) & 2125 IGC_PKTTYPE_MASK; 2126 2127 const uint32_t hash __unused = 2128 le32toh(rxdesc->wb.lower.hi_dword.rss); 2129 const uint16_t hashtype __unused = 2130 le16toh(rxdesc->wb.lower.lo_dword.hs_rss.pkt_info) & 2131 IGC_RXDADV_RSSTYPE_MASK; 2132 2133 igc_rxdesc_sync(rxr, id, 2134 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); 2135 2136 if (__predict_false(staterr & IGC_RXDEXT_STATERR_RXE)) { 2137 if (rxbuf->fmp) { 2138 m_freem(rxbuf->fmp); 2139 rxbuf->fmp = NULL; 2140 } 2141 2142 m_freem(mp); 2143 m = NULL; 2144 2145 if_statinc(ifp, if_ierrors); 2146 IGC_QUEUE_EVENT(q, rx_discard, 1); 2147 2148 DPRINTF(RX, "ierrors++\n"); 2149 2150 goto next_desc; 2151 } 2152 2153 if (__predict_false(mp == NULL)) { 2154 panic("%s: igc_rxeof: NULL mbuf in slot %d " 2155 "(filled %d)", device_xname(sc->sc_dev), 2156 id, rxr->last_desc_filled); 2157 } 2158 2159 if (!eop) { 2160 /* 2161 * Figure out the next descriptor of this frame. 2162 */ 2163 int nextp = igc_rxdesc_incr(sc, id); 2164 2165 nxbuf = &rxr->rx_buffers[nextp]; 2166 /* 2167 * TODO prefetch(nxbuf); 2168 */ 2169 } 2170 2171 mp->m_len = len; 2172 2173 m = rxbuf->fmp; 2174 rxbuf->fmp = NULL; 2175 2176 if (m != NULL) { 2177 m->m_pkthdr.len += mp->m_len; 2178 } else { 2179 m = mp; 2180 m->m_pkthdr.len = mp->m_len; 2181 #if NVLAN > 0 2182 if (staterr & IGC_RXD_STAT_VP) 2183 vlan_set_tag(m, vtag); 2184 #endif 2185 } 2186 2187 /* Pass the head pointer on */ 2188 if (!eop) { 2189 nxbuf->fmp = m; 2190 m = NULL; 2191 mp->m_next = nxbuf->buf; 2192 } else { 2193 m_set_rcvif(m, ifp); 2194 2195 m->m_pkthdr.csum_flags = igc_rx_checksum(q, 2196 ifp->if_capenable, staterr, ptype); 2197 2198 #ifdef notyet 2199 if (hashtype != IGC_RXDADV_RSSTYPE_NONE) { 2200 m->m_pkthdr.ph_flowid = hash; 2201 SET(m->m_pkthdr.csum_flags, M_FLOWID); 2202 } 2203 ml_enqueue(&ml, m); 2204 #endif 2205 2206 if_percpuq_enqueue(sc->sc_ipq, m); 2207 2208 if_statinc(ifp, if_ipackets); 2209 IGC_QUEUE_EVENT(q, rx_packets, 1); 2210 IGC_QUEUE_EVENT(q, rx_bytes, m->m_pkthdr.len); 2211 } 2212 next_desc: 2213 /* Advance our pointers to the next descriptor. */ 2214 id = igc_rxdesc_incr(sc, id); 2215 } 2216 2217 DPRINTF(RX, "fill queue[%d]\n", rxr->me); 2218 igc_rxrefill(rxr, id); 2219 2220 DPRINTF(RX, "%s n2c %d id %d\n", 2221 rxr->next_to_check == id ? "same" : "diff", 2222 rxr->next_to_check, id); 2223 rxr->next_to_check = id; 2224 2225 #ifdef OPENBSD 2226 if (!(staterr & IGC_RXD_STAT_DD)) 2227 return 0; 2228 #endif 2229 2230 return more; 2231 } 2232 2233 /********************************************************************* 2234 * 2235 * Verify that the hardware indicated that the checksum is valid. 2236 * Inform the stack about the status of checksum so that stack 2237 * doesn't spend time verifying the checksum. 2238 * 2239 *********************************************************************/ 2240 static int 2241 igc_rx_checksum(struct igc_queue *q, uint64_t capenable, uint32_t staterr, 2242 uint32_t ptype) 2243 { 2244 const uint16_t status = (uint16_t)staterr; 2245 const uint8_t errors = (uint8_t)(staterr >> 24); 2246 int flags = 0; 2247 2248 if ((status & IGC_RXD_STAT_IPCS) != 0 && 2249 (capenable & IFCAP_CSUM_IPv4_Rx) != 0) { 2250 IGC_DRIVER_EVENT(q, rx_ipcs, 1); 2251 flags |= M_CSUM_IPv4; 2252 if (__predict_false((errors & IGC_RXD_ERR_IPE) != 0)) { 2253 IGC_DRIVER_EVENT(q, rx_ipcs_bad, 1); 2254 flags |= M_CSUM_IPv4_BAD; 2255 } 2256 } 2257 2258 if ((status & IGC_RXD_STAT_TCPCS) != 0) { 2259 IGC_DRIVER_EVENT(q, rx_tcpcs, 1); 2260 if ((capenable & IFCAP_CSUM_TCPv4_Rx) != 0) 2261 flags |= M_CSUM_TCPv4; 2262 if ((capenable & IFCAP_CSUM_TCPv6_Rx) != 0) 2263 flags |= M_CSUM_TCPv6; 2264 } 2265 2266 if ((status & IGC_RXD_STAT_UDPCS) != 0) { 2267 IGC_DRIVER_EVENT(q, rx_udpcs, 1); 2268 if ((capenable & IFCAP_CSUM_UDPv4_Rx) != 0) 2269 flags |= M_CSUM_UDPv4; 2270 if ((capenable & IFCAP_CSUM_UDPv6_Rx) != 0) 2271 flags |= M_CSUM_UDPv6; 2272 } 2273 2274 if (__predict_false((errors & IGC_RXD_ERR_TCPE) != 0)) { 2275 IGC_DRIVER_EVENT(q, rx_l4cs_bad, 1); 2276 if ((flags & ~M_CSUM_IPv4) != 0) 2277 flags |= M_CSUM_TCP_UDP_BAD; 2278 } 2279 2280 return flags; 2281 } 2282 2283 static void 2284 igc_watchdog(struct ifnet * ifp) 2285 { 2286 } 2287 2288 static void 2289 igc_tick(void *arg) 2290 { 2291 struct igc_softc *sc = arg; 2292 2293 mutex_enter(&sc->sc_core_lock); 2294 2295 if (__predict_false(sc->sc_core_stopping)) { 2296 mutex_exit(&sc->sc_core_lock); 2297 return; 2298 } 2299 2300 /* XXX watchdog */ 2301 if (0) { 2302 IGC_GLOBAL_EVENT(sc, watchdog, 1); 2303 } 2304 2305 igc_update_counters(sc); 2306 2307 mutex_exit(&sc->sc_core_lock); 2308 2309 callout_schedule(&sc->sc_tick_ch, hz); 2310 } 2311 2312 /********************************************************************* 2313 * 2314 * Media Ioctl callback 2315 * 2316 * This routine is called whenever the user queries the status of 2317 * the interface using ifconfig. 2318 * 2319 **********************************************************************/ 2320 static void 2321 igc_media_status(struct ifnet *ifp, struct ifmediareq *ifmr) 2322 { 2323 struct igc_softc *sc = ifp->if_softc; 2324 struct igc_hw *hw = &sc->hw; 2325 2326 igc_update_link_status(sc); 2327 2328 ifmr->ifm_status = IFM_AVALID; 2329 ifmr->ifm_active = IFM_ETHER; 2330 2331 if (!sc->link_active) { 2332 ifmr->ifm_active |= IFM_NONE; 2333 return; 2334 } 2335 2336 ifmr->ifm_status |= IFM_ACTIVE; 2337 2338 switch (sc->link_speed) { 2339 case 10: 2340 ifmr->ifm_active |= IFM_10_T; 2341 break; 2342 case 100: 2343 ifmr->ifm_active |= IFM_100_TX; 2344 break; 2345 case 1000: 2346 ifmr->ifm_active |= IFM_1000_T; 2347 break; 2348 case 2500: 2349 ifmr->ifm_active |= IFM_2500_T; 2350 break; 2351 } 2352 2353 if (sc->link_duplex == FULL_DUPLEX) 2354 ifmr->ifm_active |= IFM_FDX; 2355 else 2356 ifmr->ifm_active |= IFM_HDX; 2357 2358 switch (hw->fc.current_mode) { 2359 case igc_fc_tx_pause: 2360 ifmr->ifm_active |= IFM_FLOW | IFM_ETH_TXPAUSE; 2361 break; 2362 case igc_fc_rx_pause: 2363 ifmr->ifm_active |= IFM_FLOW | IFM_ETH_RXPAUSE; 2364 break; 2365 case igc_fc_full: 2366 ifmr->ifm_active |= IFM_FLOW | 2367 IFM_ETH_TXPAUSE | IFM_ETH_RXPAUSE; 2368 break; 2369 case igc_fc_none: 2370 default: 2371 break; 2372 } 2373 } 2374 2375 /********************************************************************* 2376 * 2377 * Media Ioctl callback 2378 * 2379 * This routine is called when the user changes speed/duplex using 2380 * media/mediopt option with ifconfig. 2381 * 2382 **********************************************************************/ 2383 static int 2384 igc_media_change(struct ifnet *ifp) 2385 { 2386 struct igc_softc *sc = ifp->if_softc; 2387 struct ifmedia *ifm = &sc->media; 2388 2389 if (IFM_TYPE(ifm->ifm_media) != IFM_ETHER) 2390 return EINVAL; 2391 2392 sc->hw.mac.autoneg = DO_AUTO_NEG; 2393 2394 switch (IFM_SUBTYPE(ifm->ifm_media)) { 2395 case IFM_AUTO: 2396 sc->hw.phy.autoneg_advertised = AUTONEG_ADV_DEFAULT; 2397 break; 2398 case IFM_2500_T: 2399 sc->hw.phy.autoneg_advertised = ADVERTISE_2500_FULL; 2400 break; 2401 case IFM_1000_T: 2402 sc->hw.phy.autoneg_advertised = ADVERTISE_1000_FULL; 2403 break; 2404 case IFM_100_TX: 2405 if ((ifm->ifm_media & IFM_GMASK) == IFM_FDX) 2406 sc->hw.phy.autoneg_advertised = ADVERTISE_100_FULL; 2407 else 2408 sc->hw.phy.autoneg_advertised = ADVERTISE_100_HALF; 2409 break; 2410 case IFM_10_T: 2411 if ((ifm->ifm_media & IFM_GMASK) == IFM_FDX) 2412 sc->hw.phy.autoneg_advertised = ADVERTISE_10_FULL; 2413 else 2414 sc->hw.phy.autoneg_advertised = ADVERTISE_10_HALF; 2415 break; 2416 default: 2417 return EINVAL; 2418 } 2419 2420 igc_init_locked(sc); 2421 2422 return 0; 2423 } 2424 2425 static int 2426 igc_ifflags_cb(struct ethercom *ec) 2427 { 2428 struct ifnet *ifp = &ec->ec_if; 2429 struct igc_softc *sc = ifp->if_softc; 2430 int rc = 0; 2431 u_short iffchange; 2432 bool needreset = false; 2433 2434 DPRINTF(CFG, "called\n"); 2435 2436 KASSERT(IFNET_LOCKED(ifp)); 2437 2438 mutex_enter(&sc->sc_core_lock); 2439 2440 /* 2441 * Check for if_flags. 2442 * Main usage is to prevent linkdown when opening bpf. 2443 */ 2444 iffchange = ifp->if_flags ^ sc->sc_if_flags; 2445 sc->sc_if_flags = ifp->if_flags; 2446 if ((iffchange & ~(IFF_CANTCHANGE | IFF_DEBUG)) != 0) { 2447 needreset = true; 2448 goto ec; 2449 } 2450 2451 /* iff related updates */ 2452 if ((iffchange & IFF_PROMISC) != 0) 2453 igc_set_filter(sc); 2454 2455 #ifdef notyet 2456 igc_set_vlan(sc); 2457 #endif 2458 2459 ec: 2460 #ifdef notyet 2461 /* Check for ec_capenable. */ 2462 ecchange = ec->ec_capenable ^ sc->sc_ec_capenable; 2463 sc->sc_ec_capenable = ec->ec_capenable; 2464 if ((ecchange & ~ETHERCAP_SOMETHING) != 0) { 2465 needreset = true; 2466 goto out; 2467 } 2468 #endif 2469 if (needreset) 2470 rc = ENETRESET; 2471 2472 mutex_exit(&sc->sc_core_lock); 2473 2474 return rc; 2475 } 2476 2477 static void 2478 igc_set_filter(struct igc_softc *sc) 2479 { 2480 struct ethercom *ec = &sc->sc_ec; 2481 uint32_t rctl; 2482 2483 rctl = IGC_READ_REG(&sc->hw, IGC_RCTL); 2484 rctl &= ~(IGC_RCTL_BAM |IGC_RCTL_UPE | IGC_RCTL_MPE); 2485 2486 if ((sc->sc_if_flags & IFF_BROADCAST) != 0) 2487 rctl |= IGC_RCTL_BAM; 2488 if ((sc->sc_if_flags & IFF_PROMISC) != 0) { 2489 DPRINTF(CFG, "promisc\n"); 2490 rctl |= IGC_RCTL_UPE; 2491 ETHER_LOCK(ec); 2492 allmulti: 2493 ec->ec_flags |= ETHER_F_ALLMULTI; 2494 ETHER_UNLOCK(ec); 2495 rctl |= IGC_RCTL_MPE; 2496 } else { 2497 struct ether_multistep step; 2498 struct ether_multi *enm; 2499 int mcnt = 0; 2500 2501 memset(sc->mta, 0, IGC_MTA_LEN); 2502 2503 ETHER_LOCK(ec); 2504 ETHER_FIRST_MULTI(step, ec, enm); 2505 while (enm != NULL) { 2506 if (((memcmp(enm->enm_addrlo, enm->enm_addrhi, 2507 ETHER_ADDR_LEN)) != 0) || 2508 (mcnt >= MAX_NUM_MULTICAST_ADDRESSES)) { 2509 /* 2510 * We must listen to a range of multicast 2511 * addresses. For now, just accept all 2512 * multicasts, rather than trying to set only 2513 * those filter bits needed to match the range. 2514 * (At this time, the only use of address 2515 * ranges is for IP multicast routing, for 2516 * which the range is big enough to require all 2517 * bits set.) 2518 */ 2519 goto allmulti; 2520 } 2521 DPRINTF(CFG, "%d: %s\n", mcnt, 2522 ether_sprintf(enm->enm_addrlo)); 2523 memcpy(&sc->mta[mcnt * ETHER_ADDR_LEN], 2524 enm->enm_addrlo, ETHER_ADDR_LEN); 2525 2526 mcnt++; 2527 ETHER_NEXT_MULTI(step, enm); 2528 } 2529 ec->ec_flags &= ~ETHER_F_ALLMULTI; 2530 ETHER_UNLOCK(ec); 2531 2532 DPRINTF(CFG, "hw filter\n"); 2533 igc_update_mc_addr_list(&sc->hw, sc->mta, mcnt); 2534 } 2535 2536 IGC_WRITE_REG(&sc->hw, IGC_RCTL, rctl); 2537 } 2538 2539 static void 2540 igc_update_link_status(struct igc_softc *sc) 2541 { 2542 struct ifnet *ifp = &sc->sc_ec.ec_if; 2543 struct igc_hw *hw = &sc->hw; 2544 2545 if (hw->mac.get_link_status == true) 2546 igc_check_for_link(hw); 2547 2548 if (IGC_READ_REG(&sc->hw, IGC_STATUS) & IGC_STATUS_LU) { 2549 if (sc->link_active == 0) { 2550 igc_get_speed_and_duplex(hw, &sc->link_speed, 2551 &sc->link_duplex); 2552 sc->link_active = 1; 2553 ifp->if_baudrate = IF_Mbps(sc->link_speed); 2554 if_link_state_change(ifp, LINK_STATE_UP); 2555 } 2556 } else { 2557 if (sc->link_active == 1) { 2558 ifp->if_baudrate = sc->link_speed = 0; 2559 sc->link_duplex = 0; 2560 sc->link_active = 0; 2561 if_link_state_change(ifp, LINK_STATE_DOWN); 2562 } 2563 } 2564 } 2565 2566 /********************************************************************* 2567 * 2568 * Get a buffer from system mbuf buffer pool. 2569 * 2570 **********************************************************************/ 2571 static int 2572 igc_get_buf(struct rx_ring *rxr, int id, bool strict) 2573 { 2574 struct igc_softc *sc = rxr->sc; 2575 struct igc_queue *q = rxr->rxr_igcq; 2576 struct igc_rx_buf *rxbuf = &rxr->rx_buffers[id]; 2577 bus_dmamap_t map = rxbuf->map; 2578 struct mbuf *m; 2579 int error; 2580 2581 if (__predict_false(rxbuf->buf)) { 2582 if (strict) { 2583 DPRINTF(RX, "slot %d already has an mbuf\n", id); 2584 return EINVAL; 2585 } 2586 return 0; 2587 } 2588 2589 MGETHDR(m, M_DONTWAIT, MT_DATA); 2590 if (__predict_false(m == NULL)) { 2591 enobuf: 2592 IGC_QUEUE_EVENT(q, rx_no_mbuf, 1); 2593 return ENOBUFS; 2594 } 2595 2596 MCLGET(m, M_DONTWAIT); 2597 if (__predict_false(!(m->m_flags & M_EXT))) { 2598 m_freem(m); 2599 goto enobuf; 2600 } 2601 2602 m->m_len = m->m_pkthdr.len = sc->rx_mbuf_sz; 2603 2604 error = bus_dmamap_load_mbuf(rxr->rxdma.dma_tag, map, m, 2605 BUS_DMA_READ | BUS_DMA_NOWAIT); 2606 if (error) { 2607 m_freem(m); 2608 return error; 2609 } 2610 2611 bus_dmamap_sync(rxr->rxdma.dma_tag, map, 0, 2612 map->dm_mapsize, BUS_DMASYNC_PREREAD); 2613 rxbuf->buf = m; 2614 2615 union igc_adv_rx_desc *rxdesc = &rxr->rx_base[id]; 2616 igc_rxdesc_sync(rxr, id, BUS_DMASYNC_POSTWRITE | BUS_DMASYNC_POSTREAD); 2617 rxdesc->read.pkt_addr = htole64(map->dm_segs[0].ds_addr); 2618 igc_rxdesc_sync(rxr, id, BUS_DMASYNC_PREWRITE | BUS_DMASYNC_PREREAD); 2619 2620 return 0; 2621 } 2622 2623 static void 2624 igc_configure_queues(struct igc_softc *sc) 2625 { 2626 struct igc_hw *hw = &sc->hw; 2627 uint32_t ivar; 2628 2629 /* First turn on RSS capability */ 2630 IGC_WRITE_REG(hw, IGC_GPIE, IGC_GPIE_MSIX_MODE | IGC_GPIE_EIAME | 2631 IGC_GPIE_PBA | IGC_GPIE_NSICR); 2632 2633 /* Set the starting interrupt rate */ 2634 uint32_t newitr = (4000000 / MAX_INTS_PER_SEC) & 0x7FFC; 2635 newitr |= IGC_EITR_CNT_IGNR; 2636 2637 /* Turn on MSI-X */ 2638 uint32_t newmask = 0; 2639 for (int iq = 0; iq < sc->sc_nqueues; iq++) { 2640 struct igc_queue *q = &sc->queues[iq]; 2641 2642 /* RX entries */ 2643 igc_set_queues(sc, iq, q->msix, 0); 2644 /* TX entries */ 2645 igc_set_queues(sc, iq, q->msix, 1); 2646 newmask |= q->eims; 2647 IGC_WRITE_REG(hw, IGC_EITR(q->msix), newitr); 2648 } 2649 sc->msix_queuesmask = newmask; 2650 2651 #if 1 2652 ivar = IGC_READ_REG_ARRAY(hw, IGC_IVAR0, 0); 2653 DPRINTF(CFG, "ivar(0)=0x%x\n", ivar); 2654 ivar = IGC_READ_REG_ARRAY(hw, IGC_IVAR0, 1); 2655 DPRINTF(CFG, "ivar(1)=0x%x\n", ivar); 2656 #endif 2657 2658 /* And for the link interrupt */ 2659 ivar = (sc->linkvec | IGC_IVAR_VALID) << 8; 2660 sc->msix_linkmask = 1 << sc->linkvec; 2661 IGC_WRITE_REG(hw, IGC_IVAR_MISC, ivar); 2662 } 2663 2664 static void 2665 igc_set_queues(struct igc_softc *sc, uint32_t entry, uint32_t vector, int type) 2666 { 2667 struct igc_hw *hw = &sc->hw; 2668 const uint32_t index = entry >> 1; 2669 uint32_t ivar = IGC_READ_REG_ARRAY(hw, IGC_IVAR0, index); 2670 2671 if (type) { 2672 if (entry & 1) { 2673 ivar &= 0x00FFFFFF; 2674 ivar |= (vector | IGC_IVAR_VALID) << 24; 2675 } else { 2676 ivar &= 0xFFFF00FF; 2677 ivar |= (vector | IGC_IVAR_VALID) << 8; 2678 } 2679 } else { 2680 if (entry & 1) { 2681 ivar &= 0xFF00FFFF; 2682 ivar |= (vector | IGC_IVAR_VALID) << 16; 2683 } else { 2684 ivar &= 0xFFFFFF00; 2685 ivar |= vector | IGC_IVAR_VALID; 2686 } 2687 } 2688 IGC_WRITE_REG_ARRAY(hw, IGC_IVAR0, index, ivar); 2689 } 2690 2691 static void 2692 igc_enable_queue(struct igc_softc *sc, uint32_t eims) 2693 { 2694 IGC_WRITE_REG(&sc->hw, IGC_EIMS, eims); 2695 } 2696 2697 static void 2698 igc_enable_intr(struct igc_softc *sc) 2699 { 2700 struct igc_hw *hw = &sc->hw; 2701 2702 if (sc->sc_intr_type == PCI_INTR_TYPE_MSIX) { 2703 const uint32_t mask = sc->msix_queuesmask | sc->msix_linkmask; 2704 2705 IGC_WRITE_REG(hw, IGC_EIAC, mask); 2706 IGC_WRITE_REG(hw, IGC_EIAM, mask); 2707 IGC_WRITE_REG(hw, IGC_EIMS, mask); 2708 IGC_WRITE_REG(hw, IGC_IMS, IGC_IMS_LSC); 2709 } else { 2710 IGC_WRITE_REG(hw, IGC_IMS, IMS_ENABLE_MASK); 2711 } 2712 IGC_WRITE_FLUSH(hw); 2713 } 2714 2715 static void 2716 igc_disable_intr(struct igc_softc *sc) 2717 { 2718 struct igc_hw *hw = &sc->hw; 2719 2720 if (sc->sc_intr_type == PCI_INTR_TYPE_MSIX) { 2721 IGC_WRITE_REG(hw, IGC_EIMC, 0xffffffff); 2722 IGC_WRITE_REG(hw, IGC_EIAC, 0); 2723 } 2724 IGC_WRITE_REG(hw, IGC_IMC, 0xffffffff); 2725 IGC_WRITE_FLUSH(hw); 2726 } 2727 2728 static int 2729 igc_intr_link(void *arg) 2730 { 2731 struct igc_softc *sc = (struct igc_softc *)arg; 2732 const uint32_t reg_icr = IGC_READ_REG(&sc->hw, IGC_ICR); 2733 2734 IGC_GLOBAL_EVENT(sc, link, 1); 2735 2736 if (reg_icr & IGC_ICR_LSC) { 2737 mutex_enter(&sc->sc_core_lock); 2738 sc->hw.mac.get_link_status = true; 2739 igc_update_link_status(sc); 2740 mutex_exit(&sc->sc_core_lock); 2741 } 2742 2743 IGC_WRITE_REG(&sc->hw, IGC_IMS, IGC_IMS_LSC); 2744 IGC_WRITE_REG(&sc->hw, IGC_EIMS, sc->msix_linkmask); 2745 2746 return 1; 2747 } 2748 2749 static int 2750 igc_intr_queue(void *arg) 2751 { 2752 struct igc_queue *iq = arg; 2753 struct igc_softc *sc = iq->sc; 2754 struct ifnet *ifp = &sc->sc_ec.ec_if; 2755 struct rx_ring *rxr = iq->rxr; 2756 struct tx_ring *txr = iq->txr; 2757 const u_int txlimit = sc->sc_tx_intr_process_limit, 2758 rxlimit = sc->sc_rx_intr_process_limit; 2759 bool txmore, rxmore; 2760 2761 IGC_QUEUE_EVENT(iq, irqs, 1); 2762 2763 if (__predict_false(!ISSET(ifp->if_flags, IFF_RUNNING))) 2764 return 0; 2765 2766 mutex_enter(&txr->txr_lock); 2767 txmore = igc_txeof(txr, txlimit); 2768 mutex_exit(&txr->txr_lock); 2769 mutex_enter(&rxr->rxr_lock); 2770 rxmore = igc_rxeof(rxr, rxlimit); 2771 mutex_exit(&rxr->rxr_lock); 2772 2773 if (txmore || rxmore) { 2774 IGC_QUEUE_EVENT(iq, req, 1); 2775 igc_sched_handle_queue(sc, iq); 2776 } else { 2777 igc_enable_queue(sc, iq->eims); 2778 } 2779 2780 return 1; 2781 } 2782 2783 static int 2784 igc_intr(void *arg) 2785 { 2786 struct igc_softc *sc = arg; 2787 struct ifnet *ifp = &sc->sc_ec.ec_if; 2788 struct igc_queue *iq = &sc->queues[0]; 2789 struct rx_ring *rxr = iq->rxr; 2790 struct tx_ring *txr = iq->txr; 2791 const u_int txlimit = sc->sc_tx_intr_process_limit, 2792 rxlimit = sc->sc_rx_intr_process_limit; 2793 bool txmore, rxmore; 2794 2795 if (__predict_false(!ISSET(ifp->if_flags, IFF_RUNNING))) 2796 return 0; 2797 2798 const uint32_t reg_icr = IGC_READ_REG(&sc->hw, IGC_ICR); 2799 DPRINTF(MISC, "reg_icr=0x%x\n", reg_icr); 2800 2801 /* Definitely not our interrupt. */ 2802 if (reg_icr == 0x0) { 2803 DPRINTF(MISC, "not for me"); 2804 return 0; 2805 } 2806 2807 IGC_QUEUE_EVENT(iq, irqs, 1); 2808 2809 /* Hot eject? */ 2810 if (__predict_false(reg_icr == 0xffffffff)) { 2811 DPRINTF(MISC, "hot eject\n"); 2812 return 0; 2813 } 2814 2815 if (__predict_false(!(reg_icr & IGC_ICR_INT_ASSERTED))) { 2816 DPRINTF(MISC, "not set IGC_ICR_INT_ASSERTED"); 2817 return 0; 2818 } 2819 2820 /* 2821 * Only MSI-X interrupts have one-shot behavior by taking advantage 2822 * of the EIAC register. Thus, explicitly disable interrupts. This 2823 * also works around the MSI message reordering errata on certain 2824 * systems. 2825 */ 2826 igc_disable_intr(sc); 2827 2828 mutex_enter(&txr->txr_lock); 2829 txmore = igc_txeof(txr, txlimit); 2830 mutex_exit(&txr->txr_lock); 2831 mutex_enter(&rxr->rxr_lock); 2832 rxmore = igc_rxeof(rxr, rxlimit); 2833 mutex_exit(&rxr->rxr_lock); 2834 2835 /* Link status change */ 2836 // XXXX FreeBSD checks IGC_ICR_RXSEQ 2837 if (__predict_false(reg_icr & IGC_ICR_LSC)) { 2838 IGC_GLOBAL_EVENT(sc, link, 1); 2839 mutex_enter(&sc->sc_core_lock); 2840 sc->hw.mac.get_link_status = true; 2841 igc_update_link_status(sc); 2842 mutex_exit(&sc->sc_core_lock); 2843 } 2844 2845 if (txmore || rxmore) { 2846 IGC_QUEUE_EVENT(iq, req, 1); 2847 igc_sched_handle_queue(sc, iq); 2848 } else { 2849 igc_enable_intr(sc); 2850 } 2851 2852 return 1; 2853 } 2854 2855 static void 2856 igc_handle_queue(void *arg) 2857 { 2858 struct igc_queue *iq = arg; 2859 struct igc_softc *sc = iq->sc; 2860 struct tx_ring *txr = iq->txr; 2861 struct rx_ring *rxr = iq->rxr; 2862 const u_int txlimit = sc->sc_tx_process_limit, 2863 rxlimit = sc->sc_rx_process_limit; 2864 bool txmore, rxmore; 2865 2866 IGC_QUEUE_EVENT(iq, handleq, 1); 2867 2868 mutex_enter(&txr->txr_lock); 2869 txmore = igc_txeof(txr, txlimit); 2870 /* for ALTQ, dequeue from if_snd */ 2871 if (txr->me == 0) { 2872 struct ifnet *ifp = &sc->sc_ec.ec_if; 2873 2874 igc_tx_common_locked(ifp, txr, IGC_TX_START); 2875 } 2876 mutex_exit(&txr->txr_lock); 2877 2878 mutex_enter(&rxr->rxr_lock); 2879 rxmore = igc_rxeof(rxr, rxlimit); 2880 mutex_exit(&rxr->rxr_lock); 2881 2882 if (txmore || rxmore) { 2883 igc_sched_handle_queue(sc, iq); 2884 } else { 2885 if (sc->sc_intr_type == PCI_INTR_TYPE_MSIX) 2886 igc_enable_queue(sc, iq->eims); 2887 else 2888 igc_enable_intr(sc); 2889 } 2890 } 2891 2892 static void 2893 igc_handle_queue_work(struct work *wk, void *context) 2894 { 2895 struct igc_queue *iq = 2896 container_of(wk, struct igc_queue, igcq_wq_cookie); 2897 2898 igc_handle_queue(iq); 2899 } 2900 2901 static void 2902 igc_sched_handle_queue(struct igc_softc *sc, struct igc_queue *iq) 2903 { 2904 2905 if (iq->igcq_workqueue) { 2906 /* XXXRO notyet */ 2907 workqueue_enqueue(sc->sc_queue_wq, &iq->igcq_wq_cookie, 2908 curcpu()); 2909 } else { 2910 softint_schedule(iq->igcq_si); 2911 } 2912 } 2913 2914 static void 2915 igc_barrier_handle_queue(struct igc_softc *sc) 2916 { 2917 2918 if (sc->sc_txrx_workqueue) { 2919 for (int iq = 0; iq < sc->sc_nqueues; iq++) { 2920 struct igc_queue *q = &sc->queues[iq]; 2921 2922 workqueue_wait(sc->sc_queue_wq, &q->igcq_wq_cookie); 2923 } 2924 } else { 2925 xc_barrier(0); 2926 } 2927 } 2928 2929 /********************************************************************* 2930 * 2931 * Allocate memory for tx_buffer structures. The tx_buffer stores all 2932 * the information needed to transmit a packet on the wire. 2933 * 2934 **********************************************************************/ 2935 static int 2936 igc_allocate_transmit_buffers(struct tx_ring *txr) 2937 { 2938 struct igc_softc *sc = txr->sc; 2939 int error; 2940 2941 txr->tx_buffers = 2942 kmem_zalloc(sc->num_tx_desc * sizeof(struct igc_tx_buf), KM_SLEEP); 2943 txr->txtag = txr->txdma.dma_tag; 2944 2945 /* Create the descriptor buffer dma maps. */ 2946 for (int id = 0; id < sc->num_tx_desc; id++) { 2947 struct igc_tx_buf *txbuf = &txr->tx_buffers[id]; 2948 2949 error = bus_dmamap_create(txr->txdma.dma_tag, 2950 round_page(IGC_TSO_SIZE + sizeof(struct ether_vlan_header)), 2951 IGC_MAX_SCATTER, PAGE_SIZE, 0, BUS_DMA_NOWAIT, &txbuf->map); 2952 if (error != 0) { 2953 aprint_error_dev(sc->sc_dev, 2954 "unable to create TX DMA map\n"); 2955 goto fail; 2956 } 2957 2958 txbuf->eop_index = -1; 2959 } 2960 2961 return 0; 2962 fail: 2963 return error; 2964 } 2965 2966 2967 /********************************************************************* 2968 * 2969 * Allocate and initialize transmit structures. 2970 * 2971 **********************************************************************/ 2972 static int 2973 igc_setup_transmit_structures(struct igc_softc *sc) 2974 { 2975 2976 for (int iq = 0; iq < sc->sc_nqueues; iq++) { 2977 struct tx_ring *txr = &sc->tx_rings[iq]; 2978 2979 if (igc_setup_transmit_ring(txr)) 2980 goto fail; 2981 } 2982 2983 return 0; 2984 fail: 2985 igc_free_transmit_structures(sc); 2986 return ENOBUFS; 2987 } 2988 2989 /********************************************************************* 2990 * 2991 * Initialize a transmit ring. 2992 * 2993 **********************************************************************/ 2994 static int 2995 igc_setup_transmit_ring(struct tx_ring *txr) 2996 { 2997 struct igc_softc *sc = txr->sc; 2998 2999 /* Now allocate transmit buffers for the ring. */ 3000 if (igc_allocate_transmit_buffers(txr)) 3001 return ENOMEM; 3002 3003 /* Clear the old ring contents */ 3004 memset(txr->tx_base, 0, 3005 sizeof(union igc_adv_tx_desc) * sc->num_tx_desc); 3006 3007 /* Reset indices. */ 3008 txr->next_avail_desc = 0; 3009 txr->next_to_clean = 0; 3010 3011 bus_dmamap_sync(txr->txdma.dma_tag, txr->txdma.dma_map, 0, 3012 txr->txdma.dma_map->dm_mapsize, 3013 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); 3014 3015 txr->txr_interq = pcq_create(sc->num_tx_desc, KM_SLEEP); 3016 3017 mutex_init(&txr->txr_lock, MUTEX_DEFAULT, IPL_NET); 3018 3019 return 0; 3020 } 3021 3022 /********************************************************************* 3023 * 3024 * Enable transmit unit. 3025 * 3026 **********************************************************************/ 3027 static void 3028 igc_initialize_transmit_unit(struct igc_softc *sc) 3029 { 3030 struct ifnet *ifp = &sc->sc_ec.ec_if; 3031 struct igc_hw *hw = &sc->hw; 3032 3033 /* Setup the Base and Length of the TX descriptor ring. */ 3034 for (int iq = 0; iq < sc->sc_nqueues; iq++) { 3035 struct tx_ring *txr = &sc->tx_rings[iq]; 3036 const uint64_t bus_addr = 3037 txr->txdma.dma_map->dm_segs[0].ds_addr; 3038 3039 /* Base and len of TX ring */ 3040 IGC_WRITE_REG(hw, IGC_TDLEN(iq), 3041 sc->num_tx_desc * sizeof(union igc_adv_tx_desc)); 3042 IGC_WRITE_REG(hw, IGC_TDBAH(iq), (uint32_t)(bus_addr >> 32)); 3043 IGC_WRITE_REG(hw, IGC_TDBAL(iq), (uint32_t)bus_addr); 3044 3045 /* Init the HEAD/TAIL indices */ 3046 IGC_WRITE_REG(hw, IGC_TDT(iq), 0 /* XXX txr->next_avail_desc */); 3047 IGC_WRITE_REG(hw, IGC_TDH(iq), 0); 3048 3049 txr->watchdog_timer = 0; 3050 3051 uint32_t txdctl = 0; /* Clear txdctl */ 3052 txdctl |= 0x1f; /* PTHRESH */ 3053 txdctl |= 1 << 8; /* HTHRESH */ 3054 txdctl |= 1 << 16; /* WTHRESH */ 3055 txdctl |= 1 << 22; /* Reserved bit 22 must always be 1 */ 3056 txdctl |= IGC_TXDCTL_GRAN; 3057 txdctl |= 1 << 25; /* LWTHRESH */ 3058 3059 IGC_WRITE_REG(hw, IGC_TXDCTL(iq), txdctl); 3060 } 3061 ifp->if_timer = 0; 3062 3063 /* Program the Transmit Control Register */ 3064 uint32_t tctl = IGC_READ_REG(&sc->hw, IGC_TCTL); 3065 tctl &= ~IGC_TCTL_CT; 3066 tctl |= (IGC_TCTL_PSP | IGC_TCTL_RTLC | IGC_TCTL_EN | 3067 (IGC_COLLISION_THRESHOLD << IGC_CT_SHIFT)); 3068 3069 /* This write will effectively turn on the transmit unit. */ 3070 IGC_WRITE_REG(&sc->hw, IGC_TCTL, tctl); 3071 } 3072 3073 /********************************************************************* 3074 * 3075 * Free all transmit rings. 3076 * 3077 **********************************************************************/ 3078 static void 3079 igc_free_transmit_structures(struct igc_softc *sc) 3080 { 3081 3082 for (int iq = 0; iq < sc->sc_nqueues; iq++) { 3083 struct tx_ring *txr = &sc->tx_rings[iq]; 3084 3085 igc_free_transmit_buffers(txr); 3086 } 3087 } 3088 3089 /********************************************************************* 3090 * 3091 * Free transmit ring related data structures. 3092 * 3093 **********************************************************************/ 3094 static void 3095 igc_free_transmit_buffers(struct tx_ring *txr) 3096 { 3097 struct igc_softc *sc = txr->sc; 3098 3099 if (txr->tx_buffers == NULL) 3100 return; 3101 3102 igc_withdraw_transmit_packets(txr, true); 3103 3104 kmem_free(txr->tx_buffers, 3105 sc->num_tx_desc * sizeof(struct igc_tx_buf)); 3106 txr->tx_buffers = NULL; 3107 txr->txtag = NULL; 3108 3109 pcq_destroy(txr->txr_interq); 3110 mutex_destroy(&txr->txr_lock); 3111 } 3112 3113 /********************************************************************* 3114 * 3115 * Withdraw transmit packets. 3116 * 3117 **********************************************************************/ 3118 static void 3119 igc_withdraw_transmit_packets(struct tx_ring *txr, bool destroy) 3120 { 3121 struct igc_softc *sc = txr->sc; 3122 struct igc_queue *q = txr->txr_igcq; 3123 3124 mutex_enter(&txr->txr_lock); 3125 3126 for (int id = 0; id < sc->num_tx_desc; id++) { 3127 union igc_adv_tx_desc *txdesc = &txr->tx_base[id]; 3128 3129 igc_txdesc_sync(txr, id, 3130 BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE); 3131 txdesc->read.buffer_addr = 0; 3132 txdesc->read.cmd_type_len = 0; 3133 txdesc->read.olinfo_status = 0; 3134 igc_txdesc_sync(txr, id, 3135 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); 3136 3137 struct igc_tx_buf *txbuf = &txr->tx_buffers[id]; 3138 bus_dmamap_t map = txbuf->map; 3139 3140 if (map != NULL && map->dm_nsegs > 0) { 3141 bus_dmamap_sync(txr->txdma.dma_tag, map, 3142 0, map->dm_mapsize, BUS_DMASYNC_POSTWRITE); 3143 bus_dmamap_unload(txr->txdma.dma_tag, map); 3144 } 3145 if (txbuf->m_head != NULL) { 3146 m_freem(txbuf->m_head); 3147 txbuf->m_head = NULL; 3148 } 3149 if (map != NULL && destroy) { 3150 bus_dmamap_destroy(txr->txdma.dma_tag, map); 3151 txbuf->map = NULL; 3152 } 3153 txbuf->eop_index = -1; 3154 3155 txr->next_avail_desc = 0; 3156 txr->next_to_clean = 0; 3157 } 3158 3159 struct mbuf *m; 3160 while ((m = pcq_get(txr->txr_interq)) != NULL) { 3161 IGC_QUEUE_EVENT(q, tx_pcq_drop, 1); 3162 m_freem(m); 3163 } 3164 3165 mutex_exit(&txr->txr_lock); 3166 } 3167 3168 3169 /********************************************************************* 3170 * 3171 * Advanced Context Descriptor setup for VLAN, CSUM or TSO 3172 * 3173 **********************************************************************/ 3174 3175 static int 3176 igc_tx_ctx_setup(struct tx_ring *txr, struct mbuf *mp, int prod, 3177 uint32_t *cmd_type_len, uint32_t *olinfo_status) 3178 { 3179 struct ether_vlan_header *evl; 3180 uint32_t type_tucmd_mlhl = 0; 3181 uint32_t vlan_macip_lens = 0; 3182 uint32_t ehlen, iphlen; 3183 uint16_t ehtype; 3184 int off = 0; 3185 3186 const int csum_flags = mp->m_pkthdr.csum_flags; 3187 3188 /* First check if TSO is to be used */ 3189 if ((csum_flags & (M_CSUM_TSOv4 | M_CSUM_TSOv6)) != 0) { 3190 return igc_tso_setup(txr, mp, prod, cmd_type_len, 3191 olinfo_status); 3192 } 3193 3194 const bool v4 = (csum_flags & 3195 (M_CSUM_IPv4 | M_CSUM_TCPv4 | M_CSUM_UDPv4)) != 0; 3196 const bool v6 = (csum_flags & (M_CSUM_UDPv6 | M_CSUM_TCPv6)) != 0; 3197 3198 /* Indicate the whole packet as payload when not doing TSO */ 3199 *olinfo_status |= mp->m_pkthdr.len << IGC_ADVTXD_PAYLEN_SHIFT; 3200 3201 /* 3202 * In advanced descriptors the vlan tag must 3203 * be placed into the context descriptor. Hence 3204 * we need to make one even if not doing offloads. 3205 */ 3206 #if NVLAN > 0 3207 if (vlan_has_tag(mp)) { 3208 vlan_macip_lens |= (uint32_t)vlan_get_tag(mp) 3209 << IGC_ADVTXD_VLAN_SHIFT; 3210 off = 1; 3211 } else 3212 #endif 3213 if (!v4 && !v6) 3214 return 0; 3215 3216 KASSERT(mp->m_len >= sizeof(struct ether_header)); 3217 evl = mtod(mp, struct ether_vlan_header *); 3218 if (evl->evl_encap_proto == htons(ETHERTYPE_VLAN)) { 3219 KASSERT(mp->m_len >= sizeof(struct ether_vlan_header)); 3220 ehlen = ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN; 3221 ehtype = evl->evl_proto; 3222 } else { 3223 ehlen = ETHER_HDR_LEN; 3224 ehtype = evl->evl_encap_proto; 3225 } 3226 3227 vlan_macip_lens |= ehlen << IGC_ADVTXD_MACLEN_SHIFT; 3228 3229 #ifdef IGC_DEBUG 3230 /* 3231 * For checksum offloading, L3 headers are not mandatory. 3232 * We use these only for consistency checks. 3233 */ 3234 struct ip *ip; 3235 struct ip6_hdr *ip6; 3236 uint8_t ipproto; 3237 char *l3d; 3238 3239 if (mp->m_len == ehlen && mp->m_next != NULL) 3240 l3d = mtod(mp->m_next, char *); 3241 else 3242 l3d = mtod(mp, char *) + ehlen; 3243 #endif 3244 3245 switch (ntohs(ehtype)) { 3246 case ETHERTYPE_IP: 3247 iphlen = M_CSUM_DATA_IPv4_IPHL(mp->m_pkthdr.csum_data); 3248 type_tucmd_mlhl |= IGC_ADVTXD_TUCMD_IPV4; 3249 3250 if ((csum_flags & M_CSUM_IPv4) != 0) { 3251 *olinfo_status |= IGC_TXD_POPTS_IXSM << 8; 3252 off = 1; 3253 } 3254 #ifdef IGC_DEBUG 3255 KASSERT(!v6); 3256 ip = (void *)l3d; 3257 ipproto = ip->ip_p; 3258 KASSERT(iphlen == ip->ip_hl << 2); 3259 KASSERT((mp->m_pkthdr.csum_flags & M_CSUM_IPv4) == 0 || 3260 ip->ip_sum == 0); 3261 #endif 3262 break; 3263 case ETHERTYPE_IPV6: 3264 iphlen = M_CSUM_DATA_IPv6_IPHL(mp->m_pkthdr.csum_data); 3265 type_tucmd_mlhl |= IGC_ADVTXD_TUCMD_IPV6; 3266 #ifdef IGC_DEBUG 3267 KASSERT(!v4); 3268 ip6 = (void *)l3d; 3269 ipproto = ip6->ip6_nxt; /* XXX */ 3270 KASSERT(iphlen == sizeof(struct ip6_hdr)); 3271 #endif 3272 break; 3273 default: 3274 /* 3275 * Unknown L3 protocol. Clear L3 header length and proceed for 3276 * LAN as done by Linux driver. 3277 */ 3278 iphlen = 0; 3279 #ifdef IGC_DEBUG 3280 KASSERT(!v4 && !v6); 3281 ipproto = 0; 3282 #endif 3283 break; 3284 } 3285 3286 vlan_macip_lens |= iphlen; 3287 3288 const bool tcp = (csum_flags & (M_CSUM_TCPv4 | M_CSUM_TCPv6)) != 0; 3289 const bool udp = (csum_flags & (M_CSUM_UDPv4 | M_CSUM_UDPv6)) != 0; 3290 3291 if (tcp) { 3292 #ifdef IGC_DEBUG 3293 KASSERTMSG(ipproto == IPPROTO_TCP, "ipproto = %d", ipproto); 3294 #endif 3295 type_tucmd_mlhl |= IGC_ADVTXD_TUCMD_L4T_TCP; 3296 *olinfo_status |= IGC_TXD_POPTS_TXSM << 8; 3297 off = 1; 3298 } else if (udp) { 3299 #ifdef IGC_DEBUG 3300 KASSERTMSG(ipproto == IPPROTO_UDP, "ipproto = %d", ipproto); 3301 #endif 3302 type_tucmd_mlhl |= IGC_ADVTXD_TUCMD_L4T_UDP; 3303 *olinfo_status |= IGC_TXD_POPTS_TXSM << 8; 3304 off = 1; 3305 } 3306 3307 if (off == 0) 3308 return 0; 3309 3310 type_tucmd_mlhl |= IGC_ADVTXD_DCMD_DEXT | IGC_ADVTXD_DTYP_CTXT; 3311 3312 /* Now ready a context descriptor */ 3313 struct igc_adv_tx_context_desc *txdesc = 3314 (struct igc_adv_tx_context_desc *)&txr->tx_base[prod]; 3315 3316 /* Now copy bits into descriptor */ 3317 igc_txdesc_sync(txr, prod, 3318 BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE); 3319 htolem32(&txdesc->vlan_macip_lens, vlan_macip_lens); 3320 htolem32(&txdesc->type_tucmd_mlhl, type_tucmd_mlhl); 3321 htolem32(&txdesc->seqnum_seed, 0); 3322 htolem32(&txdesc->mss_l4len_idx, 0); 3323 igc_txdesc_sync(txr, prod, 3324 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); 3325 3326 return 1; 3327 } 3328 3329 /********************************************************************* 3330 * 3331 * Advanced Context Descriptor setup for TSO 3332 * 3333 * XXX XXXRO 3334 * Not working. Some packets are sent with correct csums, but 3335 * others aren't. th->th_sum may be adjusted. 3336 * 3337 **********************************************************************/ 3338 3339 static int 3340 igc_tso_setup(struct tx_ring *txr, struct mbuf *mp, int prod, 3341 uint32_t *cmd_type_len, uint32_t *olinfo_status) 3342 { 3343 #if 1 /* notyet */ 3344 return 0; 3345 #else 3346 struct ether_vlan_header *evl; 3347 struct ip *ip; 3348 struct ip6_hdr *ip6; 3349 struct tcphdr *th; 3350 uint32_t type_tucmd_mlhl = 0; 3351 uint32_t vlan_macip_lens = 0; 3352 uint32_t mss_l4len_idx = 0; 3353 uint32_t ehlen, iphlen, tcphlen, paylen; 3354 uint16_t ehtype; 3355 3356 /* 3357 * In advanced descriptors the vlan tag must 3358 * be placed into the context descriptor. Hence 3359 * we need to make one even if not doing offloads. 3360 */ 3361 #if NVLAN > 0 3362 if (vlan_has_tag(mp)) { 3363 vlan_macip_lens |= (uint32_t)vlan_get_tag(mp) 3364 << IGC_ADVTXD_VLAN_SHIFT; 3365 } 3366 #endif 3367 3368 KASSERT(mp->m_len >= sizeof(struct ether_header)); 3369 evl = mtod(mp, struct ether_vlan_header *); 3370 if (evl->evl_encap_proto == htons(ETHERTYPE_VLAN)) { 3371 KASSERT(mp->m_len >= sizeof(struct ether_vlan_header)); 3372 ehlen = ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN; 3373 ehtype = evl->evl_proto; 3374 } else { 3375 ehlen = ETHER_HDR_LEN; 3376 ehtype = evl->evl_encap_proto; 3377 } 3378 3379 vlan_macip_lens |= ehlen << IGC_ADVTXD_MACLEN_SHIFT; 3380 3381 switch (ntohs(ehtype)) { 3382 case ETHERTYPE_IP: 3383 iphlen = M_CSUM_DATA_IPv4_IPHL(mp->m_pkthdr.csum_data); 3384 type_tucmd_mlhl |= IGC_ADVTXD_TUCMD_IPV4; 3385 *olinfo_status |= IGC_TXD_POPTS_IXSM << 8; 3386 3387 KASSERT(mp->m_len >= ehlen + sizeof(*ip)); 3388 ip = (void *)(mtod(mp, char *) + ehlen); 3389 ip->ip_len = 0; 3390 KASSERT(iphlen == ip->ip_hl << 2); 3391 KASSERT(ip->ip_sum == 0); 3392 KASSERT(ip->ip_p == IPPROTO_TCP); 3393 3394 KASSERT(mp->m_len >= ehlen + iphlen + sizeof(*th)); 3395 th = (void *)((char *)ip + iphlen); 3396 th->th_sum = in_cksum_phdr(ip->ip_src.s_addr, ip->ip_dst.s_addr, 3397 htons(IPPROTO_TCP)); 3398 break; 3399 case ETHERTYPE_IPV6: 3400 iphlen = M_CSUM_DATA_IPv6_IPHL(mp->m_pkthdr.csum_data); 3401 type_tucmd_mlhl |= IGC_ADVTXD_TUCMD_IPV6; 3402 3403 KASSERT(mp->m_len >= ehlen + sizeof(*ip6)); 3404 ip6 = (void *)(mtod(mp, char *) + ehlen); 3405 ip6->ip6_plen = 0; 3406 KASSERT(iphlen == sizeof(struct ip6_hdr)); 3407 KASSERT(ip6->ip6_nxt == IPPROTO_TCP); 3408 3409 KASSERT(mp->m_len >= ehlen + iphlen + sizeof(*th)); 3410 th = (void *)((char *)ip6 + iphlen); 3411 tcphlen = th->th_off << 2; 3412 paylen = mp->m_pkthdr.len - ehlen - iphlen - tcphlen; 3413 th->th_sum = in6_cksum_phdr(&ip6->ip6_src, &ip6->ip6_dst, 0, 3414 htonl(IPPROTO_TCP)); 3415 break; 3416 default: 3417 panic("%s", __func__); 3418 } 3419 3420 tcphlen = th->th_off << 2; 3421 paylen = mp->m_pkthdr.len - ehlen - iphlen - tcphlen; 3422 3423 vlan_macip_lens |= iphlen; 3424 3425 type_tucmd_mlhl |= IGC_ADVTXD_DCMD_DEXT | IGC_ADVTXD_DTYP_CTXT; 3426 type_tucmd_mlhl |= IGC_ADVTXD_TUCMD_L4T_TCP; 3427 3428 mss_l4len_idx |= mp->m_pkthdr.segsz << IGC_ADVTXD_MSS_SHIFT; 3429 mss_l4len_idx |= tcphlen << IGC_ADVTXD_L4LEN_SHIFT; 3430 3431 /* Now ready a context descriptor */ 3432 struct igc_adv_tx_context_desc *txdesc = 3433 (struct igc_adv_tx_context_desc *)&txr->tx_base[prod]; 3434 3435 /* Now copy bits into descriptor */ 3436 igc_txdesc_sync(txr, prod, 3437 BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE); 3438 htolem32(&txdesc->vlan_macip_lens, vlan_macip_lens); 3439 htolem32(&txdesc->type_tucmd_mlhl, type_tucmd_mlhl); 3440 htolem32(&txdesc->seqnum_seed, 0); 3441 htolem32(&txdesc->mss_l4len_idx, mss_l4len_idx); 3442 igc_txdesc_sync(txr, prod, 3443 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); 3444 3445 *cmd_type_len |= IGC_ADVTXD_DCMD_TSE; 3446 *olinfo_status |= IGC_TXD_POPTS_TXSM << 8; 3447 *olinfo_status |= paylen << IGC_ADVTXD_PAYLEN_SHIFT; 3448 3449 return 1; 3450 #endif /* notyet */ 3451 } 3452 3453 /********************************************************************* 3454 * 3455 * Allocate memory for rx_buffer structures. Since we use one 3456 * rx_buffer per received packet, the maximum number of rx_buffer's 3457 * that we'll need is equal to the number of receive descriptors 3458 * that we've allocated. 3459 * 3460 **********************************************************************/ 3461 static int 3462 igc_allocate_receive_buffers(struct rx_ring *rxr) 3463 { 3464 struct igc_softc *sc = rxr->sc; 3465 int error; 3466 3467 rxr->rx_buffers = 3468 kmem_zalloc(sc->num_rx_desc * sizeof(struct igc_rx_buf), KM_SLEEP); 3469 3470 for (int id = 0; id < sc->num_rx_desc; id++) { 3471 struct igc_rx_buf *rxbuf = &rxr->rx_buffers[id]; 3472 3473 error = bus_dmamap_create(rxr->rxdma.dma_tag, MCLBYTES, 1, 3474 MCLBYTES, 0, BUS_DMA_WAITOK, &rxbuf->map); 3475 if (error) { 3476 aprint_error_dev(sc->sc_dev, 3477 "unable to create RX DMA map\n"); 3478 goto fail; 3479 } 3480 } 3481 bus_dmamap_sync(rxr->rxdma.dma_tag, rxr->rxdma.dma_map, 0, 3482 rxr->rxdma.dma_map->dm_mapsize, 3483 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); 3484 3485 return 0; 3486 fail: 3487 return error; 3488 } 3489 3490 /********************************************************************* 3491 * 3492 * Allocate and initialize receive structures. 3493 * 3494 **********************************************************************/ 3495 static int 3496 igc_setup_receive_structures(struct igc_softc *sc) 3497 { 3498 3499 for (int iq = 0; iq < sc->sc_nqueues; iq++) { 3500 struct rx_ring *rxr = &sc->rx_rings[iq]; 3501 3502 if (igc_setup_receive_ring(rxr)) 3503 goto fail; 3504 } 3505 3506 return 0; 3507 fail: 3508 igc_free_receive_structures(sc); 3509 return ENOBUFS; 3510 } 3511 3512 /********************************************************************* 3513 * 3514 * Initialize a receive ring and its buffers. 3515 * 3516 **********************************************************************/ 3517 static int 3518 igc_setup_receive_ring(struct rx_ring *rxr) 3519 { 3520 struct igc_softc *sc = rxr->sc; 3521 const int rsize = roundup2( 3522 sc->num_rx_desc * sizeof(union igc_adv_rx_desc), IGC_DBA_ALIGN); 3523 3524 /* Clear the ring contents. */ 3525 memset(rxr->rx_base, 0, rsize); 3526 3527 if (igc_allocate_receive_buffers(rxr)) 3528 return ENOMEM; 3529 3530 /* Setup our descriptor indices. */ 3531 rxr->next_to_check = 0; 3532 rxr->last_desc_filled = 0; 3533 3534 mutex_init(&rxr->rxr_lock, MUTEX_DEFAULT, IPL_NET); 3535 3536 return 0; 3537 } 3538 3539 /********************************************************************* 3540 * 3541 * Enable receive unit. 3542 * 3543 **********************************************************************/ 3544 static void 3545 igc_initialize_receive_unit(struct igc_softc *sc) 3546 { 3547 struct ifnet *ifp = &sc->sc_ec.ec_if; 3548 struct igc_hw *hw = &sc->hw; 3549 uint32_t rctl, rxcsum, srrctl; 3550 3551 DPRINTF(RX, "called\n"); 3552 3553 /* 3554 * Make sure receives are disabled while setting 3555 * up the descriptor ring. 3556 */ 3557 rctl = IGC_READ_REG(hw, IGC_RCTL); 3558 IGC_WRITE_REG(hw, IGC_RCTL, rctl & ~IGC_RCTL_EN); 3559 3560 /* Setup the Receive Control Register */ 3561 rctl &= ~(3 << IGC_RCTL_MO_SHIFT); 3562 rctl |= IGC_RCTL_EN | IGC_RCTL_BAM | IGC_RCTL_LBM_NO | 3563 IGC_RCTL_RDMTS_HALF | (hw->mac.mc_filter_type << IGC_RCTL_MO_SHIFT); 3564 3565 #if 1 3566 /* Do not store bad packets */ 3567 rctl &= ~IGC_RCTL_SBP; 3568 #else 3569 /* for debug */ 3570 rctl |= IGC_RCTL_SBP; 3571 #endif 3572 3573 /* Enable Long Packet receive */ 3574 if (sc->hw.mac.max_frame_size > ETHER_MAX_LEN) 3575 rctl |= IGC_RCTL_LPE; 3576 else 3577 rctl &= ~IGC_RCTL_LPE; 3578 3579 /* Strip the CRC */ 3580 rctl |= IGC_RCTL_SECRC; 3581 3582 /* 3583 * Set the interrupt throttling rate. Value is calculated 3584 * as DEFAULT_ITR = 1/(MAX_INTS_PER_SEC * 256ns) 3585 * 3586 * XXX Sync with Linux, especially for jumbo MTU or TSO. 3587 * XXX Shouldn't be here? 3588 */ 3589 IGC_WRITE_REG(hw, IGC_ITR, DEFAULT_ITR); 3590 3591 rxcsum = IGC_READ_REG(hw, IGC_RXCSUM); 3592 rxcsum &= ~(IGC_RXCSUM_IPOFL | IGC_RXCSUM_TUOFL | IGC_RXCSUM_PCSD); 3593 if (ifp->if_capenable & IFCAP_CSUM_IPv4_Rx) 3594 rxcsum |= IGC_RXCSUM_IPOFL; 3595 if (ifp->if_capenable & (IFCAP_CSUM_TCPv4_Rx | IFCAP_CSUM_UDPv4_Rx | 3596 IFCAP_CSUM_TCPv6_Rx | IFCAP_CSUM_UDPv6_Rx)) 3597 rxcsum |= IGC_RXCSUM_TUOFL; 3598 if (sc->sc_nqueues > 1) 3599 rxcsum |= IGC_RXCSUM_PCSD; 3600 IGC_WRITE_REG(hw, IGC_RXCSUM, rxcsum); 3601 3602 if (sc->sc_nqueues > 1) 3603 igc_initialize_rss_mapping(sc); 3604 3605 srrctl = 0; 3606 #if 0 3607 srrctl |= 4096 >> IGC_SRRCTL_BSIZEPKT_SHIFT; 3608 rctl |= IGC_RCTL_SZ_4096 | IGC_RCTL_BSEX; 3609 #else 3610 srrctl |= 2048 >> IGC_SRRCTL_BSIZEPKT_SHIFT; 3611 rctl |= IGC_RCTL_SZ_2048; 3612 #endif 3613 3614 /* 3615 * If TX flow control is disabled and there's > 1 queue defined, 3616 * enable DROP. 3617 * 3618 * This drops frames rather than hanging the RX MAC for all queues. 3619 */ 3620 if (sc->sc_nqueues > 1 && 3621 (sc->fc == igc_fc_none || sc->fc == igc_fc_rx_pause)) 3622 srrctl |= IGC_SRRCTL_DROP_EN; 3623 3624 /* Setup the Base and Length of the RX descriptor rings. */ 3625 for (int iq = 0; iq < sc->sc_nqueues; iq++) { 3626 struct rx_ring *rxr = &sc->rx_rings[iq]; 3627 const uint64_t bus_addr = 3628 rxr->rxdma.dma_map->dm_segs[0].ds_addr; 3629 3630 IGC_WRITE_REG(hw, IGC_RXDCTL(iq), 0); 3631 3632 srrctl |= IGC_SRRCTL_DESCTYPE_ADV_ONEBUF; 3633 3634 IGC_WRITE_REG(hw, IGC_RDLEN(iq), 3635 sc->num_rx_desc * sizeof(union igc_adv_rx_desc)); 3636 IGC_WRITE_REG(hw, IGC_RDBAH(iq), (uint32_t)(bus_addr >> 32)); 3637 IGC_WRITE_REG(hw, IGC_RDBAL(iq), (uint32_t)bus_addr); 3638 IGC_WRITE_REG(hw, IGC_SRRCTL(iq), srrctl); 3639 3640 /* Setup the Head and Tail Descriptor Pointers */ 3641 IGC_WRITE_REG(hw, IGC_RDH(iq), 0); 3642 IGC_WRITE_REG(hw, IGC_RDT(iq), 0 /* XXX rxr->last_desc_filled */); 3643 3644 /* Enable this Queue */ 3645 uint32_t rxdctl = IGC_READ_REG(hw, IGC_RXDCTL(iq)); 3646 rxdctl |= IGC_RXDCTL_QUEUE_ENABLE; 3647 rxdctl &= 0xFFF00000; 3648 rxdctl |= IGC_RX_PTHRESH; 3649 rxdctl |= IGC_RX_HTHRESH << 8; 3650 rxdctl |= IGC_RX_WTHRESH << 16; 3651 IGC_WRITE_REG(hw, IGC_RXDCTL(iq), rxdctl); 3652 } 3653 3654 /* Make sure VLAN Filters are off */ 3655 rctl &= ~IGC_RCTL_VFE; 3656 3657 /* Write out the settings */ 3658 IGC_WRITE_REG(hw, IGC_RCTL, rctl); 3659 } 3660 3661 /********************************************************************* 3662 * 3663 * Free all receive rings. 3664 * 3665 **********************************************************************/ 3666 static void 3667 igc_free_receive_structures(struct igc_softc *sc) 3668 { 3669 3670 for (int iq = 0; iq < sc->sc_nqueues; iq++) { 3671 struct rx_ring *rxr = &sc->rx_rings[iq]; 3672 3673 igc_free_receive_buffers(rxr); 3674 } 3675 } 3676 3677 /********************************************************************* 3678 * 3679 * Free receive ring data structures 3680 * 3681 **********************************************************************/ 3682 static void 3683 igc_free_receive_buffers(struct rx_ring *rxr) 3684 { 3685 struct igc_softc *sc = rxr->sc; 3686 3687 if (rxr->rx_buffers != NULL) { 3688 for (int id = 0; id < sc->num_rx_desc; id++) { 3689 struct igc_rx_buf *rxbuf = &rxr->rx_buffers[id]; 3690 bus_dmamap_t map = rxbuf->map; 3691 3692 if (rxbuf->buf != NULL) { 3693 bus_dmamap_sync(rxr->rxdma.dma_tag, map, 3694 0, map->dm_mapsize, BUS_DMASYNC_POSTREAD); 3695 bus_dmamap_unload(rxr->rxdma.dma_tag, map); 3696 m_freem(rxbuf->buf); 3697 rxbuf->buf = NULL; 3698 } 3699 bus_dmamap_destroy(rxr->rxdma.dma_tag, map); 3700 rxbuf->map = NULL; 3701 } 3702 kmem_free(rxr->rx_buffers, 3703 sc->num_rx_desc * sizeof(struct igc_rx_buf)); 3704 rxr->rx_buffers = NULL; 3705 } 3706 3707 mutex_destroy(&rxr->rxr_lock); 3708 } 3709 3710 /********************************************************************* 3711 * 3712 * Clear status registers in all RX descriptors. 3713 * 3714 **********************************************************************/ 3715 static void 3716 igc_clear_receive_status(struct rx_ring *rxr) 3717 { 3718 struct igc_softc *sc = rxr->sc; 3719 3720 mutex_enter(&rxr->rxr_lock); 3721 3722 for (int id = 0; id < sc->num_rx_desc; id++) { 3723 union igc_adv_rx_desc *rxdesc = &rxr->rx_base[id]; 3724 3725 igc_rxdesc_sync(rxr, id, 3726 BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE); 3727 rxdesc->wb.upper.status_error = 0; 3728 igc_rxdesc_sync(rxr, id, 3729 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); 3730 } 3731 3732 mutex_exit(&rxr->rxr_lock); 3733 } 3734 3735 /* 3736 * Initialise the RSS mapping for NICs that support multiple transmit/ 3737 * receive rings. 3738 */ 3739 static void 3740 igc_initialize_rss_mapping(struct igc_softc *sc) 3741 { 3742 struct igc_hw *hw = &sc->hw; 3743 3744 /* 3745 * The redirection table controls which destination 3746 * queue each bucket redirects traffic to. 3747 * Each DWORD represents four queues, with the LSB 3748 * being the first queue in the DWORD. 3749 * 3750 * This just allocates buckets to queues using round-robin 3751 * allocation. 3752 * 3753 * NOTE: It Just Happens to line up with the default 3754 * RSS allocation method. 3755 */ 3756 3757 /* Warning FM follows */ 3758 uint32_t reta = 0; 3759 for (int i = 0; i < 128; i++) { 3760 const int shift = 0; /* XXXRO */ 3761 int queue_id = i % sc->sc_nqueues; 3762 /* Adjust if required */ 3763 queue_id <<= shift; 3764 3765 /* 3766 * The low 8 bits are for hash value (n+0); 3767 * The next 8 bits are for hash value (n+1), etc. 3768 */ 3769 reta >>= 8; 3770 reta |= ((uint32_t)queue_id) << 24; 3771 if ((i & 3) == 3) { 3772 IGC_WRITE_REG(hw, IGC_RETA(i >> 2), reta); 3773 reta = 0; 3774 } 3775 } 3776 3777 /* 3778 * MRQC: Multiple Receive Queues Command 3779 * Set queuing to RSS control, number depends on the device. 3780 */ 3781 3782 /* Set up random bits */ 3783 uint32_t rss_key[RSS_KEYSIZE / sizeof(uint32_t)]; 3784 rss_getkey((uint8_t *)rss_key); 3785 3786 /* Now fill our hash function seeds */ 3787 for (int i = 0; i < __arraycount(rss_key); i++) 3788 IGC_WRITE_REG_ARRAY(hw, IGC_RSSRK(0), i, rss_key[i]); 3789 3790 /* 3791 * Configure the RSS fields to hash upon. 3792 */ 3793 uint32_t mrqc = IGC_MRQC_ENABLE_RSS_4Q; 3794 mrqc |= IGC_MRQC_RSS_FIELD_IPV4 | IGC_MRQC_RSS_FIELD_IPV4_TCP; 3795 mrqc |= IGC_MRQC_RSS_FIELD_IPV6 | IGC_MRQC_RSS_FIELD_IPV6_TCP; 3796 mrqc |= IGC_MRQC_RSS_FIELD_IPV6_TCP_EX; 3797 3798 IGC_WRITE_REG(hw, IGC_MRQC, mrqc); 3799 } 3800 3801 /* 3802 * igc_get_hw_control sets the {CTRL_EXT|FWSM}:DRV_LOAD bit. 3803 * For ASF and Pass Through versions of f/w this means 3804 * that the driver is loaded. For AMT version type f/w 3805 * this means that the network i/f is open. 3806 */ 3807 static void 3808 igc_get_hw_control(struct igc_softc *sc) 3809 { 3810 const uint32_t ctrl_ext = IGC_READ_REG(&sc->hw, IGC_CTRL_EXT); 3811 3812 IGC_WRITE_REG(&sc->hw, IGC_CTRL_EXT, ctrl_ext | IGC_CTRL_EXT_DRV_LOAD); 3813 } 3814 3815 /* 3816 * igc_release_hw_control resets {CTRL_EXT|FWSM}:DRV_LOAD bit. 3817 * For ASF and Pass Through versions of f/w this means that 3818 * the driver is no longer loaded. For AMT versions of the 3819 * f/w this means that the network i/f is closed. 3820 */ 3821 static void 3822 igc_release_hw_control(struct igc_softc *sc) 3823 { 3824 const uint32_t ctrl_ext = IGC_READ_REG(&sc->hw, IGC_CTRL_EXT); 3825 3826 IGC_WRITE_REG(&sc->hw, IGC_CTRL_EXT, ctrl_ext & ~IGC_CTRL_EXT_DRV_LOAD); 3827 } 3828 3829 static int 3830 igc_is_valid_ether_addr(uint8_t *addr) 3831 { 3832 const char zero_addr[6] = { 0, 0, 0, 0, 0, 0 }; 3833 3834 if ((addr[0] & 1) || !bcmp(addr, zero_addr, ETHER_ADDR_LEN)) 3835 return 0; 3836 3837 return 1; 3838 } 3839 3840 static void 3841 igc_print_devinfo(struct igc_softc *sc) 3842 { 3843 device_t dev = sc->sc_dev; 3844 struct igc_hw *hw = &sc->hw; 3845 struct igc_phy_info *phy = &hw->phy; 3846 u_int oui, model, rev; 3847 uint16_t id1, id2, nvm_ver, phy_ver, etk_lo, etk_hi; 3848 char descr[MII_MAX_DESCR_LEN]; 3849 3850 /* Print PHY Info */ 3851 id1 = phy->id >> 16; 3852 /* The revision field in phy->id is cleard and it's in phy->revision */ 3853 id2 = (phy->id & 0xfff0) | phy->revision; 3854 oui = MII_OUI(id1, id2); 3855 model = MII_MODEL(id2); 3856 rev = MII_REV(id2); 3857 mii_get_descr(descr, sizeof(descr), oui, model); 3858 if (descr[0]) 3859 aprint_normal_dev(dev, "PHY: %s, rev. %d", 3860 descr, rev); 3861 else 3862 aprint_normal_dev(dev, 3863 "PHY OUI 0x%06x, model 0x%04x, rev. %d", 3864 oui, model, rev); 3865 3866 /* PHY FW version */ 3867 phy->ops.read_reg(hw, 0x1e, &phy_ver); 3868 aprint_normal(", PHY FW version 0x%04hx\n", phy_ver); 3869 3870 /* NVM version */ 3871 hw->nvm.ops.read(hw, NVM_VERSION, 1, &nvm_ver); 3872 3873 /* EtrackID */ 3874 hw->nvm.ops.read(hw, NVM_ETKID_LO, 1, &etk_lo); 3875 hw->nvm.ops.read(hw, NVM_ETKID_HI, 1, &etk_hi); 3876 3877 aprint_normal_dev(dev, 3878 "NVM image version %x.%02x, EtrackID %04hx%04hx\n", 3879 (nvm_ver & NVM_VERSION_MAJOR) >> NVM_VERSION_MAJOR_SHIFT, 3880 nvm_ver & NVM_VERSION_MINOR, etk_hi, etk_lo); 3881 } 3882
Indexes created Tue Sep 30 11:09:46 GMT 2025