if_igc.c revision 1.5
1 /* $NetBSD: if_igc.c,v 1.5 2023/10/11 18:19:43 riastradh Exp $ */ 2 /* $OpenBSD: if_igc.c,v 1.13 2023/04/28 10:18:57 bluhm Exp $ */ 3 /*- 4 * SPDX-License-Identifier: BSD-2-Clause 5 * 6 * Copyright (c) 2016 Nicole Graziano <nicole (at) nextbsd.org> 7 * All rights reserved. 8 * Copyright (c) 2021 Rubicon Communications, LLC (Netgate) 9 * 10 * Redistribution and use in source and binary forms, with or without 11 * modification, are permitted provided that the following conditions 12 * are met: 13 * 1. Redistributions of source code must retain the above copyright 14 * notice, this list of conditions and the following disclaimer. 15 * 2. Redistributions in binary form must reproduce the above copyright 16 * notice, this list of conditions and the following disclaimer in the 17 * documentation and/or other materials provided with the distribution. 18 * 19 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 20 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 21 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 22 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 23 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 24 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 25 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 26 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 27 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 28 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 29 * SUCH DAMAGE. 30 */ 31 32 #include <sys/cdefs.h> 33 __KERNEL_RCSID(0, "$NetBSD: if_igc.c,v 1.5 2023/10/11 18:19:43 riastradh Exp $"); 34 35 #ifdef _KERNEL_OPT 36 #include "opt_net_mpsafe.h" 37 #include "opt_if_igc.h" 38 #if 0 /* notyet */ 39 #include "vlan.h" 40 #endif 41 #endif 42 43 #include <sys/param.h> 44 #include <sys/systm.h> 45 #include <sys/bus.h> 46 #include <sys/cpu.h> 47 #include <sys/device.h> 48 #include <sys/endian.h> 49 #include <sys/intr.h> 50 #include <sys/interrupt.h> 51 #include <sys/kernel.h> 52 #include <sys/kmem.h> 53 #include <sys/mbuf.h> 54 #include <sys/mutex.h> 55 #include <sys/socket.h> 56 #include <sys/workqueue.h> 57 #include <sys/xcall.h> 58 59 #include <net/bpf.h> 60 #include <net/if.h> 61 #include <net/if_ether.h> 62 #include <net/if_media.h> 63 #include <net/if_vlanvar.h> 64 #include <net/rss_config.h> 65 66 #include <netinet/in.h> 67 #include <netinet/ip.h> 68 #include <netinet/ip6.h> 69 #include <netinet/tcp.h> 70 71 #include <dev/pci/pcivar.h> 72 #include <dev/pci/pcireg.h> 73 #include <dev/pci/pcidevs.h> 74 75 #include <dev/pci/igc/if_igc.h> 76 #include <dev/pci/igc/igc_evcnt.h> 77 #include <dev/pci/igc/igc_hw.h> 78 #include <dev/mii/miivar.h> 79 80 #define IGC_WORKQUEUE_PRI PRI_SOFTNET 81 82 #ifndef IGC_RX_INTR_PROCESS_LIMIT_DEFAULT 83 #define IGC_RX_INTR_PROCESS_LIMIT_DEFAULT 0 84 #endif 85 #ifndef IGC_TX_INTR_PROCESS_LIMIT_DEFAULT 86 #define IGC_TX_INTR_PROCESS_LIMIT_DEFAULT 0 87 #endif 88 89 #ifndef IGC_RX_PROCESS_LIMIT_DEFAULT 90 #define IGC_RX_PROCESS_LIMIT_DEFAULT 256 91 #endif 92 #ifndef IGC_TX_PROCESS_LIMIT_DEFAULT 93 #define IGC_TX_PROCESS_LIMIT_DEFAULT 256 94 #endif 95 96 #define htolem32(p, x) (*((uint32_t *)(p)) = htole32(x)) 97 #define htolem64(p, x) (*((uint64_t *)(p)) = htole64(x)) 98 99 static const struct igc_product { 100 pci_vendor_id_t igcp_vendor; 101 pci_product_id_t igcp_product; 102 const char *igcp_name; 103 } igc_products[] = { 104 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_I225_IT, 105 "Intel(R) Ethernet Controller I225-IT(2)" }, 106 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_I226_LM, 107 "Intel(R) Ethernet Controller I226-LM" }, 108 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_I226_V, 109 "Intel(R) Ethernet Controller I226-V" }, 110 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_I226_IT, 111 "Intel(R) Ethernet Controller I226-IT" }, 112 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_I221_V, 113 "Intel(R) Ethernet Controller I221-V" }, 114 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_I226_BLANK_NVM, 115 "Intel(R) Ethernet Controller I226(blankNVM)" }, 116 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_I225_LM, 117 "Intel(R) Ethernet Controller I225-LM" }, 118 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_I225_V, 119 "Intel(R) Ethernet Controller I225-V" }, 120 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_I220_V, 121 "Intel(R) Ethernet Controller I220-V" }, 122 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_I225_I, 123 "Intel(R) Ethernet Controller I225-I" }, 124 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_I225_BLANK_NVM, 125 "Intel(R) Ethernet Controller I225(blankNVM)" }, 126 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_I225_K, 127 "Intel(R) Ethernet Controller I225-K" }, 128 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_I225_K2, 129 "Intel(R) Ethernet Controller I225-K(2)" }, 130 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_I226_K, 131 "Intel(R) Ethernet Controller I226-K" }, 132 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_I225_LMVP, 133 "Intel(R) Ethernet Controller I225-LMvP(2)" }, 134 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_I226_LMVP, 135 "Intel(R) Ethernet Controller I226-LMvP" }, 136 { 0, 0, NULL }, 137 }; 138 139 #define IGC_DF_CFG 0x1 140 #define IGC_DF_TX 0x2 141 #define IGC_DF_RX 0x4 142 #define IGC_DF_MISC 0x8 143 144 #ifdef IGC_DEBUG_FLAGS 145 int igc_debug_flags = IGC_DEBUG_FLAGS; 146 #else 147 int igc_debug_flags = 0; 148 #endif 149 150 #define DPRINTF(flag, fmt, args...) do { \ 151 if (igc_debug_flags & (IGC_DF_ ## flag)) \ 152 printf("%s: %d: " fmt, __func__, __LINE__, ##args); \ 153 } while (0) 154 155 /********************************************************************* 156 * Function Prototypes 157 *********************************************************************/ 158 static int igc_match(device_t, cfdata_t, void *); 159 static void igc_attach(device_t, device_t, void *); 160 static int igc_detach(device_t, int); 161 162 static void igc_identify_hardware(struct igc_softc *); 163 static int igc_adjust_nqueues(struct igc_softc *); 164 static int igc_allocate_pci_resources(struct igc_softc *); 165 static int igc_allocate_interrupts(struct igc_softc *); 166 static int igc_allocate_queues(struct igc_softc *); 167 static void igc_free_pci_resources(struct igc_softc *); 168 static void igc_free_interrupts(struct igc_softc *); 169 static void igc_free_queues(struct igc_softc *); 170 static void igc_reset(struct igc_softc *); 171 static void igc_init_dmac(struct igc_softc *, uint32_t); 172 static int igc_setup_interrupts(struct igc_softc *); 173 static void igc_attach_counters(struct igc_softc *sc); 174 static void igc_detach_counters(struct igc_softc *sc); 175 static void igc_update_counters(struct igc_softc *sc); 176 static void igc_clear_counters(struct igc_softc *sc); 177 static int igc_setup_msix(struct igc_softc *); 178 static int igc_setup_msi(struct igc_softc *); 179 static int igc_setup_intx(struct igc_softc *); 180 static int igc_dma_malloc(struct igc_softc *, bus_size_t, 181 struct igc_dma_alloc *); 182 static void igc_dma_free(struct igc_softc *, struct igc_dma_alloc *); 183 static void igc_setup_interface(struct igc_softc *); 184 185 static int igc_init(struct ifnet *); 186 static int igc_init_locked(struct igc_softc *); 187 static void igc_start(struct ifnet *); 188 static int igc_transmit(struct ifnet *, struct mbuf *); 189 static void igc_tx_common_locked(struct ifnet *, struct tx_ring *, int); 190 static bool igc_txeof(struct tx_ring *, u_int); 191 static void igc_intr_barrier(struct igc_softc *); 192 static void igc_stop(struct ifnet *, int); 193 static void igc_stop_locked(struct igc_softc *); 194 static int igc_ioctl(struct ifnet *, u_long, void *); 195 #ifdef IF_RXR 196 static int igc_rxrinfo(struct igc_softc *, struct if_rxrinfo *); 197 #endif 198 static void igc_rxfill(struct rx_ring *); 199 static void igc_rxrefill(struct rx_ring *, int); 200 static bool igc_rxeof(struct rx_ring *, u_int); 201 static int igc_rx_checksum(struct igc_queue *, uint64_t, uint32_t, 202 uint32_t); 203 static void igc_watchdog(struct ifnet *); 204 static void igc_tick(void *); 205 static void igc_media_status(struct ifnet *, struct ifmediareq *); 206 static int igc_media_change(struct ifnet *); 207 static int igc_ifflags_cb(struct ethercom *); 208 static void igc_set_filter(struct igc_softc *); 209 static void igc_update_link_status(struct igc_softc *); 210 static int igc_get_buf(struct rx_ring *, int, bool); 211 static int igc_tx_ctx_setup(struct tx_ring *, struct mbuf *, int, 212 uint32_t *, uint32_t *); 213 static int igc_tso_setup(struct tx_ring *, struct mbuf *, int, 214 uint32_t *, uint32_t *); 215 216 static void igc_configure_queues(struct igc_softc *); 217 static void igc_set_queues(struct igc_softc *, uint32_t, uint32_t, int); 218 static void igc_enable_queue(struct igc_softc *, uint32_t); 219 static void igc_enable_intr(struct igc_softc *); 220 static void igc_disable_intr(struct igc_softc *); 221 static int igc_intr_link(void *); 222 static int igc_intr_queue(void *); 223 static int igc_intr(void *); 224 static void igc_handle_queue(void *); 225 static void igc_handle_queue_work(struct work *, void *); 226 static void igc_sched_handle_queue(struct igc_softc *, struct igc_queue *); 227 static void igc_barrier_handle_queue(struct igc_softc *); 228 229 static int igc_allocate_transmit_buffers(struct tx_ring *); 230 static int igc_setup_transmit_structures(struct igc_softc *); 231 static int igc_setup_transmit_ring(struct tx_ring *); 232 static void igc_initialize_transmit_unit(struct igc_softc *); 233 static void igc_free_transmit_structures(struct igc_softc *); 234 static void igc_free_transmit_buffers(struct tx_ring *); 235 static void igc_withdraw_transmit_packets(struct tx_ring *, bool); 236 static int igc_allocate_receive_buffers(struct rx_ring *); 237 static int igc_setup_receive_structures(struct igc_softc *); 238 static int igc_setup_receive_ring(struct rx_ring *); 239 static void igc_initialize_receive_unit(struct igc_softc *); 240 static void igc_free_receive_structures(struct igc_softc *); 241 static void igc_free_receive_buffers(struct rx_ring *); 242 static void igc_clear_receive_status(struct rx_ring *); 243 static void igc_initialize_rss_mapping(struct igc_softc *); 244 245 static void igc_get_hw_control(struct igc_softc *); 246 static void igc_release_hw_control(struct igc_softc *); 247 static int igc_is_valid_ether_addr(uint8_t *); 248 static void igc_print_devinfo(struct igc_softc *); 249 250 CFATTACH_DECL3_NEW(igc, sizeof(struct igc_softc), 251 igc_match, igc_attach, igc_detach, NULL, NULL, NULL, 0); 252 253 static inline int 254 igc_txdesc_incr(struct igc_softc *sc, int id) 255 { 256 257 if (++id == sc->num_tx_desc) 258 id = 0; 259 return id; 260 } 261 262 static inline int __unused 263 igc_txdesc_decr(struct igc_softc *sc, int id) 264 { 265 266 if (--id < 0) 267 id = sc->num_tx_desc - 1; 268 return id; 269 } 270 271 static inline void 272 igc_txdesc_sync(struct tx_ring *txr, int id, int ops) 273 { 274 275 bus_dmamap_sync(txr->txdma.dma_tag, txr->txdma.dma_map, 276 id * sizeof(union igc_adv_tx_desc), sizeof(union igc_adv_tx_desc), 277 ops); 278 } 279 280 static inline int 281 igc_rxdesc_incr(struct igc_softc *sc, int id) 282 { 283 284 if (++id == sc->num_rx_desc) 285 id = 0; 286 return id; 287 } 288 289 static inline int 290 igc_rxdesc_decr(struct igc_softc *sc, int id) 291 { 292 293 if (--id < 0) 294 id = sc->num_rx_desc - 1; 295 return id; 296 } 297 298 static inline void 299 igc_rxdesc_sync(struct rx_ring *rxr, int id, int ops) 300 { 301 302 bus_dmamap_sync(rxr->rxdma.dma_tag, rxr->rxdma.dma_map, 303 id * sizeof(union igc_adv_rx_desc), sizeof(union igc_adv_rx_desc), 304 ops); 305 } 306 307 static const struct igc_product * 308 igc_lookup(const struct pci_attach_args *pa) 309 { 310 const struct igc_product *igcp; 311 312 for (igcp = igc_products; igcp->igcp_name != NULL; igcp++) { 313 if (PCI_VENDOR(pa->pa_id) == igcp->igcp_vendor && 314 PCI_PRODUCT(pa->pa_id) == igcp->igcp_product) 315 return igcp; 316 } 317 return NULL; 318 } 319 320 /********************************************************************* 321 * Device identification routine 322 * 323 * igc_match determines if the driver should be loaded on 324 * adapter based on PCI vendor/device id of the adapter. 325 * 326 * return 0 on success, positive on failure 327 *********************************************************************/ 328 static int 329 igc_match(device_t parent, cfdata_t match, void *aux) 330 { 331 struct pci_attach_args *pa = aux; 332 333 if (igc_lookup(pa) != NULL) 334 return 1; 335 336 return 0; 337 } 338 339 /********************************************************************* 340 * Device initialization routine 341 * 342 * The attach entry point is called when the driver is being loaded. 343 * This routine identifies the type of hardware, allocates all resources 344 * and initializes the hardware. 345 * 346 * return 0 on success, positive on failure 347 *********************************************************************/ 348 static void 349 igc_attach(device_t parent, device_t self, void *aux) 350 { 351 struct pci_attach_args *pa = aux; 352 struct igc_softc *sc = device_private(self); 353 struct igc_hw *hw = &sc->hw; 354 355 const struct igc_product *igcp = igc_lookup(pa); 356 KASSERT(igcp != NULL); 357 pci_aprint_devinfo_fancy(pa, "Ethernet controller", igcp->igcp_name, 1); 358 359 sc->sc_dev = self; 360 callout_init(&sc->sc_tick_ch, CALLOUT_MPSAFE); 361 callout_setfunc(&sc->sc_tick_ch, igc_tick, sc); 362 sc->sc_core_stopping = false; 363 364 sc->osdep.os_sc = sc; 365 sc->osdep.os_pa = *pa; 366 #ifdef __aarch64__ 367 /* 368 * XXX PR port-arm/57643 369 * 64-bit DMA does not work at least for LX2K with 32/64GB memory. 370 * smmu(4) support may be required. 371 */ 372 sc->osdep.os_dmat = pa->pa_dmat; 373 #else 374 sc->osdep.os_dmat = pci_dma64_available(pa) ? 375 pa->pa_dmat64 : pa->pa_dmat; 376 #endif 377 378 /* Determine hardware and mac info */ 379 igc_identify_hardware(sc); 380 381 sc->num_tx_desc = IGC_DEFAULT_TXD; 382 sc->num_rx_desc = IGC_DEFAULT_RXD; 383 384 /* Setup PCI resources */ 385 if (igc_allocate_pci_resources(sc)) { 386 aprint_error_dev(sc->sc_dev, 387 "unable to allocate PCI resources\n"); 388 goto err_pci; 389 } 390 391 if (igc_allocate_interrupts(sc)) { 392 aprint_error_dev(sc->sc_dev, "unable to allocate interrupts\n"); 393 goto err_pci; 394 } 395 396 /* Allocate TX/RX queues */ 397 if (igc_allocate_queues(sc)) { 398 aprint_error_dev(sc->sc_dev, "unable to allocate queues\n"); 399 goto err_alloc_intr; 400 } 401 402 /* Do shared code initialization */ 403 if (igc_setup_init_funcs(hw, true)) { 404 aprint_error_dev(sc->sc_dev, "unable to initialize\n"); 405 goto err_alloc_intr; 406 } 407 408 hw->mac.autoneg = DO_AUTO_NEG; 409 hw->phy.autoneg_wait_to_complete = false; 410 hw->phy.autoneg_advertised = AUTONEG_ADV_DEFAULT; 411 412 /* Copper options. */ 413 if (hw->phy.media_type == igc_media_type_copper) 414 hw->phy.mdix = AUTO_ALL_MODES; 415 416 /* Set the max frame size. */ 417 sc->hw.mac.max_frame_size = 9234; 418 419 /* Allocate multicast array memory. */ 420 sc->mta = kmem_alloc(IGC_MTA_LEN, KM_SLEEP); 421 422 /* Check SOL/IDER usage. */ 423 if (igc_check_reset_block(hw)) { 424 aprint_error_dev(sc->sc_dev, 425 "PHY reset is blocked due to SOL/IDER session\n"); 426 } 427 428 /* Disable Energy Efficient Ethernet. */ 429 sc->hw.dev_spec._i225.eee_disable = true; 430 431 igc_reset_hw(hw); 432 433 /* Make sure we have a good EEPROM before we read from it. */ 434 if (igc_validate_nvm_checksum(hw) < 0) { 435 /* 436 * Some PCI-E parts fail the first check due to 437 * the link being in sleep state, call it again, 438 * if it fails a second time its a real issue. 439 */ 440 if (igc_validate_nvm_checksum(hw) < 0) { 441 aprint_error_dev(sc->sc_dev, 442 "EEPROM checksum invalid\n"); 443 goto err_late; 444 } 445 } 446 447 /* Copy the permanent MAC address out of the EEPROM. */ 448 if (igc_read_mac_addr(hw) < 0) { 449 aprint_error_dev(sc->sc_dev, 450 "unable to read MAC address from EEPROM\n"); 451 goto err_late; 452 } 453 454 if (!igc_is_valid_ether_addr(hw->mac.addr)) { 455 aprint_error_dev(sc->sc_dev, "invalid MAC address\n"); 456 goto err_late; 457 } 458 459 if (igc_setup_interrupts(sc)) 460 goto err_late; 461 462 /* Attach counters. */ 463 igc_attach_counters(sc); 464 465 /* Setup OS specific network interface. */ 466 igc_setup_interface(sc); 467 468 igc_print_devinfo(sc); 469 470 igc_reset(sc); 471 hw->mac.get_link_status = true; 472 igc_update_link_status(sc); 473 474 /* The driver can now take control from firmware. */ 475 igc_get_hw_control(sc); 476 477 aprint_normal_dev(sc->sc_dev, "Ethernet address %s\n", 478 ether_sprintf(sc->hw.mac.addr)); 479 480 if (pmf_device_register(self, NULL, NULL)) 481 pmf_class_network_register(self, &sc->sc_ec.ec_if); 482 else 483 aprint_error_dev(self, "couldn't establish power handler\n"); 484 485 return; 486 487 err_late: 488 igc_release_hw_control(sc); 489 err_alloc_intr: 490 igc_free_interrupts(sc); 491 err_pci: 492 igc_free_pci_resources(sc); 493 kmem_free(sc->mta, IGC_MTA_LEN); 494 } 495 496 /********************************************************************* 497 * Device removal routine 498 * 499 * The detach entry point is called when the driver is being removed. 500 * This routine stops the adapter and deallocates all the resources 501 * that were allocated for driver operation. 502 * 503 * return 0 on success, positive on failure 504 *********************************************************************/ 505 static int 506 igc_detach(device_t self, int flags) 507 { 508 struct igc_softc *sc = device_private(self); 509 struct ifnet *ifp = &sc->sc_ec.ec_if; 510 511 mutex_enter(&sc->sc_core_lock); 512 igc_stop_locked(sc); 513 mutex_exit(&sc->sc_core_lock); 514 515 igc_detach_counters(sc); 516 517 igc_free_queues(sc); 518 519 igc_phy_hw_reset(&sc->hw); 520 igc_release_hw_control(sc); 521 522 ether_ifdetach(ifp); 523 if_detach(ifp); 524 ifmedia_fini(&sc->media); 525 526 igc_free_interrupts(sc); 527 igc_free_pci_resources(sc); 528 kmem_free(sc->mta, IGC_MTA_LEN); 529 530 mutex_destroy(&sc->sc_core_lock); 531 532 return 0; 533 } 534 535 static void 536 igc_identify_hardware(struct igc_softc *sc) 537 { 538 struct igc_osdep *os = &sc->osdep; 539 struct pci_attach_args *pa = &os->os_pa; 540 541 /* Save off the information about this board. */ 542 sc->hw.device_id = PCI_PRODUCT(pa->pa_id); 543 544 /* Do shared code init and setup. */ 545 if (igc_set_mac_type(&sc->hw)) { 546 aprint_error_dev(sc->sc_dev, "unable to identify hardware\n"); 547 return; 548 } 549 } 550 551 static int 552 igc_allocate_pci_resources(struct igc_softc *sc) 553 { 554 struct igc_osdep *os = &sc->osdep; 555 struct pci_attach_args *pa = &os->os_pa; 556 557 /* 558 * Enable bus mastering and memory-mapped I/O for sure. 559 */ 560 pcireg_t csr = 561 pci_conf_read(pa->pa_pc, pa->pa_tag, PCI_COMMAND_STATUS_REG); 562 csr |= PCI_COMMAND_MASTER_ENABLE | PCI_COMMAND_MEM_ENABLE; 563 pci_conf_write(pa->pa_pc, pa->pa_tag, PCI_COMMAND_STATUS_REG, csr); 564 565 const pcireg_t memtype = 566 pci_mapreg_type(pa->pa_pc, pa->pa_tag, IGC_PCIREG); 567 if (pci_mapreg_map(pa, IGC_PCIREG, memtype, 0, &os->os_memt, 568 &os->os_memh, &os->os_membase, &os->os_memsize)) { 569 aprint_error_dev(sc->sc_dev, "unable to map registers\n"); 570 return ENXIO; 571 } 572 573 sc->hw.hw_addr = os->os_membase; 574 sc->hw.back = os; 575 576 return 0; 577 } 578 579 static int __unused 580 igc_adjust_nqueues(struct igc_softc *sc) 581 { 582 struct pci_attach_args *pa = &sc->osdep.os_pa; 583 int nqueues = MIN(IGC_MAX_NQUEUES, ncpu); 584 585 const int nmsix = pci_msix_count(pa->pa_pc, pa->pa_tag); 586 if (nmsix <= 1) 587 nqueues = 1; 588 else if (nmsix < nqueues + 1) 589 nqueues = nmsix - 1; 590 591 return nqueues; 592 } 593 594 static int 595 igc_allocate_interrupts(struct igc_softc *sc) 596 { 597 struct pci_attach_args *pa = &sc->osdep.os_pa; 598 int error; 599 600 #ifndef IGC_DISABLE_MSIX 601 const int nqueues = igc_adjust_nqueues(sc); 602 if (nqueues > 1) { 603 sc->sc_nintrs = nqueues + 1; 604 error = pci_msix_alloc_exact(pa, &sc->sc_intrs, sc->sc_nintrs); 605 if (!error) { 606 sc->sc_nqueues = nqueues; 607 sc->sc_intr_type = PCI_INTR_TYPE_MSIX; 608 return 0; 609 } 610 } 611 #endif 612 613 /* fallback to MSI */ 614 sc->sc_nintrs = sc->sc_nqueues = 1; 615 616 #ifndef IGC_DISABLE_MSI 617 error = pci_msi_alloc_exact(pa, &sc->sc_intrs, sc->sc_nintrs); 618 if (!error) { 619 sc->sc_intr_type = PCI_INTR_TYPE_MSI; 620 return 0; 621 } 622 #endif 623 624 /* fallback to INTx */ 625 626 error = pci_intx_alloc(pa, &sc->sc_intrs); 627 if (!error) { 628 sc->sc_intr_type = PCI_INTR_TYPE_INTX; 629 return 0; 630 } 631 632 return error; 633 } 634 635 static int 636 igc_allocate_queues(struct igc_softc *sc) 637 { 638 device_t dev = sc->sc_dev; 639 int rxconf = 0, txconf = 0; 640 641 /* Allocate the top level queue structs. */ 642 sc->queues = 643 kmem_zalloc(sc->sc_nqueues * sizeof(struct igc_queue), KM_SLEEP); 644 645 /* Allocate the TX ring. */ 646 sc->tx_rings = 647 kmem_zalloc(sc->sc_nqueues * sizeof(struct tx_ring), KM_SLEEP); 648 649 /* Allocate the RX ring. */ 650 sc->rx_rings = 651 kmem_zalloc(sc->sc_nqueues * sizeof(struct rx_ring), KM_SLEEP); 652 653 /* Set up the TX queues. */ 654 for (int iq = 0; iq < sc->sc_nqueues; iq++, txconf++) { 655 struct tx_ring *txr = &sc->tx_rings[iq]; 656 const int tsize = roundup2( 657 sc->num_tx_desc * sizeof(union igc_adv_tx_desc), 658 IGC_DBA_ALIGN); 659 660 txr->sc = sc; 661 txr->txr_igcq = &sc->queues[iq]; 662 txr->me = iq; 663 if (igc_dma_malloc(sc, tsize, &txr->txdma)) { 664 aprint_error_dev(dev, 665 "unable to allocate TX descriptor\n"); 666 goto fail; 667 } 668 txr->tx_base = (union igc_adv_tx_desc *)txr->txdma.dma_vaddr; 669 memset(txr->tx_base, 0, tsize); 670 } 671 672 /* Prepare transmit descriptors and buffers. */ 673 if (igc_setup_transmit_structures(sc)) { 674 aprint_error_dev(dev, "unable to setup transmit structures\n"); 675 goto fail; 676 } 677 678 /* Set up the RX queues. */ 679 for (int iq = 0; iq < sc->sc_nqueues; iq++, rxconf++) { 680 struct rx_ring *rxr = &sc->rx_rings[iq]; 681 const int rsize = roundup2( 682 sc->num_rx_desc * sizeof(union igc_adv_rx_desc), 683 IGC_DBA_ALIGN); 684 685 rxr->sc = sc; 686 rxr->rxr_igcq = &sc->queues[iq]; 687 rxr->me = iq; 688 #ifdef OPENBSD 689 timeout_set(&rxr->rx_refill, igc_rxrefill, rxr); 690 #endif 691 if (igc_dma_malloc(sc, rsize, &rxr->rxdma)) { 692 aprint_error_dev(dev, 693 "unable to allocate RX descriptor\n"); 694 goto fail; 695 } 696 rxr->rx_base = (union igc_adv_rx_desc *)rxr->rxdma.dma_vaddr; 697 memset(rxr->rx_base, 0, rsize); 698 } 699 700 sc->rx_mbuf_sz = MCLBYTES; 701 /* Prepare receive descriptors and buffers. */ 702 if (igc_setup_receive_structures(sc)) { 703 aprint_error_dev(sc->sc_dev, 704 "unable to setup receive structures\n"); 705 goto fail; 706 } 707 708 /* Set up the queue holding structs. */ 709 for (int iq = 0; iq < sc->sc_nqueues; iq++) { 710 struct igc_queue *q = &sc->queues[iq]; 711 712 q->sc = sc; 713 q->txr = &sc->tx_rings[iq]; 714 q->rxr = &sc->rx_rings[iq]; 715 } 716 717 return 0; 718 719 fail: 720 for (struct rx_ring *rxr = sc->rx_rings; rxconf > 0; rxr++, rxconf--) 721 igc_dma_free(sc, &rxr->rxdma); 722 for (struct tx_ring *txr = sc->tx_rings; txconf > 0; txr++, txconf--) 723 igc_dma_free(sc, &txr->txdma); 724 725 kmem_free(sc->rx_rings, sc->sc_nqueues * sizeof(struct rx_ring)); 726 sc->rx_rings = NULL; 727 kmem_free(sc->tx_rings, sc->sc_nqueues * sizeof(struct tx_ring)); 728 sc->tx_rings = NULL; 729 kmem_free(sc->queues, sc->sc_nqueues * sizeof(struct igc_queue)); 730 sc->queues = NULL; 731 732 return ENOMEM; 733 } 734 735 static void 736 igc_free_pci_resources(struct igc_softc *sc) 737 { 738 struct igc_osdep *os = &sc->osdep; 739 740 if (os->os_membase != 0) 741 bus_space_unmap(os->os_memt, os->os_memh, os->os_memsize); 742 os->os_membase = 0; 743 } 744 745 static void 746 igc_free_interrupts(struct igc_softc *sc) 747 { 748 struct pci_attach_args *pa = &sc->osdep.os_pa; 749 pci_chipset_tag_t pc = pa->pa_pc; 750 751 for (int i = 0; i < sc->sc_nintrs; i++) { 752 if (sc->sc_ihs[i] != NULL) { 753 pci_intr_disestablish(pc, sc->sc_ihs[i]); 754 sc->sc_ihs[i] = NULL; 755 } 756 } 757 pci_intr_release(pc, sc->sc_intrs, sc->sc_nintrs); 758 } 759 760 static void 761 igc_free_queues(struct igc_softc *sc) 762 { 763 764 igc_free_receive_structures(sc); 765 for (int iq = 0; iq < sc->sc_nqueues; iq++) { 766 struct rx_ring *rxr = &sc->rx_rings[iq]; 767 768 igc_dma_free(sc, &rxr->rxdma); 769 } 770 771 igc_free_transmit_structures(sc); 772 for (int iq = 0; iq < sc->sc_nqueues; iq++) { 773 struct tx_ring *txr = &sc->tx_rings[iq]; 774 775 igc_dma_free(sc, &txr->txdma); 776 } 777 778 kmem_free(sc->rx_rings, sc->sc_nqueues * sizeof(struct rx_ring)); 779 kmem_free(sc->tx_rings, sc->sc_nqueues * sizeof(struct tx_ring)); 780 kmem_free(sc->queues, sc->sc_nqueues * sizeof(struct igc_queue)); 781 } 782 783 /********************************************************************* 784 * 785 * Initialize the hardware to a configuration as specified by the 786 * adapter structure. 787 * 788 **********************************************************************/ 789 static void 790 igc_reset(struct igc_softc *sc) 791 { 792 struct igc_hw *hw = &sc->hw; 793 794 /* Let the firmware know the OS is in control */ 795 igc_get_hw_control(sc); 796 797 /* 798 * Packet Buffer Allocation (PBA) 799 * Writing PBA sets the receive portion of the buffer 800 * the remainder is used for the transmit buffer. 801 */ 802 const uint32_t pba = IGC_PBA_34K; 803 804 /* 805 * These parameters control the automatic generation (Tx) and 806 * response (Rx) to Ethernet PAUSE frames. 807 * - High water mark should allow for at least two frames to be 808 * received after sending an XOFF. 809 * - Low water mark works best when it is very near the high water mark. 810 * This allows the receiver to restart by sending XON when it has 811 * drained a bit. Here we use an arbitrary value of 1500 which will 812 * restart after one full frame is pulled from the buffer. There 813 * could be several smaller frames in the buffer and if so they will 814 * not trigger the XON until their total number reduces the buffer 815 * by 1500. 816 * - The pause time is fairly large at 1000 x 512ns = 512 usec. 817 */ 818 const uint16_t rx_buffer_size = (pba & 0xffff) << 10; 819 820 hw->fc.high_water = rx_buffer_size - 821 roundup2(sc->hw.mac.max_frame_size, 1024); 822 /* 16-byte granularity */ 823 hw->fc.low_water = hw->fc.high_water - 16; 824 825 if (sc->fc) /* locally set flow control value? */ 826 hw->fc.requested_mode = sc->fc; 827 else 828 hw->fc.requested_mode = igc_fc_full; 829 830 hw->fc.pause_time = IGC_FC_PAUSE_TIME; 831 832 hw->fc.send_xon = true; 833 834 /* Issue a global reset */ 835 igc_reset_hw(hw); 836 IGC_WRITE_REG(hw, IGC_WUC, 0); 837 838 /* and a re-init */ 839 if (igc_init_hw(hw) < 0) { 840 aprint_error_dev(sc->sc_dev, "unable to reset hardware\n"); 841 return; 842 } 843 844 /* Setup DMA Coalescing */ 845 igc_init_dmac(sc, pba); 846 847 IGC_WRITE_REG(hw, IGC_VET, ETHERTYPE_VLAN); 848 igc_get_phy_info(hw); 849 igc_check_for_link(hw); 850 } 851 852 /********************************************************************* 853 * 854 * Initialize the DMA Coalescing feature 855 * 856 **********************************************************************/ 857 static void 858 igc_init_dmac(struct igc_softc *sc, uint32_t pba) 859 { 860 struct igc_hw *hw = &sc->hw; 861 const uint16_t max_frame_size = sc->hw.mac.max_frame_size; 862 uint32_t reg, status; 863 864 if (sc->dmac == 0) { /* Disabling it */ 865 reg = ~IGC_DMACR_DMAC_EN; /* XXXRO */ 866 IGC_WRITE_REG(hw, IGC_DMACR, reg); 867 DPRINTF(MISC, "DMA coalescing disabled\n"); 868 return; 869 } else { 870 device_printf(sc->sc_dev, "DMA coalescing enabled\n"); 871 } 872 873 /* Set starting threshold */ 874 IGC_WRITE_REG(hw, IGC_DMCTXTH, 0); 875 876 uint16_t hwm = 64 * pba - max_frame_size / 16; 877 if (hwm < 64 * (pba - 6)) 878 hwm = 64 * (pba - 6); 879 reg = IGC_READ_REG(hw, IGC_FCRTC); 880 reg &= ~IGC_FCRTC_RTH_COAL_MASK; 881 reg |= (hwm << IGC_FCRTC_RTH_COAL_SHIFT) & IGC_FCRTC_RTH_COAL_MASK; 882 IGC_WRITE_REG(hw, IGC_FCRTC, reg); 883 884 uint32_t dmac = pba - max_frame_size / 512; 885 if (dmac < pba - 10) 886 dmac = pba - 10; 887 reg = IGC_READ_REG(hw, IGC_DMACR); 888 reg &= ~IGC_DMACR_DMACTHR_MASK; 889 reg |= (dmac << IGC_DMACR_DMACTHR_SHIFT) & IGC_DMACR_DMACTHR_MASK; 890 891 /* transition to L0x or L1 if available..*/ 892 reg |= IGC_DMACR_DMAC_EN | IGC_DMACR_DMAC_LX_MASK; 893 894 /* Check if status is 2.5Gb backplane connection 895 * before configuration of watchdog timer, which is 896 * in msec values in 12.8usec intervals 897 * watchdog timer= msec values in 32usec intervals 898 * for non 2.5Gb connection 899 */ 900 status = IGC_READ_REG(hw, IGC_STATUS); 901 if ((status & IGC_STATUS_2P5_SKU) && 902 !(status & IGC_STATUS_2P5_SKU_OVER)) 903 reg |= (sc->dmac * 5) >> 6; 904 else 905 reg |= sc->dmac >> 5; 906 907 IGC_WRITE_REG(hw, IGC_DMACR, reg); 908 909 IGC_WRITE_REG(hw, IGC_DMCRTRH, 0); 910 911 /* Set the interval before transition */ 912 reg = IGC_READ_REG(hw, IGC_DMCTLX); 913 reg |= IGC_DMCTLX_DCFLUSH_DIS; 914 915 /* 916 * in 2.5Gb connection, TTLX unit is 0.4 usec 917 * which is 0x4*2 = 0xA. But delay is still 4 usec 918 */ 919 status = IGC_READ_REG(hw, IGC_STATUS); 920 if ((status & IGC_STATUS_2P5_SKU) && 921 !(status & IGC_STATUS_2P5_SKU_OVER)) 922 reg |= 0xA; 923 else 924 reg |= 0x4; 925 926 IGC_WRITE_REG(hw, IGC_DMCTLX, reg); 927 928 /* free space in tx packet buffer to wake from DMA coal */ 929 IGC_WRITE_REG(hw, IGC_DMCTXTH, 930 (IGC_TXPBSIZE - (2 * max_frame_size)) >> 6); 931 932 /* make low power state decision controlled by DMA coal */ 933 reg = IGC_READ_REG(hw, IGC_PCIEMISC); 934 reg &= ~IGC_PCIEMISC_LX_DECISION; 935 IGC_WRITE_REG(hw, IGC_PCIEMISC, reg); 936 } 937 938 static int 939 igc_setup_interrupts(struct igc_softc *sc) 940 { 941 int error; 942 943 switch (sc->sc_intr_type) { 944 case PCI_INTR_TYPE_MSIX: 945 error = igc_setup_msix(sc); 946 break; 947 case PCI_INTR_TYPE_MSI: 948 error = igc_setup_msi(sc); 949 break; 950 case PCI_INTR_TYPE_INTX: 951 error = igc_setup_intx(sc); 952 break; 953 default: 954 panic("%s: invalid interrupt type: %d", 955 device_xname(sc->sc_dev), sc->sc_intr_type); 956 } 957 958 return error; 959 } 960 961 static void 962 igc_attach_counters(struct igc_softc *sc) 963 { 964 #ifdef IGC_EVENT_COUNTERS 965 966 /* Global counters */ 967 sc->sc_global_evcnts = kmem_zalloc( 968 IGC_GLOBAL_COUNTERS * sizeof(sc->sc_global_evcnts[0]), KM_SLEEP); 969 970 for (int cnt = 0; cnt < IGC_GLOBAL_COUNTERS; cnt++) { 971 evcnt_attach_dynamic(&sc->sc_global_evcnts[cnt], 972 igc_global_counters[cnt].type, NULL, 973 device_xname(sc->sc_dev), igc_global_counters[cnt].name); 974 } 975 976 /* Driver counters */ 977 sc->sc_driver_evcnts = kmem_zalloc( 978 IGC_DRIVER_COUNTERS * sizeof(sc->sc_driver_evcnts[0]), KM_SLEEP); 979 980 for (int cnt = 0; cnt < IGC_DRIVER_COUNTERS; cnt++) { 981 evcnt_attach_dynamic(&sc->sc_driver_evcnts[cnt], 982 igc_driver_counters[cnt].type, NULL, 983 device_xname(sc->sc_dev), igc_driver_counters[cnt].name); 984 } 985 986 for (int iq = 0; iq < sc->sc_nqueues; iq++) { 987 struct igc_queue *q = &sc->queues[iq]; 988 989 q->igcq_driver_counters = kmem_zalloc( 990 IGC_DRIVER_COUNTERS * sizeof(q->igcq_driver_counters[0]), 991 KM_SLEEP); 992 } 993 994 /* Queue counters */ 995 for (int iq = 0; iq < sc->sc_nqueues; iq++) { 996 struct igc_queue *q = &sc->queues[iq]; 997 998 snprintf(q->igcq_queue_evname, sizeof(q->igcq_queue_evname), 999 "%s q%d", device_xname(sc->sc_dev), iq); 1000 1001 q->igcq_queue_evcnts = kmem_zalloc( 1002 IGC_QUEUE_COUNTERS * sizeof(q->igcq_queue_evcnts[0]), 1003 KM_SLEEP); 1004 1005 for (int cnt = 0; cnt < IGC_QUEUE_COUNTERS; cnt++) { 1006 evcnt_attach_dynamic(&q->igcq_queue_evcnts[cnt], 1007 igc_queue_counters[cnt].type, NULL, 1008 q->igcq_queue_evname, igc_queue_counters[cnt].name); 1009 } 1010 } 1011 1012 /* MAC counters */ 1013 snprintf(sc->sc_mac_evname, sizeof(sc->sc_mac_evname), 1014 "%s Mac Statistics", device_xname(sc->sc_dev)); 1015 1016 sc->sc_mac_evcnts = kmem_zalloc( 1017 IGC_MAC_COUNTERS * sizeof(sc->sc_mac_evcnts[0]), KM_SLEEP); 1018 1019 for (int cnt = 0; cnt < IGC_MAC_COUNTERS; cnt++) { 1020 evcnt_attach_dynamic(&sc->sc_mac_evcnts[cnt], EVCNT_TYPE_MISC, 1021 NULL, sc->sc_mac_evname, igc_mac_counters[cnt].name); 1022 } 1023 #endif 1024 } 1025 1026 static void 1027 igc_detach_counters(struct igc_softc *sc) 1028 { 1029 #ifdef IGC_EVENT_COUNTERS 1030 1031 /* Global counters */ 1032 for (int cnt = 0; cnt < IGC_GLOBAL_COUNTERS; cnt++) 1033 evcnt_detach(&sc->sc_global_evcnts[cnt]); 1034 1035 kmem_free(sc->sc_global_evcnts, 1036 IGC_GLOBAL_COUNTERS * sizeof(sc->sc_global_evcnts)); 1037 1038 /* Driver counters */ 1039 for (int iq = 0; iq < sc->sc_nqueues; iq++) { 1040 struct igc_queue *q = &sc->queues[iq]; 1041 1042 kmem_free(q->igcq_driver_counters, 1043 IGC_DRIVER_COUNTERS * sizeof(q->igcq_driver_counters[0])); 1044 } 1045 1046 for (int cnt = 0; cnt < IGC_DRIVER_COUNTERS; cnt++) 1047 evcnt_detach(&sc->sc_driver_evcnts[cnt]); 1048 1049 kmem_free(sc->sc_driver_evcnts, 1050 IGC_DRIVER_COUNTERS * sizeof(sc->sc_driver_evcnts)); 1051 1052 /* Queue counters */ 1053 for (int iq = 0; iq < sc->sc_nqueues; iq++) { 1054 struct igc_queue *q = &sc->queues[iq]; 1055 1056 for (int cnt = 0; cnt < IGC_QUEUE_COUNTERS; cnt++) 1057 evcnt_detach(&q->igcq_queue_evcnts[cnt]); 1058 1059 kmem_free(q->igcq_queue_evcnts, 1060 IGC_QUEUE_COUNTERS * sizeof(q->igcq_queue_evcnts[0])); 1061 } 1062 1063 /* MAC statistics */ 1064 for (int cnt = 0; cnt < IGC_MAC_COUNTERS; cnt++) 1065 evcnt_detach(&sc->sc_mac_evcnts[cnt]); 1066 1067 kmem_free(sc->sc_mac_evcnts, 1068 IGC_MAC_COUNTERS * sizeof(sc->sc_mac_evcnts[0])); 1069 #endif 1070 } 1071 1072 /* 1073 * XXX 1074 * FreeBSD uses 4-byte-wise read for 64-bit counters, while Linux just 1075 * drops hi words. 1076 */ 1077 static inline uint64_t __unused 1078 igc_read_mac_counter(struct igc_hw *hw, bus_size_t reg, bool is64) 1079 { 1080 uint64_t val; 1081 1082 val = IGC_READ_REG(hw, reg); 1083 if (is64) 1084 val += ((uint64_t)IGC_READ_REG(hw, reg + 4)) << 32; 1085 return val; 1086 } 1087 1088 static void 1089 igc_update_counters(struct igc_softc *sc) 1090 { 1091 #ifdef IGC_EVENT_COUNTERS 1092 1093 /* Global counters: nop */ 1094 1095 /* Driver counters */ 1096 uint64_t sum[IGC_DRIVER_COUNTERS]; 1097 1098 memset(sum, 0, sizeof(sum)); 1099 for (int iq = 0; iq < sc->sc_nqueues; iq++) { 1100 struct igc_queue *q = &sc->queues[iq]; 1101 1102 for (int cnt = 0; cnt < IGC_DRIVER_COUNTERS; cnt++) { 1103 sum[cnt] += IGC_QUEUE_DRIVER_COUNTER_VAL(q, cnt); 1104 IGC_QUEUE_DRIVER_COUNTER_STORE(q, cnt, 0); 1105 } 1106 } 1107 1108 for (int cnt = 0; cnt < IGC_DRIVER_COUNTERS; cnt++) 1109 IGC_DRIVER_COUNTER_ADD(sc, cnt, sum[cnt]); 1110 1111 /* Queue counters: nop */ 1112 1113 /* Mac statistics */ 1114 struct igc_hw *hw = &sc->hw; 1115 1116 for (int cnt = 0; cnt < IGC_MAC_COUNTERS; cnt++) { 1117 IGC_MAC_COUNTER_ADD(sc, cnt, igc_read_mac_counter(hw, 1118 igc_mac_counters[cnt].reg, igc_mac_counters[cnt].is64)); 1119 } 1120 #endif 1121 } 1122 1123 static void 1124 igc_clear_counters(struct igc_softc *sc) 1125 { 1126 #ifdef IGC_EVENT_COUNTERS 1127 1128 /* Global counters */ 1129 for (int cnt = 0; cnt < IGC_GLOBAL_COUNTERS; cnt++) 1130 IGC_GLOBAL_COUNTER_STORE(sc, cnt, 0); 1131 1132 /* Driver counters */ 1133 for (int iq = 0; iq < sc->sc_nqueues; iq++) { 1134 struct igc_queue *q = &sc->queues[iq]; 1135 1136 for (int cnt = 0; cnt < IGC_DRIVER_COUNTERS; cnt++) 1137 IGC_QUEUE_DRIVER_COUNTER_STORE(q, cnt, 0); 1138 } 1139 1140 for (int cnt = 0; cnt < IGC_DRIVER_COUNTERS; cnt++) 1141 IGC_DRIVER_COUNTER_STORE(sc, cnt, 0); 1142 1143 /* Queue counters */ 1144 for (int iq = 0; iq < sc->sc_nqueues; iq++) { 1145 struct igc_queue *q = &sc->queues[iq]; 1146 1147 for (int cnt = 0; cnt < IGC_QUEUE_COUNTERS; cnt++) 1148 IGC_QUEUE_COUNTER_STORE(q, cnt, 0); 1149 } 1150 1151 /* Mac statistics */ 1152 struct igc_hw *hw = &sc->hw; 1153 1154 for (int cnt = 0; cnt < IGC_MAC_COUNTERS; cnt++) { 1155 (void)igc_read_mac_counter(hw, igc_mac_counters[cnt].reg, 1156 igc_mac_counters[cnt].is64); 1157 IGC_MAC_COUNTER_STORE(sc, cnt, 0); 1158 } 1159 #endif 1160 } 1161 1162 static int 1163 igc_setup_msix(struct igc_softc *sc) 1164 { 1165 pci_chipset_tag_t pc = sc->osdep.os_pa.pa_pc; 1166 device_t dev = sc->sc_dev; 1167 pci_intr_handle_t *intrs; 1168 void **ihs; 1169 const char *intrstr; 1170 char intrbuf[PCI_INTRSTR_LEN]; 1171 char xnamebuf[MAX(32, MAXCOMLEN)]; 1172 int iq, error; 1173 1174 for (iq = 0, intrs = sc->sc_intrs, ihs = sc->sc_ihs; 1175 iq < sc->sc_nqueues; iq++, intrs++, ihs++) { 1176 struct igc_queue *q = &sc->queues[iq]; 1177 1178 snprintf(xnamebuf, sizeof(xnamebuf), "%s: txrx %d", 1179 device_xname(dev), iq); 1180 1181 intrstr = pci_intr_string(pc, *intrs, intrbuf, sizeof(intrbuf)); 1182 1183 pci_intr_setattr(pc, intrs, PCI_INTR_MPSAFE, true); 1184 *ihs = pci_intr_establish_xname(pc, *intrs, IPL_NET, 1185 igc_intr_queue, q, xnamebuf); 1186 if (*ihs == NULL) { 1187 aprint_error_dev(dev, 1188 "unable to establish txrx interrupt at %s\n", 1189 intrstr); 1190 return ENOBUFS; 1191 } 1192 aprint_normal_dev(dev, "txrx interrupting at %s\n", intrstr); 1193 1194 kcpuset_t *affinity; 1195 kcpuset_create(&affinity, true); 1196 kcpuset_set(affinity, iq % ncpu); 1197 error = interrupt_distribute(*ihs, affinity, NULL); 1198 if (error) { 1199 aprint_normal_dev(dev, 1200 "%s: unable to change affinity, use default CPU\n", 1201 intrstr); 1202 } 1203 kcpuset_destroy(affinity); 1204 1205 q->igcq_si = softint_establish(SOFTINT_NET | SOFTINT_MPSAFE, 1206 igc_handle_queue, q); 1207 if (q->igcq_si == NULL) { 1208 aprint_error_dev(dev, 1209 "%s: unable to establish softint\n", intrstr); 1210 return ENOBUFS; 1211 } 1212 1213 q->msix = iq; 1214 q->eims = 1 << iq; 1215 } 1216 1217 snprintf(xnamebuf, MAXCOMLEN, "%s_tx_rx", device_xname(dev)); 1218 error = workqueue_create(&sc->sc_queue_wq, xnamebuf, 1219 igc_handle_queue_work, sc, IGC_WORKQUEUE_PRI, IPL_NET, 1220 WQ_PERCPU | WQ_MPSAFE); 1221 if (error) { 1222 aprint_error_dev(dev, "workqueue_create failed\n"); 1223 return ENOBUFS; 1224 } 1225 sc->sc_txrx_workqueue = false; 1226 1227 intrstr = pci_intr_string(pc, *intrs, intrbuf, sizeof(intrbuf)); 1228 snprintf(xnamebuf, sizeof(xnamebuf), "%s: link", device_xname(dev)); 1229 pci_intr_setattr(pc, intrs, PCI_INTR_MPSAFE, true); 1230 *ihs = pci_intr_establish_xname(pc, *intrs, IPL_NET, 1231 igc_intr_link, sc, xnamebuf); 1232 if (*ihs == NULL) { 1233 aprint_error_dev(dev, 1234 "unable to establish link interrupt at %s\n", intrstr); 1235 return ENOBUFS; 1236 } 1237 aprint_normal_dev(dev, "link interrupting at %s\n", intrstr); 1238 /* use later in igc_configure_queues() */ 1239 sc->linkvec = iq; 1240 1241 return 0; 1242 } 1243 1244 static int 1245 igc_setup_msi(struct igc_softc *sc) 1246 { 1247 pci_chipset_tag_t pc = sc->osdep.os_pa.pa_pc; 1248 device_t dev = sc->sc_dev; 1249 pci_intr_handle_t *intr = sc->sc_intrs; 1250 void **ihs = sc->sc_ihs; 1251 const char *intrstr; 1252 char intrbuf[PCI_INTRSTR_LEN]; 1253 char xnamebuf[MAX(32, MAXCOMLEN)]; 1254 int error; 1255 1256 intrstr = pci_intr_string(pc, *intr, intrbuf, sizeof(intrbuf)); 1257 1258 snprintf(xnamebuf, sizeof(xnamebuf), "%s: msi", device_xname(dev)); 1259 pci_intr_setattr(pc, intr, PCI_INTR_MPSAFE, true); 1260 *ihs = pci_intr_establish_xname(pc, *intr, IPL_NET, 1261 igc_intr, sc, xnamebuf); 1262 if (*ihs == NULL) { 1263 aprint_error_dev(dev, 1264 "unable to establish interrupt at %s\n", intrstr); 1265 return ENOBUFS; 1266 } 1267 aprint_normal_dev(dev, "interrupting at %s\n", intrstr); 1268 1269 struct igc_queue *iq = sc->queues; 1270 iq->igcq_si = softint_establish(SOFTINT_NET | SOFTINT_MPSAFE, 1271 igc_handle_queue, iq); 1272 if (iq->igcq_si == NULL) { 1273 aprint_error_dev(dev, 1274 "%s: unable to establish softint\n", intrstr); 1275 return ENOBUFS; 1276 } 1277 1278 snprintf(xnamebuf, MAXCOMLEN, "%s_tx_rx", device_xname(dev)); 1279 error = workqueue_create(&sc->sc_queue_wq, xnamebuf, 1280 igc_handle_queue_work, sc, IGC_WORKQUEUE_PRI, IPL_NET, 1281 WQ_PERCPU | WQ_MPSAFE); 1282 if (error) { 1283 aprint_error_dev(dev, "workqueue_create failed\n"); 1284 return ENOBUFS; 1285 } 1286 sc->sc_txrx_workqueue = false; 1287 1288 sc->queues[0].msix = 0; 1289 sc->linkvec = 0; 1290 1291 return 0; 1292 } 1293 1294 static int 1295 igc_setup_intx(struct igc_softc *sc) 1296 { 1297 pci_chipset_tag_t pc = sc->osdep.os_pa.pa_pc; 1298 device_t dev = sc->sc_dev; 1299 pci_intr_handle_t *intr = sc->sc_intrs; 1300 void **ihs = sc->sc_ihs; 1301 const char *intrstr; 1302 char intrbuf[PCI_INTRSTR_LEN]; 1303 char xnamebuf[32]; 1304 1305 intrstr = pci_intr_string(pc, *intr, intrbuf, sizeof(intrbuf)); 1306 1307 snprintf(xnamebuf, sizeof(xnamebuf), "%s:intx", device_xname(dev)); 1308 pci_intr_setattr(pc, intr, PCI_INTR_MPSAFE, true); 1309 *ihs = pci_intr_establish_xname(pc, *intr, IPL_NET, 1310 igc_intr, sc, xnamebuf); 1311 if (*ihs == NULL) { 1312 aprint_error_dev(dev, 1313 "unable to establish interrupt at %s\n", intrstr); 1314 return ENOBUFS; 1315 } 1316 aprint_normal_dev(dev, "interrupting at %s\n", intrstr); 1317 1318 struct igc_queue *iq = sc->queues; 1319 iq->igcq_si = softint_establish(SOFTINT_NET | SOFTINT_MPSAFE, 1320 igc_handle_queue, iq); 1321 if (iq->igcq_si == NULL) { 1322 aprint_error_dev(dev, 1323 "%s: unable to establish softint\n", intrstr); 1324 return ENOBUFS; 1325 } 1326 1327 /* create workqueue? */ 1328 sc->sc_txrx_workqueue = false; 1329 1330 sc->queues[0].msix = 0; 1331 sc->linkvec = 0; 1332 1333 return 0; 1334 } 1335 1336 static int 1337 igc_dma_malloc(struct igc_softc *sc, bus_size_t size, struct igc_dma_alloc *dma) 1338 { 1339 struct igc_osdep *os = &sc->osdep; 1340 1341 dma->dma_tag = os->os_dmat; 1342 1343 if (bus_dmamap_create(dma->dma_tag, size, 1, size, 0, 1344 BUS_DMA_WAITOK | BUS_DMA_ALLOCNOW, &dma->dma_map)) 1345 return 1; 1346 if (bus_dmamem_alloc(dma->dma_tag, size, PAGE_SIZE, 0, &dma->dma_seg, 1347 1, &dma->dma_nseg, BUS_DMA_WAITOK)) 1348 goto destroy; 1349 /* 1350 * XXXRO 1351 * 1352 * Coherent mapping for descriptors is required for now. 1353 * 1354 * Both TX and RX descriptors are 16-byte length, which is shorter 1355 * than dcache lines on modern CPUs. Therefore, sync for a descriptor 1356 * may overwrite DMA read for descriptors in the same cache line. 1357 * 1358 * Can't we avoid this by use cache-line-aligned descriptors at once? 1359 */ 1360 if (bus_dmamem_map(dma->dma_tag, &dma->dma_seg, dma->dma_nseg, size, 1361 &dma->dma_vaddr, BUS_DMA_WAITOK | BUS_DMA_COHERENT /* XXXRO */)) 1362 goto free; 1363 if (bus_dmamap_load(dma->dma_tag, dma->dma_map, dma->dma_vaddr, size, 1364 NULL, BUS_DMA_WAITOK)) 1365 goto unmap; 1366 1367 dma->dma_size = size; 1368 1369 return 0; 1370 unmap: 1371 bus_dmamem_unmap(dma->dma_tag, dma->dma_vaddr, size); 1372 free: 1373 bus_dmamem_free(dma->dma_tag, &dma->dma_seg, dma->dma_nseg); 1374 destroy: 1375 bus_dmamap_destroy(dma->dma_tag, dma->dma_map); 1376 dma->dma_map = NULL; 1377 dma->dma_tag = NULL; 1378 return 1; 1379 } 1380 1381 static void 1382 igc_dma_free(struct igc_softc *sc, struct igc_dma_alloc *dma) 1383 { 1384 1385 if (dma->dma_tag == NULL) 1386 return; 1387 1388 if (dma->dma_map != NULL) { 1389 bus_dmamap_sync(dma->dma_tag, dma->dma_map, 0, 1390 dma->dma_map->dm_mapsize, 1391 BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE); 1392 bus_dmamap_unload(dma->dma_tag, dma->dma_map); 1393 bus_dmamem_unmap(dma->dma_tag, dma->dma_vaddr, dma->dma_size); 1394 bus_dmamem_free(dma->dma_tag, &dma->dma_seg, dma->dma_nseg); 1395 bus_dmamap_destroy(dma->dma_tag, dma->dma_map); 1396 dma->dma_map = NULL; 1397 } 1398 } 1399 1400 /********************************************************************* 1401 * 1402 * Setup networking device structure and register an interface. 1403 * 1404 **********************************************************************/ 1405 static void 1406 igc_setup_interface(struct igc_softc *sc) 1407 { 1408 struct ifnet *ifp = &sc->sc_ec.ec_if; 1409 1410 strlcpy(ifp->if_xname, device_xname(sc->sc_dev), sizeof(ifp->if_xname)); 1411 ifp->if_softc = sc; 1412 ifp->if_flags = IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST; 1413 ifp->if_extflags = IFEF_MPSAFE; 1414 ifp->if_ioctl = igc_ioctl; 1415 ifp->if_start = igc_start; 1416 if (sc->sc_nqueues > 1) 1417 ifp->if_transmit = igc_transmit; 1418 ifp->if_watchdog = igc_watchdog; 1419 ifp->if_init = igc_init; 1420 ifp->if_stop = igc_stop; 1421 1422 #if 0 /* notyet */ 1423 ifp->if_capabilities = IFCAP_TSOv4 | IFCAP_TSOv6; 1424 #endif 1425 1426 ifp->if_capabilities |= 1427 IFCAP_CSUM_IPv4_Tx | IFCAP_CSUM_IPv4_Rx | 1428 IFCAP_CSUM_TCPv4_Tx | IFCAP_CSUM_TCPv4_Rx | 1429 IFCAP_CSUM_UDPv4_Tx | IFCAP_CSUM_UDPv4_Rx | 1430 IFCAP_CSUM_TCPv6_Tx | IFCAP_CSUM_TCPv6_Rx | 1431 IFCAP_CSUM_UDPv6_Tx | IFCAP_CSUM_UDPv6_Rx; 1432 1433 ifp->if_capenable = 0; 1434 1435 sc->sc_ec.ec_capabilities |= 1436 ETHERCAP_JUMBO_MTU | ETHERCAP_VLAN_MTU; 1437 1438 IFQ_SET_MAXLEN(&ifp->if_snd, sc->num_tx_desc - 1); 1439 IFQ_SET_READY(&ifp->if_snd); 1440 1441 #if NVLAN > 0 1442 sc->sc_ec.ec_capabilities |= ETHERCAP_VLAN_HWTAGGING; 1443 #endif 1444 1445 mutex_init(&sc->sc_core_lock, MUTEX_DEFAULT, IPL_NET); 1446 1447 /* Initialize ifmedia structures. */ 1448 sc->sc_ec.ec_ifmedia = &sc->media; 1449 ifmedia_init_with_lock(&sc->media, IFM_IMASK, igc_media_change, 1450 igc_media_status, &sc->sc_core_lock); 1451 ifmedia_add(&sc->media, IFM_ETHER | IFM_10_T, 0, NULL); 1452 ifmedia_add(&sc->media, IFM_ETHER | IFM_10_T | IFM_FDX, 0, NULL); 1453 ifmedia_add(&sc->media, IFM_ETHER | IFM_100_TX, 0, NULL); 1454 ifmedia_add(&sc->media, IFM_ETHER | IFM_100_TX | IFM_FDX, 0, NULL); 1455 ifmedia_add(&sc->media, IFM_ETHER | IFM_1000_T | IFM_FDX, 0, NULL); 1456 ifmedia_add(&sc->media, IFM_ETHER | IFM_2500_T | IFM_FDX, 0, NULL); 1457 ifmedia_add(&sc->media, IFM_ETHER | IFM_AUTO, 0, NULL); 1458 ifmedia_set(&sc->media, IFM_ETHER | IFM_AUTO); 1459 1460 sc->sc_rx_intr_process_limit = IGC_RX_INTR_PROCESS_LIMIT_DEFAULT; 1461 sc->sc_tx_intr_process_limit = IGC_TX_INTR_PROCESS_LIMIT_DEFAULT; 1462 sc->sc_rx_process_limit = IGC_RX_PROCESS_LIMIT_DEFAULT; 1463 sc->sc_tx_process_limit = IGC_TX_PROCESS_LIMIT_DEFAULT; 1464 1465 if_initialize(ifp); 1466 sc->sc_ipq = if_percpuq_create(ifp); 1467 if_deferred_start_init(ifp, NULL); 1468 ether_ifattach(ifp, sc->hw.mac.addr); 1469 ether_set_ifflags_cb(&sc->sc_ec, igc_ifflags_cb); 1470 if_register(ifp); 1471 } 1472 1473 static int 1474 igc_init(struct ifnet *ifp) 1475 { 1476 struct igc_softc *sc = ifp->if_softc; 1477 int error; 1478 1479 mutex_enter(&sc->sc_core_lock); 1480 error = igc_init_locked(sc); 1481 mutex_exit(&sc->sc_core_lock); 1482 1483 return error; 1484 } 1485 1486 static int 1487 igc_init_locked(struct igc_softc *sc) 1488 { 1489 struct ethercom *ec = &sc->sc_ec; 1490 struct ifnet *ifp = &ec->ec_if; 1491 1492 DPRINTF(CFG, "called\n"); 1493 1494 KASSERT(mutex_owned(&sc->sc_core_lock)); 1495 1496 if (ISSET(ifp->if_flags, IFF_RUNNING)) 1497 igc_stop_locked(sc); 1498 1499 /* Put the address into the receive address array. */ 1500 igc_rar_set(&sc->hw, sc->hw.mac.addr, 0); 1501 1502 /* Initialize the hardware. */ 1503 igc_reset(sc); 1504 igc_update_link_status(sc); 1505 1506 /* Setup VLAN support, basic and offload if available. */ 1507 IGC_WRITE_REG(&sc->hw, IGC_VET, ETHERTYPE_VLAN); 1508 1509 igc_initialize_transmit_unit(sc); 1510 igc_initialize_receive_unit(sc); 1511 1512 if (ec->ec_capenable & ETHERCAP_VLAN_HWTAGGING) { 1513 uint32_t ctrl = IGC_READ_REG(&sc->hw, IGC_CTRL); 1514 ctrl |= IGC_CTRL_VME; 1515 IGC_WRITE_REG(&sc->hw, IGC_CTRL, ctrl); 1516 } 1517 1518 /* Setup multicast table. */ 1519 igc_set_filter(sc); 1520 1521 igc_clear_hw_cntrs_base_generic(&sc->hw); 1522 1523 if (sc->sc_intr_type == PCI_INTR_TYPE_MSIX) 1524 igc_configure_queues(sc); 1525 1526 /* This clears any pending interrupts */ 1527 IGC_READ_REG(&sc->hw, IGC_ICR); 1528 IGC_WRITE_REG(&sc->hw, IGC_ICS, IGC_ICS_LSC); 1529 1530 /* The driver can now take control from firmware. */ 1531 igc_get_hw_control(sc); 1532 1533 /* Set Energy Efficient Ethernet. */ 1534 igc_set_eee_i225(&sc->hw, true, true, true); 1535 1536 for (int iq = 0; iq < sc->sc_nqueues; iq++) { 1537 struct rx_ring *rxr = &sc->rx_rings[iq]; 1538 1539 mutex_enter(&rxr->rxr_lock); 1540 igc_rxfill(rxr); 1541 mutex_exit(&rxr->rxr_lock); 1542 } 1543 1544 igc_enable_intr(sc); 1545 1546 sc->sc_core_stopping = false; 1547 1548 callout_schedule(&sc->sc_tick_ch, hz); 1549 1550 ifp->if_flags |= IFF_RUNNING; 1551 1552 /* Save last flags for the callback */ 1553 sc->sc_if_flags = ifp->if_flags; 1554 1555 return 0; 1556 } 1557 1558 static inline int 1559 igc_load_mbuf(struct igc_queue *q, bus_dma_tag_t dmat, bus_dmamap_t map, 1560 struct mbuf *m) 1561 { 1562 int error; 1563 1564 error = bus_dmamap_load_mbuf(dmat, map, m, 1565 BUS_DMA_WRITE | BUS_DMA_NOWAIT); 1566 1567 if (__predict_false(error == EFBIG)) { 1568 IGC_DRIVER_EVENT(q, txdma_efbig, 1); 1569 m = m_defrag(m, M_NOWAIT); 1570 if (__predict_false(m == NULL)) { 1571 IGC_DRIVER_EVENT(q, txdma_defrag, 1); 1572 return ENOBUFS; 1573 } 1574 error = bus_dmamap_load_mbuf(dmat, map, m, 1575 BUS_DMA_WRITE | BUS_DMA_NOWAIT); 1576 } 1577 1578 switch (error) { 1579 case 0: 1580 break; 1581 case ENOMEM: 1582 IGC_DRIVER_EVENT(q, txdma_enomem, 1); 1583 break; 1584 case EINVAL: 1585 IGC_DRIVER_EVENT(q, txdma_einval, 1); 1586 break; 1587 case EAGAIN: 1588 IGC_DRIVER_EVENT(q, txdma_eagain, 1); 1589 break; 1590 default: 1591 IGC_DRIVER_EVENT(q, txdma_other, 1); 1592 break; 1593 } 1594 1595 return error; 1596 } 1597 1598 #define IGC_TX_START 1 1599 #define IGC_TX_TRANSMIT 2 1600 1601 static void 1602 igc_start(struct ifnet *ifp) 1603 { 1604 struct igc_softc *sc = ifp->if_softc; 1605 1606 if (__predict_false(!sc->link_active)) { 1607 IFQ_PURGE(&ifp->if_snd); 1608 return; 1609 } 1610 1611 struct tx_ring *txr = &sc->tx_rings[0]; /* queue 0 */ 1612 mutex_enter(&txr->txr_lock); 1613 igc_tx_common_locked(ifp, txr, IGC_TX_START); 1614 mutex_exit(&txr->txr_lock); 1615 } 1616 1617 static inline u_int 1618 igc_select_txqueue(struct igc_softc *sc, struct mbuf *m __unused) 1619 { 1620 const u_int cpuid = cpu_index(curcpu()); 1621 1622 return cpuid % sc->sc_nqueues; 1623 } 1624 1625 static int 1626 igc_transmit(struct ifnet *ifp, struct mbuf *m) 1627 { 1628 struct igc_softc *sc = ifp->if_softc; 1629 const u_int qid = igc_select_txqueue(sc, m); 1630 struct tx_ring *txr = &sc->tx_rings[qid]; 1631 struct igc_queue *q = txr->txr_igcq; 1632 1633 if (__predict_false(!pcq_put(txr->txr_interq, m))) { 1634 IGC_QUEUE_EVENT(q, tx_pcq_drop, 1); 1635 m_freem(m); 1636 return ENOBUFS; 1637 } 1638 1639 mutex_enter(&txr->txr_lock); 1640 igc_tx_common_locked(ifp, txr, IGC_TX_TRANSMIT); 1641 mutex_exit(&txr->txr_lock); 1642 1643 return 0; 1644 } 1645 1646 static void 1647 igc_tx_common_locked(struct ifnet *ifp, struct tx_ring *txr, int caller) 1648 { 1649 struct igc_softc *sc = ifp->if_softc; 1650 struct igc_queue *q = txr->txr_igcq; 1651 net_stat_ref_t nsr = IF_STAT_GETREF(ifp); 1652 int prod, free, last = -1; 1653 bool post = false; 1654 1655 prod = txr->next_avail_desc; 1656 free = txr->next_to_clean; 1657 if (free <= prod) 1658 free += sc->num_tx_desc; 1659 free -= prod; 1660 1661 DPRINTF(TX, "%s: begin: msix %d prod %d n2c %d free %d\n", 1662 caller == IGC_TX_TRANSMIT ? "transmit" : "start", 1663 txr->me, prod, txr->next_to_clean, free); 1664 1665 for (;;) { 1666 struct mbuf *m; 1667 1668 if (__predict_false(free <= IGC_MAX_SCATTER)) { 1669 IGC_QUEUE_EVENT(q, tx_no_desc, 1); 1670 break; 1671 } 1672 1673 if (caller == IGC_TX_TRANSMIT) 1674 m = pcq_get(txr->txr_interq); 1675 else 1676 IFQ_DEQUEUE(&ifp->if_snd, m); 1677 if (__predict_false(m == NULL)) 1678 break; 1679 1680 struct igc_tx_buf *txbuf = &txr->tx_buffers[prod]; 1681 bus_dmamap_t map = txbuf->map; 1682 1683 if (__predict_false( 1684 igc_load_mbuf(q, txr->txdma.dma_tag, map, m))) { 1685 if (caller == IGC_TX_TRANSMIT) 1686 IGC_QUEUE_EVENT(q, tx_pcq_drop, 1); 1687 m_freem(m); 1688 if_statinc_ref(nsr, if_oerrors); 1689 continue; 1690 } 1691 1692 bus_dmamap_sync(txr->txdma.dma_tag, map, 0, 1693 map->dm_mapsize, BUS_DMASYNC_PREWRITE); 1694 1695 uint32_t ctx_cmd_type_len = 0, olinfo_status = 0; 1696 if (igc_tx_ctx_setup(txr, m, prod, &ctx_cmd_type_len, 1697 &olinfo_status)) { 1698 IGC_QUEUE_EVENT(q, tx_ctx, 1); 1699 /* Consume the first descriptor */ 1700 prod = igc_txdesc_incr(sc, prod); 1701 free--; 1702 } 1703 for (int i = 0; i < map->dm_nsegs; i++) { 1704 union igc_adv_tx_desc *txdesc = &txr->tx_base[prod]; 1705 1706 uint32_t cmd_type_len = ctx_cmd_type_len | 1707 IGC_ADVTXD_DCMD_IFCS | IGC_ADVTXD_DTYP_DATA | 1708 IGC_ADVTXD_DCMD_DEXT | map->dm_segs[i].ds_len; 1709 if (i == map->dm_nsegs - 1) { 1710 cmd_type_len |= 1711 IGC_ADVTXD_DCMD_EOP | IGC_ADVTXD_DCMD_RS; 1712 } 1713 1714 igc_txdesc_sync(txr, prod, 1715 BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE); 1716 htolem64(&txdesc->read.buffer_addr, 1717 map->dm_segs[i].ds_addr); 1718 htolem32(&txdesc->read.cmd_type_len, cmd_type_len); 1719 htolem32(&txdesc->read.olinfo_status, olinfo_status); 1720 igc_txdesc_sync(txr, prod, 1721 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); 1722 1723 last = prod; 1724 prod = igc_txdesc_incr(sc, prod); 1725 } 1726 1727 txbuf->m_head = m; 1728 txbuf->eop_index = last; 1729 1730 bpf_mtap(ifp, m, BPF_D_OUT); 1731 1732 if_statadd_ref(nsr, if_obytes, m->m_pkthdr.len); 1733 if (m->m_flags & M_MCAST) 1734 if_statinc_ref(nsr, if_omcasts); 1735 IGC_QUEUE_EVENT(q, tx_packets, 1); 1736 IGC_QUEUE_EVENT(q, tx_bytes, m->m_pkthdr.len); 1737 1738 free -= map->dm_nsegs; 1739 post = true; 1740 } 1741 1742 if (post) { 1743 txr->next_avail_desc = prod; 1744 IGC_WRITE_REG(&sc->hw, IGC_TDT(txr->me), prod); 1745 } 1746 1747 DPRINTF(TX, "%s: done : msix %d prod %d n2c %d free %d\n", 1748 caller == IGC_TX_TRANSMIT ? "transmit" : "start", 1749 txr->me, prod, txr->next_to_clean, free); 1750 1751 IF_STAT_PUTREF(ifp); 1752 } 1753 1754 static bool 1755 igc_txeof(struct tx_ring *txr, u_int limit) 1756 { 1757 struct igc_softc *sc = txr->sc; 1758 struct ifnet *ifp = &sc->sc_ec.ec_if; 1759 int cons, prod; 1760 bool more = false; 1761 1762 prod = txr->next_avail_desc; 1763 cons = txr->next_to_clean; 1764 1765 if (cons == prod) { 1766 DPRINTF(TX, "false: msix %d cons %d prod %d\n", 1767 txr->me, cons, prod); 1768 return false; 1769 } 1770 1771 do { 1772 struct igc_tx_buf *txbuf = &txr->tx_buffers[cons]; 1773 const int last = txbuf->eop_index; 1774 1775 membar_consumer(); /* XXXRO necessary? */ 1776 1777 KASSERT(last != -1); 1778 union igc_adv_tx_desc *txdesc = &txr->tx_base[last]; 1779 igc_txdesc_sync(txr, last, BUS_DMASYNC_POSTREAD); 1780 const uint32_t status = le32toh(txdesc->wb.status); 1781 igc_txdesc_sync(txr, last, BUS_DMASYNC_PREREAD); 1782 1783 if (!(status & IGC_TXD_STAT_DD)) 1784 break; 1785 1786 if (limit-- == 0) { 1787 more = true; 1788 DPRINTF(TX, "pending TX " 1789 "msix %d cons %d last %d prod %d " 1790 "status 0x%08x\n", 1791 txr->me, cons, last, prod, status); 1792 break; 1793 } 1794 1795 DPRINTF(TX, "handled TX " 1796 "msix %d cons %d last %d prod %d " 1797 "status 0x%08x\n", 1798 txr->me, cons, last, prod, status); 1799 1800 if_statinc(ifp, if_opackets); 1801 1802 bus_dmamap_t map = txbuf->map; 1803 bus_dmamap_sync(txr->txdma.dma_tag, map, 0, map->dm_mapsize, 1804 BUS_DMASYNC_POSTWRITE); 1805 bus_dmamap_unload(txr->txdma.dma_tag, map); 1806 m_freem(txbuf->m_head); 1807 1808 txbuf->m_head = NULL; 1809 txbuf->eop_index = -1; 1810 1811 cons = igc_txdesc_incr(sc, last); 1812 } while (cons != prod); 1813 1814 txr->next_to_clean = cons; 1815 1816 return more; 1817 } 1818 1819 static void 1820 igc_intr_barrier(struct igc_softc *sc __unused) 1821 { 1822 1823 xc_barrier(0); 1824 } 1825 1826 static void 1827 igc_stop(struct ifnet *ifp, int disable) 1828 { 1829 struct igc_softc *sc = ifp->if_softc; 1830 1831 mutex_enter(&sc->sc_core_lock); 1832 igc_stop_locked(sc); 1833 mutex_exit(&sc->sc_core_lock); 1834 } 1835 1836 /********************************************************************* 1837 * 1838 * This routine disables all traffic on the adapter by issuing a 1839 * global reset on the MAC. 1840 * 1841 **********************************************************************/ 1842 static void 1843 igc_stop_locked(struct igc_softc *sc) 1844 { 1845 struct ifnet *ifp = &sc->sc_ec.ec_if; 1846 1847 DPRINTF(CFG, "called\n"); 1848 1849 KASSERT(mutex_owned(&sc->sc_core_lock)); 1850 1851 /* 1852 * If stopping processing has already started, do nothing. 1853 */ 1854 if ((ifp->if_flags & IFF_RUNNING) == 0) 1855 return; 1856 1857 /* Tell the stack that the interface is no longer active. */ 1858 ifp->if_flags &= ~IFF_RUNNING; 1859 1860 /* 1861 * igc_handle_queue() can enable interrupts, so wait for completion of 1862 * last igc_handle_queue() after unset IFF_RUNNING. 1863 */ 1864 mutex_exit(&sc->sc_core_lock); 1865 igc_barrier_handle_queue(sc); 1866 mutex_enter(&sc->sc_core_lock); 1867 1868 sc->sc_core_stopping = true; 1869 1870 igc_disable_intr(sc); 1871 1872 callout_halt(&sc->sc_tick_ch, &sc->sc_core_lock); 1873 1874 igc_reset_hw(&sc->hw); 1875 IGC_WRITE_REG(&sc->hw, IGC_WUC, 0); 1876 1877 /* 1878 * Wait for completion of interrupt handlers. 1879 */ 1880 mutex_exit(&sc->sc_core_lock); 1881 igc_intr_barrier(sc); 1882 mutex_enter(&sc->sc_core_lock); 1883 1884 igc_update_link_status(sc); 1885 1886 for (int iq = 0; iq < sc->sc_nqueues; iq++) { 1887 struct tx_ring *txr = &sc->tx_rings[iq]; 1888 1889 igc_withdraw_transmit_packets(txr, false); 1890 } 1891 1892 for (int iq = 0; iq < sc->sc_nqueues; iq++) { 1893 struct rx_ring *rxr = &sc->rx_rings[iq]; 1894 1895 igc_clear_receive_status(rxr); 1896 } 1897 1898 /* Save last flags for the callback */ 1899 sc->sc_if_flags = ifp->if_flags; 1900 } 1901 1902 /********************************************************************* 1903 * Ioctl entry point 1904 * 1905 * igc_ioctl is called when the user wants to configure the 1906 * interface. 1907 * 1908 * return 0 on success, positive on failure 1909 **********************************************************************/ 1910 static int 1911 igc_ioctl(struct ifnet * ifp, u_long cmd, void *data) 1912 { 1913 struct igc_softc *sc __unused = ifp->if_softc; 1914 int s; 1915 int error; 1916 1917 DPRINTF(CFG, "cmd 0x%016lx\n", cmd); 1918 1919 switch (cmd) { 1920 case SIOCADDMULTI: 1921 case SIOCDELMULTI: 1922 break; 1923 default: 1924 KASSERT(IFNET_LOCKED(ifp)); 1925 } 1926 1927 if (cmd == SIOCZIFDATA) { 1928 mutex_enter(&sc->sc_core_lock); 1929 igc_clear_counters(sc); 1930 mutex_exit(&sc->sc_core_lock); 1931 } 1932 1933 switch (cmd) { 1934 #ifdef IF_RXR 1935 case SIOCGIFRXR: 1936 s = splnet(); 1937 error = igc_rxrinfo(sc, (struct if_rxrinfo *)ifr->ifr_data); 1938 splx(s); 1939 break; 1940 #endif 1941 default: 1942 s = splnet(); 1943 error = ether_ioctl(ifp, cmd, data); 1944 splx(s); 1945 break; 1946 } 1947 1948 if (error != ENETRESET) 1949 return error; 1950 1951 error = 0; 1952 1953 if (cmd == SIOCSIFCAP) 1954 error = if_init(ifp); 1955 else if ((cmd == SIOCADDMULTI) || (cmd == SIOCDELMULTI)) { 1956 mutex_enter(&sc->sc_core_lock); 1957 if (sc->sc_if_flags & IFF_RUNNING) { 1958 /* 1959 * Multicast list has changed; set the hardware filter 1960 * accordingly. 1961 */ 1962 igc_disable_intr(sc); 1963 igc_set_filter(sc); 1964 igc_enable_intr(sc); 1965 } 1966 mutex_exit(&sc->sc_core_lock); 1967 } 1968 1969 return error; 1970 } 1971 1972 #ifdef IF_RXR 1973 static int 1974 igc_rxrinfo(struct igc_softc *sc, struct if_rxrinfo *ifri) 1975 { 1976 struct if_rxring_info *ifr, ifr1; 1977 int error; 1978 1979 if (sc->sc_nqueues > 1) { 1980 ifr = kmem_zalloc(sc->sc_nqueues * sizeof(*ifr), KM_SLEEP); 1981 } else { 1982 ifr = &ifr1; 1983 memset(ifr, 0, sizeof(*ifr)); 1984 } 1985 1986 for (int iq = 0; iq < sc->sc_nqueues; iq++) { 1987 struct rx_ring *rxr = &sc->rx_rings[iq]; 1988 1989 ifr[iq].ifr_size = MCLBYTES; 1990 snprintf(ifr[iq].ifr_name, sizeof(ifr[iq].ifr_name), "%d", iq); 1991 ifr[iq].ifr_info = rxr->rx_ring; 1992 } 1993 1994 error = if_rxr_info_ioctl(ifri, sc->sc_nqueues, ifr); 1995 if (sc->sc_nqueues > 1) 1996 kmem_free(ifr, sc->sc_nqueues * sizeof(*ifr)); 1997 1998 return error; 1999 } 2000 #endif 2001 2002 static void 2003 igc_rxfill(struct rx_ring *rxr) 2004 { 2005 struct igc_softc *sc = rxr->sc; 2006 int id; 2007 2008 for (id = 0; id < sc->num_rx_desc; id++) { 2009 if (igc_get_buf(rxr, id, false)) { 2010 panic("%s: msix=%d i=%d\n", __func__, rxr->me, id); 2011 } 2012 } 2013 2014 id = sc->num_rx_desc - 1; 2015 rxr->last_desc_filled = id; 2016 IGC_WRITE_REG(&sc->hw, IGC_RDT(rxr->me), id); 2017 rxr->next_to_check = 0; 2018 } 2019 2020 static void 2021 igc_rxrefill(struct rx_ring *rxr, int end) 2022 { 2023 struct igc_softc *sc = rxr->sc; 2024 int id; 2025 2026 for (id = rxr->next_to_check; id != end; id = igc_rxdesc_incr(sc, id)) { 2027 if (igc_get_buf(rxr, id, true)) { 2028 /* XXXRO */ 2029 panic("%s: msix=%d id=%d\n", __func__, rxr->me, id); 2030 } 2031 } 2032 2033 id = igc_rxdesc_decr(sc, id); 2034 DPRINTF(RX, "%s RDT %d id %d\n", 2035 rxr->last_desc_filled == id ? "same" : "diff", 2036 rxr->last_desc_filled, id); 2037 rxr->last_desc_filled = id; 2038 IGC_WRITE_REG(&sc->hw, IGC_RDT(rxr->me), id); 2039 } 2040 2041 /********************************************************************* 2042 * 2043 * This routine executes in interrupt context. It replenishes 2044 * the mbufs in the descriptor and sends data which has been 2045 * dma'ed into host memory to upper layer. 2046 * 2047 *********************************************************************/ 2048 static bool 2049 igc_rxeof(struct rx_ring *rxr, u_int limit) 2050 { 2051 struct igc_softc *sc = rxr->sc; 2052 struct igc_queue *q = rxr->rxr_igcq; 2053 struct ifnet *ifp = &sc->sc_ec.ec_if; 2054 int id; 2055 bool more = false; 2056 2057 id = rxr->next_to_check; 2058 for (;;) { 2059 union igc_adv_rx_desc *rxdesc = &rxr->rx_base[id]; 2060 struct igc_rx_buf *rxbuf, *nxbuf; 2061 struct mbuf *mp, *m; 2062 2063 igc_rxdesc_sync(rxr, id, 2064 BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE); 2065 2066 const uint32_t staterr = le32toh(rxdesc->wb.upper.status_error); 2067 2068 if (!ISSET(staterr, IGC_RXD_STAT_DD)) { 2069 igc_rxdesc_sync(rxr, id, 2070 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); 2071 break; 2072 } 2073 2074 if (limit-- == 0) { 2075 igc_rxdesc_sync(rxr, id, 2076 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); 2077 DPRINTF(RX, "more=true\n"); 2078 more = true; 2079 break; 2080 } 2081 2082 /* Zero out the receive descriptors status. */ 2083 rxdesc->wb.upper.status_error = 0; 2084 2085 /* Pull the mbuf off the ring. */ 2086 rxbuf = &rxr->rx_buffers[id]; 2087 bus_dmamap_t map = rxbuf->map; 2088 bus_dmamap_sync(rxr->rxdma.dma_tag, map, 2089 0, map->dm_mapsize, BUS_DMASYNC_POSTREAD); 2090 bus_dmamap_unload(rxr->rxdma.dma_tag, map); 2091 2092 mp = rxbuf->buf; 2093 rxbuf->buf = NULL; 2094 2095 const bool eop = staterr & IGC_RXD_STAT_EOP; 2096 const uint16_t len = le16toh(rxdesc->wb.upper.length); 2097 2098 const uint16_t vtag = le16toh(rxdesc->wb.upper.vlan); 2099 2100 const uint32_t ptype = le32toh(rxdesc->wb.lower.lo_dword.data) & 2101 IGC_PKTTYPE_MASK; 2102 2103 const uint32_t hash __unused = 2104 le32toh(rxdesc->wb.lower.hi_dword.rss); 2105 const uint16_t hashtype __unused = 2106 le16toh(rxdesc->wb.lower.lo_dword.hs_rss.pkt_info) & 2107 IGC_RXDADV_RSSTYPE_MASK; 2108 2109 igc_rxdesc_sync(rxr, id, 2110 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); 2111 2112 if (__predict_false(staterr & IGC_RXDEXT_STATERR_RXE)) { 2113 if (rxbuf->fmp) { 2114 m_freem(rxbuf->fmp); 2115 rxbuf->fmp = NULL; 2116 } 2117 2118 m_freem(mp); 2119 m = NULL; 2120 2121 if_statinc(ifp, if_ierrors); 2122 IGC_QUEUE_EVENT(q, rx_discard, 1); 2123 2124 DPRINTF(RX, "ierrors++\n"); 2125 2126 goto next_desc; 2127 } 2128 2129 if (__predict_false(mp == NULL)) { 2130 panic("%s: igc_rxeof: NULL mbuf in slot %d " 2131 "(filled %d)", device_xname(sc->sc_dev), 2132 id, rxr->last_desc_filled); 2133 } 2134 2135 if (!eop) { 2136 /* 2137 * Figure out the next descriptor of this frame. 2138 */ 2139 int nextp = igc_rxdesc_incr(sc, id); 2140 2141 nxbuf = &rxr->rx_buffers[nextp]; 2142 /* 2143 * TODO prefetch(nxbuf); 2144 */ 2145 } 2146 2147 mp->m_len = len; 2148 2149 m = rxbuf->fmp; 2150 rxbuf->fmp = NULL; 2151 2152 if (m != NULL) { 2153 m->m_pkthdr.len += mp->m_len; 2154 } else { 2155 m = mp; 2156 m->m_pkthdr.len = mp->m_len; 2157 #if NVLAN > 0 2158 if (staterr & IGC_RXD_STAT_VP) 2159 vlan_set_tag(m, vtag); 2160 #endif 2161 } 2162 2163 /* Pass the head pointer on */ 2164 if (!eop) { 2165 nxbuf->fmp = m; 2166 m = NULL; 2167 mp->m_next = nxbuf->buf; 2168 } else { 2169 m_set_rcvif(m, ifp); 2170 2171 m->m_pkthdr.csum_flags = igc_rx_checksum(q, 2172 ifp->if_capenable, staterr, ptype); 2173 2174 #ifdef notyet 2175 if (hashtype != IGC_RXDADV_RSSTYPE_NONE) { 2176 m->m_pkthdr.ph_flowid = hash; 2177 SET(m->m_pkthdr.csum_flags, M_FLOWID); 2178 } 2179 ml_enqueue(&ml, m); 2180 #endif 2181 2182 if_percpuq_enqueue(sc->sc_ipq, m); 2183 2184 if_statinc(ifp, if_ipackets); 2185 IGC_QUEUE_EVENT(q, rx_packets, 1); 2186 IGC_QUEUE_EVENT(q, rx_bytes, m->m_pkthdr.len); 2187 } 2188 next_desc: 2189 /* Advance our pointers to the next descriptor. */ 2190 id = igc_rxdesc_incr(sc, id); 2191 } 2192 2193 DPRINTF(RX, "fill queue[%d]\n", rxr->me); 2194 igc_rxrefill(rxr, id); 2195 2196 DPRINTF(RX, "%s n2c %d id %d\n", 2197 rxr->next_to_check == id ? "same" : "diff", 2198 rxr->next_to_check, id); 2199 rxr->next_to_check = id; 2200 2201 #ifdef OPENBSD 2202 if (!(staterr & IGC_RXD_STAT_DD)) 2203 return 0; 2204 #endif 2205 2206 return more; 2207 } 2208 2209 /********************************************************************* 2210 * 2211 * Verify that the hardware indicated that the checksum is valid. 2212 * Inform the stack about the status of checksum so that stack 2213 * doesn't spend time verifying the checksum. 2214 * 2215 *********************************************************************/ 2216 static int 2217 igc_rx_checksum(struct igc_queue *q, uint64_t capenable, uint32_t staterr, 2218 uint32_t ptype) 2219 { 2220 const uint16_t status = (uint16_t)staterr; 2221 const uint8_t errors = (uint8_t)(staterr >> 24); 2222 int flags = 0; 2223 2224 if ((status & IGC_RXD_STAT_IPCS) != 0 && 2225 (capenable & IFCAP_CSUM_IPv4_Rx) != 0) { 2226 IGC_DRIVER_EVENT(q, rx_ipcs, 1); 2227 flags |= M_CSUM_IPv4; 2228 if (__predict_false((errors & IGC_RXD_ERR_IPE) != 0)) { 2229 IGC_DRIVER_EVENT(q, rx_ipcs_bad, 1); 2230 flags |= M_CSUM_IPv4_BAD; 2231 } 2232 } 2233 2234 if ((status & IGC_RXD_STAT_TCPCS) != 0) { 2235 IGC_DRIVER_EVENT(q, rx_tcpcs, 1); 2236 if ((capenable & IFCAP_CSUM_TCPv4_Rx) != 0) 2237 flags |= M_CSUM_TCPv4; 2238 if ((capenable & IFCAP_CSUM_TCPv6_Rx) != 0) 2239 flags |= M_CSUM_TCPv6; 2240 } 2241 2242 if ((status & IGC_RXD_STAT_UDPCS) != 0) { 2243 IGC_DRIVER_EVENT(q, rx_udpcs, 1); 2244 if ((capenable & IFCAP_CSUM_UDPv4_Rx) != 0) 2245 flags |= M_CSUM_UDPv4; 2246 if ((capenable & IFCAP_CSUM_UDPv6_Rx) != 0) 2247 flags |= M_CSUM_UDPv6; 2248 } 2249 2250 if (__predict_false((errors & IGC_RXD_ERR_TCPE) != 0)) { 2251 IGC_DRIVER_EVENT(q, rx_l4cs_bad, 1); 2252 if ((flags & ~M_CSUM_IPv4) != 0) 2253 flags |= M_CSUM_TCP_UDP_BAD; 2254 } 2255 2256 return flags; 2257 } 2258 2259 static void 2260 igc_watchdog(struct ifnet * ifp) 2261 { 2262 } 2263 2264 static void 2265 igc_tick(void *arg) 2266 { 2267 struct igc_softc *sc = arg; 2268 2269 mutex_enter(&sc->sc_core_lock); 2270 2271 if (__predict_false(sc->sc_core_stopping)) { 2272 mutex_exit(&sc->sc_core_lock); 2273 return; 2274 } 2275 2276 /* XXX watchdog */ 2277 if (0) { 2278 IGC_GLOBAL_EVENT(sc, watchdog, 1); 2279 } 2280 2281 igc_update_counters(sc); 2282 2283 mutex_exit(&sc->sc_core_lock); 2284 2285 callout_schedule(&sc->sc_tick_ch, hz); 2286 } 2287 2288 /********************************************************************* 2289 * 2290 * Media Ioctl callback 2291 * 2292 * This routine is called whenever the user queries the status of 2293 * the interface using ifconfig. 2294 * 2295 **********************************************************************/ 2296 static void 2297 igc_media_status(struct ifnet *ifp, struct ifmediareq *ifmr) 2298 { 2299 struct igc_softc *sc = ifp->if_softc; 2300 struct igc_hw *hw = &sc->hw; 2301 2302 igc_update_link_status(sc); 2303 2304 ifmr->ifm_status = IFM_AVALID; 2305 ifmr->ifm_active = IFM_ETHER; 2306 2307 if (!sc->link_active) { 2308 ifmr->ifm_active |= IFM_NONE; 2309 return; 2310 } 2311 2312 ifmr->ifm_status |= IFM_ACTIVE; 2313 2314 switch (sc->link_speed) { 2315 case 10: 2316 ifmr->ifm_active |= IFM_10_T; 2317 break; 2318 case 100: 2319 ifmr->ifm_active |= IFM_100_TX; 2320 break; 2321 case 1000: 2322 ifmr->ifm_active |= IFM_1000_T; 2323 break; 2324 case 2500: 2325 ifmr->ifm_active |= IFM_2500_T; 2326 break; 2327 } 2328 2329 if (sc->link_duplex == FULL_DUPLEX) 2330 ifmr->ifm_active |= IFM_FDX; 2331 else 2332 ifmr->ifm_active |= IFM_HDX; 2333 2334 switch (hw->fc.current_mode) { 2335 case igc_fc_tx_pause: 2336 ifmr->ifm_active |= IFM_FLOW | IFM_ETH_TXPAUSE; 2337 break; 2338 case igc_fc_rx_pause: 2339 ifmr->ifm_active |= IFM_FLOW | IFM_ETH_RXPAUSE; 2340 break; 2341 case igc_fc_full: 2342 ifmr->ifm_active |= IFM_FLOW | 2343 IFM_ETH_TXPAUSE | IFM_ETH_RXPAUSE; 2344 break; 2345 case igc_fc_none: 2346 default: 2347 break; 2348 } 2349 } 2350 2351 /********************************************************************* 2352 * 2353 * Media Ioctl callback 2354 * 2355 * This routine is called when the user changes speed/duplex using 2356 * media/mediopt option with ifconfig. 2357 * 2358 **********************************************************************/ 2359 static int 2360 igc_media_change(struct ifnet *ifp) 2361 { 2362 struct igc_softc *sc = ifp->if_softc; 2363 struct ifmedia *ifm = &sc->media; 2364 2365 if (IFM_TYPE(ifm->ifm_media) != IFM_ETHER) 2366 return EINVAL; 2367 2368 sc->hw.mac.autoneg = DO_AUTO_NEG; 2369 2370 switch (IFM_SUBTYPE(ifm->ifm_media)) { 2371 case IFM_AUTO: 2372 sc->hw.phy.autoneg_advertised = AUTONEG_ADV_DEFAULT; 2373 break; 2374 case IFM_2500_T: 2375 sc->hw.phy.autoneg_advertised = ADVERTISE_2500_FULL; 2376 break; 2377 case IFM_1000_T: 2378 sc->hw.phy.autoneg_advertised = ADVERTISE_1000_FULL; 2379 break; 2380 case IFM_100_TX: 2381 if ((ifm->ifm_media & IFM_GMASK) == IFM_FDX) 2382 sc->hw.phy.autoneg_advertised = ADVERTISE_100_FULL; 2383 else 2384 sc->hw.phy.autoneg_advertised = ADVERTISE_100_HALF; 2385 break; 2386 case IFM_10_T: 2387 if ((ifm->ifm_media & IFM_GMASK) == IFM_FDX) 2388 sc->hw.phy.autoneg_advertised = ADVERTISE_10_FULL; 2389 else 2390 sc->hw.phy.autoneg_advertised = ADVERTISE_10_HALF; 2391 break; 2392 default: 2393 return EINVAL; 2394 } 2395 2396 igc_init_locked(sc); 2397 2398 return 0; 2399 } 2400 2401 static int 2402 igc_ifflags_cb(struct ethercom *ec) 2403 { 2404 struct ifnet *ifp = &ec->ec_if; 2405 struct igc_softc *sc = ifp->if_softc; 2406 int rc = 0; 2407 u_short iffchange; 2408 bool needreset = false; 2409 2410 DPRINTF(CFG, "called\n"); 2411 2412 KASSERT(IFNET_LOCKED(ifp)); 2413 2414 mutex_enter(&sc->sc_core_lock); 2415 2416 /* 2417 * Check for if_flags. 2418 * Main usage is to prevent linkdown when opening bpf. 2419 */ 2420 iffchange = ifp->if_flags ^ sc->sc_if_flags; 2421 sc->sc_if_flags = ifp->if_flags; 2422 if ((iffchange & ~(IFF_CANTCHANGE | IFF_DEBUG)) != 0) { 2423 needreset = true; 2424 goto ec; 2425 } 2426 2427 /* iff related updates */ 2428 if ((iffchange & IFF_PROMISC) != 0) 2429 igc_set_filter(sc); 2430 2431 #ifdef notyet 2432 igc_set_vlan(sc); 2433 #endif 2434 2435 ec: 2436 #ifdef notyet 2437 /* Check for ec_capenable. */ 2438 ecchange = ec->ec_capenable ^ sc->sc_ec_capenable; 2439 sc->sc_ec_capenable = ec->ec_capenable; 2440 if ((ecchange & ~ETHERCAP_SOMETHING) != 0) { 2441 needreset = true; 2442 goto out; 2443 } 2444 #endif 2445 if (needreset) 2446 rc = ENETRESET; 2447 2448 mutex_exit(&sc->sc_core_lock); 2449 2450 return rc; 2451 } 2452 2453 static void 2454 igc_set_filter(struct igc_softc *sc) 2455 { 2456 struct ethercom *ec = &sc->sc_ec; 2457 uint32_t rctl; 2458 2459 rctl = IGC_READ_REG(&sc->hw, IGC_RCTL); 2460 rctl &= ~(IGC_RCTL_BAM |IGC_RCTL_UPE | IGC_RCTL_MPE); 2461 2462 if ((sc->sc_if_flags & IFF_BROADCAST) != 0) 2463 rctl |= IGC_RCTL_BAM; 2464 if ((sc->sc_if_flags & IFF_PROMISC) != 0) { 2465 DPRINTF(CFG, "promisc\n"); 2466 rctl |= IGC_RCTL_UPE; 2467 ETHER_LOCK(ec); 2468 allmulti: 2469 ec->ec_flags |= ETHER_F_ALLMULTI; 2470 ETHER_UNLOCK(ec); 2471 rctl |= IGC_RCTL_MPE; 2472 } else { 2473 struct ether_multistep step; 2474 struct ether_multi *enm; 2475 int mcnt = 0; 2476 2477 memset(sc->mta, 0, IGC_MTA_LEN); 2478 2479 ETHER_LOCK(ec); 2480 ETHER_FIRST_MULTI(step, ec, enm); 2481 while (enm != NULL) { 2482 if (((memcmp(enm->enm_addrlo, enm->enm_addrhi, 2483 ETHER_ADDR_LEN)) != 0) || 2484 (mcnt >= MAX_NUM_MULTICAST_ADDRESSES)) { 2485 /* 2486 * We must listen to a range of multicast 2487 * addresses. For now, just accept all 2488 * multicasts, rather than trying to set only 2489 * those filter bits needed to match the range. 2490 * (At this time, the only use of address 2491 * ranges is for IP multicast routing, for 2492 * which the range is big enough to require all 2493 * bits set.) 2494 */ 2495 goto allmulti; 2496 } 2497 DPRINTF(CFG, "%d: %s\n", mcnt, 2498 ether_sprintf(enm->enm_addrlo)); 2499 memcpy(&sc->mta[mcnt * ETHER_ADDR_LEN], 2500 enm->enm_addrlo, ETHER_ADDR_LEN); 2501 2502 mcnt++; 2503 ETHER_NEXT_MULTI(step, enm); 2504 } 2505 ec->ec_flags &= ~ETHER_F_ALLMULTI; 2506 ETHER_UNLOCK(ec); 2507 2508 DPRINTF(CFG, "hw filter\n"); 2509 igc_update_mc_addr_list(&sc->hw, sc->mta, mcnt); 2510 } 2511 2512 IGC_WRITE_REG(&sc->hw, IGC_RCTL, rctl); 2513 } 2514 2515 static void 2516 igc_update_link_status(struct igc_softc *sc) 2517 { 2518 struct ifnet *ifp = &sc->sc_ec.ec_if; 2519 struct igc_hw *hw = &sc->hw; 2520 2521 if (IGC_READ_REG(&sc->hw, IGC_STATUS) & IGC_STATUS_LU) { 2522 if (sc->link_active == 0) { 2523 igc_get_speed_and_duplex(hw, &sc->link_speed, 2524 &sc->link_duplex); 2525 sc->link_active = 1; 2526 ifp->if_baudrate = IF_Mbps(sc->link_speed); 2527 if_link_state_change(ifp, LINK_STATE_UP); 2528 } 2529 } else { 2530 if (sc->link_active == 1) { 2531 ifp->if_baudrate = sc->link_speed = 0; 2532 sc->link_duplex = 0; 2533 sc->link_active = 0; 2534 if_link_state_change(ifp, LINK_STATE_DOWN); 2535 } 2536 } 2537 } 2538 2539 /********************************************************************* 2540 * 2541 * Get a buffer from system mbuf buffer pool. 2542 * 2543 **********************************************************************/ 2544 static int 2545 igc_get_buf(struct rx_ring *rxr, int id, bool strict) 2546 { 2547 struct igc_softc *sc = rxr->sc; 2548 struct igc_queue *q = rxr->rxr_igcq; 2549 struct igc_rx_buf *rxbuf = &rxr->rx_buffers[id]; 2550 bus_dmamap_t map = rxbuf->map; 2551 struct mbuf *m; 2552 int error; 2553 2554 if (__predict_false(rxbuf->buf)) { 2555 if (strict) { 2556 DPRINTF(RX, "slot %d already has an mbuf\n", id); 2557 return EINVAL; 2558 } 2559 return 0; 2560 } 2561 2562 MGETHDR(m, M_DONTWAIT, MT_DATA); 2563 if (__predict_false(m == NULL)) { 2564 enobuf: 2565 IGC_QUEUE_EVENT(q, rx_no_mbuf, 1); 2566 return ENOBUFS; 2567 } 2568 2569 MCLGET(m, M_DONTWAIT); 2570 if (__predict_false(!(m->m_flags & M_EXT))) { 2571 m_freem(m); 2572 goto enobuf; 2573 } 2574 2575 m->m_len = m->m_pkthdr.len = sc->rx_mbuf_sz; 2576 2577 error = bus_dmamap_load_mbuf(rxr->rxdma.dma_tag, map, m, 2578 BUS_DMA_READ | BUS_DMA_NOWAIT); 2579 if (error) { 2580 m_freem(m); 2581 return error; 2582 } 2583 2584 bus_dmamap_sync(rxr->rxdma.dma_tag, map, 0, 2585 map->dm_mapsize, BUS_DMASYNC_PREREAD); 2586 rxbuf->buf = m; 2587 2588 union igc_adv_rx_desc *rxdesc = &rxr->rx_base[id]; 2589 igc_rxdesc_sync(rxr, id, BUS_DMASYNC_POSTWRITE | BUS_DMASYNC_POSTREAD); 2590 rxdesc->read.pkt_addr = htole64(map->dm_segs[0].ds_addr); 2591 igc_rxdesc_sync(rxr, id, BUS_DMASYNC_PREWRITE | BUS_DMASYNC_PREREAD); 2592 2593 return 0; 2594 } 2595 2596 static void 2597 igc_configure_queues(struct igc_softc *sc) 2598 { 2599 struct igc_hw *hw = &sc->hw; 2600 uint32_t ivar; 2601 2602 /* First turn on RSS capability */ 2603 IGC_WRITE_REG(hw, IGC_GPIE, IGC_GPIE_MSIX_MODE | IGC_GPIE_EIAME | 2604 IGC_GPIE_PBA | IGC_GPIE_NSICR); 2605 2606 /* Set the starting interrupt rate */ 2607 uint32_t newitr = (4000000 / MAX_INTS_PER_SEC) & 0x7FFC; 2608 newitr |= IGC_EITR_CNT_IGNR; 2609 2610 /* Turn on MSI-X */ 2611 uint32_t newmask = 0; 2612 for (int iq = 0; iq < sc->sc_nqueues; iq++) { 2613 struct igc_queue *q = &sc->queues[iq]; 2614 2615 /* RX entries */ 2616 igc_set_queues(sc, iq, q->msix, 0); 2617 /* TX entries */ 2618 igc_set_queues(sc, iq, q->msix, 1); 2619 newmask |= q->eims; 2620 IGC_WRITE_REG(hw, IGC_EITR(q->msix), newitr); 2621 } 2622 sc->msix_queuesmask = newmask; 2623 2624 #if 1 2625 ivar = IGC_READ_REG_ARRAY(hw, IGC_IVAR0, 0); 2626 DPRINTF(CFG, "ivar(0)=0x%x\n", ivar); 2627 ivar = IGC_READ_REG_ARRAY(hw, IGC_IVAR0, 1); 2628 DPRINTF(CFG, "ivar(1)=0x%x\n", ivar); 2629 #endif 2630 2631 /* And for the link interrupt */ 2632 ivar = (sc->linkvec | IGC_IVAR_VALID) << 8; 2633 sc->msix_linkmask = 1 << sc->linkvec; 2634 IGC_WRITE_REG(hw, IGC_IVAR_MISC, ivar); 2635 } 2636 2637 static void 2638 igc_set_queues(struct igc_softc *sc, uint32_t entry, uint32_t vector, int type) 2639 { 2640 struct igc_hw *hw = &sc->hw; 2641 const uint32_t index = entry >> 1; 2642 uint32_t ivar = IGC_READ_REG_ARRAY(hw, IGC_IVAR0, index); 2643 2644 if (type) { 2645 if (entry & 1) { 2646 ivar &= 0x00FFFFFF; 2647 ivar |= (vector | IGC_IVAR_VALID) << 24; 2648 } else { 2649 ivar &= 0xFFFF00FF; 2650 ivar |= (vector | IGC_IVAR_VALID) << 8; 2651 } 2652 } else { 2653 if (entry & 1) { 2654 ivar &= 0xFF00FFFF; 2655 ivar |= (vector | IGC_IVAR_VALID) << 16; 2656 } else { 2657 ivar &= 0xFFFFFF00; 2658 ivar |= vector | IGC_IVAR_VALID; 2659 } 2660 } 2661 IGC_WRITE_REG_ARRAY(hw, IGC_IVAR0, index, ivar); 2662 } 2663 2664 static void 2665 igc_enable_queue(struct igc_softc *sc, uint32_t eims) 2666 { 2667 IGC_WRITE_REG(&sc->hw, IGC_EIMS, eims); 2668 } 2669 2670 static void 2671 igc_enable_intr(struct igc_softc *sc) 2672 { 2673 struct igc_hw *hw = &sc->hw; 2674 2675 if (sc->sc_intr_type == PCI_INTR_TYPE_MSIX) { 2676 const uint32_t mask = sc->msix_queuesmask | sc->msix_linkmask; 2677 2678 IGC_WRITE_REG(hw, IGC_EIAC, mask); 2679 IGC_WRITE_REG(hw, IGC_EIAM, mask); 2680 IGC_WRITE_REG(hw, IGC_EIMS, mask); 2681 IGC_WRITE_REG(hw, IGC_IMS, IGC_IMS_LSC); 2682 } else { 2683 IGC_WRITE_REG(hw, IGC_IMS, IMS_ENABLE_MASK); 2684 } 2685 IGC_WRITE_FLUSH(hw); 2686 } 2687 2688 static void 2689 igc_disable_intr(struct igc_softc *sc) 2690 { 2691 struct igc_hw *hw = &sc->hw; 2692 2693 if (sc->sc_intr_type == PCI_INTR_TYPE_MSIX) { 2694 IGC_WRITE_REG(hw, IGC_EIMC, 0xffffffff); 2695 IGC_WRITE_REG(hw, IGC_EIAC, 0); 2696 } 2697 IGC_WRITE_REG(hw, IGC_IMC, 0xffffffff); 2698 IGC_WRITE_FLUSH(hw); 2699 } 2700 2701 static int 2702 igc_intr_link(void *arg) 2703 { 2704 struct igc_softc *sc = (struct igc_softc *)arg; 2705 const uint32_t reg_icr = IGC_READ_REG(&sc->hw, IGC_ICR); 2706 2707 IGC_GLOBAL_EVENT(sc, link, 1); 2708 2709 if (reg_icr & IGC_ICR_LSC) { 2710 mutex_enter(&sc->sc_core_lock); 2711 sc->hw.mac.get_link_status = true; 2712 igc_update_link_status(sc); 2713 mutex_exit(&sc->sc_core_lock); 2714 } 2715 2716 IGC_WRITE_REG(&sc->hw, IGC_IMS, IGC_IMS_LSC); 2717 IGC_WRITE_REG(&sc->hw, IGC_EIMS, sc->msix_linkmask); 2718 2719 return 1; 2720 } 2721 2722 static int 2723 igc_intr_queue(void *arg) 2724 { 2725 struct igc_queue *iq = arg; 2726 struct igc_softc *sc = iq->sc; 2727 struct ifnet *ifp = &sc->sc_ec.ec_if; 2728 struct rx_ring *rxr = iq->rxr; 2729 struct tx_ring *txr = iq->txr; 2730 const u_int txlimit = sc->sc_tx_intr_process_limit, 2731 rxlimit = sc->sc_rx_intr_process_limit; 2732 bool txmore, rxmore; 2733 2734 IGC_QUEUE_EVENT(iq, irqs, 1); 2735 2736 if (__predict_false(!ISSET(ifp->if_flags, IFF_RUNNING))) 2737 return 0; 2738 2739 mutex_enter(&txr->txr_lock); 2740 txmore = igc_txeof(txr, txlimit); 2741 mutex_exit(&txr->txr_lock); 2742 mutex_enter(&rxr->rxr_lock); 2743 rxmore = igc_rxeof(rxr, rxlimit); 2744 mutex_exit(&rxr->rxr_lock); 2745 2746 if (txmore || rxmore) { 2747 IGC_QUEUE_EVENT(iq, req, 1); 2748 igc_sched_handle_queue(sc, iq); 2749 } else { 2750 igc_enable_queue(sc, iq->eims); 2751 } 2752 2753 return 1; 2754 } 2755 2756 static int 2757 igc_intr(void *arg) 2758 { 2759 struct igc_softc *sc = arg; 2760 struct ifnet *ifp = &sc->sc_ec.ec_if; 2761 struct igc_queue *iq = &sc->queues[0]; 2762 struct rx_ring *rxr = iq->rxr; 2763 struct tx_ring *txr = iq->txr; 2764 const u_int txlimit = sc->sc_tx_intr_process_limit, 2765 rxlimit = sc->sc_rx_intr_process_limit; 2766 bool txmore, rxmore; 2767 2768 if (__predict_false(!ISSET(ifp->if_flags, IFF_RUNNING))) 2769 return 0; 2770 2771 const uint32_t reg_icr = IGC_READ_REG(&sc->hw, IGC_ICR); 2772 DPRINTF(MISC, "reg_icr=0x%x\n", reg_icr); 2773 2774 /* Definitely not our interrupt. */ 2775 if (reg_icr == 0x0) { 2776 DPRINTF(MISC, "not for me"); 2777 return 0; 2778 } 2779 2780 IGC_QUEUE_EVENT(iq, irqs, 1); 2781 2782 /* Hot eject? */ 2783 if (__predict_false(reg_icr == 0xffffffff)) { 2784 DPRINTF(MISC, "hot eject\n"); 2785 return 0; 2786 } 2787 2788 if (__predict_false(!(reg_icr & IGC_ICR_INT_ASSERTED))) { 2789 DPRINTF(MISC, "not set IGC_ICR_INT_ASSERTED"); 2790 return 0; 2791 } 2792 2793 /* 2794 * Only MSI-X interrupts have one-shot behavior by taking advantage 2795 * of the EIAC register. Thus, explicitly disable interrupts. This 2796 * also works around the MSI message reordering errata on certain 2797 * systems. 2798 */ 2799 igc_disable_intr(sc); 2800 2801 mutex_enter(&txr->txr_lock); 2802 txmore = igc_txeof(txr, txlimit); 2803 mutex_exit(&txr->txr_lock); 2804 mutex_enter(&rxr->rxr_lock); 2805 rxmore = igc_rxeof(rxr, rxlimit); 2806 mutex_exit(&rxr->rxr_lock); 2807 2808 /* Link status change */ 2809 // XXXX FreeBSD checks IGC_ICR_RXSEQ 2810 if (__predict_false(reg_icr & IGC_ICR_LSC)) { 2811 IGC_GLOBAL_EVENT(sc, link, 1); 2812 mutex_enter(&sc->sc_core_lock); 2813 sc->hw.mac.get_link_status = true; 2814 igc_update_link_status(sc); 2815 mutex_exit(&sc->sc_core_lock); 2816 } 2817 2818 if (txmore || rxmore) { 2819 IGC_QUEUE_EVENT(iq, req, 1); 2820 igc_sched_handle_queue(sc, iq); 2821 } else { 2822 igc_enable_intr(sc); 2823 } 2824 2825 return 1; 2826 } 2827 2828 static void 2829 igc_handle_queue(void *arg) 2830 { 2831 struct igc_queue *iq = arg; 2832 struct igc_softc *sc = iq->sc; 2833 struct tx_ring *txr = iq->txr; 2834 struct rx_ring *rxr = iq->rxr; 2835 const u_int txlimit = sc->sc_tx_process_limit, 2836 rxlimit = sc->sc_rx_process_limit; 2837 bool txmore, rxmore; 2838 2839 IGC_QUEUE_EVENT(iq, handleq, 1); 2840 2841 mutex_enter(&txr->txr_lock); 2842 txmore = igc_txeof(txr, txlimit); 2843 /* for ALTQ, dequeue from if_snd */ 2844 if (txr->me == 0) { 2845 struct ifnet *ifp = &sc->sc_ec.ec_if; 2846 2847 igc_tx_common_locked(ifp, txr, IGC_TX_START); 2848 } 2849 mutex_exit(&txr->txr_lock); 2850 2851 mutex_enter(&rxr->rxr_lock); 2852 rxmore = igc_rxeof(rxr, rxlimit); 2853 mutex_exit(&rxr->rxr_lock); 2854 2855 if (txmore || rxmore) { 2856 igc_sched_handle_queue(sc, iq); 2857 } else { 2858 if (sc->sc_intr_type == PCI_INTR_TYPE_MSIX) 2859 igc_enable_queue(sc, iq->eims); 2860 else 2861 igc_enable_intr(sc); 2862 } 2863 } 2864 2865 static void 2866 igc_handle_queue_work(struct work *wk, void *context) 2867 { 2868 struct igc_queue *iq = 2869 container_of(wk, struct igc_queue, igcq_wq_cookie); 2870 2871 igc_handle_queue(iq); 2872 } 2873 2874 static void 2875 igc_sched_handle_queue(struct igc_softc *sc, struct igc_queue *iq) 2876 { 2877 2878 if (iq->igcq_workqueue) { 2879 /* XXXRO notyet */ 2880 workqueue_enqueue(sc->sc_queue_wq, &iq->igcq_wq_cookie, 2881 curcpu()); 2882 } else { 2883 softint_schedule(iq->igcq_si); 2884 } 2885 } 2886 2887 static void 2888 igc_barrier_handle_queue(struct igc_softc *sc) 2889 { 2890 2891 if (sc->sc_txrx_workqueue) { 2892 for (int iq = 0; iq < sc->sc_nqueues; iq++) { 2893 struct igc_queue *q = &sc->queues[iq]; 2894 2895 workqueue_wait(sc->sc_queue_wq, &q->igcq_wq_cookie); 2896 } 2897 } else { 2898 xc_barrier(0); 2899 } 2900 } 2901 2902 /********************************************************************* 2903 * 2904 * Allocate memory for tx_buffer structures. The tx_buffer stores all 2905 * the information needed to transmit a packet on the wire. 2906 * 2907 **********************************************************************/ 2908 static int 2909 igc_allocate_transmit_buffers(struct tx_ring *txr) 2910 { 2911 struct igc_softc *sc = txr->sc; 2912 int error; 2913 2914 txr->tx_buffers = 2915 kmem_zalloc(sc->num_tx_desc * sizeof(struct igc_tx_buf), KM_SLEEP); 2916 txr->txtag = txr->txdma.dma_tag; 2917 2918 /* Create the descriptor buffer dma maps. */ 2919 for (int id = 0; id < sc->num_tx_desc; id++) { 2920 struct igc_tx_buf *txbuf = &txr->tx_buffers[id]; 2921 2922 error = bus_dmamap_create(txr->txdma.dma_tag, 2923 round_page(IGC_TSO_SIZE + sizeof(struct ether_vlan_header)), 2924 IGC_MAX_SCATTER, PAGE_SIZE, 0, BUS_DMA_NOWAIT, &txbuf->map); 2925 if (error != 0) { 2926 aprint_error_dev(sc->sc_dev, 2927 "unable to create TX DMA map\n"); 2928 goto fail; 2929 } 2930 2931 txbuf->eop_index = -1; 2932 } 2933 2934 return 0; 2935 fail: 2936 return error; 2937 } 2938 2939 2940 /********************************************************************* 2941 * 2942 * Allocate and initialize transmit structures. 2943 * 2944 **********************************************************************/ 2945 static int 2946 igc_setup_transmit_structures(struct igc_softc *sc) 2947 { 2948 2949 for (int iq = 0; iq < sc->sc_nqueues; iq++) { 2950 struct tx_ring *txr = &sc->tx_rings[iq]; 2951 2952 if (igc_setup_transmit_ring(txr)) 2953 goto fail; 2954 } 2955 2956 return 0; 2957 fail: 2958 igc_free_transmit_structures(sc); 2959 return ENOBUFS; 2960 } 2961 2962 /********************************************************************* 2963 * 2964 * Initialize a transmit ring. 2965 * 2966 **********************************************************************/ 2967 static int 2968 igc_setup_transmit_ring(struct tx_ring *txr) 2969 { 2970 struct igc_softc *sc = txr->sc; 2971 2972 /* Now allocate transmit buffers for the ring. */ 2973 if (igc_allocate_transmit_buffers(txr)) 2974 return ENOMEM; 2975 2976 /* Clear the old ring contents */ 2977 memset(txr->tx_base, 0, 2978 sizeof(union igc_adv_tx_desc) * sc->num_tx_desc); 2979 2980 /* Reset indices. */ 2981 txr->next_avail_desc = 0; 2982 txr->next_to_clean = 0; 2983 2984 bus_dmamap_sync(txr->txdma.dma_tag, txr->txdma.dma_map, 0, 2985 txr->txdma.dma_map->dm_mapsize, 2986 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); 2987 2988 txr->txr_interq = pcq_create(sc->num_tx_desc, KM_SLEEP); 2989 2990 mutex_init(&txr->txr_lock, MUTEX_DEFAULT, IPL_NET); 2991 2992 return 0; 2993 } 2994 2995 /********************************************************************* 2996 * 2997 * Enable transmit unit. 2998 * 2999 **********************************************************************/ 3000 static void 3001 igc_initialize_transmit_unit(struct igc_softc *sc) 3002 { 3003 struct ifnet *ifp = &sc->sc_ec.ec_if; 3004 struct igc_hw *hw = &sc->hw; 3005 3006 /* Setup the Base and Length of the TX descriptor ring. */ 3007 for (int iq = 0; iq < sc->sc_nqueues; iq++) { 3008 struct tx_ring *txr = &sc->tx_rings[iq]; 3009 const uint64_t bus_addr = 3010 txr->txdma.dma_map->dm_segs[0].ds_addr; 3011 3012 /* Base and len of TX ring */ 3013 IGC_WRITE_REG(hw, IGC_TDLEN(iq), 3014 sc->num_tx_desc * sizeof(union igc_adv_tx_desc)); 3015 IGC_WRITE_REG(hw, IGC_TDBAH(iq), (uint32_t)(bus_addr >> 32)); 3016 IGC_WRITE_REG(hw, IGC_TDBAL(iq), (uint32_t)bus_addr); 3017 3018 /* Init the HEAD/TAIL indices */ 3019 IGC_WRITE_REG(hw, IGC_TDT(iq), 0 /* XXX txr->next_avail_desc */); 3020 IGC_WRITE_REG(hw, IGC_TDH(iq), 0); 3021 3022 txr->watchdog_timer = 0; 3023 3024 uint32_t txdctl = 0; /* Clear txdctl */ 3025 txdctl |= 0x1f; /* PTHRESH */ 3026 txdctl |= 1 << 8; /* HTHRESH */ 3027 txdctl |= 1 << 16; /* WTHRESH */ 3028 txdctl |= 1 << 22; /* Reserved bit 22 must always be 1 */ 3029 txdctl |= IGC_TXDCTL_GRAN; 3030 txdctl |= 1 << 25; /* LWTHRESH */ 3031 3032 IGC_WRITE_REG(hw, IGC_TXDCTL(iq), txdctl); 3033 } 3034 ifp->if_timer = 0; 3035 3036 /* Program the Transmit Control Register */ 3037 uint32_t tctl = IGC_READ_REG(&sc->hw, IGC_TCTL); 3038 tctl &= ~IGC_TCTL_CT; 3039 tctl |= (IGC_TCTL_PSP | IGC_TCTL_RTLC | IGC_TCTL_EN | 3040 (IGC_COLLISION_THRESHOLD << IGC_CT_SHIFT)); 3041 3042 /* This write will effectively turn on the transmit unit. */ 3043 IGC_WRITE_REG(&sc->hw, IGC_TCTL, tctl); 3044 } 3045 3046 /********************************************************************* 3047 * 3048 * Free all transmit rings. 3049 * 3050 **********************************************************************/ 3051 static void 3052 igc_free_transmit_structures(struct igc_softc *sc) 3053 { 3054 3055 for (int iq = 0; iq < sc->sc_nqueues; iq++) { 3056 struct tx_ring *txr = &sc->tx_rings[iq]; 3057 3058 igc_free_transmit_buffers(txr); 3059 } 3060 } 3061 3062 /********************************************************************* 3063 * 3064 * Free transmit ring related data structures. 3065 * 3066 **********************************************************************/ 3067 static void 3068 igc_free_transmit_buffers(struct tx_ring *txr) 3069 { 3070 struct igc_softc *sc = txr->sc; 3071 3072 if (txr->tx_buffers == NULL) 3073 return; 3074 3075 igc_withdraw_transmit_packets(txr, true); 3076 3077 kmem_free(txr->tx_buffers, 3078 sc->num_tx_desc * sizeof(struct igc_tx_buf)); 3079 txr->tx_buffers = NULL; 3080 txr->txtag = NULL; 3081 3082 pcq_destroy(txr->txr_interq); 3083 mutex_destroy(&txr->txr_lock); 3084 } 3085 3086 /********************************************************************* 3087 * 3088 * Withdraw transmit packets. 3089 * 3090 **********************************************************************/ 3091 static void 3092 igc_withdraw_transmit_packets(struct tx_ring *txr, bool destroy) 3093 { 3094 struct igc_softc *sc = txr->sc; 3095 struct igc_queue *q = txr->txr_igcq; 3096 3097 mutex_enter(&txr->txr_lock); 3098 3099 for (int id = 0; id < sc->num_tx_desc; id++) { 3100 union igc_adv_tx_desc *txdesc = &txr->tx_base[id]; 3101 3102 igc_txdesc_sync(txr, id, 3103 BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE); 3104 txdesc->read.buffer_addr = 0; 3105 txdesc->read.cmd_type_len = 0; 3106 txdesc->read.olinfo_status = 0; 3107 igc_txdesc_sync(txr, id, 3108 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); 3109 3110 struct igc_tx_buf *txbuf = &txr->tx_buffers[id]; 3111 bus_dmamap_t map = txbuf->map; 3112 3113 if (map != NULL && map->dm_nsegs > 0) { 3114 bus_dmamap_sync(txr->txdma.dma_tag, map, 3115 0, map->dm_mapsize, BUS_DMASYNC_POSTWRITE); 3116 bus_dmamap_unload(txr->txdma.dma_tag, map); 3117 } 3118 if (txbuf->m_head != NULL) { 3119 m_freem(txbuf->m_head); 3120 txbuf->m_head = NULL; 3121 } 3122 if (map != NULL && destroy) { 3123 bus_dmamap_destroy(txr->txdma.dma_tag, map); 3124 txbuf->map = NULL; 3125 } 3126 txbuf->eop_index = -1; 3127 3128 txr->next_avail_desc = 0; 3129 txr->next_to_clean = 0; 3130 } 3131 3132 struct mbuf *m; 3133 while ((m = pcq_get(txr->txr_interq)) != NULL) { 3134 IGC_QUEUE_EVENT(q, tx_pcq_drop, 1); 3135 m_freem(m); 3136 } 3137 3138 mutex_exit(&txr->txr_lock); 3139 } 3140 3141 3142 /********************************************************************* 3143 * 3144 * Advanced Context Descriptor setup for VLAN, CSUM or TSO 3145 * 3146 **********************************************************************/ 3147 3148 static int 3149 igc_tx_ctx_setup(struct tx_ring *txr, struct mbuf *mp, int prod, 3150 uint32_t *cmd_type_len, uint32_t *olinfo_status) 3151 { 3152 struct ether_vlan_header *evl; 3153 uint32_t type_tucmd_mlhl = 0; 3154 uint32_t vlan_macip_lens = 0; 3155 uint32_t ehlen, iphlen; 3156 uint16_t ehtype; 3157 int off = 0; 3158 3159 const int csum_flags = mp->m_pkthdr.csum_flags; 3160 3161 /* First check if TSO is to be used */ 3162 if ((csum_flags & (M_CSUM_TSOv4 | M_CSUM_TSOv6)) != 0) { 3163 return igc_tso_setup(txr, mp, prod, cmd_type_len, 3164 olinfo_status); 3165 } 3166 3167 const bool v4 = (csum_flags & 3168 (M_CSUM_IPv4 | M_CSUM_TCPv4 | M_CSUM_UDPv4)) != 0; 3169 const bool v6 = (csum_flags & (M_CSUM_UDPv6 | M_CSUM_TCPv6)) != 0; 3170 3171 /* Indicate the whole packet as payload when not doing TSO */ 3172 *olinfo_status |= mp->m_pkthdr.len << IGC_ADVTXD_PAYLEN_SHIFT; 3173 3174 /* 3175 * In advanced descriptors the vlan tag must 3176 * be placed into the context descriptor. Hence 3177 * we need to make one even if not doing offloads. 3178 */ 3179 #if NVLAN > 0 3180 if (vlan_has_tag(mp)) { 3181 vlan_macip_lens |= (uint32_t)vlan_get_tag(mp) 3182 << IGC_ADVTXD_VLAN_SHIFT; 3183 off = 1; 3184 } else 3185 #endif 3186 if (!v4 && !v6) 3187 return 0; 3188 3189 KASSERT(mp->m_len >= sizeof(struct ether_header)); 3190 evl = mtod(mp, struct ether_vlan_header *); 3191 if (evl->evl_encap_proto == htons(ETHERTYPE_VLAN)) { 3192 KASSERT(mp->m_len >= sizeof(struct ether_vlan_header)); 3193 ehlen = ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN; 3194 ehtype = evl->evl_proto; 3195 } else { 3196 ehlen = ETHER_HDR_LEN; 3197 ehtype = evl->evl_encap_proto; 3198 } 3199 3200 vlan_macip_lens |= ehlen << IGC_ADVTXD_MACLEN_SHIFT; 3201 3202 #ifdef IGC_DEBUG 3203 /* 3204 * For checksum offloading, L3 headers are not mandatory. 3205 * We use these only for consistency checks. 3206 */ 3207 struct ip *ip; 3208 struct ip6_hdr *ip6; 3209 uint8_t ipproto; 3210 char *l3d; 3211 3212 if (mp->m_len == ehlen && mp->m_next != NULL) 3213 l3d = mtod(mp->m_next, char *); 3214 else 3215 l3d = mtod(mp, char *) + ehlen; 3216 #endif 3217 3218 switch (ntohs(ehtype)) { 3219 case ETHERTYPE_IP: 3220 iphlen = M_CSUM_DATA_IPv4_IPHL(mp->m_pkthdr.csum_data); 3221 type_tucmd_mlhl |= IGC_ADVTXD_TUCMD_IPV4; 3222 3223 if ((csum_flags & M_CSUM_IPv4) != 0) { 3224 *olinfo_status |= IGC_TXD_POPTS_IXSM << 8; 3225 off = 1; 3226 } 3227 #ifdef IGC_DEBUG 3228 KASSERT(!v6); 3229 ip = (void *)l3d; 3230 ipproto = ip->ip_p; 3231 KASSERT(iphlen == ip->ip_hl << 2); 3232 KASSERT((mp->m_pkthdr.csum_flags & M_CSUM_IPv4) == 0 || 3233 ip->ip_sum == 0); 3234 #endif 3235 break; 3236 case ETHERTYPE_IPV6: 3237 iphlen = M_CSUM_DATA_IPv6_IPHL(mp->m_pkthdr.csum_data); 3238 type_tucmd_mlhl |= IGC_ADVTXD_TUCMD_IPV6; 3239 #ifdef IGC_DEBUG 3240 KASSERT(!v4); 3241 ip6 = (void *)l3d; 3242 ipproto = ip6->ip6_nxt; /* XXX */ 3243 KASSERT(iphlen == sizeof(struct ip6_hdr)); 3244 #endif 3245 break; 3246 default: 3247 /* 3248 * Unknown L3 protocol. Clear L3 header length and proceed for 3249 * LAN as done by Linux driver. 3250 */ 3251 iphlen = 0; 3252 #ifdef IGC_DEBUG 3253 KASSERT(!v4 && !v6); 3254 ipproto = 0; 3255 #endif 3256 break; 3257 } 3258 3259 vlan_macip_lens |= iphlen; 3260 3261 const bool tcp = (csum_flags & (M_CSUM_TCPv4 | M_CSUM_TCPv6)) != 0; 3262 const bool udp = (csum_flags & (M_CSUM_UDPv4 | M_CSUM_UDPv6)) != 0; 3263 3264 if (tcp) { 3265 #ifdef IGC_DEBUG 3266 KASSERTMSG(ipproto == IPPROTO_TCP, "ipproto = %d", ipproto); 3267 #endif 3268 type_tucmd_mlhl |= IGC_ADVTXD_TUCMD_L4T_TCP; 3269 *olinfo_status |= IGC_TXD_POPTS_TXSM << 8; 3270 off = 1; 3271 } else if (udp) { 3272 #ifdef IGC_DEBUG 3273 KASSERTMSG(ipproto == IPPROTO_UDP, "ipproto = %d", ipproto); 3274 #endif 3275 type_tucmd_mlhl |= IGC_ADVTXD_TUCMD_L4T_UDP; 3276 *olinfo_status |= IGC_TXD_POPTS_TXSM << 8; 3277 off = 1; 3278 } 3279 3280 if (off == 0) 3281 return 0; 3282 3283 type_tucmd_mlhl |= IGC_ADVTXD_DCMD_DEXT | IGC_ADVTXD_DTYP_CTXT; 3284 3285 /* Now ready a context descriptor */ 3286 struct igc_adv_tx_context_desc *txdesc = 3287 (struct igc_adv_tx_context_desc *)&txr->tx_base[prod]; 3288 3289 /* Now copy bits into descriptor */ 3290 igc_txdesc_sync(txr, prod, 3291 BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE); 3292 htolem32(&txdesc->vlan_macip_lens, vlan_macip_lens); 3293 htolem32(&txdesc->type_tucmd_mlhl, type_tucmd_mlhl); 3294 htolem32(&txdesc->seqnum_seed, 0); 3295 htolem32(&txdesc->mss_l4len_idx, 0); 3296 igc_txdesc_sync(txr, prod, 3297 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); 3298 3299 return 1; 3300 } 3301 3302 /********************************************************************* 3303 * 3304 * Advanced Context Descriptor setup for TSO 3305 * 3306 * XXX XXXRO 3307 * Not working. Some packets are sent with correct csums, but 3308 * others aren't. th->th_sum may be adjusted. 3309 * 3310 **********************************************************************/ 3311 3312 static int 3313 igc_tso_setup(struct tx_ring *txr, struct mbuf *mp, int prod, 3314 uint32_t *cmd_type_len, uint32_t *olinfo_status) 3315 { 3316 #if 1 /* notyet */ 3317 return 0; 3318 #else 3319 struct ether_vlan_header *evl; 3320 struct ip *ip; 3321 struct ip6_hdr *ip6; 3322 struct tcphdr *th; 3323 uint32_t type_tucmd_mlhl = 0; 3324 uint32_t vlan_macip_lens = 0; 3325 uint32_t mss_l4len_idx = 0; 3326 uint32_t ehlen, iphlen, tcphlen, paylen; 3327 uint16_t ehtype; 3328 3329 /* 3330 * In advanced descriptors the vlan tag must 3331 * be placed into the context descriptor. Hence 3332 * we need to make one even if not doing offloads. 3333 */ 3334 #if NVLAN > 0 3335 if (vlan_has_tag(mp)) { 3336 vlan_macip_lens |= (uint32_t)vlan_get_tag(mp) 3337 << IGC_ADVTXD_VLAN_SHIFT; 3338 } 3339 #endif 3340 3341 KASSERT(mp->m_len >= sizeof(struct ether_header)); 3342 evl = mtod(mp, struct ether_vlan_header *); 3343 if (evl->evl_encap_proto == htons(ETHERTYPE_VLAN)) { 3344 KASSERT(mp->m_len >= sizeof(struct ether_vlan_header)); 3345 ehlen = ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN; 3346 ehtype = evl->evl_proto; 3347 } else { 3348 ehlen = ETHER_HDR_LEN; 3349 ehtype = evl->evl_encap_proto; 3350 } 3351 3352 vlan_macip_lens |= ehlen << IGC_ADVTXD_MACLEN_SHIFT; 3353 3354 switch (ntohs(ehtype)) { 3355 case ETHERTYPE_IP: 3356 iphlen = M_CSUM_DATA_IPv4_IPHL(mp->m_pkthdr.csum_data); 3357 type_tucmd_mlhl |= IGC_ADVTXD_TUCMD_IPV4; 3358 *olinfo_status |= IGC_TXD_POPTS_IXSM << 8; 3359 3360 KASSERT(mp->m_len >= ehlen + sizeof(*ip)); 3361 ip = (void *)(mtod(mp, char *) + ehlen); 3362 ip->ip_len = 0; 3363 KASSERT(iphlen == ip->ip_hl << 2); 3364 KASSERT(ip->ip_sum == 0); 3365 KASSERT(ip->ip_p == IPPROTO_TCP); 3366 3367 KASSERT(mp->m_len >= ehlen + iphlen + sizeof(*th)); 3368 th = (void *)((char *)ip + iphlen); 3369 th->th_sum = in_cksum_phdr(ip->ip_src.s_addr, ip->ip_dst.s_addr, 3370 htons(IPPROTO_TCP)); 3371 break; 3372 case ETHERTYPE_IPV6: 3373 iphlen = M_CSUM_DATA_IPv6_IPHL(mp->m_pkthdr.csum_data); 3374 type_tucmd_mlhl |= IGC_ADVTXD_TUCMD_IPV6; 3375 3376 KASSERT(mp->m_len >= ehlen + sizeof(*ip6)); 3377 ip6 = (void *)(mtod(mp, char *) + ehlen); 3378 ip6->ip6_plen = 0; 3379 KASSERT(iphlen == sizeof(struct ip6_hdr)); 3380 KASSERT(ip6->ip6_nxt == IPPROTO_TCP); 3381 3382 KASSERT(mp->m_len >= ehlen + iphlen + sizeof(*th)); 3383 th = (void *)((char *)ip6 + iphlen); 3384 tcphlen = th->th_off << 2; 3385 paylen = mp->m_pkthdr.len - ehlen - iphlen - tcphlen; 3386 th->th_sum = in6_cksum_phdr(&ip6->ip6_src, &ip6->ip6_dst, 0, 3387 htonl(IPPROTO_TCP)); 3388 break; 3389 default: 3390 panic("%s", __func__); 3391 } 3392 3393 tcphlen = th->th_off << 2; 3394 paylen = mp->m_pkthdr.len - ehlen - iphlen - tcphlen; 3395 3396 vlan_macip_lens |= iphlen; 3397 3398 type_tucmd_mlhl |= IGC_ADVTXD_DCMD_DEXT | IGC_ADVTXD_DTYP_CTXT; 3399 type_tucmd_mlhl |= IGC_ADVTXD_TUCMD_L4T_TCP; 3400 3401 mss_l4len_idx |= mp->m_pkthdr.segsz << IGC_ADVTXD_MSS_SHIFT; 3402 mss_l4len_idx |= tcphlen << IGC_ADVTXD_L4LEN_SHIFT; 3403 3404 /* Now ready a context descriptor */ 3405 struct igc_adv_tx_context_desc *txdesc = 3406 (struct igc_adv_tx_context_desc *)&txr->tx_base[prod]; 3407 3408 /* Now copy bits into descriptor */ 3409 igc_txdesc_sync(txr, prod, 3410 BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE); 3411 htolem32(&txdesc->vlan_macip_lens, vlan_macip_lens); 3412 htolem32(&txdesc->type_tucmd_mlhl, type_tucmd_mlhl); 3413 htolem32(&txdesc->seqnum_seed, 0); 3414 htolem32(&txdesc->mss_l4len_idx, mss_l4len_idx); 3415 igc_txdesc_sync(txr, prod, 3416 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); 3417 3418 *cmd_type_len |= IGC_ADVTXD_DCMD_TSE; 3419 *olinfo_status |= IGC_TXD_POPTS_TXSM << 8; 3420 *olinfo_status |= paylen << IGC_ADVTXD_PAYLEN_SHIFT; 3421 3422 return 1; 3423 #endif /* notyet */ 3424 } 3425 3426 /********************************************************************* 3427 * 3428 * Allocate memory for rx_buffer structures. Since we use one 3429 * rx_buffer per received packet, the maximum number of rx_buffer's 3430 * that we'll need is equal to the number of receive descriptors 3431 * that we've allocated. 3432 * 3433 **********************************************************************/ 3434 static int 3435 igc_allocate_receive_buffers(struct rx_ring *rxr) 3436 { 3437 struct igc_softc *sc = rxr->sc; 3438 int error; 3439 3440 rxr->rx_buffers = 3441 kmem_zalloc(sc->num_rx_desc * sizeof(struct igc_rx_buf), KM_SLEEP); 3442 3443 for (int id = 0; id < sc->num_rx_desc; id++) { 3444 struct igc_rx_buf *rxbuf = &rxr->rx_buffers[id]; 3445 3446 error = bus_dmamap_create(rxr->rxdma.dma_tag, MCLBYTES, 1, 3447 MCLBYTES, 0, BUS_DMA_WAITOK, &rxbuf->map); 3448 if (error) { 3449 aprint_error_dev(sc->sc_dev, 3450 "unable to create RX DMA map\n"); 3451 goto fail; 3452 } 3453 } 3454 bus_dmamap_sync(rxr->rxdma.dma_tag, rxr->rxdma.dma_map, 0, 3455 rxr->rxdma.dma_map->dm_mapsize, 3456 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); 3457 3458 return 0; 3459 fail: 3460 return error; 3461 } 3462 3463 /********************************************************************* 3464 * 3465 * Allocate and initialize receive structures. 3466 * 3467 **********************************************************************/ 3468 static int 3469 igc_setup_receive_structures(struct igc_softc *sc) 3470 { 3471 3472 for (int iq = 0; iq < sc->sc_nqueues; iq++) { 3473 struct rx_ring *rxr = &sc->rx_rings[iq]; 3474 3475 if (igc_setup_receive_ring(rxr)) 3476 goto fail; 3477 } 3478 3479 return 0; 3480 fail: 3481 igc_free_receive_structures(sc); 3482 return ENOBUFS; 3483 } 3484 3485 /********************************************************************* 3486 * 3487 * Initialize a receive ring and its buffers. 3488 * 3489 **********************************************************************/ 3490 static int 3491 igc_setup_receive_ring(struct rx_ring *rxr) 3492 { 3493 struct igc_softc *sc = rxr->sc; 3494 const int rsize = roundup2( 3495 sc->num_rx_desc * sizeof(union igc_adv_rx_desc), IGC_DBA_ALIGN); 3496 3497 /* Clear the ring contents. */ 3498 memset(rxr->rx_base, 0, rsize); 3499 3500 if (igc_allocate_receive_buffers(rxr)) 3501 return ENOMEM; 3502 3503 /* Setup our descriptor indices. */ 3504 rxr->next_to_check = 0; 3505 rxr->last_desc_filled = 0; 3506 3507 mutex_init(&rxr->rxr_lock, MUTEX_DEFAULT, IPL_NET); 3508 3509 return 0; 3510 } 3511 3512 /********************************************************************* 3513 * 3514 * Enable receive unit. 3515 * 3516 **********************************************************************/ 3517 static void 3518 igc_initialize_receive_unit(struct igc_softc *sc) 3519 { 3520 struct ifnet *ifp = &sc->sc_ec.ec_if; 3521 struct igc_hw *hw = &sc->hw; 3522 uint32_t rctl, rxcsum, srrctl; 3523 3524 DPRINTF(RX, "called\n"); 3525 3526 /* 3527 * Make sure receives are disabled while setting 3528 * up the descriptor ring. 3529 */ 3530 rctl = IGC_READ_REG(hw, IGC_RCTL); 3531 IGC_WRITE_REG(hw, IGC_RCTL, rctl & ~IGC_RCTL_EN); 3532 3533 /* Setup the Receive Control Register */ 3534 rctl &= ~(3 << IGC_RCTL_MO_SHIFT); 3535 rctl |= IGC_RCTL_EN | IGC_RCTL_BAM | IGC_RCTL_LBM_NO | 3536 IGC_RCTL_RDMTS_HALF | (hw->mac.mc_filter_type << IGC_RCTL_MO_SHIFT); 3537 3538 #if 1 3539 /* Do not store bad packets */ 3540 rctl &= ~IGC_RCTL_SBP; 3541 #else 3542 /* for debug */ 3543 rctl |= IGC_RCTL_SBP; 3544 #endif 3545 3546 /* Enable Long Packet receive */ 3547 if (sc->hw.mac.max_frame_size > ETHER_MAX_LEN) 3548 rctl |= IGC_RCTL_LPE; 3549 else 3550 rctl &= ~IGC_RCTL_LPE; 3551 3552 /* Strip the CRC */ 3553 rctl |= IGC_RCTL_SECRC; 3554 3555 /* 3556 * Set the interrupt throttling rate. Value is calculated 3557 * as DEFAULT_ITR = 1/(MAX_INTS_PER_SEC * 256ns) 3558 * 3559 * XXX Sync with Linux, especially for jumbo MTU or TSO. 3560 * XXX Shouldn't be here? 3561 */ 3562 IGC_WRITE_REG(hw, IGC_ITR, DEFAULT_ITR); 3563 3564 rxcsum = IGC_READ_REG(hw, IGC_RXCSUM); 3565 rxcsum &= ~(IGC_RXCSUM_IPOFL | IGC_RXCSUM_TUOFL | IGC_RXCSUM_PCSD); 3566 if (ifp->if_capenable & IFCAP_CSUM_IPv4_Rx) 3567 rxcsum |= IGC_RXCSUM_IPOFL; 3568 if (ifp->if_capenable & (IFCAP_CSUM_TCPv4_Rx | IFCAP_CSUM_UDPv4_Rx | 3569 IFCAP_CSUM_TCPv6_Rx | IFCAP_CSUM_UDPv6_Rx)) 3570 rxcsum |= IGC_RXCSUM_TUOFL; 3571 if (sc->sc_nqueues > 1) 3572 rxcsum |= IGC_RXCSUM_PCSD; 3573 IGC_WRITE_REG(hw, IGC_RXCSUM, rxcsum); 3574 3575 if (sc->sc_nqueues > 1) 3576 igc_initialize_rss_mapping(sc); 3577 3578 srrctl = 0; 3579 #if 0 3580 srrctl |= 4096 >> IGC_SRRCTL_BSIZEPKT_SHIFT; 3581 rctl |= IGC_RCTL_SZ_4096 | IGC_RCTL_BSEX; 3582 #else 3583 srrctl |= 2048 >> IGC_SRRCTL_BSIZEPKT_SHIFT; 3584 rctl |= IGC_RCTL_SZ_2048; 3585 #endif 3586 3587 /* 3588 * If TX flow control is disabled and there's > 1 queue defined, 3589 * enable DROP. 3590 * 3591 * This drops frames rather than hanging the RX MAC for all queues. 3592 */ 3593 if (sc->sc_nqueues > 1 && 3594 (sc->fc == igc_fc_none || sc->fc == igc_fc_rx_pause)) 3595 srrctl |= IGC_SRRCTL_DROP_EN; 3596 3597 /* Setup the Base and Length of the RX descriptor rings. */ 3598 for (int iq = 0; iq < sc->sc_nqueues; iq++) { 3599 struct rx_ring *rxr = &sc->rx_rings[iq]; 3600 const uint64_t bus_addr = 3601 rxr->rxdma.dma_map->dm_segs[0].ds_addr; 3602 3603 IGC_WRITE_REG(hw, IGC_RXDCTL(iq), 0); 3604 3605 srrctl |= IGC_SRRCTL_DESCTYPE_ADV_ONEBUF; 3606 3607 IGC_WRITE_REG(hw, IGC_RDLEN(iq), 3608 sc->num_rx_desc * sizeof(union igc_adv_rx_desc)); 3609 IGC_WRITE_REG(hw, IGC_RDBAH(iq), (uint32_t)(bus_addr >> 32)); 3610 IGC_WRITE_REG(hw, IGC_RDBAL(iq), (uint32_t)bus_addr); 3611 IGC_WRITE_REG(hw, IGC_SRRCTL(iq), srrctl); 3612 3613 /* Setup the Head and Tail Descriptor Pointers */ 3614 IGC_WRITE_REG(hw, IGC_RDH(iq), 0); 3615 IGC_WRITE_REG(hw, IGC_RDT(iq), 0 /* XXX rxr->last_desc_filled */); 3616 3617 /* Enable this Queue */ 3618 uint32_t rxdctl = IGC_READ_REG(hw, IGC_RXDCTL(iq)); 3619 rxdctl |= IGC_RXDCTL_QUEUE_ENABLE; 3620 rxdctl &= 0xFFF00000; 3621 rxdctl |= IGC_RX_PTHRESH; 3622 rxdctl |= IGC_RX_HTHRESH << 8; 3623 rxdctl |= IGC_RX_WTHRESH << 16; 3624 IGC_WRITE_REG(hw, IGC_RXDCTL(iq), rxdctl); 3625 } 3626 3627 /* Make sure VLAN Filters are off */ 3628 rctl &= ~IGC_RCTL_VFE; 3629 3630 /* Write out the settings */ 3631 IGC_WRITE_REG(hw, IGC_RCTL, rctl); 3632 } 3633 3634 /********************************************************************* 3635 * 3636 * Free all receive rings. 3637 * 3638 **********************************************************************/ 3639 static void 3640 igc_free_receive_structures(struct igc_softc *sc) 3641 { 3642 3643 for (int iq = 0; iq < sc->sc_nqueues; iq++) { 3644 struct rx_ring *rxr = &sc->rx_rings[iq]; 3645 3646 igc_free_receive_buffers(rxr); 3647 } 3648 } 3649 3650 /********************************************************************* 3651 * 3652 * Free receive ring data structures 3653 * 3654 **********************************************************************/ 3655 static void 3656 igc_free_receive_buffers(struct rx_ring *rxr) 3657 { 3658 struct igc_softc *sc = rxr->sc; 3659 3660 if (rxr->rx_buffers != NULL) { 3661 for (int id = 0; id < sc->num_rx_desc; id++) { 3662 struct igc_rx_buf *rxbuf = &rxr->rx_buffers[id]; 3663 bus_dmamap_t map = rxbuf->map; 3664 3665 if (rxbuf->buf != NULL) { 3666 bus_dmamap_sync(rxr->rxdma.dma_tag, map, 3667 0, map->dm_mapsize, BUS_DMASYNC_POSTREAD); 3668 bus_dmamap_unload(rxr->rxdma.dma_tag, map); 3669 m_freem(rxbuf->buf); 3670 rxbuf->buf = NULL; 3671 } 3672 bus_dmamap_destroy(rxr->rxdma.dma_tag, map); 3673 rxbuf->map = NULL; 3674 } 3675 kmem_free(rxr->rx_buffers, 3676 sc->num_rx_desc * sizeof(struct igc_rx_buf)); 3677 rxr->rx_buffers = NULL; 3678 } 3679 3680 mutex_destroy(&rxr->rxr_lock); 3681 } 3682 3683 /********************************************************************* 3684 * 3685 * Clear status registers in all RX descriptors. 3686 * 3687 **********************************************************************/ 3688 static void 3689 igc_clear_receive_status(struct rx_ring *rxr) 3690 { 3691 struct igc_softc *sc = rxr->sc; 3692 3693 mutex_enter(&rxr->rxr_lock); 3694 3695 for (int id = 0; id < sc->num_rx_desc; id++) { 3696 union igc_adv_rx_desc *rxdesc = &rxr->rx_base[id]; 3697 3698 igc_rxdesc_sync(rxr, id, 3699 BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE); 3700 rxdesc->wb.upper.status_error = 0; 3701 igc_rxdesc_sync(rxr, id, 3702 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); 3703 } 3704 3705 mutex_exit(&rxr->rxr_lock); 3706 } 3707 3708 /* 3709 * Initialise the RSS mapping for NICs that support multiple transmit/ 3710 * receive rings. 3711 */ 3712 static void 3713 igc_initialize_rss_mapping(struct igc_softc *sc) 3714 { 3715 struct igc_hw *hw = &sc->hw; 3716 3717 /* 3718 * The redirection table controls which destination 3719 * queue each bucket redirects traffic to. 3720 * Each DWORD represents four queues, with the LSB 3721 * being the first queue in the DWORD. 3722 * 3723 * This just allocates buckets to queues using round-robin 3724 * allocation. 3725 * 3726 * NOTE: It Just Happens to line up with the default 3727 * RSS allocation method. 3728 */ 3729 3730 /* Warning FM follows */ 3731 uint32_t reta = 0; 3732 for (int i = 0; i < 128; i++) { 3733 const int shift = 0; /* XXXRO */ 3734 int queue_id = i % sc->sc_nqueues; 3735 /* Adjust if required */ 3736 queue_id <<= shift; 3737 3738 /* 3739 * The low 8 bits are for hash value (n+0); 3740 * The next 8 bits are for hash value (n+1), etc. 3741 */ 3742 reta >>= 8; 3743 reta |= ((uint32_t)queue_id) << 24; 3744 if ((i & 3) == 3) { 3745 IGC_WRITE_REG(hw, IGC_RETA(i >> 2), reta); 3746 reta = 0; 3747 } 3748 } 3749 3750 /* 3751 * MRQC: Multiple Receive Queues Command 3752 * Set queuing to RSS control, number depends on the device. 3753 */ 3754 3755 /* Set up random bits */ 3756 uint32_t rss_key[RSS_KEYSIZE / sizeof(uint32_t)]; 3757 rss_getkey((uint8_t *)rss_key); 3758 3759 /* Now fill our hash function seeds */ 3760 for (int i = 0; i < __arraycount(rss_key); i++) 3761 IGC_WRITE_REG_ARRAY(hw, IGC_RSSRK(0), i, rss_key[i]); 3762 3763 /* 3764 * Configure the RSS fields to hash upon. 3765 */ 3766 uint32_t mrqc = IGC_MRQC_ENABLE_RSS_4Q; 3767 mrqc |= IGC_MRQC_RSS_FIELD_IPV4 | IGC_MRQC_RSS_FIELD_IPV4_TCP; 3768 mrqc |= IGC_MRQC_RSS_FIELD_IPV6 | IGC_MRQC_RSS_FIELD_IPV6_TCP; 3769 mrqc |= IGC_MRQC_RSS_FIELD_IPV6_TCP_EX; 3770 3771 IGC_WRITE_REG(hw, IGC_MRQC, mrqc); 3772 } 3773 3774 /* 3775 * igc_get_hw_control sets the {CTRL_EXT|FWSM}:DRV_LOAD bit. 3776 * For ASF and Pass Through versions of f/w this means 3777 * that the driver is loaded. For AMT version type f/w 3778 * this means that the network i/f is open. 3779 */ 3780 static void 3781 igc_get_hw_control(struct igc_softc *sc) 3782 { 3783 const uint32_t ctrl_ext = IGC_READ_REG(&sc->hw, IGC_CTRL_EXT); 3784 3785 IGC_WRITE_REG(&sc->hw, IGC_CTRL_EXT, ctrl_ext | IGC_CTRL_EXT_DRV_LOAD); 3786 } 3787 3788 /* 3789 * igc_release_hw_control resets {CTRL_EXT|FWSM}:DRV_LOAD bit. 3790 * For ASF and Pass Through versions of f/w this means that 3791 * the driver is no longer loaded. For AMT versions of the 3792 * f/w this means that the network i/f is closed. 3793 */ 3794 static void 3795 igc_release_hw_control(struct igc_softc *sc) 3796 { 3797 const uint32_t ctrl_ext = IGC_READ_REG(&sc->hw, IGC_CTRL_EXT); 3798 3799 IGC_WRITE_REG(&sc->hw, IGC_CTRL_EXT, ctrl_ext & ~IGC_CTRL_EXT_DRV_LOAD); 3800 } 3801 3802 static int 3803 igc_is_valid_ether_addr(uint8_t *addr) 3804 { 3805 const char zero_addr[6] = { 0, 0, 0, 0, 0, 0 }; 3806 3807 if ((addr[0] & 1) || !bcmp(addr, zero_addr, ETHER_ADDR_LEN)) 3808 return 0; 3809 3810 return 1; 3811 } 3812 3813 static void 3814 igc_print_devinfo(struct igc_softc *sc) 3815 { 3816 device_t dev = sc->sc_dev; 3817 struct igc_hw *hw = &sc->hw; 3818 struct igc_phy_info *phy = &hw->phy; 3819 u_int oui, model, rev; 3820 uint16_t id1, id2, nvm_ver, phy_ver; 3821 char descr[MII_MAX_DESCR_LEN]; 3822 3823 /* Print PHY Info */ 3824 id1 = phy->id >> 16; 3825 /* The revision field in phy->id is cleard and it's in phy->revision */ 3826 id2 = (phy->id & 0xfff0) | phy->revision; 3827 oui = MII_OUI(id1, id2); 3828 model = MII_MODEL(id2); 3829 rev = MII_REV(id2); 3830 mii_get_descr(descr, sizeof(descr), oui, model); 3831 if (descr[0]) 3832 aprint_normal_dev(dev, "PHY: %s, rev. %d\n", 3833 descr, rev); 3834 else 3835 aprint_normal_dev(dev, 3836 "PHY OUI 0x%06x, model 0x%04x, rev. %d\n", 3837 oui, model, rev); 3838 3839 /* Get NVM version */ 3840 hw->nvm.ops.read(hw, NVM_VERSION, 1, &nvm_ver); 3841 3842 /* Get PHY FW version */ 3843 phy->ops.read_reg(hw, 0x1e, &phy_ver); 3844 3845 aprint_normal_dev(dev, "ROM image version %x.%02x", 3846 (nvm_ver & NVM_VERSION_MAJOR) >> NVM_VERSION_MAJOR_SHIFT, 3847 (nvm_ver & NVM_VERSION_MINOR)); 3848 aprint_debug("(0x%04hx)", nvm_ver); 3849 3850 aprint_normal(", PHY FW version 0x%04hx\n", phy_ver); 3851 } 3852
Indexes created Tue Sep 30 11:09:46 GMT 2025