if_igc.c revision 1.8
1 /* $NetBSD: if_igc.c,v 1.8 2023/11/02 09:29:30 rin Exp $ */ 2 /* $OpenBSD: if_igc.c,v 1.13 2023/04/28 10:18:57 bluhm Exp $ */ 3 /*- 4 * SPDX-License-Identifier: BSD-2-Clause 5 * 6 * Copyright (c) 2016 Nicole Graziano <nicole (at) nextbsd.org> 7 * All rights reserved. 8 * Copyright (c) 2021 Rubicon Communications, LLC (Netgate) 9 * 10 * Redistribution and use in source and binary forms, with or without 11 * modification, are permitted provided that the following conditions 12 * are met: 13 * 1. Redistributions of source code must retain the above copyright 14 * notice, this list of conditions and the following disclaimer. 15 * 2. Redistributions in binary form must reproduce the above copyright 16 * notice, this list of conditions and the following disclaimer in the 17 * documentation and/or other materials provided with the distribution. 18 * 19 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 20 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 21 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 22 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 23 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 24 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 25 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 26 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 27 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 28 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 29 * SUCH DAMAGE. 30 */ 31 32 #include <sys/cdefs.h> 33 __KERNEL_RCSID(0, "$NetBSD: if_igc.c,v 1.8 2023/11/02 09:29:30 rin Exp $"); 34 35 #ifdef _KERNEL_OPT 36 #include "opt_net_mpsafe.h" 37 #include "opt_if_igc.h" 38 #if 0 /* notyet */ 39 #include "vlan.h" 40 #endif 41 #endif 42 43 #include <sys/param.h> 44 #include <sys/systm.h> 45 #include <sys/bus.h> 46 #include <sys/cpu.h> 47 #include <sys/device.h> 48 #include <sys/endian.h> 49 #include <sys/intr.h> 50 #include <sys/interrupt.h> 51 #include <sys/kernel.h> 52 #include <sys/kmem.h> 53 #include <sys/mbuf.h> 54 #include <sys/mutex.h> 55 #include <sys/socket.h> 56 #include <sys/workqueue.h> 57 #include <sys/xcall.h> 58 59 #include <net/bpf.h> 60 #include <net/if.h> 61 #include <net/if_ether.h> 62 #include <net/if_media.h> 63 #include <net/if_vlanvar.h> 64 #include <net/rss_config.h> 65 66 #include <netinet/in.h> 67 #include <netinet/ip.h> 68 #include <netinet/ip6.h> 69 #include <netinet/tcp.h> 70 71 #include <dev/pci/pcivar.h> 72 #include <dev/pci/pcireg.h> 73 #include <dev/pci/pcidevs.h> 74 75 #include <dev/pci/igc/if_igc.h> 76 #include <dev/pci/igc/igc_evcnt.h> 77 #include <dev/pci/igc/igc_hw.h> 78 #include <dev/mii/miivar.h> 79 80 #define IGC_WORKQUEUE_PRI PRI_SOFTNET 81 82 #ifndef IGC_RX_INTR_PROCESS_LIMIT_DEFAULT 83 #define IGC_RX_INTR_PROCESS_LIMIT_DEFAULT 0 84 #endif 85 #ifndef IGC_TX_INTR_PROCESS_LIMIT_DEFAULT 86 #define IGC_TX_INTR_PROCESS_LIMIT_DEFAULT 0 87 #endif 88 89 #ifndef IGC_RX_PROCESS_LIMIT_DEFAULT 90 #define IGC_RX_PROCESS_LIMIT_DEFAULT 256 91 #endif 92 #ifndef IGC_TX_PROCESS_LIMIT_DEFAULT 93 #define IGC_TX_PROCESS_LIMIT_DEFAULT 256 94 #endif 95 96 #define htolem32(p, x) (*((uint32_t *)(p)) = htole32(x)) 97 #define htolem64(p, x) (*((uint64_t *)(p)) = htole64(x)) 98 99 static const struct igc_product { 100 pci_vendor_id_t igcp_vendor; 101 pci_product_id_t igcp_product; 102 const char *igcp_name; 103 } igc_products[] = { 104 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_I225_IT, 105 "Intel(R) Ethernet Controller I225-IT(2)" }, 106 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_I226_LM, 107 "Intel(R) Ethernet Controller I226-LM" }, 108 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_I226_V, 109 "Intel(R) Ethernet Controller I226-V" }, 110 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_I226_IT, 111 "Intel(R) Ethernet Controller I226-IT" }, 112 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_I221_V, 113 "Intel(R) Ethernet Controller I221-V" }, 114 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_I226_BLANK_NVM, 115 "Intel(R) Ethernet Controller I226(blankNVM)" }, 116 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_I225_LM, 117 "Intel(R) Ethernet Controller I225-LM" }, 118 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_I225_V, 119 "Intel(R) Ethernet Controller I225-V" }, 120 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_I220_V, 121 "Intel(R) Ethernet Controller I220-V" }, 122 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_I225_I, 123 "Intel(R) Ethernet Controller I225-I" }, 124 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_I225_BLANK_NVM, 125 "Intel(R) Ethernet Controller I225(blankNVM)" }, 126 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_I225_K, 127 "Intel(R) Ethernet Controller I225-K" }, 128 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_I225_K2, 129 "Intel(R) Ethernet Controller I225-K(2)" }, 130 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_I226_K, 131 "Intel(R) Ethernet Controller I226-K" }, 132 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_I225_LMVP, 133 "Intel(R) Ethernet Controller I225-LMvP(2)" }, 134 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_I226_LMVP, 135 "Intel(R) Ethernet Controller I226-LMvP" }, 136 { 0, 0, NULL }, 137 }; 138 139 #define IGC_DF_CFG 0x1 140 #define IGC_DF_TX 0x2 141 #define IGC_DF_RX 0x4 142 #define IGC_DF_MISC 0x8 143 144 #ifdef IGC_DEBUG_FLAGS 145 int igc_debug_flags = IGC_DEBUG_FLAGS; 146 #else 147 int igc_debug_flags = 0; 148 #endif 149 150 #define DPRINTF(flag, fmt, args...) do { \ 151 if (igc_debug_flags & (IGC_DF_ ## flag)) \ 152 printf("%s: %d: " fmt, __func__, __LINE__, ##args); \ 153 } while (0) 154 155 /********************************************************************* 156 * Function Prototypes 157 *********************************************************************/ 158 static int igc_match(device_t, cfdata_t, void *); 159 static void igc_attach(device_t, device_t, void *); 160 static int igc_detach(device_t, int); 161 162 static void igc_identify_hardware(struct igc_softc *); 163 static int igc_adjust_nqueues(struct igc_softc *); 164 static int igc_allocate_pci_resources(struct igc_softc *); 165 static int igc_allocate_interrupts(struct igc_softc *); 166 static int igc_allocate_queues(struct igc_softc *); 167 static void igc_free_pci_resources(struct igc_softc *); 168 static void igc_free_interrupts(struct igc_softc *); 169 static void igc_free_queues(struct igc_softc *); 170 static void igc_reset(struct igc_softc *); 171 static void igc_init_dmac(struct igc_softc *, uint32_t); 172 static int igc_setup_interrupts(struct igc_softc *); 173 static void igc_attach_counters(struct igc_softc *sc); 174 static void igc_detach_counters(struct igc_softc *sc); 175 static void igc_update_counters(struct igc_softc *sc); 176 static void igc_clear_counters(struct igc_softc *sc); 177 static int igc_setup_msix(struct igc_softc *); 178 static int igc_setup_msi(struct igc_softc *); 179 static int igc_setup_intx(struct igc_softc *); 180 static int igc_dma_malloc(struct igc_softc *, bus_size_t, 181 struct igc_dma_alloc *); 182 static void igc_dma_free(struct igc_softc *, struct igc_dma_alloc *); 183 static void igc_setup_interface(struct igc_softc *); 184 185 static int igc_init(struct ifnet *); 186 static int igc_init_locked(struct igc_softc *); 187 static void igc_start(struct ifnet *); 188 static int igc_transmit(struct ifnet *, struct mbuf *); 189 static void igc_tx_common_locked(struct ifnet *, struct tx_ring *, int); 190 static bool igc_txeof(struct tx_ring *, u_int); 191 static void igc_intr_barrier(struct igc_softc *); 192 static void igc_stop(struct ifnet *, int); 193 static void igc_stop_locked(struct igc_softc *); 194 static int igc_ioctl(struct ifnet *, u_long, void *); 195 #ifdef IF_RXR 196 static int igc_rxrinfo(struct igc_softc *, struct if_rxrinfo *); 197 #endif 198 static void igc_rxfill(struct rx_ring *); 199 static void igc_rxrefill(struct rx_ring *, int); 200 static bool igc_rxeof(struct rx_ring *, u_int); 201 static int igc_rx_checksum(struct igc_queue *, uint64_t, uint32_t, 202 uint32_t); 203 static void igc_watchdog(struct ifnet *); 204 static void igc_tick(void *); 205 static void igc_media_status(struct ifnet *, struct ifmediareq *); 206 static int igc_media_change(struct ifnet *); 207 static int igc_ifflags_cb(struct ethercom *); 208 static void igc_set_filter(struct igc_softc *); 209 static void igc_update_link_status(struct igc_softc *); 210 static int igc_get_buf(struct rx_ring *, int, bool); 211 static int igc_tx_ctx_setup(struct tx_ring *, struct mbuf *, int, 212 uint32_t *, uint32_t *); 213 static int igc_tso_setup(struct tx_ring *, struct mbuf *, int, 214 uint32_t *, uint32_t *); 215 216 static void igc_configure_queues(struct igc_softc *); 217 static void igc_set_queues(struct igc_softc *, uint32_t, uint32_t, int); 218 static void igc_enable_queue(struct igc_softc *, uint32_t); 219 static void igc_enable_intr(struct igc_softc *); 220 static void igc_disable_intr(struct igc_softc *); 221 static int igc_intr_link(void *); 222 static int igc_intr_queue(void *); 223 static int igc_intr(void *); 224 static void igc_handle_queue(void *); 225 static void igc_handle_queue_work(struct work *, void *); 226 static void igc_sched_handle_queue(struct igc_softc *, struct igc_queue *); 227 static void igc_barrier_handle_queue(struct igc_softc *); 228 229 static int igc_allocate_transmit_buffers(struct tx_ring *); 230 static int igc_setup_transmit_structures(struct igc_softc *); 231 static int igc_setup_transmit_ring(struct tx_ring *); 232 static void igc_initialize_transmit_unit(struct igc_softc *); 233 static void igc_free_transmit_structures(struct igc_softc *); 234 static void igc_free_transmit_buffers(struct tx_ring *); 235 static void igc_withdraw_transmit_packets(struct tx_ring *, bool); 236 static int igc_allocate_receive_buffers(struct rx_ring *); 237 static int igc_setup_receive_structures(struct igc_softc *); 238 static int igc_setup_receive_ring(struct rx_ring *); 239 static void igc_initialize_receive_unit(struct igc_softc *); 240 static void igc_free_receive_structures(struct igc_softc *); 241 static void igc_free_receive_buffers(struct rx_ring *); 242 static void igc_clear_receive_status(struct rx_ring *); 243 static void igc_initialize_rss_mapping(struct igc_softc *); 244 245 static void igc_get_hw_control(struct igc_softc *); 246 static void igc_release_hw_control(struct igc_softc *); 247 static int igc_is_valid_ether_addr(uint8_t *); 248 static void igc_print_devinfo(struct igc_softc *); 249 250 CFATTACH_DECL3_NEW(igc, sizeof(struct igc_softc), 251 igc_match, igc_attach, igc_detach, NULL, NULL, NULL, 0); 252 253 static inline int 254 igc_txdesc_incr(struct igc_softc *sc, int id) 255 { 256 257 if (++id == sc->num_tx_desc) 258 id = 0; 259 return id; 260 } 261 262 static inline int __unused 263 igc_txdesc_decr(struct igc_softc *sc, int id) 264 { 265 266 if (--id < 0) 267 id = sc->num_tx_desc - 1; 268 return id; 269 } 270 271 static inline void 272 igc_txdesc_sync(struct tx_ring *txr, int id, int ops) 273 { 274 275 bus_dmamap_sync(txr->txdma.dma_tag, txr->txdma.dma_map, 276 id * sizeof(union igc_adv_tx_desc), sizeof(union igc_adv_tx_desc), 277 ops); 278 } 279 280 static inline int 281 igc_rxdesc_incr(struct igc_softc *sc, int id) 282 { 283 284 if (++id == sc->num_rx_desc) 285 id = 0; 286 return id; 287 } 288 289 static inline int 290 igc_rxdesc_decr(struct igc_softc *sc, int id) 291 { 292 293 if (--id < 0) 294 id = sc->num_rx_desc - 1; 295 return id; 296 } 297 298 static inline void 299 igc_rxdesc_sync(struct rx_ring *rxr, int id, int ops) 300 { 301 302 bus_dmamap_sync(rxr->rxdma.dma_tag, rxr->rxdma.dma_map, 303 id * sizeof(union igc_adv_rx_desc), sizeof(union igc_adv_rx_desc), 304 ops); 305 } 306 307 static const struct igc_product * 308 igc_lookup(const struct pci_attach_args *pa) 309 { 310 const struct igc_product *igcp; 311 312 for (igcp = igc_products; igcp->igcp_name != NULL; igcp++) { 313 if (PCI_VENDOR(pa->pa_id) == igcp->igcp_vendor && 314 PCI_PRODUCT(pa->pa_id) == igcp->igcp_product) 315 return igcp; 316 } 317 return NULL; 318 } 319 320 /********************************************************************* 321 * Device identification routine 322 * 323 * igc_match determines if the driver should be loaded on 324 * adapter based on PCI vendor/device id of the adapter. 325 * 326 * return 0 on success, positive on failure 327 *********************************************************************/ 328 static int 329 igc_match(device_t parent, cfdata_t match, void *aux) 330 { 331 struct pci_attach_args *pa = aux; 332 333 if (igc_lookup(pa) != NULL) 334 return 1; 335 336 return 0; 337 } 338 339 /********************************************************************* 340 * Device initialization routine 341 * 342 * The attach entry point is called when the driver is being loaded. 343 * This routine identifies the type of hardware, allocates all resources 344 * and initializes the hardware. 345 * 346 * return 0 on success, positive on failure 347 *********************************************************************/ 348 static void 349 igc_attach(device_t parent, device_t self, void *aux) 350 { 351 struct pci_attach_args *pa = aux; 352 struct igc_softc *sc = device_private(self); 353 struct igc_hw *hw = &sc->hw; 354 355 const struct igc_product *igcp = igc_lookup(pa); 356 KASSERT(igcp != NULL); 357 358 sc->sc_dev = self; 359 callout_init(&sc->sc_tick_ch, CALLOUT_MPSAFE); 360 callout_setfunc(&sc->sc_tick_ch, igc_tick, sc); 361 sc->sc_core_stopping = false; 362 363 sc->osdep.os_sc = sc; 364 sc->osdep.os_pa = *pa; 365 #ifndef __aarch64__ 366 /* 367 * XXX PR port-arm/57643 368 * 64-bit DMA does not work at least for LX2K with 32/64GB memory. 369 * smmu(4) support may be required. 370 */ 371 if (pci_dma64_available(pa)) { 372 aprint_verbose(", 64-bit DMA"); 373 sc->osdep.os_dmat = pa->pa_dmat64; 374 } else 375 #endif 376 { 377 aprint_verbose(", 32-bit DMA"); 378 sc->osdep.os_dmat = pa->pa_dmat; 379 } 380 381 pci_aprint_devinfo_fancy(pa, "Ethernet controller", igcp->igcp_name, 1); 382 383 /* Determine hardware and mac info */ 384 igc_identify_hardware(sc); 385 386 sc->num_tx_desc = IGC_DEFAULT_TXD; 387 sc->num_rx_desc = IGC_DEFAULT_RXD; 388 389 /* Setup PCI resources */ 390 if (igc_allocate_pci_resources(sc)) { 391 aprint_error_dev(sc->sc_dev, 392 "unable to allocate PCI resources\n"); 393 goto err_pci; 394 } 395 396 if (igc_allocate_interrupts(sc)) { 397 aprint_error_dev(sc->sc_dev, "unable to allocate interrupts\n"); 398 goto err_pci; 399 } 400 401 /* Allocate TX/RX queues */ 402 if (igc_allocate_queues(sc)) { 403 aprint_error_dev(sc->sc_dev, "unable to allocate queues\n"); 404 goto err_alloc_intr; 405 } 406 407 /* Do shared code initialization */ 408 if (igc_setup_init_funcs(hw, true)) { 409 aprint_error_dev(sc->sc_dev, "unable to initialize\n"); 410 goto err_alloc_intr; 411 } 412 413 hw->mac.autoneg = DO_AUTO_NEG; 414 hw->phy.autoneg_wait_to_complete = false; 415 hw->phy.autoneg_advertised = AUTONEG_ADV_DEFAULT; 416 417 /* Copper options. */ 418 if (hw->phy.media_type == igc_media_type_copper) 419 hw->phy.mdix = AUTO_ALL_MODES; 420 421 /* Set the max frame size. */ 422 sc->hw.mac.max_frame_size = 9234; 423 424 /* Allocate multicast array memory. */ 425 sc->mta = kmem_alloc(IGC_MTA_LEN, KM_SLEEP); 426 427 /* Check SOL/IDER usage. */ 428 if (igc_check_reset_block(hw)) { 429 aprint_error_dev(sc->sc_dev, 430 "PHY reset is blocked due to SOL/IDER session\n"); 431 } 432 433 /* Disable Energy Efficient Ethernet. */ 434 sc->hw.dev_spec._i225.eee_disable = true; 435 436 igc_reset_hw(hw); 437 438 /* Make sure we have a good EEPROM before we read from it. */ 439 if (igc_validate_nvm_checksum(hw) < 0) { 440 /* 441 * Some PCI-E parts fail the first check due to 442 * the link being in sleep state, call it again, 443 * if it fails a second time its a real issue. 444 */ 445 if (igc_validate_nvm_checksum(hw) < 0) { 446 aprint_error_dev(sc->sc_dev, 447 "EEPROM checksum invalid\n"); 448 goto err_late; 449 } 450 } 451 452 /* Copy the permanent MAC address out of the EEPROM. */ 453 if (igc_read_mac_addr(hw) < 0) { 454 aprint_error_dev(sc->sc_dev, 455 "unable to read MAC address from EEPROM\n"); 456 goto err_late; 457 } 458 459 if (!igc_is_valid_ether_addr(hw->mac.addr)) { 460 aprint_error_dev(sc->sc_dev, "invalid MAC address\n"); 461 goto err_late; 462 } 463 464 if (igc_setup_interrupts(sc)) 465 goto err_late; 466 467 /* Attach counters. */ 468 igc_attach_counters(sc); 469 470 /* Setup OS specific network interface. */ 471 igc_setup_interface(sc); 472 473 igc_print_devinfo(sc); 474 475 igc_reset(sc); 476 hw->mac.get_link_status = true; 477 igc_update_link_status(sc); 478 479 /* The driver can now take control from firmware. */ 480 igc_get_hw_control(sc); 481 482 aprint_normal_dev(sc->sc_dev, "Ethernet address %s\n", 483 ether_sprintf(sc->hw.mac.addr)); 484 485 if (pmf_device_register(self, NULL, NULL)) 486 pmf_class_network_register(self, &sc->sc_ec.ec_if); 487 else 488 aprint_error_dev(self, "couldn't establish power handler\n"); 489 490 return; 491 492 err_late: 493 igc_release_hw_control(sc); 494 err_alloc_intr: 495 igc_free_interrupts(sc); 496 err_pci: 497 igc_free_pci_resources(sc); 498 kmem_free(sc->mta, IGC_MTA_LEN); 499 } 500 501 /********************************************************************* 502 * Device removal routine 503 * 504 * The detach entry point is called when the driver is being removed. 505 * This routine stops the adapter and deallocates all the resources 506 * that were allocated for driver operation. 507 * 508 * return 0 on success, positive on failure 509 *********************************************************************/ 510 static int 511 igc_detach(device_t self, int flags) 512 { 513 struct igc_softc *sc = device_private(self); 514 struct ifnet *ifp = &sc->sc_ec.ec_if; 515 516 mutex_enter(&sc->sc_core_lock); 517 igc_stop_locked(sc); 518 mutex_exit(&sc->sc_core_lock); 519 520 igc_detach_counters(sc); 521 522 igc_free_queues(sc); 523 524 igc_phy_hw_reset(&sc->hw); 525 igc_release_hw_control(sc); 526 527 ether_ifdetach(ifp); 528 if_detach(ifp); 529 ifmedia_fini(&sc->media); 530 531 igc_free_interrupts(sc); 532 igc_free_pci_resources(sc); 533 kmem_free(sc->mta, IGC_MTA_LEN); 534 535 mutex_destroy(&sc->sc_core_lock); 536 537 return 0; 538 } 539 540 static void 541 igc_identify_hardware(struct igc_softc *sc) 542 { 543 struct igc_osdep *os = &sc->osdep; 544 struct pci_attach_args *pa = &os->os_pa; 545 546 /* Save off the information about this board. */ 547 sc->hw.device_id = PCI_PRODUCT(pa->pa_id); 548 549 /* Do shared code init and setup. */ 550 if (igc_set_mac_type(&sc->hw)) { 551 aprint_error_dev(sc->sc_dev, "unable to identify hardware\n"); 552 return; 553 } 554 } 555 556 static int 557 igc_allocate_pci_resources(struct igc_softc *sc) 558 { 559 struct igc_osdep *os = &sc->osdep; 560 struct pci_attach_args *pa = &os->os_pa; 561 562 /* 563 * Enable bus mastering and memory-mapped I/O for sure. 564 */ 565 pcireg_t csr = 566 pci_conf_read(pa->pa_pc, pa->pa_tag, PCI_COMMAND_STATUS_REG); 567 csr |= PCI_COMMAND_MASTER_ENABLE | PCI_COMMAND_MEM_ENABLE; 568 pci_conf_write(pa->pa_pc, pa->pa_tag, PCI_COMMAND_STATUS_REG, csr); 569 570 const pcireg_t memtype = 571 pci_mapreg_type(pa->pa_pc, pa->pa_tag, IGC_PCIREG); 572 if (pci_mapreg_map(pa, IGC_PCIREG, memtype, 0, &os->os_memt, 573 &os->os_memh, &os->os_membase, &os->os_memsize)) { 574 aprint_error_dev(sc->sc_dev, "unable to map registers\n"); 575 return ENXIO; 576 } 577 578 sc->hw.hw_addr = os->os_membase; 579 sc->hw.back = os; 580 581 return 0; 582 } 583 584 static int __unused 585 igc_adjust_nqueues(struct igc_softc *sc) 586 { 587 struct pci_attach_args *pa = &sc->osdep.os_pa; 588 int nqueues = MIN(IGC_MAX_NQUEUES, ncpu); 589 590 const int nmsix = pci_msix_count(pa->pa_pc, pa->pa_tag); 591 if (nmsix <= 1) 592 nqueues = 1; 593 else if (nmsix < nqueues + 1) 594 nqueues = nmsix - 1; 595 596 return nqueues; 597 } 598 599 static int 600 igc_allocate_interrupts(struct igc_softc *sc) 601 { 602 struct pci_attach_args *pa = &sc->osdep.os_pa; 603 int error; 604 605 #ifndef IGC_DISABLE_MSIX 606 const int nqueues = igc_adjust_nqueues(sc); 607 if (nqueues > 1) { 608 sc->sc_nintrs = nqueues + 1; 609 error = pci_msix_alloc_exact(pa, &sc->sc_intrs, sc->sc_nintrs); 610 if (!error) { 611 sc->sc_nqueues = nqueues; 612 sc->sc_intr_type = PCI_INTR_TYPE_MSIX; 613 return 0; 614 } 615 } 616 #endif 617 618 /* fallback to MSI */ 619 sc->sc_nintrs = sc->sc_nqueues = 1; 620 621 #ifndef IGC_DISABLE_MSI 622 error = pci_msi_alloc_exact(pa, &sc->sc_intrs, sc->sc_nintrs); 623 if (!error) { 624 sc->sc_intr_type = PCI_INTR_TYPE_MSI; 625 return 0; 626 } 627 #endif 628 629 /* fallback to INTx */ 630 631 error = pci_intx_alloc(pa, &sc->sc_intrs); 632 if (!error) { 633 sc->sc_intr_type = PCI_INTR_TYPE_INTX; 634 return 0; 635 } 636 637 return error; 638 } 639 640 static int 641 igc_allocate_queues(struct igc_softc *sc) 642 { 643 device_t dev = sc->sc_dev; 644 int rxconf = 0, txconf = 0; 645 646 /* Allocate the top level queue structs. */ 647 sc->queues = 648 kmem_zalloc(sc->sc_nqueues * sizeof(struct igc_queue), KM_SLEEP); 649 650 /* Allocate the TX ring. */ 651 sc->tx_rings = 652 kmem_zalloc(sc->sc_nqueues * sizeof(struct tx_ring), KM_SLEEP); 653 654 /* Allocate the RX ring. */ 655 sc->rx_rings = 656 kmem_zalloc(sc->sc_nqueues * sizeof(struct rx_ring), KM_SLEEP); 657 658 /* Set up the TX queues. */ 659 for (int iq = 0; iq < sc->sc_nqueues; iq++, txconf++) { 660 struct tx_ring *txr = &sc->tx_rings[iq]; 661 const int tsize = roundup2( 662 sc->num_tx_desc * sizeof(union igc_adv_tx_desc), 663 IGC_DBA_ALIGN); 664 665 txr->sc = sc; 666 txr->txr_igcq = &sc->queues[iq]; 667 txr->me = iq; 668 if (igc_dma_malloc(sc, tsize, &txr->txdma)) { 669 aprint_error_dev(dev, 670 "unable to allocate TX descriptor\n"); 671 goto fail; 672 } 673 txr->tx_base = (union igc_adv_tx_desc *)txr->txdma.dma_vaddr; 674 memset(txr->tx_base, 0, tsize); 675 } 676 677 /* Prepare transmit descriptors and buffers. */ 678 if (igc_setup_transmit_structures(sc)) { 679 aprint_error_dev(dev, "unable to setup transmit structures\n"); 680 goto fail; 681 } 682 683 /* Set up the RX queues. */ 684 for (int iq = 0; iq < sc->sc_nqueues; iq++, rxconf++) { 685 struct rx_ring *rxr = &sc->rx_rings[iq]; 686 const int rsize = roundup2( 687 sc->num_rx_desc * sizeof(union igc_adv_rx_desc), 688 IGC_DBA_ALIGN); 689 690 rxr->sc = sc; 691 rxr->rxr_igcq = &sc->queues[iq]; 692 rxr->me = iq; 693 #ifdef OPENBSD 694 timeout_set(&rxr->rx_refill, igc_rxrefill, rxr); 695 #endif 696 if (igc_dma_malloc(sc, rsize, &rxr->rxdma)) { 697 aprint_error_dev(dev, 698 "unable to allocate RX descriptor\n"); 699 goto fail; 700 } 701 rxr->rx_base = (union igc_adv_rx_desc *)rxr->rxdma.dma_vaddr; 702 memset(rxr->rx_base, 0, rsize); 703 } 704 705 sc->rx_mbuf_sz = MCLBYTES; 706 /* Prepare receive descriptors and buffers. */ 707 if (igc_setup_receive_structures(sc)) { 708 aprint_error_dev(sc->sc_dev, 709 "unable to setup receive structures\n"); 710 goto fail; 711 } 712 713 /* Set up the queue holding structs. */ 714 for (int iq = 0; iq < sc->sc_nqueues; iq++) { 715 struct igc_queue *q = &sc->queues[iq]; 716 717 q->sc = sc; 718 q->txr = &sc->tx_rings[iq]; 719 q->rxr = &sc->rx_rings[iq]; 720 } 721 722 return 0; 723 724 fail: 725 for (struct rx_ring *rxr = sc->rx_rings; rxconf > 0; rxr++, rxconf--) 726 igc_dma_free(sc, &rxr->rxdma); 727 for (struct tx_ring *txr = sc->tx_rings; txconf > 0; txr++, txconf--) 728 igc_dma_free(sc, &txr->txdma); 729 730 kmem_free(sc->rx_rings, sc->sc_nqueues * sizeof(struct rx_ring)); 731 sc->rx_rings = NULL; 732 kmem_free(sc->tx_rings, sc->sc_nqueues * sizeof(struct tx_ring)); 733 sc->tx_rings = NULL; 734 kmem_free(sc->queues, sc->sc_nqueues * sizeof(struct igc_queue)); 735 sc->queues = NULL; 736 737 return ENOMEM; 738 } 739 740 static void 741 igc_free_pci_resources(struct igc_softc *sc) 742 { 743 struct igc_osdep *os = &sc->osdep; 744 745 if (os->os_membase != 0) 746 bus_space_unmap(os->os_memt, os->os_memh, os->os_memsize); 747 os->os_membase = 0; 748 } 749 750 static void 751 igc_free_interrupts(struct igc_softc *sc) 752 { 753 struct pci_attach_args *pa = &sc->osdep.os_pa; 754 pci_chipset_tag_t pc = pa->pa_pc; 755 756 for (int i = 0; i < sc->sc_nintrs; i++) { 757 if (sc->sc_ihs[i] != NULL) { 758 pci_intr_disestablish(pc, sc->sc_ihs[i]); 759 sc->sc_ihs[i] = NULL; 760 } 761 } 762 pci_intr_release(pc, sc->sc_intrs, sc->sc_nintrs); 763 } 764 765 static void 766 igc_free_queues(struct igc_softc *sc) 767 { 768 769 igc_free_receive_structures(sc); 770 for (int iq = 0; iq < sc->sc_nqueues; iq++) { 771 struct rx_ring *rxr = &sc->rx_rings[iq]; 772 773 igc_dma_free(sc, &rxr->rxdma); 774 } 775 776 igc_free_transmit_structures(sc); 777 for (int iq = 0; iq < sc->sc_nqueues; iq++) { 778 struct tx_ring *txr = &sc->tx_rings[iq]; 779 780 igc_dma_free(sc, &txr->txdma); 781 } 782 783 kmem_free(sc->rx_rings, sc->sc_nqueues * sizeof(struct rx_ring)); 784 kmem_free(sc->tx_rings, sc->sc_nqueues * sizeof(struct tx_ring)); 785 kmem_free(sc->queues, sc->sc_nqueues * sizeof(struct igc_queue)); 786 } 787 788 /********************************************************************* 789 * 790 * Initialize the hardware to a configuration as specified by the 791 * adapter structure. 792 * 793 **********************************************************************/ 794 static void 795 igc_reset(struct igc_softc *sc) 796 { 797 struct igc_hw *hw = &sc->hw; 798 799 /* Let the firmware know the OS is in control */ 800 igc_get_hw_control(sc); 801 802 /* 803 * Packet Buffer Allocation (PBA) 804 * Writing PBA sets the receive portion of the buffer 805 * the remainder is used for the transmit buffer. 806 */ 807 const uint32_t pba = IGC_PBA_34K; 808 809 /* 810 * These parameters control the automatic generation (Tx) and 811 * response (Rx) to Ethernet PAUSE frames. 812 * - High water mark should allow for at least two frames to be 813 * received after sending an XOFF. 814 * - Low water mark works best when it is very near the high water mark. 815 * This allows the receiver to restart by sending XON when it has 816 * drained a bit. Here we use an arbitrary value of 1500 which will 817 * restart after one full frame is pulled from the buffer. There 818 * could be several smaller frames in the buffer and if so they will 819 * not trigger the XON until their total number reduces the buffer 820 * by 1500. 821 * - The pause time is fairly large at 1000 x 512ns = 512 usec. 822 */ 823 const uint16_t rx_buffer_size = (pba & 0xffff) << 10; 824 825 hw->fc.high_water = rx_buffer_size - 826 roundup2(sc->hw.mac.max_frame_size, 1024); 827 /* 16-byte granularity */ 828 hw->fc.low_water = hw->fc.high_water - 16; 829 830 if (sc->fc) /* locally set flow control value? */ 831 hw->fc.requested_mode = sc->fc; 832 else 833 hw->fc.requested_mode = igc_fc_full; 834 835 hw->fc.pause_time = IGC_FC_PAUSE_TIME; 836 837 hw->fc.send_xon = true; 838 839 /* Issue a global reset */ 840 igc_reset_hw(hw); 841 IGC_WRITE_REG(hw, IGC_WUC, 0); 842 843 /* and a re-init */ 844 if (igc_init_hw(hw) < 0) { 845 aprint_error_dev(sc->sc_dev, "unable to reset hardware\n"); 846 return; 847 } 848 849 /* Setup DMA Coalescing */ 850 igc_init_dmac(sc, pba); 851 852 IGC_WRITE_REG(hw, IGC_VET, ETHERTYPE_VLAN); 853 igc_get_phy_info(hw); 854 igc_check_for_link(hw); 855 } 856 857 /********************************************************************* 858 * 859 * Initialize the DMA Coalescing feature 860 * 861 **********************************************************************/ 862 static void 863 igc_init_dmac(struct igc_softc *sc, uint32_t pba) 864 { 865 struct igc_hw *hw = &sc->hw; 866 const uint16_t max_frame_size = sc->hw.mac.max_frame_size; 867 uint32_t reg, status; 868 869 if (sc->dmac == 0) { /* Disabling it */ 870 reg = ~IGC_DMACR_DMAC_EN; /* XXXRO */ 871 IGC_WRITE_REG(hw, IGC_DMACR, reg); 872 DPRINTF(MISC, "DMA coalescing disabled\n"); 873 return; 874 } else { 875 device_printf(sc->sc_dev, "DMA coalescing enabled\n"); 876 } 877 878 /* Set starting threshold */ 879 IGC_WRITE_REG(hw, IGC_DMCTXTH, 0); 880 881 uint16_t hwm = 64 * pba - max_frame_size / 16; 882 if (hwm < 64 * (pba - 6)) 883 hwm = 64 * (pba - 6); 884 reg = IGC_READ_REG(hw, IGC_FCRTC); 885 reg &= ~IGC_FCRTC_RTH_COAL_MASK; 886 reg |= (hwm << IGC_FCRTC_RTH_COAL_SHIFT) & IGC_FCRTC_RTH_COAL_MASK; 887 IGC_WRITE_REG(hw, IGC_FCRTC, reg); 888 889 uint32_t dmac = pba - max_frame_size / 512; 890 if (dmac < pba - 10) 891 dmac = pba - 10; 892 reg = IGC_READ_REG(hw, IGC_DMACR); 893 reg &= ~IGC_DMACR_DMACTHR_MASK; 894 reg |= (dmac << IGC_DMACR_DMACTHR_SHIFT) & IGC_DMACR_DMACTHR_MASK; 895 896 /* transition to L0x or L1 if available..*/ 897 reg |= IGC_DMACR_DMAC_EN | IGC_DMACR_DMAC_LX_MASK; 898 899 /* Check if status is 2.5Gb backplane connection 900 * before configuration of watchdog timer, which is 901 * in msec values in 12.8usec intervals 902 * watchdog timer= msec values in 32usec intervals 903 * for non 2.5Gb connection 904 */ 905 status = IGC_READ_REG(hw, IGC_STATUS); 906 if ((status & IGC_STATUS_2P5_SKU) && 907 !(status & IGC_STATUS_2P5_SKU_OVER)) 908 reg |= (sc->dmac * 5) >> 6; 909 else 910 reg |= sc->dmac >> 5; 911 912 IGC_WRITE_REG(hw, IGC_DMACR, reg); 913 914 IGC_WRITE_REG(hw, IGC_DMCRTRH, 0); 915 916 /* Set the interval before transition */ 917 reg = IGC_READ_REG(hw, IGC_DMCTLX); 918 reg |= IGC_DMCTLX_DCFLUSH_DIS; 919 920 /* 921 * in 2.5Gb connection, TTLX unit is 0.4 usec 922 * which is 0x4*2 = 0xA. But delay is still 4 usec 923 */ 924 status = IGC_READ_REG(hw, IGC_STATUS); 925 if ((status & IGC_STATUS_2P5_SKU) && 926 !(status & IGC_STATUS_2P5_SKU_OVER)) 927 reg |= 0xA; 928 else 929 reg |= 0x4; 930 931 IGC_WRITE_REG(hw, IGC_DMCTLX, reg); 932 933 /* free space in tx packet buffer to wake from DMA coal */ 934 IGC_WRITE_REG(hw, IGC_DMCTXTH, 935 (IGC_TXPBSIZE - (2 * max_frame_size)) >> 6); 936 937 /* make low power state decision controlled by DMA coal */ 938 reg = IGC_READ_REG(hw, IGC_PCIEMISC); 939 reg &= ~IGC_PCIEMISC_LX_DECISION; 940 IGC_WRITE_REG(hw, IGC_PCIEMISC, reg); 941 } 942 943 static int 944 igc_setup_interrupts(struct igc_softc *sc) 945 { 946 int error; 947 948 switch (sc->sc_intr_type) { 949 case PCI_INTR_TYPE_MSIX: 950 error = igc_setup_msix(sc); 951 break; 952 case PCI_INTR_TYPE_MSI: 953 error = igc_setup_msi(sc); 954 break; 955 case PCI_INTR_TYPE_INTX: 956 error = igc_setup_intx(sc); 957 break; 958 default: 959 panic("%s: invalid interrupt type: %d", 960 device_xname(sc->sc_dev), sc->sc_intr_type); 961 } 962 963 return error; 964 } 965 966 static void 967 igc_attach_counters(struct igc_softc *sc) 968 { 969 #ifdef IGC_EVENT_COUNTERS 970 971 /* Global counters */ 972 sc->sc_global_evcnts = kmem_zalloc( 973 IGC_GLOBAL_COUNTERS * sizeof(sc->sc_global_evcnts[0]), KM_SLEEP); 974 975 for (int cnt = 0; cnt < IGC_GLOBAL_COUNTERS; cnt++) { 976 evcnt_attach_dynamic(&sc->sc_global_evcnts[cnt], 977 igc_global_counters[cnt].type, NULL, 978 device_xname(sc->sc_dev), igc_global_counters[cnt].name); 979 } 980 981 /* Driver counters */ 982 sc->sc_driver_evcnts = kmem_zalloc( 983 IGC_DRIVER_COUNTERS * sizeof(sc->sc_driver_evcnts[0]), KM_SLEEP); 984 985 for (int cnt = 0; cnt < IGC_DRIVER_COUNTERS; cnt++) { 986 evcnt_attach_dynamic(&sc->sc_driver_evcnts[cnt], 987 igc_driver_counters[cnt].type, NULL, 988 device_xname(sc->sc_dev), igc_driver_counters[cnt].name); 989 } 990 991 for (int iq = 0; iq < sc->sc_nqueues; iq++) { 992 struct igc_queue *q = &sc->queues[iq]; 993 994 q->igcq_driver_counters = kmem_zalloc( 995 IGC_DRIVER_COUNTERS * sizeof(q->igcq_driver_counters[0]), 996 KM_SLEEP); 997 } 998 999 /* Queue counters */ 1000 for (int iq = 0; iq < sc->sc_nqueues; iq++) { 1001 struct igc_queue *q = &sc->queues[iq]; 1002 1003 snprintf(q->igcq_queue_evname, sizeof(q->igcq_queue_evname), 1004 "%s q%d", device_xname(sc->sc_dev), iq); 1005 1006 q->igcq_queue_evcnts = kmem_zalloc( 1007 IGC_QUEUE_COUNTERS * sizeof(q->igcq_queue_evcnts[0]), 1008 KM_SLEEP); 1009 1010 for (int cnt = 0; cnt < IGC_QUEUE_COUNTERS; cnt++) { 1011 evcnt_attach_dynamic(&q->igcq_queue_evcnts[cnt], 1012 igc_queue_counters[cnt].type, NULL, 1013 q->igcq_queue_evname, igc_queue_counters[cnt].name); 1014 } 1015 } 1016 1017 /* MAC counters */ 1018 snprintf(sc->sc_mac_evname, sizeof(sc->sc_mac_evname), 1019 "%s Mac Statistics", device_xname(sc->sc_dev)); 1020 1021 sc->sc_mac_evcnts = kmem_zalloc( 1022 IGC_MAC_COUNTERS * sizeof(sc->sc_mac_evcnts[0]), KM_SLEEP); 1023 1024 for (int cnt = 0; cnt < IGC_MAC_COUNTERS; cnt++) { 1025 evcnt_attach_dynamic(&sc->sc_mac_evcnts[cnt], EVCNT_TYPE_MISC, 1026 NULL, sc->sc_mac_evname, igc_mac_counters[cnt].name); 1027 } 1028 #endif 1029 } 1030 1031 static void 1032 igc_detach_counters(struct igc_softc *sc) 1033 { 1034 #ifdef IGC_EVENT_COUNTERS 1035 1036 /* Global counters */ 1037 for (int cnt = 0; cnt < IGC_GLOBAL_COUNTERS; cnt++) 1038 evcnt_detach(&sc->sc_global_evcnts[cnt]); 1039 1040 kmem_free(sc->sc_global_evcnts, 1041 IGC_GLOBAL_COUNTERS * sizeof(sc->sc_global_evcnts)); 1042 1043 /* Driver counters */ 1044 for (int iq = 0; iq < sc->sc_nqueues; iq++) { 1045 struct igc_queue *q = &sc->queues[iq]; 1046 1047 kmem_free(q->igcq_driver_counters, 1048 IGC_DRIVER_COUNTERS * sizeof(q->igcq_driver_counters[0])); 1049 } 1050 1051 for (int cnt = 0; cnt < IGC_DRIVER_COUNTERS; cnt++) 1052 evcnt_detach(&sc->sc_driver_evcnts[cnt]); 1053 1054 kmem_free(sc->sc_driver_evcnts, 1055 IGC_DRIVER_COUNTERS * sizeof(sc->sc_driver_evcnts)); 1056 1057 /* Queue counters */ 1058 for (int iq = 0; iq < sc->sc_nqueues; iq++) { 1059 struct igc_queue *q = &sc->queues[iq]; 1060 1061 for (int cnt = 0; cnt < IGC_QUEUE_COUNTERS; cnt++) 1062 evcnt_detach(&q->igcq_queue_evcnts[cnt]); 1063 1064 kmem_free(q->igcq_queue_evcnts, 1065 IGC_QUEUE_COUNTERS * sizeof(q->igcq_queue_evcnts[0])); 1066 } 1067 1068 /* MAC statistics */ 1069 for (int cnt = 0; cnt < IGC_MAC_COUNTERS; cnt++) 1070 evcnt_detach(&sc->sc_mac_evcnts[cnt]); 1071 1072 kmem_free(sc->sc_mac_evcnts, 1073 IGC_MAC_COUNTERS * sizeof(sc->sc_mac_evcnts[0])); 1074 #endif 1075 } 1076 1077 /* 1078 * XXX 1079 * FreeBSD uses 4-byte-wise read for 64-bit counters, while Linux just 1080 * drops hi words. 1081 */ 1082 static inline uint64_t __unused 1083 igc_read_mac_counter(struct igc_hw *hw, bus_size_t reg, bool is64) 1084 { 1085 uint64_t val; 1086 1087 val = IGC_READ_REG(hw, reg); 1088 if (is64) 1089 val += ((uint64_t)IGC_READ_REG(hw, reg + 4)) << 32; 1090 return val; 1091 } 1092 1093 static void 1094 igc_update_counters(struct igc_softc *sc) 1095 { 1096 #ifdef IGC_EVENT_COUNTERS 1097 1098 /* Global counters: nop */ 1099 1100 /* Driver counters */ 1101 uint64_t sum[IGC_DRIVER_COUNTERS]; 1102 1103 memset(sum, 0, sizeof(sum)); 1104 for (int iq = 0; iq < sc->sc_nqueues; iq++) { 1105 struct igc_queue *q = &sc->queues[iq]; 1106 1107 for (int cnt = 0; cnt < IGC_DRIVER_COUNTERS; cnt++) { 1108 sum[cnt] += IGC_QUEUE_DRIVER_COUNTER_VAL(q, cnt); 1109 IGC_QUEUE_DRIVER_COUNTER_STORE(q, cnt, 0); 1110 } 1111 } 1112 1113 for (int cnt = 0; cnt < IGC_DRIVER_COUNTERS; cnt++) 1114 IGC_DRIVER_COUNTER_ADD(sc, cnt, sum[cnt]); 1115 1116 /* Queue counters: nop */ 1117 1118 /* Mac statistics */ 1119 struct igc_hw *hw = &sc->hw; 1120 1121 for (int cnt = 0; cnt < IGC_MAC_COUNTERS; cnt++) { 1122 IGC_MAC_COUNTER_ADD(sc, cnt, igc_read_mac_counter(hw, 1123 igc_mac_counters[cnt].reg, igc_mac_counters[cnt].is64)); 1124 } 1125 #endif 1126 } 1127 1128 static void 1129 igc_clear_counters(struct igc_softc *sc) 1130 { 1131 #ifdef IGC_EVENT_COUNTERS 1132 1133 /* Global counters */ 1134 for (int cnt = 0; cnt < IGC_GLOBAL_COUNTERS; cnt++) 1135 IGC_GLOBAL_COUNTER_STORE(sc, cnt, 0); 1136 1137 /* Driver counters */ 1138 for (int iq = 0; iq < sc->sc_nqueues; iq++) { 1139 struct igc_queue *q = &sc->queues[iq]; 1140 1141 for (int cnt = 0; cnt < IGC_DRIVER_COUNTERS; cnt++) 1142 IGC_QUEUE_DRIVER_COUNTER_STORE(q, cnt, 0); 1143 } 1144 1145 for (int cnt = 0; cnt < IGC_DRIVER_COUNTERS; cnt++) 1146 IGC_DRIVER_COUNTER_STORE(sc, cnt, 0); 1147 1148 /* Queue counters */ 1149 for (int iq = 0; iq < sc->sc_nqueues; iq++) { 1150 struct igc_queue *q = &sc->queues[iq]; 1151 1152 for (int cnt = 0; cnt < IGC_QUEUE_COUNTERS; cnt++) 1153 IGC_QUEUE_COUNTER_STORE(q, cnt, 0); 1154 } 1155 1156 /* Mac statistics */ 1157 struct igc_hw *hw = &sc->hw; 1158 1159 for (int cnt = 0; cnt < IGC_MAC_COUNTERS; cnt++) { 1160 (void)igc_read_mac_counter(hw, igc_mac_counters[cnt].reg, 1161 igc_mac_counters[cnt].is64); 1162 IGC_MAC_COUNTER_STORE(sc, cnt, 0); 1163 } 1164 #endif 1165 } 1166 1167 static int 1168 igc_setup_msix(struct igc_softc *sc) 1169 { 1170 pci_chipset_tag_t pc = sc->osdep.os_pa.pa_pc; 1171 device_t dev = sc->sc_dev; 1172 pci_intr_handle_t *intrs; 1173 void **ihs; 1174 const char *intrstr; 1175 char intrbuf[PCI_INTRSTR_LEN]; 1176 char xnamebuf[MAX(32, MAXCOMLEN)]; 1177 int iq, error; 1178 1179 for (iq = 0, intrs = sc->sc_intrs, ihs = sc->sc_ihs; 1180 iq < sc->sc_nqueues; iq++, intrs++, ihs++) { 1181 struct igc_queue *q = &sc->queues[iq]; 1182 1183 snprintf(xnamebuf, sizeof(xnamebuf), "%s: txrx %d", 1184 device_xname(dev), iq); 1185 1186 intrstr = pci_intr_string(pc, *intrs, intrbuf, sizeof(intrbuf)); 1187 1188 pci_intr_setattr(pc, intrs, PCI_INTR_MPSAFE, true); 1189 *ihs = pci_intr_establish_xname(pc, *intrs, IPL_NET, 1190 igc_intr_queue, q, xnamebuf); 1191 if (*ihs == NULL) { 1192 aprint_error_dev(dev, 1193 "unable to establish txrx interrupt at %s\n", 1194 intrstr); 1195 return ENOBUFS; 1196 } 1197 aprint_normal_dev(dev, "txrx interrupting at %s\n", intrstr); 1198 1199 kcpuset_t *affinity; 1200 kcpuset_create(&affinity, true); 1201 kcpuset_set(affinity, iq % ncpu); 1202 error = interrupt_distribute(*ihs, affinity, NULL); 1203 if (error) { 1204 aprint_normal_dev(dev, 1205 "%s: unable to change affinity, use default CPU\n", 1206 intrstr); 1207 } 1208 kcpuset_destroy(affinity); 1209 1210 q->igcq_si = softint_establish(SOFTINT_NET | SOFTINT_MPSAFE, 1211 igc_handle_queue, q); 1212 if (q->igcq_si == NULL) { 1213 aprint_error_dev(dev, 1214 "%s: unable to establish softint\n", intrstr); 1215 return ENOBUFS; 1216 } 1217 1218 q->msix = iq; 1219 q->eims = 1 << iq; 1220 } 1221 1222 snprintf(xnamebuf, MAXCOMLEN, "%s_tx_rx", device_xname(dev)); 1223 error = workqueue_create(&sc->sc_queue_wq, xnamebuf, 1224 igc_handle_queue_work, sc, IGC_WORKQUEUE_PRI, IPL_NET, 1225 WQ_PERCPU | WQ_MPSAFE); 1226 if (error) { 1227 aprint_error_dev(dev, "workqueue_create failed\n"); 1228 return ENOBUFS; 1229 } 1230 sc->sc_txrx_workqueue = false; 1231 1232 intrstr = pci_intr_string(pc, *intrs, intrbuf, sizeof(intrbuf)); 1233 snprintf(xnamebuf, sizeof(xnamebuf), "%s: link", device_xname(dev)); 1234 pci_intr_setattr(pc, intrs, PCI_INTR_MPSAFE, true); 1235 *ihs = pci_intr_establish_xname(pc, *intrs, IPL_NET, 1236 igc_intr_link, sc, xnamebuf); 1237 if (*ihs == NULL) { 1238 aprint_error_dev(dev, 1239 "unable to establish link interrupt at %s\n", intrstr); 1240 return ENOBUFS; 1241 } 1242 aprint_normal_dev(dev, "link interrupting at %s\n", intrstr); 1243 /* use later in igc_configure_queues() */ 1244 sc->linkvec = iq; 1245 1246 return 0; 1247 } 1248 1249 static int 1250 igc_setup_msi(struct igc_softc *sc) 1251 { 1252 pci_chipset_tag_t pc = sc->osdep.os_pa.pa_pc; 1253 device_t dev = sc->sc_dev; 1254 pci_intr_handle_t *intr = sc->sc_intrs; 1255 void **ihs = sc->sc_ihs; 1256 const char *intrstr; 1257 char intrbuf[PCI_INTRSTR_LEN]; 1258 char xnamebuf[MAX(32, MAXCOMLEN)]; 1259 int error; 1260 1261 intrstr = pci_intr_string(pc, *intr, intrbuf, sizeof(intrbuf)); 1262 1263 snprintf(xnamebuf, sizeof(xnamebuf), "%s: msi", device_xname(dev)); 1264 pci_intr_setattr(pc, intr, PCI_INTR_MPSAFE, true); 1265 *ihs = pci_intr_establish_xname(pc, *intr, IPL_NET, 1266 igc_intr, sc, xnamebuf); 1267 if (*ihs == NULL) { 1268 aprint_error_dev(dev, 1269 "unable to establish interrupt at %s\n", intrstr); 1270 return ENOBUFS; 1271 } 1272 aprint_normal_dev(dev, "interrupting at %s\n", intrstr); 1273 1274 struct igc_queue *iq = sc->queues; 1275 iq->igcq_si = softint_establish(SOFTINT_NET | SOFTINT_MPSAFE, 1276 igc_handle_queue, iq); 1277 if (iq->igcq_si == NULL) { 1278 aprint_error_dev(dev, 1279 "%s: unable to establish softint\n", intrstr); 1280 return ENOBUFS; 1281 } 1282 1283 snprintf(xnamebuf, MAXCOMLEN, "%s_tx_rx", device_xname(dev)); 1284 error = workqueue_create(&sc->sc_queue_wq, xnamebuf, 1285 igc_handle_queue_work, sc, IGC_WORKQUEUE_PRI, IPL_NET, 1286 WQ_PERCPU | WQ_MPSAFE); 1287 if (error) { 1288 aprint_error_dev(dev, "workqueue_create failed\n"); 1289 return ENOBUFS; 1290 } 1291 sc->sc_txrx_workqueue = false; 1292 1293 sc->queues[0].msix = 0; 1294 sc->linkvec = 0; 1295 1296 return 0; 1297 } 1298 1299 static int 1300 igc_setup_intx(struct igc_softc *sc) 1301 { 1302 pci_chipset_tag_t pc = sc->osdep.os_pa.pa_pc; 1303 device_t dev = sc->sc_dev; 1304 pci_intr_handle_t *intr = sc->sc_intrs; 1305 void **ihs = sc->sc_ihs; 1306 const char *intrstr; 1307 char intrbuf[PCI_INTRSTR_LEN]; 1308 char xnamebuf[32]; 1309 1310 intrstr = pci_intr_string(pc, *intr, intrbuf, sizeof(intrbuf)); 1311 1312 snprintf(xnamebuf, sizeof(xnamebuf), "%s:intx", device_xname(dev)); 1313 pci_intr_setattr(pc, intr, PCI_INTR_MPSAFE, true); 1314 *ihs = pci_intr_establish_xname(pc, *intr, IPL_NET, 1315 igc_intr, sc, xnamebuf); 1316 if (*ihs == NULL) { 1317 aprint_error_dev(dev, 1318 "unable to establish interrupt at %s\n", intrstr); 1319 return ENOBUFS; 1320 } 1321 aprint_normal_dev(dev, "interrupting at %s\n", intrstr); 1322 1323 struct igc_queue *iq = sc->queues; 1324 iq->igcq_si = softint_establish(SOFTINT_NET | SOFTINT_MPSAFE, 1325 igc_handle_queue, iq); 1326 if (iq->igcq_si == NULL) { 1327 aprint_error_dev(dev, 1328 "%s: unable to establish softint\n", intrstr); 1329 return ENOBUFS; 1330 } 1331 1332 /* create workqueue? */ 1333 sc->sc_txrx_workqueue = false; 1334 1335 sc->queues[0].msix = 0; 1336 sc->linkvec = 0; 1337 1338 return 0; 1339 } 1340 1341 static int 1342 igc_dma_malloc(struct igc_softc *sc, bus_size_t size, struct igc_dma_alloc *dma) 1343 { 1344 struct igc_osdep *os = &sc->osdep; 1345 1346 dma->dma_tag = os->os_dmat; 1347 1348 if (bus_dmamap_create(dma->dma_tag, size, 1, size, 0, 1349 BUS_DMA_WAITOK | BUS_DMA_ALLOCNOW, &dma->dma_map)) 1350 return 1; 1351 if (bus_dmamem_alloc(dma->dma_tag, size, PAGE_SIZE, 0, &dma->dma_seg, 1352 1, &dma->dma_nseg, BUS_DMA_WAITOK)) 1353 goto destroy; 1354 /* 1355 * XXXRO 1356 * 1357 * Coherent mapping for descriptors is required for now. 1358 * 1359 * Both TX and RX descriptors are 16-byte length, which is shorter 1360 * than dcache lines on modern CPUs. Therefore, sync for a descriptor 1361 * may overwrite DMA read for descriptors in the same cache line. 1362 * 1363 * Can't we avoid this by use cache-line-aligned descriptors at once? 1364 */ 1365 if (bus_dmamem_map(dma->dma_tag, &dma->dma_seg, dma->dma_nseg, size, 1366 &dma->dma_vaddr, BUS_DMA_WAITOK | BUS_DMA_COHERENT /* XXXRO */)) 1367 goto free; 1368 if (bus_dmamap_load(dma->dma_tag, dma->dma_map, dma->dma_vaddr, size, 1369 NULL, BUS_DMA_WAITOK)) 1370 goto unmap; 1371 1372 dma->dma_size = size; 1373 1374 return 0; 1375 unmap: 1376 bus_dmamem_unmap(dma->dma_tag, dma->dma_vaddr, size); 1377 free: 1378 bus_dmamem_free(dma->dma_tag, &dma->dma_seg, dma->dma_nseg); 1379 destroy: 1380 bus_dmamap_destroy(dma->dma_tag, dma->dma_map); 1381 dma->dma_map = NULL; 1382 dma->dma_tag = NULL; 1383 return 1; 1384 } 1385 1386 static void 1387 igc_dma_free(struct igc_softc *sc, struct igc_dma_alloc *dma) 1388 { 1389 1390 if (dma->dma_tag == NULL) 1391 return; 1392 1393 if (dma->dma_map != NULL) { 1394 bus_dmamap_sync(dma->dma_tag, dma->dma_map, 0, 1395 dma->dma_map->dm_mapsize, 1396 BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE); 1397 bus_dmamap_unload(dma->dma_tag, dma->dma_map); 1398 bus_dmamem_unmap(dma->dma_tag, dma->dma_vaddr, dma->dma_size); 1399 bus_dmamem_free(dma->dma_tag, &dma->dma_seg, dma->dma_nseg); 1400 bus_dmamap_destroy(dma->dma_tag, dma->dma_map); 1401 dma->dma_map = NULL; 1402 } 1403 } 1404 1405 /********************************************************************* 1406 * 1407 * Setup networking device structure and register an interface. 1408 * 1409 **********************************************************************/ 1410 static void 1411 igc_setup_interface(struct igc_softc *sc) 1412 { 1413 struct ifnet *ifp = &sc->sc_ec.ec_if; 1414 1415 strlcpy(ifp->if_xname, device_xname(sc->sc_dev), sizeof(ifp->if_xname)); 1416 ifp->if_softc = sc; 1417 ifp->if_flags = IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST; 1418 ifp->if_extflags = IFEF_MPSAFE; 1419 ifp->if_ioctl = igc_ioctl; 1420 ifp->if_start = igc_start; 1421 if (sc->sc_nqueues > 1) 1422 ifp->if_transmit = igc_transmit; 1423 ifp->if_watchdog = igc_watchdog; 1424 ifp->if_init = igc_init; 1425 ifp->if_stop = igc_stop; 1426 1427 #if 0 /* notyet */ 1428 ifp->if_capabilities = IFCAP_TSOv4 | IFCAP_TSOv6; 1429 #endif 1430 1431 ifp->if_capabilities |= 1432 IFCAP_CSUM_IPv4_Tx | IFCAP_CSUM_IPv4_Rx | 1433 IFCAP_CSUM_TCPv4_Tx | IFCAP_CSUM_TCPv4_Rx | 1434 IFCAP_CSUM_UDPv4_Tx | IFCAP_CSUM_UDPv4_Rx | 1435 IFCAP_CSUM_TCPv6_Tx | IFCAP_CSUM_TCPv6_Rx | 1436 IFCAP_CSUM_UDPv6_Tx | IFCAP_CSUM_UDPv6_Rx; 1437 1438 ifp->if_capenable = 0; 1439 1440 sc->sc_ec.ec_capabilities |= 1441 ETHERCAP_JUMBO_MTU | ETHERCAP_VLAN_MTU; 1442 1443 IFQ_SET_MAXLEN(&ifp->if_snd, sc->num_tx_desc - 1); 1444 IFQ_SET_READY(&ifp->if_snd); 1445 1446 #if NVLAN > 0 1447 sc->sc_ec.ec_capabilities |= ETHERCAP_VLAN_HWTAGGING; 1448 #endif 1449 1450 mutex_init(&sc->sc_core_lock, MUTEX_DEFAULT, IPL_NET); 1451 1452 /* Initialize ifmedia structures. */ 1453 sc->sc_ec.ec_ifmedia = &sc->media; 1454 ifmedia_init_with_lock(&sc->media, IFM_IMASK, igc_media_change, 1455 igc_media_status, &sc->sc_core_lock); 1456 ifmedia_add(&sc->media, IFM_ETHER | IFM_10_T, 0, NULL); 1457 ifmedia_add(&sc->media, IFM_ETHER | IFM_10_T | IFM_FDX, 0, NULL); 1458 ifmedia_add(&sc->media, IFM_ETHER | IFM_100_TX, 0, NULL); 1459 ifmedia_add(&sc->media, IFM_ETHER | IFM_100_TX | IFM_FDX, 0, NULL); 1460 ifmedia_add(&sc->media, IFM_ETHER | IFM_1000_T | IFM_FDX, 0, NULL); 1461 ifmedia_add(&sc->media, IFM_ETHER | IFM_2500_T | IFM_FDX, 0, NULL); 1462 ifmedia_add(&sc->media, IFM_ETHER | IFM_AUTO, 0, NULL); 1463 ifmedia_set(&sc->media, IFM_ETHER | IFM_AUTO); 1464 1465 sc->sc_rx_intr_process_limit = IGC_RX_INTR_PROCESS_LIMIT_DEFAULT; 1466 sc->sc_tx_intr_process_limit = IGC_TX_INTR_PROCESS_LIMIT_DEFAULT; 1467 sc->sc_rx_process_limit = IGC_RX_PROCESS_LIMIT_DEFAULT; 1468 sc->sc_tx_process_limit = IGC_TX_PROCESS_LIMIT_DEFAULT; 1469 1470 if_initialize(ifp); 1471 sc->sc_ipq = if_percpuq_create(ifp); 1472 if_deferred_start_init(ifp, NULL); 1473 ether_ifattach(ifp, sc->hw.mac.addr); 1474 ether_set_ifflags_cb(&sc->sc_ec, igc_ifflags_cb); 1475 if_register(ifp); 1476 } 1477 1478 static int 1479 igc_init(struct ifnet *ifp) 1480 { 1481 struct igc_softc *sc = ifp->if_softc; 1482 int error; 1483 1484 mutex_enter(&sc->sc_core_lock); 1485 error = igc_init_locked(sc); 1486 mutex_exit(&sc->sc_core_lock); 1487 1488 return error; 1489 } 1490 1491 static int 1492 igc_init_locked(struct igc_softc *sc) 1493 { 1494 struct ethercom *ec = &sc->sc_ec; 1495 struct ifnet *ifp = &ec->ec_if; 1496 1497 DPRINTF(CFG, "called\n"); 1498 1499 KASSERT(mutex_owned(&sc->sc_core_lock)); 1500 1501 if (ISSET(ifp->if_flags, IFF_RUNNING)) 1502 igc_stop_locked(sc); 1503 1504 /* Put the address into the receive address array. */ 1505 igc_rar_set(&sc->hw, sc->hw.mac.addr, 0); 1506 1507 /* Initialize the hardware. */ 1508 igc_reset(sc); 1509 igc_update_link_status(sc); 1510 1511 /* Setup VLAN support, basic and offload if available. */ 1512 IGC_WRITE_REG(&sc->hw, IGC_VET, ETHERTYPE_VLAN); 1513 1514 igc_initialize_transmit_unit(sc); 1515 igc_initialize_receive_unit(sc); 1516 1517 if (ec->ec_capenable & ETHERCAP_VLAN_HWTAGGING) { 1518 uint32_t ctrl = IGC_READ_REG(&sc->hw, IGC_CTRL); 1519 ctrl |= IGC_CTRL_VME; 1520 IGC_WRITE_REG(&sc->hw, IGC_CTRL, ctrl); 1521 } 1522 1523 /* Setup multicast table. */ 1524 igc_set_filter(sc); 1525 1526 igc_clear_hw_cntrs_base_generic(&sc->hw); 1527 1528 if (sc->sc_intr_type == PCI_INTR_TYPE_MSIX) 1529 igc_configure_queues(sc); 1530 1531 /* This clears any pending interrupts */ 1532 IGC_READ_REG(&sc->hw, IGC_ICR); 1533 IGC_WRITE_REG(&sc->hw, IGC_ICS, IGC_ICS_LSC); 1534 1535 /* The driver can now take control from firmware. */ 1536 igc_get_hw_control(sc); 1537 1538 /* Set Energy Efficient Ethernet. */ 1539 igc_set_eee_i225(&sc->hw, true, true, true); 1540 1541 for (int iq = 0; iq < sc->sc_nqueues; iq++) { 1542 struct rx_ring *rxr = &sc->rx_rings[iq]; 1543 1544 mutex_enter(&rxr->rxr_lock); 1545 igc_rxfill(rxr); 1546 mutex_exit(&rxr->rxr_lock); 1547 } 1548 1549 sc->sc_core_stopping = false; 1550 1551 ifp->if_flags |= IFF_RUNNING; 1552 1553 /* Save last flags for the callback */ 1554 sc->sc_if_flags = ifp->if_flags; 1555 1556 callout_schedule(&sc->sc_tick_ch, hz); 1557 1558 igc_enable_intr(sc); 1559 1560 return 0; 1561 } 1562 1563 static inline int 1564 igc_load_mbuf(struct igc_queue *q, bus_dma_tag_t dmat, bus_dmamap_t map, 1565 struct mbuf *m) 1566 { 1567 int error; 1568 1569 error = bus_dmamap_load_mbuf(dmat, map, m, 1570 BUS_DMA_WRITE | BUS_DMA_NOWAIT); 1571 1572 if (__predict_false(error == EFBIG)) { 1573 IGC_DRIVER_EVENT(q, txdma_efbig, 1); 1574 m = m_defrag(m, M_NOWAIT); 1575 if (__predict_false(m == NULL)) { 1576 IGC_DRIVER_EVENT(q, txdma_defrag, 1); 1577 return ENOBUFS; 1578 } 1579 error = bus_dmamap_load_mbuf(dmat, map, m, 1580 BUS_DMA_WRITE | BUS_DMA_NOWAIT); 1581 } 1582 1583 switch (error) { 1584 case 0: 1585 break; 1586 case ENOMEM: 1587 IGC_DRIVER_EVENT(q, txdma_enomem, 1); 1588 break; 1589 case EINVAL: 1590 IGC_DRIVER_EVENT(q, txdma_einval, 1); 1591 break; 1592 case EAGAIN: 1593 IGC_DRIVER_EVENT(q, txdma_eagain, 1); 1594 break; 1595 default: 1596 IGC_DRIVER_EVENT(q, txdma_other, 1); 1597 break; 1598 } 1599 1600 return error; 1601 } 1602 1603 #define IGC_TX_START 1 1604 #define IGC_TX_TRANSMIT 2 1605 1606 static void 1607 igc_start(struct ifnet *ifp) 1608 { 1609 struct igc_softc *sc = ifp->if_softc; 1610 1611 if (__predict_false(!sc->link_active)) { 1612 IFQ_PURGE(&ifp->if_snd); 1613 return; 1614 } 1615 1616 struct tx_ring *txr = &sc->tx_rings[0]; /* queue 0 */ 1617 mutex_enter(&txr->txr_lock); 1618 igc_tx_common_locked(ifp, txr, IGC_TX_START); 1619 mutex_exit(&txr->txr_lock); 1620 } 1621 1622 static inline u_int 1623 igc_select_txqueue(struct igc_softc *sc, struct mbuf *m __unused) 1624 { 1625 const u_int cpuid = cpu_index(curcpu()); 1626 1627 return cpuid % sc->sc_nqueues; 1628 } 1629 1630 static int 1631 igc_transmit(struct ifnet *ifp, struct mbuf *m) 1632 { 1633 struct igc_softc *sc = ifp->if_softc; 1634 const u_int qid = igc_select_txqueue(sc, m); 1635 struct tx_ring *txr = &sc->tx_rings[qid]; 1636 struct igc_queue *q = txr->txr_igcq; 1637 1638 if (__predict_false(!pcq_put(txr->txr_interq, m))) { 1639 IGC_QUEUE_EVENT(q, tx_pcq_drop, 1); 1640 m_freem(m); 1641 return ENOBUFS; 1642 } 1643 1644 mutex_enter(&txr->txr_lock); 1645 igc_tx_common_locked(ifp, txr, IGC_TX_TRANSMIT); 1646 mutex_exit(&txr->txr_lock); 1647 1648 return 0; 1649 } 1650 1651 static void 1652 igc_tx_common_locked(struct ifnet *ifp, struct tx_ring *txr, int caller) 1653 { 1654 struct igc_softc *sc = ifp->if_softc; 1655 struct igc_queue *q = txr->txr_igcq; 1656 net_stat_ref_t nsr = IF_STAT_GETREF(ifp); 1657 int prod, free, last = -1; 1658 bool post = false; 1659 1660 prod = txr->next_avail_desc; 1661 free = txr->next_to_clean; 1662 if (free <= prod) 1663 free += sc->num_tx_desc; 1664 free -= prod; 1665 1666 DPRINTF(TX, "%s: begin: msix %d prod %d n2c %d free %d\n", 1667 caller == IGC_TX_TRANSMIT ? "transmit" : "start", 1668 txr->me, prod, txr->next_to_clean, free); 1669 1670 for (;;) { 1671 struct mbuf *m; 1672 1673 if (__predict_false(free <= IGC_MAX_SCATTER)) { 1674 IGC_QUEUE_EVENT(q, tx_no_desc, 1); 1675 break; 1676 } 1677 1678 if (caller == IGC_TX_TRANSMIT) 1679 m = pcq_get(txr->txr_interq); 1680 else 1681 IFQ_DEQUEUE(&ifp->if_snd, m); 1682 if (__predict_false(m == NULL)) 1683 break; 1684 1685 struct igc_tx_buf *txbuf = &txr->tx_buffers[prod]; 1686 bus_dmamap_t map = txbuf->map; 1687 1688 if (__predict_false( 1689 igc_load_mbuf(q, txr->txdma.dma_tag, map, m))) { 1690 if (caller == IGC_TX_TRANSMIT) 1691 IGC_QUEUE_EVENT(q, tx_pcq_drop, 1); 1692 m_freem(m); 1693 if_statinc_ref(nsr, if_oerrors); 1694 continue; 1695 } 1696 1697 bus_dmamap_sync(txr->txdma.dma_tag, map, 0, 1698 map->dm_mapsize, BUS_DMASYNC_PREWRITE); 1699 1700 uint32_t ctx_cmd_type_len = 0, olinfo_status = 0; 1701 if (igc_tx_ctx_setup(txr, m, prod, &ctx_cmd_type_len, 1702 &olinfo_status)) { 1703 IGC_QUEUE_EVENT(q, tx_ctx, 1); 1704 /* Consume the first descriptor */ 1705 prod = igc_txdesc_incr(sc, prod); 1706 free--; 1707 } 1708 for (int i = 0; i < map->dm_nsegs; i++) { 1709 union igc_adv_tx_desc *txdesc = &txr->tx_base[prod]; 1710 1711 uint32_t cmd_type_len = ctx_cmd_type_len | 1712 IGC_ADVTXD_DCMD_IFCS | IGC_ADVTXD_DTYP_DATA | 1713 IGC_ADVTXD_DCMD_DEXT | map->dm_segs[i].ds_len; 1714 if (i == map->dm_nsegs - 1) { 1715 cmd_type_len |= 1716 IGC_ADVTXD_DCMD_EOP | IGC_ADVTXD_DCMD_RS; 1717 } 1718 1719 igc_txdesc_sync(txr, prod, 1720 BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE); 1721 htolem64(&txdesc->read.buffer_addr, 1722 map->dm_segs[i].ds_addr); 1723 htolem32(&txdesc->read.cmd_type_len, cmd_type_len); 1724 htolem32(&txdesc->read.olinfo_status, olinfo_status); 1725 igc_txdesc_sync(txr, prod, 1726 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); 1727 1728 last = prod; 1729 prod = igc_txdesc_incr(sc, prod); 1730 } 1731 1732 txbuf->m_head = m; 1733 txbuf->eop_index = last; 1734 1735 bpf_mtap(ifp, m, BPF_D_OUT); 1736 1737 if_statadd_ref(nsr, if_obytes, m->m_pkthdr.len); 1738 if (m->m_flags & M_MCAST) 1739 if_statinc_ref(nsr, if_omcasts); 1740 IGC_QUEUE_EVENT(q, tx_packets, 1); 1741 IGC_QUEUE_EVENT(q, tx_bytes, m->m_pkthdr.len); 1742 1743 free -= map->dm_nsegs; 1744 post = true; 1745 } 1746 1747 if (post) { 1748 txr->next_avail_desc = prod; 1749 IGC_WRITE_REG(&sc->hw, IGC_TDT(txr->me), prod); 1750 } 1751 1752 DPRINTF(TX, "%s: done : msix %d prod %d n2c %d free %d\n", 1753 caller == IGC_TX_TRANSMIT ? "transmit" : "start", 1754 txr->me, prod, txr->next_to_clean, free); 1755 1756 IF_STAT_PUTREF(ifp); 1757 } 1758 1759 static bool 1760 igc_txeof(struct tx_ring *txr, u_int limit) 1761 { 1762 struct igc_softc *sc = txr->sc; 1763 struct ifnet *ifp = &sc->sc_ec.ec_if; 1764 int cons, prod; 1765 bool more = false; 1766 1767 prod = txr->next_avail_desc; 1768 cons = txr->next_to_clean; 1769 1770 if (cons == prod) { 1771 DPRINTF(TX, "false: msix %d cons %d prod %d\n", 1772 txr->me, cons, prod); 1773 return false; 1774 } 1775 1776 do { 1777 struct igc_tx_buf *txbuf = &txr->tx_buffers[cons]; 1778 const int last = txbuf->eop_index; 1779 1780 membar_consumer(); /* XXXRO necessary? */ 1781 1782 KASSERT(last != -1); 1783 union igc_adv_tx_desc *txdesc = &txr->tx_base[last]; 1784 igc_txdesc_sync(txr, last, BUS_DMASYNC_POSTREAD); 1785 const uint32_t status = le32toh(txdesc->wb.status); 1786 igc_txdesc_sync(txr, last, BUS_DMASYNC_PREREAD); 1787 1788 if (!(status & IGC_TXD_STAT_DD)) 1789 break; 1790 1791 if (limit-- == 0) { 1792 more = true; 1793 DPRINTF(TX, "pending TX " 1794 "msix %d cons %d last %d prod %d " 1795 "status 0x%08x\n", 1796 txr->me, cons, last, prod, status); 1797 break; 1798 } 1799 1800 DPRINTF(TX, "handled TX " 1801 "msix %d cons %d last %d prod %d " 1802 "status 0x%08x\n", 1803 txr->me, cons, last, prod, status); 1804 1805 if_statinc(ifp, if_opackets); 1806 1807 bus_dmamap_t map = txbuf->map; 1808 bus_dmamap_sync(txr->txdma.dma_tag, map, 0, map->dm_mapsize, 1809 BUS_DMASYNC_POSTWRITE); 1810 bus_dmamap_unload(txr->txdma.dma_tag, map); 1811 m_freem(txbuf->m_head); 1812 1813 txbuf->m_head = NULL; 1814 txbuf->eop_index = -1; 1815 1816 cons = igc_txdesc_incr(sc, last); 1817 } while (cons != prod); 1818 1819 txr->next_to_clean = cons; 1820 1821 return more; 1822 } 1823 1824 static void 1825 igc_intr_barrier(struct igc_softc *sc __unused) 1826 { 1827 1828 xc_barrier(0); 1829 } 1830 1831 static void 1832 igc_stop(struct ifnet *ifp, int disable) 1833 { 1834 struct igc_softc *sc = ifp->if_softc; 1835 1836 mutex_enter(&sc->sc_core_lock); 1837 igc_stop_locked(sc); 1838 mutex_exit(&sc->sc_core_lock); 1839 } 1840 1841 /********************************************************************* 1842 * 1843 * This routine disables all traffic on the adapter by issuing a 1844 * global reset on the MAC. 1845 * 1846 **********************************************************************/ 1847 static void 1848 igc_stop_locked(struct igc_softc *sc) 1849 { 1850 struct ifnet *ifp = &sc->sc_ec.ec_if; 1851 1852 DPRINTF(CFG, "called\n"); 1853 1854 KASSERT(mutex_owned(&sc->sc_core_lock)); 1855 1856 /* 1857 * If stopping processing has already started, do nothing. 1858 */ 1859 if ((ifp->if_flags & IFF_RUNNING) == 0) 1860 return; 1861 1862 /* Tell the stack that the interface is no longer active. */ 1863 ifp->if_flags &= ~IFF_RUNNING; 1864 1865 /* 1866 * igc_handle_queue() can enable interrupts, so wait for completion of 1867 * last igc_handle_queue() after unset IFF_RUNNING. 1868 */ 1869 mutex_exit(&sc->sc_core_lock); 1870 igc_barrier_handle_queue(sc); 1871 mutex_enter(&sc->sc_core_lock); 1872 1873 sc->sc_core_stopping = true; 1874 1875 igc_disable_intr(sc); 1876 1877 callout_halt(&sc->sc_tick_ch, &sc->sc_core_lock); 1878 1879 igc_reset_hw(&sc->hw); 1880 IGC_WRITE_REG(&sc->hw, IGC_WUC, 0); 1881 1882 /* 1883 * Wait for completion of interrupt handlers. 1884 */ 1885 mutex_exit(&sc->sc_core_lock); 1886 igc_intr_barrier(sc); 1887 mutex_enter(&sc->sc_core_lock); 1888 1889 igc_update_link_status(sc); 1890 1891 for (int iq = 0; iq < sc->sc_nqueues; iq++) { 1892 struct tx_ring *txr = &sc->tx_rings[iq]; 1893 1894 igc_withdraw_transmit_packets(txr, false); 1895 } 1896 1897 for (int iq = 0; iq < sc->sc_nqueues; iq++) { 1898 struct rx_ring *rxr = &sc->rx_rings[iq]; 1899 1900 igc_clear_receive_status(rxr); 1901 } 1902 1903 /* Save last flags for the callback */ 1904 sc->sc_if_flags = ifp->if_flags; 1905 } 1906 1907 /********************************************************************* 1908 * Ioctl entry point 1909 * 1910 * igc_ioctl is called when the user wants to configure the 1911 * interface. 1912 * 1913 * return 0 on success, positive on failure 1914 **********************************************************************/ 1915 static int 1916 igc_ioctl(struct ifnet * ifp, u_long cmd, void *data) 1917 { 1918 struct igc_softc *sc __unused = ifp->if_softc; 1919 int s; 1920 int error; 1921 1922 DPRINTF(CFG, "cmd 0x%016lx\n", cmd); 1923 1924 switch (cmd) { 1925 case SIOCADDMULTI: 1926 case SIOCDELMULTI: 1927 break; 1928 default: 1929 KASSERT(IFNET_LOCKED(ifp)); 1930 } 1931 1932 if (cmd == SIOCZIFDATA) { 1933 mutex_enter(&sc->sc_core_lock); 1934 igc_clear_counters(sc); 1935 mutex_exit(&sc->sc_core_lock); 1936 } 1937 1938 switch (cmd) { 1939 #ifdef IF_RXR 1940 case SIOCGIFRXR: 1941 s = splnet(); 1942 error = igc_rxrinfo(sc, (struct if_rxrinfo *)ifr->ifr_data); 1943 splx(s); 1944 break; 1945 #endif 1946 default: 1947 s = splnet(); 1948 error = ether_ioctl(ifp, cmd, data); 1949 splx(s); 1950 break; 1951 } 1952 1953 if (error != ENETRESET) 1954 return error; 1955 1956 error = 0; 1957 1958 if (cmd == SIOCSIFCAP) 1959 error = if_init(ifp); 1960 else if ((cmd == SIOCADDMULTI) || (cmd == SIOCDELMULTI)) { 1961 mutex_enter(&sc->sc_core_lock); 1962 if (sc->sc_if_flags & IFF_RUNNING) { 1963 /* 1964 * Multicast list has changed; set the hardware filter 1965 * accordingly. 1966 */ 1967 igc_disable_intr(sc); 1968 igc_set_filter(sc); 1969 igc_enable_intr(sc); 1970 } 1971 mutex_exit(&sc->sc_core_lock); 1972 } 1973 1974 return error; 1975 } 1976 1977 #ifdef IF_RXR 1978 static int 1979 igc_rxrinfo(struct igc_softc *sc, struct if_rxrinfo *ifri) 1980 { 1981 struct if_rxring_info *ifr, ifr1; 1982 int error; 1983 1984 if (sc->sc_nqueues > 1) { 1985 ifr = kmem_zalloc(sc->sc_nqueues * sizeof(*ifr), KM_SLEEP); 1986 } else { 1987 ifr = &ifr1; 1988 memset(ifr, 0, sizeof(*ifr)); 1989 } 1990 1991 for (int iq = 0; iq < sc->sc_nqueues; iq++) { 1992 struct rx_ring *rxr = &sc->rx_rings[iq]; 1993 1994 ifr[iq].ifr_size = MCLBYTES; 1995 snprintf(ifr[iq].ifr_name, sizeof(ifr[iq].ifr_name), "%d", iq); 1996 ifr[iq].ifr_info = rxr->rx_ring; 1997 } 1998 1999 error = if_rxr_info_ioctl(ifri, sc->sc_nqueues, ifr); 2000 if (sc->sc_nqueues > 1) 2001 kmem_free(ifr, sc->sc_nqueues * sizeof(*ifr)); 2002 2003 return error; 2004 } 2005 #endif 2006 2007 static void 2008 igc_rxfill(struct rx_ring *rxr) 2009 { 2010 struct igc_softc *sc = rxr->sc; 2011 int id; 2012 2013 for (id = 0; id < sc->num_rx_desc; id++) { 2014 if (igc_get_buf(rxr, id, false)) { 2015 panic("%s: msix=%d i=%d\n", __func__, rxr->me, id); 2016 } 2017 } 2018 2019 id = sc->num_rx_desc - 1; 2020 rxr->last_desc_filled = id; 2021 IGC_WRITE_REG(&sc->hw, IGC_RDT(rxr->me), id); 2022 rxr->next_to_check = 0; 2023 } 2024 2025 static void 2026 igc_rxrefill(struct rx_ring *rxr, int end) 2027 { 2028 struct igc_softc *sc = rxr->sc; 2029 int id; 2030 2031 for (id = rxr->next_to_check; id != end; id = igc_rxdesc_incr(sc, id)) { 2032 if (igc_get_buf(rxr, id, true)) { 2033 /* XXXRO */ 2034 panic("%s: msix=%d id=%d\n", __func__, rxr->me, id); 2035 } 2036 } 2037 2038 id = igc_rxdesc_decr(sc, id); 2039 DPRINTF(RX, "%s RDT %d id %d\n", 2040 rxr->last_desc_filled == id ? "same" : "diff", 2041 rxr->last_desc_filled, id); 2042 rxr->last_desc_filled = id; 2043 IGC_WRITE_REG(&sc->hw, IGC_RDT(rxr->me), id); 2044 } 2045 2046 /********************************************************************* 2047 * 2048 * This routine executes in interrupt context. It replenishes 2049 * the mbufs in the descriptor and sends data which has been 2050 * dma'ed into host memory to upper layer. 2051 * 2052 *********************************************************************/ 2053 static bool 2054 igc_rxeof(struct rx_ring *rxr, u_int limit) 2055 { 2056 struct igc_softc *sc = rxr->sc; 2057 struct igc_queue *q = rxr->rxr_igcq; 2058 struct ifnet *ifp = &sc->sc_ec.ec_if; 2059 int id; 2060 bool more = false; 2061 2062 id = rxr->next_to_check; 2063 for (;;) { 2064 union igc_adv_rx_desc *rxdesc = &rxr->rx_base[id]; 2065 struct igc_rx_buf *rxbuf, *nxbuf; 2066 struct mbuf *mp, *m; 2067 2068 igc_rxdesc_sync(rxr, id, 2069 BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE); 2070 2071 const uint32_t staterr = le32toh(rxdesc->wb.upper.status_error); 2072 2073 if (!ISSET(staterr, IGC_RXD_STAT_DD)) { 2074 igc_rxdesc_sync(rxr, id, 2075 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); 2076 break; 2077 } 2078 2079 if (limit-- == 0) { 2080 igc_rxdesc_sync(rxr, id, 2081 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); 2082 DPRINTF(RX, "more=true\n"); 2083 more = true; 2084 break; 2085 } 2086 2087 /* Zero out the receive descriptors status. */ 2088 rxdesc->wb.upper.status_error = 0; 2089 2090 /* Pull the mbuf off the ring. */ 2091 rxbuf = &rxr->rx_buffers[id]; 2092 bus_dmamap_t map = rxbuf->map; 2093 bus_dmamap_sync(rxr->rxdma.dma_tag, map, 2094 0, map->dm_mapsize, BUS_DMASYNC_POSTREAD); 2095 bus_dmamap_unload(rxr->rxdma.dma_tag, map); 2096 2097 mp = rxbuf->buf; 2098 rxbuf->buf = NULL; 2099 2100 const bool eop = staterr & IGC_RXD_STAT_EOP; 2101 const uint16_t len = le16toh(rxdesc->wb.upper.length); 2102 2103 #if NVLAN > 0 2104 const uint16_t vtag = le16toh(rxdesc->wb.upper.vlan); 2105 #endif 2106 2107 const uint32_t ptype = le32toh(rxdesc->wb.lower.lo_dword.data) & 2108 IGC_PKTTYPE_MASK; 2109 2110 const uint32_t hash __unused = 2111 le32toh(rxdesc->wb.lower.hi_dword.rss); 2112 const uint16_t hashtype __unused = 2113 le16toh(rxdesc->wb.lower.lo_dword.hs_rss.pkt_info) & 2114 IGC_RXDADV_RSSTYPE_MASK; 2115 2116 igc_rxdesc_sync(rxr, id, 2117 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); 2118 2119 if (__predict_false(staterr & IGC_RXDEXT_STATERR_RXE)) { 2120 if (rxbuf->fmp) { 2121 m_freem(rxbuf->fmp); 2122 rxbuf->fmp = NULL; 2123 } 2124 2125 m_freem(mp); 2126 m = NULL; 2127 2128 if_statinc(ifp, if_ierrors); 2129 IGC_QUEUE_EVENT(q, rx_discard, 1); 2130 2131 DPRINTF(RX, "ierrors++\n"); 2132 2133 goto next_desc; 2134 } 2135 2136 if (__predict_false(mp == NULL)) { 2137 panic("%s: igc_rxeof: NULL mbuf in slot %d " 2138 "(filled %d)", device_xname(sc->sc_dev), 2139 id, rxr->last_desc_filled); 2140 } 2141 2142 if (!eop) { 2143 /* 2144 * Figure out the next descriptor of this frame. 2145 */ 2146 int nextp = igc_rxdesc_incr(sc, id); 2147 2148 nxbuf = &rxr->rx_buffers[nextp]; 2149 /* 2150 * TODO prefetch(nxbuf); 2151 */ 2152 } 2153 2154 mp->m_len = len; 2155 2156 m = rxbuf->fmp; 2157 rxbuf->fmp = NULL; 2158 2159 if (m != NULL) { 2160 m->m_pkthdr.len += mp->m_len; 2161 } else { 2162 m = mp; 2163 m->m_pkthdr.len = mp->m_len; 2164 #if NVLAN > 0 2165 if (staterr & IGC_RXD_STAT_VP) 2166 vlan_set_tag(m, vtag); 2167 #endif 2168 } 2169 2170 /* Pass the head pointer on */ 2171 if (!eop) { 2172 nxbuf->fmp = m; 2173 m = NULL; 2174 mp->m_next = nxbuf->buf; 2175 } else { 2176 m_set_rcvif(m, ifp); 2177 2178 m->m_pkthdr.csum_flags = igc_rx_checksum(q, 2179 ifp->if_capenable, staterr, ptype); 2180 2181 #ifdef notyet 2182 if (hashtype != IGC_RXDADV_RSSTYPE_NONE) { 2183 m->m_pkthdr.ph_flowid = hash; 2184 SET(m->m_pkthdr.csum_flags, M_FLOWID); 2185 } 2186 ml_enqueue(&ml, m); 2187 #endif 2188 2189 if_percpuq_enqueue(sc->sc_ipq, m); 2190 2191 if_statinc(ifp, if_ipackets); 2192 IGC_QUEUE_EVENT(q, rx_packets, 1); 2193 IGC_QUEUE_EVENT(q, rx_bytes, m->m_pkthdr.len); 2194 } 2195 next_desc: 2196 /* Advance our pointers to the next descriptor. */ 2197 id = igc_rxdesc_incr(sc, id); 2198 } 2199 2200 DPRINTF(RX, "fill queue[%d]\n", rxr->me); 2201 igc_rxrefill(rxr, id); 2202 2203 DPRINTF(RX, "%s n2c %d id %d\n", 2204 rxr->next_to_check == id ? "same" : "diff", 2205 rxr->next_to_check, id); 2206 rxr->next_to_check = id; 2207 2208 #ifdef OPENBSD 2209 if (!(staterr & IGC_RXD_STAT_DD)) 2210 return 0; 2211 #endif 2212 2213 return more; 2214 } 2215 2216 /********************************************************************* 2217 * 2218 * Verify that the hardware indicated that the checksum is valid. 2219 * Inform the stack about the status of checksum so that stack 2220 * doesn't spend time verifying the checksum. 2221 * 2222 *********************************************************************/ 2223 static int 2224 igc_rx_checksum(struct igc_queue *q, uint64_t capenable, uint32_t staterr, 2225 uint32_t ptype) 2226 { 2227 const uint16_t status = (uint16_t)staterr; 2228 const uint8_t errors = (uint8_t)(staterr >> 24); 2229 int flags = 0; 2230 2231 if ((status & IGC_RXD_STAT_IPCS) != 0 && 2232 (capenable & IFCAP_CSUM_IPv4_Rx) != 0) { 2233 IGC_DRIVER_EVENT(q, rx_ipcs, 1); 2234 flags |= M_CSUM_IPv4; 2235 if (__predict_false((errors & IGC_RXD_ERR_IPE) != 0)) { 2236 IGC_DRIVER_EVENT(q, rx_ipcs_bad, 1); 2237 flags |= M_CSUM_IPv4_BAD; 2238 } 2239 } 2240 2241 if ((status & IGC_RXD_STAT_TCPCS) != 0) { 2242 IGC_DRIVER_EVENT(q, rx_tcpcs, 1); 2243 if ((capenable & IFCAP_CSUM_TCPv4_Rx) != 0) 2244 flags |= M_CSUM_TCPv4; 2245 if ((capenable & IFCAP_CSUM_TCPv6_Rx) != 0) 2246 flags |= M_CSUM_TCPv6; 2247 } 2248 2249 if ((status & IGC_RXD_STAT_UDPCS) != 0) { 2250 IGC_DRIVER_EVENT(q, rx_udpcs, 1); 2251 if ((capenable & IFCAP_CSUM_UDPv4_Rx) != 0) 2252 flags |= M_CSUM_UDPv4; 2253 if ((capenable & IFCAP_CSUM_UDPv6_Rx) != 0) 2254 flags |= M_CSUM_UDPv6; 2255 } 2256 2257 if (__predict_false((errors & IGC_RXD_ERR_TCPE) != 0)) { 2258 IGC_DRIVER_EVENT(q, rx_l4cs_bad, 1); 2259 if ((flags & ~M_CSUM_IPv4) != 0) 2260 flags |= M_CSUM_TCP_UDP_BAD; 2261 } 2262 2263 return flags; 2264 } 2265 2266 static void 2267 igc_watchdog(struct ifnet * ifp) 2268 { 2269 } 2270 2271 static void 2272 igc_tick(void *arg) 2273 { 2274 struct igc_softc *sc = arg; 2275 2276 mutex_enter(&sc->sc_core_lock); 2277 2278 if (__predict_false(sc->sc_core_stopping)) { 2279 mutex_exit(&sc->sc_core_lock); 2280 return; 2281 } 2282 2283 /* XXX watchdog */ 2284 if (0) { 2285 IGC_GLOBAL_EVENT(sc, watchdog, 1); 2286 } 2287 2288 igc_update_counters(sc); 2289 2290 mutex_exit(&sc->sc_core_lock); 2291 2292 callout_schedule(&sc->sc_tick_ch, hz); 2293 } 2294 2295 /********************************************************************* 2296 * 2297 * Media Ioctl callback 2298 * 2299 * This routine is called whenever the user queries the status of 2300 * the interface using ifconfig. 2301 * 2302 **********************************************************************/ 2303 static void 2304 igc_media_status(struct ifnet *ifp, struct ifmediareq *ifmr) 2305 { 2306 struct igc_softc *sc = ifp->if_softc; 2307 struct igc_hw *hw = &sc->hw; 2308 2309 igc_update_link_status(sc); 2310 2311 ifmr->ifm_status = IFM_AVALID; 2312 ifmr->ifm_active = IFM_ETHER; 2313 2314 if (!sc->link_active) { 2315 ifmr->ifm_active |= IFM_NONE; 2316 return; 2317 } 2318 2319 ifmr->ifm_status |= IFM_ACTIVE; 2320 2321 switch (sc->link_speed) { 2322 case 10: 2323 ifmr->ifm_active |= IFM_10_T; 2324 break; 2325 case 100: 2326 ifmr->ifm_active |= IFM_100_TX; 2327 break; 2328 case 1000: 2329 ifmr->ifm_active |= IFM_1000_T; 2330 break; 2331 case 2500: 2332 ifmr->ifm_active |= IFM_2500_T; 2333 break; 2334 } 2335 2336 if (sc->link_duplex == FULL_DUPLEX) 2337 ifmr->ifm_active |= IFM_FDX; 2338 else 2339 ifmr->ifm_active |= IFM_HDX; 2340 2341 switch (hw->fc.current_mode) { 2342 case igc_fc_tx_pause: 2343 ifmr->ifm_active |= IFM_FLOW | IFM_ETH_TXPAUSE; 2344 break; 2345 case igc_fc_rx_pause: 2346 ifmr->ifm_active |= IFM_FLOW | IFM_ETH_RXPAUSE; 2347 break; 2348 case igc_fc_full: 2349 ifmr->ifm_active |= IFM_FLOW | 2350 IFM_ETH_TXPAUSE | IFM_ETH_RXPAUSE; 2351 break; 2352 case igc_fc_none: 2353 default: 2354 break; 2355 } 2356 } 2357 2358 /********************************************************************* 2359 * 2360 * Media Ioctl callback 2361 * 2362 * This routine is called when the user changes speed/duplex using 2363 * media/mediopt option with ifconfig. 2364 * 2365 **********************************************************************/ 2366 static int 2367 igc_media_change(struct ifnet *ifp) 2368 { 2369 struct igc_softc *sc = ifp->if_softc; 2370 struct ifmedia *ifm = &sc->media; 2371 2372 if (IFM_TYPE(ifm->ifm_media) != IFM_ETHER) 2373 return EINVAL; 2374 2375 sc->hw.mac.autoneg = DO_AUTO_NEG; 2376 2377 switch (IFM_SUBTYPE(ifm->ifm_media)) { 2378 case IFM_AUTO: 2379 sc->hw.phy.autoneg_advertised = AUTONEG_ADV_DEFAULT; 2380 break; 2381 case IFM_2500_T: 2382 sc->hw.phy.autoneg_advertised = ADVERTISE_2500_FULL; 2383 break; 2384 case IFM_1000_T: 2385 sc->hw.phy.autoneg_advertised = ADVERTISE_1000_FULL; 2386 break; 2387 case IFM_100_TX: 2388 if ((ifm->ifm_media & IFM_GMASK) == IFM_FDX) 2389 sc->hw.phy.autoneg_advertised = ADVERTISE_100_FULL; 2390 else 2391 sc->hw.phy.autoneg_advertised = ADVERTISE_100_HALF; 2392 break; 2393 case IFM_10_T: 2394 if ((ifm->ifm_media & IFM_GMASK) == IFM_FDX) 2395 sc->hw.phy.autoneg_advertised = ADVERTISE_10_FULL; 2396 else 2397 sc->hw.phy.autoneg_advertised = ADVERTISE_10_HALF; 2398 break; 2399 default: 2400 return EINVAL; 2401 } 2402 2403 igc_init_locked(sc); 2404 2405 return 0; 2406 } 2407 2408 static int 2409 igc_ifflags_cb(struct ethercom *ec) 2410 { 2411 struct ifnet *ifp = &ec->ec_if; 2412 struct igc_softc *sc = ifp->if_softc; 2413 int rc = 0; 2414 u_short iffchange; 2415 bool needreset = false; 2416 2417 DPRINTF(CFG, "called\n"); 2418 2419 KASSERT(IFNET_LOCKED(ifp)); 2420 2421 mutex_enter(&sc->sc_core_lock); 2422 2423 /* 2424 * Check for if_flags. 2425 * Main usage is to prevent linkdown when opening bpf. 2426 */ 2427 iffchange = ifp->if_flags ^ sc->sc_if_flags; 2428 sc->sc_if_flags = ifp->if_flags; 2429 if ((iffchange & ~(IFF_CANTCHANGE | IFF_DEBUG)) != 0) { 2430 needreset = true; 2431 goto ec; 2432 } 2433 2434 /* iff related updates */ 2435 if ((iffchange & IFF_PROMISC) != 0) 2436 igc_set_filter(sc); 2437 2438 #ifdef notyet 2439 igc_set_vlan(sc); 2440 #endif 2441 2442 ec: 2443 #ifdef notyet 2444 /* Check for ec_capenable. */ 2445 ecchange = ec->ec_capenable ^ sc->sc_ec_capenable; 2446 sc->sc_ec_capenable = ec->ec_capenable; 2447 if ((ecchange & ~ETHERCAP_SOMETHING) != 0) { 2448 needreset = true; 2449 goto out; 2450 } 2451 #endif 2452 if (needreset) 2453 rc = ENETRESET; 2454 2455 mutex_exit(&sc->sc_core_lock); 2456 2457 return rc; 2458 } 2459 2460 static void 2461 igc_set_filter(struct igc_softc *sc) 2462 { 2463 struct ethercom *ec = &sc->sc_ec; 2464 uint32_t rctl; 2465 2466 rctl = IGC_READ_REG(&sc->hw, IGC_RCTL); 2467 rctl &= ~(IGC_RCTL_BAM |IGC_RCTL_UPE | IGC_RCTL_MPE); 2468 2469 if ((sc->sc_if_flags & IFF_BROADCAST) != 0) 2470 rctl |= IGC_RCTL_BAM; 2471 if ((sc->sc_if_flags & IFF_PROMISC) != 0) { 2472 DPRINTF(CFG, "promisc\n"); 2473 rctl |= IGC_RCTL_UPE; 2474 ETHER_LOCK(ec); 2475 allmulti: 2476 ec->ec_flags |= ETHER_F_ALLMULTI; 2477 ETHER_UNLOCK(ec); 2478 rctl |= IGC_RCTL_MPE; 2479 } else { 2480 struct ether_multistep step; 2481 struct ether_multi *enm; 2482 int mcnt = 0; 2483 2484 memset(sc->mta, 0, IGC_MTA_LEN); 2485 2486 ETHER_LOCK(ec); 2487 ETHER_FIRST_MULTI(step, ec, enm); 2488 while (enm != NULL) { 2489 if (((memcmp(enm->enm_addrlo, enm->enm_addrhi, 2490 ETHER_ADDR_LEN)) != 0) || 2491 (mcnt >= MAX_NUM_MULTICAST_ADDRESSES)) { 2492 /* 2493 * We must listen to a range of multicast 2494 * addresses. For now, just accept all 2495 * multicasts, rather than trying to set only 2496 * those filter bits needed to match the range. 2497 * (At this time, the only use of address 2498 * ranges is for IP multicast routing, for 2499 * which the range is big enough to require all 2500 * bits set.) 2501 */ 2502 goto allmulti; 2503 } 2504 DPRINTF(CFG, "%d: %s\n", mcnt, 2505 ether_sprintf(enm->enm_addrlo)); 2506 memcpy(&sc->mta[mcnt * ETHER_ADDR_LEN], 2507 enm->enm_addrlo, ETHER_ADDR_LEN); 2508 2509 mcnt++; 2510 ETHER_NEXT_MULTI(step, enm); 2511 } 2512 ec->ec_flags &= ~ETHER_F_ALLMULTI; 2513 ETHER_UNLOCK(ec); 2514 2515 DPRINTF(CFG, "hw filter\n"); 2516 igc_update_mc_addr_list(&sc->hw, sc->mta, mcnt); 2517 } 2518 2519 IGC_WRITE_REG(&sc->hw, IGC_RCTL, rctl); 2520 } 2521 2522 static void 2523 igc_update_link_status(struct igc_softc *sc) 2524 { 2525 struct ifnet *ifp = &sc->sc_ec.ec_if; 2526 struct igc_hw *hw = &sc->hw; 2527 2528 if (IGC_READ_REG(&sc->hw, IGC_STATUS) & IGC_STATUS_LU) { 2529 if (sc->link_active == 0) { 2530 igc_get_speed_and_duplex(hw, &sc->link_speed, 2531 &sc->link_duplex); 2532 sc->link_active = 1; 2533 ifp->if_baudrate = IF_Mbps(sc->link_speed); 2534 if_link_state_change(ifp, LINK_STATE_UP); 2535 } 2536 } else { 2537 if (sc->link_active == 1) { 2538 ifp->if_baudrate = sc->link_speed = 0; 2539 sc->link_duplex = 0; 2540 sc->link_active = 0; 2541 if_link_state_change(ifp, LINK_STATE_DOWN); 2542 } 2543 } 2544 } 2545 2546 /********************************************************************* 2547 * 2548 * Get a buffer from system mbuf buffer pool. 2549 * 2550 **********************************************************************/ 2551 static int 2552 igc_get_buf(struct rx_ring *rxr, int id, bool strict) 2553 { 2554 struct igc_softc *sc = rxr->sc; 2555 struct igc_queue *q = rxr->rxr_igcq; 2556 struct igc_rx_buf *rxbuf = &rxr->rx_buffers[id]; 2557 bus_dmamap_t map = rxbuf->map; 2558 struct mbuf *m; 2559 int error; 2560 2561 if (__predict_false(rxbuf->buf)) { 2562 if (strict) { 2563 DPRINTF(RX, "slot %d already has an mbuf\n", id); 2564 return EINVAL; 2565 } 2566 return 0; 2567 } 2568 2569 MGETHDR(m, M_DONTWAIT, MT_DATA); 2570 if (__predict_false(m == NULL)) { 2571 enobuf: 2572 IGC_QUEUE_EVENT(q, rx_no_mbuf, 1); 2573 return ENOBUFS; 2574 } 2575 2576 MCLGET(m, M_DONTWAIT); 2577 if (__predict_false(!(m->m_flags & M_EXT))) { 2578 m_freem(m); 2579 goto enobuf; 2580 } 2581 2582 m->m_len = m->m_pkthdr.len = sc->rx_mbuf_sz; 2583 2584 error = bus_dmamap_load_mbuf(rxr->rxdma.dma_tag, map, m, 2585 BUS_DMA_READ | BUS_DMA_NOWAIT); 2586 if (error) { 2587 m_freem(m); 2588 return error; 2589 } 2590 2591 bus_dmamap_sync(rxr->rxdma.dma_tag, map, 0, 2592 map->dm_mapsize, BUS_DMASYNC_PREREAD); 2593 rxbuf->buf = m; 2594 2595 union igc_adv_rx_desc *rxdesc = &rxr->rx_base[id]; 2596 igc_rxdesc_sync(rxr, id, BUS_DMASYNC_POSTWRITE | BUS_DMASYNC_POSTREAD); 2597 rxdesc->read.pkt_addr = htole64(map->dm_segs[0].ds_addr); 2598 igc_rxdesc_sync(rxr, id, BUS_DMASYNC_PREWRITE | BUS_DMASYNC_PREREAD); 2599 2600 return 0; 2601 } 2602 2603 static void 2604 igc_configure_queues(struct igc_softc *sc) 2605 { 2606 struct igc_hw *hw = &sc->hw; 2607 uint32_t ivar; 2608 2609 /* First turn on RSS capability */ 2610 IGC_WRITE_REG(hw, IGC_GPIE, IGC_GPIE_MSIX_MODE | IGC_GPIE_EIAME | 2611 IGC_GPIE_PBA | IGC_GPIE_NSICR); 2612 2613 /* Set the starting interrupt rate */ 2614 uint32_t newitr = (4000000 / MAX_INTS_PER_SEC) & 0x7FFC; 2615 newitr |= IGC_EITR_CNT_IGNR; 2616 2617 /* Turn on MSI-X */ 2618 uint32_t newmask = 0; 2619 for (int iq = 0; iq < sc->sc_nqueues; iq++) { 2620 struct igc_queue *q = &sc->queues[iq]; 2621 2622 /* RX entries */ 2623 igc_set_queues(sc, iq, q->msix, 0); 2624 /* TX entries */ 2625 igc_set_queues(sc, iq, q->msix, 1); 2626 newmask |= q->eims; 2627 IGC_WRITE_REG(hw, IGC_EITR(q->msix), newitr); 2628 } 2629 sc->msix_queuesmask = newmask; 2630 2631 #if 1 2632 ivar = IGC_READ_REG_ARRAY(hw, IGC_IVAR0, 0); 2633 DPRINTF(CFG, "ivar(0)=0x%x\n", ivar); 2634 ivar = IGC_READ_REG_ARRAY(hw, IGC_IVAR0, 1); 2635 DPRINTF(CFG, "ivar(1)=0x%x\n", ivar); 2636 #endif 2637 2638 /* And for the link interrupt */ 2639 ivar = (sc->linkvec | IGC_IVAR_VALID) << 8; 2640 sc->msix_linkmask = 1 << sc->linkvec; 2641 IGC_WRITE_REG(hw, IGC_IVAR_MISC, ivar); 2642 } 2643 2644 static void 2645 igc_set_queues(struct igc_softc *sc, uint32_t entry, uint32_t vector, int type) 2646 { 2647 struct igc_hw *hw = &sc->hw; 2648 const uint32_t index = entry >> 1; 2649 uint32_t ivar = IGC_READ_REG_ARRAY(hw, IGC_IVAR0, index); 2650 2651 if (type) { 2652 if (entry & 1) { 2653 ivar &= 0x00FFFFFF; 2654 ivar |= (vector | IGC_IVAR_VALID) << 24; 2655 } else { 2656 ivar &= 0xFFFF00FF; 2657 ivar |= (vector | IGC_IVAR_VALID) << 8; 2658 } 2659 } else { 2660 if (entry & 1) { 2661 ivar &= 0xFF00FFFF; 2662 ivar |= (vector | IGC_IVAR_VALID) << 16; 2663 } else { 2664 ivar &= 0xFFFFFF00; 2665 ivar |= vector | IGC_IVAR_VALID; 2666 } 2667 } 2668 IGC_WRITE_REG_ARRAY(hw, IGC_IVAR0, index, ivar); 2669 } 2670 2671 static void 2672 igc_enable_queue(struct igc_softc *sc, uint32_t eims) 2673 { 2674 IGC_WRITE_REG(&sc->hw, IGC_EIMS, eims); 2675 } 2676 2677 static void 2678 igc_enable_intr(struct igc_softc *sc) 2679 { 2680 struct igc_hw *hw = &sc->hw; 2681 2682 if (sc->sc_intr_type == PCI_INTR_TYPE_MSIX) { 2683 const uint32_t mask = sc->msix_queuesmask | sc->msix_linkmask; 2684 2685 IGC_WRITE_REG(hw, IGC_EIAC, mask); 2686 IGC_WRITE_REG(hw, IGC_EIAM, mask); 2687 IGC_WRITE_REG(hw, IGC_EIMS, mask); 2688 IGC_WRITE_REG(hw, IGC_IMS, IGC_IMS_LSC); 2689 } else { 2690 IGC_WRITE_REG(hw, IGC_IMS, IMS_ENABLE_MASK); 2691 } 2692 IGC_WRITE_FLUSH(hw); 2693 } 2694 2695 static void 2696 igc_disable_intr(struct igc_softc *sc) 2697 { 2698 struct igc_hw *hw = &sc->hw; 2699 2700 if (sc->sc_intr_type == PCI_INTR_TYPE_MSIX) { 2701 IGC_WRITE_REG(hw, IGC_EIMC, 0xffffffff); 2702 IGC_WRITE_REG(hw, IGC_EIAC, 0); 2703 } 2704 IGC_WRITE_REG(hw, IGC_IMC, 0xffffffff); 2705 IGC_WRITE_FLUSH(hw); 2706 } 2707 2708 static int 2709 igc_intr_link(void *arg) 2710 { 2711 struct igc_softc *sc = (struct igc_softc *)arg; 2712 const uint32_t reg_icr = IGC_READ_REG(&sc->hw, IGC_ICR); 2713 2714 IGC_GLOBAL_EVENT(sc, link, 1); 2715 2716 if (reg_icr & IGC_ICR_LSC) { 2717 mutex_enter(&sc->sc_core_lock); 2718 sc->hw.mac.get_link_status = true; 2719 igc_update_link_status(sc); 2720 mutex_exit(&sc->sc_core_lock); 2721 } 2722 2723 IGC_WRITE_REG(&sc->hw, IGC_IMS, IGC_IMS_LSC); 2724 IGC_WRITE_REG(&sc->hw, IGC_EIMS, sc->msix_linkmask); 2725 2726 return 1; 2727 } 2728 2729 static int 2730 igc_intr_queue(void *arg) 2731 { 2732 struct igc_queue *iq = arg; 2733 struct igc_softc *sc = iq->sc; 2734 struct ifnet *ifp = &sc->sc_ec.ec_if; 2735 struct rx_ring *rxr = iq->rxr; 2736 struct tx_ring *txr = iq->txr; 2737 const u_int txlimit = sc->sc_tx_intr_process_limit, 2738 rxlimit = sc->sc_rx_intr_process_limit; 2739 bool txmore, rxmore; 2740 2741 IGC_QUEUE_EVENT(iq, irqs, 1); 2742 2743 if (__predict_false(!ISSET(ifp->if_flags, IFF_RUNNING))) 2744 return 0; 2745 2746 mutex_enter(&txr->txr_lock); 2747 txmore = igc_txeof(txr, txlimit); 2748 mutex_exit(&txr->txr_lock); 2749 mutex_enter(&rxr->rxr_lock); 2750 rxmore = igc_rxeof(rxr, rxlimit); 2751 mutex_exit(&rxr->rxr_lock); 2752 2753 if (txmore || rxmore) { 2754 IGC_QUEUE_EVENT(iq, req, 1); 2755 igc_sched_handle_queue(sc, iq); 2756 } else { 2757 igc_enable_queue(sc, iq->eims); 2758 } 2759 2760 return 1; 2761 } 2762 2763 static int 2764 igc_intr(void *arg) 2765 { 2766 struct igc_softc *sc = arg; 2767 struct ifnet *ifp = &sc->sc_ec.ec_if; 2768 struct igc_queue *iq = &sc->queues[0]; 2769 struct rx_ring *rxr = iq->rxr; 2770 struct tx_ring *txr = iq->txr; 2771 const u_int txlimit = sc->sc_tx_intr_process_limit, 2772 rxlimit = sc->sc_rx_intr_process_limit; 2773 bool txmore, rxmore; 2774 2775 if (__predict_false(!ISSET(ifp->if_flags, IFF_RUNNING))) 2776 return 0; 2777 2778 const uint32_t reg_icr = IGC_READ_REG(&sc->hw, IGC_ICR); 2779 DPRINTF(MISC, "reg_icr=0x%x\n", reg_icr); 2780 2781 /* Definitely not our interrupt. */ 2782 if (reg_icr == 0x0) { 2783 DPRINTF(MISC, "not for me"); 2784 return 0; 2785 } 2786 2787 IGC_QUEUE_EVENT(iq, irqs, 1); 2788 2789 /* Hot eject? */ 2790 if (__predict_false(reg_icr == 0xffffffff)) { 2791 DPRINTF(MISC, "hot eject\n"); 2792 return 0; 2793 } 2794 2795 if (__predict_false(!(reg_icr & IGC_ICR_INT_ASSERTED))) { 2796 DPRINTF(MISC, "not set IGC_ICR_INT_ASSERTED"); 2797 return 0; 2798 } 2799 2800 /* 2801 * Only MSI-X interrupts have one-shot behavior by taking advantage 2802 * of the EIAC register. Thus, explicitly disable interrupts. This 2803 * also works around the MSI message reordering errata on certain 2804 * systems. 2805 */ 2806 igc_disable_intr(sc); 2807 2808 mutex_enter(&txr->txr_lock); 2809 txmore = igc_txeof(txr, txlimit); 2810 mutex_exit(&txr->txr_lock); 2811 mutex_enter(&rxr->rxr_lock); 2812 rxmore = igc_rxeof(rxr, rxlimit); 2813 mutex_exit(&rxr->rxr_lock); 2814 2815 /* Link status change */ 2816 // XXXX FreeBSD checks IGC_ICR_RXSEQ 2817 if (__predict_false(reg_icr & IGC_ICR_LSC)) { 2818 IGC_GLOBAL_EVENT(sc, link, 1); 2819 mutex_enter(&sc->sc_core_lock); 2820 sc->hw.mac.get_link_status = true; 2821 igc_update_link_status(sc); 2822 mutex_exit(&sc->sc_core_lock); 2823 } 2824 2825 if (txmore || rxmore) { 2826 IGC_QUEUE_EVENT(iq, req, 1); 2827 igc_sched_handle_queue(sc, iq); 2828 } else { 2829 igc_enable_intr(sc); 2830 } 2831 2832 return 1; 2833 } 2834 2835 static void 2836 igc_handle_queue(void *arg) 2837 { 2838 struct igc_queue *iq = arg; 2839 struct igc_softc *sc = iq->sc; 2840 struct tx_ring *txr = iq->txr; 2841 struct rx_ring *rxr = iq->rxr; 2842 const u_int txlimit = sc->sc_tx_process_limit, 2843 rxlimit = sc->sc_rx_process_limit; 2844 bool txmore, rxmore; 2845 2846 IGC_QUEUE_EVENT(iq, handleq, 1); 2847 2848 mutex_enter(&txr->txr_lock); 2849 txmore = igc_txeof(txr, txlimit); 2850 /* for ALTQ, dequeue from if_snd */ 2851 if (txr->me == 0) { 2852 struct ifnet *ifp = &sc->sc_ec.ec_if; 2853 2854 igc_tx_common_locked(ifp, txr, IGC_TX_START); 2855 } 2856 mutex_exit(&txr->txr_lock); 2857 2858 mutex_enter(&rxr->rxr_lock); 2859 rxmore = igc_rxeof(rxr, rxlimit); 2860 mutex_exit(&rxr->rxr_lock); 2861 2862 if (txmore || rxmore) { 2863 igc_sched_handle_queue(sc, iq); 2864 } else { 2865 if (sc->sc_intr_type == PCI_INTR_TYPE_MSIX) 2866 igc_enable_queue(sc, iq->eims); 2867 else 2868 igc_enable_intr(sc); 2869 } 2870 } 2871 2872 static void 2873 igc_handle_queue_work(struct work *wk, void *context) 2874 { 2875 struct igc_queue *iq = 2876 container_of(wk, struct igc_queue, igcq_wq_cookie); 2877 2878 igc_handle_queue(iq); 2879 } 2880 2881 static void 2882 igc_sched_handle_queue(struct igc_softc *sc, struct igc_queue *iq) 2883 { 2884 2885 if (iq->igcq_workqueue) { 2886 /* XXXRO notyet */ 2887 workqueue_enqueue(sc->sc_queue_wq, &iq->igcq_wq_cookie, 2888 curcpu()); 2889 } else { 2890 softint_schedule(iq->igcq_si); 2891 } 2892 } 2893 2894 static void 2895 igc_barrier_handle_queue(struct igc_softc *sc) 2896 { 2897 2898 if (sc->sc_txrx_workqueue) { 2899 for (int iq = 0; iq < sc->sc_nqueues; iq++) { 2900 struct igc_queue *q = &sc->queues[iq]; 2901 2902 workqueue_wait(sc->sc_queue_wq, &q->igcq_wq_cookie); 2903 } 2904 } else { 2905 xc_barrier(0); 2906 } 2907 } 2908 2909 /********************************************************************* 2910 * 2911 * Allocate memory for tx_buffer structures. The tx_buffer stores all 2912 * the information needed to transmit a packet on the wire. 2913 * 2914 **********************************************************************/ 2915 static int 2916 igc_allocate_transmit_buffers(struct tx_ring *txr) 2917 { 2918 struct igc_softc *sc = txr->sc; 2919 int error; 2920 2921 txr->tx_buffers = 2922 kmem_zalloc(sc->num_tx_desc * sizeof(struct igc_tx_buf), KM_SLEEP); 2923 txr->txtag = txr->txdma.dma_tag; 2924 2925 /* Create the descriptor buffer dma maps. */ 2926 for (int id = 0; id < sc->num_tx_desc; id++) { 2927 struct igc_tx_buf *txbuf = &txr->tx_buffers[id]; 2928 2929 error = bus_dmamap_create(txr->txdma.dma_tag, 2930 round_page(IGC_TSO_SIZE + sizeof(struct ether_vlan_header)), 2931 IGC_MAX_SCATTER, PAGE_SIZE, 0, BUS_DMA_NOWAIT, &txbuf->map); 2932 if (error != 0) { 2933 aprint_error_dev(sc->sc_dev, 2934 "unable to create TX DMA map\n"); 2935 goto fail; 2936 } 2937 2938 txbuf->eop_index = -1; 2939 } 2940 2941 return 0; 2942 fail: 2943 return error; 2944 } 2945 2946 2947 /********************************************************************* 2948 * 2949 * Allocate and initialize transmit structures. 2950 * 2951 **********************************************************************/ 2952 static int 2953 igc_setup_transmit_structures(struct igc_softc *sc) 2954 { 2955 2956 for (int iq = 0; iq < sc->sc_nqueues; iq++) { 2957 struct tx_ring *txr = &sc->tx_rings[iq]; 2958 2959 if (igc_setup_transmit_ring(txr)) 2960 goto fail; 2961 } 2962 2963 return 0; 2964 fail: 2965 igc_free_transmit_structures(sc); 2966 return ENOBUFS; 2967 } 2968 2969 /********************************************************************* 2970 * 2971 * Initialize a transmit ring. 2972 * 2973 **********************************************************************/ 2974 static int 2975 igc_setup_transmit_ring(struct tx_ring *txr) 2976 { 2977 struct igc_softc *sc = txr->sc; 2978 2979 /* Now allocate transmit buffers for the ring. */ 2980 if (igc_allocate_transmit_buffers(txr)) 2981 return ENOMEM; 2982 2983 /* Clear the old ring contents */ 2984 memset(txr->tx_base, 0, 2985 sizeof(union igc_adv_tx_desc) * sc->num_tx_desc); 2986 2987 /* Reset indices. */ 2988 txr->next_avail_desc = 0; 2989 txr->next_to_clean = 0; 2990 2991 bus_dmamap_sync(txr->txdma.dma_tag, txr->txdma.dma_map, 0, 2992 txr->txdma.dma_map->dm_mapsize, 2993 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); 2994 2995 txr->txr_interq = pcq_create(sc->num_tx_desc, KM_SLEEP); 2996 2997 mutex_init(&txr->txr_lock, MUTEX_DEFAULT, IPL_NET); 2998 2999 return 0; 3000 } 3001 3002 /********************************************************************* 3003 * 3004 * Enable transmit unit. 3005 * 3006 **********************************************************************/ 3007 static void 3008 igc_initialize_transmit_unit(struct igc_softc *sc) 3009 { 3010 struct ifnet *ifp = &sc->sc_ec.ec_if; 3011 struct igc_hw *hw = &sc->hw; 3012 3013 /* Setup the Base and Length of the TX descriptor ring. */ 3014 for (int iq = 0; iq < sc->sc_nqueues; iq++) { 3015 struct tx_ring *txr = &sc->tx_rings[iq]; 3016 const uint64_t bus_addr = 3017 txr->txdma.dma_map->dm_segs[0].ds_addr; 3018 3019 /* Base and len of TX ring */ 3020 IGC_WRITE_REG(hw, IGC_TDLEN(iq), 3021 sc->num_tx_desc * sizeof(union igc_adv_tx_desc)); 3022 IGC_WRITE_REG(hw, IGC_TDBAH(iq), (uint32_t)(bus_addr >> 32)); 3023 IGC_WRITE_REG(hw, IGC_TDBAL(iq), (uint32_t)bus_addr); 3024 3025 /* Init the HEAD/TAIL indices */ 3026 IGC_WRITE_REG(hw, IGC_TDT(iq), 0 /* XXX txr->next_avail_desc */); 3027 IGC_WRITE_REG(hw, IGC_TDH(iq), 0); 3028 3029 txr->watchdog_timer = 0; 3030 3031 uint32_t txdctl = 0; /* Clear txdctl */ 3032 txdctl |= 0x1f; /* PTHRESH */ 3033 txdctl |= 1 << 8; /* HTHRESH */ 3034 txdctl |= 1 << 16; /* WTHRESH */ 3035 txdctl |= 1 << 22; /* Reserved bit 22 must always be 1 */ 3036 txdctl |= IGC_TXDCTL_GRAN; 3037 txdctl |= 1 << 25; /* LWTHRESH */ 3038 3039 IGC_WRITE_REG(hw, IGC_TXDCTL(iq), txdctl); 3040 } 3041 ifp->if_timer = 0; 3042 3043 /* Program the Transmit Control Register */ 3044 uint32_t tctl = IGC_READ_REG(&sc->hw, IGC_TCTL); 3045 tctl &= ~IGC_TCTL_CT; 3046 tctl |= (IGC_TCTL_PSP | IGC_TCTL_RTLC | IGC_TCTL_EN | 3047 (IGC_COLLISION_THRESHOLD << IGC_CT_SHIFT)); 3048 3049 /* This write will effectively turn on the transmit unit. */ 3050 IGC_WRITE_REG(&sc->hw, IGC_TCTL, tctl); 3051 } 3052 3053 /********************************************************************* 3054 * 3055 * Free all transmit rings. 3056 * 3057 **********************************************************************/ 3058 static void 3059 igc_free_transmit_structures(struct igc_softc *sc) 3060 { 3061 3062 for (int iq = 0; iq < sc->sc_nqueues; iq++) { 3063 struct tx_ring *txr = &sc->tx_rings[iq]; 3064 3065 igc_free_transmit_buffers(txr); 3066 } 3067 } 3068 3069 /********************************************************************* 3070 * 3071 * Free transmit ring related data structures. 3072 * 3073 **********************************************************************/ 3074 static void 3075 igc_free_transmit_buffers(struct tx_ring *txr) 3076 { 3077 struct igc_softc *sc = txr->sc; 3078 3079 if (txr->tx_buffers == NULL) 3080 return; 3081 3082 igc_withdraw_transmit_packets(txr, true); 3083 3084 kmem_free(txr->tx_buffers, 3085 sc->num_tx_desc * sizeof(struct igc_tx_buf)); 3086 txr->tx_buffers = NULL; 3087 txr->txtag = NULL; 3088 3089 pcq_destroy(txr->txr_interq); 3090 mutex_destroy(&txr->txr_lock); 3091 } 3092 3093 /********************************************************************* 3094 * 3095 * Withdraw transmit packets. 3096 * 3097 **********************************************************************/ 3098 static void 3099 igc_withdraw_transmit_packets(struct tx_ring *txr, bool destroy) 3100 { 3101 struct igc_softc *sc = txr->sc; 3102 struct igc_queue *q = txr->txr_igcq; 3103 3104 mutex_enter(&txr->txr_lock); 3105 3106 for (int id = 0; id < sc->num_tx_desc; id++) { 3107 union igc_adv_tx_desc *txdesc = &txr->tx_base[id]; 3108 3109 igc_txdesc_sync(txr, id, 3110 BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE); 3111 txdesc->read.buffer_addr = 0; 3112 txdesc->read.cmd_type_len = 0; 3113 txdesc->read.olinfo_status = 0; 3114 igc_txdesc_sync(txr, id, 3115 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); 3116 3117 struct igc_tx_buf *txbuf = &txr->tx_buffers[id]; 3118 bus_dmamap_t map = txbuf->map; 3119 3120 if (map != NULL && map->dm_nsegs > 0) { 3121 bus_dmamap_sync(txr->txdma.dma_tag, map, 3122 0, map->dm_mapsize, BUS_DMASYNC_POSTWRITE); 3123 bus_dmamap_unload(txr->txdma.dma_tag, map); 3124 } 3125 if (txbuf->m_head != NULL) { 3126 m_freem(txbuf->m_head); 3127 txbuf->m_head = NULL; 3128 } 3129 if (map != NULL && destroy) { 3130 bus_dmamap_destroy(txr->txdma.dma_tag, map); 3131 txbuf->map = NULL; 3132 } 3133 txbuf->eop_index = -1; 3134 3135 txr->next_avail_desc = 0; 3136 txr->next_to_clean = 0; 3137 } 3138 3139 struct mbuf *m; 3140 while ((m = pcq_get(txr->txr_interq)) != NULL) { 3141 IGC_QUEUE_EVENT(q, tx_pcq_drop, 1); 3142 m_freem(m); 3143 } 3144 3145 mutex_exit(&txr->txr_lock); 3146 } 3147 3148 3149 /********************************************************************* 3150 * 3151 * Advanced Context Descriptor setup for VLAN, CSUM or TSO 3152 * 3153 **********************************************************************/ 3154 3155 static int 3156 igc_tx_ctx_setup(struct tx_ring *txr, struct mbuf *mp, int prod, 3157 uint32_t *cmd_type_len, uint32_t *olinfo_status) 3158 { 3159 struct ether_vlan_header *evl; 3160 uint32_t type_tucmd_mlhl = 0; 3161 uint32_t vlan_macip_lens = 0; 3162 uint32_t ehlen, iphlen; 3163 uint16_t ehtype; 3164 int off = 0; 3165 3166 const int csum_flags = mp->m_pkthdr.csum_flags; 3167 3168 /* First check if TSO is to be used */ 3169 if ((csum_flags & (M_CSUM_TSOv4 | M_CSUM_TSOv6)) != 0) { 3170 return igc_tso_setup(txr, mp, prod, cmd_type_len, 3171 olinfo_status); 3172 } 3173 3174 const bool v4 = (csum_flags & 3175 (M_CSUM_IPv4 | M_CSUM_TCPv4 | M_CSUM_UDPv4)) != 0; 3176 const bool v6 = (csum_flags & (M_CSUM_UDPv6 | M_CSUM_TCPv6)) != 0; 3177 3178 /* Indicate the whole packet as payload when not doing TSO */ 3179 *olinfo_status |= mp->m_pkthdr.len << IGC_ADVTXD_PAYLEN_SHIFT; 3180 3181 /* 3182 * In advanced descriptors the vlan tag must 3183 * be placed into the context descriptor. Hence 3184 * we need to make one even if not doing offloads. 3185 */ 3186 #if NVLAN > 0 3187 if (vlan_has_tag(mp)) { 3188 vlan_macip_lens |= (uint32_t)vlan_get_tag(mp) 3189 << IGC_ADVTXD_VLAN_SHIFT; 3190 off = 1; 3191 } else 3192 #endif 3193 if (!v4 && !v6) 3194 return 0; 3195 3196 KASSERT(mp->m_len >= sizeof(struct ether_header)); 3197 evl = mtod(mp, struct ether_vlan_header *); 3198 if (evl->evl_encap_proto == htons(ETHERTYPE_VLAN)) { 3199 KASSERT(mp->m_len >= sizeof(struct ether_vlan_header)); 3200 ehlen = ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN; 3201 ehtype = evl->evl_proto; 3202 } else { 3203 ehlen = ETHER_HDR_LEN; 3204 ehtype = evl->evl_encap_proto; 3205 } 3206 3207 vlan_macip_lens |= ehlen << IGC_ADVTXD_MACLEN_SHIFT; 3208 3209 #ifdef IGC_DEBUG 3210 /* 3211 * For checksum offloading, L3 headers are not mandatory. 3212 * We use these only for consistency checks. 3213 */ 3214 struct ip *ip; 3215 struct ip6_hdr *ip6; 3216 uint8_t ipproto; 3217 char *l3d; 3218 3219 if (mp->m_len == ehlen && mp->m_next != NULL) 3220 l3d = mtod(mp->m_next, char *); 3221 else 3222 l3d = mtod(mp, char *) + ehlen; 3223 #endif 3224 3225 switch (ntohs(ehtype)) { 3226 case ETHERTYPE_IP: 3227 iphlen = M_CSUM_DATA_IPv4_IPHL(mp->m_pkthdr.csum_data); 3228 type_tucmd_mlhl |= IGC_ADVTXD_TUCMD_IPV4; 3229 3230 if ((csum_flags & M_CSUM_IPv4) != 0) { 3231 *olinfo_status |= IGC_TXD_POPTS_IXSM << 8; 3232 off = 1; 3233 } 3234 #ifdef IGC_DEBUG 3235 KASSERT(!v6); 3236 ip = (void *)l3d; 3237 ipproto = ip->ip_p; 3238 KASSERT(iphlen == ip->ip_hl << 2); 3239 KASSERT((mp->m_pkthdr.csum_flags & M_CSUM_IPv4) == 0 || 3240 ip->ip_sum == 0); 3241 #endif 3242 break; 3243 case ETHERTYPE_IPV6: 3244 iphlen = M_CSUM_DATA_IPv6_IPHL(mp->m_pkthdr.csum_data); 3245 type_tucmd_mlhl |= IGC_ADVTXD_TUCMD_IPV6; 3246 #ifdef IGC_DEBUG 3247 KASSERT(!v4); 3248 ip6 = (void *)l3d; 3249 ipproto = ip6->ip6_nxt; /* XXX */ 3250 KASSERT(iphlen == sizeof(struct ip6_hdr)); 3251 #endif 3252 break; 3253 default: 3254 /* 3255 * Unknown L3 protocol. Clear L3 header length and proceed for 3256 * LAN as done by Linux driver. 3257 */ 3258 iphlen = 0; 3259 #ifdef IGC_DEBUG 3260 KASSERT(!v4 && !v6); 3261 ipproto = 0; 3262 #endif 3263 break; 3264 } 3265 3266 vlan_macip_lens |= iphlen; 3267 3268 const bool tcp = (csum_flags & (M_CSUM_TCPv4 | M_CSUM_TCPv6)) != 0; 3269 const bool udp = (csum_flags & (M_CSUM_UDPv4 | M_CSUM_UDPv6)) != 0; 3270 3271 if (tcp) { 3272 #ifdef IGC_DEBUG 3273 KASSERTMSG(ipproto == IPPROTO_TCP, "ipproto = %d", ipproto); 3274 #endif 3275 type_tucmd_mlhl |= IGC_ADVTXD_TUCMD_L4T_TCP; 3276 *olinfo_status |= IGC_TXD_POPTS_TXSM << 8; 3277 off = 1; 3278 } else if (udp) { 3279 #ifdef IGC_DEBUG 3280 KASSERTMSG(ipproto == IPPROTO_UDP, "ipproto = %d", ipproto); 3281 #endif 3282 type_tucmd_mlhl |= IGC_ADVTXD_TUCMD_L4T_UDP; 3283 *olinfo_status |= IGC_TXD_POPTS_TXSM << 8; 3284 off = 1; 3285 } 3286 3287 if (off == 0) 3288 return 0; 3289 3290 type_tucmd_mlhl |= IGC_ADVTXD_DCMD_DEXT | IGC_ADVTXD_DTYP_CTXT; 3291 3292 /* Now ready a context descriptor */ 3293 struct igc_adv_tx_context_desc *txdesc = 3294 (struct igc_adv_tx_context_desc *)&txr->tx_base[prod]; 3295 3296 /* Now copy bits into descriptor */ 3297 igc_txdesc_sync(txr, prod, 3298 BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE); 3299 htolem32(&txdesc->vlan_macip_lens, vlan_macip_lens); 3300 htolem32(&txdesc->type_tucmd_mlhl, type_tucmd_mlhl); 3301 htolem32(&txdesc->seqnum_seed, 0); 3302 htolem32(&txdesc->mss_l4len_idx, 0); 3303 igc_txdesc_sync(txr, prod, 3304 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); 3305 3306 return 1; 3307 } 3308 3309 /********************************************************************* 3310 * 3311 * Advanced Context Descriptor setup for TSO 3312 * 3313 * XXX XXXRO 3314 * Not working. Some packets are sent with correct csums, but 3315 * others aren't. th->th_sum may be adjusted. 3316 * 3317 **********************************************************************/ 3318 3319 static int 3320 igc_tso_setup(struct tx_ring *txr, struct mbuf *mp, int prod, 3321 uint32_t *cmd_type_len, uint32_t *olinfo_status) 3322 { 3323 #if 1 /* notyet */ 3324 return 0; 3325 #else 3326 struct ether_vlan_header *evl; 3327 struct ip *ip; 3328 struct ip6_hdr *ip6; 3329 struct tcphdr *th; 3330 uint32_t type_tucmd_mlhl = 0; 3331 uint32_t vlan_macip_lens = 0; 3332 uint32_t mss_l4len_idx = 0; 3333 uint32_t ehlen, iphlen, tcphlen, paylen; 3334 uint16_t ehtype; 3335 3336 /* 3337 * In advanced descriptors the vlan tag must 3338 * be placed into the context descriptor. Hence 3339 * we need to make one even if not doing offloads. 3340 */ 3341 #if NVLAN > 0 3342 if (vlan_has_tag(mp)) { 3343 vlan_macip_lens |= (uint32_t)vlan_get_tag(mp) 3344 << IGC_ADVTXD_VLAN_SHIFT; 3345 } 3346 #endif 3347 3348 KASSERT(mp->m_len >= sizeof(struct ether_header)); 3349 evl = mtod(mp, struct ether_vlan_header *); 3350 if (evl->evl_encap_proto == htons(ETHERTYPE_VLAN)) { 3351 KASSERT(mp->m_len >= sizeof(struct ether_vlan_header)); 3352 ehlen = ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN; 3353 ehtype = evl->evl_proto; 3354 } else { 3355 ehlen = ETHER_HDR_LEN; 3356 ehtype = evl->evl_encap_proto; 3357 } 3358 3359 vlan_macip_lens |= ehlen << IGC_ADVTXD_MACLEN_SHIFT; 3360 3361 switch (ntohs(ehtype)) { 3362 case ETHERTYPE_IP: 3363 iphlen = M_CSUM_DATA_IPv4_IPHL(mp->m_pkthdr.csum_data); 3364 type_tucmd_mlhl |= IGC_ADVTXD_TUCMD_IPV4; 3365 *olinfo_status |= IGC_TXD_POPTS_IXSM << 8; 3366 3367 KASSERT(mp->m_len >= ehlen + sizeof(*ip)); 3368 ip = (void *)(mtod(mp, char *) + ehlen); 3369 ip->ip_len = 0; 3370 KASSERT(iphlen == ip->ip_hl << 2); 3371 KASSERT(ip->ip_sum == 0); 3372 KASSERT(ip->ip_p == IPPROTO_TCP); 3373 3374 KASSERT(mp->m_len >= ehlen + iphlen + sizeof(*th)); 3375 th = (void *)((char *)ip + iphlen); 3376 th->th_sum = in_cksum_phdr(ip->ip_src.s_addr, ip->ip_dst.s_addr, 3377 htons(IPPROTO_TCP)); 3378 break; 3379 case ETHERTYPE_IPV6: 3380 iphlen = M_CSUM_DATA_IPv6_IPHL(mp->m_pkthdr.csum_data); 3381 type_tucmd_mlhl |= IGC_ADVTXD_TUCMD_IPV6; 3382 3383 KASSERT(mp->m_len >= ehlen + sizeof(*ip6)); 3384 ip6 = (void *)(mtod(mp, char *) + ehlen); 3385 ip6->ip6_plen = 0; 3386 KASSERT(iphlen == sizeof(struct ip6_hdr)); 3387 KASSERT(ip6->ip6_nxt == IPPROTO_TCP); 3388 3389 KASSERT(mp->m_len >= ehlen + iphlen + sizeof(*th)); 3390 th = (void *)((char *)ip6 + iphlen); 3391 tcphlen = th->th_off << 2; 3392 paylen = mp->m_pkthdr.len - ehlen - iphlen - tcphlen; 3393 th->th_sum = in6_cksum_phdr(&ip6->ip6_src, &ip6->ip6_dst, 0, 3394 htonl(IPPROTO_TCP)); 3395 break; 3396 default: 3397 panic("%s", __func__); 3398 } 3399 3400 tcphlen = th->th_off << 2; 3401 paylen = mp->m_pkthdr.len - ehlen - iphlen - tcphlen; 3402 3403 vlan_macip_lens |= iphlen; 3404 3405 type_tucmd_mlhl |= IGC_ADVTXD_DCMD_DEXT | IGC_ADVTXD_DTYP_CTXT; 3406 type_tucmd_mlhl |= IGC_ADVTXD_TUCMD_L4T_TCP; 3407 3408 mss_l4len_idx |= mp->m_pkthdr.segsz << IGC_ADVTXD_MSS_SHIFT; 3409 mss_l4len_idx |= tcphlen << IGC_ADVTXD_L4LEN_SHIFT; 3410 3411 /* Now ready a context descriptor */ 3412 struct igc_adv_tx_context_desc *txdesc = 3413 (struct igc_adv_tx_context_desc *)&txr->tx_base[prod]; 3414 3415 /* Now copy bits into descriptor */ 3416 igc_txdesc_sync(txr, prod, 3417 BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE); 3418 htolem32(&txdesc->vlan_macip_lens, vlan_macip_lens); 3419 htolem32(&txdesc->type_tucmd_mlhl, type_tucmd_mlhl); 3420 htolem32(&txdesc->seqnum_seed, 0); 3421 htolem32(&txdesc->mss_l4len_idx, mss_l4len_idx); 3422 igc_txdesc_sync(txr, prod, 3423 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); 3424 3425 *cmd_type_len |= IGC_ADVTXD_DCMD_TSE; 3426 *olinfo_status |= IGC_TXD_POPTS_TXSM << 8; 3427 *olinfo_status |= paylen << IGC_ADVTXD_PAYLEN_SHIFT; 3428 3429 return 1; 3430 #endif /* notyet */ 3431 } 3432 3433 /********************************************************************* 3434 * 3435 * Allocate memory for rx_buffer structures. Since we use one 3436 * rx_buffer per received packet, the maximum number of rx_buffer's 3437 * that we'll need is equal to the number of receive descriptors 3438 * that we've allocated. 3439 * 3440 **********************************************************************/ 3441 static int 3442 igc_allocate_receive_buffers(struct rx_ring *rxr) 3443 { 3444 struct igc_softc *sc = rxr->sc; 3445 int error; 3446 3447 rxr->rx_buffers = 3448 kmem_zalloc(sc->num_rx_desc * sizeof(struct igc_rx_buf), KM_SLEEP); 3449 3450 for (int id = 0; id < sc->num_rx_desc; id++) { 3451 struct igc_rx_buf *rxbuf = &rxr->rx_buffers[id]; 3452 3453 error = bus_dmamap_create(rxr->rxdma.dma_tag, MCLBYTES, 1, 3454 MCLBYTES, 0, BUS_DMA_WAITOK, &rxbuf->map); 3455 if (error) { 3456 aprint_error_dev(sc->sc_dev, 3457 "unable to create RX DMA map\n"); 3458 goto fail; 3459 } 3460 } 3461 bus_dmamap_sync(rxr->rxdma.dma_tag, rxr->rxdma.dma_map, 0, 3462 rxr->rxdma.dma_map->dm_mapsize, 3463 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); 3464 3465 return 0; 3466 fail: 3467 return error; 3468 } 3469 3470 /********************************************************************* 3471 * 3472 * Allocate and initialize receive structures. 3473 * 3474 **********************************************************************/ 3475 static int 3476 igc_setup_receive_structures(struct igc_softc *sc) 3477 { 3478 3479 for (int iq = 0; iq < sc->sc_nqueues; iq++) { 3480 struct rx_ring *rxr = &sc->rx_rings[iq]; 3481 3482 if (igc_setup_receive_ring(rxr)) 3483 goto fail; 3484 } 3485 3486 return 0; 3487 fail: 3488 igc_free_receive_structures(sc); 3489 return ENOBUFS; 3490 } 3491 3492 /********************************************************************* 3493 * 3494 * Initialize a receive ring and its buffers. 3495 * 3496 **********************************************************************/ 3497 static int 3498 igc_setup_receive_ring(struct rx_ring *rxr) 3499 { 3500 struct igc_softc *sc = rxr->sc; 3501 const int rsize = roundup2( 3502 sc->num_rx_desc * sizeof(union igc_adv_rx_desc), IGC_DBA_ALIGN); 3503 3504 /* Clear the ring contents. */ 3505 memset(rxr->rx_base, 0, rsize); 3506 3507 if (igc_allocate_receive_buffers(rxr)) 3508 return ENOMEM; 3509 3510 /* Setup our descriptor indices. */ 3511 rxr->next_to_check = 0; 3512 rxr->last_desc_filled = 0; 3513 3514 mutex_init(&rxr->rxr_lock, MUTEX_DEFAULT, IPL_NET); 3515 3516 return 0; 3517 } 3518 3519 /********************************************************************* 3520 * 3521 * Enable receive unit. 3522 * 3523 **********************************************************************/ 3524 static void 3525 igc_initialize_receive_unit(struct igc_softc *sc) 3526 { 3527 struct ifnet *ifp = &sc->sc_ec.ec_if; 3528 struct igc_hw *hw = &sc->hw; 3529 uint32_t rctl, rxcsum, srrctl; 3530 3531 DPRINTF(RX, "called\n"); 3532 3533 /* 3534 * Make sure receives are disabled while setting 3535 * up the descriptor ring. 3536 */ 3537 rctl = IGC_READ_REG(hw, IGC_RCTL); 3538 IGC_WRITE_REG(hw, IGC_RCTL, rctl & ~IGC_RCTL_EN); 3539 3540 /* Setup the Receive Control Register */ 3541 rctl &= ~(3 << IGC_RCTL_MO_SHIFT); 3542 rctl |= IGC_RCTL_EN | IGC_RCTL_BAM | IGC_RCTL_LBM_NO | 3543 IGC_RCTL_RDMTS_HALF | (hw->mac.mc_filter_type << IGC_RCTL_MO_SHIFT); 3544 3545 #if 1 3546 /* Do not store bad packets */ 3547 rctl &= ~IGC_RCTL_SBP; 3548 #else 3549 /* for debug */ 3550 rctl |= IGC_RCTL_SBP; 3551 #endif 3552 3553 /* Enable Long Packet receive */ 3554 if (sc->hw.mac.max_frame_size > ETHER_MAX_LEN) 3555 rctl |= IGC_RCTL_LPE; 3556 else 3557 rctl &= ~IGC_RCTL_LPE; 3558 3559 /* Strip the CRC */ 3560 rctl |= IGC_RCTL_SECRC; 3561 3562 /* 3563 * Set the interrupt throttling rate. Value is calculated 3564 * as DEFAULT_ITR = 1/(MAX_INTS_PER_SEC * 256ns) 3565 * 3566 * XXX Sync with Linux, especially for jumbo MTU or TSO. 3567 * XXX Shouldn't be here? 3568 */ 3569 IGC_WRITE_REG(hw, IGC_ITR, DEFAULT_ITR); 3570 3571 rxcsum = IGC_READ_REG(hw, IGC_RXCSUM); 3572 rxcsum &= ~(IGC_RXCSUM_IPOFL | IGC_RXCSUM_TUOFL | IGC_RXCSUM_PCSD); 3573 if (ifp->if_capenable & IFCAP_CSUM_IPv4_Rx) 3574 rxcsum |= IGC_RXCSUM_IPOFL; 3575 if (ifp->if_capenable & (IFCAP_CSUM_TCPv4_Rx | IFCAP_CSUM_UDPv4_Rx | 3576 IFCAP_CSUM_TCPv6_Rx | IFCAP_CSUM_UDPv6_Rx)) 3577 rxcsum |= IGC_RXCSUM_TUOFL; 3578 if (sc->sc_nqueues > 1) 3579 rxcsum |= IGC_RXCSUM_PCSD; 3580 IGC_WRITE_REG(hw, IGC_RXCSUM, rxcsum); 3581 3582 if (sc->sc_nqueues > 1) 3583 igc_initialize_rss_mapping(sc); 3584 3585 srrctl = 0; 3586 #if 0 3587 srrctl |= 4096 >> IGC_SRRCTL_BSIZEPKT_SHIFT; 3588 rctl |= IGC_RCTL_SZ_4096 | IGC_RCTL_BSEX; 3589 #else 3590 srrctl |= 2048 >> IGC_SRRCTL_BSIZEPKT_SHIFT; 3591 rctl |= IGC_RCTL_SZ_2048; 3592 #endif 3593 3594 /* 3595 * If TX flow control is disabled and there's > 1 queue defined, 3596 * enable DROP. 3597 * 3598 * This drops frames rather than hanging the RX MAC for all queues. 3599 */ 3600 if (sc->sc_nqueues > 1 && 3601 (sc->fc == igc_fc_none || sc->fc == igc_fc_rx_pause)) 3602 srrctl |= IGC_SRRCTL_DROP_EN; 3603 3604 /* Setup the Base and Length of the RX descriptor rings. */ 3605 for (int iq = 0; iq < sc->sc_nqueues; iq++) { 3606 struct rx_ring *rxr = &sc->rx_rings[iq]; 3607 const uint64_t bus_addr = 3608 rxr->rxdma.dma_map->dm_segs[0].ds_addr; 3609 3610 IGC_WRITE_REG(hw, IGC_RXDCTL(iq), 0); 3611 3612 srrctl |= IGC_SRRCTL_DESCTYPE_ADV_ONEBUF; 3613 3614 IGC_WRITE_REG(hw, IGC_RDLEN(iq), 3615 sc->num_rx_desc * sizeof(union igc_adv_rx_desc)); 3616 IGC_WRITE_REG(hw, IGC_RDBAH(iq), (uint32_t)(bus_addr >> 32)); 3617 IGC_WRITE_REG(hw, IGC_RDBAL(iq), (uint32_t)bus_addr); 3618 IGC_WRITE_REG(hw, IGC_SRRCTL(iq), srrctl); 3619 3620 /* Setup the Head and Tail Descriptor Pointers */ 3621 IGC_WRITE_REG(hw, IGC_RDH(iq), 0); 3622 IGC_WRITE_REG(hw, IGC_RDT(iq), 0 /* XXX rxr->last_desc_filled */); 3623 3624 /* Enable this Queue */ 3625 uint32_t rxdctl = IGC_READ_REG(hw, IGC_RXDCTL(iq)); 3626 rxdctl |= IGC_RXDCTL_QUEUE_ENABLE; 3627 rxdctl &= 0xFFF00000; 3628 rxdctl |= IGC_RX_PTHRESH; 3629 rxdctl |= IGC_RX_HTHRESH << 8; 3630 rxdctl |= IGC_RX_WTHRESH << 16; 3631 IGC_WRITE_REG(hw, IGC_RXDCTL(iq), rxdctl); 3632 } 3633 3634 /* Make sure VLAN Filters are off */ 3635 rctl &= ~IGC_RCTL_VFE; 3636 3637 /* Write out the settings */ 3638 IGC_WRITE_REG(hw, IGC_RCTL, rctl); 3639 } 3640 3641 /********************************************************************* 3642 * 3643 * Free all receive rings. 3644 * 3645 **********************************************************************/ 3646 static void 3647 igc_free_receive_structures(struct igc_softc *sc) 3648 { 3649 3650 for (int iq = 0; iq < sc->sc_nqueues; iq++) { 3651 struct rx_ring *rxr = &sc->rx_rings[iq]; 3652 3653 igc_free_receive_buffers(rxr); 3654 } 3655 } 3656 3657 /********************************************************************* 3658 * 3659 * Free receive ring data structures 3660 * 3661 **********************************************************************/ 3662 static void 3663 igc_free_receive_buffers(struct rx_ring *rxr) 3664 { 3665 struct igc_softc *sc = rxr->sc; 3666 3667 if (rxr->rx_buffers != NULL) { 3668 for (int id = 0; id < sc->num_rx_desc; id++) { 3669 struct igc_rx_buf *rxbuf = &rxr->rx_buffers[id]; 3670 bus_dmamap_t map = rxbuf->map; 3671 3672 if (rxbuf->buf != NULL) { 3673 bus_dmamap_sync(rxr->rxdma.dma_tag, map, 3674 0, map->dm_mapsize, BUS_DMASYNC_POSTREAD); 3675 bus_dmamap_unload(rxr->rxdma.dma_tag, map); 3676 m_freem(rxbuf->buf); 3677 rxbuf->buf = NULL; 3678 } 3679 bus_dmamap_destroy(rxr->rxdma.dma_tag, map); 3680 rxbuf->map = NULL; 3681 } 3682 kmem_free(rxr->rx_buffers, 3683 sc->num_rx_desc * sizeof(struct igc_rx_buf)); 3684 rxr->rx_buffers = NULL; 3685 } 3686 3687 mutex_destroy(&rxr->rxr_lock); 3688 } 3689 3690 /********************************************************************* 3691 * 3692 * Clear status registers in all RX descriptors. 3693 * 3694 **********************************************************************/ 3695 static void 3696 igc_clear_receive_status(struct rx_ring *rxr) 3697 { 3698 struct igc_softc *sc = rxr->sc; 3699 3700 mutex_enter(&rxr->rxr_lock); 3701 3702 for (int id = 0; id < sc->num_rx_desc; id++) { 3703 union igc_adv_rx_desc *rxdesc = &rxr->rx_base[id]; 3704 3705 igc_rxdesc_sync(rxr, id, 3706 BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE); 3707 rxdesc->wb.upper.status_error = 0; 3708 igc_rxdesc_sync(rxr, id, 3709 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); 3710 } 3711 3712 mutex_exit(&rxr->rxr_lock); 3713 } 3714 3715 /* 3716 * Initialise the RSS mapping for NICs that support multiple transmit/ 3717 * receive rings. 3718 */ 3719 static void 3720 igc_initialize_rss_mapping(struct igc_softc *sc) 3721 { 3722 struct igc_hw *hw = &sc->hw; 3723 3724 /* 3725 * The redirection table controls which destination 3726 * queue each bucket redirects traffic to. 3727 * Each DWORD represents four queues, with the LSB 3728 * being the first queue in the DWORD. 3729 * 3730 * This just allocates buckets to queues using round-robin 3731 * allocation. 3732 * 3733 * NOTE: It Just Happens to line up with the default 3734 * RSS allocation method. 3735 */ 3736 3737 /* Warning FM follows */ 3738 uint32_t reta = 0; 3739 for (int i = 0; i < 128; i++) { 3740 const int shift = 0; /* XXXRO */ 3741 int queue_id = i % sc->sc_nqueues; 3742 /* Adjust if required */ 3743 queue_id <<= shift; 3744 3745 /* 3746 * The low 8 bits are for hash value (n+0); 3747 * The next 8 bits are for hash value (n+1), etc. 3748 */ 3749 reta >>= 8; 3750 reta |= ((uint32_t)queue_id) << 24; 3751 if ((i & 3) == 3) { 3752 IGC_WRITE_REG(hw, IGC_RETA(i >> 2), reta); 3753 reta = 0; 3754 } 3755 } 3756 3757 /* 3758 * MRQC: Multiple Receive Queues Command 3759 * Set queuing to RSS control, number depends on the device. 3760 */ 3761 3762 /* Set up random bits */ 3763 uint32_t rss_key[RSS_KEYSIZE / sizeof(uint32_t)]; 3764 rss_getkey((uint8_t *)rss_key); 3765 3766 /* Now fill our hash function seeds */ 3767 for (int i = 0; i < __arraycount(rss_key); i++) 3768 IGC_WRITE_REG_ARRAY(hw, IGC_RSSRK(0), i, rss_key[i]); 3769 3770 /* 3771 * Configure the RSS fields to hash upon. 3772 */ 3773 uint32_t mrqc = IGC_MRQC_ENABLE_RSS_4Q; 3774 mrqc |= IGC_MRQC_RSS_FIELD_IPV4 | IGC_MRQC_RSS_FIELD_IPV4_TCP; 3775 mrqc |= IGC_MRQC_RSS_FIELD_IPV6 | IGC_MRQC_RSS_FIELD_IPV6_TCP; 3776 mrqc |= IGC_MRQC_RSS_FIELD_IPV6_TCP_EX; 3777 3778 IGC_WRITE_REG(hw, IGC_MRQC, mrqc); 3779 } 3780 3781 /* 3782 * igc_get_hw_control sets the {CTRL_EXT|FWSM}:DRV_LOAD bit. 3783 * For ASF and Pass Through versions of f/w this means 3784 * that the driver is loaded. For AMT version type f/w 3785 * this means that the network i/f is open. 3786 */ 3787 static void 3788 igc_get_hw_control(struct igc_softc *sc) 3789 { 3790 const uint32_t ctrl_ext = IGC_READ_REG(&sc->hw, IGC_CTRL_EXT); 3791 3792 IGC_WRITE_REG(&sc->hw, IGC_CTRL_EXT, ctrl_ext | IGC_CTRL_EXT_DRV_LOAD); 3793 } 3794 3795 /* 3796 * igc_release_hw_control resets {CTRL_EXT|FWSM}:DRV_LOAD bit. 3797 * For ASF and Pass Through versions of f/w this means that 3798 * the driver is no longer loaded. For AMT versions of the 3799 * f/w this means that the network i/f is closed. 3800 */ 3801 static void 3802 igc_release_hw_control(struct igc_softc *sc) 3803 { 3804 const uint32_t ctrl_ext = IGC_READ_REG(&sc->hw, IGC_CTRL_EXT); 3805 3806 IGC_WRITE_REG(&sc->hw, IGC_CTRL_EXT, ctrl_ext & ~IGC_CTRL_EXT_DRV_LOAD); 3807 } 3808 3809 static int 3810 igc_is_valid_ether_addr(uint8_t *addr) 3811 { 3812 const char zero_addr[6] = { 0, 0, 0, 0, 0, 0 }; 3813 3814 if ((addr[0] & 1) || !bcmp(addr, zero_addr, ETHER_ADDR_LEN)) 3815 return 0; 3816 3817 return 1; 3818 } 3819 3820 static void 3821 igc_print_devinfo(struct igc_softc *sc) 3822 { 3823 device_t dev = sc->sc_dev; 3824 struct igc_hw *hw = &sc->hw; 3825 struct igc_phy_info *phy = &hw->phy; 3826 u_int oui, model, rev; 3827 uint16_t id1, id2, nvm_ver, phy_ver; 3828 char descr[MII_MAX_DESCR_LEN]; 3829 3830 /* Print PHY Info */ 3831 id1 = phy->id >> 16; 3832 /* The revision field in phy->id is cleard and it's in phy->revision */ 3833 id2 = (phy->id & 0xfff0) | phy->revision; 3834 oui = MII_OUI(id1, id2); 3835 model = MII_MODEL(id2); 3836 rev = MII_REV(id2); 3837 mii_get_descr(descr, sizeof(descr), oui, model); 3838 if (descr[0]) 3839 aprint_normal_dev(dev, "PHY: %s, rev. %d\n", 3840 descr, rev); 3841 else 3842 aprint_normal_dev(dev, 3843 "PHY OUI 0x%06x, model 0x%04x, rev. %d\n", 3844 oui, model, rev); 3845 3846 /* Get NVM version */ 3847 hw->nvm.ops.read(hw, NVM_VERSION, 1, &nvm_ver); 3848 3849 /* Get PHY FW version */ 3850 phy->ops.read_reg(hw, 0x1e, &phy_ver); 3851 3852 aprint_normal_dev(dev, "ROM image version %x.%02x", 3853 (nvm_ver & NVM_VERSION_MAJOR) >> NVM_VERSION_MAJOR_SHIFT, 3854 (nvm_ver & NVM_VERSION_MINOR)); 3855 aprint_debug("(0x%04hx)", nvm_ver); 3856 3857 aprint_normal(", PHY FW version 0x%04hx\n", phy_ver); 3858 } 3859
Indexes created Tue Sep 30 11:09:46 GMT 2025