if_igc.c revision 1.3.2.7
1 1.3.2.7 martin /* $NetBSD: if_igc.c,v 1.3.2.7 2025/06/01 12:32:06 martin Exp $ */ 2 1.3.2.2 martin /* $OpenBSD: if_igc.c,v 1.13 2023/04/28 10:18:57 bluhm Exp $ */ 3 1.3.2.2 martin /*- 4 1.3.2.2 martin * SPDX-License-Identifier: BSD-2-Clause 5 1.3.2.2 martin * 6 1.3.2.2 martin * Copyright (c) 2016 Nicole Graziano <nicole (at) nextbsd.org> 7 1.3.2.2 martin * All rights reserved. 8 1.3.2.2 martin * Copyright (c) 2021 Rubicon Communications, LLC (Netgate) 9 1.3.2.2 martin * 10 1.3.2.2 martin * Redistribution and use in source and binary forms, with or without 11 1.3.2.2 martin * modification, are permitted provided that the following conditions 12 1.3.2.2 martin * are met: 13 1.3.2.2 martin * 1. Redistributions of source code must retain the above copyright 14 1.3.2.2 martin * notice, this list of conditions and the following disclaimer. 15 1.3.2.2 martin * 2. Redistributions in binary form must reproduce the above copyright 16 1.3.2.2 martin * notice, this list of conditions and the following disclaimer in the 17 1.3.2.2 martin * documentation and/or other materials provided with the distribution. 18 1.3.2.2 martin * 19 1.3.2.2 martin * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 20 1.3.2.2 martin * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 21 1.3.2.2 martin * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 22 1.3.2.2 martin * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 23 1.3.2.2 martin * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 24 1.3.2.2 martin * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 25 1.3.2.2 martin * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 26 1.3.2.2 martin * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 27 1.3.2.2 martin * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 28 1.3.2.2 martin * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 29 1.3.2.2 martin * SUCH DAMAGE. 30 1.3.2.2 martin */ 31 1.3.2.2 martin 32 1.3.2.2 martin #include <sys/cdefs.h> 33 1.3.2.7 martin __KERNEL_RCSID(0, "$NetBSD: if_igc.c,v 1.3.2.7 2025/06/01 12:32:06 martin Exp $"); 34 1.3.2.2 martin 35 1.3.2.2 martin #ifdef _KERNEL_OPT 36 1.3.2.2 martin #include "opt_if_igc.h" 37 1.3.2.2 martin #if 0 /* notyet */ 38 1.3.2.2 martin #include "vlan.h" 39 1.3.2.2 martin #endif 40 1.3.2.2 martin #endif 41 1.3.2.2 martin 42 1.3.2.2 martin #include <sys/param.h> 43 1.3.2.2 martin #include <sys/systm.h> 44 1.3.2.2 martin #include <sys/bus.h> 45 1.3.2.2 martin #include <sys/cpu.h> 46 1.3.2.2 martin #include <sys/device.h> 47 1.3.2.2 martin #include <sys/endian.h> 48 1.3.2.2 martin #include <sys/intr.h> 49 1.3.2.2 martin #include <sys/interrupt.h> 50 1.3.2.2 martin #include <sys/kernel.h> 51 1.3.2.2 martin #include <sys/kmem.h> 52 1.3.2.2 martin #include <sys/mbuf.h> 53 1.3.2.2 martin #include <sys/mutex.h> 54 1.3.2.2 martin #include <sys/socket.h> 55 1.3.2.2 martin #include <sys/workqueue.h> 56 1.3.2.2 martin #include <sys/xcall.h> 57 1.3.2.2 martin 58 1.3.2.2 martin #include <net/bpf.h> 59 1.3.2.2 martin #include <net/if.h> 60 1.3.2.2 martin #include <net/if_ether.h> 61 1.3.2.2 martin #include <net/if_media.h> 62 1.3.2.2 martin #include <net/if_vlanvar.h> 63 1.3.2.2 martin #include <net/rss_config.h> 64 1.3.2.2 martin 65 1.3.2.2 martin #include <netinet/in.h> 66 1.3.2.2 martin #include <netinet/ip.h> 67 1.3.2.2 martin #include <netinet/ip6.h> 68 1.3.2.2 martin #include <netinet/tcp.h> 69 1.3.2.2 martin 70 1.3.2.2 martin #include <dev/pci/pcivar.h> 71 1.3.2.2 martin #include <dev/pci/pcireg.h> 72 1.3.2.2 martin #include <dev/pci/pcidevs.h> 73 1.3.2.2 martin 74 1.3.2.2 martin #include <dev/pci/igc/if_igc.h> 75 1.3.2.2 martin #include <dev/pci/igc/igc_evcnt.h> 76 1.3.2.2 martin #include <dev/pci/igc/igc_hw.h> 77 1.3.2.2 martin #include <dev/mii/miivar.h> 78 1.3.2.2 martin 79 1.3.2.2 martin #define IGC_WORKQUEUE_PRI PRI_SOFTNET 80 1.3.2.2 martin 81 1.3.2.2 martin #ifndef IGC_RX_INTR_PROCESS_LIMIT_DEFAULT 82 1.3.2.2 martin #define IGC_RX_INTR_PROCESS_LIMIT_DEFAULT 0 83 1.3.2.2 martin #endif 84 1.3.2.2 martin #ifndef IGC_TX_INTR_PROCESS_LIMIT_DEFAULT 85 1.3.2.2 martin #define IGC_TX_INTR_PROCESS_LIMIT_DEFAULT 0 86 1.3.2.2 martin #endif 87 1.3.2.2 martin 88 1.3.2.2 martin #ifndef IGC_RX_PROCESS_LIMIT_DEFAULT 89 1.3.2.2 martin #define IGC_RX_PROCESS_LIMIT_DEFAULT 256 90 1.3.2.2 martin #endif 91 1.3.2.2 martin #ifndef IGC_TX_PROCESS_LIMIT_DEFAULT 92 1.3.2.2 martin #define IGC_TX_PROCESS_LIMIT_DEFAULT 256 93 1.3.2.2 martin #endif 94 1.3.2.2 martin 95 1.3.2.2 martin #define htolem32(p, x) (*((uint32_t *)(p)) = htole32(x)) 96 1.3.2.2 martin #define htolem64(p, x) (*((uint64_t *)(p)) = htole64(x)) 97 1.3.2.2 martin 98 1.3.2.2 martin static const struct igc_product { 99 1.3.2.2 martin pci_vendor_id_t igcp_vendor; 100 1.3.2.2 martin pci_product_id_t igcp_product; 101 1.3.2.2 martin const char *igcp_name; 102 1.3.2.2 martin } igc_products[] = { 103 1.3.2.2 martin { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_I225_IT, 104 1.3.2.2 martin "Intel(R) Ethernet Controller I225-IT(2)" }, 105 1.3.2.2 martin { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_I226_LM, 106 1.3.2.2 martin "Intel(R) Ethernet Controller I226-LM" }, 107 1.3.2.2 martin { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_I226_V, 108 1.3.2.2 martin "Intel(R) Ethernet Controller I226-V" }, 109 1.3.2.2 martin { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_I226_IT, 110 1.3.2.2 martin "Intel(R) Ethernet Controller I226-IT" }, 111 1.3.2.2 martin { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_I221_V, 112 1.3.2.2 martin "Intel(R) Ethernet Controller I221-V" }, 113 1.3.2.2 martin { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_I226_BLANK_NVM, 114 1.3.2.2 martin "Intel(R) Ethernet Controller I226(blankNVM)" }, 115 1.3.2.2 martin { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_I225_LM, 116 1.3.2.2 martin "Intel(R) Ethernet Controller I225-LM" }, 117 1.3.2.2 martin { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_I225_V, 118 1.3.2.2 martin "Intel(R) Ethernet Controller I225-V" }, 119 1.3.2.2 martin { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_I220_V, 120 1.3.2.2 martin "Intel(R) Ethernet Controller I220-V" }, 121 1.3.2.2 martin { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_I225_I, 122 1.3.2.2 martin "Intel(R) Ethernet Controller I225-I" }, 123 1.3.2.2 martin { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_I225_BLANK_NVM, 124 1.3.2.2 martin "Intel(R) Ethernet Controller I225(blankNVM)" }, 125 1.3.2.2 martin { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_I225_K, 126 1.3.2.2 martin "Intel(R) Ethernet Controller I225-K" }, 127 1.3.2.2 martin { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_I225_K2, 128 1.3.2.2 martin "Intel(R) Ethernet Controller I225-K(2)" }, 129 1.3.2.2 martin { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_I226_K, 130 1.3.2.2 martin "Intel(R) Ethernet Controller I226-K" }, 131 1.3.2.2 martin { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_I225_LMVP, 132 1.3.2.2 martin "Intel(R) Ethernet Controller I225-LMvP(2)" }, 133 1.3.2.2 martin { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_I226_LMVP, 134 1.3.2.2 martin "Intel(R) Ethernet Controller I226-LMvP" }, 135 1.3.2.2 martin { 0, 0, NULL }, 136 1.3.2.2 martin }; 137 1.3.2.2 martin 138 1.3.2.2 martin #define IGC_DF_CFG 0x1 139 1.3.2.2 martin #define IGC_DF_TX 0x2 140 1.3.2.2 martin #define IGC_DF_RX 0x4 141 1.3.2.2 martin #define IGC_DF_MISC 0x8 142 1.3.2.2 martin 143 1.3.2.2 martin #ifdef IGC_DEBUG_FLAGS 144 1.3.2.2 martin int igc_debug_flags = IGC_DEBUG_FLAGS; 145 1.3.2.2 martin #else 146 1.3.2.2 martin int igc_debug_flags = 0; 147 1.3.2.2 martin #endif 148 1.3.2.2 martin 149 1.3.2.2 martin #define DPRINTF(flag, fmt, args...) do { \ 150 1.3.2.2 martin if (igc_debug_flags & (IGC_DF_ ## flag)) \ 151 1.3.2.2 martin printf("%s: %d: " fmt, __func__, __LINE__, ##args); \ 152 1.3.2.2 martin } while (0) 153 1.3.2.2 martin 154 1.3.2.2 martin /********************************************************************* 155 1.3.2.2 martin * Function Prototypes 156 1.3.2.2 martin *********************************************************************/ 157 1.3.2.2 martin static int igc_match(device_t, cfdata_t, void *); 158 1.3.2.2 martin static void igc_attach(device_t, device_t, void *); 159 1.3.2.2 martin static int igc_detach(device_t, int); 160 1.3.2.2 martin 161 1.3.2.2 martin static void igc_identify_hardware(struct igc_softc *); 162 1.3.2.2 martin static int igc_adjust_nqueues(struct igc_softc *); 163 1.3.2.2 martin static int igc_allocate_pci_resources(struct igc_softc *); 164 1.3.2.2 martin static int igc_allocate_interrupts(struct igc_softc *); 165 1.3.2.2 martin static int igc_allocate_queues(struct igc_softc *); 166 1.3.2.2 martin static void igc_free_pci_resources(struct igc_softc *); 167 1.3.2.2 martin static void igc_free_interrupts(struct igc_softc *); 168 1.3.2.2 martin static void igc_free_queues(struct igc_softc *); 169 1.3.2.2 martin static void igc_reset(struct igc_softc *); 170 1.3.2.2 martin static void igc_init_dmac(struct igc_softc *, uint32_t); 171 1.3.2.2 martin static int igc_setup_interrupts(struct igc_softc *); 172 1.3.2.2 martin static void igc_attach_counters(struct igc_softc *sc); 173 1.3.2.2 martin static void igc_detach_counters(struct igc_softc *sc); 174 1.3.2.2 martin static void igc_update_counters(struct igc_softc *sc); 175 1.3.2.2 martin static void igc_clear_counters(struct igc_softc *sc); 176 1.3.2.2 martin static int igc_setup_msix(struct igc_softc *); 177 1.3.2.2 martin static int igc_setup_msi(struct igc_softc *); 178 1.3.2.2 martin static int igc_setup_intx(struct igc_softc *); 179 1.3.2.2 martin static int igc_dma_malloc(struct igc_softc *, bus_size_t, 180 1.3.2.2 martin struct igc_dma_alloc *); 181 1.3.2.2 martin static void igc_dma_free(struct igc_softc *, struct igc_dma_alloc *); 182 1.3.2.2 martin static void igc_setup_interface(struct igc_softc *); 183 1.3.2.2 martin 184 1.3.2.2 martin static int igc_init(struct ifnet *); 185 1.3.2.2 martin static int igc_init_locked(struct igc_softc *); 186 1.3.2.2 martin static void igc_start(struct ifnet *); 187 1.3.2.2 martin static int igc_transmit(struct ifnet *, struct mbuf *); 188 1.3.2.2 martin static void igc_tx_common_locked(struct ifnet *, struct tx_ring *, int); 189 1.3.2.2 martin static bool igc_txeof(struct tx_ring *, u_int); 190 1.3.2.2 martin static void igc_intr_barrier(struct igc_softc *); 191 1.3.2.2 martin static void igc_stop(struct ifnet *, int); 192 1.3.2.2 martin static void igc_stop_locked(struct igc_softc *); 193 1.3.2.2 martin static int igc_ioctl(struct ifnet *, u_long, void *); 194 1.3.2.2 martin #ifdef IF_RXR 195 1.3.2.2 martin static int igc_rxrinfo(struct igc_softc *, struct if_rxrinfo *); 196 1.3.2.2 martin #endif 197 1.3.2.2 martin static void igc_rxfill(struct rx_ring *); 198 1.3.2.2 martin static void igc_rxrefill(struct rx_ring *, int); 199 1.3.2.2 martin static bool igc_rxeof(struct rx_ring *, u_int); 200 1.3.2.2 martin static int igc_rx_checksum(struct igc_queue *, uint64_t, uint32_t, 201 1.3.2.2 martin uint32_t); 202 1.3.2.2 martin static void igc_watchdog(struct ifnet *); 203 1.3.2.2 martin static void igc_tick(void *); 204 1.3.2.2 martin static void igc_media_status(struct ifnet *, struct ifmediareq *); 205 1.3.2.2 martin static int igc_media_change(struct ifnet *); 206 1.3.2.2 martin static int igc_ifflags_cb(struct ethercom *); 207 1.3.2.2 martin static void igc_set_filter(struct igc_softc *); 208 1.3.2.2 martin static void igc_update_link_status(struct igc_softc *); 209 1.3.2.2 martin static int igc_get_buf(struct rx_ring *, int, bool); 210 1.3.2.7 martin static bool igc_tx_ctx_setup(struct tx_ring *, struct mbuf *, int, 211 1.3.2.2 martin uint32_t *, uint32_t *); 212 1.3.2.2 martin 213 1.3.2.2 martin static void igc_configure_queues(struct igc_softc *); 214 1.3.2.2 martin static void igc_set_queues(struct igc_softc *, uint32_t, uint32_t, int); 215 1.3.2.2 martin static void igc_enable_queue(struct igc_softc *, uint32_t); 216 1.3.2.2 martin static void igc_enable_intr(struct igc_softc *); 217 1.3.2.2 martin static void igc_disable_intr(struct igc_softc *); 218 1.3.2.2 martin static int igc_intr_link(void *); 219 1.3.2.2 martin static int igc_intr_queue(void *); 220 1.3.2.2 martin static int igc_intr(void *); 221 1.3.2.2 martin static void igc_handle_queue(void *); 222 1.3.2.2 martin static void igc_handle_queue_work(struct work *, void *); 223 1.3.2.2 martin static void igc_sched_handle_queue(struct igc_softc *, struct igc_queue *); 224 1.3.2.2 martin static void igc_barrier_handle_queue(struct igc_softc *); 225 1.3.2.2 martin 226 1.3.2.2 martin static int igc_allocate_transmit_buffers(struct tx_ring *); 227 1.3.2.2 martin static int igc_setup_transmit_structures(struct igc_softc *); 228 1.3.2.2 martin static int igc_setup_transmit_ring(struct tx_ring *); 229 1.3.2.2 martin static void igc_initialize_transmit_unit(struct igc_softc *); 230 1.3.2.2 martin static void igc_free_transmit_structures(struct igc_softc *); 231 1.3.2.2 martin static void igc_free_transmit_buffers(struct tx_ring *); 232 1.3.2.2 martin static void igc_withdraw_transmit_packets(struct tx_ring *, bool); 233 1.3.2.2 martin static int igc_allocate_receive_buffers(struct rx_ring *); 234 1.3.2.2 martin static int igc_setup_receive_structures(struct igc_softc *); 235 1.3.2.2 martin static int igc_setup_receive_ring(struct rx_ring *); 236 1.3.2.2 martin static void igc_initialize_receive_unit(struct igc_softc *); 237 1.3.2.2 martin static void igc_free_receive_structures(struct igc_softc *); 238 1.3.2.2 martin static void igc_free_receive_buffers(struct rx_ring *); 239 1.3.2.2 martin static void igc_clear_receive_status(struct rx_ring *); 240 1.3.2.2 martin static void igc_initialize_rss_mapping(struct igc_softc *); 241 1.3.2.2 martin 242 1.3.2.2 martin static void igc_get_hw_control(struct igc_softc *); 243 1.3.2.2 martin static void igc_release_hw_control(struct igc_softc *); 244 1.3.2.2 martin static int igc_is_valid_ether_addr(uint8_t *); 245 1.3.2.2 martin static void igc_print_devinfo(struct igc_softc *); 246 1.3.2.2 martin 247 1.3.2.2 martin CFATTACH_DECL3_NEW(igc, sizeof(struct igc_softc), 248 1.3.2.2 martin igc_match, igc_attach, igc_detach, NULL, NULL, NULL, 0); 249 1.3.2.2 martin 250 1.3.2.2 martin static inline int 251 1.3.2.2 martin igc_txdesc_incr(struct igc_softc *sc, int id) 252 1.3.2.2 martin { 253 1.3.2.2 martin 254 1.3.2.2 martin if (++id == sc->num_tx_desc) 255 1.3.2.2 martin id = 0; 256 1.3.2.2 martin return id; 257 1.3.2.2 martin } 258 1.3.2.2 martin 259 1.3.2.2 martin static inline int __unused 260 1.3.2.2 martin igc_txdesc_decr(struct igc_softc *sc, int id) 261 1.3.2.2 martin { 262 1.3.2.2 martin 263 1.3.2.2 martin if (--id < 0) 264 1.3.2.2 martin id = sc->num_tx_desc - 1; 265 1.3.2.2 martin return id; 266 1.3.2.2 martin } 267 1.3.2.2 martin 268 1.3.2.2 martin static inline void 269 1.3.2.2 martin igc_txdesc_sync(struct tx_ring *txr, int id, int ops) 270 1.3.2.2 martin { 271 1.3.2.2 martin 272 1.3.2.2 martin bus_dmamap_sync(txr->txdma.dma_tag, txr->txdma.dma_map, 273 1.3.2.2 martin id * sizeof(union igc_adv_tx_desc), sizeof(union igc_adv_tx_desc), 274 1.3.2.2 martin ops); 275 1.3.2.2 martin } 276 1.3.2.2 martin 277 1.3.2.2 martin static inline int 278 1.3.2.2 martin igc_rxdesc_incr(struct igc_softc *sc, int id) 279 1.3.2.2 martin { 280 1.3.2.2 martin 281 1.3.2.2 martin if (++id == sc->num_rx_desc) 282 1.3.2.2 martin id = 0; 283 1.3.2.2 martin return id; 284 1.3.2.2 martin } 285 1.3.2.2 martin 286 1.3.2.2 martin static inline int 287 1.3.2.2 martin igc_rxdesc_decr(struct igc_softc *sc, int id) 288 1.3.2.2 martin { 289 1.3.2.2 martin 290 1.3.2.2 martin if (--id < 0) 291 1.3.2.2 martin id = sc->num_rx_desc - 1; 292 1.3.2.2 martin return id; 293 1.3.2.2 martin } 294 1.3.2.2 martin 295 1.3.2.2 martin static inline void 296 1.3.2.2 martin igc_rxdesc_sync(struct rx_ring *rxr, int id, int ops) 297 1.3.2.2 martin { 298 1.3.2.2 martin 299 1.3.2.2 martin bus_dmamap_sync(rxr->rxdma.dma_tag, rxr->rxdma.dma_map, 300 1.3.2.2 martin id * sizeof(union igc_adv_rx_desc), sizeof(union igc_adv_rx_desc), 301 1.3.2.2 martin ops); 302 1.3.2.2 martin } 303 1.3.2.2 martin 304 1.3.2.2 martin static const struct igc_product * 305 1.3.2.2 martin igc_lookup(const struct pci_attach_args *pa) 306 1.3.2.2 martin { 307 1.3.2.2 martin const struct igc_product *igcp; 308 1.3.2.2 martin 309 1.3.2.2 martin for (igcp = igc_products; igcp->igcp_name != NULL; igcp++) { 310 1.3.2.2 martin if (PCI_VENDOR(pa->pa_id) == igcp->igcp_vendor && 311 1.3.2.2 martin PCI_PRODUCT(pa->pa_id) == igcp->igcp_product) 312 1.3.2.2 martin return igcp; 313 1.3.2.2 martin } 314 1.3.2.2 martin return NULL; 315 1.3.2.2 martin } 316 1.3.2.2 martin 317 1.3.2.2 martin /********************************************************************* 318 1.3.2.2 martin * Device identification routine 319 1.3.2.2 martin * 320 1.3.2.2 martin * igc_match determines if the driver should be loaded on 321 1.3.2.2 martin * adapter based on PCI vendor/device id of the adapter. 322 1.3.2.2 martin * 323 1.3.2.2 martin * return 0 on success, positive on failure 324 1.3.2.2 martin *********************************************************************/ 325 1.3.2.2 martin static int 326 1.3.2.2 martin igc_match(device_t parent, cfdata_t match, void *aux) 327 1.3.2.2 martin { 328 1.3.2.2 martin struct pci_attach_args *pa = aux; 329 1.3.2.2 martin 330 1.3.2.2 martin if (igc_lookup(pa) != NULL) 331 1.3.2.2 martin return 1; 332 1.3.2.2 martin 333 1.3.2.2 martin return 0; 334 1.3.2.2 martin } 335 1.3.2.2 martin 336 1.3.2.2 martin /********************************************************************* 337 1.3.2.2 martin * Device initialization routine 338 1.3.2.2 martin * 339 1.3.2.2 martin * The attach entry point is called when the driver is being loaded. 340 1.3.2.2 martin * This routine identifies the type of hardware, allocates all resources 341 1.3.2.2 martin * and initializes the hardware. 342 1.3.2.2 martin * 343 1.3.2.2 martin * return 0 on success, positive on failure 344 1.3.2.2 martin *********************************************************************/ 345 1.3.2.2 martin static void 346 1.3.2.2 martin igc_attach(device_t parent, device_t self, void *aux) 347 1.3.2.2 martin { 348 1.3.2.2 martin struct pci_attach_args *pa = aux; 349 1.3.2.2 martin struct igc_softc *sc = device_private(self); 350 1.3.2.2 martin struct igc_hw *hw = &sc->hw; 351 1.3.2.2 martin 352 1.3.2.2 martin const struct igc_product *igcp = igc_lookup(pa); 353 1.3.2.2 martin KASSERT(igcp != NULL); 354 1.3.2.2 martin 355 1.3.2.2 martin sc->sc_dev = self; 356 1.3.2.2 martin callout_init(&sc->sc_tick_ch, CALLOUT_MPSAFE); 357 1.3.2.2 martin callout_setfunc(&sc->sc_tick_ch, igc_tick, sc); 358 1.3.2.2 martin sc->sc_core_stopping = false; 359 1.3.2.2 martin 360 1.3.2.2 martin sc->osdep.os_sc = sc; 361 1.3.2.2 martin sc->osdep.os_pa = *pa; 362 1.3.2.5 martin #ifndef __aarch64__ 363 1.3.2.2 martin /* 364 1.3.2.2 martin * XXX PR port-arm/57643 365 1.3.2.2 martin * 64-bit DMA does not work at least for LX2K with 32/64GB memory. 366 1.3.2.2 martin * smmu(4) support may be required. 367 1.3.2.2 martin */ 368 1.3.2.5 martin if (pci_dma64_available(pa)) { 369 1.3.2.5 martin aprint_verbose(", 64-bit DMA"); 370 1.3.2.5 martin sc->osdep.os_dmat = pa->pa_dmat64; 371 1.3.2.5 martin } else 372 1.3.2.2 martin #endif 373 1.3.2.5 martin { 374 1.3.2.5 martin aprint_verbose(", 32-bit DMA"); 375 1.3.2.5 martin sc->osdep.os_dmat = pa->pa_dmat; 376 1.3.2.5 martin } 377 1.3.2.5 martin 378 1.3.2.5 martin pci_aprint_devinfo_fancy(pa, "Ethernet controller", igcp->igcp_name, 1); 379 1.3.2.2 martin 380 1.3.2.2 martin /* Determine hardware and mac info */ 381 1.3.2.2 martin igc_identify_hardware(sc); 382 1.3.2.2 martin 383 1.3.2.2 martin sc->num_tx_desc = IGC_DEFAULT_TXD; 384 1.3.2.2 martin sc->num_rx_desc = IGC_DEFAULT_RXD; 385 1.3.2.2 martin 386 1.3.2.2 martin /* Setup PCI resources */ 387 1.3.2.2 martin if (igc_allocate_pci_resources(sc)) { 388 1.3.2.2 martin aprint_error_dev(sc->sc_dev, 389 1.3.2.2 martin "unable to allocate PCI resources\n"); 390 1.3.2.2 martin goto err_pci; 391 1.3.2.2 martin } 392 1.3.2.2 martin 393 1.3.2.2 martin if (igc_allocate_interrupts(sc)) { 394 1.3.2.2 martin aprint_error_dev(sc->sc_dev, "unable to allocate interrupts\n"); 395 1.3.2.2 martin goto err_pci; 396 1.3.2.2 martin } 397 1.3.2.2 martin 398 1.3.2.2 martin /* Allocate TX/RX queues */ 399 1.3.2.2 martin if (igc_allocate_queues(sc)) { 400 1.3.2.2 martin aprint_error_dev(sc->sc_dev, "unable to allocate queues\n"); 401 1.3.2.2 martin goto err_alloc_intr; 402 1.3.2.2 martin } 403 1.3.2.2 martin 404 1.3.2.2 martin /* Do shared code initialization */ 405 1.3.2.2 martin if (igc_setup_init_funcs(hw, true)) { 406 1.3.2.2 martin aprint_error_dev(sc->sc_dev, "unable to initialize\n"); 407 1.3.2.2 martin goto err_alloc_intr; 408 1.3.2.2 martin } 409 1.3.2.2 martin 410 1.3.2.2 martin hw->mac.autoneg = DO_AUTO_NEG; 411 1.3.2.2 martin hw->phy.autoneg_wait_to_complete = false; 412 1.3.2.2 martin hw->phy.autoneg_advertised = AUTONEG_ADV_DEFAULT; 413 1.3.2.2 martin 414 1.3.2.2 martin /* Copper options. */ 415 1.3.2.2 martin if (hw->phy.media_type == igc_media_type_copper) 416 1.3.2.2 martin hw->phy.mdix = AUTO_ALL_MODES; 417 1.3.2.2 martin 418 1.3.2.2 martin /* Set the max frame size. */ 419 1.3.2.2 martin sc->hw.mac.max_frame_size = 9234; 420 1.3.2.2 martin 421 1.3.2.2 martin /* Allocate multicast array memory. */ 422 1.3.2.2 martin sc->mta = kmem_alloc(IGC_MTA_LEN, KM_SLEEP); 423 1.3.2.2 martin 424 1.3.2.2 martin /* Check SOL/IDER usage. */ 425 1.3.2.2 martin if (igc_check_reset_block(hw)) { 426 1.3.2.2 martin aprint_error_dev(sc->sc_dev, 427 1.3.2.2 martin "PHY reset is blocked due to SOL/IDER session\n"); 428 1.3.2.2 martin } 429 1.3.2.2 martin 430 1.3.2.2 martin /* Disable Energy Efficient Ethernet. */ 431 1.3.2.2 martin sc->hw.dev_spec._i225.eee_disable = true; 432 1.3.2.2 martin 433 1.3.2.2 martin igc_reset_hw(hw); 434 1.3.2.2 martin 435 1.3.2.2 martin /* Make sure we have a good EEPROM before we read from it. */ 436 1.3.2.2 martin if (igc_validate_nvm_checksum(hw) < 0) { 437 1.3.2.2 martin /* 438 1.3.2.2 martin * Some PCI-E parts fail the first check due to 439 1.3.2.2 martin * the link being in sleep state, call it again, 440 1.3.2.2 martin * if it fails a second time its a real issue. 441 1.3.2.2 martin */ 442 1.3.2.2 martin if (igc_validate_nvm_checksum(hw) < 0) { 443 1.3.2.2 martin aprint_error_dev(sc->sc_dev, 444 1.3.2.2 martin "EEPROM checksum invalid\n"); 445 1.3.2.2 martin goto err_late; 446 1.3.2.2 martin } 447 1.3.2.2 martin } 448 1.3.2.2 martin 449 1.3.2.2 martin /* Copy the permanent MAC address out of the EEPROM. */ 450 1.3.2.2 martin if (igc_read_mac_addr(hw) < 0) { 451 1.3.2.2 martin aprint_error_dev(sc->sc_dev, 452 1.3.2.2 martin "unable to read MAC address from EEPROM\n"); 453 1.3.2.2 martin goto err_late; 454 1.3.2.2 martin } 455 1.3.2.2 martin 456 1.3.2.2 martin if (!igc_is_valid_ether_addr(hw->mac.addr)) { 457 1.3.2.2 martin aprint_error_dev(sc->sc_dev, "invalid MAC address\n"); 458 1.3.2.2 martin goto err_late; 459 1.3.2.2 martin } 460 1.3.2.2 martin 461 1.3.2.2 martin if (igc_setup_interrupts(sc)) 462 1.3.2.2 martin goto err_late; 463 1.3.2.2 martin 464 1.3.2.2 martin /* Attach counters. */ 465 1.3.2.2 martin igc_attach_counters(sc); 466 1.3.2.2 martin 467 1.3.2.2 martin /* Setup OS specific network interface. */ 468 1.3.2.2 martin igc_setup_interface(sc); 469 1.3.2.2 martin 470 1.3.2.2 martin igc_print_devinfo(sc); 471 1.3.2.2 martin 472 1.3.2.2 martin igc_reset(sc); 473 1.3.2.2 martin hw->mac.get_link_status = true; 474 1.3.2.2 martin igc_update_link_status(sc); 475 1.3.2.2 martin 476 1.3.2.2 martin /* The driver can now take control from firmware. */ 477 1.3.2.2 martin igc_get_hw_control(sc); 478 1.3.2.2 martin 479 1.3.2.2 martin aprint_normal_dev(sc->sc_dev, "Ethernet address %s\n", 480 1.3.2.2 martin ether_sprintf(sc->hw.mac.addr)); 481 1.3.2.2 martin 482 1.3.2.2 martin if (pmf_device_register(self, NULL, NULL)) 483 1.3.2.2 martin pmf_class_network_register(self, &sc->sc_ec.ec_if); 484 1.3.2.2 martin else 485 1.3.2.2 martin aprint_error_dev(self, "couldn't establish power handler\n"); 486 1.3.2.2 martin 487 1.3.2.2 martin return; 488 1.3.2.2 martin 489 1.3.2.2 martin err_late: 490 1.3.2.2 martin igc_release_hw_control(sc); 491 1.3.2.2 martin err_alloc_intr: 492 1.3.2.2 martin igc_free_interrupts(sc); 493 1.3.2.2 martin err_pci: 494 1.3.2.2 martin igc_free_pci_resources(sc); 495 1.3.2.2 martin kmem_free(sc->mta, IGC_MTA_LEN); 496 1.3.2.2 martin } 497 1.3.2.2 martin 498 1.3.2.2 martin /********************************************************************* 499 1.3.2.2 martin * Device removal routine 500 1.3.2.2 martin * 501 1.3.2.2 martin * The detach entry point is called when the driver is being removed. 502 1.3.2.2 martin * This routine stops the adapter and deallocates all the resources 503 1.3.2.2 martin * that were allocated for driver operation. 504 1.3.2.2 martin * 505 1.3.2.2 martin * return 0 on success, positive on failure 506 1.3.2.2 martin *********************************************************************/ 507 1.3.2.2 martin static int 508 1.3.2.2 martin igc_detach(device_t self, int flags) 509 1.3.2.2 martin { 510 1.3.2.2 martin struct igc_softc *sc = device_private(self); 511 1.3.2.2 martin struct ifnet *ifp = &sc->sc_ec.ec_if; 512 1.3.2.2 martin 513 1.3.2.2 martin mutex_enter(&sc->sc_core_lock); 514 1.3.2.2 martin igc_stop_locked(sc); 515 1.3.2.2 martin mutex_exit(&sc->sc_core_lock); 516 1.3.2.2 martin 517 1.3.2.2 martin igc_detach_counters(sc); 518 1.3.2.2 martin 519 1.3.2.2 martin igc_free_queues(sc); 520 1.3.2.2 martin 521 1.3.2.2 martin igc_phy_hw_reset(&sc->hw); 522 1.3.2.2 martin igc_release_hw_control(sc); 523 1.3.2.2 martin 524 1.3.2.2 martin ether_ifdetach(ifp); 525 1.3.2.2 martin if_detach(ifp); 526 1.3.2.2 martin ifmedia_fini(&sc->media); 527 1.3.2.2 martin 528 1.3.2.2 martin igc_free_interrupts(sc); 529 1.3.2.2 martin igc_free_pci_resources(sc); 530 1.3.2.2 martin kmem_free(sc->mta, IGC_MTA_LEN); 531 1.3.2.2 martin 532 1.3.2.2 martin mutex_destroy(&sc->sc_core_lock); 533 1.3.2.2 martin 534 1.3.2.2 martin return 0; 535 1.3.2.2 martin } 536 1.3.2.2 martin 537 1.3.2.2 martin static void 538 1.3.2.2 martin igc_identify_hardware(struct igc_softc *sc) 539 1.3.2.2 martin { 540 1.3.2.2 martin struct igc_osdep *os = &sc->osdep; 541 1.3.2.2 martin struct pci_attach_args *pa = &os->os_pa; 542 1.3.2.2 martin 543 1.3.2.2 martin /* Save off the information about this board. */ 544 1.3.2.2 martin sc->hw.device_id = PCI_PRODUCT(pa->pa_id); 545 1.3.2.2 martin 546 1.3.2.2 martin /* Do shared code init and setup. */ 547 1.3.2.2 martin if (igc_set_mac_type(&sc->hw)) { 548 1.3.2.2 martin aprint_error_dev(sc->sc_dev, "unable to identify hardware\n"); 549 1.3.2.2 martin return; 550 1.3.2.2 martin } 551 1.3.2.2 martin } 552 1.3.2.2 martin 553 1.3.2.2 martin static int 554 1.3.2.2 martin igc_allocate_pci_resources(struct igc_softc *sc) 555 1.3.2.2 martin { 556 1.3.2.2 martin struct igc_osdep *os = &sc->osdep; 557 1.3.2.2 martin struct pci_attach_args *pa = &os->os_pa; 558 1.3.2.2 martin 559 1.3.2.2 martin /* 560 1.3.2.2 martin * Enable bus mastering and memory-mapped I/O for sure. 561 1.3.2.2 martin */ 562 1.3.2.2 martin pcireg_t csr = 563 1.3.2.2 martin pci_conf_read(pa->pa_pc, pa->pa_tag, PCI_COMMAND_STATUS_REG); 564 1.3.2.2 martin csr |= PCI_COMMAND_MASTER_ENABLE | PCI_COMMAND_MEM_ENABLE; 565 1.3.2.2 martin pci_conf_write(pa->pa_pc, pa->pa_tag, PCI_COMMAND_STATUS_REG, csr); 566 1.3.2.2 martin 567 1.3.2.2 martin const pcireg_t memtype = 568 1.3.2.2 martin pci_mapreg_type(pa->pa_pc, pa->pa_tag, IGC_PCIREG); 569 1.3.2.2 martin if (pci_mapreg_map(pa, IGC_PCIREG, memtype, 0, &os->os_memt, 570 1.3.2.2 martin &os->os_memh, &os->os_membase, &os->os_memsize)) { 571 1.3.2.2 martin aprint_error_dev(sc->sc_dev, "unable to map registers\n"); 572 1.3.2.2 martin return ENXIO; 573 1.3.2.2 martin } 574 1.3.2.2 martin 575 1.3.2.2 martin sc->hw.hw_addr = os->os_membase; 576 1.3.2.2 martin sc->hw.back = os; 577 1.3.2.2 martin 578 1.3.2.2 martin return 0; 579 1.3.2.2 martin } 580 1.3.2.2 martin 581 1.3.2.2 martin static int __unused 582 1.3.2.2 martin igc_adjust_nqueues(struct igc_softc *sc) 583 1.3.2.2 martin { 584 1.3.2.2 martin struct pci_attach_args *pa = &sc->osdep.os_pa; 585 1.3.2.2 martin int nqueues = MIN(IGC_MAX_NQUEUES, ncpu); 586 1.3.2.2 martin 587 1.3.2.2 martin const int nmsix = pci_msix_count(pa->pa_pc, pa->pa_tag); 588 1.3.2.2 martin if (nmsix <= 1) 589 1.3.2.2 martin nqueues = 1; 590 1.3.2.2 martin else if (nmsix < nqueues + 1) 591 1.3.2.2 martin nqueues = nmsix - 1; 592 1.3.2.2 martin 593 1.3.2.2 martin return nqueues; 594 1.3.2.2 martin } 595 1.3.2.2 martin 596 1.3.2.2 martin static int 597 1.3.2.2 martin igc_allocate_interrupts(struct igc_softc *sc) 598 1.3.2.2 martin { 599 1.3.2.2 martin struct pci_attach_args *pa = &sc->osdep.os_pa; 600 1.3.2.2 martin int error; 601 1.3.2.2 martin 602 1.3.2.2 martin #ifndef IGC_DISABLE_MSIX 603 1.3.2.2 martin const int nqueues = igc_adjust_nqueues(sc); 604 1.3.2.2 martin if (nqueues > 1) { 605 1.3.2.2 martin sc->sc_nintrs = nqueues + 1; 606 1.3.2.2 martin error = pci_msix_alloc_exact(pa, &sc->sc_intrs, sc->sc_nintrs); 607 1.3.2.2 martin if (!error) { 608 1.3.2.2 martin sc->sc_nqueues = nqueues; 609 1.3.2.2 martin sc->sc_intr_type = PCI_INTR_TYPE_MSIX; 610 1.3.2.2 martin return 0; 611 1.3.2.2 martin } 612 1.3.2.2 martin } 613 1.3.2.2 martin #endif 614 1.3.2.2 martin 615 1.3.2.2 martin /* fallback to MSI */ 616 1.3.2.2 martin sc->sc_nintrs = sc->sc_nqueues = 1; 617 1.3.2.2 martin 618 1.3.2.2 martin #ifndef IGC_DISABLE_MSI 619 1.3.2.2 martin error = pci_msi_alloc_exact(pa, &sc->sc_intrs, sc->sc_nintrs); 620 1.3.2.2 martin if (!error) { 621 1.3.2.2 martin sc->sc_intr_type = PCI_INTR_TYPE_MSI; 622 1.3.2.2 martin return 0; 623 1.3.2.2 martin } 624 1.3.2.2 martin #endif 625 1.3.2.2 martin 626 1.3.2.2 martin /* fallback to INTx */ 627 1.3.2.2 martin 628 1.3.2.2 martin error = pci_intx_alloc(pa, &sc->sc_intrs); 629 1.3.2.2 martin if (!error) { 630 1.3.2.2 martin sc->sc_intr_type = PCI_INTR_TYPE_INTX; 631 1.3.2.2 martin return 0; 632 1.3.2.2 martin } 633 1.3.2.2 martin 634 1.3.2.2 martin return error; 635 1.3.2.2 martin } 636 1.3.2.2 martin 637 1.3.2.2 martin static int 638 1.3.2.2 martin igc_allocate_queues(struct igc_softc *sc) 639 1.3.2.2 martin { 640 1.3.2.2 martin device_t dev = sc->sc_dev; 641 1.3.2.2 martin int rxconf = 0, txconf = 0; 642 1.3.2.2 martin 643 1.3.2.2 martin /* Allocate the top level queue structs. */ 644 1.3.2.2 martin sc->queues = 645 1.3.2.2 martin kmem_zalloc(sc->sc_nqueues * sizeof(struct igc_queue), KM_SLEEP); 646 1.3.2.2 martin 647 1.3.2.2 martin /* Allocate the TX ring. */ 648 1.3.2.2 martin sc->tx_rings = 649 1.3.2.2 martin kmem_zalloc(sc->sc_nqueues * sizeof(struct tx_ring), KM_SLEEP); 650 1.3.2.2 martin 651 1.3.2.2 martin /* Allocate the RX ring. */ 652 1.3.2.2 martin sc->rx_rings = 653 1.3.2.2 martin kmem_zalloc(sc->sc_nqueues * sizeof(struct rx_ring), KM_SLEEP); 654 1.3.2.2 martin 655 1.3.2.2 martin /* Set up the TX queues. */ 656 1.3.2.2 martin for (int iq = 0; iq < sc->sc_nqueues; iq++, txconf++) { 657 1.3.2.2 martin struct tx_ring *txr = &sc->tx_rings[iq]; 658 1.3.2.2 martin const int tsize = roundup2( 659 1.3.2.2 martin sc->num_tx_desc * sizeof(union igc_adv_tx_desc), 660 1.3.2.2 martin IGC_DBA_ALIGN); 661 1.3.2.2 martin 662 1.3.2.2 martin txr->sc = sc; 663 1.3.2.2 martin txr->txr_igcq = &sc->queues[iq]; 664 1.3.2.2 martin txr->me = iq; 665 1.3.2.2 martin if (igc_dma_malloc(sc, tsize, &txr->txdma)) { 666 1.3.2.2 martin aprint_error_dev(dev, 667 1.3.2.2 martin "unable to allocate TX descriptor\n"); 668 1.3.2.2 martin goto fail; 669 1.3.2.2 martin } 670 1.3.2.2 martin txr->tx_base = (union igc_adv_tx_desc *)txr->txdma.dma_vaddr; 671 1.3.2.2 martin memset(txr->tx_base, 0, tsize); 672 1.3.2.2 martin } 673 1.3.2.2 martin 674 1.3.2.2 martin /* Prepare transmit descriptors and buffers. */ 675 1.3.2.2 martin if (igc_setup_transmit_structures(sc)) { 676 1.3.2.2 martin aprint_error_dev(dev, "unable to setup transmit structures\n"); 677 1.3.2.2 martin goto fail; 678 1.3.2.2 martin } 679 1.3.2.2 martin 680 1.3.2.2 martin /* Set up the RX queues. */ 681 1.3.2.2 martin for (int iq = 0; iq < sc->sc_nqueues; iq++, rxconf++) { 682 1.3.2.2 martin struct rx_ring *rxr = &sc->rx_rings[iq]; 683 1.3.2.2 martin const int rsize = roundup2( 684 1.3.2.2 martin sc->num_rx_desc * sizeof(union igc_adv_rx_desc), 685 1.3.2.2 martin IGC_DBA_ALIGN); 686 1.3.2.2 martin 687 1.3.2.2 martin rxr->sc = sc; 688 1.3.2.2 martin rxr->rxr_igcq = &sc->queues[iq]; 689 1.3.2.2 martin rxr->me = iq; 690 1.3.2.2 martin #ifdef OPENBSD 691 1.3.2.2 martin timeout_set(&rxr->rx_refill, igc_rxrefill, rxr); 692 1.3.2.2 martin #endif 693 1.3.2.2 martin if (igc_dma_malloc(sc, rsize, &rxr->rxdma)) { 694 1.3.2.2 martin aprint_error_dev(dev, 695 1.3.2.2 martin "unable to allocate RX descriptor\n"); 696 1.3.2.2 martin goto fail; 697 1.3.2.2 martin } 698 1.3.2.2 martin rxr->rx_base = (union igc_adv_rx_desc *)rxr->rxdma.dma_vaddr; 699 1.3.2.2 martin memset(rxr->rx_base, 0, rsize); 700 1.3.2.2 martin } 701 1.3.2.2 martin 702 1.3.2.2 martin sc->rx_mbuf_sz = MCLBYTES; 703 1.3.2.2 martin /* Prepare receive descriptors and buffers. */ 704 1.3.2.2 martin if (igc_setup_receive_structures(sc)) { 705 1.3.2.2 martin aprint_error_dev(sc->sc_dev, 706 1.3.2.2 martin "unable to setup receive structures\n"); 707 1.3.2.2 martin goto fail; 708 1.3.2.2 martin } 709 1.3.2.2 martin 710 1.3.2.2 martin /* Set up the queue holding structs. */ 711 1.3.2.2 martin for (int iq = 0; iq < sc->sc_nqueues; iq++) { 712 1.3.2.2 martin struct igc_queue *q = &sc->queues[iq]; 713 1.3.2.2 martin 714 1.3.2.2 martin q->sc = sc; 715 1.3.2.2 martin q->txr = &sc->tx_rings[iq]; 716 1.3.2.2 martin q->rxr = &sc->rx_rings[iq]; 717 1.3.2.2 martin } 718 1.3.2.2 martin 719 1.3.2.2 martin return 0; 720 1.3.2.2 martin 721 1.3.2.2 martin fail: 722 1.3.2.2 martin for (struct rx_ring *rxr = sc->rx_rings; rxconf > 0; rxr++, rxconf--) 723 1.3.2.2 martin igc_dma_free(sc, &rxr->rxdma); 724 1.3.2.2 martin for (struct tx_ring *txr = sc->tx_rings; txconf > 0; txr++, txconf--) 725 1.3.2.2 martin igc_dma_free(sc, &txr->txdma); 726 1.3.2.2 martin 727 1.3.2.2 martin kmem_free(sc->rx_rings, sc->sc_nqueues * sizeof(struct rx_ring)); 728 1.3.2.2 martin sc->rx_rings = NULL; 729 1.3.2.2 martin kmem_free(sc->tx_rings, sc->sc_nqueues * sizeof(struct tx_ring)); 730 1.3.2.2 martin sc->tx_rings = NULL; 731 1.3.2.2 martin kmem_free(sc->queues, sc->sc_nqueues * sizeof(struct igc_queue)); 732 1.3.2.2 martin sc->queues = NULL; 733 1.3.2.2 martin 734 1.3.2.2 martin return ENOMEM; 735 1.3.2.2 martin } 736 1.3.2.2 martin 737 1.3.2.2 martin static void 738 1.3.2.2 martin igc_free_pci_resources(struct igc_softc *sc) 739 1.3.2.2 martin { 740 1.3.2.2 martin struct igc_osdep *os = &sc->osdep; 741 1.3.2.2 martin 742 1.3.2.2 martin if (os->os_membase != 0) 743 1.3.2.2 martin bus_space_unmap(os->os_memt, os->os_memh, os->os_memsize); 744 1.3.2.2 martin os->os_membase = 0; 745 1.3.2.2 martin } 746 1.3.2.2 martin 747 1.3.2.2 martin static void 748 1.3.2.2 martin igc_free_interrupts(struct igc_softc *sc) 749 1.3.2.2 martin { 750 1.3.2.2 martin struct pci_attach_args *pa = &sc->osdep.os_pa; 751 1.3.2.2 martin pci_chipset_tag_t pc = pa->pa_pc; 752 1.3.2.2 martin 753 1.3.2.2 martin for (int i = 0; i < sc->sc_nintrs; i++) { 754 1.3.2.2 martin if (sc->sc_ihs[i] != NULL) { 755 1.3.2.2 martin pci_intr_disestablish(pc, sc->sc_ihs[i]); 756 1.3.2.2 martin sc->sc_ihs[i] = NULL; 757 1.3.2.2 martin } 758 1.3.2.2 martin } 759 1.3.2.2 martin pci_intr_release(pc, sc->sc_intrs, sc->sc_nintrs); 760 1.3.2.2 martin } 761 1.3.2.2 martin 762 1.3.2.2 martin static void 763 1.3.2.2 martin igc_free_queues(struct igc_softc *sc) 764 1.3.2.2 martin { 765 1.3.2.2 martin 766 1.3.2.2 martin igc_free_receive_structures(sc); 767 1.3.2.2 martin for (int iq = 0; iq < sc->sc_nqueues; iq++) { 768 1.3.2.2 martin struct rx_ring *rxr = &sc->rx_rings[iq]; 769 1.3.2.2 martin 770 1.3.2.2 martin igc_dma_free(sc, &rxr->rxdma); 771 1.3.2.2 martin } 772 1.3.2.2 martin 773 1.3.2.2 martin igc_free_transmit_structures(sc); 774 1.3.2.2 martin for (int iq = 0; iq < sc->sc_nqueues; iq++) { 775 1.3.2.2 martin struct tx_ring *txr = &sc->tx_rings[iq]; 776 1.3.2.2 martin 777 1.3.2.2 martin igc_dma_free(sc, &txr->txdma); 778 1.3.2.2 martin } 779 1.3.2.2 martin 780 1.3.2.2 martin kmem_free(sc->rx_rings, sc->sc_nqueues * sizeof(struct rx_ring)); 781 1.3.2.2 martin kmem_free(sc->tx_rings, sc->sc_nqueues * sizeof(struct tx_ring)); 782 1.3.2.2 martin kmem_free(sc->queues, sc->sc_nqueues * sizeof(struct igc_queue)); 783 1.3.2.2 martin } 784 1.3.2.2 martin 785 1.3.2.2 martin /********************************************************************* 786 1.3.2.2 martin * 787 1.3.2.2 martin * Initialize the hardware to a configuration as specified by the 788 1.3.2.2 martin * adapter structure. 789 1.3.2.2 martin * 790 1.3.2.2 martin **********************************************************************/ 791 1.3.2.2 martin static void 792 1.3.2.2 martin igc_reset(struct igc_softc *sc) 793 1.3.2.2 martin { 794 1.3.2.2 martin struct igc_hw *hw = &sc->hw; 795 1.3.2.2 martin 796 1.3.2.2 martin /* Let the firmware know the OS is in control */ 797 1.3.2.2 martin igc_get_hw_control(sc); 798 1.3.2.2 martin 799 1.3.2.2 martin /* 800 1.3.2.2 martin * Packet Buffer Allocation (PBA) 801 1.3.2.2 martin * Writing PBA sets the receive portion of the buffer 802 1.3.2.2 martin * the remainder is used for the transmit buffer. 803 1.3.2.2 martin */ 804 1.3.2.2 martin const uint32_t pba = IGC_PBA_34K; 805 1.3.2.2 martin 806 1.3.2.2 martin /* 807 1.3.2.2 martin * These parameters control the automatic generation (Tx) and 808 1.3.2.2 martin * response (Rx) to Ethernet PAUSE frames. 809 1.3.2.2 martin * - High water mark should allow for at least two frames to be 810 1.3.2.2 martin * received after sending an XOFF. 811 1.3.2.2 martin * - Low water mark works best when it is very near the high water mark. 812 1.3.2.2 martin * This allows the receiver to restart by sending XON when it has 813 1.3.2.2 martin * drained a bit. Here we use an arbitrary value of 1500 which will 814 1.3.2.2 martin * restart after one full frame is pulled from the buffer. There 815 1.3.2.2 martin * could be several smaller frames in the buffer and if so they will 816 1.3.2.2 martin * not trigger the XON until their total number reduces the buffer 817 1.3.2.2 martin * by 1500. 818 1.3.2.2 martin * - The pause time is fairly large at 1000 x 512ns = 512 usec. 819 1.3.2.2 martin */ 820 1.3.2.2 martin const uint16_t rx_buffer_size = (pba & 0xffff) << 10; 821 1.3.2.2 martin 822 1.3.2.2 martin hw->fc.high_water = rx_buffer_size - 823 1.3.2.2 martin roundup2(sc->hw.mac.max_frame_size, 1024); 824 1.3.2.2 martin /* 16-byte granularity */ 825 1.3.2.2 martin hw->fc.low_water = hw->fc.high_water - 16; 826 1.3.2.2 martin 827 1.3.2.2 martin if (sc->fc) /* locally set flow control value? */ 828 1.3.2.2 martin hw->fc.requested_mode = sc->fc; 829 1.3.2.2 martin else 830 1.3.2.2 martin hw->fc.requested_mode = igc_fc_full; 831 1.3.2.2 martin 832 1.3.2.2 martin hw->fc.pause_time = IGC_FC_PAUSE_TIME; 833 1.3.2.2 martin 834 1.3.2.2 martin hw->fc.send_xon = true; 835 1.3.2.2 martin 836 1.3.2.2 martin /* Issue a global reset */ 837 1.3.2.2 martin igc_reset_hw(hw); 838 1.3.2.2 martin IGC_WRITE_REG(hw, IGC_WUC, 0); 839 1.3.2.2 martin 840 1.3.2.2 martin /* and a re-init */ 841 1.3.2.2 martin if (igc_init_hw(hw) < 0) { 842 1.3.2.2 martin aprint_error_dev(sc->sc_dev, "unable to reset hardware\n"); 843 1.3.2.2 martin return; 844 1.3.2.2 martin } 845 1.3.2.2 martin 846 1.3.2.2 martin /* Setup DMA Coalescing */ 847 1.3.2.2 martin igc_init_dmac(sc, pba); 848 1.3.2.2 martin 849 1.3.2.2 martin IGC_WRITE_REG(hw, IGC_VET, ETHERTYPE_VLAN); 850 1.3.2.2 martin igc_get_phy_info(hw); 851 1.3.2.2 martin igc_check_for_link(hw); 852 1.3.2.2 martin } 853 1.3.2.2 martin 854 1.3.2.2 martin /********************************************************************* 855 1.3.2.2 martin * 856 1.3.2.2 martin * Initialize the DMA Coalescing feature 857 1.3.2.2 martin * 858 1.3.2.2 martin **********************************************************************/ 859 1.3.2.2 martin static void 860 1.3.2.2 martin igc_init_dmac(struct igc_softc *sc, uint32_t pba) 861 1.3.2.2 martin { 862 1.3.2.2 martin struct igc_hw *hw = &sc->hw; 863 1.3.2.2 martin const uint16_t max_frame_size = sc->hw.mac.max_frame_size; 864 1.3.2.2 martin uint32_t reg, status; 865 1.3.2.2 martin 866 1.3.2.2 martin if (sc->dmac == 0) { /* Disabling it */ 867 1.3.2.2 martin reg = ~IGC_DMACR_DMAC_EN; /* XXXRO */ 868 1.3.2.2 martin IGC_WRITE_REG(hw, IGC_DMACR, reg); 869 1.3.2.2 martin DPRINTF(MISC, "DMA coalescing disabled\n"); 870 1.3.2.2 martin return; 871 1.3.2.2 martin } else { 872 1.3.2.2 martin device_printf(sc->sc_dev, "DMA coalescing enabled\n"); 873 1.3.2.2 martin } 874 1.3.2.2 martin 875 1.3.2.2 martin /* Set starting threshold */ 876 1.3.2.2 martin IGC_WRITE_REG(hw, IGC_DMCTXTH, 0); 877 1.3.2.2 martin 878 1.3.2.2 martin uint16_t hwm = 64 * pba - max_frame_size / 16; 879 1.3.2.2 martin if (hwm < 64 * (pba - 6)) 880 1.3.2.2 martin hwm = 64 * (pba - 6); 881 1.3.2.2 martin reg = IGC_READ_REG(hw, IGC_FCRTC); 882 1.3.2.2 martin reg &= ~IGC_FCRTC_RTH_COAL_MASK; 883 1.3.2.2 martin reg |= (hwm << IGC_FCRTC_RTH_COAL_SHIFT) & IGC_FCRTC_RTH_COAL_MASK; 884 1.3.2.2 martin IGC_WRITE_REG(hw, IGC_FCRTC, reg); 885 1.3.2.2 martin 886 1.3.2.2 martin uint32_t dmac = pba - max_frame_size / 512; 887 1.3.2.2 martin if (dmac < pba - 10) 888 1.3.2.2 martin dmac = pba - 10; 889 1.3.2.2 martin reg = IGC_READ_REG(hw, IGC_DMACR); 890 1.3.2.2 martin reg &= ~IGC_DMACR_DMACTHR_MASK; 891 1.3.2.2 martin reg |= (dmac << IGC_DMACR_DMACTHR_SHIFT) & IGC_DMACR_DMACTHR_MASK; 892 1.3.2.2 martin 893 1.3.2.2 martin /* transition to L0x or L1 if available..*/ 894 1.3.2.2 martin reg |= IGC_DMACR_DMAC_EN | IGC_DMACR_DMAC_LX_MASK; 895 1.3.2.2 martin 896 1.3.2.2 martin /* Check if status is 2.5Gb backplane connection 897 1.3.2.2 martin * before configuration of watchdog timer, which is 898 1.3.2.2 martin * in msec values in 12.8usec intervals 899 1.3.2.2 martin * watchdog timer= msec values in 32usec intervals 900 1.3.2.2 martin * for non 2.5Gb connection 901 1.3.2.2 martin */ 902 1.3.2.2 martin status = IGC_READ_REG(hw, IGC_STATUS); 903 1.3.2.2 martin if ((status & IGC_STATUS_2P5_SKU) && 904 1.3.2.2 martin !(status & IGC_STATUS_2P5_SKU_OVER)) 905 1.3.2.2 martin reg |= (sc->dmac * 5) >> 6; 906 1.3.2.2 martin else 907 1.3.2.2 martin reg |= sc->dmac >> 5; 908 1.3.2.2 martin 909 1.3.2.2 martin IGC_WRITE_REG(hw, IGC_DMACR, reg); 910 1.3.2.2 martin 911 1.3.2.2 martin IGC_WRITE_REG(hw, IGC_DMCRTRH, 0); 912 1.3.2.2 martin 913 1.3.2.2 martin /* Set the interval before transition */ 914 1.3.2.2 martin reg = IGC_READ_REG(hw, IGC_DMCTLX); 915 1.3.2.2 martin reg |= IGC_DMCTLX_DCFLUSH_DIS; 916 1.3.2.2 martin 917 1.3.2.2 martin /* 918 1.3.2.2 martin * in 2.5Gb connection, TTLX unit is 0.4 usec 919 1.3.2.2 martin * which is 0x4*2 = 0xA. But delay is still 4 usec 920 1.3.2.2 martin */ 921 1.3.2.2 martin status = IGC_READ_REG(hw, IGC_STATUS); 922 1.3.2.2 martin if ((status & IGC_STATUS_2P5_SKU) && 923 1.3.2.2 martin !(status & IGC_STATUS_2P5_SKU_OVER)) 924 1.3.2.2 martin reg |= 0xA; 925 1.3.2.2 martin else 926 1.3.2.2 martin reg |= 0x4; 927 1.3.2.2 martin 928 1.3.2.2 martin IGC_WRITE_REG(hw, IGC_DMCTLX, reg); 929 1.3.2.2 martin 930 1.3.2.2 martin /* free space in tx packet buffer to wake from DMA coal */ 931 1.3.2.2 martin IGC_WRITE_REG(hw, IGC_DMCTXTH, 932 1.3.2.2 martin (IGC_TXPBSIZE - (2 * max_frame_size)) >> 6); 933 1.3.2.2 martin 934 1.3.2.2 martin /* make low power state decision controlled by DMA coal */ 935 1.3.2.2 martin reg = IGC_READ_REG(hw, IGC_PCIEMISC); 936 1.3.2.2 martin reg &= ~IGC_PCIEMISC_LX_DECISION; 937 1.3.2.2 martin IGC_WRITE_REG(hw, IGC_PCIEMISC, reg); 938 1.3.2.2 martin } 939 1.3.2.2 martin 940 1.3.2.2 martin static int 941 1.3.2.2 martin igc_setup_interrupts(struct igc_softc *sc) 942 1.3.2.2 martin { 943 1.3.2.2 martin int error; 944 1.3.2.2 martin 945 1.3.2.2 martin switch (sc->sc_intr_type) { 946 1.3.2.2 martin case PCI_INTR_TYPE_MSIX: 947 1.3.2.2 martin error = igc_setup_msix(sc); 948 1.3.2.2 martin break; 949 1.3.2.2 martin case PCI_INTR_TYPE_MSI: 950 1.3.2.2 martin error = igc_setup_msi(sc); 951 1.3.2.2 martin break; 952 1.3.2.2 martin case PCI_INTR_TYPE_INTX: 953 1.3.2.2 martin error = igc_setup_intx(sc); 954 1.3.2.2 martin break; 955 1.3.2.2 martin default: 956 1.3.2.2 martin panic("%s: invalid interrupt type: %d", 957 1.3.2.2 martin device_xname(sc->sc_dev), sc->sc_intr_type); 958 1.3.2.2 martin } 959 1.3.2.2 martin 960 1.3.2.2 martin return error; 961 1.3.2.2 martin } 962 1.3.2.2 martin 963 1.3.2.2 martin static void 964 1.3.2.2 martin igc_attach_counters(struct igc_softc *sc) 965 1.3.2.2 martin { 966 1.3.2.2 martin #ifdef IGC_EVENT_COUNTERS 967 1.3.2.2 martin 968 1.3.2.2 martin /* Global counters */ 969 1.3.2.2 martin sc->sc_global_evcnts = kmem_zalloc( 970 1.3.2.2 martin IGC_GLOBAL_COUNTERS * sizeof(sc->sc_global_evcnts[0]), KM_SLEEP); 971 1.3.2.2 martin 972 1.3.2.2 martin for (int cnt = 0; cnt < IGC_GLOBAL_COUNTERS; cnt++) { 973 1.3.2.2 martin evcnt_attach_dynamic(&sc->sc_global_evcnts[cnt], 974 1.3.2.2 martin igc_global_counters[cnt].type, NULL, 975 1.3.2.2 martin device_xname(sc->sc_dev), igc_global_counters[cnt].name); 976 1.3.2.2 martin } 977 1.3.2.2 martin 978 1.3.2.2 martin /* Driver counters */ 979 1.3.2.2 martin sc->sc_driver_evcnts = kmem_zalloc( 980 1.3.2.2 martin IGC_DRIVER_COUNTERS * sizeof(sc->sc_driver_evcnts[0]), KM_SLEEP); 981 1.3.2.2 martin 982 1.3.2.2 martin for (int cnt = 0; cnt < IGC_DRIVER_COUNTERS; cnt++) { 983 1.3.2.2 martin evcnt_attach_dynamic(&sc->sc_driver_evcnts[cnt], 984 1.3.2.2 martin igc_driver_counters[cnt].type, NULL, 985 1.3.2.2 martin device_xname(sc->sc_dev), igc_driver_counters[cnt].name); 986 1.3.2.2 martin } 987 1.3.2.2 martin 988 1.3.2.2 martin for (int iq = 0; iq < sc->sc_nqueues; iq++) { 989 1.3.2.2 martin struct igc_queue *q = &sc->queues[iq]; 990 1.3.2.2 martin 991 1.3.2.2 martin q->igcq_driver_counters = kmem_zalloc( 992 1.3.2.2 martin IGC_DRIVER_COUNTERS * sizeof(q->igcq_driver_counters[0]), 993 1.3.2.2 martin KM_SLEEP); 994 1.3.2.2 martin } 995 1.3.2.2 martin 996 1.3.2.2 martin /* Queue counters */ 997 1.3.2.2 martin for (int iq = 0; iq < sc->sc_nqueues; iq++) { 998 1.3.2.2 martin struct igc_queue *q = &sc->queues[iq]; 999 1.3.2.2 martin 1000 1.3.2.2 martin snprintf(q->igcq_queue_evname, sizeof(q->igcq_queue_evname), 1001 1.3.2.2 martin "%s q%d", device_xname(sc->sc_dev), iq); 1002 1.3.2.2 martin 1003 1.3.2.2 martin q->igcq_queue_evcnts = kmem_zalloc( 1004 1.3.2.2 martin IGC_QUEUE_COUNTERS * sizeof(q->igcq_queue_evcnts[0]), 1005 1.3.2.2 martin KM_SLEEP); 1006 1.3.2.2 martin 1007 1.3.2.2 martin for (int cnt = 0; cnt < IGC_QUEUE_COUNTERS; cnt++) { 1008 1.3.2.2 martin evcnt_attach_dynamic(&q->igcq_queue_evcnts[cnt], 1009 1.3.2.2 martin igc_queue_counters[cnt].type, NULL, 1010 1.3.2.2 martin q->igcq_queue_evname, igc_queue_counters[cnt].name); 1011 1.3.2.2 martin } 1012 1.3.2.2 martin } 1013 1.3.2.2 martin 1014 1.3.2.2 martin /* MAC counters */ 1015 1.3.2.2 martin snprintf(sc->sc_mac_evname, sizeof(sc->sc_mac_evname), 1016 1.3.2.2 martin "%s Mac Statistics", device_xname(sc->sc_dev)); 1017 1.3.2.2 martin 1018 1.3.2.2 martin sc->sc_mac_evcnts = kmem_zalloc( 1019 1.3.2.2 martin IGC_MAC_COUNTERS * sizeof(sc->sc_mac_evcnts[0]), KM_SLEEP); 1020 1.3.2.2 martin 1021 1.3.2.2 martin for (int cnt = 0; cnt < IGC_MAC_COUNTERS; cnt++) { 1022 1.3.2.2 martin evcnt_attach_dynamic(&sc->sc_mac_evcnts[cnt], EVCNT_TYPE_MISC, 1023 1.3.2.2 martin NULL, sc->sc_mac_evname, igc_mac_counters[cnt].name); 1024 1.3.2.2 martin } 1025 1.3.2.2 martin #endif 1026 1.3.2.2 martin } 1027 1.3.2.2 martin 1028 1.3.2.2 martin static void 1029 1.3.2.2 martin igc_detach_counters(struct igc_softc *sc) 1030 1.3.2.2 martin { 1031 1.3.2.2 martin #ifdef IGC_EVENT_COUNTERS 1032 1.3.2.2 martin 1033 1.3.2.2 martin /* Global counters */ 1034 1.3.2.2 martin for (int cnt = 0; cnt < IGC_GLOBAL_COUNTERS; cnt++) 1035 1.3.2.2 martin evcnt_detach(&sc->sc_global_evcnts[cnt]); 1036 1.3.2.2 martin 1037 1.3.2.2 martin kmem_free(sc->sc_global_evcnts, 1038 1.3.2.6 martin IGC_GLOBAL_COUNTERS * sizeof(sc->sc_global_evcnts[0])); 1039 1.3.2.2 martin 1040 1.3.2.2 martin /* Driver counters */ 1041 1.3.2.2 martin for (int iq = 0; iq < sc->sc_nqueues; iq++) { 1042 1.3.2.2 martin struct igc_queue *q = &sc->queues[iq]; 1043 1.3.2.2 martin 1044 1.3.2.2 martin kmem_free(q->igcq_driver_counters, 1045 1.3.2.2 martin IGC_DRIVER_COUNTERS * sizeof(q->igcq_driver_counters[0])); 1046 1.3.2.2 martin } 1047 1.3.2.2 martin 1048 1.3.2.2 martin for (int cnt = 0; cnt < IGC_DRIVER_COUNTERS; cnt++) 1049 1.3.2.2 martin evcnt_detach(&sc->sc_driver_evcnts[cnt]); 1050 1.3.2.2 martin 1051 1.3.2.2 martin kmem_free(sc->sc_driver_evcnts, 1052 1.3.2.6 martin IGC_DRIVER_COUNTERS * sizeof(sc->sc_driver_evcnts[0])); 1053 1.3.2.2 martin 1054 1.3.2.2 martin /* Queue counters */ 1055 1.3.2.2 martin for (int iq = 0; iq < sc->sc_nqueues; iq++) { 1056 1.3.2.2 martin struct igc_queue *q = &sc->queues[iq]; 1057 1.3.2.2 martin 1058 1.3.2.2 martin for (int cnt = 0; cnt < IGC_QUEUE_COUNTERS; cnt++) 1059 1.3.2.2 martin evcnt_detach(&q->igcq_queue_evcnts[cnt]); 1060 1.3.2.2 martin 1061 1.3.2.2 martin kmem_free(q->igcq_queue_evcnts, 1062 1.3.2.2 martin IGC_QUEUE_COUNTERS * sizeof(q->igcq_queue_evcnts[0])); 1063 1.3.2.2 martin } 1064 1.3.2.2 martin 1065 1.3.2.2 martin /* MAC statistics */ 1066 1.3.2.2 martin for (int cnt = 0; cnt < IGC_MAC_COUNTERS; cnt++) 1067 1.3.2.2 martin evcnt_detach(&sc->sc_mac_evcnts[cnt]); 1068 1.3.2.2 martin 1069 1.3.2.2 martin kmem_free(sc->sc_mac_evcnts, 1070 1.3.2.2 martin IGC_MAC_COUNTERS * sizeof(sc->sc_mac_evcnts[0])); 1071 1.3.2.2 martin #endif 1072 1.3.2.2 martin } 1073 1.3.2.2 martin 1074 1.3.2.2 martin /* 1075 1.3.2.2 martin * XXX 1076 1.3.2.2 martin * FreeBSD uses 4-byte-wise read for 64-bit counters, while Linux just 1077 1.3.2.2 martin * drops hi words. 1078 1.3.2.2 martin */ 1079 1.3.2.2 martin static inline uint64_t __unused 1080 1.3.2.2 martin igc_read_mac_counter(struct igc_hw *hw, bus_size_t reg, bool is64) 1081 1.3.2.2 martin { 1082 1.3.2.2 martin uint64_t val; 1083 1.3.2.2 martin 1084 1.3.2.2 martin val = IGC_READ_REG(hw, reg); 1085 1.3.2.2 martin if (is64) 1086 1.3.2.2 martin val += ((uint64_t)IGC_READ_REG(hw, reg + 4)) << 32; 1087 1.3.2.2 martin return val; 1088 1.3.2.2 martin } 1089 1.3.2.2 martin 1090 1.3.2.2 martin static void 1091 1.3.2.2 martin igc_update_counters(struct igc_softc *sc) 1092 1.3.2.2 martin { 1093 1.3.2.2 martin #ifdef IGC_EVENT_COUNTERS 1094 1.3.2.2 martin 1095 1.3.2.2 martin /* Global counters: nop */ 1096 1.3.2.2 martin 1097 1.3.2.2 martin /* Driver counters */ 1098 1.3.2.2 martin uint64_t sum[IGC_DRIVER_COUNTERS]; 1099 1.3.2.2 martin 1100 1.3.2.2 martin memset(sum, 0, sizeof(sum)); 1101 1.3.2.2 martin for (int iq = 0; iq < sc->sc_nqueues; iq++) { 1102 1.3.2.2 martin struct igc_queue *q = &sc->queues[iq]; 1103 1.3.2.2 martin 1104 1.3.2.2 martin for (int cnt = 0; cnt < IGC_DRIVER_COUNTERS; cnt++) { 1105 1.3.2.2 martin sum[cnt] += IGC_QUEUE_DRIVER_COUNTER_VAL(q, cnt); 1106 1.3.2.2 martin IGC_QUEUE_DRIVER_COUNTER_STORE(q, cnt, 0); 1107 1.3.2.2 martin } 1108 1.3.2.2 martin } 1109 1.3.2.2 martin 1110 1.3.2.2 martin for (int cnt = 0; cnt < IGC_DRIVER_COUNTERS; cnt++) 1111 1.3.2.2 martin IGC_DRIVER_COUNTER_ADD(sc, cnt, sum[cnt]); 1112 1.3.2.2 martin 1113 1.3.2.2 martin /* Queue counters: nop */ 1114 1.3.2.2 martin 1115 1.3.2.2 martin /* Mac statistics */ 1116 1.3.2.2 martin struct igc_hw *hw = &sc->hw; 1117 1.3.2.4 martin struct ifnet *ifp = &sc->sc_ec.ec_if; 1118 1.3.2.4 martin uint64_t iqdrops = 0; 1119 1.3.2.2 martin 1120 1.3.2.2 martin for (int cnt = 0; cnt < IGC_MAC_COUNTERS; cnt++) { 1121 1.3.2.4 martin uint64_t val; 1122 1.3.2.4 martin bus_size_t regaddr = igc_mac_counters[cnt].reg; 1123 1.3.2.4 martin 1124 1.3.2.4 martin val = igc_read_mac_counter(hw, regaddr, 1125 1.3.2.4 martin igc_mac_counters[cnt].is64); 1126 1.3.2.4 martin IGC_MAC_COUNTER_ADD(sc, cnt, val); 1127 1.3.2.4 martin /* XXX Count MPC to iqdrops. */ 1128 1.3.2.4 martin if (regaddr == IGC_MPC) 1129 1.3.2.4 martin iqdrops += val; 1130 1.3.2.4 martin } 1131 1.3.2.4 martin 1132 1.3.2.4 martin for (int iq = 0; iq < sc->sc_nqueues; iq++) { 1133 1.3.2.4 martin uint32_t val; 1134 1.3.2.4 martin 1135 1.3.2.4 martin /* XXX RQDPC should be visible via evcnt(9). */ 1136 1.3.2.4 martin val = IGC_READ_REG(hw, IGC_RQDPC(iq)); 1137 1.3.2.4 martin 1138 1.3.2.4 martin /* RQDPC is not cleard on read. */ 1139 1.3.2.4 martin if (val != 0) 1140 1.3.2.4 martin IGC_WRITE_REG(hw, IGC_RQDPC(iq), 0); 1141 1.3.2.4 martin iqdrops += val; 1142 1.3.2.2 martin } 1143 1.3.2.4 martin 1144 1.3.2.4 martin if (iqdrops != 0) 1145 1.3.2.4 martin if_statadd(ifp, if_iqdrops, iqdrops); 1146 1.3.2.2 martin #endif 1147 1.3.2.2 martin } 1148 1.3.2.2 martin 1149 1.3.2.2 martin static void 1150 1.3.2.2 martin igc_clear_counters(struct igc_softc *sc) 1151 1.3.2.2 martin { 1152 1.3.2.2 martin #ifdef IGC_EVENT_COUNTERS 1153 1.3.2.2 martin 1154 1.3.2.2 martin /* Global counters */ 1155 1.3.2.2 martin for (int cnt = 0; cnt < IGC_GLOBAL_COUNTERS; cnt++) 1156 1.3.2.2 martin IGC_GLOBAL_COUNTER_STORE(sc, cnt, 0); 1157 1.3.2.2 martin 1158 1.3.2.2 martin /* Driver counters */ 1159 1.3.2.2 martin for (int iq = 0; iq < sc->sc_nqueues; iq++) { 1160 1.3.2.2 martin struct igc_queue *q = &sc->queues[iq]; 1161 1.3.2.2 martin 1162 1.3.2.2 martin for (int cnt = 0; cnt < IGC_DRIVER_COUNTERS; cnt++) 1163 1.3.2.2 martin IGC_QUEUE_DRIVER_COUNTER_STORE(q, cnt, 0); 1164 1.3.2.2 martin } 1165 1.3.2.2 martin 1166 1.3.2.2 martin for (int cnt = 0; cnt < IGC_DRIVER_COUNTERS; cnt++) 1167 1.3.2.2 martin IGC_DRIVER_COUNTER_STORE(sc, cnt, 0); 1168 1.3.2.2 martin 1169 1.3.2.2 martin /* Queue counters */ 1170 1.3.2.2 martin for (int iq = 0; iq < sc->sc_nqueues; iq++) { 1171 1.3.2.2 martin struct igc_queue *q = &sc->queues[iq]; 1172 1.3.2.2 martin 1173 1.3.2.2 martin for (int cnt = 0; cnt < IGC_QUEUE_COUNTERS; cnt++) 1174 1.3.2.2 martin IGC_QUEUE_COUNTER_STORE(q, cnt, 0); 1175 1.3.2.2 martin } 1176 1.3.2.2 martin 1177 1.3.2.2 martin /* Mac statistics */ 1178 1.3.2.2 martin struct igc_hw *hw = &sc->hw; 1179 1.3.2.2 martin 1180 1.3.2.2 martin for (int cnt = 0; cnt < IGC_MAC_COUNTERS; cnt++) { 1181 1.3.2.2 martin (void)igc_read_mac_counter(hw, igc_mac_counters[cnt].reg, 1182 1.3.2.2 martin igc_mac_counters[cnt].is64); 1183 1.3.2.2 martin IGC_MAC_COUNTER_STORE(sc, cnt, 0); 1184 1.3.2.2 martin } 1185 1.3.2.2 martin #endif 1186 1.3.2.2 martin } 1187 1.3.2.2 martin 1188 1.3.2.2 martin static int 1189 1.3.2.2 martin igc_setup_msix(struct igc_softc *sc) 1190 1.3.2.2 martin { 1191 1.3.2.2 martin pci_chipset_tag_t pc = sc->osdep.os_pa.pa_pc; 1192 1.3.2.2 martin device_t dev = sc->sc_dev; 1193 1.3.2.2 martin pci_intr_handle_t *intrs; 1194 1.3.2.2 martin void **ihs; 1195 1.3.2.2 martin const char *intrstr; 1196 1.3.2.2 martin char intrbuf[PCI_INTRSTR_LEN]; 1197 1.3.2.2 martin char xnamebuf[MAX(32, MAXCOMLEN)]; 1198 1.3.2.2 martin int iq, error; 1199 1.3.2.2 martin 1200 1.3.2.2 martin for (iq = 0, intrs = sc->sc_intrs, ihs = sc->sc_ihs; 1201 1.3.2.2 martin iq < sc->sc_nqueues; iq++, intrs++, ihs++) { 1202 1.3.2.2 martin struct igc_queue *q = &sc->queues[iq]; 1203 1.3.2.2 martin 1204 1.3.2.2 martin snprintf(xnamebuf, sizeof(xnamebuf), "%s: txrx %d", 1205 1.3.2.2 martin device_xname(dev), iq); 1206 1.3.2.2 martin 1207 1.3.2.2 martin intrstr = pci_intr_string(pc, *intrs, intrbuf, sizeof(intrbuf)); 1208 1.3.2.2 martin 1209 1.3.2.2 martin pci_intr_setattr(pc, intrs, PCI_INTR_MPSAFE, true); 1210 1.3.2.2 martin *ihs = pci_intr_establish_xname(pc, *intrs, IPL_NET, 1211 1.3.2.2 martin igc_intr_queue, q, xnamebuf); 1212 1.3.2.2 martin if (*ihs == NULL) { 1213 1.3.2.2 martin aprint_error_dev(dev, 1214 1.3.2.2 martin "unable to establish txrx interrupt at %s\n", 1215 1.3.2.2 martin intrstr); 1216 1.3.2.2 martin return ENOBUFS; 1217 1.3.2.2 martin } 1218 1.3.2.2 martin aprint_normal_dev(dev, "txrx interrupting at %s\n", intrstr); 1219 1.3.2.2 martin 1220 1.3.2.2 martin kcpuset_t *affinity; 1221 1.3.2.2 martin kcpuset_create(&affinity, true); 1222 1.3.2.2 martin kcpuset_set(affinity, iq % ncpu); 1223 1.3.2.2 martin error = interrupt_distribute(*ihs, affinity, NULL); 1224 1.3.2.2 martin if (error) { 1225 1.3.2.2 martin aprint_normal_dev(dev, 1226 1.3.2.2 martin "%s: unable to change affinity, use default CPU\n", 1227 1.3.2.2 martin intrstr); 1228 1.3.2.2 martin } 1229 1.3.2.2 martin kcpuset_destroy(affinity); 1230 1.3.2.2 martin 1231 1.3.2.2 martin q->igcq_si = softint_establish(SOFTINT_NET | SOFTINT_MPSAFE, 1232 1.3.2.2 martin igc_handle_queue, q); 1233 1.3.2.2 martin if (q->igcq_si == NULL) { 1234 1.3.2.2 martin aprint_error_dev(dev, 1235 1.3.2.2 martin "%s: unable to establish softint\n", intrstr); 1236 1.3.2.2 martin return ENOBUFS; 1237 1.3.2.2 martin } 1238 1.3.2.2 martin 1239 1.3.2.2 martin q->msix = iq; 1240 1.3.2.2 martin q->eims = 1 << iq; 1241 1.3.2.2 martin } 1242 1.3.2.2 martin 1243 1.3.2.2 martin snprintf(xnamebuf, MAXCOMLEN, "%s_tx_rx", device_xname(dev)); 1244 1.3.2.2 martin error = workqueue_create(&sc->sc_queue_wq, xnamebuf, 1245 1.3.2.2 martin igc_handle_queue_work, sc, IGC_WORKQUEUE_PRI, IPL_NET, 1246 1.3.2.2 martin WQ_PERCPU | WQ_MPSAFE); 1247 1.3.2.2 martin if (error) { 1248 1.3.2.2 martin aprint_error_dev(dev, "workqueue_create failed\n"); 1249 1.3.2.2 martin return ENOBUFS; 1250 1.3.2.2 martin } 1251 1.3.2.2 martin sc->sc_txrx_workqueue = false; 1252 1.3.2.2 martin 1253 1.3.2.2 martin intrstr = pci_intr_string(pc, *intrs, intrbuf, sizeof(intrbuf)); 1254 1.3.2.2 martin snprintf(xnamebuf, sizeof(xnamebuf), "%s: link", device_xname(dev)); 1255 1.3.2.2 martin pci_intr_setattr(pc, intrs, PCI_INTR_MPSAFE, true); 1256 1.3.2.2 martin *ihs = pci_intr_establish_xname(pc, *intrs, IPL_NET, 1257 1.3.2.2 martin igc_intr_link, sc, xnamebuf); 1258 1.3.2.2 martin if (*ihs == NULL) { 1259 1.3.2.2 martin aprint_error_dev(dev, 1260 1.3.2.2 martin "unable to establish link interrupt at %s\n", intrstr); 1261 1.3.2.2 martin return ENOBUFS; 1262 1.3.2.2 martin } 1263 1.3.2.2 martin aprint_normal_dev(dev, "link interrupting at %s\n", intrstr); 1264 1.3.2.2 martin /* use later in igc_configure_queues() */ 1265 1.3.2.2 martin sc->linkvec = iq; 1266 1.3.2.2 martin 1267 1.3.2.2 martin return 0; 1268 1.3.2.2 martin } 1269 1.3.2.2 martin 1270 1.3.2.2 martin static int 1271 1.3.2.2 martin igc_setup_msi(struct igc_softc *sc) 1272 1.3.2.2 martin { 1273 1.3.2.2 martin pci_chipset_tag_t pc = sc->osdep.os_pa.pa_pc; 1274 1.3.2.2 martin device_t dev = sc->sc_dev; 1275 1.3.2.2 martin pci_intr_handle_t *intr = sc->sc_intrs; 1276 1.3.2.2 martin void **ihs = sc->sc_ihs; 1277 1.3.2.2 martin const char *intrstr; 1278 1.3.2.2 martin char intrbuf[PCI_INTRSTR_LEN]; 1279 1.3.2.2 martin char xnamebuf[MAX(32, MAXCOMLEN)]; 1280 1.3.2.2 martin int error; 1281 1.3.2.2 martin 1282 1.3.2.2 martin intrstr = pci_intr_string(pc, *intr, intrbuf, sizeof(intrbuf)); 1283 1.3.2.2 martin 1284 1.3.2.2 martin snprintf(xnamebuf, sizeof(xnamebuf), "%s: msi", device_xname(dev)); 1285 1.3.2.2 martin pci_intr_setattr(pc, intr, PCI_INTR_MPSAFE, true); 1286 1.3.2.2 martin *ihs = pci_intr_establish_xname(pc, *intr, IPL_NET, 1287 1.3.2.2 martin igc_intr, sc, xnamebuf); 1288 1.3.2.2 martin if (*ihs == NULL) { 1289 1.3.2.2 martin aprint_error_dev(dev, 1290 1.3.2.2 martin "unable to establish interrupt at %s\n", intrstr); 1291 1.3.2.2 martin return ENOBUFS; 1292 1.3.2.2 martin } 1293 1.3.2.2 martin aprint_normal_dev(dev, "interrupting at %s\n", intrstr); 1294 1.3.2.2 martin 1295 1.3.2.2 martin struct igc_queue *iq = sc->queues; 1296 1.3.2.2 martin iq->igcq_si = softint_establish(SOFTINT_NET | SOFTINT_MPSAFE, 1297 1.3.2.2 martin igc_handle_queue, iq); 1298 1.3.2.2 martin if (iq->igcq_si == NULL) { 1299 1.3.2.2 martin aprint_error_dev(dev, 1300 1.3.2.2 martin "%s: unable to establish softint\n", intrstr); 1301 1.3.2.2 martin return ENOBUFS; 1302 1.3.2.2 martin } 1303 1.3.2.2 martin 1304 1.3.2.2 martin snprintf(xnamebuf, MAXCOMLEN, "%s_tx_rx", device_xname(dev)); 1305 1.3.2.2 martin error = workqueue_create(&sc->sc_queue_wq, xnamebuf, 1306 1.3.2.2 martin igc_handle_queue_work, sc, IGC_WORKQUEUE_PRI, IPL_NET, 1307 1.3.2.2 martin WQ_PERCPU | WQ_MPSAFE); 1308 1.3.2.2 martin if (error) { 1309 1.3.2.2 martin aprint_error_dev(dev, "workqueue_create failed\n"); 1310 1.3.2.2 martin return ENOBUFS; 1311 1.3.2.2 martin } 1312 1.3.2.2 martin sc->sc_txrx_workqueue = false; 1313 1.3.2.2 martin 1314 1.3.2.2 martin sc->queues[0].msix = 0; 1315 1.3.2.2 martin sc->linkvec = 0; 1316 1.3.2.2 martin 1317 1.3.2.2 martin return 0; 1318 1.3.2.2 martin } 1319 1.3.2.2 martin 1320 1.3.2.2 martin static int 1321 1.3.2.2 martin igc_setup_intx(struct igc_softc *sc) 1322 1.3.2.2 martin { 1323 1.3.2.2 martin pci_chipset_tag_t pc = sc->osdep.os_pa.pa_pc; 1324 1.3.2.2 martin device_t dev = sc->sc_dev; 1325 1.3.2.2 martin pci_intr_handle_t *intr = sc->sc_intrs; 1326 1.3.2.2 martin void **ihs = sc->sc_ihs; 1327 1.3.2.2 martin const char *intrstr; 1328 1.3.2.2 martin char intrbuf[PCI_INTRSTR_LEN]; 1329 1.3.2.2 martin char xnamebuf[32]; 1330 1.3.2.2 martin 1331 1.3.2.2 martin intrstr = pci_intr_string(pc, *intr, intrbuf, sizeof(intrbuf)); 1332 1.3.2.2 martin 1333 1.3.2.2 martin snprintf(xnamebuf, sizeof(xnamebuf), "%s:intx", device_xname(dev)); 1334 1.3.2.2 martin pci_intr_setattr(pc, intr, PCI_INTR_MPSAFE, true); 1335 1.3.2.2 martin *ihs = pci_intr_establish_xname(pc, *intr, IPL_NET, 1336 1.3.2.2 martin igc_intr, sc, xnamebuf); 1337 1.3.2.2 martin if (*ihs == NULL) { 1338 1.3.2.2 martin aprint_error_dev(dev, 1339 1.3.2.2 martin "unable to establish interrupt at %s\n", intrstr); 1340 1.3.2.2 martin return ENOBUFS; 1341 1.3.2.2 martin } 1342 1.3.2.2 martin aprint_normal_dev(dev, "interrupting at %s\n", intrstr); 1343 1.3.2.2 martin 1344 1.3.2.2 martin struct igc_queue *iq = sc->queues; 1345 1.3.2.2 martin iq->igcq_si = softint_establish(SOFTINT_NET | SOFTINT_MPSAFE, 1346 1.3.2.2 martin igc_handle_queue, iq); 1347 1.3.2.2 martin if (iq->igcq_si == NULL) { 1348 1.3.2.2 martin aprint_error_dev(dev, 1349 1.3.2.2 martin "%s: unable to establish softint\n", intrstr); 1350 1.3.2.2 martin return ENOBUFS; 1351 1.3.2.2 martin } 1352 1.3.2.2 martin 1353 1.3.2.2 martin /* create workqueue? */ 1354 1.3.2.2 martin sc->sc_txrx_workqueue = false; 1355 1.3.2.2 martin 1356 1.3.2.2 martin sc->queues[0].msix = 0; 1357 1.3.2.2 martin sc->linkvec = 0; 1358 1.3.2.2 martin 1359 1.3.2.2 martin return 0; 1360 1.3.2.2 martin } 1361 1.3.2.2 martin 1362 1.3.2.2 martin static int 1363 1.3.2.2 martin igc_dma_malloc(struct igc_softc *sc, bus_size_t size, struct igc_dma_alloc *dma) 1364 1.3.2.2 martin { 1365 1.3.2.2 martin struct igc_osdep *os = &sc->osdep; 1366 1.3.2.2 martin 1367 1.3.2.2 martin dma->dma_tag = os->os_dmat; 1368 1.3.2.2 martin 1369 1.3.2.2 martin if (bus_dmamap_create(dma->dma_tag, size, 1, size, 0, 1370 1.3.2.2 martin BUS_DMA_WAITOK | BUS_DMA_ALLOCNOW, &dma->dma_map)) 1371 1.3.2.2 martin return 1; 1372 1.3.2.2 martin if (bus_dmamem_alloc(dma->dma_tag, size, PAGE_SIZE, 0, &dma->dma_seg, 1373 1.3.2.2 martin 1, &dma->dma_nseg, BUS_DMA_WAITOK)) 1374 1.3.2.2 martin goto destroy; 1375 1.3.2.2 martin /* 1376 1.3.2.2 martin * XXXRO 1377 1.3.2.2 martin * 1378 1.3.2.2 martin * Coherent mapping for descriptors is required for now. 1379 1.3.2.2 martin * 1380 1.3.2.2 martin * Both TX and RX descriptors are 16-byte length, which is shorter 1381 1.3.2.2 martin * than dcache lines on modern CPUs. Therefore, sync for a descriptor 1382 1.3.2.2 martin * may overwrite DMA read for descriptors in the same cache line. 1383 1.3.2.2 martin * 1384 1.3.2.2 martin * Can't we avoid this by use cache-line-aligned descriptors at once? 1385 1.3.2.2 martin */ 1386 1.3.2.2 martin if (bus_dmamem_map(dma->dma_tag, &dma->dma_seg, dma->dma_nseg, size, 1387 1.3.2.2 martin &dma->dma_vaddr, BUS_DMA_WAITOK | BUS_DMA_COHERENT /* XXXRO */)) 1388 1.3.2.2 martin goto free; 1389 1.3.2.2 martin if (bus_dmamap_load(dma->dma_tag, dma->dma_map, dma->dma_vaddr, size, 1390 1.3.2.2 martin NULL, BUS_DMA_WAITOK)) 1391 1.3.2.2 martin goto unmap; 1392 1.3.2.2 martin 1393 1.3.2.2 martin dma->dma_size = size; 1394 1.3.2.2 martin 1395 1.3.2.2 martin return 0; 1396 1.3.2.2 martin unmap: 1397 1.3.2.2 martin bus_dmamem_unmap(dma->dma_tag, dma->dma_vaddr, size); 1398 1.3.2.2 martin free: 1399 1.3.2.2 martin bus_dmamem_free(dma->dma_tag, &dma->dma_seg, dma->dma_nseg); 1400 1.3.2.2 martin destroy: 1401 1.3.2.2 martin bus_dmamap_destroy(dma->dma_tag, dma->dma_map); 1402 1.3.2.2 martin dma->dma_map = NULL; 1403 1.3.2.2 martin dma->dma_tag = NULL; 1404 1.3.2.2 martin return 1; 1405 1.3.2.2 martin } 1406 1.3.2.2 martin 1407 1.3.2.2 martin static void 1408 1.3.2.2 martin igc_dma_free(struct igc_softc *sc, struct igc_dma_alloc *dma) 1409 1.3.2.2 martin { 1410 1.3.2.2 martin 1411 1.3.2.2 martin if (dma->dma_tag == NULL) 1412 1.3.2.2 martin return; 1413 1.3.2.2 martin 1414 1.3.2.2 martin if (dma->dma_map != NULL) { 1415 1.3.2.2 martin bus_dmamap_sync(dma->dma_tag, dma->dma_map, 0, 1416 1.3.2.2 martin dma->dma_map->dm_mapsize, 1417 1.3.2.2 martin BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE); 1418 1.3.2.2 martin bus_dmamap_unload(dma->dma_tag, dma->dma_map); 1419 1.3.2.2 martin bus_dmamem_unmap(dma->dma_tag, dma->dma_vaddr, dma->dma_size); 1420 1.3.2.2 martin bus_dmamem_free(dma->dma_tag, &dma->dma_seg, dma->dma_nseg); 1421 1.3.2.2 martin bus_dmamap_destroy(dma->dma_tag, dma->dma_map); 1422 1.3.2.2 martin dma->dma_map = NULL; 1423 1.3.2.2 martin } 1424 1.3.2.2 martin } 1425 1.3.2.2 martin 1426 1.3.2.2 martin /********************************************************************* 1427 1.3.2.2 martin * 1428 1.3.2.2 martin * Setup networking device structure and register an interface. 1429 1.3.2.2 martin * 1430 1.3.2.2 martin **********************************************************************/ 1431 1.3.2.2 martin static void 1432 1.3.2.2 martin igc_setup_interface(struct igc_softc *sc) 1433 1.3.2.2 martin { 1434 1.3.2.2 martin struct ifnet *ifp = &sc->sc_ec.ec_if; 1435 1.3.2.2 martin 1436 1.3.2.2 martin strlcpy(ifp->if_xname, device_xname(sc->sc_dev), sizeof(ifp->if_xname)); 1437 1.3.2.2 martin ifp->if_softc = sc; 1438 1.3.2.2 martin ifp->if_flags = IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST; 1439 1.3.2.2 martin ifp->if_extflags = IFEF_MPSAFE; 1440 1.3.2.2 martin ifp->if_ioctl = igc_ioctl; 1441 1.3.2.2 martin ifp->if_start = igc_start; 1442 1.3.2.2 martin if (sc->sc_nqueues > 1) 1443 1.3.2.2 martin ifp->if_transmit = igc_transmit; 1444 1.3.2.2 martin ifp->if_watchdog = igc_watchdog; 1445 1.3.2.2 martin ifp->if_init = igc_init; 1446 1.3.2.2 martin ifp->if_stop = igc_stop; 1447 1.3.2.2 martin 1448 1.3.2.2 martin ifp->if_capabilities = IFCAP_TSOv4 | IFCAP_TSOv6; 1449 1.3.2.2 martin 1450 1.3.2.2 martin ifp->if_capabilities |= 1451 1.3.2.2 martin IFCAP_CSUM_IPv4_Tx | IFCAP_CSUM_IPv4_Rx | 1452 1.3.2.2 martin IFCAP_CSUM_TCPv4_Tx | IFCAP_CSUM_TCPv4_Rx | 1453 1.3.2.2 martin IFCAP_CSUM_UDPv4_Tx | IFCAP_CSUM_UDPv4_Rx | 1454 1.3.2.2 martin IFCAP_CSUM_TCPv6_Tx | IFCAP_CSUM_TCPv6_Rx | 1455 1.3.2.2 martin IFCAP_CSUM_UDPv6_Tx | IFCAP_CSUM_UDPv6_Rx; 1456 1.3.2.2 martin 1457 1.3.2.2 martin ifp->if_capenable = 0; 1458 1.3.2.2 martin 1459 1.3.2.2 martin sc->sc_ec.ec_capabilities |= 1460 1.3.2.2 martin ETHERCAP_JUMBO_MTU | ETHERCAP_VLAN_MTU; 1461 1.3.2.2 martin 1462 1.3.2.2 martin IFQ_SET_MAXLEN(&ifp->if_snd, sc->num_tx_desc - 1); 1463 1.3.2.2 martin IFQ_SET_READY(&ifp->if_snd); 1464 1.3.2.2 martin 1465 1.3.2.2 martin #if NVLAN > 0 1466 1.3.2.2 martin sc->sc_ec.ec_capabilities |= ETHERCAP_VLAN_HWTAGGING; 1467 1.3.2.2 martin #endif 1468 1.3.2.2 martin 1469 1.3.2.2 martin mutex_init(&sc->sc_core_lock, MUTEX_DEFAULT, IPL_NET); 1470 1.3.2.2 martin 1471 1.3.2.2 martin /* Initialize ifmedia structures. */ 1472 1.3.2.2 martin sc->sc_ec.ec_ifmedia = &sc->media; 1473 1.3.2.2 martin ifmedia_init_with_lock(&sc->media, IFM_IMASK, igc_media_change, 1474 1.3.2.2 martin igc_media_status, &sc->sc_core_lock); 1475 1.3.2.2 martin ifmedia_add(&sc->media, IFM_ETHER | IFM_10_T, 0, NULL); 1476 1.3.2.2 martin ifmedia_add(&sc->media, IFM_ETHER | IFM_10_T | IFM_FDX, 0, NULL); 1477 1.3.2.2 martin ifmedia_add(&sc->media, IFM_ETHER | IFM_100_TX, 0, NULL); 1478 1.3.2.2 martin ifmedia_add(&sc->media, IFM_ETHER | IFM_100_TX | IFM_FDX, 0, NULL); 1479 1.3.2.2 martin ifmedia_add(&sc->media, IFM_ETHER | IFM_1000_T | IFM_FDX, 0, NULL); 1480 1.3.2.2 martin ifmedia_add(&sc->media, IFM_ETHER | IFM_2500_T | IFM_FDX, 0, NULL); 1481 1.3.2.2 martin ifmedia_add(&sc->media, IFM_ETHER | IFM_AUTO, 0, NULL); 1482 1.3.2.2 martin ifmedia_set(&sc->media, IFM_ETHER | IFM_AUTO); 1483 1.3.2.2 martin 1484 1.3.2.2 martin sc->sc_rx_intr_process_limit = IGC_RX_INTR_PROCESS_LIMIT_DEFAULT; 1485 1.3.2.2 martin sc->sc_tx_intr_process_limit = IGC_TX_INTR_PROCESS_LIMIT_DEFAULT; 1486 1.3.2.2 martin sc->sc_rx_process_limit = IGC_RX_PROCESS_LIMIT_DEFAULT; 1487 1.3.2.2 martin sc->sc_tx_process_limit = IGC_TX_PROCESS_LIMIT_DEFAULT; 1488 1.3.2.2 martin 1489 1.3.2.2 martin if_initialize(ifp); 1490 1.3.2.2 martin sc->sc_ipq = if_percpuq_create(ifp); 1491 1.3.2.2 martin if_deferred_start_init(ifp, NULL); 1492 1.3.2.2 martin ether_ifattach(ifp, sc->hw.mac.addr); 1493 1.3.2.2 martin ether_set_ifflags_cb(&sc->sc_ec, igc_ifflags_cb); 1494 1.3.2.2 martin if_register(ifp); 1495 1.3.2.2 martin } 1496 1.3.2.2 martin 1497 1.3.2.2 martin static int 1498 1.3.2.2 martin igc_init(struct ifnet *ifp) 1499 1.3.2.2 martin { 1500 1.3.2.2 martin struct igc_softc *sc = ifp->if_softc; 1501 1.3.2.2 martin int error; 1502 1.3.2.2 martin 1503 1.3.2.2 martin mutex_enter(&sc->sc_core_lock); 1504 1.3.2.2 martin error = igc_init_locked(sc); 1505 1.3.2.2 martin mutex_exit(&sc->sc_core_lock); 1506 1.3.2.2 martin 1507 1.3.2.2 martin return error; 1508 1.3.2.2 martin } 1509 1.3.2.2 martin 1510 1.3.2.2 martin static int 1511 1.3.2.2 martin igc_init_locked(struct igc_softc *sc) 1512 1.3.2.2 martin { 1513 1.3.2.2 martin struct ethercom *ec = &sc->sc_ec; 1514 1.3.2.2 martin struct ifnet *ifp = &ec->ec_if; 1515 1.3.2.2 martin 1516 1.3.2.2 martin DPRINTF(CFG, "called\n"); 1517 1.3.2.2 martin 1518 1.3.2.2 martin KASSERT(mutex_owned(&sc->sc_core_lock)); 1519 1.3.2.2 martin 1520 1.3.2.2 martin if (ISSET(ifp->if_flags, IFF_RUNNING)) 1521 1.3.2.2 martin igc_stop_locked(sc); 1522 1.3.2.2 martin 1523 1.3.2.2 martin /* Put the address into the receive address array. */ 1524 1.3.2.2 martin igc_rar_set(&sc->hw, sc->hw.mac.addr, 0); 1525 1.3.2.2 martin 1526 1.3.2.2 martin /* Initialize the hardware. */ 1527 1.3.2.2 martin igc_reset(sc); 1528 1.3.2.2 martin igc_update_link_status(sc); 1529 1.3.2.2 martin 1530 1.3.2.2 martin /* Setup VLAN support, basic and offload if available. */ 1531 1.3.2.2 martin IGC_WRITE_REG(&sc->hw, IGC_VET, ETHERTYPE_VLAN); 1532 1.3.2.2 martin 1533 1.3.2.2 martin igc_initialize_transmit_unit(sc); 1534 1.3.2.2 martin igc_initialize_receive_unit(sc); 1535 1.3.2.2 martin 1536 1.3.2.2 martin if (ec->ec_capenable & ETHERCAP_VLAN_HWTAGGING) { 1537 1.3.2.2 martin uint32_t ctrl = IGC_READ_REG(&sc->hw, IGC_CTRL); 1538 1.3.2.2 martin ctrl |= IGC_CTRL_VME; 1539 1.3.2.2 martin IGC_WRITE_REG(&sc->hw, IGC_CTRL, ctrl); 1540 1.3.2.2 martin } 1541 1.3.2.2 martin 1542 1.3.2.2 martin /* Setup multicast table. */ 1543 1.3.2.2 martin igc_set_filter(sc); 1544 1.3.2.2 martin 1545 1.3.2.2 martin igc_clear_hw_cntrs_base_generic(&sc->hw); 1546 1.3.2.2 martin 1547 1.3.2.2 martin if (sc->sc_intr_type == PCI_INTR_TYPE_MSIX) 1548 1.3.2.2 martin igc_configure_queues(sc); 1549 1.3.2.2 martin 1550 1.3.2.2 martin /* This clears any pending interrupts */ 1551 1.3.2.2 martin IGC_READ_REG(&sc->hw, IGC_ICR); 1552 1.3.2.2 martin IGC_WRITE_REG(&sc->hw, IGC_ICS, IGC_ICS_LSC); 1553 1.3.2.2 martin 1554 1.3.2.2 martin /* The driver can now take control from firmware. */ 1555 1.3.2.2 martin igc_get_hw_control(sc); 1556 1.3.2.2 martin 1557 1.3.2.2 martin /* Set Energy Efficient Ethernet. */ 1558 1.3.2.2 martin igc_set_eee_i225(&sc->hw, true, true, true); 1559 1.3.2.2 martin 1560 1.3.2.2 martin for (int iq = 0; iq < sc->sc_nqueues; iq++) { 1561 1.3.2.2 martin struct rx_ring *rxr = &sc->rx_rings[iq]; 1562 1.3.2.2 martin 1563 1.3.2.2 martin mutex_enter(&rxr->rxr_lock); 1564 1.3.2.2 martin igc_rxfill(rxr); 1565 1.3.2.2 martin mutex_exit(&rxr->rxr_lock); 1566 1.3.2.2 martin } 1567 1.3.2.2 martin 1568 1.3.2.2 martin sc->sc_core_stopping = false; 1569 1.3.2.2 martin 1570 1.3.2.2 martin ifp->if_flags |= IFF_RUNNING; 1571 1.3.2.2 martin 1572 1.3.2.2 martin /* Save last flags for the callback */ 1573 1.3.2.2 martin sc->sc_if_flags = ifp->if_flags; 1574 1.3.2.2 martin 1575 1.3.2.3 martin callout_schedule(&sc->sc_tick_ch, hz); 1576 1.3.2.3 martin 1577 1.3.2.3 martin igc_enable_intr(sc); 1578 1.3.2.3 martin 1579 1.3.2.2 martin return 0; 1580 1.3.2.2 martin } 1581 1.3.2.2 martin 1582 1.3.2.2 martin static inline int 1583 1.3.2.2 martin igc_load_mbuf(struct igc_queue *q, bus_dma_tag_t dmat, bus_dmamap_t map, 1584 1.3.2.2 martin struct mbuf *m) 1585 1.3.2.2 martin { 1586 1.3.2.2 martin int error; 1587 1.3.2.2 martin 1588 1.3.2.2 martin error = bus_dmamap_load_mbuf(dmat, map, m, 1589 1.3.2.2 martin BUS_DMA_WRITE | BUS_DMA_NOWAIT); 1590 1.3.2.2 martin 1591 1.3.2.2 martin if (__predict_false(error == EFBIG)) { 1592 1.3.2.2 martin IGC_DRIVER_EVENT(q, txdma_efbig, 1); 1593 1.3.2.2 martin m = m_defrag(m, M_NOWAIT); 1594 1.3.2.2 martin if (__predict_false(m == NULL)) { 1595 1.3.2.2 martin IGC_DRIVER_EVENT(q, txdma_defrag, 1); 1596 1.3.2.2 martin return ENOBUFS; 1597 1.3.2.2 martin } 1598 1.3.2.2 martin error = bus_dmamap_load_mbuf(dmat, map, m, 1599 1.3.2.2 martin BUS_DMA_WRITE | BUS_DMA_NOWAIT); 1600 1.3.2.2 martin } 1601 1.3.2.2 martin 1602 1.3.2.2 martin switch (error) { 1603 1.3.2.2 martin case 0: 1604 1.3.2.2 martin break; 1605 1.3.2.2 martin case ENOMEM: 1606 1.3.2.2 martin IGC_DRIVER_EVENT(q, txdma_enomem, 1); 1607 1.3.2.2 martin break; 1608 1.3.2.2 martin case EINVAL: 1609 1.3.2.2 martin IGC_DRIVER_EVENT(q, txdma_einval, 1); 1610 1.3.2.2 martin break; 1611 1.3.2.2 martin case EAGAIN: 1612 1.3.2.2 martin IGC_DRIVER_EVENT(q, txdma_eagain, 1); 1613 1.3.2.2 martin break; 1614 1.3.2.2 martin default: 1615 1.3.2.2 martin IGC_DRIVER_EVENT(q, txdma_other, 1); 1616 1.3.2.2 martin break; 1617 1.3.2.2 martin } 1618 1.3.2.2 martin 1619 1.3.2.2 martin return error; 1620 1.3.2.2 martin } 1621 1.3.2.2 martin 1622 1.3.2.2 martin #define IGC_TX_START 1 1623 1.3.2.2 martin #define IGC_TX_TRANSMIT 2 1624 1.3.2.2 martin 1625 1.3.2.2 martin static void 1626 1.3.2.2 martin igc_start(struct ifnet *ifp) 1627 1.3.2.2 martin { 1628 1.3.2.2 martin struct igc_softc *sc = ifp->if_softc; 1629 1.3.2.2 martin 1630 1.3.2.2 martin if (__predict_false(!sc->link_active)) { 1631 1.3.2.2 martin IFQ_PURGE(&ifp->if_snd); 1632 1.3.2.2 martin return; 1633 1.3.2.2 martin } 1634 1.3.2.2 martin 1635 1.3.2.2 martin struct tx_ring *txr = &sc->tx_rings[0]; /* queue 0 */ 1636 1.3.2.2 martin mutex_enter(&txr->txr_lock); 1637 1.3.2.2 martin igc_tx_common_locked(ifp, txr, IGC_TX_START); 1638 1.3.2.2 martin mutex_exit(&txr->txr_lock); 1639 1.3.2.2 martin } 1640 1.3.2.2 martin 1641 1.3.2.2 martin static inline u_int 1642 1.3.2.2 martin igc_select_txqueue(struct igc_softc *sc, struct mbuf *m __unused) 1643 1.3.2.2 martin { 1644 1.3.2.2 martin const u_int cpuid = cpu_index(curcpu()); 1645 1.3.2.2 martin 1646 1.3.2.2 martin return cpuid % sc->sc_nqueues; 1647 1.3.2.2 martin } 1648 1.3.2.2 martin 1649 1.3.2.2 martin static int 1650 1.3.2.2 martin igc_transmit(struct ifnet *ifp, struct mbuf *m) 1651 1.3.2.2 martin { 1652 1.3.2.2 martin struct igc_softc *sc = ifp->if_softc; 1653 1.3.2.2 martin const u_int qid = igc_select_txqueue(sc, m); 1654 1.3.2.2 martin struct tx_ring *txr = &sc->tx_rings[qid]; 1655 1.3.2.2 martin struct igc_queue *q = txr->txr_igcq; 1656 1.3.2.2 martin 1657 1.3.2.2 martin if (__predict_false(!pcq_put(txr->txr_interq, m))) { 1658 1.3.2.2 martin IGC_QUEUE_EVENT(q, tx_pcq_drop, 1); 1659 1.3.2.2 martin m_freem(m); 1660 1.3.2.2 martin return ENOBUFS; 1661 1.3.2.2 martin } 1662 1.3.2.2 martin 1663 1.3.2.2 martin mutex_enter(&txr->txr_lock); 1664 1.3.2.2 martin igc_tx_common_locked(ifp, txr, IGC_TX_TRANSMIT); 1665 1.3.2.2 martin mutex_exit(&txr->txr_lock); 1666 1.3.2.2 martin 1667 1.3.2.2 martin return 0; 1668 1.3.2.2 martin } 1669 1.3.2.2 martin 1670 1.3.2.2 martin static void 1671 1.3.2.2 martin igc_tx_common_locked(struct ifnet *ifp, struct tx_ring *txr, int caller) 1672 1.3.2.2 martin { 1673 1.3.2.2 martin struct igc_softc *sc = ifp->if_softc; 1674 1.3.2.2 martin struct igc_queue *q = txr->txr_igcq; 1675 1.3.2.2 martin net_stat_ref_t nsr = IF_STAT_GETREF(ifp); 1676 1.3.2.2 martin int prod, free, last = -1; 1677 1.3.2.2 martin bool post = false; 1678 1.3.2.2 martin 1679 1.3.2.2 martin prod = txr->next_avail_desc; 1680 1.3.2.2 martin free = txr->next_to_clean; 1681 1.3.2.2 martin if (free <= prod) 1682 1.3.2.2 martin free += sc->num_tx_desc; 1683 1.3.2.2 martin free -= prod; 1684 1.3.2.2 martin 1685 1.3.2.2 martin DPRINTF(TX, "%s: begin: msix %d prod %d n2c %d free %d\n", 1686 1.3.2.2 martin caller == IGC_TX_TRANSMIT ? "transmit" : "start", 1687 1.3.2.2 martin txr->me, prod, txr->next_to_clean, free); 1688 1.3.2.2 martin 1689 1.3.2.2 martin for (;;) { 1690 1.3.2.2 martin struct mbuf *m; 1691 1.3.2.2 martin 1692 1.3.2.2 martin if (__predict_false(free <= IGC_MAX_SCATTER)) { 1693 1.3.2.2 martin IGC_QUEUE_EVENT(q, tx_no_desc, 1); 1694 1.3.2.2 martin break; 1695 1.3.2.2 martin } 1696 1.3.2.2 martin 1697 1.3.2.2 martin if (caller == IGC_TX_TRANSMIT) 1698 1.3.2.2 martin m = pcq_get(txr->txr_interq); 1699 1.3.2.2 martin else 1700 1.3.2.2 martin IFQ_DEQUEUE(&ifp->if_snd, m); 1701 1.3.2.2 martin if (__predict_false(m == NULL)) 1702 1.3.2.2 martin break; 1703 1.3.2.2 martin 1704 1.3.2.2 martin struct igc_tx_buf *txbuf = &txr->tx_buffers[prod]; 1705 1.3.2.2 martin bus_dmamap_t map = txbuf->map; 1706 1.3.2.2 martin 1707 1.3.2.2 martin if (__predict_false( 1708 1.3.2.2 martin igc_load_mbuf(q, txr->txdma.dma_tag, map, m))) { 1709 1.3.2.2 martin if (caller == IGC_TX_TRANSMIT) 1710 1.3.2.2 martin IGC_QUEUE_EVENT(q, tx_pcq_drop, 1); 1711 1.3.2.2 martin m_freem(m); 1712 1.3.2.2 martin if_statinc_ref(nsr, if_oerrors); 1713 1.3.2.2 martin continue; 1714 1.3.2.2 martin } 1715 1.3.2.2 martin 1716 1.3.2.2 martin uint32_t ctx_cmd_type_len = 0, olinfo_status = 0; 1717 1.3.2.2 martin if (igc_tx_ctx_setup(txr, m, prod, &ctx_cmd_type_len, 1718 1.3.2.2 martin &olinfo_status)) { 1719 1.3.2.2 martin IGC_QUEUE_EVENT(q, tx_ctx, 1); 1720 1.3.2.2 martin /* Consume the first descriptor */ 1721 1.3.2.2 martin prod = igc_txdesc_incr(sc, prod); 1722 1.3.2.2 martin free--; 1723 1.3.2.2 martin } 1724 1.3.2.7 martin 1725 1.3.2.7 martin bus_dmamap_sync(txr->txdma.dma_tag, map, 0, 1726 1.3.2.7 martin map->dm_mapsize, BUS_DMASYNC_PREWRITE); 1727 1.3.2.7 martin 1728 1.3.2.2 martin for (int i = 0; i < map->dm_nsegs; i++) { 1729 1.3.2.2 martin union igc_adv_tx_desc *txdesc = &txr->tx_base[prod]; 1730 1.3.2.2 martin 1731 1.3.2.2 martin uint32_t cmd_type_len = ctx_cmd_type_len | 1732 1.3.2.2 martin IGC_ADVTXD_DCMD_IFCS | IGC_ADVTXD_DTYP_DATA | 1733 1.3.2.2 martin IGC_ADVTXD_DCMD_DEXT | map->dm_segs[i].ds_len; 1734 1.3.2.2 martin if (i == map->dm_nsegs - 1) { 1735 1.3.2.2 martin cmd_type_len |= 1736 1.3.2.2 martin IGC_ADVTXD_DCMD_EOP | IGC_ADVTXD_DCMD_RS; 1737 1.3.2.2 martin } 1738 1.3.2.2 martin 1739 1.3.2.2 martin igc_txdesc_sync(txr, prod, 1740 1.3.2.2 martin BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE); 1741 1.3.2.2 martin htolem64(&txdesc->read.buffer_addr, 1742 1.3.2.2 martin map->dm_segs[i].ds_addr); 1743 1.3.2.2 martin htolem32(&txdesc->read.cmd_type_len, cmd_type_len); 1744 1.3.2.2 martin htolem32(&txdesc->read.olinfo_status, olinfo_status); 1745 1.3.2.2 martin igc_txdesc_sync(txr, prod, 1746 1.3.2.2 martin BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); 1747 1.3.2.2 martin 1748 1.3.2.2 martin last = prod; 1749 1.3.2.2 martin prod = igc_txdesc_incr(sc, prod); 1750 1.3.2.2 martin } 1751 1.3.2.2 martin 1752 1.3.2.2 martin txbuf->m_head = m; 1753 1.3.2.2 martin txbuf->eop_index = last; 1754 1.3.2.2 martin 1755 1.3.2.2 martin bpf_mtap(ifp, m, BPF_D_OUT); 1756 1.3.2.2 martin 1757 1.3.2.2 martin if_statadd_ref(nsr, if_obytes, m->m_pkthdr.len); 1758 1.3.2.2 martin if (m->m_flags & M_MCAST) 1759 1.3.2.2 martin if_statinc_ref(nsr, if_omcasts); 1760 1.3.2.2 martin IGC_QUEUE_EVENT(q, tx_packets, 1); 1761 1.3.2.2 martin IGC_QUEUE_EVENT(q, tx_bytes, m->m_pkthdr.len); 1762 1.3.2.2 martin 1763 1.3.2.2 martin free -= map->dm_nsegs; 1764 1.3.2.2 martin post = true; 1765 1.3.2.2 martin } 1766 1.3.2.2 martin 1767 1.3.2.2 martin if (post) { 1768 1.3.2.2 martin txr->next_avail_desc = prod; 1769 1.3.2.2 martin IGC_WRITE_REG(&sc->hw, IGC_TDT(txr->me), prod); 1770 1.3.2.2 martin } 1771 1.3.2.2 martin 1772 1.3.2.2 martin DPRINTF(TX, "%s: done : msix %d prod %d n2c %d free %d\n", 1773 1.3.2.2 martin caller == IGC_TX_TRANSMIT ? "transmit" : "start", 1774 1.3.2.2 martin txr->me, prod, txr->next_to_clean, free); 1775 1.3.2.2 martin 1776 1.3.2.2 martin IF_STAT_PUTREF(ifp); 1777 1.3.2.2 martin } 1778 1.3.2.2 martin 1779 1.3.2.2 martin static bool 1780 1.3.2.2 martin igc_txeof(struct tx_ring *txr, u_int limit) 1781 1.3.2.2 martin { 1782 1.3.2.2 martin struct igc_softc *sc = txr->sc; 1783 1.3.2.2 martin struct ifnet *ifp = &sc->sc_ec.ec_if; 1784 1.3.2.2 martin int cons, prod; 1785 1.3.2.2 martin bool more = false; 1786 1.3.2.2 martin 1787 1.3.2.2 martin prod = txr->next_avail_desc; 1788 1.3.2.2 martin cons = txr->next_to_clean; 1789 1.3.2.2 martin 1790 1.3.2.2 martin if (cons == prod) { 1791 1.3.2.2 martin DPRINTF(TX, "false: msix %d cons %d prod %d\n", 1792 1.3.2.2 martin txr->me, cons, prod); 1793 1.3.2.2 martin return false; 1794 1.3.2.2 martin } 1795 1.3.2.2 martin 1796 1.3.2.2 martin do { 1797 1.3.2.2 martin struct igc_tx_buf *txbuf = &txr->tx_buffers[cons]; 1798 1.3.2.2 martin const int last = txbuf->eop_index; 1799 1.3.2.2 martin 1800 1.3.2.2 martin membar_consumer(); /* XXXRO necessary? */ 1801 1.3.2.2 martin 1802 1.3.2.2 martin KASSERT(last != -1); 1803 1.3.2.2 martin union igc_adv_tx_desc *txdesc = &txr->tx_base[last]; 1804 1.3.2.2 martin igc_txdesc_sync(txr, last, BUS_DMASYNC_POSTREAD); 1805 1.3.2.2 martin const uint32_t status = le32toh(txdesc->wb.status); 1806 1.3.2.2 martin igc_txdesc_sync(txr, last, BUS_DMASYNC_PREREAD); 1807 1.3.2.2 martin 1808 1.3.2.2 martin if (!(status & IGC_TXD_STAT_DD)) 1809 1.3.2.2 martin break; 1810 1.3.2.2 martin 1811 1.3.2.2 martin if (limit-- == 0) { 1812 1.3.2.2 martin more = true; 1813 1.3.2.2 martin DPRINTF(TX, "pending TX " 1814 1.3.2.2 martin "msix %d cons %d last %d prod %d " 1815 1.3.2.2 martin "status 0x%08x\n", 1816 1.3.2.2 martin txr->me, cons, last, prod, status); 1817 1.3.2.2 martin break; 1818 1.3.2.2 martin } 1819 1.3.2.2 martin 1820 1.3.2.2 martin DPRINTF(TX, "handled TX " 1821 1.3.2.2 martin "msix %d cons %d last %d prod %d " 1822 1.3.2.2 martin "status 0x%08x\n", 1823 1.3.2.2 martin txr->me, cons, last, prod, status); 1824 1.3.2.2 martin 1825 1.3.2.2 martin if_statinc(ifp, if_opackets); 1826 1.3.2.2 martin 1827 1.3.2.2 martin bus_dmamap_t map = txbuf->map; 1828 1.3.2.2 martin bus_dmamap_sync(txr->txdma.dma_tag, map, 0, map->dm_mapsize, 1829 1.3.2.2 martin BUS_DMASYNC_POSTWRITE); 1830 1.3.2.2 martin bus_dmamap_unload(txr->txdma.dma_tag, map); 1831 1.3.2.2 martin m_freem(txbuf->m_head); 1832 1.3.2.2 martin 1833 1.3.2.2 martin txbuf->m_head = NULL; 1834 1.3.2.2 martin txbuf->eop_index = -1; 1835 1.3.2.2 martin 1836 1.3.2.2 martin cons = igc_txdesc_incr(sc, last); 1837 1.3.2.2 martin } while (cons != prod); 1838 1.3.2.2 martin 1839 1.3.2.2 martin txr->next_to_clean = cons; 1840 1.3.2.2 martin 1841 1.3.2.2 martin return more; 1842 1.3.2.2 martin } 1843 1.3.2.2 martin 1844 1.3.2.2 martin static void 1845 1.3.2.2 martin igc_intr_barrier(struct igc_softc *sc __unused) 1846 1.3.2.2 martin { 1847 1.3.2.2 martin 1848 1.3.2.2 martin xc_barrier(0); 1849 1.3.2.2 martin } 1850 1.3.2.2 martin 1851 1.3.2.2 martin static void 1852 1.3.2.2 martin igc_stop(struct ifnet *ifp, int disable) 1853 1.3.2.2 martin { 1854 1.3.2.2 martin struct igc_softc *sc = ifp->if_softc; 1855 1.3.2.2 martin 1856 1.3.2.2 martin mutex_enter(&sc->sc_core_lock); 1857 1.3.2.2 martin igc_stop_locked(sc); 1858 1.3.2.2 martin mutex_exit(&sc->sc_core_lock); 1859 1.3.2.2 martin } 1860 1.3.2.2 martin 1861 1.3.2.2 martin /********************************************************************* 1862 1.3.2.2 martin * 1863 1.3.2.2 martin * This routine disables all traffic on the adapter by issuing a 1864 1.3.2.2 martin * global reset on the MAC. 1865 1.3.2.2 martin * 1866 1.3.2.2 martin **********************************************************************/ 1867 1.3.2.2 martin static void 1868 1.3.2.2 martin igc_stop_locked(struct igc_softc *sc) 1869 1.3.2.2 martin { 1870 1.3.2.2 martin struct ifnet *ifp = &sc->sc_ec.ec_if; 1871 1.3.2.2 martin 1872 1.3.2.2 martin DPRINTF(CFG, "called\n"); 1873 1.3.2.2 martin 1874 1.3.2.2 martin KASSERT(mutex_owned(&sc->sc_core_lock)); 1875 1.3.2.2 martin 1876 1.3.2.2 martin /* 1877 1.3.2.2 martin * If stopping processing has already started, do nothing. 1878 1.3.2.2 martin */ 1879 1.3.2.2 martin if ((ifp->if_flags & IFF_RUNNING) == 0) 1880 1.3.2.2 martin return; 1881 1.3.2.2 martin 1882 1.3.2.2 martin /* Tell the stack that the interface is no longer active. */ 1883 1.3.2.2 martin ifp->if_flags &= ~IFF_RUNNING; 1884 1.3.2.2 martin 1885 1.3.2.2 martin /* 1886 1.3.2.2 martin * igc_handle_queue() can enable interrupts, so wait for completion of 1887 1.3.2.2 martin * last igc_handle_queue() after unset IFF_RUNNING. 1888 1.3.2.2 martin */ 1889 1.3.2.2 martin mutex_exit(&sc->sc_core_lock); 1890 1.3.2.2 martin igc_barrier_handle_queue(sc); 1891 1.3.2.2 martin mutex_enter(&sc->sc_core_lock); 1892 1.3.2.2 martin 1893 1.3.2.2 martin sc->sc_core_stopping = true; 1894 1.3.2.2 martin 1895 1.3.2.2 martin igc_disable_intr(sc); 1896 1.3.2.2 martin 1897 1.3.2.2 martin callout_halt(&sc->sc_tick_ch, &sc->sc_core_lock); 1898 1.3.2.2 martin 1899 1.3.2.2 martin igc_reset_hw(&sc->hw); 1900 1.3.2.2 martin IGC_WRITE_REG(&sc->hw, IGC_WUC, 0); 1901 1.3.2.2 martin 1902 1.3.2.2 martin /* 1903 1.3.2.2 martin * Wait for completion of interrupt handlers. 1904 1.3.2.2 martin */ 1905 1.3.2.2 martin mutex_exit(&sc->sc_core_lock); 1906 1.3.2.2 martin igc_intr_barrier(sc); 1907 1.3.2.2 martin mutex_enter(&sc->sc_core_lock); 1908 1.3.2.2 martin 1909 1.3.2.2 martin igc_update_link_status(sc); 1910 1.3.2.2 martin 1911 1.3.2.2 martin for (int iq = 0; iq < sc->sc_nqueues; iq++) { 1912 1.3.2.2 martin struct tx_ring *txr = &sc->tx_rings[iq]; 1913 1.3.2.2 martin 1914 1.3.2.2 martin igc_withdraw_transmit_packets(txr, false); 1915 1.3.2.2 martin } 1916 1.3.2.2 martin 1917 1.3.2.2 martin for (int iq = 0; iq < sc->sc_nqueues; iq++) { 1918 1.3.2.2 martin struct rx_ring *rxr = &sc->rx_rings[iq]; 1919 1.3.2.2 martin 1920 1.3.2.2 martin igc_clear_receive_status(rxr); 1921 1.3.2.2 martin } 1922 1.3.2.2 martin 1923 1.3.2.2 martin /* Save last flags for the callback */ 1924 1.3.2.2 martin sc->sc_if_flags = ifp->if_flags; 1925 1.3.2.2 martin } 1926 1.3.2.2 martin 1927 1.3.2.2 martin /********************************************************************* 1928 1.3.2.2 martin * Ioctl entry point 1929 1.3.2.2 martin * 1930 1.3.2.2 martin * igc_ioctl is called when the user wants to configure the 1931 1.3.2.2 martin * interface. 1932 1.3.2.2 martin * 1933 1.3.2.2 martin * return 0 on success, positive on failure 1934 1.3.2.2 martin **********************************************************************/ 1935 1.3.2.2 martin static int 1936 1.3.2.2 martin igc_ioctl(struct ifnet * ifp, u_long cmd, void *data) 1937 1.3.2.2 martin { 1938 1.3.2.2 martin struct igc_softc *sc __unused = ifp->if_softc; 1939 1.3.2.2 martin int s; 1940 1.3.2.2 martin int error; 1941 1.3.2.2 martin 1942 1.3.2.2 martin DPRINTF(CFG, "cmd 0x%016lx\n", cmd); 1943 1.3.2.2 martin 1944 1.3.2.2 martin switch (cmd) { 1945 1.3.2.2 martin case SIOCADDMULTI: 1946 1.3.2.2 martin case SIOCDELMULTI: 1947 1.3.2.2 martin break; 1948 1.3.2.2 martin default: 1949 1.3.2.2 martin KASSERT(IFNET_LOCKED(ifp)); 1950 1.3.2.2 martin } 1951 1.3.2.2 martin 1952 1.3.2.2 martin if (cmd == SIOCZIFDATA) { 1953 1.3.2.2 martin mutex_enter(&sc->sc_core_lock); 1954 1.3.2.2 martin igc_clear_counters(sc); 1955 1.3.2.2 martin mutex_exit(&sc->sc_core_lock); 1956 1.3.2.2 martin } 1957 1.3.2.2 martin 1958 1.3.2.2 martin switch (cmd) { 1959 1.3.2.2 martin #ifdef IF_RXR 1960 1.3.2.2 martin case SIOCGIFRXR: 1961 1.3.2.2 martin s = splnet(); 1962 1.3.2.2 martin error = igc_rxrinfo(sc, (struct if_rxrinfo *)ifr->ifr_data); 1963 1.3.2.2 martin splx(s); 1964 1.3.2.2 martin break; 1965 1.3.2.2 martin #endif 1966 1.3.2.2 martin default: 1967 1.3.2.2 martin s = splnet(); 1968 1.3.2.2 martin error = ether_ioctl(ifp, cmd, data); 1969 1.3.2.2 martin splx(s); 1970 1.3.2.2 martin break; 1971 1.3.2.2 martin } 1972 1.3.2.2 martin 1973 1.3.2.2 martin if (error != ENETRESET) 1974 1.3.2.2 martin return error; 1975 1.3.2.2 martin 1976 1.3.2.2 martin error = 0; 1977 1.3.2.2 martin 1978 1.3.2.2 martin if (cmd == SIOCSIFCAP) 1979 1.3.2.2 martin error = if_init(ifp); 1980 1.3.2.2 martin else if ((cmd == SIOCADDMULTI) || (cmd == SIOCDELMULTI)) { 1981 1.3.2.2 martin mutex_enter(&sc->sc_core_lock); 1982 1.3.2.2 martin if (sc->sc_if_flags & IFF_RUNNING) { 1983 1.3.2.2 martin /* 1984 1.3.2.2 martin * Multicast list has changed; set the hardware filter 1985 1.3.2.2 martin * accordingly. 1986 1.3.2.2 martin */ 1987 1.3.2.2 martin igc_disable_intr(sc); 1988 1.3.2.2 martin igc_set_filter(sc); 1989 1.3.2.2 martin igc_enable_intr(sc); 1990 1.3.2.2 martin } 1991 1.3.2.2 martin mutex_exit(&sc->sc_core_lock); 1992 1.3.2.2 martin } 1993 1.3.2.2 martin 1994 1.3.2.2 martin return error; 1995 1.3.2.2 martin } 1996 1.3.2.2 martin 1997 1.3.2.2 martin #ifdef IF_RXR 1998 1.3.2.2 martin static int 1999 1.3.2.2 martin igc_rxrinfo(struct igc_softc *sc, struct if_rxrinfo *ifri) 2000 1.3.2.2 martin { 2001 1.3.2.2 martin struct if_rxring_info *ifr, ifr1; 2002 1.3.2.2 martin int error; 2003 1.3.2.2 martin 2004 1.3.2.2 martin if (sc->sc_nqueues > 1) { 2005 1.3.2.2 martin ifr = kmem_zalloc(sc->sc_nqueues * sizeof(*ifr), KM_SLEEP); 2006 1.3.2.2 martin } else { 2007 1.3.2.2 martin ifr = &ifr1; 2008 1.3.2.2 martin memset(ifr, 0, sizeof(*ifr)); 2009 1.3.2.2 martin } 2010 1.3.2.2 martin 2011 1.3.2.2 martin for (int iq = 0; iq < sc->sc_nqueues; iq++) { 2012 1.3.2.2 martin struct rx_ring *rxr = &sc->rx_rings[iq]; 2013 1.3.2.2 martin 2014 1.3.2.2 martin ifr[iq].ifr_size = MCLBYTES; 2015 1.3.2.2 martin snprintf(ifr[iq].ifr_name, sizeof(ifr[iq].ifr_name), "%d", iq); 2016 1.3.2.2 martin ifr[iq].ifr_info = rxr->rx_ring; 2017 1.3.2.2 martin } 2018 1.3.2.2 martin 2019 1.3.2.2 martin error = if_rxr_info_ioctl(ifri, sc->sc_nqueues, ifr); 2020 1.3.2.2 martin if (sc->sc_nqueues > 1) 2021 1.3.2.2 martin kmem_free(ifr, sc->sc_nqueues * sizeof(*ifr)); 2022 1.3.2.2 martin 2023 1.3.2.2 martin return error; 2024 1.3.2.2 martin } 2025 1.3.2.2 martin #endif 2026 1.3.2.2 martin 2027 1.3.2.2 martin static void 2028 1.3.2.2 martin igc_rxfill(struct rx_ring *rxr) 2029 1.3.2.2 martin { 2030 1.3.2.2 martin struct igc_softc *sc = rxr->sc; 2031 1.3.2.2 martin int id; 2032 1.3.2.2 martin 2033 1.3.2.2 martin for (id = 0; id < sc->num_rx_desc; id++) { 2034 1.3.2.2 martin if (igc_get_buf(rxr, id, false)) { 2035 1.3.2.2 martin panic("%s: msix=%d i=%d\n", __func__, rxr->me, id); 2036 1.3.2.2 martin } 2037 1.3.2.2 martin } 2038 1.3.2.2 martin 2039 1.3.2.2 martin id = sc->num_rx_desc - 1; 2040 1.3.2.2 martin rxr->last_desc_filled = id; 2041 1.3.2.2 martin IGC_WRITE_REG(&sc->hw, IGC_RDT(rxr->me), id); 2042 1.3.2.2 martin rxr->next_to_check = 0; 2043 1.3.2.2 martin } 2044 1.3.2.2 martin 2045 1.3.2.2 martin static void 2046 1.3.2.2 martin igc_rxrefill(struct rx_ring *rxr, int end) 2047 1.3.2.2 martin { 2048 1.3.2.2 martin struct igc_softc *sc = rxr->sc; 2049 1.3.2.2 martin int id; 2050 1.3.2.2 martin 2051 1.3.2.2 martin for (id = rxr->next_to_check; id != end; id = igc_rxdesc_incr(sc, id)) { 2052 1.3.2.2 martin if (igc_get_buf(rxr, id, true)) { 2053 1.3.2.2 martin /* XXXRO */ 2054 1.3.2.2 martin panic("%s: msix=%d id=%d\n", __func__, rxr->me, id); 2055 1.3.2.2 martin } 2056 1.3.2.2 martin } 2057 1.3.2.2 martin 2058 1.3.2.2 martin id = igc_rxdesc_decr(sc, id); 2059 1.3.2.2 martin DPRINTF(RX, "%s RDT %d id %d\n", 2060 1.3.2.2 martin rxr->last_desc_filled == id ? "same" : "diff", 2061 1.3.2.2 martin rxr->last_desc_filled, id); 2062 1.3.2.2 martin rxr->last_desc_filled = id; 2063 1.3.2.2 martin IGC_WRITE_REG(&sc->hw, IGC_RDT(rxr->me), id); 2064 1.3.2.2 martin } 2065 1.3.2.2 martin 2066 1.3.2.2 martin /********************************************************************* 2067 1.3.2.2 martin * 2068 1.3.2.2 martin * This routine executes in interrupt context. It replenishes 2069 1.3.2.2 martin * the mbufs in the descriptor and sends data which has been 2070 1.3.2.2 martin * dma'ed into host memory to upper layer. 2071 1.3.2.2 martin * 2072 1.3.2.2 martin *********************************************************************/ 2073 1.3.2.2 martin static bool 2074 1.3.2.2 martin igc_rxeof(struct rx_ring *rxr, u_int limit) 2075 1.3.2.2 martin { 2076 1.3.2.2 martin struct igc_softc *sc = rxr->sc; 2077 1.3.2.2 martin struct igc_queue *q = rxr->rxr_igcq; 2078 1.3.2.2 martin struct ifnet *ifp = &sc->sc_ec.ec_if; 2079 1.3.2.2 martin int id; 2080 1.3.2.2 martin bool more = false; 2081 1.3.2.2 martin 2082 1.3.2.2 martin id = rxr->next_to_check; 2083 1.3.2.2 martin for (;;) { 2084 1.3.2.2 martin union igc_adv_rx_desc *rxdesc = &rxr->rx_base[id]; 2085 1.3.2.2 martin struct igc_rx_buf *rxbuf, *nxbuf; 2086 1.3.2.2 martin struct mbuf *mp, *m; 2087 1.3.2.2 martin 2088 1.3.2.2 martin igc_rxdesc_sync(rxr, id, 2089 1.3.2.2 martin BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE); 2090 1.3.2.2 martin 2091 1.3.2.2 martin const uint32_t staterr = le32toh(rxdesc->wb.upper.status_error); 2092 1.3.2.2 martin 2093 1.3.2.2 martin if (!ISSET(staterr, IGC_RXD_STAT_DD)) { 2094 1.3.2.2 martin igc_rxdesc_sync(rxr, id, 2095 1.3.2.2 martin BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); 2096 1.3.2.2 martin break; 2097 1.3.2.2 martin } 2098 1.3.2.2 martin 2099 1.3.2.2 martin if (limit-- == 0) { 2100 1.3.2.2 martin igc_rxdesc_sync(rxr, id, 2101 1.3.2.2 martin BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); 2102 1.3.2.2 martin DPRINTF(RX, "more=true\n"); 2103 1.3.2.2 martin more = true; 2104 1.3.2.2 martin break; 2105 1.3.2.2 martin } 2106 1.3.2.2 martin 2107 1.3.2.2 martin /* Zero out the receive descriptors status. */ 2108 1.3.2.2 martin rxdesc->wb.upper.status_error = 0; 2109 1.3.2.2 martin 2110 1.3.2.2 martin /* Pull the mbuf off the ring. */ 2111 1.3.2.2 martin rxbuf = &rxr->rx_buffers[id]; 2112 1.3.2.2 martin bus_dmamap_t map = rxbuf->map; 2113 1.3.2.2 martin bus_dmamap_sync(rxr->rxdma.dma_tag, map, 2114 1.3.2.2 martin 0, map->dm_mapsize, BUS_DMASYNC_POSTREAD); 2115 1.3.2.2 martin bus_dmamap_unload(rxr->rxdma.dma_tag, map); 2116 1.3.2.2 martin 2117 1.3.2.2 martin mp = rxbuf->buf; 2118 1.3.2.2 martin rxbuf->buf = NULL; 2119 1.3.2.2 martin 2120 1.3.2.2 martin const bool eop = staterr & IGC_RXD_STAT_EOP; 2121 1.3.2.2 martin const uint16_t len = le16toh(rxdesc->wb.upper.length); 2122 1.3.2.2 martin 2123 1.3.2.5 martin #if NVLAN > 0 2124 1.3.2.2 martin const uint16_t vtag = le16toh(rxdesc->wb.upper.vlan); 2125 1.3.2.5 martin #endif 2126 1.3.2.2 martin 2127 1.3.2.2 martin const uint32_t ptype = le32toh(rxdesc->wb.lower.lo_dword.data) & 2128 1.3.2.2 martin IGC_PKTTYPE_MASK; 2129 1.3.2.2 martin 2130 1.3.2.2 martin const uint32_t hash __unused = 2131 1.3.2.2 martin le32toh(rxdesc->wb.lower.hi_dword.rss); 2132 1.3.2.2 martin const uint16_t hashtype __unused = 2133 1.3.2.2 martin le16toh(rxdesc->wb.lower.lo_dword.hs_rss.pkt_info) & 2134 1.3.2.2 martin IGC_RXDADV_RSSTYPE_MASK; 2135 1.3.2.2 martin 2136 1.3.2.2 martin igc_rxdesc_sync(rxr, id, 2137 1.3.2.2 martin BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); 2138 1.3.2.2 martin 2139 1.3.2.2 martin if (__predict_false(staterr & IGC_RXDEXT_STATERR_RXE)) { 2140 1.3.2.2 martin if (rxbuf->fmp) { 2141 1.3.2.2 martin m_freem(rxbuf->fmp); 2142 1.3.2.2 martin rxbuf->fmp = NULL; 2143 1.3.2.2 martin } 2144 1.3.2.2 martin 2145 1.3.2.2 martin m_freem(mp); 2146 1.3.2.2 martin m = NULL; 2147 1.3.2.2 martin 2148 1.3.2.2 martin if_statinc(ifp, if_ierrors); 2149 1.3.2.2 martin IGC_QUEUE_EVENT(q, rx_discard, 1); 2150 1.3.2.2 martin 2151 1.3.2.2 martin DPRINTF(RX, "ierrors++\n"); 2152 1.3.2.2 martin 2153 1.3.2.2 martin goto next_desc; 2154 1.3.2.2 martin } 2155 1.3.2.2 martin 2156 1.3.2.2 martin if (__predict_false(mp == NULL)) { 2157 1.3.2.2 martin panic("%s: igc_rxeof: NULL mbuf in slot %d " 2158 1.3.2.2 martin "(filled %d)", device_xname(sc->sc_dev), 2159 1.3.2.2 martin id, rxr->last_desc_filled); 2160 1.3.2.2 martin } 2161 1.3.2.2 martin 2162 1.3.2.2 martin if (!eop) { 2163 1.3.2.2 martin /* 2164 1.3.2.2 martin * Figure out the next descriptor of this frame. 2165 1.3.2.2 martin */ 2166 1.3.2.2 martin int nextp = igc_rxdesc_incr(sc, id); 2167 1.3.2.2 martin 2168 1.3.2.2 martin nxbuf = &rxr->rx_buffers[nextp]; 2169 1.3.2.2 martin /* 2170 1.3.2.2 martin * TODO prefetch(nxbuf); 2171 1.3.2.2 martin */ 2172 1.3.2.2 martin } 2173 1.3.2.2 martin 2174 1.3.2.2 martin mp->m_len = len; 2175 1.3.2.2 martin 2176 1.3.2.2 martin m = rxbuf->fmp; 2177 1.3.2.2 martin rxbuf->fmp = NULL; 2178 1.3.2.2 martin 2179 1.3.2.2 martin if (m != NULL) { 2180 1.3.2.2 martin m->m_pkthdr.len += mp->m_len; 2181 1.3.2.2 martin } else { 2182 1.3.2.2 martin m = mp; 2183 1.3.2.2 martin m->m_pkthdr.len = mp->m_len; 2184 1.3.2.2 martin #if NVLAN > 0 2185 1.3.2.2 martin if (staterr & IGC_RXD_STAT_VP) 2186 1.3.2.2 martin vlan_set_tag(m, vtag); 2187 1.3.2.2 martin #endif 2188 1.3.2.2 martin } 2189 1.3.2.2 martin 2190 1.3.2.2 martin /* Pass the head pointer on */ 2191 1.3.2.2 martin if (!eop) { 2192 1.3.2.2 martin nxbuf->fmp = m; 2193 1.3.2.2 martin m = NULL; 2194 1.3.2.2 martin mp->m_next = nxbuf->buf; 2195 1.3.2.2 martin } else { 2196 1.3.2.2 martin m_set_rcvif(m, ifp); 2197 1.3.2.2 martin 2198 1.3.2.2 martin m->m_pkthdr.csum_flags = igc_rx_checksum(q, 2199 1.3.2.2 martin ifp->if_capenable, staterr, ptype); 2200 1.3.2.2 martin 2201 1.3.2.2 martin #ifdef notyet 2202 1.3.2.2 martin if (hashtype != IGC_RXDADV_RSSTYPE_NONE) { 2203 1.3.2.2 martin m->m_pkthdr.ph_flowid = hash; 2204 1.3.2.2 martin SET(m->m_pkthdr.csum_flags, M_FLOWID); 2205 1.3.2.2 martin } 2206 1.3.2.2 martin ml_enqueue(&ml, m); 2207 1.3.2.2 martin #endif 2208 1.3.2.2 martin 2209 1.3.2.2 martin if_percpuq_enqueue(sc->sc_ipq, m); 2210 1.3.2.2 martin 2211 1.3.2.2 martin if_statinc(ifp, if_ipackets); 2212 1.3.2.2 martin IGC_QUEUE_EVENT(q, rx_packets, 1); 2213 1.3.2.2 martin IGC_QUEUE_EVENT(q, rx_bytes, m->m_pkthdr.len); 2214 1.3.2.2 martin } 2215 1.3.2.2 martin next_desc: 2216 1.3.2.2 martin /* Advance our pointers to the next descriptor. */ 2217 1.3.2.2 martin id = igc_rxdesc_incr(sc, id); 2218 1.3.2.2 martin } 2219 1.3.2.2 martin 2220 1.3.2.2 martin DPRINTF(RX, "fill queue[%d]\n", rxr->me); 2221 1.3.2.2 martin igc_rxrefill(rxr, id); 2222 1.3.2.2 martin 2223 1.3.2.2 martin DPRINTF(RX, "%s n2c %d id %d\n", 2224 1.3.2.2 martin rxr->next_to_check == id ? "same" : "diff", 2225 1.3.2.2 martin rxr->next_to_check, id); 2226 1.3.2.2 martin rxr->next_to_check = id; 2227 1.3.2.2 martin 2228 1.3.2.2 martin #ifdef OPENBSD 2229 1.3.2.2 martin if (!(staterr & IGC_RXD_STAT_DD)) 2230 1.3.2.2 martin return 0; 2231 1.3.2.2 martin #endif 2232 1.3.2.2 martin 2233 1.3.2.2 martin return more; 2234 1.3.2.2 martin } 2235 1.3.2.2 martin 2236 1.3.2.2 martin /********************************************************************* 2237 1.3.2.2 martin * 2238 1.3.2.2 martin * Verify that the hardware indicated that the checksum is valid. 2239 1.3.2.2 martin * Inform the stack about the status of checksum so that stack 2240 1.3.2.2 martin * doesn't spend time verifying the checksum. 2241 1.3.2.2 martin * 2242 1.3.2.2 martin *********************************************************************/ 2243 1.3.2.2 martin static int 2244 1.3.2.2 martin igc_rx_checksum(struct igc_queue *q, uint64_t capenable, uint32_t staterr, 2245 1.3.2.2 martin uint32_t ptype) 2246 1.3.2.2 martin { 2247 1.3.2.2 martin const uint16_t status = (uint16_t)staterr; 2248 1.3.2.2 martin const uint8_t errors = (uint8_t)(staterr >> 24); 2249 1.3.2.2 martin int flags = 0; 2250 1.3.2.2 martin 2251 1.3.2.2 martin if ((status & IGC_RXD_STAT_IPCS) != 0 && 2252 1.3.2.2 martin (capenable & IFCAP_CSUM_IPv4_Rx) != 0) { 2253 1.3.2.2 martin IGC_DRIVER_EVENT(q, rx_ipcs, 1); 2254 1.3.2.2 martin flags |= M_CSUM_IPv4; 2255 1.3.2.2 martin if (__predict_false((errors & IGC_RXD_ERR_IPE) != 0)) { 2256 1.3.2.2 martin IGC_DRIVER_EVENT(q, rx_ipcs_bad, 1); 2257 1.3.2.2 martin flags |= M_CSUM_IPv4_BAD; 2258 1.3.2.2 martin } 2259 1.3.2.2 martin } 2260 1.3.2.2 martin 2261 1.3.2.2 martin if ((status & IGC_RXD_STAT_TCPCS) != 0) { 2262 1.3.2.2 martin IGC_DRIVER_EVENT(q, rx_tcpcs, 1); 2263 1.3.2.2 martin if ((capenable & IFCAP_CSUM_TCPv4_Rx) != 0) 2264 1.3.2.2 martin flags |= M_CSUM_TCPv4; 2265 1.3.2.2 martin if ((capenable & IFCAP_CSUM_TCPv6_Rx) != 0) 2266 1.3.2.2 martin flags |= M_CSUM_TCPv6; 2267 1.3.2.2 martin } 2268 1.3.2.2 martin 2269 1.3.2.2 martin if ((status & IGC_RXD_STAT_UDPCS) != 0) { 2270 1.3.2.2 martin IGC_DRIVER_EVENT(q, rx_udpcs, 1); 2271 1.3.2.2 martin if ((capenable & IFCAP_CSUM_UDPv4_Rx) != 0) 2272 1.3.2.2 martin flags |= M_CSUM_UDPv4; 2273 1.3.2.2 martin if ((capenable & IFCAP_CSUM_UDPv6_Rx) != 0) 2274 1.3.2.2 martin flags |= M_CSUM_UDPv6; 2275 1.3.2.2 martin } 2276 1.3.2.2 martin 2277 1.3.2.2 martin if (__predict_false((errors & IGC_RXD_ERR_TCPE) != 0)) { 2278 1.3.2.2 martin IGC_DRIVER_EVENT(q, rx_l4cs_bad, 1); 2279 1.3.2.2 martin if ((flags & ~M_CSUM_IPv4) != 0) 2280 1.3.2.2 martin flags |= M_CSUM_TCP_UDP_BAD; 2281 1.3.2.2 martin } 2282 1.3.2.2 martin 2283 1.3.2.2 martin return flags; 2284 1.3.2.2 martin } 2285 1.3.2.2 martin 2286 1.3.2.2 martin static void 2287 1.3.2.2 martin igc_watchdog(struct ifnet * ifp) 2288 1.3.2.2 martin { 2289 1.3.2.2 martin } 2290 1.3.2.2 martin 2291 1.3.2.2 martin static void 2292 1.3.2.2 martin igc_tick(void *arg) 2293 1.3.2.2 martin { 2294 1.3.2.2 martin struct igc_softc *sc = arg; 2295 1.3.2.2 martin 2296 1.3.2.2 martin mutex_enter(&sc->sc_core_lock); 2297 1.3.2.2 martin 2298 1.3.2.2 martin if (__predict_false(sc->sc_core_stopping)) { 2299 1.3.2.2 martin mutex_exit(&sc->sc_core_lock); 2300 1.3.2.2 martin return; 2301 1.3.2.2 martin } 2302 1.3.2.2 martin 2303 1.3.2.2 martin /* XXX watchdog */ 2304 1.3.2.2 martin if (0) { 2305 1.3.2.2 martin IGC_GLOBAL_EVENT(sc, watchdog, 1); 2306 1.3.2.2 martin } 2307 1.3.2.2 martin 2308 1.3.2.2 martin igc_update_counters(sc); 2309 1.3.2.2 martin 2310 1.3.2.2 martin mutex_exit(&sc->sc_core_lock); 2311 1.3.2.2 martin 2312 1.3.2.2 martin callout_schedule(&sc->sc_tick_ch, hz); 2313 1.3.2.2 martin } 2314 1.3.2.2 martin 2315 1.3.2.2 martin /********************************************************************* 2316 1.3.2.2 martin * 2317 1.3.2.2 martin * Media Ioctl callback 2318 1.3.2.2 martin * 2319 1.3.2.2 martin * This routine is called whenever the user queries the status of 2320 1.3.2.2 martin * the interface using ifconfig. 2321 1.3.2.2 martin * 2322 1.3.2.2 martin **********************************************************************/ 2323 1.3.2.2 martin static void 2324 1.3.2.2 martin igc_media_status(struct ifnet *ifp, struct ifmediareq *ifmr) 2325 1.3.2.2 martin { 2326 1.3.2.2 martin struct igc_softc *sc = ifp->if_softc; 2327 1.3.2.2 martin struct igc_hw *hw = &sc->hw; 2328 1.3.2.2 martin 2329 1.3.2.2 martin igc_update_link_status(sc); 2330 1.3.2.2 martin 2331 1.3.2.2 martin ifmr->ifm_status = IFM_AVALID; 2332 1.3.2.2 martin ifmr->ifm_active = IFM_ETHER; 2333 1.3.2.2 martin 2334 1.3.2.2 martin if (!sc->link_active) { 2335 1.3.2.2 martin ifmr->ifm_active |= IFM_NONE; 2336 1.3.2.2 martin return; 2337 1.3.2.2 martin } 2338 1.3.2.2 martin 2339 1.3.2.2 martin ifmr->ifm_status |= IFM_ACTIVE; 2340 1.3.2.2 martin 2341 1.3.2.2 martin switch (sc->link_speed) { 2342 1.3.2.2 martin case 10: 2343 1.3.2.2 martin ifmr->ifm_active |= IFM_10_T; 2344 1.3.2.2 martin break; 2345 1.3.2.2 martin case 100: 2346 1.3.2.2 martin ifmr->ifm_active |= IFM_100_TX; 2347 1.3.2.2 martin break; 2348 1.3.2.2 martin case 1000: 2349 1.3.2.2 martin ifmr->ifm_active |= IFM_1000_T; 2350 1.3.2.2 martin break; 2351 1.3.2.2 martin case 2500: 2352 1.3.2.2 martin ifmr->ifm_active |= IFM_2500_T; 2353 1.3.2.2 martin break; 2354 1.3.2.2 martin } 2355 1.3.2.2 martin 2356 1.3.2.2 martin if (sc->link_duplex == FULL_DUPLEX) 2357 1.3.2.2 martin ifmr->ifm_active |= IFM_FDX; 2358 1.3.2.2 martin else 2359 1.3.2.2 martin ifmr->ifm_active |= IFM_HDX; 2360 1.3.2.2 martin 2361 1.3.2.2 martin switch (hw->fc.current_mode) { 2362 1.3.2.2 martin case igc_fc_tx_pause: 2363 1.3.2.2 martin ifmr->ifm_active |= IFM_FLOW | IFM_ETH_TXPAUSE; 2364 1.3.2.2 martin break; 2365 1.3.2.2 martin case igc_fc_rx_pause: 2366 1.3.2.2 martin ifmr->ifm_active |= IFM_FLOW | IFM_ETH_RXPAUSE; 2367 1.3.2.2 martin break; 2368 1.3.2.2 martin case igc_fc_full: 2369 1.3.2.2 martin ifmr->ifm_active |= IFM_FLOW | 2370 1.3.2.2 martin IFM_ETH_TXPAUSE | IFM_ETH_RXPAUSE; 2371 1.3.2.2 martin break; 2372 1.3.2.2 martin case igc_fc_none: 2373 1.3.2.2 martin default: 2374 1.3.2.2 martin break; 2375 1.3.2.2 martin } 2376 1.3.2.2 martin } 2377 1.3.2.2 martin 2378 1.3.2.2 martin /********************************************************************* 2379 1.3.2.2 martin * 2380 1.3.2.2 martin * Media Ioctl callback 2381 1.3.2.2 martin * 2382 1.3.2.2 martin * This routine is called when the user changes speed/duplex using 2383 1.3.2.2 martin * media/mediopt option with ifconfig. 2384 1.3.2.2 martin * 2385 1.3.2.2 martin **********************************************************************/ 2386 1.3.2.2 martin static int 2387 1.3.2.2 martin igc_media_change(struct ifnet *ifp) 2388 1.3.2.2 martin { 2389 1.3.2.2 martin struct igc_softc *sc = ifp->if_softc; 2390 1.3.2.2 martin struct ifmedia *ifm = &sc->media; 2391 1.3.2.2 martin 2392 1.3.2.2 martin if (IFM_TYPE(ifm->ifm_media) != IFM_ETHER) 2393 1.3.2.2 martin return EINVAL; 2394 1.3.2.2 martin 2395 1.3.2.2 martin sc->hw.mac.autoneg = DO_AUTO_NEG; 2396 1.3.2.2 martin 2397 1.3.2.2 martin switch (IFM_SUBTYPE(ifm->ifm_media)) { 2398 1.3.2.2 martin case IFM_AUTO: 2399 1.3.2.2 martin sc->hw.phy.autoneg_advertised = AUTONEG_ADV_DEFAULT; 2400 1.3.2.2 martin break; 2401 1.3.2.2 martin case IFM_2500_T: 2402 1.3.2.2 martin sc->hw.phy.autoneg_advertised = ADVERTISE_2500_FULL; 2403 1.3.2.2 martin break; 2404 1.3.2.2 martin case IFM_1000_T: 2405 1.3.2.2 martin sc->hw.phy.autoneg_advertised = ADVERTISE_1000_FULL; 2406 1.3.2.2 martin break; 2407 1.3.2.2 martin case IFM_100_TX: 2408 1.3.2.3 martin if ((ifm->ifm_media & IFM_GMASK) == IFM_FDX) 2409 1.3.2.2 martin sc->hw.phy.autoneg_advertised = ADVERTISE_100_FULL; 2410 1.3.2.3 martin else 2411 1.3.2.3 martin sc->hw.phy.autoneg_advertised = ADVERTISE_100_HALF; 2412 1.3.2.2 martin break; 2413 1.3.2.2 martin case IFM_10_T: 2414 1.3.2.3 martin if ((ifm->ifm_media & IFM_GMASK) == IFM_FDX) 2415 1.3.2.2 martin sc->hw.phy.autoneg_advertised = ADVERTISE_10_FULL; 2416 1.3.2.3 martin else 2417 1.3.2.3 martin sc->hw.phy.autoneg_advertised = ADVERTISE_10_HALF; 2418 1.3.2.2 martin break; 2419 1.3.2.2 martin default: 2420 1.3.2.2 martin return EINVAL; 2421 1.3.2.2 martin } 2422 1.3.2.2 martin 2423 1.3.2.2 martin igc_init_locked(sc); 2424 1.3.2.2 martin 2425 1.3.2.2 martin return 0; 2426 1.3.2.2 martin } 2427 1.3.2.2 martin 2428 1.3.2.2 martin static int 2429 1.3.2.2 martin igc_ifflags_cb(struct ethercom *ec) 2430 1.3.2.2 martin { 2431 1.3.2.2 martin struct ifnet *ifp = &ec->ec_if; 2432 1.3.2.2 martin struct igc_softc *sc = ifp->if_softc; 2433 1.3.2.2 martin int rc = 0; 2434 1.3.2.2 martin u_short iffchange; 2435 1.3.2.2 martin bool needreset = false; 2436 1.3.2.2 martin 2437 1.3.2.2 martin DPRINTF(CFG, "called\n"); 2438 1.3.2.2 martin 2439 1.3.2.2 martin KASSERT(IFNET_LOCKED(ifp)); 2440 1.3.2.2 martin 2441 1.3.2.2 martin mutex_enter(&sc->sc_core_lock); 2442 1.3.2.2 martin 2443 1.3.2.2 martin /* 2444 1.3.2.2 martin * Check for if_flags. 2445 1.3.2.2 martin * Main usage is to prevent linkdown when opening bpf. 2446 1.3.2.2 martin */ 2447 1.3.2.2 martin iffchange = ifp->if_flags ^ sc->sc_if_flags; 2448 1.3.2.2 martin sc->sc_if_flags = ifp->if_flags; 2449 1.3.2.2 martin if ((iffchange & ~(IFF_CANTCHANGE | IFF_DEBUG)) != 0) { 2450 1.3.2.2 martin needreset = true; 2451 1.3.2.2 martin goto ec; 2452 1.3.2.2 martin } 2453 1.3.2.2 martin 2454 1.3.2.2 martin /* iff related updates */ 2455 1.3.2.2 martin if ((iffchange & IFF_PROMISC) != 0) 2456 1.3.2.2 martin igc_set_filter(sc); 2457 1.3.2.2 martin 2458 1.3.2.2 martin #ifdef notyet 2459 1.3.2.2 martin igc_set_vlan(sc); 2460 1.3.2.2 martin #endif 2461 1.3.2.2 martin 2462 1.3.2.2 martin ec: 2463 1.3.2.2 martin #ifdef notyet 2464 1.3.2.2 martin /* Check for ec_capenable. */ 2465 1.3.2.2 martin ecchange = ec->ec_capenable ^ sc->sc_ec_capenable; 2466 1.3.2.2 martin sc->sc_ec_capenable = ec->ec_capenable; 2467 1.3.2.2 martin if ((ecchange & ~ETHERCAP_SOMETHING) != 0) { 2468 1.3.2.2 martin needreset = true; 2469 1.3.2.2 martin goto out; 2470 1.3.2.2 martin } 2471 1.3.2.2 martin #endif 2472 1.3.2.2 martin if (needreset) 2473 1.3.2.2 martin rc = ENETRESET; 2474 1.3.2.2 martin 2475 1.3.2.2 martin mutex_exit(&sc->sc_core_lock); 2476 1.3.2.2 martin 2477 1.3.2.2 martin return rc; 2478 1.3.2.2 martin } 2479 1.3.2.2 martin 2480 1.3.2.2 martin static void 2481 1.3.2.2 martin igc_set_filter(struct igc_softc *sc) 2482 1.3.2.2 martin { 2483 1.3.2.2 martin struct ethercom *ec = &sc->sc_ec; 2484 1.3.2.2 martin uint32_t rctl; 2485 1.3.2.2 martin 2486 1.3.2.2 martin rctl = IGC_READ_REG(&sc->hw, IGC_RCTL); 2487 1.3.2.2 martin rctl &= ~(IGC_RCTL_BAM |IGC_RCTL_UPE | IGC_RCTL_MPE); 2488 1.3.2.2 martin 2489 1.3.2.2 martin if ((sc->sc_if_flags & IFF_BROADCAST) != 0) 2490 1.3.2.2 martin rctl |= IGC_RCTL_BAM; 2491 1.3.2.2 martin if ((sc->sc_if_flags & IFF_PROMISC) != 0) { 2492 1.3.2.2 martin DPRINTF(CFG, "promisc\n"); 2493 1.3.2.2 martin rctl |= IGC_RCTL_UPE; 2494 1.3.2.2 martin ETHER_LOCK(ec); 2495 1.3.2.2 martin allmulti: 2496 1.3.2.2 martin ec->ec_flags |= ETHER_F_ALLMULTI; 2497 1.3.2.2 martin ETHER_UNLOCK(ec); 2498 1.3.2.2 martin rctl |= IGC_RCTL_MPE; 2499 1.3.2.2 martin } else { 2500 1.3.2.2 martin struct ether_multistep step; 2501 1.3.2.2 martin struct ether_multi *enm; 2502 1.3.2.2 martin int mcnt = 0; 2503 1.3.2.2 martin 2504 1.3.2.2 martin memset(sc->mta, 0, IGC_MTA_LEN); 2505 1.3.2.2 martin 2506 1.3.2.2 martin ETHER_LOCK(ec); 2507 1.3.2.2 martin ETHER_FIRST_MULTI(step, ec, enm); 2508 1.3.2.2 martin while (enm != NULL) { 2509 1.3.2.2 martin if (((memcmp(enm->enm_addrlo, enm->enm_addrhi, 2510 1.3.2.2 martin ETHER_ADDR_LEN)) != 0) || 2511 1.3.2.2 martin (mcnt >= MAX_NUM_MULTICAST_ADDRESSES)) { 2512 1.3.2.2 martin /* 2513 1.3.2.2 martin * We must listen to a range of multicast 2514 1.3.2.2 martin * addresses. For now, just accept all 2515 1.3.2.2 martin * multicasts, rather than trying to set only 2516 1.3.2.2 martin * those filter bits needed to match the range. 2517 1.3.2.2 martin * (At this time, the only use of address 2518 1.3.2.2 martin * ranges is for IP multicast routing, for 2519 1.3.2.2 martin * which the range is big enough to require all 2520 1.3.2.2 martin * bits set.) 2521 1.3.2.2 martin */ 2522 1.3.2.2 martin goto allmulti; 2523 1.3.2.2 martin } 2524 1.3.2.2 martin DPRINTF(CFG, "%d: %s\n", mcnt, 2525 1.3.2.2 martin ether_sprintf(enm->enm_addrlo)); 2526 1.3.2.2 martin memcpy(&sc->mta[mcnt * ETHER_ADDR_LEN], 2527 1.3.2.2 martin enm->enm_addrlo, ETHER_ADDR_LEN); 2528 1.3.2.2 martin 2529 1.3.2.2 martin mcnt++; 2530 1.3.2.2 martin ETHER_NEXT_MULTI(step, enm); 2531 1.3.2.2 martin } 2532 1.3.2.2 martin ec->ec_flags &= ~ETHER_F_ALLMULTI; 2533 1.3.2.2 martin ETHER_UNLOCK(ec); 2534 1.3.2.2 martin 2535 1.3.2.2 martin DPRINTF(CFG, "hw filter\n"); 2536 1.3.2.2 martin igc_update_mc_addr_list(&sc->hw, sc->mta, mcnt); 2537 1.3.2.2 martin } 2538 1.3.2.2 martin 2539 1.3.2.2 martin IGC_WRITE_REG(&sc->hw, IGC_RCTL, rctl); 2540 1.3.2.2 martin } 2541 1.3.2.2 martin 2542 1.3.2.2 martin static void 2543 1.3.2.2 martin igc_update_link_status(struct igc_softc *sc) 2544 1.3.2.2 martin { 2545 1.3.2.2 martin struct ifnet *ifp = &sc->sc_ec.ec_if; 2546 1.3.2.2 martin struct igc_hw *hw = &sc->hw; 2547 1.3.2.2 martin 2548 1.3.2.4 martin if (hw->mac.get_link_status == true) 2549 1.3.2.4 martin igc_check_for_link(hw); 2550 1.3.2.4 martin 2551 1.3.2.2 martin if (IGC_READ_REG(&sc->hw, IGC_STATUS) & IGC_STATUS_LU) { 2552 1.3.2.2 martin if (sc->link_active == 0) { 2553 1.3.2.2 martin igc_get_speed_and_duplex(hw, &sc->link_speed, 2554 1.3.2.2 martin &sc->link_duplex); 2555 1.3.2.2 martin sc->link_active = 1; 2556 1.3.2.2 martin ifp->if_baudrate = IF_Mbps(sc->link_speed); 2557 1.3.2.2 martin if_link_state_change(ifp, LINK_STATE_UP); 2558 1.3.2.2 martin } 2559 1.3.2.2 martin } else { 2560 1.3.2.2 martin if (sc->link_active == 1) { 2561 1.3.2.2 martin ifp->if_baudrate = sc->link_speed = 0; 2562 1.3.2.2 martin sc->link_duplex = 0; 2563 1.3.2.2 martin sc->link_active = 0; 2564 1.3.2.2 martin if_link_state_change(ifp, LINK_STATE_DOWN); 2565 1.3.2.2 martin } 2566 1.3.2.2 martin } 2567 1.3.2.2 martin } 2568 1.3.2.2 martin 2569 1.3.2.2 martin /********************************************************************* 2570 1.3.2.2 martin * 2571 1.3.2.2 martin * Get a buffer from system mbuf buffer pool. 2572 1.3.2.2 martin * 2573 1.3.2.2 martin **********************************************************************/ 2574 1.3.2.2 martin static int 2575 1.3.2.2 martin igc_get_buf(struct rx_ring *rxr, int id, bool strict) 2576 1.3.2.2 martin { 2577 1.3.2.2 martin struct igc_softc *sc = rxr->sc; 2578 1.3.2.2 martin struct igc_queue *q = rxr->rxr_igcq; 2579 1.3.2.2 martin struct igc_rx_buf *rxbuf = &rxr->rx_buffers[id]; 2580 1.3.2.2 martin bus_dmamap_t map = rxbuf->map; 2581 1.3.2.2 martin struct mbuf *m; 2582 1.3.2.2 martin int error; 2583 1.3.2.2 martin 2584 1.3.2.2 martin if (__predict_false(rxbuf->buf)) { 2585 1.3.2.2 martin if (strict) { 2586 1.3.2.2 martin DPRINTF(RX, "slot %d already has an mbuf\n", id); 2587 1.3.2.2 martin return EINVAL; 2588 1.3.2.2 martin } 2589 1.3.2.2 martin return 0; 2590 1.3.2.2 martin } 2591 1.3.2.2 martin 2592 1.3.2.2 martin MGETHDR(m, M_DONTWAIT, MT_DATA); 2593 1.3.2.2 martin if (__predict_false(m == NULL)) { 2594 1.3.2.2 martin enobuf: 2595 1.3.2.2 martin IGC_QUEUE_EVENT(q, rx_no_mbuf, 1); 2596 1.3.2.2 martin return ENOBUFS; 2597 1.3.2.2 martin } 2598 1.3.2.2 martin 2599 1.3.2.2 martin MCLGET(m, M_DONTWAIT); 2600 1.3.2.2 martin if (__predict_false(!(m->m_flags & M_EXT))) { 2601 1.3.2.2 martin m_freem(m); 2602 1.3.2.2 martin goto enobuf; 2603 1.3.2.2 martin } 2604 1.3.2.2 martin 2605 1.3.2.2 martin m->m_len = m->m_pkthdr.len = sc->rx_mbuf_sz; 2606 1.3.2.2 martin 2607 1.3.2.2 martin error = bus_dmamap_load_mbuf(rxr->rxdma.dma_tag, map, m, 2608 1.3.2.2 martin BUS_DMA_READ | BUS_DMA_NOWAIT); 2609 1.3.2.2 martin if (error) { 2610 1.3.2.2 martin m_freem(m); 2611 1.3.2.2 martin return error; 2612 1.3.2.2 martin } 2613 1.3.2.2 martin 2614 1.3.2.2 martin bus_dmamap_sync(rxr->rxdma.dma_tag, map, 0, 2615 1.3.2.2 martin map->dm_mapsize, BUS_DMASYNC_PREREAD); 2616 1.3.2.2 martin rxbuf->buf = m; 2617 1.3.2.2 martin 2618 1.3.2.2 martin union igc_adv_rx_desc *rxdesc = &rxr->rx_base[id]; 2619 1.3.2.2 martin igc_rxdesc_sync(rxr, id, BUS_DMASYNC_POSTWRITE | BUS_DMASYNC_POSTREAD); 2620 1.3.2.2 martin rxdesc->read.pkt_addr = htole64(map->dm_segs[0].ds_addr); 2621 1.3.2.2 martin igc_rxdesc_sync(rxr, id, BUS_DMASYNC_PREWRITE | BUS_DMASYNC_PREREAD); 2622 1.3.2.2 martin 2623 1.3.2.2 martin return 0; 2624 1.3.2.2 martin } 2625 1.3.2.2 martin 2626 1.3.2.2 martin static void 2627 1.3.2.2 martin igc_configure_queues(struct igc_softc *sc) 2628 1.3.2.2 martin { 2629 1.3.2.2 martin struct igc_hw *hw = &sc->hw; 2630 1.3.2.2 martin uint32_t ivar; 2631 1.3.2.2 martin 2632 1.3.2.2 martin /* First turn on RSS capability */ 2633 1.3.2.2 martin IGC_WRITE_REG(hw, IGC_GPIE, IGC_GPIE_MSIX_MODE | IGC_GPIE_EIAME | 2634 1.3.2.2 martin IGC_GPIE_PBA | IGC_GPIE_NSICR); 2635 1.3.2.2 martin 2636 1.3.2.2 martin /* Set the starting interrupt rate */ 2637 1.3.2.2 martin uint32_t newitr = (4000000 / MAX_INTS_PER_SEC) & 0x7FFC; 2638 1.3.2.2 martin newitr |= IGC_EITR_CNT_IGNR; 2639 1.3.2.2 martin 2640 1.3.2.2 martin /* Turn on MSI-X */ 2641 1.3.2.2 martin uint32_t newmask = 0; 2642 1.3.2.2 martin for (int iq = 0; iq < sc->sc_nqueues; iq++) { 2643 1.3.2.2 martin struct igc_queue *q = &sc->queues[iq]; 2644 1.3.2.2 martin 2645 1.3.2.2 martin /* RX entries */ 2646 1.3.2.2 martin igc_set_queues(sc, iq, q->msix, 0); 2647 1.3.2.2 martin /* TX entries */ 2648 1.3.2.2 martin igc_set_queues(sc, iq, q->msix, 1); 2649 1.3.2.2 martin newmask |= q->eims; 2650 1.3.2.2 martin IGC_WRITE_REG(hw, IGC_EITR(q->msix), newitr); 2651 1.3.2.2 martin } 2652 1.3.2.2 martin sc->msix_queuesmask = newmask; 2653 1.3.2.2 martin 2654 1.3.2.2 martin #if 1 2655 1.3.2.2 martin ivar = IGC_READ_REG_ARRAY(hw, IGC_IVAR0, 0); 2656 1.3.2.2 martin DPRINTF(CFG, "ivar(0)=0x%x\n", ivar); 2657 1.3.2.2 martin ivar = IGC_READ_REG_ARRAY(hw, IGC_IVAR0, 1); 2658 1.3.2.2 martin DPRINTF(CFG, "ivar(1)=0x%x\n", ivar); 2659 1.3.2.2 martin #endif 2660 1.3.2.2 martin 2661 1.3.2.2 martin /* And for the link interrupt */ 2662 1.3.2.2 martin ivar = (sc->linkvec | IGC_IVAR_VALID) << 8; 2663 1.3.2.2 martin sc->msix_linkmask = 1 << sc->linkvec; 2664 1.3.2.2 martin IGC_WRITE_REG(hw, IGC_IVAR_MISC, ivar); 2665 1.3.2.2 martin } 2666 1.3.2.2 martin 2667 1.3.2.2 martin static void 2668 1.3.2.2 martin igc_set_queues(struct igc_softc *sc, uint32_t entry, uint32_t vector, int type) 2669 1.3.2.2 martin { 2670 1.3.2.2 martin struct igc_hw *hw = &sc->hw; 2671 1.3.2.2 martin const uint32_t index = entry >> 1; 2672 1.3.2.2 martin uint32_t ivar = IGC_READ_REG_ARRAY(hw, IGC_IVAR0, index); 2673 1.3.2.2 martin 2674 1.3.2.2 martin if (type) { 2675 1.3.2.2 martin if (entry & 1) { 2676 1.3.2.2 martin ivar &= 0x00FFFFFF; 2677 1.3.2.2 martin ivar |= (vector | IGC_IVAR_VALID) << 24; 2678 1.3.2.2 martin } else { 2679 1.3.2.2 martin ivar &= 0xFFFF00FF; 2680 1.3.2.2 martin ivar |= (vector | IGC_IVAR_VALID) << 8; 2681 1.3.2.2 martin } 2682 1.3.2.2 martin } else { 2683 1.3.2.2 martin if (entry & 1) { 2684 1.3.2.2 martin ivar &= 0xFF00FFFF; 2685 1.3.2.2 martin ivar |= (vector | IGC_IVAR_VALID) << 16; 2686 1.3.2.2 martin } else { 2687 1.3.2.2 martin ivar &= 0xFFFFFF00; 2688 1.3.2.2 martin ivar |= vector | IGC_IVAR_VALID; 2689 1.3.2.2 martin } 2690 1.3.2.2 martin } 2691 1.3.2.2 martin IGC_WRITE_REG_ARRAY(hw, IGC_IVAR0, index, ivar); 2692 1.3.2.2 martin } 2693 1.3.2.2 martin 2694 1.3.2.2 martin static void 2695 1.3.2.2 martin igc_enable_queue(struct igc_softc *sc, uint32_t eims) 2696 1.3.2.2 martin { 2697 1.3.2.2 martin IGC_WRITE_REG(&sc->hw, IGC_EIMS, eims); 2698 1.3.2.2 martin } 2699 1.3.2.2 martin 2700 1.3.2.2 martin static void 2701 1.3.2.2 martin igc_enable_intr(struct igc_softc *sc) 2702 1.3.2.2 martin { 2703 1.3.2.2 martin struct igc_hw *hw = &sc->hw; 2704 1.3.2.2 martin 2705 1.3.2.2 martin if (sc->sc_intr_type == PCI_INTR_TYPE_MSIX) { 2706 1.3.2.2 martin const uint32_t mask = sc->msix_queuesmask | sc->msix_linkmask; 2707 1.3.2.2 martin 2708 1.3.2.2 martin IGC_WRITE_REG(hw, IGC_EIAC, mask); 2709 1.3.2.2 martin IGC_WRITE_REG(hw, IGC_EIAM, mask); 2710 1.3.2.2 martin IGC_WRITE_REG(hw, IGC_EIMS, mask); 2711 1.3.2.2 martin IGC_WRITE_REG(hw, IGC_IMS, IGC_IMS_LSC); 2712 1.3.2.2 martin } else { 2713 1.3.2.2 martin IGC_WRITE_REG(hw, IGC_IMS, IMS_ENABLE_MASK); 2714 1.3.2.2 martin } 2715 1.3.2.2 martin IGC_WRITE_FLUSH(hw); 2716 1.3.2.2 martin } 2717 1.3.2.2 martin 2718 1.3.2.2 martin static void 2719 1.3.2.2 martin igc_disable_intr(struct igc_softc *sc) 2720 1.3.2.2 martin { 2721 1.3.2.2 martin struct igc_hw *hw = &sc->hw; 2722 1.3.2.2 martin 2723 1.3.2.2 martin if (sc->sc_intr_type == PCI_INTR_TYPE_MSIX) { 2724 1.3.2.2 martin IGC_WRITE_REG(hw, IGC_EIMC, 0xffffffff); 2725 1.3.2.2 martin IGC_WRITE_REG(hw, IGC_EIAC, 0); 2726 1.3.2.2 martin } 2727 1.3.2.2 martin IGC_WRITE_REG(hw, IGC_IMC, 0xffffffff); 2728 1.3.2.2 martin IGC_WRITE_FLUSH(hw); 2729 1.3.2.2 martin } 2730 1.3.2.2 martin 2731 1.3.2.2 martin static int 2732 1.3.2.2 martin igc_intr_link(void *arg) 2733 1.3.2.2 martin { 2734 1.3.2.2 martin struct igc_softc *sc = (struct igc_softc *)arg; 2735 1.3.2.2 martin const uint32_t reg_icr = IGC_READ_REG(&sc->hw, IGC_ICR); 2736 1.3.2.2 martin 2737 1.3.2.2 martin IGC_GLOBAL_EVENT(sc, link, 1); 2738 1.3.2.2 martin 2739 1.3.2.2 martin if (reg_icr & IGC_ICR_LSC) { 2740 1.3.2.2 martin mutex_enter(&sc->sc_core_lock); 2741 1.3.2.2 martin sc->hw.mac.get_link_status = true; 2742 1.3.2.2 martin igc_update_link_status(sc); 2743 1.3.2.2 martin mutex_exit(&sc->sc_core_lock); 2744 1.3.2.2 martin } 2745 1.3.2.2 martin 2746 1.3.2.2 martin IGC_WRITE_REG(&sc->hw, IGC_IMS, IGC_IMS_LSC); 2747 1.3.2.2 martin IGC_WRITE_REG(&sc->hw, IGC_EIMS, sc->msix_linkmask); 2748 1.3.2.2 martin 2749 1.3.2.2 martin return 1; 2750 1.3.2.2 martin } 2751 1.3.2.2 martin 2752 1.3.2.2 martin static int 2753 1.3.2.2 martin igc_intr_queue(void *arg) 2754 1.3.2.2 martin { 2755 1.3.2.2 martin struct igc_queue *iq = arg; 2756 1.3.2.2 martin struct igc_softc *sc = iq->sc; 2757 1.3.2.2 martin struct ifnet *ifp = &sc->sc_ec.ec_if; 2758 1.3.2.2 martin struct rx_ring *rxr = iq->rxr; 2759 1.3.2.2 martin struct tx_ring *txr = iq->txr; 2760 1.3.2.2 martin const u_int txlimit = sc->sc_tx_intr_process_limit, 2761 1.3.2.2 martin rxlimit = sc->sc_rx_intr_process_limit; 2762 1.3.2.2 martin bool txmore, rxmore; 2763 1.3.2.2 martin 2764 1.3.2.2 martin IGC_QUEUE_EVENT(iq, irqs, 1); 2765 1.3.2.2 martin 2766 1.3.2.2 martin if (__predict_false(!ISSET(ifp->if_flags, IFF_RUNNING))) 2767 1.3.2.2 martin return 0; 2768 1.3.2.2 martin 2769 1.3.2.2 martin mutex_enter(&txr->txr_lock); 2770 1.3.2.2 martin txmore = igc_txeof(txr, txlimit); 2771 1.3.2.2 martin mutex_exit(&txr->txr_lock); 2772 1.3.2.2 martin mutex_enter(&rxr->rxr_lock); 2773 1.3.2.2 martin rxmore = igc_rxeof(rxr, rxlimit); 2774 1.3.2.2 martin mutex_exit(&rxr->rxr_lock); 2775 1.3.2.2 martin 2776 1.3.2.2 martin if (txmore || rxmore) { 2777 1.3.2.2 martin IGC_QUEUE_EVENT(iq, req, 1); 2778 1.3.2.2 martin igc_sched_handle_queue(sc, iq); 2779 1.3.2.2 martin } else { 2780 1.3.2.2 martin igc_enable_queue(sc, iq->eims); 2781 1.3.2.2 martin } 2782 1.3.2.2 martin 2783 1.3.2.2 martin return 1; 2784 1.3.2.2 martin } 2785 1.3.2.2 martin 2786 1.3.2.2 martin static int 2787 1.3.2.2 martin igc_intr(void *arg) 2788 1.3.2.2 martin { 2789 1.3.2.2 martin struct igc_softc *sc = arg; 2790 1.3.2.2 martin struct ifnet *ifp = &sc->sc_ec.ec_if; 2791 1.3.2.2 martin struct igc_queue *iq = &sc->queues[0]; 2792 1.3.2.2 martin struct rx_ring *rxr = iq->rxr; 2793 1.3.2.2 martin struct tx_ring *txr = iq->txr; 2794 1.3.2.2 martin const u_int txlimit = sc->sc_tx_intr_process_limit, 2795 1.3.2.2 martin rxlimit = sc->sc_rx_intr_process_limit; 2796 1.3.2.2 martin bool txmore, rxmore; 2797 1.3.2.2 martin 2798 1.3.2.2 martin if (__predict_false(!ISSET(ifp->if_flags, IFF_RUNNING))) 2799 1.3.2.2 martin return 0; 2800 1.3.2.2 martin 2801 1.3.2.2 martin const uint32_t reg_icr = IGC_READ_REG(&sc->hw, IGC_ICR); 2802 1.3.2.2 martin DPRINTF(MISC, "reg_icr=0x%x\n", reg_icr); 2803 1.3.2.2 martin 2804 1.3.2.2 martin /* Definitely not our interrupt. */ 2805 1.3.2.2 martin if (reg_icr == 0x0) { 2806 1.3.2.5 martin DPRINTF(MISC, "not for me\n"); 2807 1.3.2.2 martin return 0; 2808 1.3.2.2 martin } 2809 1.3.2.2 martin 2810 1.3.2.2 martin IGC_QUEUE_EVENT(iq, irqs, 1); 2811 1.3.2.2 martin 2812 1.3.2.2 martin /* Hot eject? */ 2813 1.3.2.2 martin if (__predict_false(reg_icr == 0xffffffff)) { 2814 1.3.2.2 martin DPRINTF(MISC, "hot eject\n"); 2815 1.3.2.2 martin return 0; 2816 1.3.2.2 martin } 2817 1.3.2.2 martin 2818 1.3.2.2 martin if (__predict_false(!(reg_icr & IGC_ICR_INT_ASSERTED))) { 2819 1.3.2.2 martin DPRINTF(MISC, "not set IGC_ICR_INT_ASSERTED"); 2820 1.3.2.2 martin return 0; 2821 1.3.2.2 martin } 2822 1.3.2.2 martin 2823 1.3.2.2 martin /* 2824 1.3.2.2 martin * Only MSI-X interrupts have one-shot behavior by taking advantage 2825 1.3.2.2 martin * of the EIAC register. Thus, explicitly disable interrupts. This 2826 1.3.2.2 martin * also works around the MSI message reordering errata on certain 2827 1.3.2.2 martin * systems. 2828 1.3.2.2 martin */ 2829 1.3.2.2 martin igc_disable_intr(sc); 2830 1.3.2.2 martin 2831 1.3.2.2 martin mutex_enter(&txr->txr_lock); 2832 1.3.2.2 martin txmore = igc_txeof(txr, txlimit); 2833 1.3.2.2 martin mutex_exit(&txr->txr_lock); 2834 1.3.2.2 martin mutex_enter(&rxr->rxr_lock); 2835 1.3.2.2 martin rxmore = igc_rxeof(rxr, rxlimit); 2836 1.3.2.2 martin mutex_exit(&rxr->rxr_lock); 2837 1.3.2.2 martin 2838 1.3.2.2 martin /* Link status change */ 2839 1.3.2.2 martin // XXXX FreeBSD checks IGC_ICR_RXSEQ 2840 1.3.2.2 martin if (__predict_false(reg_icr & IGC_ICR_LSC)) { 2841 1.3.2.2 martin IGC_GLOBAL_EVENT(sc, link, 1); 2842 1.3.2.2 martin mutex_enter(&sc->sc_core_lock); 2843 1.3.2.2 martin sc->hw.mac.get_link_status = true; 2844 1.3.2.2 martin igc_update_link_status(sc); 2845 1.3.2.2 martin mutex_exit(&sc->sc_core_lock); 2846 1.3.2.2 martin } 2847 1.3.2.2 martin 2848 1.3.2.2 martin if (txmore || rxmore) { 2849 1.3.2.2 martin IGC_QUEUE_EVENT(iq, req, 1); 2850 1.3.2.2 martin igc_sched_handle_queue(sc, iq); 2851 1.3.2.2 martin } else { 2852 1.3.2.2 martin igc_enable_intr(sc); 2853 1.3.2.2 martin } 2854 1.3.2.2 martin 2855 1.3.2.2 martin return 1; 2856 1.3.2.2 martin } 2857 1.3.2.2 martin 2858 1.3.2.2 martin static void 2859 1.3.2.2 martin igc_handle_queue(void *arg) 2860 1.3.2.2 martin { 2861 1.3.2.2 martin struct igc_queue *iq = arg; 2862 1.3.2.2 martin struct igc_softc *sc = iq->sc; 2863 1.3.2.2 martin struct tx_ring *txr = iq->txr; 2864 1.3.2.2 martin struct rx_ring *rxr = iq->rxr; 2865 1.3.2.2 martin const u_int txlimit = sc->sc_tx_process_limit, 2866 1.3.2.2 martin rxlimit = sc->sc_rx_process_limit; 2867 1.3.2.2 martin bool txmore, rxmore; 2868 1.3.2.2 martin 2869 1.3.2.2 martin IGC_QUEUE_EVENT(iq, handleq, 1); 2870 1.3.2.2 martin 2871 1.3.2.2 martin mutex_enter(&txr->txr_lock); 2872 1.3.2.2 martin txmore = igc_txeof(txr, txlimit); 2873 1.3.2.2 martin /* for ALTQ, dequeue from if_snd */ 2874 1.3.2.2 martin if (txr->me == 0) { 2875 1.3.2.2 martin struct ifnet *ifp = &sc->sc_ec.ec_if; 2876 1.3.2.2 martin 2877 1.3.2.2 martin igc_tx_common_locked(ifp, txr, IGC_TX_START); 2878 1.3.2.2 martin } 2879 1.3.2.2 martin mutex_exit(&txr->txr_lock); 2880 1.3.2.2 martin 2881 1.3.2.2 martin mutex_enter(&rxr->rxr_lock); 2882 1.3.2.2 martin rxmore = igc_rxeof(rxr, rxlimit); 2883 1.3.2.2 martin mutex_exit(&rxr->rxr_lock); 2884 1.3.2.2 martin 2885 1.3.2.2 martin if (txmore || rxmore) { 2886 1.3.2.2 martin igc_sched_handle_queue(sc, iq); 2887 1.3.2.2 martin } else { 2888 1.3.2.2 martin if (sc->sc_intr_type == PCI_INTR_TYPE_MSIX) 2889 1.3.2.2 martin igc_enable_queue(sc, iq->eims); 2890 1.3.2.2 martin else 2891 1.3.2.2 martin igc_enable_intr(sc); 2892 1.3.2.2 martin } 2893 1.3.2.2 martin } 2894 1.3.2.2 martin 2895 1.3.2.2 martin static void 2896 1.3.2.2 martin igc_handle_queue_work(struct work *wk, void *context) 2897 1.3.2.2 martin { 2898 1.3.2.2 martin struct igc_queue *iq = 2899 1.3.2.2 martin container_of(wk, struct igc_queue, igcq_wq_cookie); 2900 1.3.2.2 martin 2901 1.3.2.2 martin igc_handle_queue(iq); 2902 1.3.2.2 martin } 2903 1.3.2.2 martin 2904 1.3.2.2 martin static void 2905 1.3.2.2 martin igc_sched_handle_queue(struct igc_softc *sc, struct igc_queue *iq) 2906 1.3.2.2 martin { 2907 1.3.2.2 martin 2908 1.3.2.2 martin if (iq->igcq_workqueue) { 2909 1.3.2.2 martin /* XXXRO notyet */ 2910 1.3.2.2 martin workqueue_enqueue(sc->sc_queue_wq, &iq->igcq_wq_cookie, 2911 1.3.2.2 martin curcpu()); 2912 1.3.2.2 martin } else { 2913 1.3.2.2 martin softint_schedule(iq->igcq_si); 2914 1.3.2.2 martin } 2915 1.3.2.2 martin } 2916 1.3.2.2 martin 2917 1.3.2.2 martin static void 2918 1.3.2.2 martin igc_barrier_handle_queue(struct igc_softc *sc) 2919 1.3.2.2 martin { 2920 1.3.2.2 martin 2921 1.3.2.2 martin if (sc->sc_txrx_workqueue) { 2922 1.3.2.2 martin for (int iq = 0; iq < sc->sc_nqueues; iq++) { 2923 1.3.2.2 martin struct igc_queue *q = &sc->queues[iq]; 2924 1.3.2.2 martin 2925 1.3.2.2 martin workqueue_wait(sc->sc_queue_wq, &q->igcq_wq_cookie); 2926 1.3.2.2 martin } 2927 1.3.2.2 martin } else { 2928 1.3.2.2 martin xc_barrier(0); 2929 1.3.2.2 martin } 2930 1.3.2.2 martin } 2931 1.3.2.2 martin 2932 1.3.2.2 martin /********************************************************************* 2933 1.3.2.2 martin * 2934 1.3.2.2 martin * Allocate memory for tx_buffer structures. The tx_buffer stores all 2935 1.3.2.2 martin * the information needed to transmit a packet on the wire. 2936 1.3.2.2 martin * 2937 1.3.2.2 martin **********************************************************************/ 2938 1.3.2.2 martin static int 2939 1.3.2.2 martin igc_allocate_transmit_buffers(struct tx_ring *txr) 2940 1.3.2.2 martin { 2941 1.3.2.2 martin struct igc_softc *sc = txr->sc; 2942 1.3.2.2 martin int error; 2943 1.3.2.2 martin 2944 1.3.2.2 martin txr->tx_buffers = 2945 1.3.2.2 martin kmem_zalloc(sc->num_tx_desc * sizeof(struct igc_tx_buf), KM_SLEEP); 2946 1.3.2.2 martin txr->txtag = txr->txdma.dma_tag; 2947 1.3.2.2 martin 2948 1.3.2.2 martin /* Create the descriptor buffer dma maps. */ 2949 1.3.2.2 martin for (int id = 0; id < sc->num_tx_desc; id++) { 2950 1.3.2.2 martin struct igc_tx_buf *txbuf = &txr->tx_buffers[id]; 2951 1.3.2.2 martin 2952 1.3.2.2 martin error = bus_dmamap_create(txr->txdma.dma_tag, 2953 1.3.2.2 martin round_page(IGC_TSO_SIZE + sizeof(struct ether_vlan_header)), 2954 1.3.2.2 martin IGC_MAX_SCATTER, PAGE_SIZE, 0, BUS_DMA_NOWAIT, &txbuf->map); 2955 1.3.2.2 martin if (error != 0) { 2956 1.3.2.2 martin aprint_error_dev(sc->sc_dev, 2957 1.3.2.2 martin "unable to create TX DMA map\n"); 2958 1.3.2.2 martin goto fail; 2959 1.3.2.2 martin } 2960 1.3.2.2 martin 2961 1.3.2.2 martin txbuf->eop_index = -1; 2962 1.3.2.2 martin } 2963 1.3.2.2 martin 2964 1.3.2.2 martin return 0; 2965 1.3.2.2 martin fail: 2966 1.3.2.2 martin return error; 2967 1.3.2.2 martin } 2968 1.3.2.2 martin 2969 1.3.2.2 martin 2970 1.3.2.2 martin /********************************************************************* 2971 1.3.2.2 martin * 2972 1.3.2.2 martin * Allocate and initialize transmit structures. 2973 1.3.2.2 martin * 2974 1.3.2.2 martin **********************************************************************/ 2975 1.3.2.2 martin static int 2976 1.3.2.2 martin igc_setup_transmit_structures(struct igc_softc *sc) 2977 1.3.2.2 martin { 2978 1.3.2.2 martin 2979 1.3.2.2 martin for (int iq = 0; iq < sc->sc_nqueues; iq++) { 2980 1.3.2.2 martin struct tx_ring *txr = &sc->tx_rings[iq]; 2981 1.3.2.2 martin 2982 1.3.2.2 martin if (igc_setup_transmit_ring(txr)) 2983 1.3.2.2 martin goto fail; 2984 1.3.2.2 martin } 2985 1.3.2.2 martin 2986 1.3.2.2 martin return 0; 2987 1.3.2.2 martin fail: 2988 1.3.2.2 martin igc_free_transmit_structures(sc); 2989 1.3.2.2 martin return ENOBUFS; 2990 1.3.2.2 martin } 2991 1.3.2.2 martin 2992 1.3.2.2 martin /********************************************************************* 2993 1.3.2.2 martin * 2994 1.3.2.2 martin * Initialize a transmit ring. 2995 1.3.2.2 martin * 2996 1.3.2.2 martin **********************************************************************/ 2997 1.3.2.2 martin static int 2998 1.3.2.2 martin igc_setup_transmit_ring(struct tx_ring *txr) 2999 1.3.2.2 martin { 3000 1.3.2.2 martin struct igc_softc *sc = txr->sc; 3001 1.3.2.2 martin 3002 1.3.2.2 martin /* Now allocate transmit buffers for the ring. */ 3003 1.3.2.2 martin if (igc_allocate_transmit_buffers(txr)) 3004 1.3.2.2 martin return ENOMEM; 3005 1.3.2.2 martin 3006 1.3.2.2 martin /* Clear the old ring contents */ 3007 1.3.2.2 martin memset(txr->tx_base, 0, 3008 1.3.2.2 martin sizeof(union igc_adv_tx_desc) * sc->num_tx_desc); 3009 1.3.2.2 martin 3010 1.3.2.2 martin /* Reset indices. */ 3011 1.3.2.2 martin txr->next_avail_desc = 0; 3012 1.3.2.2 martin txr->next_to_clean = 0; 3013 1.3.2.2 martin 3014 1.3.2.2 martin bus_dmamap_sync(txr->txdma.dma_tag, txr->txdma.dma_map, 0, 3015 1.3.2.2 martin txr->txdma.dma_map->dm_mapsize, 3016 1.3.2.2 martin BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); 3017 1.3.2.2 martin 3018 1.3.2.2 martin txr->txr_interq = pcq_create(sc->num_tx_desc, KM_SLEEP); 3019 1.3.2.2 martin 3020 1.3.2.2 martin mutex_init(&txr->txr_lock, MUTEX_DEFAULT, IPL_NET); 3021 1.3.2.2 martin 3022 1.3.2.2 martin return 0; 3023 1.3.2.2 martin } 3024 1.3.2.2 martin 3025 1.3.2.2 martin /********************************************************************* 3026 1.3.2.2 martin * 3027 1.3.2.2 martin * Enable transmit unit. 3028 1.3.2.2 martin * 3029 1.3.2.2 martin **********************************************************************/ 3030 1.3.2.2 martin static void 3031 1.3.2.2 martin igc_initialize_transmit_unit(struct igc_softc *sc) 3032 1.3.2.2 martin { 3033 1.3.2.2 martin struct ifnet *ifp = &sc->sc_ec.ec_if; 3034 1.3.2.2 martin struct igc_hw *hw = &sc->hw; 3035 1.3.2.2 martin 3036 1.3.2.2 martin /* Setup the Base and Length of the TX descriptor ring. */ 3037 1.3.2.2 martin for (int iq = 0; iq < sc->sc_nqueues; iq++) { 3038 1.3.2.2 martin struct tx_ring *txr = &sc->tx_rings[iq]; 3039 1.3.2.2 martin const uint64_t bus_addr = 3040 1.3.2.2 martin txr->txdma.dma_map->dm_segs[0].ds_addr; 3041 1.3.2.2 martin 3042 1.3.2.2 martin /* Base and len of TX ring */ 3043 1.3.2.2 martin IGC_WRITE_REG(hw, IGC_TDLEN(iq), 3044 1.3.2.2 martin sc->num_tx_desc * sizeof(union igc_adv_tx_desc)); 3045 1.3.2.2 martin IGC_WRITE_REG(hw, IGC_TDBAH(iq), (uint32_t)(bus_addr >> 32)); 3046 1.3.2.2 martin IGC_WRITE_REG(hw, IGC_TDBAL(iq), (uint32_t)bus_addr); 3047 1.3.2.2 martin 3048 1.3.2.2 martin /* Init the HEAD/TAIL indices */ 3049 1.3.2.2 martin IGC_WRITE_REG(hw, IGC_TDT(iq), 0 /* XXX txr->next_avail_desc */); 3050 1.3.2.2 martin IGC_WRITE_REG(hw, IGC_TDH(iq), 0); 3051 1.3.2.2 martin 3052 1.3.2.2 martin txr->watchdog_timer = 0; 3053 1.3.2.2 martin 3054 1.3.2.2 martin uint32_t txdctl = 0; /* Clear txdctl */ 3055 1.3.2.2 martin txdctl |= 0x1f; /* PTHRESH */ 3056 1.3.2.2 martin txdctl |= 1 << 8; /* HTHRESH */ 3057 1.3.2.2 martin txdctl |= 1 << 16; /* WTHRESH */ 3058 1.3.2.2 martin txdctl |= 1 << 22; /* Reserved bit 22 must always be 1 */ 3059 1.3.2.2 martin txdctl |= IGC_TXDCTL_GRAN; 3060 1.3.2.2 martin txdctl |= 1 << 25; /* LWTHRESH */ 3061 1.3.2.2 martin 3062 1.3.2.2 martin IGC_WRITE_REG(hw, IGC_TXDCTL(iq), txdctl); 3063 1.3.2.2 martin } 3064 1.3.2.2 martin ifp->if_timer = 0; 3065 1.3.2.2 martin 3066 1.3.2.2 martin /* Program the Transmit Control Register */ 3067 1.3.2.2 martin uint32_t tctl = IGC_READ_REG(&sc->hw, IGC_TCTL); 3068 1.3.2.2 martin tctl &= ~IGC_TCTL_CT; 3069 1.3.2.2 martin tctl |= (IGC_TCTL_PSP | IGC_TCTL_RTLC | IGC_TCTL_EN | 3070 1.3.2.2 martin (IGC_COLLISION_THRESHOLD << IGC_CT_SHIFT)); 3071 1.3.2.2 martin 3072 1.3.2.2 martin /* This write will effectively turn on the transmit unit. */ 3073 1.3.2.2 martin IGC_WRITE_REG(&sc->hw, IGC_TCTL, tctl); 3074 1.3.2.2 martin } 3075 1.3.2.2 martin 3076 1.3.2.2 martin /********************************************************************* 3077 1.3.2.2 martin * 3078 1.3.2.2 martin * Free all transmit rings. 3079 1.3.2.2 martin * 3080 1.3.2.2 martin **********************************************************************/ 3081 1.3.2.2 martin static void 3082 1.3.2.2 martin igc_free_transmit_structures(struct igc_softc *sc) 3083 1.3.2.2 martin { 3084 1.3.2.2 martin 3085 1.3.2.2 martin for (int iq = 0; iq < sc->sc_nqueues; iq++) { 3086 1.3.2.2 martin struct tx_ring *txr = &sc->tx_rings[iq]; 3087 1.3.2.2 martin 3088 1.3.2.2 martin igc_free_transmit_buffers(txr); 3089 1.3.2.2 martin } 3090 1.3.2.2 martin } 3091 1.3.2.2 martin 3092 1.3.2.2 martin /********************************************************************* 3093 1.3.2.2 martin * 3094 1.3.2.2 martin * Free transmit ring related data structures. 3095 1.3.2.2 martin * 3096 1.3.2.2 martin **********************************************************************/ 3097 1.3.2.2 martin static void 3098 1.3.2.2 martin igc_free_transmit_buffers(struct tx_ring *txr) 3099 1.3.2.2 martin { 3100 1.3.2.2 martin struct igc_softc *sc = txr->sc; 3101 1.3.2.2 martin 3102 1.3.2.2 martin if (txr->tx_buffers == NULL) 3103 1.3.2.2 martin return; 3104 1.3.2.2 martin 3105 1.3.2.2 martin igc_withdraw_transmit_packets(txr, true); 3106 1.3.2.2 martin 3107 1.3.2.2 martin kmem_free(txr->tx_buffers, 3108 1.3.2.2 martin sc->num_tx_desc * sizeof(struct igc_tx_buf)); 3109 1.3.2.2 martin txr->tx_buffers = NULL; 3110 1.3.2.2 martin txr->txtag = NULL; 3111 1.3.2.2 martin 3112 1.3.2.2 martin pcq_destroy(txr->txr_interq); 3113 1.3.2.2 martin mutex_destroy(&txr->txr_lock); 3114 1.3.2.2 martin } 3115 1.3.2.2 martin 3116 1.3.2.2 martin /********************************************************************* 3117 1.3.2.2 martin * 3118 1.3.2.2 martin * Withdraw transmit packets. 3119 1.3.2.2 martin * 3120 1.3.2.2 martin **********************************************************************/ 3121 1.3.2.2 martin static void 3122 1.3.2.2 martin igc_withdraw_transmit_packets(struct tx_ring *txr, bool destroy) 3123 1.3.2.2 martin { 3124 1.3.2.2 martin struct igc_softc *sc = txr->sc; 3125 1.3.2.2 martin struct igc_queue *q = txr->txr_igcq; 3126 1.3.2.2 martin 3127 1.3.2.2 martin mutex_enter(&txr->txr_lock); 3128 1.3.2.2 martin 3129 1.3.2.2 martin for (int id = 0; id < sc->num_tx_desc; id++) { 3130 1.3.2.2 martin union igc_adv_tx_desc *txdesc = &txr->tx_base[id]; 3131 1.3.2.2 martin 3132 1.3.2.2 martin igc_txdesc_sync(txr, id, 3133 1.3.2.2 martin BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE); 3134 1.3.2.2 martin txdesc->read.buffer_addr = 0; 3135 1.3.2.2 martin txdesc->read.cmd_type_len = 0; 3136 1.3.2.2 martin txdesc->read.olinfo_status = 0; 3137 1.3.2.2 martin igc_txdesc_sync(txr, id, 3138 1.3.2.2 martin BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); 3139 1.3.2.2 martin 3140 1.3.2.2 martin struct igc_tx_buf *txbuf = &txr->tx_buffers[id]; 3141 1.3.2.2 martin bus_dmamap_t map = txbuf->map; 3142 1.3.2.2 martin 3143 1.3.2.2 martin if (map != NULL && map->dm_nsegs > 0) { 3144 1.3.2.2 martin bus_dmamap_sync(txr->txdma.dma_tag, map, 3145 1.3.2.2 martin 0, map->dm_mapsize, BUS_DMASYNC_POSTWRITE); 3146 1.3.2.2 martin bus_dmamap_unload(txr->txdma.dma_tag, map); 3147 1.3.2.2 martin } 3148 1.3.2.2 martin if (txbuf->m_head != NULL) { 3149 1.3.2.2 martin m_freem(txbuf->m_head); 3150 1.3.2.2 martin txbuf->m_head = NULL; 3151 1.3.2.2 martin } 3152 1.3.2.2 martin if (map != NULL && destroy) { 3153 1.3.2.2 martin bus_dmamap_destroy(txr->txdma.dma_tag, map); 3154 1.3.2.2 martin txbuf->map = NULL; 3155 1.3.2.2 martin } 3156 1.3.2.2 martin txbuf->eop_index = -1; 3157 1.3.2.2 martin 3158 1.3.2.2 martin txr->next_avail_desc = 0; 3159 1.3.2.2 martin txr->next_to_clean = 0; 3160 1.3.2.2 martin } 3161 1.3.2.2 martin 3162 1.3.2.2 martin struct mbuf *m; 3163 1.3.2.2 martin while ((m = pcq_get(txr->txr_interq)) != NULL) { 3164 1.3.2.2 martin IGC_QUEUE_EVENT(q, tx_pcq_drop, 1); 3165 1.3.2.2 martin m_freem(m); 3166 1.3.2.2 martin } 3167 1.3.2.2 martin 3168 1.3.2.2 martin mutex_exit(&txr->txr_lock); 3169 1.3.2.2 martin } 3170 1.3.2.2 martin 3171 1.3.2.2 martin 3172 1.3.2.2 martin /********************************************************************* 3173 1.3.2.2 martin * 3174 1.3.2.2 martin * Advanced Context Descriptor setup for VLAN, CSUM or TSO 3175 1.3.2.2 martin * 3176 1.3.2.2 martin **********************************************************************/ 3177 1.3.2.2 martin 3178 1.3.2.7 martin static bool 3179 1.3.2.2 martin igc_tx_ctx_setup(struct tx_ring *txr, struct mbuf *mp, int prod, 3180 1.3.2.2 martin uint32_t *cmd_type_len, uint32_t *olinfo_status) 3181 1.3.2.2 martin { 3182 1.3.2.2 martin struct ether_vlan_header *evl; 3183 1.3.2.7 martin struct tcphdr *th = NULL /* XXXGCC */; 3184 1.3.2.2 martin uint32_t vlan_macip_lens = 0; 3185 1.3.2.7 martin uint32_t type_tucmd_mlhl = 0; 3186 1.3.2.7 martin uint32_t mss_l4len_idx = 0; 3187 1.3.2.2 martin uint32_t ehlen, iphlen; 3188 1.3.2.2 martin uint16_t ehtype; 3189 1.3.2.2 martin 3190 1.3.2.2 martin const int csum_flags = mp->m_pkthdr.csum_flags; 3191 1.3.2.2 martin const bool v4 = (csum_flags & 3192 1.3.2.7 martin (M_CSUM_IPv4 | M_CSUM_TCPv4 | M_CSUM_UDPv4 | M_CSUM_TSOv4)) != 0; 3193 1.3.2.7 martin const bool v6 = (csum_flags & 3194 1.3.2.7 martin (M_CSUM_UDPv6 | M_CSUM_TCPv6 | M_CSUM_TSOv6)) != 0; 3195 1.3.2.7 martin const bool tso = (csum_flags & (M_CSUM_TSOv4 | M_CSUM_TSOv6)) != 0; 3196 1.3.2.7 martin const bool tcp = tso || 3197 1.3.2.7 martin (csum_flags & (M_CSUM_TCPv4 | M_CSUM_TCPv6)) != 0; 3198 1.3.2.7 martin const bool udp = (csum_flags & (M_CSUM_UDPv4 | M_CSUM_UDPv6)) != 0; 3199 1.3.2.2 martin 3200 1.3.2.2 martin /* Indicate the whole packet as payload when not doing TSO */ 3201 1.3.2.7 martin if (!tso) { 3202 1.3.2.7 martin *olinfo_status |= mp->m_pkthdr.len << IGC_ADVTXD_PAYLEN_SHIFT; 3203 1.3.2.7 martin } else { 3204 1.3.2.7 martin /* Set L4 payload length later... */ 3205 1.3.2.7 martin } 3206 1.3.2.2 martin 3207 1.3.2.7 martin #if NVLAN > 0 3208 1.3.2.2 martin /* 3209 1.3.2.2 martin * In advanced descriptors the vlan tag must 3210 1.3.2.2 martin * be placed into the context descriptor. Hence 3211 1.3.2.2 martin * we need to make one even if not doing offloads. 3212 1.3.2.2 martin */ 3213 1.3.2.2 martin if (vlan_has_tag(mp)) { 3214 1.3.2.2 martin vlan_macip_lens |= (uint32_t)vlan_get_tag(mp) 3215 1.3.2.2 martin << IGC_ADVTXD_VLAN_SHIFT; 3216 1.3.2.2 martin } else 3217 1.3.2.2 martin #endif 3218 1.3.2.2 martin if (!v4 && !v6) 3219 1.3.2.7 martin return false; 3220 1.3.2.2 martin 3221 1.3.2.2 martin KASSERT(mp->m_len >= sizeof(struct ether_header)); 3222 1.3.2.2 martin evl = mtod(mp, struct ether_vlan_header *); 3223 1.3.2.2 martin if (evl->evl_encap_proto == htons(ETHERTYPE_VLAN)) { 3224 1.3.2.2 martin KASSERT(mp->m_len >= sizeof(struct ether_vlan_header)); 3225 1.3.2.2 martin ehlen = ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN; 3226 1.3.2.2 martin ehtype = evl->evl_proto; 3227 1.3.2.2 martin } else { 3228 1.3.2.2 martin ehlen = ETHER_HDR_LEN; 3229 1.3.2.2 martin ehtype = evl->evl_encap_proto; 3230 1.3.2.2 martin } 3231 1.3.2.2 martin 3232 1.3.2.2 martin switch (ntohs(ehtype)) { 3233 1.3.2.2 martin case ETHERTYPE_IP: 3234 1.3.2.2 martin iphlen = M_CSUM_DATA_IPv4_IPHL(mp->m_pkthdr.csum_data); 3235 1.3.2.2 martin type_tucmd_mlhl |= IGC_ADVTXD_TUCMD_IPV4; 3236 1.3.2.2 martin 3237 1.3.2.7 martin if ((csum_flags & (M_CSUM_IPv4 | M_CSUM_TSOv4)) != 0) 3238 1.3.2.2 martin *olinfo_status |= IGC_TXD_POPTS_IXSM << 8; 3239 1.3.2.7 martin 3240 1.3.2.7 martin if (!tso) 3241 1.3.2.7 martin break; 3242 1.3.2.7 martin 3243 1.3.2.7 martin struct ip *ip; 3244 1.3.2.7 martin KASSERT(mp->m_len >= ehlen + sizeof(*ip)); 3245 1.3.2.7 martin ip = (void *)(mtod(mp, char *) + ehlen); 3246 1.3.2.7 martin ip->ip_len = 0; 3247 1.3.2.7 martin 3248 1.3.2.7 martin KASSERT(mp->m_len >= ehlen + iphlen + sizeof(*th)); 3249 1.3.2.7 martin th = (void *)((char *)ip + iphlen); 3250 1.3.2.7 martin th->th_sum = in_cksum_phdr(ip->ip_src.s_addr, 3251 1.3.2.7 martin ip->ip_dst.s_addr, htons(IPPROTO_TCP)); 3252 1.3.2.2 martin break; 3253 1.3.2.2 martin case ETHERTYPE_IPV6: 3254 1.3.2.2 martin iphlen = M_CSUM_DATA_IPv6_IPHL(mp->m_pkthdr.csum_data); 3255 1.3.2.2 martin type_tucmd_mlhl |= IGC_ADVTXD_TUCMD_IPV6; 3256 1.3.2.7 martin 3257 1.3.2.7 martin if (!tso) 3258 1.3.2.7 martin break; 3259 1.3.2.7 martin 3260 1.3.2.7 martin struct ip6_hdr *ip6; 3261 1.3.2.7 martin KASSERT(mp->m_len >= ehlen + sizeof(*ip6)); 3262 1.3.2.7 martin ip6 = (void *)(mtod(mp, char *) + ehlen); 3263 1.3.2.7 martin ip6->ip6_plen = 0; 3264 1.3.2.7 martin 3265 1.3.2.7 martin KASSERT(mp->m_len >= ehlen + iphlen + sizeof(*th)); 3266 1.3.2.7 martin th = (void *)((char *)ip6 + iphlen); 3267 1.3.2.7 martin th->th_sum = in6_cksum_phdr(&ip6->ip6_src, &ip6->ip6_dst, 0, 3268 1.3.2.7 martin htonl(IPPROTO_TCP)); 3269 1.3.2.2 martin break; 3270 1.3.2.2 martin default: 3271 1.3.2.2 martin /* 3272 1.3.2.2 martin * Unknown L3 protocol. Clear L3 header length and proceed for 3273 1.3.2.2 martin * LAN as done by Linux driver. 3274 1.3.2.2 martin */ 3275 1.3.2.2 martin KASSERT(!v4 && !v6); 3276 1.3.2.7 martin iphlen = 0; 3277 1.3.2.2 martin break; 3278 1.3.2.2 martin } 3279 1.3.2.2 martin 3280 1.3.2.2 martin if (tcp) { 3281 1.3.2.2 martin type_tucmd_mlhl |= IGC_ADVTXD_TUCMD_L4T_TCP; 3282 1.3.2.2 martin *olinfo_status |= IGC_TXD_POPTS_TXSM << 8; 3283 1.3.2.2 martin } else if (udp) { 3284 1.3.2.2 martin type_tucmd_mlhl |= IGC_ADVTXD_TUCMD_L4T_UDP; 3285 1.3.2.2 martin *olinfo_status |= IGC_TXD_POPTS_TXSM << 8; 3286 1.3.2.2 martin } 3287 1.3.2.2 martin 3288 1.3.2.7 martin if (tso) { 3289 1.3.2.7 martin const uint32_t tcphlen = th->th_off << 2; 3290 1.3.2.7 martin const uint32_t paylen = 3291 1.3.2.7 martin mp->m_pkthdr.len - ehlen - iphlen - tcphlen; 3292 1.3.2.2 martin 3293 1.3.2.7 martin mss_l4len_idx |= mp->m_pkthdr.segsz << IGC_ADVTXD_MSS_SHIFT; 3294 1.3.2.7 martin mss_l4len_idx |= tcphlen << IGC_ADVTXD_L4LEN_SHIFT; 3295 1.3.2.2 martin 3296 1.3.2.7 martin *cmd_type_len |= IGC_ADVTXD_DCMD_TSE; 3297 1.3.2.2 martin 3298 1.3.2.7 martin *olinfo_status |= paylen << IGC_ADVTXD_PAYLEN_SHIFT; 3299 1.3.2.2 martin } 3300 1.3.2.2 martin 3301 1.3.2.2 martin vlan_macip_lens |= iphlen; 3302 1.3.2.7 martin vlan_macip_lens |= ehlen << IGC_ADVTXD_MACLEN_SHIFT; 3303 1.3.2.2 martin 3304 1.3.2.2 martin type_tucmd_mlhl |= IGC_ADVTXD_DCMD_DEXT | IGC_ADVTXD_DTYP_CTXT; 3305 1.3.2.2 martin 3306 1.3.2.2 martin /* Now ready a context descriptor */ 3307 1.3.2.2 martin struct igc_adv_tx_context_desc *txdesc = 3308 1.3.2.2 martin (struct igc_adv_tx_context_desc *)&txr->tx_base[prod]; 3309 1.3.2.2 martin 3310 1.3.2.2 martin igc_txdesc_sync(txr, prod, 3311 1.3.2.2 martin BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE); 3312 1.3.2.7 martin 3313 1.3.2.7 martin /* Now copy bits into descriptor */ 3314 1.3.2.2 martin htolem32(&txdesc->vlan_macip_lens, vlan_macip_lens); 3315 1.3.2.2 martin htolem32(&txdesc->type_tucmd_mlhl, type_tucmd_mlhl); 3316 1.3.2.2 martin htolem32(&txdesc->seqnum_seed, 0); 3317 1.3.2.2 martin htolem32(&txdesc->mss_l4len_idx, mss_l4len_idx); 3318 1.3.2.7 martin 3319 1.3.2.2 martin igc_txdesc_sync(txr, prod, 3320 1.3.2.2 martin BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); 3321 1.3.2.2 martin 3322 1.3.2.2 martin return 1; 3323 1.3.2.2 martin } 3324 1.3.2.2 martin 3325 1.3.2.2 martin /********************************************************************* 3326 1.3.2.2 martin * 3327 1.3.2.2 martin * Allocate memory for rx_buffer structures. Since we use one 3328 1.3.2.2 martin * rx_buffer per received packet, the maximum number of rx_buffer's 3329 1.3.2.2 martin * that we'll need is equal to the number of receive descriptors 3330 1.3.2.2 martin * that we've allocated. 3331 1.3.2.2 martin * 3332 1.3.2.2 martin **********************************************************************/ 3333 1.3.2.2 martin static int 3334 1.3.2.2 martin igc_allocate_receive_buffers(struct rx_ring *rxr) 3335 1.3.2.2 martin { 3336 1.3.2.2 martin struct igc_softc *sc = rxr->sc; 3337 1.3.2.2 martin int error; 3338 1.3.2.2 martin 3339 1.3.2.2 martin rxr->rx_buffers = 3340 1.3.2.2 martin kmem_zalloc(sc->num_rx_desc * sizeof(struct igc_rx_buf), KM_SLEEP); 3341 1.3.2.2 martin 3342 1.3.2.2 martin for (int id = 0; id < sc->num_rx_desc; id++) { 3343 1.3.2.2 martin struct igc_rx_buf *rxbuf = &rxr->rx_buffers[id]; 3344 1.3.2.2 martin 3345 1.3.2.2 martin error = bus_dmamap_create(rxr->rxdma.dma_tag, MCLBYTES, 1, 3346 1.3.2.2 martin MCLBYTES, 0, BUS_DMA_WAITOK, &rxbuf->map); 3347 1.3.2.2 martin if (error) { 3348 1.3.2.2 martin aprint_error_dev(sc->sc_dev, 3349 1.3.2.2 martin "unable to create RX DMA map\n"); 3350 1.3.2.2 martin goto fail; 3351 1.3.2.2 martin } 3352 1.3.2.2 martin } 3353 1.3.2.2 martin bus_dmamap_sync(rxr->rxdma.dma_tag, rxr->rxdma.dma_map, 0, 3354 1.3.2.2 martin rxr->rxdma.dma_map->dm_mapsize, 3355 1.3.2.2 martin BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); 3356 1.3.2.2 martin 3357 1.3.2.2 martin return 0; 3358 1.3.2.2 martin fail: 3359 1.3.2.2 martin return error; 3360 1.3.2.2 martin } 3361 1.3.2.2 martin 3362 1.3.2.2 martin /********************************************************************* 3363 1.3.2.2 martin * 3364 1.3.2.2 martin * Allocate and initialize receive structures. 3365 1.3.2.2 martin * 3366 1.3.2.2 martin **********************************************************************/ 3367 1.3.2.2 martin static int 3368 1.3.2.2 martin igc_setup_receive_structures(struct igc_softc *sc) 3369 1.3.2.2 martin { 3370 1.3.2.2 martin 3371 1.3.2.2 martin for (int iq = 0; iq < sc->sc_nqueues; iq++) { 3372 1.3.2.2 martin struct rx_ring *rxr = &sc->rx_rings[iq]; 3373 1.3.2.2 martin 3374 1.3.2.2 martin if (igc_setup_receive_ring(rxr)) 3375 1.3.2.2 martin goto fail; 3376 1.3.2.2 martin } 3377 1.3.2.2 martin 3378 1.3.2.2 martin return 0; 3379 1.3.2.2 martin fail: 3380 1.3.2.2 martin igc_free_receive_structures(sc); 3381 1.3.2.2 martin return ENOBUFS; 3382 1.3.2.2 martin } 3383 1.3.2.2 martin 3384 1.3.2.2 martin /********************************************************************* 3385 1.3.2.2 martin * 3386 1.3.2.2 martin * Initialize a receive ring and its buffers. 3387 1.3.2.2 martin * 3388 1.3.2.2 martin **********************************************************************/ 3389 1.3.2.2 martin static int 3390 1.3.2.2 martin igc_setup_receive_ring(struct rx_ring *rxr) 3391 1.3.2.2 martin { 3392 1.3.2.2 martin struct igc_softc *sc = rxr->sc; 3393 1.3.2.2 martin const int rsize = roundup2( 3394 1.3.2.2 martin sc->num_rx_desc * sizeof(union igc_adv_rx_desc), IGC_DBA_ALIGN); 3395 1.3.2.2 martin 3396 1.3.2.2 martin /* Clear the ring contents. */ 3397 1.3.2.2 martin memset(rxr->rx_base, 0, rsize); 3398 1.3.2.2 martin 3399 1.3.2.2 martin if (igc_allocate_receive_buffers(rxr)) 3400 1.3.2.2 martin return ENOMEM; 3401 1.3.2.2 martin 3402 1.3.2.2 martin /* Setup our descriptor indices. */ 3403 1.3.2.2 martin rxr->next_to_check = 0; 3404 1.3.2.2 martin rxr->last_desc_filled = 0; 3405 1.3.2.2 martin 3406 1.3.2.2 martin mutex_init(&rxr->rxr_lock, MUTEX_DEFAULT, IPL_NET); 3407 1.3.2.2 martin 3408 1.3.2.2 martin return 0; 3409 1.3.2.2 martin } 3410 1.3.2.2 martin 3411 1.3.2.2 martin /********************************************************************* 3412 1.3.2.2 martin * 3413 1.3.2.2 martin * Enable receive unit. 3414 1.3.2.2 martin * 3415 1.3.2.2 martin **********************************************************************/ 3416 1.3.2.2 martin static void 3417 1.3.2.2 martin igc_initialize_receive_unit(struct igc_softc *sc) 3418 1.3.2.2 martin { 3419 1.3.2.2 martin struct ifnet *ifp = &sc->sc_ec.ec_if; 3420 1.3.2.2 martin struct igc_hw *hw = &sc->hw; 3421 1.3.2.2 martin uint32_t rctl, rxcsum, srrctl; 3422 1.3.2.2 martin 3423 1.3.2.2 martin DPRINTF(RX, "called\n"); 3424 1.3.2.2 martin 3425 1.3.2.2 martin /* 3426 1.3.2.2 martin * Make sure receives are disabled while setting 3427 1.3.2.2 martin * up the descriptor ring. 3428 1.3.2.2 martin */ 3429 1.3.2.2 martin rctl = IGC_READ_REG(hw, IGC_RCTL); 3430 1.3.2.2 martin IGC_WRITE_REG(hw, IGC_RCTL, rctl & ~IGC_RCTL_EN); 3431 1.3.2.2 martin 3432 1.3.2.2 martin /* Setup the Receive Control Register */ 3433 1.3.2.2 martin rctl &= ~(3 << IGC_RCTL_MO_SHIFT); 3434 1.3.2.2 martin rctl |= IGC_RCTL_EN | IGC_RCTL_BAM | IGC_RCTL_LBM_NO | 3435 1.3.2.2 martin IGC_RCTL_RDMTS_HALF | (hw->mac.mc_filter_type << IGC_RCTL_MO_SHIFT); 3436 1.3.2.2 martin 3437 1.3.2.2 martin #if 1 3438 1.3.2.2 martin /* Do not store bad packets */ 3439 1.3.2.2 martin rctl &= ~IGC_RCTL_SBP; 3440 1.3.2.2 martin #else 3441 1.3.2.2 martin /* for debug */ 3442 1.3.2.2 martin rctl |= IGC_RCTL_SBP; 3443 1.3.2.2 martin #endif 3444 1.3.2.2 martin 3445 1.3.2.2 martin /* Enable Long Packet receive */ 3446 1.3.2.2 martin if (sc->hw.mac.max_frame_size > ETHER_MAX_LEN) 3447 1.3.2.2 martin rctl |= IGC_RCTL_LPE; 3448 1.3.2.2 martin else 3449 1.3.2.2 martin rctl &= ~IGC_RCTL_LPE; 3450 1.3.2.2 martin 3451 1.3.2.2 martin /* Strip the CRC */ 3452 1.3.2.2 martin rctl |= IGC_RCTL_SECRC; 3453 1.3.2.2 martin 3454 1.3.2.2 martin /* 3455 1.3.2.2 martin * Set the interrupt throttling rate. Value is calculated 3456 1.3.2.2 martin * as DEFAULT_ITR = 1/(MAX_INTS_PER_SEC * 256ns) 3457 1.3.2.2 martin * 3458 1.3.2.2 martin * XXX Sync with Linux, especially for jumbo MTU or TSO. 3459 1.3.2.2 martin * XXX Shouldn't be here? 3460 1.3.2.2 martin */ 3461 1.3.2.2 martin IGC_WRITE_REG(hw, IGC_ITR, DEFAULT_ITR); 3462 1.3.2.2 martin 3463 1.3.2.2 martin rxcsum = IGC_READ_REG(hw, IGC_RXCSUM); 3464 1.3.2.2 martin rxcsum &= ~(IGC_RXCSUM_IPOFL | IGC_RXCSUM_TUOFL | IGC_RXCSUM_PCSD); 3465 1.3.2.2 martin if (ifp->if_capenable & IFCAP_CSUM_IPv4_Rx) 3466 1.3.2.2 martin rxcsum |= IGC_RXCSUM_IPOFL; 3467 1.3.2.2 martin if (ifp->if_capenable & (IFCAP_CSUM_TCPv4_Rx | IFCAP_CSUM_UDPv4_Rx | 3468 1.3.2.2 martin IFCAP_CSUM_TCPv6_Rx | IFCAP_CSUM_UDPv6_Rx)) 3469 1.3.2.2 martin rxcsum |= IGC_RXCSUM_TUOFL; 3470 1.3.2.2 martin if (sc->sc_nqueues > 1) 3471 1.3.2.2 martin rxcsum |= IGC_RXCSUM_PCSD; 3472 1.3.2.2 martin IGC_WRITE_REG(hw, IGC_RXCSUM, rxcsum); 3473 1.3.2.2 martin 3474 1.3.2.2 martin if (sc->sc_nqueues > 1) 3475 1.3.2.2 martin igc_initialize_rss_mapping(sc); 3476 1.3.2.2 martin 3477 1.3.2.2 martin srrctl = 0; 3478 1.3.2.2 martin #if 0 3479 1.3.2.2 martin srrctl |= 4096 >> IGC_SRRCTL_BSIZEPKT_SHIFT; 3480 1.3.2.2 martin rctl |= IGC_RCTL_SZ_4096 | IGC_RCTL_BSEX; 3481 1.3.2.2 martin #else 3482 1.3.2.2 martin srrctl |= 2048 >> IGC_SRRCTL_BSIZEPKT_SHIFT; 3483 1.3.2.2 martin rctl |= IGC_RCTL_SZ_2048; 3484 1.3.2.2 martin #endif 3485 1.3.2.2 martin 3486 1.3.2.2 martin /* 3487 1.3.2.2 martin * If TX flow control is disabled and there's > 1 queue defined, 3488 1.3.2.2 martin * enable DROP. 3489 1.3.2.2 martin * 3490 1.3.2.2 martin * This drops frames rather than hanging the RX MAC for all queues. 3491 1.3.2.2 martin */ 3492 1.3.2.2 martin if (sc->sc_nqueues > 1 && 3493 1.3.2.2 martin (sc->fc == igc_fc_none || sc->fc == igc_fc_rx_pause)) 3494 1.3.2.2 martin srrctl |= IGC_SRRCTL_DROP_EN; 3495 1.3.2.2 martin 3496 1.3.2.2 martin /* Setup the Base and Length of the RX descriptor rings. */ 3497 1.3.2.2 martin for (int iq = 0; iq < sc->sc_nqueues; iq++) { 3498 1.3.2.2 martin struct rx_ring *rxr = &sc->rx_rings[iq]; 3499 1.3.2.2 martin const uint64_t bus_addr = 3500 1.3.2.2 martin rxr->rxdma.dma_map->dm_segs[0].ds_addr; 3501 1.3.2.2 martin 3502 1.3.2.2 martin IGC_WRITE_REG(hw, IGC_RXDCTL(iq), 0); 3503 1.3.2.2 martin 3504 1.3.2.2 martin srrctl |= IGC_SRRCTL_DESCTYPE_ADV_ONEBUF; 3505 1.3.2.2 martin 3506 1.3.2.2 martin IGC_WRITE_REG(hw, IGC_RDLEN(iq), 3507 1.3.2.2 martin sc->num_rx_desc * sizeof(union igc_adv_rx_desc)); 3508 1.3.2.2 martin IGC_WRITE_REG(hw, IGC_RDBAH(iq), (uint32_t)(bus_addr >> 32)); 3509 1.3.2.2 martin IGC_WRITE_REG(hw, IGC_RDBAL(iq), (uint32_t)bus_addr); 3510 1.3.2.2 martin IGC_WRITE_REG(hw, IGC_SRRCTL(iq), srrctl); 3511 1.3.2.2 martin 3512 1.3.2.2 martin /* Setup the Head and Tail Descriptor Pointers */ 3513 1.3.2.2 martin IGC_WRITE_REG(hw, IGC_RDH(iq), 0); 3514 1.3.2.2 martin IGC_WRITE_REG(hw, IGC_RDT(iq), 0 /* XXX rxr->last_desc_filled */); 3515 1.3.2.2 martin 3516 1.3.2.2 martin /* Enable this Queue */ 3517 1.3.2.2 martin uint32_t rxdctl = IGC_READ_REG(hw, IGC_RXDCTL(iq)); 3518 1.3.2.2 martin rxdctl |= IGC_RXDCTL_QUEUE_ENABLE; 3519 1.3.2.2 martin rxdctl &= 0xFFF00000; 3520 1.3.2.2 martin rxdctl |= IGC_RX_PTHRESH; 3521 1.3.2.2 martin rxdctl |= IGC_RX_HTHRESH << 8; 3522 1.3.2.2 martin rxdctl |= IGC_RX_WTHRESH << 16; 3523 1.3.2.2 martin IGC_WRITE_REG(hw, IGC_RXDCTL(iq), rxdctl); 3524 1.3.2.2 martin } 3525 1.3.2.2 martin 3526 1.3.2.2 martin /* Make sure VLAN Filters are off */ 3527 1.3.2.2 martin rctl &= ~IGC_RCTL_VFE; 3528 1.3.2.2 martin 3529 1.3.2.2 martin /* Write out the settings */ 3530 1.3.2.2 martin IGC_WRITE_REG(hw, IGC_RCTL, rctl); 3531 1.3.2.2 martin } 3532 1.3.2.2 martin 3533 1.3.2.2 martin /********************************************************************* 3534 1.3.2.2 martin * 3535 1.3.2.2 martin * Free all receive rings. 3536 1.3.2.2 martin * 3537 1.3.2.2 martin **********************************************************************/ 3538 1.3.2.2 martin static void 3539 1.3.2.2 martin igc_free_receive_structures(struct igc_softc *sc) 3540 1.3.2.2 martin { 3541 1.3.2.2 martin 3542 1.3.2.2 martin for (int iq = 0; iq < sc->sc_nqueues; iq++) { 3543 1.3.2.2 martin struct rx_ring *rxr = &sc->rx_rings[iq]; 3544 1.3.2.2 martin 3545 1.3.2.2 martin igc_free_receive_buffers(rxr); 3546 1.3.2.2 martin } 3547 1.3.2.2 martin } 3548 1.3.2.2 martin 3549 1.3.2.2 martin /********************************************************************* 3550 1.3.2.2 martin * 3551 1.3.2.2 martin * Free receive ring data structures 3552 1.3.2.2 martin * 3553 1.3.2.2 martin **********************************************************************/ 3554 1.3.2.2 martin static void 3555 1.3.2.2 martin igc_free_receive_buffers(struct rx_ring *rxr) 3556 1.3.2.2 martin { 3557 1.3.2.2 martin struct igc_softc *sc = rxr->sc; 3558 1.3.2.2 martin 3559 1.3.2.2 martin if (rxr->rx_buffers != NULL) { 3560 1.3.2.2 martin for (int id = 0; id < sc->num_rx_desc; id++) { 3561 1.3.2.2 martin struct igc_rx_buf *rxbuf = &rxr->rx_buffers[id]; 3562 1.3.2.2 martin bus_dmamap_t map = rxbuf->map; 3563 1.3.2.2 martin 3564 1.3.2.2 martin if (rxbuf->buf != NULL) { 3565 1.3.2.2 martin bus_dmamap_sync(rxr->rxdma.dma_tag, map, 3566 1.3.2.2 martin 0, map->dm_mapsize, BUS_DMASYNC_POSTREAD); 3567 1.3.2.2 martin bus_dmamap_unload(rxr->rxdma.dma_tag, map); 3568 1.3.2.2 martin m_freem(rxbuf->buf); 3569 1.3.2.2 martin rxbuf->buf = NULL; 3570 1.3.2.2 martin } 3571 1.3.2.2 martin bus_dmamap_destroy(rxr->rxdma.dma_tag, map); 3572 1.3.2.2 martin rxbuf->map = NULL; 3573 1.3.2.2 martin } 3574 1.3.2.2 martin kmem_free(rxr->rx_buffers, 3575 1.3.2.2 martin sc->num_rx_desc * sizeof(struct igc_rx_buf)); 3576 1.3.2.2 martin rxr->rx_buffers = NULL; 3577 1.3.2.2 martin } 3578 1.3.2.2 martin 3579 1.3.2.2 martin mutex_destroy(&rxr->rxr_lock); 3580 1.3.2.2 martin } 3581 1.3.2.2 martin 3582 1.3.2.2 martin /********************************************************************* 3583 1.3.2.2 martin * 3584 1.3.2.2 martin * Clear status registers in all RX descriptors. 3585 1.3.2.2 martin * 3586 1.3.2.2 martin **********************************************************************/ 3587 1.3.2.2 martin static void 3588 1.3.2.2 martin igc_clear_receive_status(struct rx_ring *rxr) 3589 1.3.2.2 martin { 3590 1.3.2.2 martin struct igc_softc *sc = rxr->sc; 3591 1.3.2.2 martin 3592 1.3.2.2 martin mutex_enter(&rxr->rxr_lock); 3593 1.3.2.2 martin 3594 1.3.2.2 martin for (int id = 0; id < sc->num_rx_desc; id++) { 3595 1.3.2.2 martin union igc_adv_rx_desc *rxdesc = &rxr->rx_base[id]; 3596 1.3.2.2 martin 3597 1.3.2.2 martin igc_rxdesc_sync(rxr, id, 3598 1.3.2.2 martin BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE); 3599 1.3.2.2 martin rxdesc->wb.upper.status_error = 0; 3600 1.3.2.2 martin igc_rxdesc_sync(rxr, id, 3601 1.3.2.2 martin BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); 3602 1.3.2.2 martin } 3603 1.3.2.2 martin 3604 1.3.2.2 martin mutex_exit(&rxr->rxr_lock); 3605 1.3.2.2 martin } 3606 1.3.2.2 martin 3607 1.3.2.2 martin /* 3608 1.3.2.2 martin * Initialise the RSS mapping for NICs that support multiple transmit/ 3609 1.3.2.2 martin * receive rings. 3610 1.3.2.2 martin */ 3611 1.3.2.2 martin static void 3612 1.3.2.2 martin igc_initialize_rss_mapping(struct igc_softc *sc) 3613 1.3.2.2 martin { 3614 1.3.2.2 martin struct igc_hw *hw = &sc->hw; 3615 1.3.2.2 martin 3616 1.3.2.2 martin /* 3617 1.3.2.2 martin * The redirection table controls which destination 3618 1.3.2.2 martin * queue each bucket redirects traffic to. 3619 1.3.2.2 martin * Each DWORD represents four queues, with the LSB 3620 1.3.2.2 martin * being the first queue in the DWORD. 3621 1.3.2.2 martin * 3622 1.3.2.2 martin * This just allocates buckets to queues using round-robin 3623 1.3.2.2 martin * allocation. 3624 1.3.2.2 martin * 3625 1.3.2.2 martin * NOTE: It Just Happens to line up with the default 3626 1.3.2.2 martin * RSS allocation method. 3627 1.3.2.2 martin */ 3628 1.3.2.2 martin 3629 1.3.2.2 martin /* Warning FM follows */ 3630 1.3.2.2 martin uint32_t reta = 0; 3631 1.3.2.2 martin for (int i = 0; i < 128; i++) { 3632 1.3.2.2 martin const int shift = 0; /* XXXRO */ 3633 1.3.2.2 martin int queue_id = i % sc->sc_nqueues; 3634 1.3.2.2 martin /* Adjust if required */ 3635 1.3.2.2 martin queue_id <<= shift; 3636 1.3.2.2 martin 3637 1.3.2.2 martin /* 3638 1.3.2.2 martin * The low 8 bits are for hash value (n+0); 3639 1.3.2.2 martin * The next 8 bits are for hash value (n+1), etc. 3640 1.3.2.2 martin */ 3641 1.3.2.2 martin reta >>= 8; 3642 1.3.2.2 martin reta |= ((uint32_t)queue_id) << 24; 3643 1.3.2.2 martin if ((i & 3) == 3) { 3644 1.3.2.2 martin IGC_WRITE_REG(hw, IGC_RETA(i >> 2), reta); 3645 1.3.2.2 martin reta = 0; 3646 1.3.2.2 martin } 3647 1.3.2.2 martin } 3648 1.3.2.2 martin 3649 1.3.2.2 martin /* 3650 1.3.2.2 martin * MRQC: Multiple Receive Queues Command 3651 1.3.2.2 martin * Set queuing to RSS control, number depends on the device. 3652 1.3.2.2 martin */ 3653 1.3.2.2 martin 3654 1.3.2.2 martin /* Set up random bits */ 3655 1.3.2.2 martin uint32_t rss_key[RSS_KEYSIZE / sizeof(uint32_t)]; 3656 1.3.2.2 martin rss_getkey((uint8_t *)rss_key); 3657 1.3.2.2 martin 3658 1.3.2.2 martin /* Now fill our hash function seeds */ 3659 1.3.2.2 martin for (int i = 0; i < __arraycount(rss_key); i++) 3660 1.3.2.2 martin IGC_WRITE_REG_ARRAY(hw, IGC_RSSRK(0), i, rss_key[i]); 3661 1.3.2.2 martin 3662 1.3.2.2 martin /* 3663 1.3.2.2 martin * Configure the RSS fields to hash upon. 3664 1.3.2.2 martin */ 3665 1.3.2.2 martin uint32_t mrqc = IGC_MRQC_ENABLE_RSS_4Q; 3666 1.3.2.2 martin mrqc |= IGC_MRQC_RSS_FIELD_IPV4 | IGC_MRQC_RSS_FIELD_IPV4_TCP; 3667 1.3.2.2 martin mrqc |= IGC_MRQC_RSS_FIELD_IPV6 | IGC_MRQC_RSS_FIELD_IPV6_TCP; 3668 1.3.2.2 martin mrqc |= IGC_MRQC_RSS_FIELD_IPV6_TCP_EX; 3669 1.3.2.2 martin 3670 1.3.2.2 martin IGC_WRITE_REG(hw, IGC_MRQC, mrqc); 3671 1.3.2.2 martin } 3672 1.3.2.2 martin 3673 1.3.2.2 martin /* 3674 1.3.2.2 martin * igc_get_hw_control sets the {CTRL_EXT|FWSM}:DRV_LOAD bit. 3675 1.3.2.2 martin * For ASF and Pass Through versions of f/w this means 3676 1.3.2.2 martin * that the driver is loaded. For AMT version type f/w 3677 1.3.2.2 martin * this means that the network i/f is open. 3678 1.3.2.2 martin */ 3679 1.3.2.2 martin static void 3680 1.3.2.2 martin igc_get_hw_control(struct igc_softc *sc) 3681 1.3.2.2 martin { 3682 1.3.2.2 martin const uint32_t ctrl_ext = IGC_READ_REG(&sc->hw, IGC_CTRL_EXT); 3683 1.3.2.2 martin 3684 1.3.2.2 martin IGC_WRITE_REG(&sc->hw, IGC_CTRL_EXT, ctrl_ext | IGC_CTRL_EXT_DRV_LOAD); 3685 1.3.2.2 martin } 3686 1.3.2.2 martin 3687 1.3.2.2 martin /* 3688 1.3.2.2 martin * igc_release_hw_control resets {CTRL_EXT|FWSM}:DRV_LOAD bit. 3689 1.3.2.2 martin * For ASF and Pass Through versions of f/w this means that 3690 1.3.2.2 martin * the driver is no longer loaded. For AMT versions of the 3691 1.3.2.2 martin * f/w this means that the network i/f is closed. 3692 1.3.2.2 martin */ 3693 1.3.2.2 martin static void 3694 1.3.2.2 martin igc_release_hw_control(struct igc_softc *sc) 3695 1.3.2.2 martin { 3696 1.3.2.2 martin const uint32_t ctrl_ext = IGC_READ_REG(&sc->hw, IGC_CTRL_EXT); 3697 1.3.2.2 martin 3698 1.3.2.2 martin IGC_WRITE_REG(&sc->hw, IGC_CTRL_EXT, ctrl_ext & ~IGC_CTRL_EXT_DRV_LOAD); 3699 1.3.2.2 martin } 3700 1.3.2.2 martin 3701 1.3.2.2 martin static int 3702 1.3.2.2 martin igc_is_valid_ether_addr(uint8_t *addr) 3703 1.3.2.2 martin { 3704 1.3.2.2 martin const char zero_addr[6] = { 0, 0, 0, 0, 0, 0 }; 3705 1.3.2.2 martin 3706 1.3.2.2 martin if ((addr[0] & 1) || !bcmp(addr, zero_addr, ETHER_ADDR_LEN)) 3707 1.3.2.2 martin return 0; 3708 1.3.2.2 martin 3709 1.3.2.2 martin return 1; 3710 1.3.2.2 martin } 3711 1.3.2.2 martin 3712 1.3.2.2 martin static void 3713 1.3.2.2 martin igc_print_devinfo(struct igc_softc *sc) 3714 1.3.2.2 martin { 3715 1.3.2.2 martin device_t dev = sc->sc_dev; 3716 1.3.2.2 martin struct igc_hw *hw = &sc->hw; 3717 1.3.2.2 martin struct igc_phy_info *phy = &hw->phy; 3718 1.3.2.2 martin u_int oui, model, rev; 3719 1.3.2.4 martin uint16_t id1, id2, nvm_ver, phy_ver, etk_lo, etk_hi; 3720 1.3.2.2 martin char descr[MII_MAX_DESCR_LEN]; 3721 1.3.2.2 martin 3722 1.3.2.2 martin /* Print PHY Info */ 3723 1.3.2.2 martin id1 = phy->id >> 16; 3724 1.3.2.2 martin /* The revision field in phy->id is cleard and it's in phy->revision */ 3725 1.3.2.2 martin id2 = (phy->id & 0xfff0) | phy->revision; 3726 1.3.2.2 martin oui = MII_OUI(id1, id2); 3727 1.3.2.2 martin model = MII_MODEL(id2); 3728 1.3.2.2 martin rev = MII_REV(id2); 3729 1.3.2.2 martin mii_get_descr(descr, sizeof(descr), oui, model); 3730 1.3.2.2 martin if (descr[0]) 3731 1.3.2.4 martin aprint_normal_dev(dev, "PHY: %s, rev. %d", 3732 1.3.2.2 martin descr, rev); 3733 1.3.2.2 martin else 3734 1.3.2.2 martin aprint_normal_dev(dev, 3735 1.3.2.4 martin "PHY OUI 0x%06x, model 0x%04x, rev. %d", 3736 1.3.2.2 martin oui, model, rev); 3737 1.3.2.2 martin 3738 1.3.2.4 martin /* PHY FW version */ 3739 1.3.2.4 martin phy->ops.read_reg(hw, 0x1e, &phy_ver); 3740 1.3.2.4 martin aprint_normal(", PHY FW version 0x%04hx\n", phy_ver); 3741 1.3.2.4 martin 3742 1.3.2.4 martin /* NVM version */ 3743 1.3.2.2 martin hw->nvm.ops.read(hw, NVM_VERSION, 1, &nvm_ver); 3744 1.3.2.2 martin 3745 1.3.2.4 martin /* EtrackID */ 3746 1.3.2.4 martin hw->nvm.ops.read(hw, NVM_ETKID_LO, 1, &etk_lo); 3747 1.3.2.4 martin hw->nvm.ops.read(hw, NVM_ETKID_HI, 1, &etk_hi); 3748 1.3.2.2 martin 3749 1.3.2.4 martin aprint_normal_dev(dev, 3750 1.3.2.4 martin "NVM image version %x.%02x, EtrackID %04hx%04hx\n", 3751 1.3.2.2 martin (nvm_ver & NVM_VERSION_MAJOR) >> NVM_VERSION_MAJOR_SHIFT, 3752 1.3.2.4 martin nvm_ver & NVM_VERSION_MINOR, etk_hi, etk_lo); 3753 1.3.2.2 martin } 3754
Indexes created Tue Sep 30 20:09:53 GMT 2025