if_igc.c revision 1.3.2.4
1 1.3.2.4 martin /* $NetBSD: if_igc.c,v 1.3.2.4 2024/02/23 18:41:02 martin Exp $ */ 2 1.3.2.2 martin /* $OpenBSD: if_igc.c,v 1.13 2023/04/28 10:18:57 bluhm Exp $ */ 3 1.3.2.2 martin /*- 4 1.3.2.2 martin * SPDX-License-Identifier: BSD-2-Clause 5 1.3.2.2 martin * 6 1.3.2.2 martin * Copyright (c) 2016 Nicole Graziano <nicole (at) nextbsd.org> 7 1.3.2.2 martin * All rights reserved. 8 1.3.2.2 martin * Copyright (c) 2021 Rubicon Communications, LLC (Netgate) 9 1.3.2.2 martin * 10 1.3.2.2 martin * Redistribution and use in source and binary forms, with or without 11 1.3.2.2 martin * modification, are permitted provided that the following conditions 12 1.3.2.2 martin * are met: 13 1.3.2.2 martin * 1. Redistributions of source code must retain the above copyright 14 1.3.2.2 martin * notice, this list of conditions and the following disclaimer. 15 1.3.2.2 martin * 2. Redistributions in binary form must reproduce the above copyright 16 1.3.2.2 martin * notice, this list of conditions and the following disclaimer in the 17 1.3.2.2 martin * documentation and/or other materials provided with the distribution. 18 1.3.2.2 martin * 19 1.3.2.2 martin * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 20 1.3.2.2 martin * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 21 1.3.2.2 martin * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 22 1.3.2.2 martin * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 23 1.3.2.2 martin * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 24 1.3.2.2 martin * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 25 1.3.2.2 martin * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 26 1.3.2.2 martin * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 27 1.3.2.2 martin * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 28 1.3.2.2 martin * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 29 1.3.2.2 martin * SUCH DAMAGE. 30 1.3.2.2 martin */ 31 1.3.2.2 martin 32 1.3.2.2 martin #include <sys/cdefs.h> 33 1.3.2.4 martin __KERNEL_RCSID(0, "$NetBSD: if_igc.c,v 1.3.2.4 2024/02/23 18:41:02 martin Exp $"); 34 1.3.2.2 martin 35 1.3.2.2 martin #ifdef _KERNEL_OPT 36 1.3.2.2 martin #include "opt_net_mpsafe.h" 37 1.3.2.2 martin #include "opt_if_igc.h" 38 1.3.2.2 martin #if 0 /* notyet */ 39 1.3.2.2 martin #include "vlan.h" 40 1.3.2.2 martin #endif 41 1.3.2.2 martin #endif 42 1.3.2.2 martin 43 1.3.2.2 martin #include <sys/param.h> 44 1.3.2.2 martin #include <sys/systm.h> 45 1.3.2.2 martin #include <sys/bus.h> 46 1.3.2.2 martin #include <sys/cpu.h> 47 1.3.2.2 martin #include <sys/device.h> 48 1.3.2.2 martin #include <sys/endian.h> 49 1.3.2.2 martin #include <sys/intr.h> 50 1.3.2.2 martin #include <sys/interrupt.h> 51 1.3.2.2 martin #include <sys/kernel.h> 52 1.3.2.2 martin #include <sys/kmem.h> 53 1.3.2.2 martin #include <sys/mbuf.h> 54 1.3.2.2 martin #include <sys/mutex.h> 55 1.3.2.2 martin #include <sys/socket.h> 56 1.3.2.2 martin #include <sys/workqueue.h> 57 1.3.2.2 martin #include <sys/xcall.h> 58 1.3.2.2 martin 59 1.3.2.2 martin #include <net/bpf.h> 60 1.3.2.2 martin #include <net/if.h> 61 1.3.2.2 martin #include <net/if_ether.h> 62 1.3.2.2 martin #include <net/if_media.h> 63 1.3.2.2 martin #include <net/if_vlanvar.h> 64 1.3.2.2 martin #include <net/rss_config.h> 65 1.3.2.2 martin 66 1.3.2.2 martin #include <netinet/in.h> 67 1.3.2.2 martin #include <netinet/ip.h> 68 1.3.2.2 martin #include <netinet/ip6.h> 69 1.3.2.2 martin #include <netinet/tcp.h> 70 1.3.2.2 martin 71 1.3.2.2 martin #include <dev/pci/pcivar.h> 72 1.3.2.2 martin #include <dev/pci/pcireg.h> 73 1.3.2.2 martin #include <dev/pci/pcidevs.h> 74 1.3.2.2 martin 75 1.3.2.2 martin #include <dev/pci/igc/if_igc.h> 76 1.3.2.2 martin #include <dev/pci/igc/igc_evcnt.h> 77 1.3.2.2 martin #include <dev/pci/igc/igc_hw.h> 78 1.3.2.2 martin #include <dev/mii/miivar.h> 79 1.3.2.2 martin 80 1.3.2.2 martin #define IGC_WORKQUEUE_PRI PRI_SOFTNET 81 1.3.2.2 martin 82 1.3.2.2 martin #ifndef IGC_RX_INTR_PROCESS_LIMIT_DEFAULT 83 1.3.2.2 martin #define IGC_RX_INTR_PROCESS_LIMIT_DEFAULT 0 84 1.3.2.2 martin #endif 85 1.3.2.2 martin #ifndef IGC_TX_INTR_PROCESS_LIMIT_DEFAULT 86 1.3.2.2 martin #define IGC_TX_INTR_PROCESS_LIMIT_DEFAULT 0 87 1.3.2.2 martin #endif 88 1.3.2.2 martin 89 1.3.2.2 martin #ifndef IGC_RX_PROCESS_LIMIT_DEFAULT 90 1.3.2.2 martin #define IGC_RX_PROCESS_LIMIT_DEFAULT 256 91 1.3.2.2 martin #endif 92 1.3.2.2 martin #ifndef IGC_TX_PROCESS_LIMIT_DEFAULT 93 1.3.2.2 martin #define IGC_TX_PROCESS_LIMIT_DEFAULT 256 94 1.3.2.2 martin #endif 95 1.3.2.2 martin 96 1.3.2.2 martin #define htolem32(p, x) (*((uint32_t *)(p)) = htole32(x)) 97 1.3.2.2 martin #define htolem64(p, x) (*((uint64_t *)(p)) = htole64(x)) 98 1.3.2.2 martin 99 1.3.2.2 martin static const struct igc_product { 100 1.3.2.2 martin pci_vendor_id_t igcp_vendor; 101 1.3.2.2 martin pci_product_id_t igcp_product; 102 1.3.2.2 martin const char *igcp_name; 103 1.3.2.2 martin } igc_products[] = { 104 1.3.2.2 martin { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_I225_IT, 105 1.3.2.2 martin "Intel(R) Ethernet Controller I225-IT(2)" }, 106 1.3.2.2 martin { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_I226_LM, 107 1.3.2.2 martin "Intel(R) Ethernet Controller I226-LM" }, 108 1.3.2.2 martin { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_I226_V, 109 1.3.2.2 martin "Intel(R) Ethernet Controller I226-V" }, 110 1.3.2.2 martin { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_I226_IT, 111 1.3.2.2 martin "Intel(R) Ethernet Controller I226-IT" }, 112 1.3.2.2 martin { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_I221_V, 113 1.3.2.2 martin "Intel(R) Ethernet Controller I221-V" }, 114 1.3.2.2 martin { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_I226_BLANK_NVM, 115 1.3.2.2 martin "Intel(R) Ethernet Controller I226(blankNVM)" }, 116 1.3.2.2 martin { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_I225_LM, 117 1.3.2.2 martin "Intel(R) Ethernet Controller I225-LM" }, 118 1.3.2.2 martin { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_I225_V, 119 1.3.2.2 martin "Intel(R) Ethernet Controller I225-V" }, 120 1.3.2.2 martin { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_I220_V, 121 1.3.2.2 martin "Intel(R) Ethernet Controller I220-V" }, 122 1.3.2.2 martin { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_I225_I, 123 1.3.2.2 martin "Intel(R) Ethernet Controller I225-I" }, 124 1.3.2.2 martin { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_I225_BLANK_NVM, 125 1.3.2.2 martin "Intel(R) Ethernet Controller I225(blankNVM)" }, 126 1.3.2.2 martin { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_I225_K, 127 1.3.2.2 martin "Intel(R) Ethernet Controller I225-K" }, 128 1.3.2.2 martin { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_I225_K2, 129 1.3.2.2 martin "Intel(R) Ethernet Controller I225-K(2)" }, 130 1.3.2.2 martin { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_I226_K, 131 1.3.2.2 martin "Intel(R) Ethernet Controller I226-K" }, 132 1.3.2.2 martin { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_I225_LMVP, 133 1.3.2.2 martin "Intel(R) Ethernet Controller I225-LMvP(2)" }, 134 1.3.2.2 martin { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_I226_LMVP, 135 1.3.2.2 martin "Intel(R) Ethernet Controller I226-LMvP" }, 136 1.3.2.2 martin { 0, 0, NULL }, 137 1.3.2.2 martin }; 138 1.3.2.2 martin 139 1.3.2.2 martin #define IGC_DF_CFG 0x1 140 1.3.2.2 martin #define IGC_DF_TX 0x2 141 1.3.2.2 martin #define IGC_DF_RX 0x4 142 1.3.2.2 martin #define IGC_DF_MISC 0x8 143 1.3.2.2 martin 144 1.3.2.2 martin #ifdef IGC_DEBUG_FLAGS 145 1.3.2.2 martin int igc_debug_flags = IGC_DEBUG_FLAGS; 146 1.3.2.2 martin #else 147 1.3.2.2 martin int igc_debug_flags = 0; 148 1.3.2.2 martin #endif 149 1.3.2.2 martin 150 1.3.2.2 martin #define DPRINTF(flag, fmt, args...) do { \ 151 1.3.2.2 martin if (igc_debug_flags & (IGC_DF_ ## flag)) \ 152 1.3.2.2 martin printf("%s: %d: " fmt, __func__, __LINE__, ##args); \ 153 1.3.2.2 martin } while (0) 154 1.3.2.2 martin 155 1.3.2.2 martin /********************************************************************* 156 1.3.2.2 martin * Function Prototypes 157 1.3.2.2 martin *********************************************************************/ 158 1.3.2.2 martin static int igc_match(device_t, cfdata_t, void *); 159 1.3.2.2 martin static void igc_attach(device_t, device_t, void *); 160 1.3.2.2 martin static int igc_detach(device_t, int); 161 1.3.2.2 martin 162 1.3.2.2 martin static void igc_identify_hardware(struct igc_softc *); 163 1.3.2.2 martin static int igc_adjust_nqueues(struct igc_softc *); 164 1.3.2.2 martin static int igc_allocate_pci_resources(struct igc_softc *); 165 1.3.2.2 martin static int igc_allocate_interrupts(struct igc_softc *); 166 1.3.2.2 martin static int igc_allocate_queues(struct igc_softc *); 167 1.3.2.2 martin static void igc_free_pci_resources(struct igc_softc *); 168 1.3.2.2 martin static void igc_free_interrupts(struct igc_softc *); 169 1.3.2.2 martin static void igc_free_queues(struct igc_softc *); 170 1.3.2.2 martin static void igc_reset(struct igc_softc *); 171 1.3.2.2 martin static void igc_init_dmac(struct igc_softc *, uint32_t); 172 1.3.2.2 martin static int igc_setup_interrupts(struct igc_softc *); 173 1.3.2.2 martin static void igc_attach_counters(struct igc_softc *sc); 174 1.3.2.2 martin static void igc_detach_counters(struct igc_softc *sc); 175 1.3.2.2 martin static void igc_update_counters(struct igc_softc *sc); 176 1.3.2.2 martin static void igc_clear_counters(struct igc_softc *sc); 177 1.3.2.2 martin static int igc_setup_msix(struct igc_softc *); 178 1.3.2.2 martin static int igc_setup_msi(struct igc_softc *); 179 1.3.2.2 martin static int igc_setup_intx(struct igc_softc *); 180 1.3.2.2 martin static int igc_dma_malloc(struct igc_softc *, bus_size_t, 181 1.3.2.2 martin struct igc_dma_alloc *); 182 1.3.2.2 martin static void igc_dma_free(struct igc_softc *, struct igc_dma_alloc *); 183 1.3.2.2 martin static void igc_setup_interface(struct igc_softc *); 184 1.3.2.2 martin 185 1.3.2.2 martin static int igc_init(struct ifnet *); 186 1.3.2.2 martin static int igc_init_locked(struct igc_softc *); 187 1.3.2.2 martin static void igc_start(struct ifnet *); 188 1.3.2.2 martin static int igc_transmit(struct ifnet *, struct mbuf *); 189 1.3.2.2 martin static void igc_tx_common_locked(struct ifnet *, struct tx_ring *, int); 190 1.3.2.2 martin static bool igc_txeof(struct tx_ring *, u_int); 191 1.3.2.2 martin static void igc_intr_barrier(struct igc_softc *); 192 1.3.2.2 martin static void igc_stop(struct ifnet *, int); 193 1.3.2.2 martin static void igc_stop_locked(struct igc_softc *); 194 1.3.2.2 martin static int igc_ioctl(struct ifnet *, u_long, void *); 195 1.3.2.2 martin #ifdef IF_RXR 196 1.3.2.2 martin static int igc_rxrinfo(struct igc_softc *, struct if_rxrinfo *); 197 1.3.2.2 martin #endif 198 1.3.2.2 martin static void igc_rxfill(struct rx_ring *); 199 1.3.2.2 martin static void igc_rxrefill(struct rx_ring *, int); 200 1.3.2.2 martin static bool igc_rxeof(struct rx_ring *, u_int); 201 1.3.2.2 martin static int igc_rx_checksum(struct igc_queue *, uint64_t, uint32_t, 202 1.3.2.2 martin uint32_t); 203 1.3.2.2 martin static void igc_watchdog(struct ifnet *); 204 1.3.2.2 martin static void igc_tick(void *); 205 1.3.2.2 martin static void igc_media_status(struct ifnet *, struct ifmediareq *); 206 1.3.2.2 martin static int igc_media_change(struct ifnet *); 207 1.3.2.2 martin static int igc_ifflags_cb(struct ethercom *); 208 1.3.2.2 martin static void igc_set_filter(struct igc_softc *); 209 1.3.2.2 martin static void igc_update_link_status(struct igc_softc *); 210 1.3.2.2 martin static int igc_get_buf(struct rx_ring *, int, bool); 211 1.3.2.2 martin static int igc_tx_ctx_setup(struct tx_ring *, struct mbuf *, int, 212 1.3.2.2 martin uint32_t *, uint32_t *); 213 1.3.2.2 martin static int igc_tso_setup(struct tx_ring *, struct mbuf *, int, 214 1.3.2.2 martin uint32_t *, uint32_t *); 215 1.3.2.2 martin 216 1.3.2.2 martin static void igc_configure_queues(struct igc_softc *); 217 1.3.2.2 martin static void igc_set_queues(struct igc_softc *, uint32_t, uint32_t, int); 218 1.3.2.2 martin static void igc_enable_queue(struct igc_softc *, uint32_t); 219 1.3.2.2 martin static void igc_enable_intr(struct igc_softc *); 220 1.3.2.2 martin static void igc_disable_intr(struct igc_softc *); 221 1.3.2.2 martin static int igc_intr_link(void *); 222 1.3.2.2 martin static int igc_intr_queue(void *); 223 1.3.2.2 martin static int igc_intr(void *); 224 1.3.2.2 martin static void igc_handle_queue(void *); 225 1.3.2.2 martin static void igc_handle_queue_work(struct work *, void *); 226 1.3.2.2 martin static void igc_sched_handle_queue(struct igc_softc *, struct igc_queue *); 227 1.3.2.2 martin static void igc_barrier_handle_queue(struct igc_softc *); 228 1.3.2.2 martin 229 1.3.2.2 martin static int igc_allocate_transmit_buffers(struct tx_ring *); 230 1.3.2.2 martin static int igc_setup_transmit_structures(struct igc_softc *); 231 1.3.2.2 martin static int igc_setup_transmit_ring(struct tx_ring *); 232 1.3.2.2 martin static void igc_initialize_transmit_unit(struct igc_softc *); 233 1.3.2.2 martin static void igc_free_transmit_structures(struct igc_softc *); 234 1.3.2.2 martin static void igc_free_transmit_buffers(struct tx_ring *); 235 1.3.2.2 martin static void igc_withdraw_transmit_packets(struct tx_ring *, bool); 236 1.3.2.2 martin static int igc_allocate_receive_buffers(struct rx_ring *); 237 1.3.2.2 martin static int igc_setup_receive_structures(struct igc_softc *); 238 1.3.2.2 martin static int igc_setup_receive_ring(struct rx_ring *); 239 1.3.2.2 martin static void igc_initialize_receive_unit(struct igc_softc *); 240 1.3.2.2 martin static void igc_free_receive_structures(struct igc_softc *); 241 1.3.2.2 martin static void igc_free_receive_buffers(struct rx_ring *); 242 1.3.2.2 martin static void igc_clear_receive_status(struct rx_ring *); 243 1.3.2.2 martin static void igc_initialize_rss_mapping(struct igc_softc *); 244 1.3.2.2 martin 245 1.3.2.2 martin static void igc_get_hw_control(struct igc_softc *); 246 1.3.2.2 martin static void igc_release_hw_control(struct igc_softc *); 247 1.3.2.2 martin static int igc_is_valid_ether_addr(uint8_t *); 248 1.3.2.2 martin static void igc_print_devinfo(struct igc_softc *); 249 1.3.2.2 martin 250 1.3.2.2 martin CFATTACH_DECL3_NEW(igc, sizeof(struct igc_softc), 251 1.3.2.2 martin igc_match, igc_attach, igc_detach, NULL, NULL, NULL, 0); 252 1.3.2.2 martin 253 1.3.2.2 martin static inline int 254 1.3.2.2 martin igc_txdesc_incr(struct igc_softc *sc, int id) 255 1.3.2.2 martin { 256 1.3.2.2 martin 257 1.3.2.2 martin if (++id == sc->num_tx_desc) 258 1.3.2.2 martin id = 0; 259 1.3.2.2 martin return id; 260 1.3.2.2 martin } 261 1.3.2.2 martin 262 1.3.2.2 martin static inline int __unused 263 1.3.2.2 martin igc_txdesc_decr(struct igc_softc *sc, int id) 264 1.3.2.2 martin { 265 1.3.2.2 martin 266 1.3.2.2 martin if (--id < 0) 267 1.3.2.2 martin id = sc->num_tx_desc - 1; 268 1.3.2.2 martin return id; 269 1.3.2.2 martin } 270 1.3.2.2 martin 271 1.3.2.2 martin static inline void 272 1.3.2.2 martin igc_txdesc_sync(struct tx_ring *txr, int id, int ops) 273 1.3.2.2 martin { 274 1.3.2.2 martin 275 1.3.2.2 martin bus_dmamap_sync(txr->txdma.dma_tag, txr->txdma.dma_map, 276 1.3.2.2 martin id * sizeof(union igc_adv_tx_desc), sizeof(union igc_adv_tx_desc), 277 1.3.2.2 martin ops); 278 1.3.2.2 martin } 279 1.3.2.2 martin 280 1.3.2.2 martin static inline int 281 1.3.2.2 martin igc_rxdesc_incr(struct igc_softc *sc, int id) 282 1.3.2.2 martin { 283 1.3.2.2 martin 284 1.3.2.2 martin if (++id == sc->num_rx_desc) 285 1.3.2.2 martin id = 0; 286 1.3.2.2 martin return id; 287 1.3.2.2 martin } 288 1.3.2.2 martin 289 1.3.2.2 martin static inline int 290 1.3.2.2 martin igc_rxdesc_decr(struct igc_softc *sc, int id) 291 1.3.2.2 martin { 292 1.3.2.2 martin 293 1.3.2.2 martin if (--id < 0) 294 1.3.2.2 martin id = sc->num_rx_desc - 1; 295 1.3.2.2 martin return id; 296 1.3.2.2 martin } 297 1.3.2.2 martin 298 1.3.2.2 martin static inline void 299 1.3.2.2 martin igc_rxdesc_sync(struct rx_ring *rxr, int id, int ops) 300 1.3.2.2 martin { 301 1.3.2.2 martin 302 1.3.2.2 martin bus_dmamap_sync(rxr->rxdma.dma_tag, rxr->rxdma.dma_map, 303 1.3.2.2 martin id * sizeof(union igc_adv_rx_desc), sizeof(union igc_adv_rx_desc), 304 1.3.2.2 martin ops); 305 1.3.2.2 martin } 306 1.3.2.2 martin 307 1.3.2.2 martin static const struct igc_product * 308 1.3.2.2 martin igc_lookup(const struct pci_attach_args *pa) 309 1.3.2.2 martin { 310 1.3.2.2 martin const struct igc_product *igcp; 311 1.3.2.2 martin 312 1.3.2.2 martin for (igcp = igc_products; igcp->igcp_name != NULL; igcp++) { 313 1.3.2.2 martin if (PCI_VENDOR(pa->pa_id) == igcp->igcp_vendor && 314 1.3.2.2 martin PCI_PRODUCT(pa->pa_id) == igcp->igcp_product) 315 1.3.2.2 martin return igcp; 316 1.3.2.2 martin } 317 1.3.2.2 martin return NULL; 318 1.3.2.2 martin } 319 1.3.2.2 martin 320 1.3.2.2 martin /********************************************************************* 321 1.3.2.2 martin * Device identification routine 322 1.3.2.2 martin * 323 1.3.2.2 martin * igc_match determines if the driver should be loaded on 324 1.3.2.2 martin * adapter based on PCI vendor/device id of the adapter. 325 1.3.2.2 martin * 326 1.3.2.2 martin * return 0 on success, positive on failure 327 1.3.2.2 martin *********************************************************************/ 328 1.3.2.2 martin static int 329 1.3.2.2 martin igc_match(device_t parent, cfdata_t match, void *aux) 330 1.3.2.2 martin { 331 1.3.2.2 martin struct pci_attach_args *pa = aux; 332 1.3.2.2 martin 333 1.3.2.2 martin if (igc_lookup(pa) != NULL) 334 1.3.2.2 martin return 1; 335 1.3.2.2 martin 336 1.3.2.2 martin return 0; 337 1.3.2.2 martin } 338 1.3.2.2 martin 339 1.3.2.2 martin /********************************************************************* 340 1.3.2.2 martin * Device initialization routine 341 1.3.2.2 martin * 342 1.3.2.2 martin * The attach entry point is called when the driver is being loaded. 343 1.3.2.2 martin * This routine identifies the type of hardware, allocates all resources 344 1.3.2.2 martin * and initializes the hardware. 345 1.3.2.2 martin * 346 1.3.2.2 martin * return 0 on success, positive on failure 347 1.3.2.2 martin *********************************************************************/ 348 1.3.2.2 martin static void 349 1.3.2.2 martin igc_attach(device_t parent, device_t self, void *aux) 350 1.3.2.2 martin { 351 1.3.2.2 martin struct pci_attach_args *pa = aux; 352 1.3.2.2 martin struct igc_softc *sc = device_private(self); 353 1.3.2.2 martin struct igc_hw *hw = &sc->hw; 354 1.3.2.2 martin 355 1.3.2.2 martin const struct igc_product *igcp = igc_lookup(pa); 356 1.3.2.2 martin KASSERT(igcp != NULL); 357 1.3.2.2 martin pci_aprint_devinfo_fancy(pa, "Ethernet controller", igcp->igcp_name, 1); 358 1.3.2.2 martin 359 1.3.2.2 martin sc->sc_dev = self; 360 1.3.2.2 martin callout_init(&sc->sc_tick_ch, CALLOUT_MPSAFE); 361 1.3.2.2 martin callout_setfunc(&sc->sc_tick_ch, igc_tick, sc); 362 1.3.2.2 martin sc->sc_core_stopping = false; 363 1.3.2.2 martin 364 1.3.2.2 martin sc->osdep.os_sc = sc; 365 1.3.2.2 martin sc->osdep.os_pa = *pa; 366 1.3.2.2 martin #ifdef __aarch64__ 367 1.3.2.2 martin /* 368 1.3.2.2 martin * XXX PR port-arm/57643 369 1.3.2.2 martin * 64-bit DMA does not work at least for LX2K with 32/64GB memory. 370 1.3.2.2 martin * smmu(4) support may be required. 371 1.3.2.2 martin */ 372 1.3.2.2 martin sc->osdep.os_dmat = pa->pa_dmat; 373 1.3.2.2 martin #else 374 1.3.2.2 martin sc->osdep.os_dmat = pci_dma64_available(pa) ? 375 1.3.2.2 martin pa->pa_dmat64 : pa->pa_dmat; 376 1.3.2.2 martin #endif 377 1.3.2.2 martin 378 1.3.2.2 martin /* Determine hardware and mac info */ 379 1.3.2.2 martin igc_identify_hardware(sc); 380 1.3.2.2 martin 381 1.3.2.2 martin sc->num_tx_desc = IGC_DEFAULT_TXD; 382 1.3.2.2 martin sc->num_rx_desc = IGC_DEFAULT_RXD; 383 1.3.2.2 martin 384 1.3.2.2 martin /* Setup PCI resources */ 385 1.3.2.2 martin if (igc_allocate_pci_resources(sc)) { 386 1.3.2.2 martin aprint_error_dev(sc->sc_dev, 387 1.3.2.2 martin "unable to allocate PCI resources\n"); 388 1.3.2.2 martin goto err_pci; 389 1.3.2.2 martin } 390 1.3.2.2 martin 391 1.3.2.2 martin if (igc_allocate_interrupts(sc)) { 392 1.3.2.2 martin aprint_error_dev(sc->sc_dev, "unable to allocate interrupts\n"); 393 1.3.2.2 martin goto err_pci; 394 1.3.2.2 martin } 395 1.3.2.2 martin 396 1.3.2.2 martin /* Allocate TX/RX queues */ 397 1.3.2.2 martin if (igc_allocate_queues(sc)) { 398 1.3.2.2 martin aprint_error_dev(sc->sc_dev, "unable to allocate queues\n"); 399 1.3.2.2 martin goto err_alloc_intr; 400 1.3.2.2 martin } 401 1.3.2.2 martin 402 1.3.2.2 martin /* Do shared code initialization */ 403 1.3.2.2 martin if (igc_setup_init_funcs(hw, true)) { 404 1.3.2.2 martin aprint_error_dev(sc->sc_dev, "unable to initialize\n"); 405 1.3.2.2 martin goto err_alloc_intr; 406 1.3.2.2 martin } 407 1.3.2.2 martin 408 1.3.2.2 martin hw->mac.autoneg = DO_AUTO_NEG; 409 1.3.2.2 martin hw->phy.autoneg_wait_to_complete = false; 410 1.3.2.2 martin hw->phy.autoneg_advertised = AUTONEG_ADV_DEFAULT; 411 1.3.2.2 martin 412 1.3.2.2 martin /* Copper options. */ 413 1.3.2.2 martin if (hw->phy.media_type == igc_media_type_copper) 414 1.3.2.2 martin hw->phy.mdix = AUTO_ALL_MODES; 415 1.3.2.2 martin 416 1.3.2.2 martin /* Set the max frame size. */ 417 1.3.2.2 martin sc->hw.mac.max_frame_size = 9234; 418 1.3.2.2 martin 419 1.3.2.2 martin /* Allocate multicast array memory. */ 420 1.3.2.2 martin sc->mta = kmem_alloc(IGC_MTA_LEN, KM_SLEEP); 421 1.3.2.2 martin 422 1.3.2.2 martin /* Check SOL/IDER usage. */ 423 1.3.2.2 martin if (igc_check_reset_block(hw)) { 424 1.3.2.2 martin aprint_error_dev(sc->sc_dev, 425 1.3.2.2 martin "PHY reset is blocked due to SOL/IDER session\n"); 426 1.3.2.2 martin } 427 1.3.2.2 martin 428 1.3.2.2 martin /* Disable Energy Efficient Ethernet. */ 429 1.3.2.2 martin sc->hw.dev_spec._i225.eee_disable = true; 430 1.3.2.2 martin 431 1.3.2.2 martin igc_reset_hw(hw); 432 1.3.2.2 martin 433 1.3.2.2 martin /* Make sure we have a good EEPROM before we read from it. */ 434 1.3.2.2 martin if (igc_validate_nvm_checksum(hw) < 0) { 435 1.3.2.2 martin /* 436 1.3.2.2 martin * Some PCI-E parts fail the first check due to 437 1.3.2.2 martin * the link being in sleep state, call it again, 438 1.3.2.2 martin * if it fails a second time its a real issue. 439 1.3.2.2 martin */ 440 1.3.2.2 martin if (igc_validate_nvm_checksum(hw) < 0) { 441 1.3.2.2 martin aprint_error_dev(sc->sc_dev, 442 1.3.2.2 martin "EEPROM checksum invalid\n"); 443 1.3.2.2 martin goto err_late; 444 1.3.2.2 martin } 445 1.3.2.2 martin } 446 1.3.2.2 martin 447 1.3.2.2 martin /* Copy the permanent MAC address out of the EEPROM. */ 448 1.3.2.2 martin if (igc_read_mac_addr(hw) < 0) { 449 1.3.2.2 martin aprint_error_dev(sc->sc_dev, 450 1.3.2.2 martin "unable to read MAC address from EEPROM\n"); 451 1.3.2.2 martin goto err_late; 452 1.3.2.2 martin } 453 1.3.2.2 martin 454 1.3.2.2 martin if (!igc_is_valid_ether_addr(hw->mac.addr)) { 455 1.3.2.2 martin aprint_error_dev(sc->sc_dev, "invalid MAC address\n"); 456 1.3.2.2 martin goto err_late; 457 1.3.2.2 martin } 458 1.3.2.2 martin 459 1.3.2.2 martin if (igc_setup_interrupts(sc)) 460 1.3.2.2 martin goto err_late; 461 1.3.2.2 martin 462 1.3.2.2 martin /* Attach counters. */ 463 1.3.2.2 martin igc_attach_counters(sc); 464 1.3.2.2 martin 465 1.3.2.2 martin /* Setup OS specific network interface. */ 466 1.3.2.2 martin igc_setup_interface(sc); 467 1.3.2.2 martin 468 1.3.2.2 martin igc_print_devinfo(sc); 469 1.3.2.2 martin 470 1.3.2.2 martin igc_reset(sc); 471 1.3.2.2 martin hw->mac.get_link_status = true; 472 1.3.2.2 martin igc_update_link_status(sc); 473 1.3.2.2 martin 474 1.3.2.2 martin /* The driver can now take control from firmware. */ 475 1.3.2.2 martin igc_get_hw_control(sc); 476 1.3.2.2 martin 477 1.3.2.2 martin aprint_normal_dev(sc->sc_dev, "Ethernet address %s\n", 478 1.3.2.2 martin ether_sprintf(sc->hw.mac.addr)); 479 1.3.2.2 martin 480 1.3.2.2 martin if (pmf_device_register(self, NULL, NULL)) 481 1.3.2.2 martin pmf_class_network_register(self, &sc->sc_ec.ec_if); 482 1.3.2.2 martin else 483 1.3.2.2 martin aprint_error_dev(self, "couldn't establish power handler\n"); 484 1.3.2.2 martin 485 1.3.2.2 martin return; 486 1.3.2.2 martin 487 1.3.2.2 martin err_late: 488 1.3.2.2 martin igc_release_hw_control(sc); 489 1.3.2.2 martin err_alloc_intr: 490 1.3.2.2 martin igc_free_interrupts(sc); 491 1.3.2.2 martin err_pci: 492 1.3.2.2 martin igc_free_pci_resources(sc); 493 1.3.2.2 martin kmem_free(sc->mta, IGC_MTA_LEN); 494 1.3.2.2 martin } 495 1.3.2.2 martin 496 1.3.2.2 martin /********************************************************************* 497 1.3.2.2 martin * Device removal routine 498 1.3.2.2 martin * 499 1.3.2.2 martin * The detach entry point is called when the driver is being removed. 500 1.3.2.2 martin * This routine stops the adapter and deallocates all the resources 501 1.3.2.2 martin * that were allocated for driver operation. 502 1.3.2.2 martin * 503 1.3.2.2 martin * return 0 on success, positive on failure 504 1.3.2.2 martin *********************************************************************/ 505 1.3.2.2 martin static int 506 1.3.2.2 martin igc_detach(device_t self, int flags) 507 1.3.2.2 martin { 508 1.3.2.2 martin struct igc_softc *sc = device_private(self); 509 1.3.2.2 martin struct ifnet *ifp = &sc->sc_ec.ec_if; 510 1.3.2.2 martin 511 1.3.2.2 martin mutex_enter(&sc->sc_core_lock); 512 1.3.2.2 martin igc_stop_locked(sc); 513 1.3.2.2 martin mutex_exit(&sc->sc_core_lock); 514 1.3.2.2 martin 515 1.3.2.2 martin igc_detach_counters(sc); 516 1.3.2.2 martin 517 1.3.2.2 martin igc_free_queues(sc); 518 1.3.2.2 martin 519 1.3.2.2 martin igc_phy_hw_reset(&sc->hw); 520 1.3.2.2 martin igc_release_hw_control(sc); 521 1.3.2.2 martin 522 1.3.2.2 martin ether_ifdetach(ifp); 523 1.3.2.2 martin if_detach(ifp); 524 1.3.2.2 martin ifmedia_fini(&sc->media); 525 1.3.2.2 martin 526 1.3.2.2 martin igc_free_interrupts(sc); 527 1.3.2.2 martin igc_free_pci_resources(sc); 528 1.3.2.2 martin kmem_free(sc->mta, IGC_MTA_LEN); 529 1.3.2.2 martin 530 1.3.2.2 martin mutex_destroy(&sc->sc_core_lock); 531 1.3.2.2 martin 532 1.3.2.2 martin return 0; 533 1.3.2.2 martin } 534 1.3.2.2 martin 535 1.3.2.2 martin static void 536 1.3.2.2 martin igc_identify_hardware(struct igc_softc *sc) 537 1.3.2.2 martin { 538 1.3.2.2 martin struct igc_osdep *os = &sc->osdep; 539 1.3.2.2 martin struct pci_attach_args *pa = &os->os_pa; 540 1.3.2.2 martin 541 1.3.2.2 martin /* Save off the information about this board. */ 542 1.3.2.2 martin sc->hw.device_id = PCI_PRODUCT(pa->pa_id); 543 1.3.2.2 martin 544 1.3.2.2 martin /* Do shared code init and setup. */ 545 1.3.2.2 martin if (igc_set_mac_type(&sc->hw)) { 546 1.3.2.2 martin aprint_error_dev(sc->sc_dev, "unable to identify hardware\n"); 547 1.3.2.2 martin return; 548 1.3.2.2 martin } 549 1.3.2.2 martin } 550 1.3.2.2 martin 551 1.3.2.2 martin static int 552 1.3.2.2 martin igc_allocate_pci_resources(struct igc_softc *sc) 553 1.3.2.2 martin { 554 1.3.2.2 martin struct igc_osdep *os = &sc->osdep; 555 1.3.2.2 martin struct pci_attach_args *pa = &os->os_pa; 556 1.3.2.2 martin 557 1.3.2.2 martin /* 558 1.3.2.2 martin * Enable bus mastering and memory-mapped I/O for sure. 559 1.3.2.2 martin */ 560 1.3.2.2 martin pcireg_t csr = 561 1.3.2.2 martin pci_conf_read(pa->pa_pc, pa->pa_tag, PCI_COMMAND_STATUS_REG); 562 1.3.2.2 martin csr |= PCI_COMMAND_MASTER_ENABLE | PCI_COMMAND_MEM_ENABLE; 563 1.3.2.2 martin pci_conf_write(pa->pa_pc, pa->pa_tag, PCI_COMMAND_STATUS_REG, csr); 564 1.3.2.2 martin 565 1.3.2.2 martin const pcireg_t memtype = 566 1.3.2.2 martin pci_mapreg_type(pa->pa_pc, pa->pa_tag, IGC_PCIREG); 567 1.3.2.2 martin if (pci_mapreg_map(pa, IGC_PCIREG, memtype, 0, &os->os_memt, 568 1.3.2.2 martin &os->os_memh, &os->os_membase, &os->os_memsize)) { 569 1.3.2.2 martin aprint_error_dev(sc->sc_dev, "unable to map registers\n"); 570 1.3.2.2 martin return ENXIO; 571 1.3.2.2 martin } 572 1.3.2.2 martin 573 1.3.2.2 martin sc->hw.hw_addr = os->os_membase; 574 1.3.2.2 martin sc->hw.back = os; 575 1.3.2.2 martin 576 1.3.2.2 martin return 0; 577 1.3.2.2 martin } 578 1.3.2.2 martin 579 1.3.2.2 martin static int __unused 580 1.3.2.2 martin igc_adjust_nqueues(struct igc_softc *sc) 581 1.3.2.2 martin { 582 1.3.2.2 martin struct pci_attach_args *pa = &sc->osdep.os_pa; 583 1.3.2.2 martin int nqueues = MIN(IGC_MAX_NQUEUES, ncpu); 584 1.3.2.2 martin 585 1.3.2.2 martin const int nmsix = pci_msix_count(pa->pa_pc, pa->pa_tag); 586 1.3.2.2 martin if (nmsix <= 1) 587 1.3.2.2 martin nqueues = 1; 588 1.3.2.2 martin else if (nmsix < nqueues + 1) 589 1.3.2.2 martin nqueues = nmsix - 1; 590 1.3.2.2 martin 591 1.3.2.2 martin return nqueues; 592 1.3.2.2 martin } 593 1.3.2.2 martin 594 1.3.2.2 martin static int 595 1.3.2.2 martin igc_allocate_interrupts(struct igc_softc *sc) 596 1.3.2.2 martin { 597 1.3.2.2 martin struct pci_attach_args *pa = &sc->osdep.os_pa; 598 1.3.2.2 martin int error; 599 1.3.2.2 martin 600 1.3.2.2 martin #ifndef IGC_DISABLE_MSIX 601 1.3.2.2 martin const int nqueues = igc_adjust_nqueues(sc); 602 1.3.2.2 martin if (nqueues > 1) { 603 1.3.2.2 martin sc->sc_nintrs = nqueues + 1; 604 1.3.2.2 martin error = pci_msix_alloc_exact(pa, &sc->sc_intrs, sc->sc_nintrs); 605 1.3.2.2 martin if (!error) { 606 1.3.2.2 martin sc->sc_nqueues = nqueues; 607 1.3.2.2 martin sc->sc_intr_type = PCI_INTR_TYPE_MSIX; 608 1.3.2.2 martin return 0; 609 1.3.2.2 martin } 610 1.3.2.2 martin } 611 1.3.2.2 martin #endif 612 1.3.2.2 martin 613 1.3.2.2 martin /* fallback to MSI */ 614 1.3.2.2 martin sc->sc_nintrs = sc->sc_nqueues = 1; 615 1.3.2.2 martin 616 1.3.2.2 martin #ifndef IGC_DISABLE_MSI 617 1.3.2.2 martin error = pci_msi_alloc_exact(pa, &sc->sc_intrs, sc->sc_nintrs); 618 1.3.2.2 martin if (!error) { 619 1.3.2.2 martin sc->sc_intr_type = PCI_INTR_TYPE_MSI; 620 1.3.2.2 martin return 0; 621 1.3.2.2 martin } 622 1.3.2.2 martin #endif 623 1.3.2.2 martin 624 1.3.2.2 martin /* fallback to INTx */ 625 1.3.2.2 martin 626 1.3.2.2 martin error = pci_intx_alloc(pa, &sc->sc_intrs); 627 1.3.2.2 martin if (!error) { 628 1.3.2.2 martin sc->sc_intr_type = PCI_INTR_TYPE_INTX; 629 1.3.2.2 martin return 0; 630 1.3.2.2 martin } 631 1.3.2.2 martin 632 1.3.2.2 martin return error; 633 1.3.2.2 martin } 634 1.3.2.2 martin 635 1.3.2.2 martin static int 636 1.3.2.2 martin igc_allocate_queues(struct igc_softc *sc) 637 1.3.2.2 martin { 638 1.3.2.2 martin device_t dev = sc->sc_dev; 639 1.3.2.2 martin int rxconf = 0, txconf = 0; 640 1.3.2.2 martin 641 1.3.2.2 martin /* Allocate the top level queue structs. */ 642 1.3.2.2 martin sc->queues = 643 1.3.2.2 martin kmem_zalloc(sc->sc_nqueues * sizeof(struct igc_queue), KM_SLEEP); 644 1.3.2.2 martin 645 1.3.2.2 martin /* Allocate the TX ring. */ 646 1.3.2.2 martin sc->tx_rings = 647 1.3.2.2 martin kmem_zalloc(sc->sc_nqueues * sizeof(struct tx_ring), KM_SLEEP); 648 1.3.2.2 martin 649 1.3.2.2 martin /* Allocate the RX ring. */ 650 1.3.2.2 martin sc->rx_rings = 651 1.3.2.2 martin kmem_zalloc(sc->sc_nqueues * sizeof(struct rx_ring), KM_SLEEP); 652 1.3.2.2 martin 653 1.3.2.2 martin /* Set up the TX queues. */ 654 1.3.2.2 martin for (int iq = 0; iq < sc->sc_nqueues; iq++, txconf++) { 655 1.3.2.2 martin struct tx_ring *txr = &sc->tx_rings[iq]; 656 1.3.2.2 martin const int tsize = roundup2( 657 1.3.2.2 martin sc->num_tx_desc * sizeof(union igc_adv_tx_desc), 658 1.3.2.2 martin IGC_DBA_ALIGN); 659 1.3.2.2 martin 660 1.3.2.2 martin txr->sc = sc; 661 1.3.2.2 martin txr->txr_igcq = &sc->queues[iq]; 662 1.3.2.2 martin txr->me = iq; 663 1.3.2.2 martin if (igc_dma_malloc(sc, tsize, &txr->txdma)) { 664 1.3.2.2 martin aprint_error_dev(dev, 665 1.3.2.2 martin "unable to allocate TX descriptor\n"); 666 1.3.2.2 martin goto fail; 667 1.3.2.2 martin } 668 1.3.2.2 martin txr->tx_base = (union igc_adv_tx_desc *)txr->txdma.dma_vaddr; 669 1.3.2.2 martin memset(txr->tx_base, 0, tsize); 670 1.3.2.2 martin } 671 1.3.2.2 martin 672 1.3.2.2 martin /* Prepare transmit descriptors and buffers. */ 673 1.3.2.2 martin if (igc_setup_transmit_structures(sc)) { 674 1.3.2.2 martin aprint_error_dev(dev, "unable to setup transmit structures\n"); 675 1.3.2.2 martin goto fail; 676 1.3.2.2 martin } 677 1.3.2.2 martin 678 1.3.2.2 martin /* Set up the RX queues. */ 679 1.3.2.2 martin for (int iq = 0; iq < sc->sc_nqueues; iq++, rxconf++) { 680 1.3.2.2 martin struct rx_ring *rxr = &sc->rx_rings[iq]; 681 1.3.2.2 martin const int rsize = roundup2( 682 1.3.2.2 martin sc->num_rx_desc * sizeof(union igc_adv_rx_desc), 683 1.3.2.2 martin IGC_DBA_ALIGN); 684 1.3.2.2 martin 685 1.3.2.2 martin rxr->sc = sc; 686 1.3.2.2 martin rxr->rxr_igcq = &sc->queues[iq]; 687 1.3.2.2 martin rxr->me = iq; 688 1.3.2.2 martin #ifdef OPENBSD 689 1.3.2.2 martin timeout_set(&rxr->rx_refill, igc_rxrefill, rxr); 690 1.3.2.2 martin #endif 691 1.3.2.2 martin if (igc_dma_malloc(sc, rsize, &rxr->rxdma)) { 692 1.3.2.2 martin aprint_error_dev(dev, 693 1.3.2.2 martin "unable to allocate RX descriptor\n"); 694 1.3.2.2 martin goto fail; 695 1.3.2.2 martin } 696 1.3.2.2 martin rxr->rx_base = (union igc_adv_rx_desc *)rxr->rxdma.dma_vaddr; 697 1.3.2.2 martin memset(rxr->rx_base, 0, rsize); 698 1.3.2.2 martin } 699 1.3.2.2 martin 700 1.3.2.2 martin sc->rx_mbuf_sz = MCLBYTES; 701 1.3.2.2 martin /* Prepare receive descriptors and buffers. */ 702 1.3.2.2 martin if (igc_setup_receive_structures(sc)) { 703 1.3.2.2 martin aprint_error_dev(sc->sc_dev, 704 1.3.2.2 martin "unable to setup receive structures\n"); 705 1.3.2.2 martin goto fail; 706 1.3.2.2 martin } 707 1.3.2.2 martin 708 1.3.2.2 martin /* Set up the queue holding structs. */ 709 1.3.2.2 martin for (int iq = 0; iq < sc->sc_nqueues; iq++) { 710 1.3.2.2 martin struct igc_queue *q = &sc->queues[iq]; 711 1.3.2.2 martin 712 1.3.2.2 martin q->sc = sc; 713 1.3.2.2 martin q->txr = &sc->tx_rings[iq]; 714 1.3.2.2 martin q->rxr = &sc->rx_rings[iq]; 715 1.3.2.2 martin } 716 1.3.2.2 martin 717 1.3.2.2 martin return 0; 718 1.3.2.2 martin 719 1.3.2.2 martin fail: 720 1.3.2.2 martin for (struct rx_ring *rxr = sc->rx_rings; rxconf > 0; rxr++, rxconf--) 721 1.3.2.2 martin igc_dma_free(sc, &rxr->rxdma); 722 1.3.2.2 martin for (struct tx_ring *txr = sc->tx_rings; txconf > 0; txr++, txconf--) 723 1.3.2.2 martin igc_dma_free(sc, &txr->txdma); 724 1.3.2.2 martin 725 1.3.2.2 martin kmem_free(sc->rx_rings, sc->sc_nqueues * sizeof(struct rx_ring)); 726 1.3.2.2 martin sc->rx_rings = NULL; 727 1.3.2.2 martin kmem_free(sc->tx_rings, sc->sc_nqueues * sizeof(struct tx_ring)); 728 1.3.2.2 martin sc->tx_rings = NULL; 729 1.3.2.2 martin kmem_free(sc->queues, sc->sc_nqueues * sizeof(struct igc_queue)); 730 1.3.2.2 martin sc->queues = NULL; 731 1.3.2.2 martin 732 1.3.2.2 martin return ENOMEM; 733 1.3.2.2 martin } 734 1.3.2.2 martin 735 1.3.2.2 martin static void 736 1.3.2.2 martin igc_free_pci_resources(struct igc_softc *sc) 737 1.3.2.2 martin { 738 1.3.2.2 martin struct igc_osdep *os = &sc->osdep; 739 1.3.2.2 martin 740 1.3.2.2 martin if (os->os_membase != 0) 741 1.3.2.2 martin bus_space_unmap(os->os_memt, os->os_memh, os->os_memsize); 742 1.3.2.2 martin os->os_membase = 0; 743 1.3.2.2 martin } 744 1.3.2.2 martin 745 1.3.2.2 martin static void 746 1.3.2.2 martin igc_free_interrupts(struct igc_softc *sc) 747 1.3.2.2 martin { 748 1.3.2.2 martin struct pci_attach_args *pa = &sc->osdep.os_pa; 749 1.3.2.2 martin pci_chipset_tag_t pc = pa->pa_pc; 750 1.3.2.2 martin 751 1.3.2.2 martin for (int i = 0; i < sc->sc_nintrs; i++) { 752 1.3.2.2 martin if (sc->sc_ihs[i] != NULL) { 753 1.3.2.2 martin pci_intr_disestablish(pc, sc->sc_ihs[i]); 754 1.3.2.2 martin sc->sc_ihs[i] = NULL; 755 1.3.2.2 martin } 756 1.3.2.2 martin } 757 1.3.2.2 martin pci_intr_release(pc, sc->sc_intrs, sc->sc_nintrs); 758 1.3.2.2 martin } 759 1.3.2.2 martin 760 1.3.2.2 martin static void 761 1.3.2.2 martin igc_free_queues(struct igc_softc *sc) 762 1.3.2.2 martin { 763 1.3.2.2 martin 764 1.3.2.2 martin igc_free_receive_structures(sc); 765 1.3.2.2 martin for (int iq = 0; iq < sc->sc_nqueues; iq++) { 766 1.3.2.2 martin struct rx_ring *rxr = &sc->rx_rings[iq]; 767 1.3.2.2 martin 768 1.3.2.2 martin igc_dma_free(sc, &rxr->rxdma); 769 1.3.2.2 martin } 770 1.3.2.2 martin 771 1.3.2.2 martin igc_free_transmit_structures(sc); 772 1.3.2.2 martin for (int iq = 0; iq < sc->sc_nqueues; iq++) { 773 1.3.2.2 martin struct tx_ring *txr = &sc->tx_rings[iq]; 774 1.3.2.2 martin 775 1.3.2.2 martin igc_dma_free(sc, &txr->txdma); 776 1.3.2.2 martin } 777 1.3.2.2 martin 778 1.3.2.2 martin kmem_free(sc->rx_rings, sc->sc_nqueues * sizeof(struct rx_ring)); 779 1.3.2.2 martin kmem_free(sc->tx_rings, sc->sc_nqueues * sizeof(struct tx_ring)); 780 1.3.2.2 martin kmem_free(sc->queues, sc->sc_nqueues * sizeof(struct igc_queue)); 781 1.3.2.2 martin } 782 1.3.2.2 martin 783 1.3.2.2 martin /********************************************************************* 784 1.3.2.2 martin * 785 1.3.2.2 martin * Initialize the hardware to a configuration as specified by the 786 1.3.2.2 martin * adapter structure. 787 1.3.2.2 martin * 788 1.3.2.2 martin **********************************************************************/ 789 1.3.2.2 martin static void 790 1.3.2.2 martin igc_reset(struct igc_softc *sc) 791 1.3.2.2 martin { 792 1.3.2.2 martin struct igc_hw *hw = &sc->hw; 793 1.3.2.2 martin 794 1.3.2.2 martin /* Let the firmware know the OS is in control */ 795 1.3.2.2 martin igc_get_hw_control(sc); 796 1.3.2.2 martin 797 1.3.2.2 martin /* 798 1.3.2.2 martin * Packet Buffer Allocation (PBA) 799 1.3.2.2 martin * Writing PBA sets the receive portion of the buffer 800 1.3.2.2 martin * the remainder is used for the transmit buffer. 801 1.3.2.2 martin */ 802 1.3.2.2 martin const uint32_t pba = IGC_PBA_34K; 803 1.3.2.2 martin 804 1.3.2.2 martin /* 805 1.3.2.2 martin * These parameters control the automatic generation (Tx) and 806 1.3.2.2 martin * response (Rx) to Ethernet PAUSE frames. 807 1.3.2.2 martin * - High water mark should allow for at least two frames to be 808 1.3.2.2 martin * received after sending an XOFF. 809 1.3.2.2 martin * - Low water mark works best when it is very near the high water mark. 810 1.3.2.2 martin * This allows the receiver to restart by sending XON when it has 811 1.3.2.2 martin * drained a bit. Here we use an arbitrary value of 1500 which will 812 1.3.2.2 martin * restart after one full frame is pulled from the buffer. There 813 1.3.2.2 martin * could be several smaller frames in the buffer and if so they will 814 1.3.2.2 martin * not trigger the XON until their total number reduces the buffer 815 1.3.2.2 martin * by 1500. 816 1.3.2.2 martin * - The pause time is fairly large at 1000 x 512ns = 512 usec. 817 1.3.2.2 martin */ 818 1.3.2.2 martin const uint16_t rx_buffer_size = (pba & 0xffff) << 10; 819 1.3.2.2 martin 820 1.3.2.2 martin hw->fc.high_water = rx_buffer_size - 821 1.3.2.2 martin roundup2(sc->hw.mac.max_frame_size, 1024); 822 1.3.2.2 martin /* 16-byte granularity */ 823 1.3.2.2 martin hw->fc.low_water = hw->fc.high_water - 16; 824 1.3.2.2 martin 825 1.3.2.2 martin if (sc->fc) /* locally set flow control value? */ 826 1.3.2.2 martin hw->fc.requested_mode = sc->fc; 827 1.3.2.2 martin else 828 1.3.2.2 martin hw->fc.requested_mode = igc_fc_full; 829 1.3.2.2 martin 830 1.3.2.2 martin hw->fc.pause_time = IGC_FC_PAUSE_TIME; 831 1.3.2.2 martin 832 1.3.2.2 martin hw->fc.send_xon = true; 833 1.3.2.2 martin 834 1.3.2.2 martin /* Issue a global reset */ 835 1.3.2.2 martin igc_reset_hw(hw); 836 1.3.2.2 martin IGC_WRITE_REG(hw, IGC_WUC, 0); 837 1.3.2.2 martin 838 1.3.2.2 martin /* and a re-init */ 839 1.3.2.2 martin if (igc_init_hw(hw) < 0) { 840 1.3.2.2 martin aprint_error_dev(sc->sc_dev, "unable to reset hardware\n"); 841 1.3.2.2 martin return; 842 1.3.2.2 martin } 843 1.3.2.2 martin 844 1.3.2.2 martin /* Setup DMA Coalescing */ 845 1.3.2.2 martin igc_init_dmac(sc, pba); 846 1.3.2.2 martin 847 1.3.2.2 martin IGC_WRITE_REG(hw, IGC_VET, ETHERTYPE_VLAN); 848 1.3.2.2 martin igc_get_phy_info(hw); 849 1.3.2.2 martin igc_check_for_link(hw); 850 1.3.2.2 martin } 851 1.3.2.2 martin 852 1.3.2.2 martin /********************************************************************* 853 1.3.2.2 martin * 854 1.3.2.2 martin * Initialize the DMA Coalescing feature 855 1.3.2.2 martin * 856 1.3.2.2 martin **********************************************************************/ 857 1.3.2.2 martin static void 858 1.3.2.2 martin igc_init_dmac(struct igc_softc *sc, uint32_t pba) 859 1.3.2.2 martin { 860 1.3.2.2 martin struct igc_hw *hw = &sc->hw; 861 1.3.2.2 martin const uint16_t max_frame_size = sc->hw.mac.max_frame_size; 862 1.3.2.2 martin uint32_t reg, status; 863 1.3.2.2 martin 864 1.3.2.2 martin if (sc->dmac == 0) { /* Disabling it */ 865 1.3.2.2 martin reg = ~IGC_DMACR_DMAC_EN; /* XXXRO */ 866 1.3.2.2 martin IGC_WRITE_REG(hw, IGC_DMACR, reg); 867 1.3.2.2 martin DPRINTF(MISC, "DMA coalescing disabled\n"); 868 1.3.2.2 martin return; 869 1.3.2.2 martin } else { 870 1.3.2.2 martin device_printf(sc->sc_dev, "DMA coalescing enabled\n"); 871 1.3.2.2 martin } 872 1.3.2.2 martin 873 1.3.2.2 martin /* Set starting threshold */ 874 1.3.2.2 martin IGC_WRITE_REG(hw, IGC_DMCTXTH, 0); 875 1.3.2.2 martin 876 1.3.2.2 martin uint16_t hwm = 64 * pba - max_frame_size / 16; 877 1.3.2.2 martin if (hwm < 64 * (pba - 6)) 878 1.3.2.2 martin hwm = 64 * (pba - 6); 879 1.3.2.2 martin reg = IGC_READ_REG(hw, IGC_FCRTC); 880 1.3.2.2 martin reg &= ~IGC_FCRTC_RTH_COAL_MASK; 881 1.3.2.2 martin reg |= (hwm << IGC_FCRTC_RTH_COAL_SHIFT) & IGC_FCRTC_RTH_COAL_MASK; 882 1.3.2.2 martin IGC_WRITE_REG(hw, IGC_FCRTC, reg); 883 1.3.2.2 martin 884 1.3.2.2 martin uint32_t dmac = pba - max_frame_size / 512; 885 1.3.2.2 martin if (dmac < pba - 10) 886 1.3.2.2 martin dmac = pba - 10; 887 1.3.2.2 martin reg = IGC_READ_REG(hw, IGC_DMACR); 888 1.3.2.2 martin reg &= ~IGC_DMACR_DMACTHR_MASK; 889 1.3.2.2 martin reg |= (dmac << IGC_DMACR_DMACTHR_SHIFT) & IGC_DMACR_DMACTHR_MASK; 890 1.3.2.2 martin 891 1.3.2.2 martin /* transition to L0x or L1 if available..*/ 892 1.3.2.2 martin reg |= IGC_DMACR_DMAC_EN | IGC_DMACR_DMAC_LX_MASK; 893 1.3.2.2 martin 894 1.3.2.2 martin /* Check if status is 2.5Gb backplane connection 895 1.3.2.2 martin * before configuration of watchdog timer, which is 896 1.3.2.2 martin * in msec values in 12.8usec intervals 897 1.3.2.2 martin * watchdog timer= msec values in 32usec intervals 898 1.3.2.2 martin * for non 2.5Gb connection 899 1.3.2.2 martin */ 900 1.3.2.2 martin status = IGC_READ_REG(hw, IGC_STATUS); 901 1.3.2.2 martin if ((status & IGC_STATUS_2P5_SKU) && 902 1.3.2.2 martin !(status & IGC_STATUS_2P5_SKU_OVER)) 903 1.3.2.2 martin reg |= (sc->dmac * 5) >> 6; 904 1.3.2.2 martin else 905 1.3.2.2 martin reg |= sc->dmac >> 5; 906 1.3.2.2 martin 907 1.3.2.2 martin IGC_WRITE_REG(hw, IGC_DMACR, reg); 908 1.3.2.2 martin 909 1.3.2.2 martin IGC_WRITE_REG(hw, IGC_DMCRTRH, 0); 910 1.3.2.2 martin 911 1.3.2.2 martin /* Set the interval before transition */ 912 1.3.2.2 martin reg = IGC_READ_REG(hw, IGC_DMCTLX); 913 1.3.2.2 martin reg |= IGC_DMCTLX_DCFLUSH_DIS; 914 1.3.2.2 martin 915 1.3.2.2 martin /* 916 1.3.2.2 martin * in 2.5Gb connection, TTLX unit is 0.4 usec 917 1.3.2.2 martin * which is 0x4*2 = 0xA. But delay is still 4 usec 918 1.3.2.2 martin */ 919 1.3.2.2 martin status = IGC_READ_REG(hw, IGC_STATUS); 920 1.3.2.2 martin if ((status & IGC_STATUS_2P5_SKU) && 921 1.3.2.2 martin !(status & IGC_STATUS_2P5_SKU_OVER)) 922 1.3.2.2 martin reg |= 0xA; 923 1.3.2.2 martin else 924 1.3.2.2 martin reg |= 0x4; 925 1.3.2.2 martin 926 1.3.2.2 martin IGC_WRITE_REG(hw, IGC_DMCTLX, reg); 927 1.3.2.2 martin 928 1.3.2.2 martin /* free space in tx packet buffer to wake from DMA coal */ 929 1.3.2.2 martin IGC_WRITE_REG(hw, IGC_DMCTXTH, 930 1.3.2.2 martin (IGC_TXPBSIZE - (2 * max_frame_size)) >> 6); 931 1.3.2.2 martin 932 1.3.2.2 martin /* make low power state decision controlled by DMA coal */ 933 1.3.2.2 martin reg = IGC_READ_REG(hw, IGC_PCIEMISC); 934 1.3.2.2 martin reg &= ~IGC_PCIEMISC_LX_DECISION; 935 1.3.2.2 martin IGC_WRITE_REG(hw, IGC_PCIEMISC, reg); 936 1.3.2.2 martin } 937 1.3.2.2 martin 938 1.3.2.2 martin static int 939 1.3.2.2 martin igc_setup_interrupts(struct igc_softc *sc) 940 1.3.2.2 martin { 941 1.3.2.2 martin int error; 942 1.3.2.2 martin 943 1.3.2.2 martin switch (sc->sc_intr_type) { 944 1.3.2.2 martin case PCI_INTR_TYPE_MSIX: 945 1.3.2.2 martin error = igc_setup_msix(sc); 946 1.3.2.2 martin break; 947 1.3.2.2 martin case PCI_INTR_TYPE_MSI: 948 1.3.2.2 martin error = igc_setup_msi(sc); 949 1.3.2.2 martin break; 950 1.3.2.2 martin case PCI_INTR_TYPE_INTX: 951 1.3.2.2 martin error = igc_setup_intx(sc); 952 1.3.2.2 martin break; 953 1.3.2.2 martin default: 954 1.3.2.2 martin panic("%s: invalid interrupt type: %d", 955 1.3.2.2 martin device_xname(sc->sc_dev), sc->sc_intr_type); 956 1.3.2.2 martin } 957 1.3.2.2 martin 958 1.3.2.2 martin return error; 959 1.3.2.2 martin } 960 1.3.2.2 martin 961 1.3.2.2 martin static void 962 1.3.2.2 martin igc_attach_counters(struct igc_softc *sc) 963 1.3.2.2 martin { 964 1.3.2.2 martin #ifdef IGC_EVENT_COUNTERS 965 1.3.2.2 martin 966 1.3.2.2 martin /* Global counters */ 967 1.3.2.2 martin sc->sc_global_evcnts = kmem_zalloc( 968 1.3.2.2 martin IGC_GLOBAL_COUNTERS * sizeof(sc->sc_global_evcnts[0]), KM_SLEEP); 969 1.3.2.2 martin 970 1.3.2.2 martin for (int cnt = 0; cnt < IGC_GLOBAL_COUNTERS; cnt++) { 971 1.3.2.2 martin evcnt_attach_dynamic(&sc->sc_global_evcnts[cnt], 972 1.3.2.2 martin igc_global_counters[cnt].type, NULL, 973 1.3.2.2 martin device_xname(sc->sc_dev), igc_global_counters[cnt].name); 974 1.3.2.2 martin } 975 1.3.2.2 martin 976 1.3.2.2 martin /* Driver counters */ 977 1.3.2.2 martin sc->sc_driver_evcnts = kmem_zalloc( 978 1.3.2.2 martin IGC_DRIVER_COUNTERS * sizeof(sc->sc_driver_evcnts[0]), KM_SLEEP); 979 1.3.2.2 martin 980 1.3.2.2 martin for (int cnt = 0; cnt < IGC_DRIVER_COUNTERS; cnt++) { 981 1.3.2.2 martin evcnt_attach_dynamic(&sc->sc_driver_evcnts[cnt], 982 1.3.2.2 martin igc_driver_counters[cnt].type, NULL, 983 1.3.2.2 martin device_xname(sc->sc_dev), igc_driver_counters[cnt].name); 984 1.3.2.2 martin } 985 1.3.2.2 martin 986 1.3.2.2 martin for (int iq = 0; iq < sc->sc_nqueues; iq++) { 987 1.3.2.2 martin struct igc_queue *q = &sc->queues[iq]; 988 1.3.2.2 martin 989 1.3.2.2 martin q->igcq_driver_counters = kmem_zalloc( 990 1.3.2.2 martin IGC_DRIVER_COUNTERS * sizeof(q->igcq_driver_counters[0]), 991 1.3.2.2 martin KM_SLEEP); 992 1.3.2.2 martin } 993 1.3.2.2 martin 994 1.3.2.2 martin /* Queue counters */ 995 1.3.2.2 martin for (int iq = 0; iq < sc->sc_nqueues; iq++) { 996 1.3.2.2 martin struct igc_queue *q = &sc->queues[iq]; 997 1.3.2.2 martin 998 1.3.2.2 martin snprintf(q->igcq_queue_evname, sizeof(q->igcq_queue_evname), 999 1.3.2.2 martin "%s q%d", device_xname(sc->sc_dev), iq); 1000 1.3.2.2 martin 1001 1.3.2.2 martin q->igcq_queue_evcnts = kmem_zalloc( 1002 1.3.2.2 martin IGC_QUEUE_COUNTERS * sizeof(q->igcq_queue_evcnts[0]), 1003 1.3.2.2 martin KM_SLEEP); 1004 1.3.2.2 martin 1005 1.3.2.2 martin for (int cnt = 0; cnt < IGC_QUEUE_COUNTERS; cnt++) { 1006 1.3.2.2 martin evcnt_attach_dynamic(&q->igcq_queue_evcnts[cnt], 1007 1.3.2.2 martin igc_queue_counters[cnt].type, NULL, 1008 1.3.2.2 martin q->igcq_queue_evname, igc_queue_counters[cnt].name); 1009 1.3.2.2 martin } 1010 1.3.2.2 martin } 1011 1.3.2.2 martin 1012 1.3.2.2 martin /* MAC counters */ 1013 1.3.2.2 martin snprintf(sc->sc_mac_evname, sizeof(sc->sc_mac_evname), 1014 1.3.2.2 martin "%s Mac Statistics", device_xname(sc->sc_dev)); 1015 1.3.2.2 martin 1016 1.3.2.2 martin sc->sc_mac_evcnts = kmem_zalloc( 1017 1.3.2.2 martin IGC_MAC_COUNTERS * sizeof(sc->sc_mac_evcnts[0]), KM_SLEEP); 1018 1.3.2.2 martin 1019 1.3.2.2 martin for (int cnt = 0; cnt < IGC_MAC_COUNTERS; cnt++) { 1020 1.3.2.2 martin evcnt_attach_dynamic(&sc->sc_mac_evcnts[cnt], EVCNT_TYPE_MISC, 1021 1.3.2.2 martin NULL, sc->sc_mac_evname, igc_mac_counters[cnt].name); 1022 1.3.2.2 martin } 1023 1.3.2.2 martin #endif 1024 1.3.2.2 martin } 1025 1.3.2.2 martin 1026 1.3.2.2 martin static void 1027 1.3.2.2 martin igc_detach_counters(struct igc_softc *sc) 1028 1.3.2.2 martin { 1029 1.3.2.2 martin #ifdef IGC_EVENT_COUNTERS 1030 1.3.2.2 martin 1031 1.3.2.2 martin /* Global counters */ 1032 1.3.2.2 martin for (int cnt = 0; cnt < IGC_GLOBAL_COUNTERS; cnt++) 1033 1.3.2.2 martin evcnt_detach(&sc->sc_global_evcnts[cnt]); 1034 1.3.2.2 martin 1035 1.3.2.2 martin kmem_free(sc->sc_global_evcnts, 1036 1.3.2.2 martin IGC_GLOBAL_COUNTERS * sizeof(sc->sc_global_evcnts)); 1037 1.3.2.2 martin 1038 1.3.2.2 martin /* Driver counters */ 1039 1.3.2.2 martin for (int iq = 0; iq < sc->sc_nqueues; iq++) { 1040 1.3.2.2 martin struct igc_queue *q = &sc->queues[iq]; 1041 1.3.2.2 martin 1042 1.3.2.2 martin kmem_free(q->igcq_driver_counters, 1043 1.3.2.2 martin IGC_DRIVER_COUNTERS * sizeof(q->igcq_driver_counters[0])); 1044 1.3.2.2 martin } 1045 1.3.2.2 martin 1046 1.3.2.2 martin for (int cnt = 0; cnt < IGC_DRIVER_COUNTERS; cnt++) 1047 1.3.2.2 martin evcnt_detach(&sc->sc_driver_evcnts[cnt]); 1048 1.3.2.2 martin 1049 1.3.2.2 martin kmem_free(sc->sc_driver_evcnts, 1050 1.3.2.2 martin IGC_DRIVER_COUNTERS * sizeof(sc->sc_driver_evcnts)); 1051 1.3.2.2 martin 1052 1.3.2.2 martin /* Queue counters */ 1053 1.3.2.2 martin for (int iq = 0; iq < sc->sc_nqueues; iq++) { 1054 1.3.2.2 martin struct igc_queue *q = &sc->queues[iq]; 1055 1.3.2.2 martin 1056 1.3.2.2 martin for (int cnt = 0; cnt < IGC_QUEUE_COUNTERS; cnt++) 1057 1.3.2.2 martin evcnt_detach(&q->igcq_queue_evcnts[cnt]); 1058 1.3.2.2 martin 1059 1.3.2.2 martin kmem_free(q->igcq_queue_evcnts, 1060 1.3.2.2 martin IGC_QUEUE_COUNTERS * sizeof(q->igcq_queue_evcnts[0])); 1061 1.3.2.2 martin } 1062 1.3.2.2 martin 1063 1.3.2.2 martin /* MAC statistics */ 1064 1.3.2.2 martin for (int cnt = 0; cnt < IGC_MAC_COUNTERS; cnt++) 1065 1.3.2.2 martin evcnt_detach(&sc->sc_mac_evcnts[cnt]); 1066 1.3.2.2 martin 1067 1.3.2.2 martin kmem_free(sc->sc_mac_evcnts, 1068 1.3.2.2 martin IGC_MAC_COUNTERS * sizeof(sc->sc_mac_evcnts[0])); 1069 1.3.2.2 martin #endif 1070 1.3.2.2 martin } 1071 1.3.2.2 martin 1072 1.3.2.2 martin /* 1073 1.3.2.2 martin * XXX 1074 1.3.2.2 martin * FreeBSD uses 4-byte-wise read for 64-bit counters, while Linux just 1075 1.3.2.2 martin * drops hi words. 1076 1.3.2.2 martin */ 1077 1.3.2.2 martin static inline uint64_t __unused 1078 1.3.2.2 martin igc_read_mac_counter(struct igc_hw *hw, bus_size_t reg, bool is64) 1079 1.3.2.2 martin { 1080 1.3.2.2 martin uint64_t val; 1081 1.3.2.2 martin 1082 1.3.2.2 martin val = IGC_READ_REG(hw, reg); 1083 1.3.2.2 martin if (is64) 1084 1.3.2.2 martin val += ((uint64_t)IGC_READ_REG(hw, reg + 4)) << 32; 1085 1.3.2.2 martin return val; 1086 1.3.2.2 martin } 1087 1.3.2.2 martin 1088 1.3.2.2 martin static void 1089 1.3.2.2 martin igc_update_counters(struct igc_softc *sc) 1090 1.3.2.2 martin { 1091 1.3.2.2 martin #ifdef IGC_EVENT_COUNTERS 1092 1.3.2.2 martin 1093 1.3.2.2 martin /* Global counters: nop */ 1094 1.3.2.2 martin 1095 1.3.2.2 martin /* Driver counters */ 1096 1.3.2.2 martin uint64_t sum[IGC_DRIVER_COUNTERS]; 1097 1.3.2.2 martin 1098 1.3.2.2 martin memset(sum, 0, sizeof(sum)); 1099 1.3.2.2 martin for (int iq = 0; iq < sc->sc_nqueues; iq++) { 1100 1.3.2.2 martin struct igc_queue *q = &sc->queues[iq]; 1101 1.3.2.2 martin 1102 1.3.2.2 martin for (int cnt = 0; cnt < IGC_DRIVER_COUNTERS; cnt++) { 1103 1.3.2.2 martin sum[cnt] += IGC_QUEUE_DRIVER_COUNTER_VAL(q, cnt); 1104 1.3.2.2 martin IGC_QUEUE_DRIVER_COUNTER_STORE(q, cnt, 0); 1105 1.3.2.2 martin } 1106 1.3.2.2 martin } 1107 1.3.2.2 martin 1108 1.3.2.2 martin for (int cnt = 0; cnt < IGC_DRIVER_COUNTERS; cnt++) 1109 1.3.2.2 martin IGC_DRIVER_COUNTER_ADD(sc, cnt, sum[cnt]); 1110 1.3.2.2 martin 1111 1.3.2.2 martin /* Queue counters: nop */ 1112 1.3.2.2 martin 1113 1.3.2.2 martin /* Mac statistics */ 1114 1.3.2.2 martin struct igc_hw *hw = &sc->hw; 1115 1.3.2.4 martin struct ifnet *ifp = &sc->sc_ec.ec_if; 1116 1.3.2.4 martin uint64_t iqdrops = 0; 1117 1.3.2.2 martin 1118 1.3.2.2 martin for (int cnt = 0; cnt < IGC_MAC_COUNTERS; cnt++) { 1119 1.3.2.4 martin uint64_t val; 1120 1.3.2.4 martin bus_size_t regaddr = igc_mac_counters[cnt].reg; 1121 1.3.2.4 martin 1122 1.3.2.4 martin val = igc_read_mac_counter(hw, regaddr, 1123 1.3.2.4 martin igc_mac_counters[cnt].is64); 1124 1.3.2.4 martin IGC_MAC_COUNTER_ADD(sc, cnt, val); 1125 1.3.2.4 martin /* XXX Count MPC to iqdrops. */ 1126 1.3.2.4 martin if (regaddr == IGC_MPC) 1127 1.3.2.4 martin iqdrops += val; 1128 1.3.2.4 martin } 1129 1.3.2.4 martin 1130 1.3.2.4 martin for (int iq = 0; iq < sc->sc_nqueues; iq++) { 1131 1.3.2.4 martin uint32_t val; 1132 1.3.2.4 martin 1133 1.3.2.4 martin /* XXX RQDPC should be visible via evcnt(9). */ 1134 1.3.2.4 martin val = IGC_READ_REG(hw, IGC_RQDPC(iq)); 1135 1.3.2.4 martin 1136 1.3.2.4 martin /* RQDPC is not cleard on read. */ 1137 1.3.2.4 martin if (val != 0) 1138 1.3.2.4 martin IGC_WRITE_REG(hw, IGC_RQDPC(iq), 0); 1139 1.3.2.4 martin iqdrops += val; 1140 1.3.2.2 martin } 1141 1.3.2.4 martin 1142 1.3.2.4 martin if (iqdrops != 0) 1143 1.3.2.4 martin if_statadd(ifp, if_iqdrops, iqdrops); 1144 1.3.2.2 martin #endif 1145 1.3.2.2 martin } 1146 1.3.2.2 martin 1147 1.3.2.2 martin static void 1148 1.3.2.2 martin igc_clear_counters(struct igc_softc *sc) 1149 1.3.2.2 martin { 1150 1.3.2.2 martin #ifdef IGC_EVENT_COUNTERS 1151 1.3.2.2 martin 1152 1.3.2.2 martin /* Global counters */ 1153 1.3.2.2 martin for (int cnt = 0; cnt < IGC_GLOBAL_COUNTERS; cnt++) 1154 1.3.2.2 martin IGC_GLOBAL_COUNTER_STORE(sc, cnt, 0); 1155 1.3.2.2 martin 1156 1.3.2.2 martin /* Driver counters */ 1157 1.3.2.2 martin for (int iq = 0; iq < sc->sc_nqueues; iq++) { 1158 1.3.2.2 martin struct igc_queue *q = &sc->queues[iq]; 1159 1.3.2.2 martin 1160 1.3.2.2 martin for (int cnt = 0; cnt < IGC_DRIVER_COUNTERS; cnt++) 1161 1.3.2.2 martin IGC_QUEUE_DRIVER_COUNTER_STORE(q, cnt, 0); 1162 1.3.2.2 martin } 1163 1.3.2.2 martin 1164 1.3.2.2 martin for (int cnt = 0; cnt < IGC_DRIVER_COUNTERS; cnt++) 1165 1.3.2.2 martin IGC_DRIVER_COUNTER_STORE(sc, cnt, 0); 1166 1.3.2.2 martin 1167 1.3.2.2 martin /* Queue counters */ 1168 1.3.2.2 martin for (int iq = 0; iq < sc->sc_nqueues; iq++) { 1169 1.3.2.2 martin struct igc_queue *q = &sc->queues[iq]; 1170 1.3.2.2 martin 1171 1.3.2.2 martin for (int cnt = 0; cnt < IGC_QUEUE_COUNTERS; cnt++) 1172 1.3.2.2 martin IGC_QUEUE_COUNTER_STORE(q, cnt, 0); 1173 1.3.2.2 martin } 1174 1.3.2.2 martin 1175 1.3.2.2 martin /* Mac statistics */ 1176 1.3.2.2 martin struct igc_hw *hw = &sc->hw; 1177 1.3.2.2 martin 1178 1.3.2.2 martin for (int cnt = 0; cnt < IGC_MAC_COUNTERS; cnt++) { 1179 1.3.2.2 martin (void)igc_read_mac_counter(hw, igc_mac_counters[cnt].reg, 1180 1.3.2.2 martin igc_mac_counters[cnt].is64); 1181 1.3.2.2 martin IGC_MAC_COUNTER_STORE(sc, cnt, 0); 1182 1.3.2.2 martin } 1183 1.3.2.2 martin #endif 1184 1.3.2.2 martin } 1185 1.3.2.2 martin 1186 1.3.2.2 martin static int 1187 1.3.2.2 martin igc_setup_msix(struct igc_softc *sc) 1188 1.3.2.2 martin { 1189 1.3.2.2 martin pci_chipset_tag_t pc = sc->osdep.os_pa.pa_pc; 1190 1.3.2.2 martin device_t dev = sc->sc_dev; 1191 1.3.2.2 martin pci_intr_handle_t *intrs; 1192 1.3.2.2 martin void **ihs; 1193 1.3.2.2 martin const char *intrstr; 1194 1.3.2.2 martin char intrbuf[PCI_INTRSTR_LEN]; 1195 1.3.2.2 martin char xnamebuf[MAX(32, MAXCOMLEN)]; 1196 1.3.2.2 martin int iq, error; 1197 1.3.2.2 martin 1198 1.3.2.2 martin for (iq = 0, intrs = sc->sc_intrs, ihs = sc->sc_ihs; 1199 1.3.2.2 martin iq < sc->sc_nqueues; iq++, intrs++, ihs++) { 1200 1.3.2.2 martin struct igc_queue *q = &sc->queues[iq]; 1201 1.3.2.2 martin 1202 1.3.2.2 martin snprintf(xnamebuf, sizeof(xnamebuf), "%s: txrx %d", 1203 1.3.2.2 martin device_xname(dev), iq); 1204 1.3.2.2 martin 1205 1.3.2.2 martin intrstr = pci_intr_string(pc, *intrs, intrbuf, sizeof(intrbuf)); 1206 1.3.2.2 martin 1207 1.3.2.2 martin pci_intr_setattr(pc, intrs, PCI_INTR_MPSAFE, true); 1208 1.3.2.2 martin *ihs = pci_intr_establish_xname(pc, *intrs, IPL_NET, 1209 1.3.2.2 martin igc_intr_queue, q, xnamebuf); 1210 1.3.2.2 martin if (*ihs == NULL) { 1211 1.3.2.2 martin aprint_error_dev(dev, 1212 1.3.2.2 martin "unable to establish txrx interrupt at %s\n", 1213 1.3.2.2 martin intrstr); 1214 1.3.2.2 martin return ENOBUFS; 1215 1.3.2.2 martin } 1216 1.3.2.2 martin aprint_normal_dev(dev, "txrx interrupting at %s\n", intrstr); 1217 1.3.2.2 martin 1218 1.3.2.2 martin kcpuset_t *affinity; 1219 1.3.2.2 martin kcpuset_create(&affinity, true); 1220 1.3.2.2 martin kcpuset_set(affinity, iq % ncpu); 1221 1.3.2.2 martin error = interrupt_distribute(*ihs, affinity, NULL); 1222 1.3.2.2 martin if (error) { 1223 1.3.2.2 martin aprint_normal_dev(dev, 1224 1.3.2.2 martin "%s: unable to change affinity, use default CPU\n", 1225 1.3.2.2 martin intrstr); 1226 1.3.2.2 martin } 1227 1.3.2.2 martin kcpuset_destroy(affinity); 1228 1.3.2.2 martin 1229 1.3.2.2 martin q->igcq_si = softint_establish(SOFTINT_NET | SOFTINT_MPSAFE, 1230 1.3.2.2 martin igc_handle_queue, q); 1231 1.3.2.2 martin if (q->igcq_si == NULL) { 1232 1.3.2.2 martin aprint_error_dev(dev, 1233 1.3.2.2 martin "%s: unable to establish softint\n", intrstr); 1234 1.3.2.2 martin return ENOBUFS; 1235 1.3.2.2 martin } 1236 1.3.2.2 martin 1237 1.3.2.2 martin q->msix = iq; 1238 1.3.2.2 martin q->eims = 1 << iq; 1239 1.3.2.2 martin } 1240 1.3.2.2 martin 1241 1.3.2.2 martin snprintf(xnamebuf, MAXCOMLEN, "%s_tx_rx", device_xname(dev)); 1242 1.3.2.2 martin error = workqueue_create(&sc->sc_queue_wq, xnamebuf, 1243 1.3.2.2 martin igc_handle_queue_work, sc, IGC_WORKQUEUE_PRI, IPL_NET, 1244 1.3.2.2 martin WQ_PERCPU | WQ_MPSAFE); 1245 1.3.2.2 martin if (error) { 1246 1.3.2.2 martin aprint_error_dev(dev, "workqueue_create failed\n"); 1247 1.3.2.2 martin return ENOBUFS; 1248 1.3.2.2 martin } 1249 1.3.2.2 martin sc->sc_txrx_workqueue = false; 1250 1.3.2.2 martin 1251 1.3.2.2 martin intrstr = pci_intr_string(pc, *intrs, intrbuf, sizeof(intrbuf)); 1252 1.3.2.2 martin snprintf(xnamebuf, sizeof(xnamebuf), "%s: link", device_xname(dev)); 1253 1.3.2.2 martin pci_intr_setattr(pc, intrs, PCI_INTR_MPSAFE, true); 1254 1.3.2.2 martin *ihs = pci_intr_establish_xname(pc, *intrs, IPL_NET, 1255 1.3.2.2 martin igc_intr_link, sc, xnamebuf); 1256 1.3.2.2 martin if (*ihs == NULL) { 1257 1.3.2.2 martin aprint_error_dev(dev, 1258 1.3.2.2 martin "unable to establish link interrupt at %s\n", intrstr); 1259 1.3.2.2 martin return ENOBUFS; 1260 1.3.2.2 martin } 1261 1.3.2.2 martin aprint_normal_dev(dev, "link interrupting at %s\n", intrstr); 1262 1.3.2.2 martin /* use later in igc_configure_queues() */ 1263 1.3.2.2 martin sc->linkvec = iq; 1264 1.3.2.2 martin 1265 1.3.2.2 martin return 0; 1266 1.3.2.2 martin } 1267 1.3.2.2 martin 1268 1.3.2.2 martin static int 1269 1.3.2.2 martin igc_setup_msi(struct igc_softc *sc) 1270 1.3.2.2 martin { 1271 1.3.2.2 martin pci_chipset_tag_t pc = sc->osdep.os_pa.pa_pc; 1272 1.3.2.2 martin device_t dev = sc->sc_dev; 1273 1.3.2.2 martin pci_intr_handle_t *intr = sc->sc_intrs; 1274 1.3.2.2 martin void **ihs = sc->sc_ihs; 1275 1.3.2.2 martin const char *intrstr; 1276 1.3.2.2 martin char intrbuf[PCI_INTRSTR_LEN]; 1277 1.3.2.2 martin char xnamebuf[MAX(32, MAXCOMLEN)]; 1278 1.3.2.2 martin int error; 1279 1.3.2.2 martin 1280 1.3.2.2 martin intrstr = pci_intr_string(pc, *intr, intrbuf, sizeof(intrbuf)); 1281 1.3.2.2 martin 1282 1.3.2.2 martin snprintf(xnamebuf, sizeof(xnamebuf), "%s: msi", device_xname(dev)); 1283 1.3.2.2 martin pci_intr_setattr(pc, intr, PCI_INTR_MPSAFE, true); 1284 1.3.2.2 martin *ihs = pci_intr_establish_xname(pc, *intr, IPL_NET, 1285 1.3.2.2 martin igc_intr, sc, xnamebuf); 1286 1.3.2.2 martin if (*ihs == NULL) { 1287 1.3.2.2 martin aprint_error_dev(dev, 1288 1.3.2.2 martin "unable to establish interrupt at %s\n", intrstr); 1289 1.3.2.2 martin return ENOBUFS; 1290 1.3.2.2 martin } 1291 1.3.2.2 martin aprint_normal_dev(dev, "interrupting at %s\n", intrstr); 1292 1.3.2.2 martin 1293 1.3.2.2 martin struct igc_queue *iq = sc->queues; 1294 1.3.2.2 martin iq->igcq_si = softint_establish(SOFTINT_NET | SOFTINT_MPSAFE, 1295 1.3.2.2 martin igc_handle_queue, iq); 1296 1.3.2.2 martin if (iq->igcq_si == NULL) { 1297 1.3.2.2 martin aprint_error_dev(dev, 1298 1.3.2.2 martin "%s: unable to establish softint\n", intrstr); 1299 1.3.2.2 martin return ENOBUFS; 1300 1.3.2.2 martin } 1301 1.3.2.2 martin 1302 1.3.2.2 martin snprintf(xnamebuf, MAXCOMLEN, "%s_tx_rx", device_xname(dev)); 1303 1.3.2.2 martin error = workqueue_create(&sc->sc_queue_wq, xnamebuf, 1304 1.3.2.2 martin igc_handle_queue_work, sc, IGC_WORKQUEUE_PRI, IPL_NET, 1305 1.3.2.2 martin WQ_PERCPU | WQ_MPSAFE); 1306 1.3.2.2 martin if (error) { 1307 1.3.2.2 martin aprint_error_dev(dev, "workqueue_create failed\n"); 1308 1.3.2.2 martin return ENOBUFS; 1309 1.3.2.2 martin } 1310 1.3.2.2 martin sc->sc_txrx_workqueue = false; 1311 1.3.2.2 martin 1312 1.3.2.2 martin sc->queues[0].msix = 0; 1313 1.3.2.2 martin sc->linkvec = 0; 1314 1.3.2.2 martin 1315 1.3.2.2 martin return 0; 1316 1.3.2.2 martin } 1317 1.3.2.2 martin 1318 1.3.2.2 martin static int 1319 1.3.2.2 martin igc_setup_intx(struct igc_softc *sc) 1320 1.3.2.2 martin { 1321 1.3.2.2 martin pci_chipset_tag_t pc = sc->osdep.os_pa.pa_pc; 1322 1.3.2.2 martin device_t dev = sc->sc_dev; 1323 1.3.2.2 martin pci_intr_handle_t *intr = sc->sc_intrs; 1324 1.3.2.2 martin void **ihs = sc->sc_ihs; 1325 1.3.2.2 martin const char *intrstr; 1326 1.3.2.2 martin char intrbuf[PCI_INTRSTR_LEN]; 1327 1.3.2.2 martin char xnamebuf[32]; 1328 1.3.2.2 martin 1329 1.3.2.2 martin intrstr = pci_intr_string(pc, *intr, intrbuf, sizeof(intrbuf)); 1330 1.3.2.2 martin 1331 1.3.2.2 martin snprintf(xnamebuf, sizeof(xnamebuf), "%s:intx", device_xname(dev)); 1332 1.3.2.2 martin pci_intr_setattr(pc, intr, PCI_INTR_MPSAFE, true); 1333 1.3.2.2 martin *ihs = pci_intr_establish_xname(pc, *intr, IPL_NET, 1334 1.3.2.2 martin igc_intr, sc, xnamebuf); 1335 1.3.2.2 martin if (*ihs == NULL) { 1336 1.3.2.2 martin aprint_error_dev(dev, 1337 1.3.2.2 martin "unable to establish interrupt at %s\n", intrstr); 1338 1.3.2.2 martin return ENOBUFS; 1339 1.3.2.2 martin } 1340 1.3.2.2 martin aprint_normal_dev(dev, "interrupting at %s\n", intrstr); 1341 1.3.2.2 martin 1342 1.3.2.2 martin struct igc_queue *iq = sc->queues; 1343 1.3.2.2 martin iq->igcq_si = softint_establish(SOFTINT_NET | SOFTINT_MPSAFE, 1344 1.3.2.2 martin igc_handle_queue, iq); 1345 1.3.2.2 martin if (iq->igcq_si == NULL) { 1346 1.3.2.2 martin aprint_error_dev(dev, 1347 1.3.2.2 martin "%s: unable to establish softint\n", intrstr); 1348 1.3.2.2 martin return ENOBUFS; 1349 1.3.2.2 martin } 1350 1.3.2.2 martin 1351 1.3.2.2 martin /* create workqueue? */ 1352 1.3.2.2 martin sc->sc_txrx_workqueue = false; 1353 1.3.2.2 martin 1354 1.3.2.2 martin sc->queues[0].msix = 0; 1355 1.3.2.2 martin sc->linkvec = 0; 1356 1.3.2.2 martin 1357 1.3.2.2 martin return 0; 1358 1.3.2.2 martin } 1359 1.3.2.2 martin 1360 1.3.2.2 martin static int 1361 1.3.2.2 martin igc_dma_malloc(struct igc_softc *sc, bus_size_t size, struct igc_dma_alloc *dma) 1362 1.3.2.2 martin { 1363 1.3.2.2 martin struct igc_osdep *os = &sc->osdep; 1364 1.3.2.2 martin 1365 1.3.2.2 martin dma->dma_tag = os->os_dmat; 1366 1.3.2.2 martin 1367 1.3.2.2 martin if (bus_dmamap_create(dma->dma_tag, size, 1, size, 0, 1368 1.3.2.2 martin BUS_DMA_WAITOK | BUS_DMA_ALLOCNOW, &dma->dma_map)) 1369 1.3.2.2 martin return 1; 1370 1.3.2.2 martin if (bus_dmamem_alloc(dma->dma_tag, size, PAGE_SIZE, 0, &dma->dma_seg, 1371 1.3.2.2 martin 1, &dma->dma_nseg, BUS_DMA_WAITOK)) 1372 1.3.2.2 martin goto destroy; 1373 1.3.2.2 martin /* 1374 1.3.2.2 martin * XXXRO 1375 1.3.2.2 martin * 1376 1.3.2.2 martin * Coherent mapping for descriptors is required for now. 1377 1.3.2.2 martin * 1378 1.3.2.2 martin * Both TX and RX descriptors are 16-byte length, which is shorter 1379 1.3.2.2 martin * than dcache lines on modern CPUs. Therefore, sync for a descriptor 1380 1.3.2.2 martin * may overwrite DMA read for descriptors in the same cache line. 1381 1.3.2.2 martin * 1382 1.3.2.2 martin * Can't we avoid this by use cache-line-aligned descriptors at once? 1383 1.3.2.2 martin */ 1384 1.3.2.2 martin if (bus_dmamem_map(dma->dma_tag, &dma->dma_seg, dma->dma_nseg, size, 1385 1.3.2.2 martin &dma->dma_vaddr, BUS_DMA_WAITOK | BUS_DMA_COHERENT /* XXXRO */)) 1386 1.3.2.2 martin goto free; 1387 1.3.2.2 martin if (bus_dmamap_load(dma->dma_tag, dma->dma_map, dma->dma_vaddr, size, 1388 1.3.2.2 martin NULL, BUS_DMA_WAITOK)) 1389 1.3.2.2 martin goto unmap; 1390 1.3.2.2 martin 1391 1.3.2.2 martin dma->dma_size = size; 1392 1.3.2.2 martin 1393 1.3.2.2 martin return 0; 1394 1.3.2.2 martin unmap: 1395 1.3.2.2 martin bus_dmamem_unmap(dma->dma_tag, dma->dma_vaddr, size); 1396 1.3.2.2 martin free: 1397 1.3.2.2 martin bus_dmamem_free(dma->dma_tag, &dma->dma_seg, dma->dma_nseg); 1398 1.3.2.2 martin destroy: 1399 1.3.2.2 martin bus_dmamap_destroy(dma->dma_tag, dma->dma_map); 1400 1.3.2.2 martin dma->dma_map = NULL; 1401 1.3.2.2 martin dma->dma_tag = NULL; 1402 1.3.2.2 martin return 1; 1403 1.3.2.2 martin } 1404 1.3.2.2 martin 1405 1.3.2.2 martin static void 1406 1.3.2.2 martin igc_dma_free(struct igc_softc *sc, struct igc_dma_alloc *dma) 1407 1.3.2.2 martin { 1408 1.3.2.2 martin 1409 1.3.2.2 martin if (dma->dma_tag == NULL) 1410 1.3.2.2 martin return; 1411 1.3.2.2 martin 1412 1.3.2.2 martin if (dma->dma_map != NULL) { 1413 1.3.2.2 martin bus_dmamap_sync(dma->dma_tag, dma->dma_map, 0, 1414 1.3.2.2 martin dma->dma_map->dm_mapsize, 1415 1.3.2.2 martin BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE); 1416 1.3.2.2 martin bus_dmamap_unload(dma->dma_tag, dma->dma_map); 1417 1.3.2.2 martin bus_dmamem_unmap(dma->dma_tag, dma->dma_vaddr, dma->dma_size); 1418 1.3.2.2 martin bus_dmamem_free(dma->dma_tag, &dma->dma_seg, dma->dma_nseg); 1419 1.3.2.2 martin bus_dmamap_destroy(dma->dma_tag, dma->dma_map); 1420 1.3.2.2 martin dma->dma_map = NULL; 1421 1.3.2.2 martin } 1422 1.3.2.2 martin } 1423 1.3.2.2 martin 1424 1.3.2.2 martin /********************************************************************* 1425 1.3.2.2 martin * 1426 1.3.2.2 martin * Setup networking device structure and register an interface. 1427 1.3.2.2 martin * 1428 1.3.2.2 martin **********************************************************************/ 1429 1.3.2.2 martin static void 1430 1.3.2.2 martin igc_setup_interface(struct igc_softc *sc) 1431 1.3.2.2 martin { 1432 1.3.2.2 martin struct ifnet *ifp = &sc->sc_ec.ec_if; 1433 1.3.2.2 martin 1434 1.3.2.2 martin strlcpy(ifp->if_xname, device_xname(sc->sc_dev), sizeof(ifp->if_xname)); 1435 1.3.2.2 martin ifp->if_softc = sc; 1436 1.3.2.2 martin ifp->if_flags = IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST; 1437 1.3.2.2 martin ifp->if_extflags = IFEF_MPSAFE; 1438 1.3.2.2 martin ifp->if_ioctl = igc_ioctl; 1439 1.3.2.2 martin ifp->if_start = igc_start; 1440 1.3.2.2 martin if (sc->sc_nqueues > 1) 1441 1.3.2.2 martin ifp->if_transmit = igc_transmit; 1442 1.3.2.2 martin ifp->if_watchdog = igc_watchdog; 1443 1.3.2.2 martin ifp->if_init = igc_init; 1444 1.3.2.2 martin ifp->if_stop = igc_stop; 1445 1.3.2.2 martin 1446 1.3.2.2 martin #if 0 /* notyet */ 1447 1.3.2.2 martin ifp->if_capabilities = IFCAP_TSOv4 | IFCAP_TSOv6; 1448 1.3.2.2 martin #endif 1449 1.3.2.2 martin 1450 1.3.2.2 martin ifp->if_capabilities |= 1451 1.3.2.2 martin IFCAP_CSUM_IPv4_Tx | IFCAP_CSUM_IPv4_Rx | 1452 1.3.2.2 martin IFCAP_CSUM_TCPv4_Tx | IFCAP_CSUM_TCPv4_Rx | 1453 1.3.2.2 martin IFCAP_CSUM_UDPv4_Tx | IFCAP_CSUM_UDPv4_Rx | 1454 1.3.2.2 martin IFCAP_CSUM_TCPv6_Tx | IFCAP_CSUM_TCPv6_Rx | 1455 1.3.2.2 martin IFCAP_CSUM_UDPv6_Tx | IFCAP_CSUM_UDPv6_Rx; 1456 1.3.2.2 martin 1457 1.3.2.2 martin ifp->if_capenable = 0; 1458 1.3.2.2 martin 1459 1.3.2.2 martin sc->sc_ec.ec_capabilities |= 1460 1.3.2.2 martin ETHERCAP_JUMBO_MTU | ETHERCAP_VLAN_MTU; 1461 1.3.2.2 martin 1462 1.3.2.2 martin IFQ_SET_MAXLEN(&ifp->if_snd, sc->num_tx_desc - 1); 1463 1.3.2.2 martin IFQ_SET_READY(&ifp->if_snd); 1464 1.3.2.2 martin 1465 1.3.2.2 martin #if NVLAN > 0 1466 1.3.2.2 martin sc->sc_ec.ec_capabilities |= ETHERCAP_VLAN_HWTAGGING; 1467 1.3.2.2 martin #endif 1468 1.3.2.2 martin 1469 1.3.2.2 martin mutex_init(&sc->sc_core_lock, MUTEX_DEFAULT, IPL_NET); 1470 1.3.2.2 martin 1471 1.3.2.2 martin /* Initialize ifmedia structures. */ 1472 1.3.2.2 martin sc->sc_ec.ec_ifmedia = &sc->media; 1473 1.3.2.2 martin ifmedia_init_with_lock(&sc->media, IFM_IMASK, igc_media_change, 1474 1.3.2.2 martin igc_media_status, &sc->sc_core_lock); 1475 1.3.2.2 martin ifmedia_add(&sc->media, IFM_ETHER | IFM_10_T, 0, NULL); 1476 1.3.2.2 martin ifmedia_add(&sc->media, IFM_ETHER | IFM_10_T | IFM_FDX, 0, NULL); 1477 1.3.2.2 martin ifmedia_add(&sc->media, IFM_ETHER | IFM_100_TX, 0, NULL); 1478 1.3.2.2 martin ifmedia_add(&sc->media, IFM_ETHER | IFM_100_TX | IFM_FDX, 0, NULL); 1479 1.3.2.2 martin ifmedia_add(&sc->media, IFM_ETHER | IFM_1000_T | IFM_FDX, 0, NULL); 1480 1.3.2.2 martin ifmedia_add(&sc->media, IFM_ETHER | IFM_2500_T | IFM_FDX, 0, NULL); 1481 1.3.2.2 martin ifmedia_add(&sc->media, IFM_ETHER | IFM_AUTO, 0, NULL); 1482 1.3.2.2 martin ifmedia_set(&sc->media, IFM_ETHER | IFM_AUTO); 1483 1.3.2.2 martin 1484 1.3.2.2 martin sc->sc_rx_intr_process_limit = IGC_RX_INTR_PROCESS_LIMIT_DEFAULT; 1485 1.3.2.2 martin sc->sc_tx_intr_process_limit = IGC_TX_INTR_PROCESS_LIMIT_DEFAULT; 1486 1.3.2.2 martin sc->sc_rx_process_limit = IGC_RX_PROCESS_LIMIT_DEFAULT; 1487 1.3.2.2 martin sc->sc_tx_process_limit = IGC_TX_PROCESS_LIMIT_DEFAULT; 1488 1.3.2.2 martin 1489 1.3.2.2 martin if_initialize(ifp); 1490 1.3.2.2 martin sc->sc_ipq = if_percpuq_create(ifp); 1491 1.3.2.2 martin if_deferred_start_init(ifp, NULL); 1492 1.3.2.2 martin ether_ifattach(ifp, sc->hw.mac.addr); 1493 1.3.2.2 martin ether_set_ifflags_cb(&sc->sc_ec, igc_ifflags_cb); 1494 1.3.2.2 martin if_register(ifp); 1495 1.3.2.2 martin } 1496 1.3.2.2 martin 1497 1.3.2.2 martin static int 1498 1.3.2.2 martin igc_init(struct ifnet *ifp) 1499 1.3.2.2 martin { 1500 1.3.2.2 martin struct igc_softc *sc = ifp->if_softc; 1501 1.3.2.2 martin int error; 1502 1.3.2.2 martin 1503 1.3.2.2 martin mutex_enter(&sc->sc_core_lock); 1504 1.3.2.2 martin error = igc_init_locked(sc); 1505 1.3.2.2 martin mutex_exit(&sc->sc_core_lock); 1506 1.3.2.2 martin 1507 1.3.2.2 martin return error; 1508 1.3.2.2 martin } 1509 1.3.2.2 martin 1510 1.3.2.2 martin static int 1511 1.3.2.2 martin igc_init_locked(struct igc_softc *sc) 1512 1.3.2.2 martin { 1513 1.3.2.2 martin struct ethercom *ec = &sc->sc_ec; 1514 1.3.2.2 martin struct ifnet *ifp = &ec->ec_if; 1515 1.3.2.2 martin 1516 1.3.2.2 martin DPRINTF(CFG, "called\n"); 1517 1.3.2.2 martin 1518 1.3.2.2 martin KASSERT(mutex_owned(&sc->sc_core_lock)); 1519 1.3.2.2 martin 1520 1.3.2.2 martin if (ISSET(ifp->if_flags, IFF_RUNNING)) 1521 1.3.2.2 martin igc_stop_locked(sc); 1522 1.3.2.2 martin 1523 1.3.2.2 martin /* Put the address into the receive address array. */ 1524 1.3.2.2 martin igc_rar_set(&sc->hw, sc->hw.mac.addr, 0); 1525 1.3.2.2 martin 1526 1.3.2.2 martin /* Initialize the hardware. */ 1527 1.3.2.2 martin igc_reset(sc); 1528 1.3.2.2 martin igc_update_link_status(sc); 1529 1.3.2.2 martin 1530 1.3.2.2 martin /* Setup VLAN support, basic and offload if available. */ 1531 1.3.2.2 martin IGC_WRITE_REG(&sc->hw, IGC_VET, ETHERTYPE_VLAN); 1532 1.3.2.2 martin 1533 1.3.2.2 martin igc_initialize_transmit_unit(sc); 1534 1.3.2.2 martin igc_initialize_receive_unit(sc); 1535 1.3.2.2 martin 1536 1.3.2.2 martin if (ec->ec_capenable & ETHERCAP_VLAN_HWTAGGING) { 1537 1.3.2.2 martin uint32_t ctrl = IGC_READ_REG(&sc->hw, IGC_CTRL); 1538 1.3.2.2 martin ctrl |= IGC_CTRL_VME; 1539 1.3.2.2 martin IGC_WRITE_REG(&sc->hw, IGC_CTRL, ctrl); 1540 1.3.2.2 martin } 1541 1.3.2.2 martin 1542 1.3.2.2 martin /* Setup multicast table. */ 1543 1.3.2.2 martin igc_set_filter(sc); 1544 1.3.2.2 martin 1545 1.3.2.2 martin igc_clear_hw_cntrs_base_generic(&sc->hw); 1546 1.3.2.2 martin 1547 1.3.2.2 martin if (sc->sc_intr_type == PCI_INTR_TYPE_MSIX) 1548 1.3.2.2 martin igc_configure_queues(sc); 1549 1.3.2.2 martin 1550 1.3.2.2 martin /* This clears any pending interrupts */ 1551 1.3.2.2 martin IGC_READ_REG(&sc->hw, IGC_ICR); 1552 1.3.2.2 martin IGC_WRITE_REG(&sc->hw, IGC_ICS, IGC_ICS_LSC); 1553 1.3.2.2 martin 1554 1.3.2.2 martin /* The driver can now take control from firmware. */ 1555 1.3.2.2 martin igc_get_hw_control(sc); 1556 1.3.2.2 martin 1557 1.3.2.2 martin /* Set Energy Efficient Ethernet. */ 1558 1.3.2.2 martin igc_set_eee_i225(&sc->hw, true, true, true); 1559 1.3.2.2 martin 1560 1.3.2.2 martin for (int iq = 0; iq < sc->sc_nqueues; iq++) { 1561 1.3.2.2 martin struct rx_ring *rxr = &sc->rx_rings[iq]; 1562 1.3.2.2 martin 1563 1.3.2.2 martin mutex_enter(&rxr->rxr_lock); 1564 1.3.2.2 martin igc_rxfill(rxr); 1565 1.3.2.2 martin mutex_exit(&rxr->rxr_lock); 1566 1.3.2.2 martin } 1567 1.3.2.2 martin 1568 1.3.2.2 martin sc->sc_core_stopping = false; 1569 1.3.2.2 martin 1570 1.3.2.2 martin ifp->if_flags |= IFF_RUNNING; 1571 1.3.2.2 martin 1572 1.3.2.2 martin /* Save last flags for the callback */ 1573 1.3.2.2 martin sc->sc_if_flags = ifp->if_flags; 1574 1.3.2.2 martin 1575 1.3.2.3 martin callout_schedule(&sc->sc_tick_ch, hz); 1576 1.3.2.3 martin 1577 1.3.2.3 martin igc_enable_intr(sc); 1578 1.3.2.3 martin 1579 1.3.2.2 martin return 0; 1580 1.3.2.2 martin } 1581 1.3.2.2 martin 1582 1.3.2.2 martin static inline int 1583 1.3.2.2 martin igc_load_mbuf(struct igc_queue *q, bus_dma_tag_t dmat, bus_dmamap_t map, 1584 1.3.2.2 martin struct mbuf *m) 1585 1.3.2.2 martin { 1586 1.3.2.2 martin int error; 1587 1.3.2.2 martin 1588 1.3.2.2 martin error = bus_dmamap_load_mbuf(dmat, map, m, 1589 1.3.2.2 martin BUS_DMA_WRITE | BUS_DMA_NOWAIT); 1590 1.3.2.2 martin 1591 1.3.2.2 martin if (__predict_false(error == EFBIG)) { 1592 1.3.2.2 martin IGC_DRIVER_EVENT(q, txdma_efbig, 1); 1593 1.3.2.2 martin m = m_defrag(m, M_NOWAIT); 1594 1.3.2.2 martin if (__predict_false(m == NULL)) { 1595 1.3.2.2 martin IGC_DRIVER_EVENT(q, txdma_defrag, 1); 1596 1.3.2.2 martin return ENOBUFS; 1597 1.3.2.2 martin } 1598 1.3.2.2 martin error = bus_dmamap_load_mbuf(dmat, map, m, 1599 1.3.2.2 martin BUS_DMA_WRITE | BUS_DMA_NOWAIT); 1600 1.3.2.2 martin } 1601 1.3.2.2 martin 1602 1.3.2.2 martin switch (error) { 1603 1.3.2.2 martin case 0: 1604 1.3.2.2 martin break; 1605 1.3.2.2 martin case ENOMEM: 1606 1.3.2.2 martin IGC_DRIVER_EVENT(q, txdma_enomem, 1); 1607 1.3.2.2 martin break; 1608 1.3.2.2 martin case EINVAL: 1609 1.3.2.2 martin IGC_DRIVER_EVENT(q, txdma_einval, 1); 1610 1.3.2.2 martin break; 1611 1.3.2.2 martin case EAGAIN: 1612 1.3.2.2 martin IGC_DRIVER_EVENT(q, txdma_eagain, 1); 1613 1.3.2.2 martin break; 1614 1.3.2.2 martin default: 1615 1.3.2.2 martin IGC_DRIVER_EVENT(q, txdma_other, 1); 1616 1.3.2.2 martin break; 1617 1.3.2.2 martin } 1618 1.3.2.2 martin 1619 1.3.2.2 martin return error; 1620 1.3.2.2 martin } 1621 1.3.2.2 martin 1622 1.3.2.2 martin #define IGC_TX_START 1 1623 1.3.2.2 martin #define IGC_TX_TRANSMIT 2 1624 1.3.2.2 martin 1625 1.3.2.2 martin static void 1626 1.3.2.2 martin igc_start(struct ifnet *ifp) 1627 1.3.2.2 martin { 1628 1.3.2.2 martin struct igc_softc *sc = ifp->if_softc; 1629 1.3.2.2 martin 1630 1.3.2.2 martin if (__predict_false(!sc->link_active)) { 1631 1.3.2.2 martin IFQ_PURGE(&ifp->if_snd); 1632 1.3.2.2 martin return; 1633 1.3.2.2 martin } 1634 1.3.2.2 martin 1635 1.3.2.2 martin struct tx_ring *txr = &sc->tx_rings[0]; /* queue 0 */ 1636 1.3.2.2 martin mutex_enter(&txr->txr_lock); 1637 1.3.2.2 martin igc_tx_common_locked(ifp, txr, IGC_TX_START); 1638 1.3.2.2 martin mutex_exit(&txr->txr_lock); 1639 1.3.2.2 martin } 1640 1.3.2.2 martin 1641 1.3.2.2 martin static inline u_int 1642 1.3.2.2 martin igc_select_txqueue(struct igc_softc *sc, struct mbuf *m __unused) 1643 1.3.2.2 martin { 1644 1.3.2.2 martin const u_int cpuid = cpu_index(curcpu()); 1645 1.3.2.2 martin 1646 1.3.2.2 martin return cpuid % sc->sc_nqueues; 1647 1.3.2.2 martin } 1648 1.3.2.2 martin 1649 1.3.2.2 martin static int 1650 1.3.2.2 martin igc_transmit(struct ifnet *ifp, struct mbuf *m) 1651 1.3.2.2 martin { 1652 1.3.2.2 martin struct igc_softc *sc = ifp->if_softc; 1653 1.3.2.2 martin const u_int qid = igc_select_txqueue(sc, m); 1654 1.3.2.2 martin struct tx_ring *txr = &sc->tx_rings[qid]; 1655 1.3.2.2 martin struct igc_queue *q = txr->txr_igcq; 1656 1.3.2.2 martin 1657 1.3.2.2 martin if (__predict_false(!pcq_put(txr->txr_interq, m))) { 1658 1.3.2.2 martin IGC_QUEUE_EVENT(q, tx_pcq_drop, 1); 1659 1.3.2.2 martin m_freem(m); 1660 1.3.2.2 martin return ENOBUFS; 1661 1.3.2.2 martin } 1662 1.3.2.2 martin 1663 1.3.2.2 martin mutex_enter(&txr->txr_lock); 1664 1.3.2.2 martin igc_tx_common_locked(ifp, txr, IGC_TX_TRANSMIT); 1665 1.3.2.2 martin mutex_exit(&txr->txr_lock); 1666 1.3.2.2 martin 1667 1.3.2.2 martin return 0; 1668 1.3.2.2 martin } 1669 1.3.2.2 martin 1670 1.3.2.2 martin static void 1671 1.3.2.2 martin igc_tx_common_locked(struct ifnet *ifp, struct tx_ring *txr, int caller) 1672 1.3.2.2 martin { 1673 1.3.2.2 martin struct igc_softc *sc = ifp->if_softc; 1674 1.3.2.2 martin struct igc_queue *q = txr->txr_igcq; 1675 1.3.2.2 martin net_stat_ref_t nsr = IF_STAT_GETREF(ifp); 1676 1.3.2.2 martin int prod, free, last = -1; 1677 1.3.2.2 martin bool post = false; 1678 1.3.2.2 martin 1679 1.3.2.2 martin prod = txr->next_avail_desc; 1680 1.3.2.2 martin free = txr->next_to_clean; 1681 1.3.2.2 martin if (free <= prod) 1682 1.3.2.2 martin free += sc->num_tx_desc; 1683 1.3.2.2 martin free -= prod; 1684 1.3.2.2 martin 1685 1.3.2.2 martin DPRINTF(TX, "%s: begin: msix %d prod %d n2c %d free %d\n", 1686 1.3.2.2 martin caller == IGC_TX_TRANSMIT ? "transmit" : "start", 1687 1.3.2.2 martin txr->me, prod, txr->next_to_clean, free); 1688 1.3.2.2 martin 1689 1.3.2.2 martin for (;;) { 1690 1.3.2.2 martin struct mbuf *m; 1691 1.3.2.2 martin 1692 1.3.2.2 martin if (__predict_false(free <= IGC_MAX_SCATTER)) { 1693 1.3.2.2 martin IGC_QUEUE_EVENT(q, tx_no_desc, 1); 1694 1.3.2.2 martin break; 1695 1.3.2.2 martin } 1696 1.3.2.2 martin 1697 1.3.2.2 martin if (caller == IGC_TX_TRANSMIT) 1698 1.3.2.2 martin m = pcq_get(txr->txr_interq); 1699 1.3.2.2 martin else 1700 1.3.2.2 martin IFQ_DEQUEUE(&ifp->if_snd, m); 1701 1.3.2.2 martin if (__predict_false(m == NULL)) 1702 1.3.2.2 martin break; 1703 1.3.2.2 martin 1704 1.3.2.2 martin struct igc_tx_buf *txbuf = &txr->tx_buffers[prod]; 1705 1.3.2.2 martin bus_dmamap_t map = txbuf->map; 1706 1.3.2.2 martin 1707 1.3.2.2 martin if (__predict_false( 1708 1.3.2.2 martin igc_load_mbuf(q, txr->txdma.dma_tag, map, m))) { 1709 1.3.2.2 martin if (caller == IGC_TX_TRANSMIT) 1710 1.3.2.2 martin IGC_QUEUE_EVENT(q, tx_pcq_drop, 1); 1711 1.3.2.2 martin m_freem(m); 1712 1.3.2.2 martin if_statinc_ref(nsr, if_oerrors); 1713 1.3.2.2 martin continue; 1714 1.3.2.2 martin } 1715 1.3.2.2 martin 1716 1.3.2.2 martin bus_dmamap_sync(txr->txdma.dma_tag, map, 0, 1717 1.3.2.2 martin map->dm_mapsize, BUS_DMASYNC_PREWRITE); 1718 1.3.2.2 martin 1719 1.3.2.2 martin uint32_t ctx_cmd_type_len = 0, olinfo_status = 0; 1720 1.3.2.2 martin if (igc_tx_ctx_setup(txr, m, prod, &ctx_cmd_type_len, 1721 1.3.2.2 martin &olinfo_status)) { 1722 1.3.2.2 martin IGC_QUEUE_EVENT(q, tx_ctx, 1); 1723 1.3.2.2 martin /* Consume the first descriptor */ 1724 1.3.2.2 martin prod = igc_txdesc_incr(sc, prod); 1725 1.3.2.2 martin free--; 1726 1.3.2.2 martin } 1727 1.3.2.2 martin for (int i = 0; i < map->dm_nsegs; i++) { 1728 1.3.2.2 martin union igc_adv_tx_desc *txdesc = &txr->tx_base[prod]; 1729 1.3.2.2 martin 1730 1.3.2.2 martin uint32_t cmd_type_len = ctx_cmd_type_len | 1731 1.3.2.2 martin IGC_ADVTXD_DCMD_IFCS | IGC_ADVTXD_DTYP_DATA | 1732 1.3.2.2 martin IGC_ADVTXD_DCMD_DEXT | map->dm_segs[i].ds_len; 1733 1.3.2.2 martin if (i == map->dm_nsegs - 1) { 1734 1.3.2.2 martin cmd_type_len |= 1735 1.3.2.2 martin IGC_ADVTXD_DCMD_EOP | IGC_ADVTXD_DCMD_RS; 1736 1.3.2.2 martin } 1737 1.3.2.2 martin 1738 1.3.2.2 martin igc_txdesc_sync(txr, prod, 1739 1.3.2.2 martin BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE); 1740 1.3.2.2 martin htolem64(&txdesc->read.buffer_addr, 1741 1.3.2.2 martin map->dm_segs[i].ds_addr); 1742 1.3.2.2 martin htolem32(&txdesc->read.cmd_type_len, cmd_type_len); 1743 1.3.2.2 martin htolem32(&txdesc->read.olinfo_status, olinfo_status); 1744 1.3.2.2 martin igc_txdesc_sync(txr, prod, 1745 1.3.2.2 martin BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); 1746 1.3.2.2 martin 1747 1.3.2.2 martin last = prod; 1748 1.3.2.2 martin prod = igc_txdesc_incr(sc, prod); 1749 1.3.2.2 martin } 1750 1.3.2.2 martin 1751 1.3.2.2 martin txbuf->m_head = m; 1752 1.3.2.2 martin txbuf->eop_index = last; 1753 1.3.2.2 martin 1754 1.3.2.2 martin bpf_mtap(ifp, m, BPF_D_OUT); 1755 1.3.2.2 martin 1756 1.3.2.2 martin if_statadd_ref(nsr, if_obytes, m->m_pkthdr.len); 1757 1.3.2.2 martin if (m->m_flags & M_MCAST) 1758 1.3.2.2 martin if_statinc_ref(nsr, if_omcasts); 1759 1.3.2.2 martin IGC_QUEUE_EVENT(q, tx_packets, 1); 1760 1.3.2.2 martin IGC_QUEUE_EVENT(q, tx_bytes, m->m_pkthdr.len); 1761 1.3.2.2 martin 1762 1.3.2.2 martin free -= map->dm_nsegs; 1763 1.3.2.2 martin post = true; 1764 1.3.2.2 martin } 1765 1.3.2.2 martin 1766 1.3.2.2 martin if (post) { 1767 1.3.2.2 martin txr->next_avail_desc = prod; 1768 1.3.2.2 martin IGC_WRITE_REG(&sc->hw, IGC_TDT(txr->me), prod); 1769 1.3.2.2 martin } 1770 1.3.2.2 martin 1771 1.3.2.2 martin DPRINTF(TX, "%s: done : msix %d prod %d n2c %d free %d\n", 1772 1.3.2.2 martin caller == IGC_TX_TRANSMIT ? "transmit" : "start", 1773 1.3.2.2 martin txr->me, prod, txr->next_to_clean, free); 1774 1.3.2.2 martin 1775 1.3.2.2 martin IF_STAT_PUTREF(ifp); 1776 1.3.2.2 martin } 1777 1.3.2.2 martin 1778 1.3.2.2 martin static bool 1779 1.3.2.2 martin igc_txeof(struct tx_ring *txr, u_int limit) 1780 1.3.2.2 martin { 1781 1.3.2.2 martin struct igc_softc *sc = txr->sc; 1782 1.3.2.2 martin struct ifnet *ifp = &sc->sc_ec.ec_if; 1783 1.3.2.2 martin int cons, prod; 1784 1.3.2.2 martin bool more = false; 1785 1.3.2.2 martin 1786 1.3.2.2 martin prod = txr->next_avail_desc; 1787 1.3.2.2 martin cons = txr->next_to_clean; 1788 1.3.2.2 martin 1789 1.3.2.2 martin if (cons == prod) { 1790 1.3.2.2 martin DPRINTF(TX, "false: msix %d cons %d prod %d\n", 1791 1.3.2.2 martin txr->me, cons, prod); 1792 1.3.2.2 martin return false; 1793 1.3.2.2 martin } 1794 1.3.2.2 martin 1795 1.3.2.2 martin do { 1796 1.3.2.2 martin struct igc_tx_buf *txbuf = &txr->tx_buffers[cons]; 1797 1.3.2.2 martin const int last = txbuf->eop_index; 1798 1.3.2.2 martin 1799 1.3.2.2 martin membar_consumer(); /* XXXRO necessary? */ 1800 1.3.2.2 martin 1801 1.3.2.2 martin KASSERT(last != -1); 1802 1.3.2.2 martin union igc_adv_tx_desc *txdesc = &txr->tx_base[last]; 1803 1.3.2.2 martin igc_txdesc_sync(txr, last, BUS_DMASYNC_POSTREAD); 1804 1.3.2.2 martin const uint32_t status = le32toh(txdesc->wb.status); 1805 1.3.2.2 martin igc_txdesc_sync(txr, last, BUS_DMASYNC_PREREAD); 1806 1.3.2.2 martin 1807 1.3.2.2 martin if (!(status & IGC_TXD_STAT_DD)) 1808 1.3.2.2 martin break; 1809 1.3.2.2 martin 1810 1.3.2.2 martin if (limit-- == 0) { 1811 1.3.2.2 martin more = true; 1812 1.3.2.2 martin DPRINTF(TX, "pending TX " 1813 1.3.2.2 martin "msix %d cons %d last %d prod %d " 1814 1.3.2.2 martin "status 0x%08x\n", 1815 1.3.2.2 martin txr->me, cons, last, prod, status); 1816 1.3.2.2 martin break; 1817 1.3.2.2 martin } 1818 1.3.2.2 martin 1819 1.3.2.2 martin DPRINTF(TX, "handled TX " 1820 1.3.2.2 martin "msix %d cons %d last %d prod %d " 1821 1.3.2.2 martin "status 0x%08x\n", 1822 1.3.2.2 martin txr->me, cons, last, prod, status); 1823 1.3.2.2 martin 1824 1.3.2.2 martin if_statinc(ifp, if_opackets); 1825 1.3.2.2 martin 1826 1.3.2.2 martin bus_dmamap_t map = txbuf->map; 1827 1.3.2.2 martin bus_dmamap_sync(txr->txdma.dma_tag, map, 0, map->dm_mapsize, 1828 1.3.2.2 martin BUS_DMASYNC_POSTWRITE); 1829 1.3.2.2 martin bus_dmamap_unload(txr->txdma.dma_tag, map); 1830 1.3.2.2 martin m_freem(txbuf->m_head); 1831 1.3.2.2 martin 1832 1.3.2.2 martin txbuf->m_head = NULL; 1833 1.3.2.2 martin txbuf->eop_index = -1; 1834 1.3.2.2 martin 1835 1.3.2.2 martin cons = igc_txdesc_incr(sc, last); 1836 1.3.2.2 martin } while (cons != prod); 1837 1.3.2.2 martin 1838 1.3.2.2 martin txr->next_to_clean = cons; 1839 1.3.2.2 martin 1840 1.3.2.2 martin return more; 1841 1.3.2.2 martin } 1842 1.3.2.2 martin 1843 1.3.2.2 martin static void 1844 1.3.2.2 martin igc_intr_barrier(struct igc_softc *sc __unused) 1845 1.3.2.2 martin { 1846 1.3.2.2 martin 1847 1.3.2.2 martin xc_barrier(0); 1848 1.3.2.2 martin } 1849 1.3.2.2 martin 1850 1.3.2.2 martin static void 1851 1.3.2.2 martin igc_stop(struct ifnet *ifp, int disable) 1852 1.3.2.2 martin { 1853 1.3.2.2 martin struct igc_softc *sc = ifp->if_softc; 1854 1.3.2.2 martin 1855 1.3.2.2 martin mutex_enter(&sc->sc_core_lock); 1856 1.3.2.2 martin igc_stop_locked(sc); 1857 1.3.2.2 martin mutex_exit(&sc->sc_core_lock); 1858 1.3.2.2 martin } 1859 1.3.2.2 martin 1860 1.3.2.2 martin /********************************************************************* 1861 1.3.2.2 martin * 1862 1.3.2.2 martin * This routine disables all traffic on the adapter by issuing a 1863 1.3.2.2 martin * global reset on the MAC. 1864 1.3.2.2 martin * 1865 1.3.2.2 martin **********************************************************************/ 1866 1.3.2.2 martin static void 1867 1.3.2.2 martin igc_stop_locked(struct igc_softc *sc) 1868 1.3.2.2 martin { 1869 1.3.2.2 martin struct ifnet *ifp = &sc->sc_ec.ec_if; 1870 1.3.2.2 martin 1871 1.3.2.2 martin DPRINTF(CFG, "called\n"); 1872 1.3.2.2 martin 1873 1.3.2.2 martin KASSERT(mutex_owned(&sc->sc_core_lock)); 1874 1.3.2.2 martin 1875 1.3.2.2 martin /* 1876 1.3.2.2 martin * If stopping processing has already started, do nothing. 1877 1.3.2.2 martin */ 1878 1.3.2.2 martin if ((ifp->if_flags & IFF_RUNNING) == 0) 1879 1.3.2.2 martin return; 1880 1.3.2.2 martin 1881 1.3.2.2 martin /* Tell the stack that the interface is no longer active. */ 1882 1.3.2.2 martin ifp->if_flags &= ~IFF_RUNNING; 1883 1.3.2.2 martin 1884 1.3.2.2 martin /* 1885 1.3.2.2 martin * igc_handle_queue() can enable interrupts, so wait for completion of 1886 1.3.2.2 martin * last igc_handle_queue() after unset IFF_RUNNING. 1887 1.3.2.2 martin */ 1888 1.3.2.2 martin mutex_exit(&sc->sc_core_lock); 1889 1.3.2.2 martin igc_barrier_handle_queue(sc); 1890 1.3.2.2 martin mutex_enter(&sc->sc_core_lock); 1891 1.3.2.2 martin 1892 1.3.2.2 martin sc->sc_core_stopping = true; 1893 1.3.2.2 martin 1894 1.3.2.2 martin igc_disable_intr(sc); 1895 1.3.2.2 martin 1896 1.3.2.2 martin callout_halt(&sc->sc_tick_ch, &sc->sc_core_lock); 1897 1.3.2.2 martin 1898 1.3.2.2 martin igc_reset_hw(&sc->hw); 1899 1.3.2.2 martin IGC_WRITE_REG(&sc->hw, IGC_WUC, 0); 1900 1.3.2.2 martin 1901 1.3.2.2 martin /* 1902 1.3.2.2 martin * Wait for completion of interrupt handlers. 1903 1.3.2.2 martin */ 1904 1.3.2.2 martin mutex_exit(&sc->sc_core_lock); 1905 1.3.2.2 martin igc_intr_barrier(sc); 1906 1.3.2.2 martin mutex_enter(&sc->sc_core_lock); 1907 1.3.2.2 martin 1908 1.3.2.2 martin igc_update_link_status(sc); 1909 1.3.2.2 martin 1910 1.3.2.2 martin for (int iq = 0; iq < sc->sc_nqueues; iq++) { 1911 1.3.2.2 martin struct tx_ring *txr = &sc->tx_rings[iq]; 1912 1.3.2.2 martin 1913 1.3.2.2 martin igc_withdraw_transmit_packets(txr, false); 1914 1.3.2.2 martin } 1915 1.3.2.2 martin 1916 1.3.2.2 martin for (int iq = 0; iq < sc->sc_nqueues; iq++) { 1917 1.3.2.2 martin struct rx_ring *rxr = &sc->rx_rings[iq]; 1918 1.3.2.2 martin 1919 1.3.2.2 martin igc_clear_receive_status(rxr); 1920 1.3.2.2 martin } 1921 1.3.2.2 martin 1922 1.3.2.2 martin /* Save last flags for the callback */ 1923 1.3.2.2 martin sc->sc_if_flags = ifp->if_flags; 1924 1.3.2.2 martin } 1925 1.3.2.2 martin 1926 1.3.2.2 martin /********************************************************************* 1927 1.3.2.2 martin * Ioctl entry point 1928 1.3.2.2 martin * 1929 1.3.2.2 martin * igc_ioctl is called when the user wants to configure the 1930 1.3.2.2 martin * interface. 1931 1.3.2.2 martin * 1932 1.3.2.2 martin * return 0 on success, positive on failure 1933 1.3.2.2 martin **********************************************************************/ 1934 1.3.2.2 martin static int 1935 1.3.2.2 martin igc_ioctl(struct ifnet * ifp, u_long cmd, void *data) 1936 1.3.2.2 martin { 1937 1.3.2.2 martin struct igc_softc *sc __unused = ifp->if_softc; 1938 1.3.2.2 martin int s; 1939 1.3.2.2 martin int error; 1940 1.3.2.2 martin 1941 1.3.2.2 martin DPRINTF(CFG, "cmd 0x%016lx\n", cmd); 1942 1.3.2.2 martin 1943 1.3.2.2 martin switch (cmd) { 1944 1.3.2.2 martin case SIOCADDMULTI: 1945 1.3.2.2 martin case SIOCDELMULTI: 1946 1.3.2.2 martin break; 1947 1.3.2.2 martin default: 1948 1.3.2.2 martin KASSERT(IFNET_LOCKED(ifp)); 1949 1.3.2.2 martin } 1950 1.3.2.2 martin 1951 1.3.2.2 martin if (cmd == SIOCZIFDATA) { 1952 1.3.2.2 martin mutex_enter(&sc->sc_core_lock); 1953 1.3.2.2 martin igc_clear_counters(sc); 1954 1.3.2.2 martin mutex_exit(&sc->sc_core_lock); 1955 1.3.2.2 martin } 1956 1.3.2.2 martin 1957 1.3.2.2 martin switch (cmd) { 1958 1.3.2.2 martin #ifdef IF_RXR 1959 1.3.2.2 martin case SIOCGIFRXR: 1960 1.3.2.2 martin s = splnet(); 1961 1.3.2.2 martin error = igc_rxrinfo(sc, (struct if_rxrinfo *)ifr->ifr_data); 1962 1.3.2.2 martin splx(s); 1963 1.3.2.2 martin break; 1964 1.3.2.2 martin #endif 1965 1.3.2.2 martin default: 1966 1.3.2.2 martin s = splnet(); 1967 1.3.2.2 martin error = ether_ioctl(ifp, cmd, data); 1968 1.3.2.2 martin splx(s); 1969 1.3.2.2 martin break; 1970 1.3.2.2 martin } 1971 1.3.2.2 martin 1972 1.3.2.2 martin if (error != ENETRESET) 1973 1.3.2.2 martin return error; 1974 1.3.2.2 martin 1975 1.3.2.2 martin error = 0; 1976 1.3.2.2 martin 1977 1.3.2.2 martin if (cmd == SIOCSIFCAP) 1978 1.3.2.2 martin error = if_init(ifp); 1979 1.3.2.2 martin else if ((cmd == SIOCADDMULTI) || (cmd == SIOCDELMULTI)) { 1980 1.3.2.2 martin mutex_enter(&sc->sc_core_lock); 1981 1.3.2.2 martin if (sc->sc_if_flags & IFF_RUNNING) { 1982 1.3.2.2 martin /* 1983 1.3.2.2 martin * Multicast list has changed; set the hardware filter 1984 1.3.2.2 martin * accordingly. 1985 1.3.2.2 martin */ 1986 1.3.2.2 martin igc_disable_intr(sc); 1987 1.3.2.2 martin igc_set_filter(sc); 1988 1.3.2.2 martin igc_enable_intr(sc); 1989 1.3.2.2 martin } 1990 1.3.2.2 martin mutex_exit(&sc->sc_core_lock); 1991 1.3.2.2 martin } 1992 1.3.2.2 martin 1993 1.3.2.2 martin return error; 1994 1.3.2.2 martin } 1995 1.3.2.2 martin 1996 1.3.2.2 martin #ifdef IF_RXR 1997 1.3.2.2 martin static int 1998 1.3.2.2 martin igc_rxrinfo(struct igc_softc *sc, struct if_rxrinfo *ifri) 1999 1.3.2.2 martin { 2000 1.3.2.2 martin struct if_rxring_info *ifr, ifr1; 2001 1.3.2.2 martin int error; 2002 1.3.2.2 martin 2003 1.3.2.2 martin if (sc->sc_nqueues > 1) { 2004 1.3.2.2 martin ifr = kmem_zalloc(sc->sc_nqueues * sizeof(*ifr), KM_SLEEP); 2005 1.3.2.2 martin } else { 2006 1.3.2.2 martin ifr = &ifr1; 2007 1.3.2.2 martin memset(ifr, 0, sizeof(*ifr)); 2008 1.3.2.2 martin } 2009 1.3.2.2 martin 2010 1.3.2.2 martin for (int iq = 0; iq < sc->sc_nqueues; iq++) { 2011 1.3.2.2 martin struct rx_ring *rxr = &sc->rx_rings[iq]; 2012 1.3.2.2 martin 2013 1.3.2.2 martin ifr[iq].ifr_size = MCLBYTES; 2014 1.3.2.2 martin snprintf(ifr[iq].ifr_name, sizeof(ifr[iq].ifr_name), "%d", iq); 2015 1.3.2.2 martin ifr[iq].ifr_info = rxr->rx_ring; 2016 1.3.2.2 martin } 2017 1.3.2.2 martin 2018 1.3.2.2 martin error = if_rxr_info_ioctl(ifri, sc->sc_nqueues, ifr); 2019 1.3.2.2 martin if (sc->sc_nqueues > 1) 2020 1.3.2.2 martin kmem_free(ifr, sc->sc_nqueues * sizeof(*ifr)); 2021 1.3.2.2 martin 2022 1.3.2.2 martin return error; 2023 1.3.2.2 martin } 2024 1.3.2.2 martin #endif 2025 1.3.2.2 martin 2026 1.3.2.2 martin static void 2027 1.3.2.2 martin igc_rxfill(struct rx_ring *rxr) 2028 1.3.2.2 martin { 2029 1.3.2.2 martin struct igc_softc *sc = rxr->sc; 2030 1.3.2.2 martin int id; 2031 1.3.2.2 martin 2032 1.3.2.2 martin for (id = 0; id < sc->num_rx_desc; id++) { 2033 1.3.2.2 martin if (igc_get_buf(rxr, id, false)) { 2034 1.3.2.2 martin panic("%s: msix=%d i=%d\n", __func__, rxr->me, id); 2035 1.3.2.2 martin } 2036 1.3.2.2 martin } 2037 1.3.2.2 martin 2038 1.3.2.2 martin id = sc->num_rx_desc - 1; 2039 1.3.2.2 martin rxr->last_desc_filled = id; 2040 1.3.2.2 martin IGC_WRITE_REG(&sc->hw, IGC_RDT(rxr->me), id); 2041 1.3.2.2 martin rxr->next_to_check = 0; 2042 1.3.2.2 martin } 2043 1.3.2.2 martin 2044 1.3.2.2 martin static void 2045 1.3.2.2 martin igc_rxrefill(struct rx_ring *rxr, int end) 2046 1.3.2.2 martin { 2047 1.3.2.2 martin struct igc_softc *sc = rxr->sc; 2048 1.3.2.2 martin int id; 2049 1.3.2.2 martin 2050 1.3.2.2 martin for (id = rxr->next_to_check; id != end; id = igc_rxdesc_incr(sc, id)) { 2051 1.3.2.2 martin if (igc_get_buf(rxr, id, true)) { 2052 1.3.2.2 martin /* XXXRO */ 2053 1.3.2.2 martin panic("%s: msix=%d id=%d\n", __func__, rxr->me, id); 2054 1.3.2.2 martin } 2055 1.3.2.2 martin } 2056 1.3.2.2 martin 2057 1.3.2.2 martin id = igc_rxdesc_decr(sc, id); 2058 1.3.2.2 martin DPRINTF(RX, "%s RDT %d id %d\n", 2059 1.3.2.2 martin rxr->last_desc_filled == id ? "same" : "diff", 2060 1.3.2.2 martin rxr->last_desc_filled, id); 2061 1.3.2.2 martin rxr->last_desc_filled = id; 2062 1.3.2.2 martin IGC_WRITE_REG(&sc->hw, IGC_RDT(rxr->me), id); 2063 1.3.2.2 martin } 2064 1.3.2.2 martin 2065 1.3.2.2 martin /********************************************************************* 2066 1.3.2.2 martin * 2067 1.3.2.2 martin * This routine executes in interrupt context. It replenishes 2068 1.3.2.2 martin * the mbufs in the descriptor and sends data which has been 2069 1.3.2.2 martin * dma'ed into host memory to upper layer. 2070 1.3.2.2 martin * 2071 1.3.2.2 martin *********************************************************************/ 2072 1.3.2.2 martin static bool 2073 1.3.2.2 martin igc_rxeof(struct rx_ring *rxr, u_int limit) 2074 1.3.2.2 martin { 2075 1.3.2.2 martin struct igc_softc *sc = rxr->sc; 2076 1.3.2.2 martin struct igc_queue *q = rxr->rxr_igcq; 2077 1.3.2.2 martin struct ifnet *ifp = &sc->sc_ec.ec_if; 2078 1.3.2.2 martin int id; 2079 1.3.2.2 martin bool more = false; 2080 1.3.2.2 martin 2081 1.3.2.2 martin id = rxr->next_to_check; 2082 1.3.2.2 martin for (;;) { 2083 1.3.2.2 martin union igc_adv_rx_desc *rxdesc = &rxr->rx_base[id]; 2084 1.3.2.2 martin struct igc_rx_buf *rxbuf, *nxbuf; 2085 1.3.2.2 martin struct mbuf *mp, *m; 2086 1.3.2.2 martin 2087 1.3.2.2 martin igc_rxdesc_sync(rxr, id, 2088 1.3.2.2 martin BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE); 2089 1.3.2.2 martin 2090 1.3.2.2 martin const uint32_t staterr = le32toh(rxdesc->wb.upper.status_error); 2091 1.3.2.2 martin 2092 1.3.2.2 martin if (!ISSET(staterr, IGC_RXD_STAT_DD)) { 2093 1.3.2.2 martin igc_rxdesc_sync(rxr, id, 2094 1.3.2.2 martin BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); 2095 1.3.2.2 martin break; 2096 1.3.2.2 martin } 2097 1.3.2.2 martin 2098 1.3.2.2 martin if (limit-- == 0) { 2099 1.3.2.2 martin igc_rxdesc_sync(rxr, id, 2100 1.3.2.2 martin BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); 2101 1.3.2.2 martin DPRINTF(RX, "more=true\n"); 2102 1.3.2.2 martin more = true; 2103 1.3.2.2 martin break; 2104 1.3.2.2 martin } 2105 1.3.2.2 martin 2106 1.3.2.2 martin /* Zero out the receive descriptors status. */ 2107 1.3.2.2 martin rxdesc->wb.upper.status_error = 0; 2108 1.3.2.2 martin 2109 1.3.2.2 martin /* Pull the mbuf off the ring. */ 2110 1.3.2.2 martin rxbuf = &rxr->rx_buffers[id]; 2111 1.3.2.2 martin bus_dmamap_t map = rxbuf->map; 2112 1.3.2.2 martin bus_dmamap_sync(rxr->rxdma.dma_tag, map, 2113 1.3.2.2 martin 0, map->dm_mapsize, BUS_DMASYNC_POSTREAD); 2114 1.3.2.2 martin bus_dmamap_unload(rxr->rxdma.dma_tag, map); 2115 1.3.2.2 martin 2116 1.3.2.2 martin mp = rxbuf->buf; 2117 1.3.2.2 martin rxbuf->buf = NULL; 2118 1.3.2.2 martin 2119 1.3.2.2 martin const bool eop = staterr & IGC_RXD_STAT_EOP; 2120 1.3.2.2 martin const uint16_t len = le16toh(rxdesc->wb.upper.length); 2121 1.3.2.2 martin 2122 1.3.2.2 martin const uint16_t vtag = le16toh(rxdesc->wb.upper.vlan); 2123 1.3.2.2 martin 2124 1.3.2.2 martin const uint32_t ptype = le32toh(rxdesc->wb.lower.lo_dword.data) & 2125 1.3.2.2 martin IGC_PKTTYPE_MASK; 2126 1.3.2.2 martin 2127 1.3.2.2 martin const uint32_t hash __unused = 2128 1.3.2.2 martin le32toh(rxdesc->wb.lower.hi_dword.rss); 2129 1.3.2.2 martin const uint16_t hashtype __unused = 2130 1.3.2.2 martin le16toh(rxdesc->wb.lower.lo_dword.hs_rss.pkt_info) & 2131 1.3.2.2 martin IGC_RXDADV_RSSTYPE_MASK; 2132 1.3.2.2 martin 2133 1.3.2.2 martin igc_rxdesc_sync(rxr, id, 2134 1.3.2.2 martin BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); 2135 1.3.2.2 martin 2136 1.3.2.2 martin if (__predict_false(staterr & IGC_RXDEXT_STATERR_RXE)) { 2137 1.3.2.2 martin if (rxbuf->fmp) { 2138 1.3.2.2 martin m_freem(rxbuf->fmp); 2139 1.3.2.2 martin rxbuf->fmp = NULL; 2140 1.3.2.2 martin } 2141 1.3.2.2 martin 2142 1.3.2.2 martin m_freem(mp); 2143 1.3.2.2 martin m = NULL; 2144 1.3.2.2 martin 2145 1.3.2.2 martin if_statinc(ifp, if_ierrors); 2146 1.3.2.2 martin IGC_QUEUE_EVENT(q, rx_discard, 1); 2147 1.3.2.2 martin 2148 1.3.2.2 martin DPRINTF(RX, "ierrors++\n"); 2149 1.3.2.2 martin 2150 1.3.2.2 martin goto next_desc; 2151 1.3.2.2 martin } 2152 1.3.2.2 martin 2153 1.3.2.2 martin if (__predict_false(mp == NULL)) { 2154 1.3.2.2 martin panic("%s: igc_rxeof: NULL mbuf in slot %d " 2155 1.3.2.2 martin "(filled %d)", device_xname(sc->sc_dev), 2156 1.3.2.2 martin id, rxr->last_desc_filled); 2157 1.3.2.2 martin } 2158 1.3.2.2 martin 2159 1.3.2.2 martin if (!eop) { 2160 1.3.2.2 martin /* 2161 1.3.2.2 martin * Figure out the next descriptor of this frame. 2162 1.3.2.2 martin */ 2163 1.3.2.2 martin int nextp = igc_rxdesc_incr(sc, id); 2164 1.3.2.2 martin 2165 1.3.2.2 martin nxbuf = &rxr->rx_buffers[nextp]; 2166 1.3.2.2 martin /* 2167 1.3.2.2 martin * TODO prefetch(nxbuf); 2168 1.3.2.2 martin */ 2169 1.3.2.2 martin } 2170 1.3.2.2 martin 2171 1.3.2.2 martin mp->m_len = len; 2172 1.3.2.2 martin 2173 1.3.2.2 martin m = rxbuf->fmp; 2174 1.3.2.2 martin rxbuf->fmp = NULL; 2175 1.3.2.2 martin 2176 1.3.2.2 martin if (m != NULL) { 2177 1.3.2.2 martin m->m_pkthdr.len += mp->m_len; 2178 1.3.2.2 martin } else { 2179 1.3.2.2 martin m = mp; 2180 1.3.2.2 martin m->m_pkthdr.len = mp->m_len; 2181 1.3.2.2 martin #if NVLAN > 0 2182 1.3.2.2 martin if (staterr & IGC_RXD_STAT_VP) 2183 1.3.2.2 martin vlan_set_tag(m, vtag); 2184 1.3.2.2 martin #endif 2185 1.3.2.2 martin } 2186 1.3.2.2 martin 2187 1.3.2.2 martin /* Pass the head pointer on */ 2188 1.3.2.2 martin if (!eop) { 2189 1.3.2.2 martin nxbuf->fmp = m; 2190 1.3.2.2 martin m = NULL; 2191 1.3.2.2 martin mp->m_next = nxbuf->buf; 2192 1.3.2.2 martin } else { 2193 1.3.2.2 martin m_set_rcvif(m, ifp); 2194 1.3.2.2 martin 2195 1.3.2.2 martin m->m_pkthdr.csum_flags = igc_rx_checksum(q, 2196 1.3.2.2 martin ifp->if_capenable, staterr, ptype); 2197 1.3.2.2 martin 2198 1.3.2.2 martin #ifdef notyet 2199 1.3.2.2 martin if (hashtype != IGC_RXDADV_RSSTYPE_NONE) { 2200 1.3.2.2 martin m->m_pkthdr.ph_flowid = hash; 2201 1.3.2.2 martin SET(m->m_pkthdr.csum_flags, M_FLOWID); 2202 1.3.2.2 martin } 2203 1.3.2.2 martin ml_enqueue(&ml, m); 2204 1.3.2.2 martin #endif 2205 1.3.2.2 martin 2206 1.3.2.2 martin if_percpuq_enqueue(sc->sc_ipq, m); 2207 1.3.2.2 martin 2208 1.3.2.2 martin if_statinc(ifp, if_ipackets); 2209 1.3.2.2 martin IGC_QUEUE_EVENT(q, rx_packets, 1); 2210 1.3.2.2 martin IGC_QUEUE_EVENT(q, rx_bytes, m->m_pkthdr.len); 2211 1.3.2.2 martin } 2212 1.3.2.2 martin next_desc: 2213 1.3.2.2 martin /* Advance our pointers to the next descriptor. */ 2214 1.3.2.2 martin id = igc_rxdesc_incr(sc, id); 2215 1.3.2.2 martin } 2216 1.3.2.2 martin 2217 1.3.2.2 martin DPRINTF(RX, "fill queue[%d]\n", rxr->me); 2218 1.3.2.2 martin igc_rxrefill(rxr, id); 2219 1.3.2.2 martin 2220 1.3.2.2 martin DPRINTF(RX, "%s n2c %d id %d\n", 2221 1.3.2.2 martin rxr->next_to_check == id ? "same" : "diff", 2222 1.3.2.2 martin rxr->next_to_check, id); 2223 1.3.2.2 martin rxr->next_to_check = id; 2224 1.3.2.2 martin 2225 1.3.2.2 martin #ifdef OPENBSD 2226 1.3.2.2 martin if (!(staterr & IGC_RXD_STAT_DD)) 2227 1.3.2.2 martin return 0; 2228 1.3.2.2 martin #endif 2229 1.3.2.2 martin 2230 1.3.2.2 martin return more; 2231 1.3.2.2 martin } 2232 1.3.2.2 martin 2233 1.3.2.2 martin /********************************************************************* 2234 1.3.2.2 martin * 2235 1.3.2.2 martin * Verify that the hardware indicated that the checksum is valid. 2236 1.3.2.2 martin * Inform the stack about the status of checksum so that stack 2237 1.3.2.2 martin * doesn't spend time verifying the checksum. 2238 1.3.2.2 martin * 2239 1.3.2.2 martin *********************************************************************/ 2240 1.3.2.2 martin static int 2241 1.3.2.2 martin igc_rx_checksum(struct igc_queue *q, uint64_t capenable, uint32_t staterr, 2242 1.3.2.2 martin uint32_t ptype) 2243 1.3.2.2 martin { 2244 1.3.2.2 martin const uint16_t status = (uint16_t)staterr; 2245 1.3.2.2 martin const uint8_t errors = (uint8_t)(staterr >> 24); 2246 1.3.2.2 martin int flags = 0; 2247 1.3.2.2 martin 2248 1.3.2.2 martin if ((status & IGC_RXD_STAT_IPCS) != 0 && 2249 1.3.2.2 martin (capenable & IFCAP_CSUM_IPv4_Rx) != 0) { 2250 1.3.2.2 martin IGC_DRIVER_EVENT(q, rx_ipcs, 1); 2251 1.3.2.2 martin flags |= M_CSUM_IPv4; 2252 1.3.2.2 martin if (__predict_false((errors & IGC_RXD_ERR_IPE) != 0)) { 2253 1.3.2.2 martin IGC_DRIVER_EVENT(q, rx_ipcs_bad, 1); 2254 1.3.2.2 martin flags |= M_CSUM_IPv4_BAD; 2255 1.3.2.2 martin } 2256 1.3.2.2 martin } 2257 1.3.2.2 martin 2258 1.3.2.2 martin if ((status & IGC_RXD_STAT_TCPCS) != 0) { 2259 1.3.2.2 martin IGC_DRIVER_EVENT(q, rx_tcpcs, 1); 2260 1.3.2.2 martin if ((capenable & IFCAP_CSUM_TCPv4_Rx) != 0) 2261 1.3.2.2 martin flags |= M_CSUM_TCPv4; 2262 1.3.2.2 martin if ((capenable & IFCAP_CSUM_TCPv6_Rx) != 0) 2263 1.3.2.2 martin flags |= M_CSUM_TCPv6; 2264 1.3.2.2 martin } 2265 1.3.2.2 martin 2266 1.3.2.2 martin if ((status & IGC_RXD_STAT_UDPCS) != 0) { 2267 1.3.2.2 martin IGC_DRIVER_EVENT(q, rx_udpcs, 1); 2268 1.3.2.2 martin if ((capenable & IFCAP_CSUM_UDPv4_Rx) != 0) 2269 1.3.2.2 martin flags |= M_CSUM_UDPv4; 2270 1.3.2.2 martin if ((capenable & IFCAP_CSUM_UDPv6_Rx) != 0) 2271 1.3.2.2 martin flags |= M_CSUM_UDPv6; 2272 1.3.2.2 martin } 2273 1.3.2.2 martin 2274 1.3.2.2 martin if (__predict_false((errors & IGC_RXD_ERR_TCPE) != 0)) { 2275 1.3.2.2 martin IGC_DRIVER_EVENT(q, rx_l4cs_bad, 1); 2276 1.3.2.2 martin if ((flags & ~M_CSUM_IPv4) != 0) 2277 1.3.2.2 martin flags |= M_CSUM_TCP_UDP_BAD; 2278 1.3.2.2 martin } 2279 1.3.2.2 martin 2280 1.3.2.2 martin return flags; 2281 1.3.2.2 martin } 2282 1.3.2.2 martin 2283 1.3.2.2 martin static void 2284 1.3.2.2 martin igc_watchdog(struct ifnet * ifp) 2285 1.3.2.2 martin { 2286 1.3.2.2 martin } 2287 1.3.2.2 martin 2288 1.3.2.2 martin static void 2289 1.3.2.2 martin igc_tick(void *arg) 2290 1.3.2.2 martin { 2291 1.3.2.2 martin struct igc_softc *sc = arg; 2292 1.3.2.2 martin 2293 1.3.2.2 martin mutex_enter(&sc->sc_core_lock); 2294 1.3.2.2 martin 2295 1.3.2.2 martin if (__predict_false(sc->sc_core_stopping)) { 2296 1.3.2.2 martin mutex_exit(&sc->sc_core_lock); 2297 1.3.2.2 martin return; 2298 1.3.2.2 martin } 2299 1.3.2.2 martin 2300 1.3.2.2 martin /* XXX watchdog */ 2301 1.3.2.2 martin if (0) { 2302 1.3.2.2 martin IGC_GLOBAL_EVENT(sc, watchdog, 1); 2303 1.3.2.2 martin } 2304 1.3.2.2 martin 2305 1.3.2.2 martin igc_update_counters(sc); 2306 1.3.2.2 martin 2307 1.3.2.2 martin mutex_exit(&sc->sc_core_lock); 2308 1.3.2.2 martin 2309 1.3.2.2 martin callout_schedule(&sc->sc_tick_ch, hz); 2310 1.3.2.2 martin } 2311 1.3.2.2 martin 2312 1.3.2.2 martin /********************************************************************* 2313 1.3.2.2 martin * 2314 1.3.2.2 martin * Media Ioctl callback 2315 1.3.2.2 martin * 2316 1.3.2.2 martin * This routine is called whenever the user queries the status of 2317 1.3.2.2 martin * the interface using ifconfig. 2318 1.3.2.2 martin * 2319 1.3.2.2 martin **********************************************************************/ 2320 1.3.2.2 martin static void 2321 1.3.2.2 martin igc_media_status(struct ifnet *ifp, struct ifmediareq *ifmr) 2322 1.3.2.2 martin { 2323 1.3.2.2 martin struct igc_softc *sc = ifp->if_softc; 2324 1.3.2.2 martin struct igc_hw *hw = &sc->hw; 2325 1.3.2.2 martin 2326 1.3.2.2 martin igc_update_link_status(sc); 2327 1.3.2.2 martin 2328 1.3.2.2 martin ifmr->ifm_status = IFM_AVALID; 2329 1.3.2.2 martin ifmr->ifm_active = IFM_ETHER; 2330 1.3.2.2 martin 2331 1.3.2.2 martin if (!sc->link_active) { 2332 1.3.2.2 martin ifmr->ifm_active |= IFM_NONE; 2333 1.3.2.2 martin return; 2334 1.3.2.2 martin } 2335 1.3.2.2 martin 2336 1.3.2.2 martin ifmr->ifm_status |= IFM_ACTIVE; 2337 1.3.2.2 martin 2338 1.3.2.2 martin switch (sc->link_speed) { 2339 1.3.2.2 martin case 10: 2340 1.3.2.2 martin ifmr->ifm_active |= IFM_10_T; 2341 1.3.2.2 martin break; 2342 1.3.2.2 martin case 100: 2343 1.3.2.2 martin ifmr->ifm_active |= IFM_100_TX; 2344 1.3.2.2 martin break; 2345 1.3.2.2 martin case 1000: 2346 1.3.2.2 martin ifmr->ifm_active |= IFM_1000_T; 2347 1.3.2.2 martin break; 2348 1.3.2.2 martin case 2500: 2349 1.3.2.2 martin ifmr->ifm_active |= IFM_2500_T; 2350 1.3.2.2 martin break; 2351 1.3.2.2 martin } 2352 1.3.2.2 martin 2353 1.3.2.2 martin if (sc->link_duplex == FULL_DUPLEX) 2354 1.3.2.2 martin ifmr->ifm_active |= IFM_FDX; 2355 1.3.2.2 martin else 2356 1.3.2.2 martin ifmr->ifm_active |= IFM_HDX; 2357 1.3.2.2 martin 2358 1.3.2.2 martin switch (hw->fc.current_mode) { 2359 1.3.2.2 martin case igc_fc_tx_pause: 2360 1.3.2.2 martin ifmr->ifm_active |= IFM_FLOW | IFM_ETH_TXPAUSE; 2361 1.3.2.2 martin break; 2362 1.3.2.2 martin case igc_fc_rx_pause: 2363 1.3.2.2 martin ifmr->ifm_active |= IFM_FLOW | IFM_ETH_RXPAUSE; 2364 1.3.2.2 martin break; 2365 1.3.2.2 martin case igc_fc_full: 2366 1.3.2.2 martin ifmr->ifm_active |= IFM_FLOW | 2367 1.3.2.2 martin IFM_ETH_TXPAUSE | IFM_ETH_RXPAUSE; 2368 1.3.2.2 martin break; 2369 1.3.2.2 martin case igc_fc_none: 2370 1.3.2.2 martin default: 2371 1.3.2.2 martin break; 2372 1.3.2.2 martin } 2373 1.3.2.2 martin } 2374 1.3.2.2 martin 2375 1.3.2.2 martin /********************************************************************* 2376 1.3.2.2 martin * 2377 1.3.2.2 martin * Media Ioctl callback 2378 1.3.2.2 martin * 2379 1.3.2.2 martin * This routine is called when the user changes speed/duplex using 2380 1.3.2.2 martin * media/mediopt option with ifconfig. 2381 1.3.2.2 martin * 2382 1.3.2.2 martin **********************************************************************/ 2383 1.3.2.2 martin static int 2384 1.3.2.2 martin igc_media_change(struct ifnet *ifp) 2385 1.3.2.2 martin { 2386 1.3.2.2 martin struct igc_softc *sc = ifp->if_softc; 2387 1.3.2.2 martin struct ifmedia *ifm = &sc->media; 2388 1.3.2.2 martin 2389 1.3.2.2 martin if (IFM_TYPE(ifm->ifm_media) != IFM_ETHER) 2390 1.3.2.2 martin return EINVAL; 2391 1.3.2.2 martin 2392 1.3.2.2 martin sc->hw.mac.autoneg = DO_AUTO_NEG; 2393 1.3.2.2 martin 2394 1.3.2.2 martin switch (IFM_SUBTYPE(ifm->ifm_media)) { 2395 1.3.2.2 martin case IFM_AUTO: 2396 1.3.2.2 martin sc->hw.phy.autoneg_advertised = AUTONEG_ADV_DEFAULT; 2397 1.3.2.2 martin break; 2398 1.3.2.2 martin case IFM_2500_T: 2399 1.3.2.2 martin sc->hw.phy.autoneg_advertised = ADVERTISE_2500_FULL; 2400 1.3.2.2 martin break; 2401 1.3.2.2 martin case IFM_1000_T: 2402 1.3.2.2 martin sc->hw.phy.autoneg_advertised = ADVERTISE_1000_FULL; 2403 1.3.2.2 martin break; 2404 1.3.2.2 martin case IFM_100_TX: 2405 1.3.2.3 martin if ((ifm->ifm_media & IFM_GMASK) == IFM_FDX) 2406 1.3.2.2 martin sc->hw.phy.autoneg_advertised = ADVERTISE_100_FULL; 2407 1.3.2.3 martin else 2408 1.3.2.3 martin sc->hw.phy.autoneg_advertised = ADVERTISE_100_HALF; 2409 1.3.2.2 martin break; 2410 1.3.2.2 martin case IFM_10_T: 2411 1.3.2.3 martin if ((ifm->ifm_media & IFM_GMASK) == IFM_FDX) 2412 1.3.2.2 martin sc->hw.phy.autoneg_advertised = ADVERTISE_10_FULL; 2413 1.3.2.3 martin else 2414 1.3.2.3 martin sc->hw.phy.autoneg_advertised = ADVERTISE_10_HALF; 2415 1.3.2.2 martin break; 2416 1.3.2.2 martin default: 2417 1.3.2.2 martin return EINVAL; 2418 1.3.2.2 martin } 2419 1.3.2.2 martin 2420 1.3.2.2 martin igc_init_locked(sc); 2421 1.3.2.2 martin 2422 1.3.2.2 martin return 0; 2423 1.3.2.2 martin } 2424 1.3.2.2 martin 2425 1.3.2.2 martin static int 2426 1.3.2.2 martin igc_ifflags_cb(struct ethercom *ec) 2427 1.3.2.2 martin { 2428 1.3.2.2 martin struct ifnet *ifp = &ec->ec_if; 2429 1.3.2.2 martin struct igc_softc *sc = ifp->if_softc; 2430 1.3.2.2 martin int rc = 0; 2431 1.3.2.2 martin u_short iffchange; 2432 1.3.2.2 martin bool needreset = false; 2433 1.3.2.2 martin 2434 1.3.2.2 martin DPRINTF(CFG, "called\n"); 2435 1.3.2.2 martin 2436 1.3.2.2 martin KASSERT(IFNET_LOCKED(ifp)); 2437 1.3.2.2 martin 2438 1.3.2.2 martin mutex_enter(&sc->sc_core_lock); 2439 1.3.2.2 martin 2440 1.3.2.2 martin /* 2441 1.3.2.2 martin * Check for if_flags. 2442 1.3.2.2 martin * Main usage is to prevent linkdown when opening bpf. 2443 1.3.2.2 martin */ 2444 1.3.2.2 martin iffchange = ifp->if_flags ^ sc->sc_if_flags; 2445 1.3.2.2 martin sc->sc_if_flags = ifp->if_flags; 2446 1.3.2.2 martin if ((iffchange & ~(IFF_CANTCHANGE | IFF_DEBUG)) != 0) { 2447 1.3.2.2 martin needreset = true; 2448 1.3.2.2 martin goto ec; 2449 1.3.2.2 martin } 2450 1.3.2.2 martin 2451 1.3.2.2 martin /* iff related updates */ 2452 1.3.2.2 martin if ((iffchange & IFF_PROMISC) != 0) 2453 1.3.2.2 martin igc_set_filter(sc); 2454 1.3.2.2 martin 2455 1.3.2.2 martin #ifdef notyet 2456 1.3.2.2 martin igc_set_vlan(sc); 2457 1.3.2.2 martin #endif 2458 1.3.2.2 martin 2459 1.3.2.2 martin ec: 2460 1.3.2.2 martin #ifdef notyet 2461 1.3.2.2 martin /* Check for ec_capenable. */ 2462 1.3.2.2 martin ecchange = ec->ec_capenable ^ sc->sc_ec_capenable; 2463 1.3.2.2 martin sc->sc_ec_capenable = ec->ec_capenable; 2464 1.3.2.2 martin if ((ecchange & ~ETHERCAP_SOMETHING) != 0) { 2465 1.3.2.2 martin needreset = true; 2466 1.3.2.2 martin goto out; 2467 1.3.2.2 martin } 2468 1.3.2.2 martin #endif 2469 1.3.2.2 martin if (needreset) 2470 1.3.2.2 martin rc = ENETRESET; 2471 1.3.2.2 martin 2472 1.3.2.2 martin mutex_exit(&sc->sc_core_lock); 2473 1.3.2.2 martin 2474 1.3.2.2 martin return rc; 2475 1.3.2.2 martin } 2476 1.3.2.2 martin 2477 1.3.2.2 martin static void 2478 1.3.2.2 martin igc_set_filter(struct igc_softc *sc) 2479 1.3.2.2 martin { 2480 1.3.2.2 martin struct ethercom *ec = &sc->sc_ec; 2481 1.3.2.2 martin uint32_t rctl; 2482 1.3.2.2 martin 2483 1.3.2.2 martin rctl = IGC_READ_REG(&sc->hw, IGC_RCTL); 2484 1.3.2.2 martin rctl &= ~(IGC_RCTL_BAM |IGC_RCTL_UPE | IGC_RCTL_MPE); 2485 1.3.2.2 martin 2486 1.3.2.2 martin if ((sc->sc_if_flags & IFF_BROADCAST) != 0) 2487 1.3.2.2 martin rctl |= IGC_RCTL_BAM; 2488 1.3.2.2 martin if ((sc->sc_if_flags & IFF_PROMISC) != 0) { 2489 1.3.2.2 martin DPRINTF(CFG, "promisc\n"); 2490 1.3.2.2 martin rctl |= IGC_RCTL_UPE; 2491 1.3.2.2 martin ETHER_LOCK(ec); 2492 1.3.2.2 martin allmulti: 2493 1.3.2.2 martin ec->ec_flags |= ETHER_F_ALLMULTI; 2494 1.3.2.2 martin ETHER_UNLOCK(ec); 2495 1.3.2.2 martin rctl |= IGC_RCTL_MPE; 2496 1.3.2.2 martin } else { 2497 1.3.2.2 martin struct ether_multistep step; 2498 1.3.2.2 martin struct ether_multi *enm; 2499 1.3.2.2 martin int mcnt = 0; 2500 1.3.2.2 martin 2501 1.3.2.2 martin memset(sc->mta, 0, IGC_MTA_LEN); 2502 1.3.2.2 martin 2503 1.3.2.2 martin ETHER_LOCK(ec); 2504 1.3.2.2 martin ETHER_FIRST_MULTI(step, ec, enm); 2505 1.3.2.2 martin while (enm != NULL) { 2506 1.3.2.2 martin if (((memcmp(enm->enm_addrlo, enm->enm_addrhi, 2507 1.3.2.2 martin ETHER_ADDR_LEN)) != 0) || 2508 1.3.2.2 martin (mcnt >= MAX_NUM_MULTICAST_ADDRESSES)) { 2509 1.3.2.2 martin /* 2510 1.3.2.2 martin * We must listen to a range of multicast 2511 1.3.2.2 martin * addresses. For now, just accept all 2512 1.3.2.2 martin * multicasts, rather than trying to set only 2513 1.3.2.2 martin * those filter bits needed to match the range. 2514 1.3.2.2 martin * (At this time, the only use of address 2515 1.3.2.2 martin * ranges is for IP multicast routing, for 2516 1.3.2.2 martin * which the range is big enough to require all 2517 1.3.2.2 martin * bits set.) 2518 1.3.2.2 martin */ 2519 1.3.2.2 martin goto allmulti; 2520 1.3.2.2 martin } 2521 1.3.2.2 martin DPRINTF(CFG, "%d: %s\n", mcnt, 2522 1.3.2.2 martin ether_sprintf(enm->enm_addrlo)); 2523 1.3.2.2 martin memcpy(&sc->mta[mcnt * ETHER_ADDR_LEN], 2524 1.3.2.2 martin enm->enm_addrlo, ETHER_ADDR_LEN); 2525 1.3.2.2 martin 2526 1.3.2.2 martin mcnt++; 2527 1.3.2.2 martin ETHER_NEXT_MULTI(step, enm); 2528 1.3.2.2 martin } 2529 1.3.2.2 martin ec->ec_flags &= ~ETHER_F_ALLMULTI; 2530 1.3.2.2 martin ETHER_UNLOCK(ec); 2531 1.3.2.2 martin 2532 1.3.2.2 martin DPRINTF(CFG, "hw filter\n"); 2533 1.3.2.2 martin igc_update_mc_addr_list(&sc->hw, sc->mta, mcnt); 2534 1.3.2.2 martin } 2535 1.3.2.2 martin 2536 1.3.2.2 martin IGC_WRITE_REG(&sc->hw, IGC_RCTL, rctl); 2537 1.3.2.2 martin } 2538 1.3.2.2 martin 2539 1.3.2.2 martin static void 2540 1.3.2.2 martin igc_update_link_status(struct igc_softc *sc) 2541 1.3.2.2 martin { 2542 1.3.2.2 martin struct ifnet *ifp = &sc->sc_ec.ec_if; 2543 1.3.2.2 martin struct igc_hw *hw = &sc->hw; 2544 1.3.2.2 martin 2545 1.3.2.4 martin if (hw->mac.get_link_status == true) 2546 1.3.2.4 martin igc_check_for_link(hw); 2547 1.3.2.4 martin 2548 1.3.2.2 martin if (IGC_READ_REG(&sc->hw, IGC_STATUS) & IGC_STATUS_LU) { 2549 1.3.2.2 martin if (sc->link_active == 0) { 2550 1.3.2.2 martin igc_get_speed_and_duplex(hw, &sc->link_speed, 2551 1.3.2.2 martin &sc->link_duplex); 2552 1.3.2.2 martin sc->link_active = 1; 2553 1.3.2.2 martin ifp->if_baudrate = IF_Mbps(sc->link_speed); 2554 1.3.2.2 martin if_link_state_change(ifp, LINK_STATE_UP); 2555 1.3.2.2 martin } 2556 1.3.2.2 martin } else { 2557 1.3.2.2 martin if (sc->link_active == 1) { 2558 1.3.2.2 martin ifp->if_baudrate = sc->link_speed = 0; 2559 1.3.2.2 martin sc->link_duplex = 0; 2560 1.3.2.2 martin sc->link_active = 0; 2561 1.3.2.2 martin if_link_state_change(ifp, LINK_STATE_DOWN); 2562 1.3.2.2 martin } 2563 1.3.2.2 martin } 2564 1.3.2.2 martin } 2565 1.3.2.2 martin 2566 1.3.2.2 martin /********************************************************************* 2567 1.3.2.2 martin * 2568 1.3.2.2 martin * Get a buffer from system mbuf buffer pool. 2569 1.3.2.2 martin * 2570 1.3.2.2 martin **********************************************************************/ 2571 1.3.2.2 martin static int 2572 1.3.2.2 martin igc_get_buf(struct rx_ring *rxr, int id, bool strict) 2573 1.3.2.2 martin { 2574 1.3.2.2 martin struct igc_softc *sc = rxr->sc; 2575 1.3.2.2 martin struct igc_queue *q = rxr->rxr_igcq; 2576 1.3.2.2 martin struct igc_rx_buf *rxbuf = &rxr->rx_buffers[id]; 2577 1.3.2.2 martin bus_dmamap_t map = rxbuf->map; 2578 1.3.2.2 martin struct mbuf *m; 2579 1.3.2.2 martin int error; 2580 1.3.2.2 martin 2581 1.3.2.2 martin if (__predict_false(rxbuf->buf)) { 2582 1.3.2.2 martin if (strict) { 2583 1.3.2.2 martin DPRINTF(RX, "slot %d already has an mbuf\n", id); 2584 1.3.2.2 martin return EINVAL; 2585 1.3.2.2 martin } 2586 1.3.2.2 martin return 0; 2587 1.3.2.2 martin } 2588 1.3.2.2 martin 2589 1.3.2.2 martin MGETHDR(m, M_DONTWAIT, MT_DATA); 2590 1.3.2.2 martin if (__predict_false(m == NULL)) { 2591 1.3.2.2 martin enobuf: 2592 1.3.2.2 martin IGC_QUEUE_EVENT(q, rx_no_mbuf, 1); 2593 1.3.2.2 martin return ENOBUFS; 2594 1.3.2.2 martin } 2595 1.3.2.2 martin 2596 1.3.2.2 martin MCLGET(m, M_DONTWAIT); 2597 1.3.2.2 martin if (__predict_false(!(m->m_flags & M_EXT))) { 2598 1.3.2.2 martin m_freem(m); 2599 1.3.2.2 martin goto enobuf; 2600 1.3.2.2 martin } 2601 1.3.2.2 martin 2602 1.3.2.2 martin m->m_len = m->m_pkthdr.len = sc->rx_mbuf_sz; 2603 1.3.2.2 martin 2604 1.3.2.2 martin error = bus_dmamap_load_mbuf(rxr->rxdma.dma_tag, map, m, 2605 1.3.2.2 martin BUS_DMA_READ | BUS_DMA_NOWAIT); 2606 1.3.2.2 martin if (error) { 2607 1.3.2.2 martin m_freem(m); 2608 1.3.2.2 martin return error; 2609 1.3.2.2 martin } 2610 1.3.2.2 martin 2611 1.3.2.2 martin bus_dmamap_sync(rxr->rxdma.dma_tag, map, 0, 2612 1.3.2.2 martin map->dm_mapsize, BUS_DMASYNC_PREREAD); 2613 1.3.2.2 martin rxbuf->buf = m; 2614 1.3.2.2 martin 2615 1.3.2.2 martin union igc_adv_rx_desc *rxdesc = &rxr->rx_base[id]; 2616 1.3.2.2 martin igc_rxdesc_sync(rxr, id, BUS_DMASYNC_POSTWRITE | BUS_DMASYNC_POSTREAD); 2617 1.3.2.2 martin rxdesc->read.pkt_addr = htole64(map->dm_segs[0].ds_addr); 2618 1.3.2.2 martin igc_rxdesc_sync(rxr, id, BUS_DMASYNC_PREWRITE | BUS_DMASYNC_PREREAD); 2619 1.3.2.2 martin 2620 1.3.2.2 martin return 0; 2621 1.3.2.2 martin } 2622 1.3.2.2 martin 2623 1.3.2.2 martin static void 2624 1.3.2.2 martin igc_configure_queues(struct igc_softc *sc) 2625 1.3.2.2 martin { 2626 1.3.2.2 martin struct igc_hw *hw = &sc->hw; 2627 1.3.2.2 martin uint32_t ivar; 2628 1.3.2.2 martin 2629 1.3.2.2 martin /* First turn on RSS capability */ 2630 1.3.2.2 martin IGC_WRITE_REG(hw, IGC_GPIE, IGC_GPIE_MSIX_MODE | IGC_GPIE_EIAME | 2631 1.3.2.2 martin IGC_GPIE_PBA | IGC_GPIE_NSICR); 2632 1.3.2.2 martin 2633 1.3.2.2 martin /* Set the starting interrupt rate */ 2634 1.3.2.2 martin uint32_t newitr = (4000000 / MAX_INTS_PER_SEC) & 0x7FFC; 2635 1.3.2.2 martin newitr |= IGC_EITR_CNT_IGNR; 2636 1.3.2.2 martin 2637 1.3.2.2 martin /* Turn on MSI-X */ 2638 1.3.2.2 martin uint32_t newmask = 0; 2639 1.3.2.2 martin for (int iq = 0; iq < sc->sc_nqueues; iq++) { 2640 1.3.2.2 martin struct igc_queue *q = &sc->queues[iq]; 2641 1.3.2.2 martin 2642 1.3.2.2 martin /* RX entries */ 2643 1.3.2.2 martin igc_set_queues(sc, iq, q->msix, 0); 2644 1.3.2.2 martin /* TX entries */ 2645 1.3.2.2 martin igc_set_queues(sc, iq, q->msix, 1); 2646 1.3.2.2 martin newmask |= q->eims; 2647 1.3.2.2 martin IGC_WRITE_REG(hw, IGC_EITR(q->msix), newitr); 2648 1.3.2.2 martin } 2649 1.3.2.2 martin sc->msix_queuesmask = newmask; 2650 1.3.2.2 martin 2651 1.3.2.2 martin #if 1 2652 1.3.2.2 martin ivar = IGC_READ_REG_ARRAY(hw, IGC_IVAR0, 0); 2653 1.3.2.2 martin DPRINTF(CFG, "ivar(0)=0x%x\n", ivar); 2654 1.3.2.2 martin ivar = IGC_READ_REG_ARRAY(hw, IGC_IVAR0, 1); 2655 1.3.2.2 martin DPRINTF(CFG, "ivar(1)=0x%x\n", ivar); 2656 1.3.2.2 martin #endif 2657 1.3.2.2 martin 2658 1.3.2.2 martin /* And for the link interrupt */ 2659 1.3.2.2 martin ivar = (sc->linkvec | IGC_IVAR_VALID) << 8; 2660 1.3.2.2 martin sc->msix_linkmask = 1 << sc->linkvec; 2661 1.3.2.2 martin IGC_WRITE_REG(hw, IGC_IVAR_MISC, ivar); 2662 1.3.2.2 martin } 2663 1.3.2.2 martin 2664 1.3.2.2 martin static void 2665 1.3.2.2 martin igc_set_queues(struct igc_softc *sc, uint32_t entry, uint32_t vector, int type) 2666 1.3.2.2 martin { 2667 1.3.2.2 martin struct igc_hw *hw = &sc->hw; 2668 1.3.2.2 martin const uint32_t index = entry >> 1; 2669 1.3.2.2 martin uint32_t ivar = IGC_READ_REG_ARRAY(hw, IGC_IVAR0, index); 2670 1.3.2.2 martin 2671 1.3.2.2 martin if (type) { 2672 1.3.2.2 martin if (entry & 1) { 2673 1.3.2.2 martin ivar &= 0x00FFFFFF; 2674 1.3.2.2 martin ivar |= (vector | IGC_IVAR_VALID) << 24; 2675 1.3.2.2 martin } else { 2676 1.3.2.2 martin ivar &= 0xFFFF00FF; 2677 1.3.2.2 martin ivar |= (vector | IGC_IVAR_VALID) << 8; 2678 1.3.2.2 martin } 2679 1.3.2.2 martin } else { 2680 1.3.2.2 martin if (entry & 1) { 2681 1.3.2.2 martin ivar &= 0xFF00FFFF; 2682 1.3.2.2 martin ivar |= (vector | IGC_IVAR_VALID) << 16; 2683 1.3.2.2 martin } else { 2684 1.3.2.2 martin ivar &= 0xFFFFFF00; 2685 1.3.2.2 martin ivar |= vector | IGC_IVAR_VALID; 2686 1.3.2.2 martin } 2687 1.3.2.2 martin } 2688 1.3.2.2 martin IGC_WRITE_REG_ARRAY(hw, IGC_IVAR0, index, ivar); 2689 1.3.2.2 martin } 2690 1.3.2.2 martin 2691 1.3.2.2 martin static void 2692 1.3.2.2 martin igc_enable_queue(struct igc_softc *sc, uint32_t eims) 2693 1.3.2.2 martin { 2694 1.3.2.2 martin IGC_WRITE_REG(&sc->hw, IGC_EIMS, eims); 2695 1.3.2.2 martin } 2696 1.3.2.2 martin 2697 1.3.2.2 martin static void 2698 1.3.2.2 martin igc_enable_intr(struct igc_softc *sc) 2699 1.3.2.2 martin { 2700 1.3.2.2 martin struct igc_hw *hw = &sc->hw; 2701 1.3.2.2 martin 2702 1.3.2.2 martin if (sc->sc_intr_type == PCI_INTR_TYPE_MSIX) { 2703 1.3.2.2 martin const uint32_t mask = sc->msix_queuesmask | sc->msix_linkmask; 2704 1.3.2.2 martin 2705 1.3.2.2 martin IGC_WRITE_REG(hw, IGC_EIAC, mask); 2706 1.3.2.2 martin IGC_WRITE_REG(hw, IGC_EIAM, mask); 2707 1.3.2.2 martin IGC_WRITE_REG(hw, IGC_EIMS, mask); 2708 1.3.2.2 martin IGC_WRITE_REG(hw, IGC_IMS, IGC_IMS_LSC); 2709 1.3.2.2 martin } else { 2710 1.3.2.2 martin IGC_WRITE_REG(hw, IGC_IMS, IMS_ENABLE_MASK); 2711 1.3.2.2 martin } 2712 1.3.2.2 martin IGC_WRITE_FLUSH(hw); 2713 1.3.2.2 martin } 2714 1.3.2.2 martin 2715 1.3.2.2 martin static void 2716 1.3.2.2 martin igc_disable_intr(struct igc_softc *sc) 2717 1.3.2.2 martin { 2718 1.3.2.2 martin struct igc_hw *hw = &sc->hw; 2719 1.3.2.2 martin 2720 1.3.2.2 martin if (sc->sc_intr_type == PCI_INTR_TYPE_MSIX) { 2721 1.3.2.2 martin IGC_WRITE_REG(hw, IGC_EIMC, 0xffffffff); 2722 1.3.2.2 martin IGC_WRITE_REG(hw, IGC_EIAC, 0); 2723 1.3.2.2 martin } 2724 1.3.2.2 martin IGC_WRITE_REG(hw, IGC_IMC, 0xffffffff); 2725 1.3.2.2 martin IGC_WRITE_FLUSH(hw); 2726 1.3.2.2 martin } 2727 1.3.2.2 martin 2728 1.3.2.2 martin static int 2729 1.3.2.2 martin igc_intr_link(void *arg) 2730 1.3.2.2 martin { 2731 1.3.2.2 martin struct igc_softc *sc = (struct igc_softc *)arg; 2732 1.3.2.2 martin const uint32_t reg_icr = IGC_READ_REG(&sc->hw, IGC_ICR); 2733 1.3.2.2 martin 2734 1.3.2.2 martin IGC_GLOBAL_EVENT(sc, link, 1); 2735 1.3.2.2 martin 2736 1.3.2.2 martin if (reg_icr & IGC_ICR_LSC) { 2737 1.3.2.2 martin mutex_enter(&sc->sc_core_lock); 2738 1.3.2.2 martin sc->hw.mac.get_link_status = true; 2739 1.3.2.2 martin igc_update_link_status(sc); 2740 1.3.2.2 martin mutex_exit(&sc->sc_core_lock); 2741 1.3.2.2 martin } 2742 1.3.2.2 martin 2743 1.3.2.2 martin IGC_WRITE_REG(&sc->hw, IGC_IMS, IGC_IMS_LSC); 2744 1.3.2.2 martin IGC_WRITE_REG(&sc->hw, IGC_EIMS, sc->msix_linkmask); 2745 1.3.2.2 martin 2746 1.3.2.2 martin return 1; 2747 1.3.2.2 martin } 2748 1.3.2.2 martin 2749 1.3.2.2 martin static int 2750 1.3.2.2 martin igc_intr_queue(void *arg) 2751 1.3.2.2 martin { 2752 1.3.2.2 martin struct igc_queue *iq = arg; 2753 1.3.2.2 martin struct igc_softc *sc = iq->sc; 2754 1.3.2.2 martin struct ifnet *ifp = &sc->sc_ec.ec_if; 2755 1.3.2.2 martin struct rx_ring *rxr = iq->rxr; 2756 1.3.2.2 martin struct tx_ring *txr = iq->txr; 2757 1.3.2.2 martin const u_int txlimit = sc->sc_tx_intr_process_limit, 2758 1.3.2.2 martin rxlimit = sc->sc_rx_intr_process_limit; 2759 1.3.2.2 martin bool txmore, rxmore; 2760 1.3.2.2 martin 2761 1.3.2.2 martin IGC_QUEUE_EVENT(iq, irqs, 1); 2762 1.3.2.2 martin 2763 1.3.2.2 martin if (__predict_false(!ISSET(ifp->if_flags, IFF_RUNNING))) 2764 1.3.2.2 martin return 0; 2765 1.3.2.2 martin 2766 1.3.2.2 martin mutex_enter(&txr->txr_lock); 2767 1.3.2.2 martin txmore = igc_txeof(txr, txlimit); 2768 1.3.2.2 martin mutex_exit(&txr->txr_lock); 2769 1.3.2.2 martin mutex_enter(&rxr->rxr_lock); 2770 1.3.2.2 martin rxmore = igc_rxeof(rxr, rxlimit); 2771 1.3.2.2 martin mutex_exit(&rxr->rxr_lock); 2772 1.3.2.2 martin 2773 1.3.2.2 martin if (txmore || rxmore) { 2774 1.3.2.2 martin IGC_QUEUE_EVENT(iq, req, 1); 2775 1.3.2.2 martin igc_sched_handle_queue(sc, iq); 2776 1.3.2.2 martin } else { 2777 1.3.2.2 martin igc_enable_queue(sc, iq->eims); 2778 1.3.2.2 martin } 2779 1.3.2.2 martin 2780 1.3.2.2 martin return 1; 2781 1.3.2.2 martin } 2782 1.3.2.2 martin 2783 1.3.2.2 martin static int 2784 1.3.2.2 martin igc_intr(void *arg) 2785 1.3.2.2 martin { 2786 1.3.2.2 martin struct igc_softc *sc = arg; 2787 1.3.2.2 martin struct ifnet *ifp = &sc->sc_ec.ec_if; 2788 1.3.2.2 martin struct igc_queue *iq = &sc->queues[0]; 2789 1.3.2.2 martin struct rx_ring *rxr = iq->rxr; 2790 1.3.2.2 martin struct tx_ring *txr = iq->txr; 2791 1.3.2.2 martin const u_int txlimit = sc->sc_tx_intr_process_limit, 2792 1.3.2.2 martin rxlimit = sc->sc_rx_intr_process_limit; 2793 1.3.2.2 martin bool txmore, rxmore; 2794 1.3.2.2 martin 2795 1.3.2.2 martin if (__predict_false(!ISSET(ifp->if_flags, IFF_RUNNING))) 2796 1.3.2.2 martin return 0; 2797 1.3.2.2 martin 2798 1.3.2.2 martin const uint32_t reg_icr = IGC_READ_REG(&sc->hw, IGC_ICR); 2799 1.3.2.2 martin DPRINTF(MISC, "reg_icr=0x%x\n", reg_icr); 2800 1.3.2.2 martin 2801 1.3.2.2 martin /* Definitely not our interrupt. */ 2802 1.3.2.2 martin if (reg_icr == 0x0) { 2803 1.3.2.2 martin DPRINTF(MISC, "not for me"); 2804 1.3.2.2 martin return 0; 2805 1.3.2.2 martin } 2806 1.3.2.2 martin 2807 1.3.2.2 martin IGC_QUEUE_EVENT(iq, irqs, 1); 2808 1.3.2.2 martin 2809 1.3.2.2 martin /* Hot eject? */ 2810 1.3.2.2 martin if (__predict_false(reg_icr == 0xffffffff)) { 2811 1.3.2.2 martin DPRINTF(MISC, "hot eject\n"); 2812 1.3.2.2 martin return 0; 2813 1.3.2.2 martin } 2814 1.3.2.2 martin 2815 1.3.2.2 martin if (__predict_false(!(reg_icr & IGC_ICR_INT_ASSERTED))) { 2816 1.3.2.2 martin DPRINTF(MISC, "not set IGC_ICR_INT_ASSERTED"); 2817 1.3.2.2 martin return 0; 2818 1.3.2.2 martin } 2819 1.3.2.2 martin 2820 1.3.2.2 martin /* 2821 1.3.2.2 martin * Only MSI-X interrupts have one-shot behavior by taking advantage 2822 1.3.2.2 martin * of the EIAC register. Thus, explicitly disable interrupts. This 2823 1.3.2.2 martin * also works around the MSI message reordering errata on certain 2824 1.3.2.2 martin * systems. 2825 1.3.2.2 martin */ 2826 1.3.2.2 martin igc_disable_intr(sc); 2827 1.3.2.2 martin 2828 1.3.2.2 martin mutex_enter(&txr->txr_lock); 2829 1.3.2.2 martin txmore = igc_txeof(txr, txlimit); 2830 1.3.2.2 martin mutex_exit(&txr->txr_lock); 2831 1.3.2.2 martin mutex_enter(&rxr->rxr_lock); 2832 1.3.2.2 martin rxmore = igc_rxeof(rxr, rxlimit); 2833 1.3.2.2 martin mutex_exit(&rxr->rxr_lock); 2834 1.3.2.2 martin 2835 1.3.2.2 martin /* Link status change */ 2836 1.3.2.2 martin // XXXX FreeBSD checks IGC_ICR_RXSEQ 2837 1.3.2.2 martin if (__predict_false(reg_icr & IGC_ICR_LSC)) { 2838 1.3.2.2 martin IGC_GLOBAL_EVENT(sc, link, 1); 2839 1.3.2.2 martin mutex_enter(&sc->sc_core_lock); 2840 1.3.2.2 martin sc->hw.mac.get_link_status = true; 2841 1.3.2.2 martin igc_update_link_status(sc); 2842 1.3.2.2 martin mutex_exit(&sc->sc_core_lock); 2843 1.3.2.2 martin } 2844 1.3.2.2 martin 2845 1.3.2.2 martin if (txmore || rxmore) { 2846 1.3.2.2 martin IGC_QUEUE_EVENT(iq, req, 1); 2847 1.3.2.2 martin igc_sched_handle_queue(sc, iq); 2848 1.3.2.2 martin } else { 2849 1.3.2.2 martin igc_enable_intr(sc); 2850 1.3.2.2 martin } 2851 1.3.2.2 martin 2852 1.3.2.2 martin return 1; 2853 1.3.2.2 martin } 2854 1.3.2.2 martin 2855 1.3.2.2 martin static void 2856 1.3.2.2 martin igc_handle_queue(void *arg) 2857 1.3.2.2 martin { 2858 1.3.2.2 martin struct igc_queue *iq = arg; 2859 1.3.2.2 martin struct igc_softc *sc = iq->sc; 2860 1.3.2.2 martin struct tx_ring *txr = iq->txr; 2861 1.3.2.2 martin struct rx_ring *rxr = iq->rxr; 2862 1.3.2.2 martin const u_int txlimit = sc->sc_tx_process_limit, 2863 1.3.2.2 martin rxlimit = sc->sc_rx_process_limit; 2864 1.3.2.2 martin bool txmore, rxmore; 2865 1.3.2.2 martin 2866 1.3.2.2 martin IGC_QUEUE_EVENT(iq, handleq, 1); 2867 1.3.2.2 martin 2868 1.3.2.2 martin mutex_enter(&txr->txr_lock); 2869 1.3.2.2 martin txmore = igc_txeof(txr, txlimit); 2870 1.3.2.2 martin /* for ALTQ, dequeue from if_snd */ 2871 1.3.2.2 martin if (txr->me == 0) { 2872 1.3.2.2 martin struct ifnet *ifp = &sc->sc_ec.ec_if; 2873 1.3.2.2 martin 2874 1.3.2.2 martin igc_tx_common_locked(ifp, txr, IGC_TX_START); 2875 1.3.2.2 martin } 2876 1.3.2.2 martin mutex_exit(&txr->txr_lock); 2877 1.3.2.2 martin 2878 1.3.2.2 martin mutex_enter(&rxr->rxr_lock); 2879 1.3.2.2 martin rxmore = igc_rxeof(rxr, rxlimit); 2880 1.3.2.2 martin mutex_exit(&rxr->rxr_lock); 2881 1.3.2.2 martin 2882 1.3.2.2 martin if (txmore || rxmore) { 2883 1.3.2.2 martin igc_sched_handle_queue(sc, iq); 2884 1.3.2.2 martin } else { 2885 1.3.2.2 martin if (sc->sc_intr_type == PCI_INTR_TYPE_MSIX) 2886 1.3.2.2 martin igc_enable_queue(sc, iq->eims); 2887 1.3.2.2 martin else 2888 1.3.2.2 martin igc_enable_intr(sc); 2889 1.3.2.2 martin } 2890 1.3.2.2 martin } 2891 1.3.2.2 martin 2892 1.3.2.2 martin static void 2893 1.3.2.2 martin igc_handle_queue_work(struct work *wk, void *context) 2894 1.3.2.2 martin { 2895 1.3.2.2 martin struct igc_queue *iq = 2896 1.3.2.2 martin container_of(wk, struct igc_queue, igcq_wq_cookie); 2897 1.3.2.2 martin 2898 1.3.2.2 martin igc_handle_queue(iq); 2899 1.3.2.2 martin } 2900 1.3.2.2 martin 2901 1.3.2.2 martin static void 2902 1.3.2.2 martin igc_sched_handle_queue(struct igc_softc *sc, struct igc_queue *iq) 2903 1.3.2.2 martin { 2904 1.3.2.2 martin 2905 1.3.2.2 martin if (iq->igcq_workqueue) { 2906 1.3.2.2 martin /* XXXRO notyet */ 2907 1.3.2.2 martin workqueue_enqueue(sc->sc_queue_wq, &iq->igcq_wq_cookie, 2908 1.3.2.2 martin curcpu()); 2909 1.3.2.2 martin } else { 2910 1.3.2.2 martin softint_schedule(iq->igcq_si); 2911 1.3.2.2 martin } 2912 1.3.2.2 martin } 2913 1.3.2.2 martin 2914 1.3.2.2 martin static void 2915 1.3.2.2 martin igc_barrier_handle_queue(struct igc_softc *sc) 2916 1.3.2.2 martin { 2917 1.3.2.2 martin 2918 1.3.2.2 martin if (sc->sc_txrx_workqueue) { 2919 1.3.2.2 martin for (int iq = 0; iq < sc->sc_nqueues; iq++) { 2920 1.3.2.2 martin struct igc_queue *q = &sc->queues[iq]; 2921 1.3.2.2 martin 2922 1.3.2.2 martin workqueue_wait(sc->sc_queue_wq, &q->igcq_wq_cookie); 2923 1.3.2.2 martin } 2924 1.3.2.2 martin } else { 2925 1.3.2.2 martin xc_barrier(0); 2926 1.3.2.2 martin } 2927 1.3.2.2 martin } 2928 1.3.2.2 martin 2929 1.3.2.2 martin /********************************************************************* 2930 1.3.2.2 martin * 2931 1.3.2.2 martin * Allocate memory for tx_buffer structures. The tx_buffer stores all 2932 1.3.2.2 martin * the information needed to transmit a packet on the wire. 2933 1.3.2.2 martin * 2934 1.3.2.2 martin **********************************************************************/ 2935 1.3.2.2 martin static int 2936 1.3.2.2 martin igc_allocate_transmit_buffers(struct tx_ring *txr) 2937 1.3.2.2 martin { 2938 1.3.2.2 martin struct igc_softc *sc = txr->sc; 2939 1.3.2.2 martin int error; 2940 1.3.2.2 martin 2941 1.3.2.2 martin txr->tx_buffers = 2942 1.3.2.2 martin kmem_zalloc(sc->num_tx_desc * sizeof(struct igc_tx_buf), KM_SLEEP); 2943 1.3.2.2 martin txr->txtag = txr->txdma.dma_tag; 2944 1.3.2.2 martin 2945 1.3.2.2 martin /* Create the descriptor buffer dma maps. */ 2946 1.3.2.2 martin for (int id = 0; id < sc->num_tx_desc; id++) { 2947 1.3.2.2 martin struct igc_tx_buf *txbuf = &txr->tx_buffers[id]; 2948 1.3.2.2 martin 2949 1.3.2.2 martin error = bus_dmamap_create(txr->txdma.dma_tag, 2950 1.3.2.2 martin round_page(IGC_TSO_SIZE + sizeof(struct ether_vlan_header)), 2951 1.3.2.2 martin IGC_MAX_SCATTER, PAGE_SIZE, 0, BUS_DMA_NOWAIT, &txbuf->map); 2952 1.3.2.2 martin if (error != 0) { 2953 1.3.2.2 martin aprint_error_dev(sc->sc_dev, 2954 1.3.2.2 martin "unable to create TX DMA map\n"); 2955 1.3.2.2 martin goto fail; 2956 1.3.2.2 martin } 2957 1.3.2.2 martin 2958 1.3.2.2 martin txbuf->eop_index = -1; 2959 1.3.2.2 martin } 2960 1.3.2.2 martin 2961 1.3.2.2 martin return 0; 2962 1.3.2.2 martin fail: 2963 1.3.2.2 martin return error; 2964 1.3.2.2 martin } 2965 1.3.2.2 martin 2966 1.3.2.2 martin 2967 1.3.2.2 martin /********************************************************************* 2968 1.3.2.2 martin * 2969 1.3.2.2 martin * Allocate and initialize transmit structures. 2970 1.3.2.2 martin * 2971 1.3.2.2 martin **********************************************************************/ 2972 1.3.2.2 martin static int 2973 1.3.2.2 martin igc_setup_transmit_structures(struct igc_softc *sc) 2974 1.3.2.2 martin { 2975 1.3.2.2 martin 2976 1.3.2.2 martin for (int iq = 0; iq < sc->sc_nqueues; iq++) { 2977 1.3.2.2 martin struct tx_ring *txr = &sc->tx_rings[iq]; 2978 1.3.2.2 martin 2979 1.3.2.2 martin if (igc_setup_transmit_ring(txr)) 2980 1.3.2.2 martin goto fail; 2981 1.3.2.2 martin } 2982 1.3.2.2 martin 2983 1.3.2.2 martin return 0; 2984 1.3.2.2 martin fail: 2985 1.3.2.2 martin igc_free_transmit_structures(sc); 2986 1.3.2.2 martin return ENOBUFS; 2987 1.3.2.2 martin } 2988 1.3.2.2 martin 2989 1.3.2.2 martin /********************************************************************* 2990 1.3.2.2 martin * 2991 1.3.2.2 martin * Initialize a transmit ring. 2992 1.3.2.2 martin * 2993 1.3.2.2 martin **********************************************************************/ 2994 1.3.2.2 martin static int 2995 1.3.2.2 martin igc_setup_transmit_ring(struct tx_ring *txr) 2996 1.3.2.2 martin { 2997 1.3.2.2 martin struct igc_softc *sc = txr->sc; 2998 1.3.2.2 martin 2999 1.3.2.2 martin /* Now allocate transmit buffers for the ring. */ 3000 1.3.2.2 martin if (igc_allocate_transmit_buffers(txr)) 3001 1.3.2.2 martin return ENOMEM; 3002 1.3.2.2 martin 3003 1.3.2.2 martin /* Clear the old ring contents */ 3004 1.3.2.2 martin memset(txr->tx_base, 0, 3005 1.3.2.2 martin sizeof(union igc_adv_tx_desc) * sc->num_tx_desc); 3006 1.3.2.2 martin 3007 1.3.2.2 martin /* Reset indices. */ 3008 1.3.2.2 martin txr->next_avail_desc = 0; 3009 1.3.2.2 martin txr->next_to_clean = 0; 3010 1.3.2.2 martin 3011 1.3.2.2 martin bus_dmamap_sync(txr->txdma.dma_tag, txr->txdma.dma_map, 0, 3012 1.3.2.2 martin txr->txdma.dma_map->dm_mapsize, 3013 1.3.2.2 martin BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); 3014 1.3.2.2 martin 3015 1.3.2.2 martin txr->txr_interq = pcq_create(sc->num_tx_desc, KM_SLEEP); 3016 1.3.2.2 martin 3017 1.3.2.2 martin mutex_init(&txr->txr_lock, MUTEX_DEFAULT, IPL_NET); 3018 1.3.2.2 martin 3019 1.3.2.2 martin return 0; 3020 1.3.2.2 martin } 3021 1.3.2.2 martin 3022 1.3.2.2 martin /********************************************************************* 3023 1.3.2.2 martin * 3024 1.3.2.2 martin * Enable transmit unit. 3025 1.3.2.2 martin * 3026 1.3.2.2 martin **********************************************************************/ 3027 1.3.2.2 martin static void 3028 1.3.2.2 martin igc_initialize_transmit_unit(struct igc_softc *sc) 3029 1.3.2.2 martin { 3030 1.3.2.2 martin struct ifnet *ifp = &sc->sc_ec.ec_if; 3031 1.3.2.2 martin struct igc_hw *hw = &sc->hw; 3032 1.3.2.2 martin 3033 1.3.2.2 martin /* Setup the Base and Length of the TX descriptor ring. */ 3034 1.3.2.2 martin for (int iq = 0; iq < sc->sc_nqueues; iq++) { 3035 1.3.2.2 martin struct tx_ring *txr = &sc->tx_rings[iq]; 3036 1.3.2.2 martin const uint64_t bus_addr = 3037 1.3.2.2 martin txr->txdma.dma_map->dm_segs[0].ds_addr; 3038 1.3.2.2 martin 3039 1.3.2.2 martin /* Base and len of TX ring */ 3040 1.3.2.2 martin IGC_WRITE_REG(hw, IGC_TDLEN(iq), 3041 1.3.2.2 martin sc->num_tx_desc * sizeof(union igc_adv_tx_desc)); 3042 1.3.2.2 martin IGC_WRITE_REG(hw, IGC_TDBAH(iq), (uint32_t)(bus_addr >> 32)); 3043 1.3.2.2 martin IGC_WRITE_REG(hw, IGC_TDBAL(iq), (uint32_t)bus_addr); 3044 1.3.2.2 martin 3045 1.3.2.2 martin /* Init the HEAD/TAIL indices */ 3046 1.3.2.2 martin IGC_WRITE_REG(hw, IGC_TDT(iq), 0 /* XXX txr->next_avail_desc */); 3047 1.3.2.2 martin IGC_WRITE_REG(hw, IGC_TDH(iq), 0); 3048 1.3.2.2 martin 3049 1.3.2.2 martin txr->watchdog_timer = 0; 3050 1.3.2.2 martin 3051 1.3.2.2 martin uint32_t txdctl = 0; /* Clear txdctl */ 3052 1.3.2.2 martin txdctl |= 0x1f; /* PTHRESH */ 3053 1.3.2.2 martin txdctl |= 1 << 8; /* HTHRESH */ 3054 1.3.2.2 martin txdctl |= 1 << 16; /* WTHRESH */ 3055 1.3.2.2 martin txdctl |= 1 << 22; /* Reserved bit 22 must always be 1 */ 3056 1.3.2.2 martin txdctl |= IGC_TXDCTL_GRAN; 3057 1.3.2.2 martin txdctl |= 1 << 25; /* LWTHRESH */ 3058 1.3.2.2 martin 3059 1.3.2.2 martin IGC_WRITE_REG(hw, IGC_TXDCTL(iq), txdctl); 3060 1.3.2.2 martin } 3061 1.3.2.2 martin ifp->if_timer = 0; 3062 1.3.2.2 martin 3063 1.3.2.2 martin /* Program the Transmit Control Register */ 3064 1.3.2.2 martin uint32_t tctl = IGC_READ_REG(&sc->hw, IGC_TCTL); 3065 1.3.2.2 martin tctl &= ~IGC_TCTL_CT; 3066 1.3.2.2 martin tctl |= (IGC_TCTL_PSP | IGC_TCTL_RTLC | IGC_TCTL_EN | 3067 1.3.2.2 martin (IGC_COLLISION_THRESHOLD << IGC_CT_SHIFT)); 3068 1.3.2.2 martin 3069 1.3.2.2 martin /* This write will effectively turn on the transmit unit. */ 3070 1.3.2.2 martin IGC_WRITE_REG(&sc->hw, IGC_TCTL, tctl); 3071 1.3.2.2 martin } 3072 1.3.2.2 martin 3073 1.3.2.2 martin /********************************************************************* 3074 1.3.2.2 martin * 3075 1.3.2.2 martin * Free all transmit rings. 3076 1.3.2.2 martin * 3077 1.3.2.2 martin **********************************************************************/ 3078 1.3.2.2 martin static void 3079 1.3.2.2 martin igc_free_transmit_structures(struct igc_softc *sc) 3080 1.3.2.2 martin { 3081 1.3.2.2 martin 3082 1.3.2.2 martin for (int iq = 0; iq < sc->sc_nqueues; iq++) { 3083 1.3.2.2 martin struct tx_ring *txr = &sc->tx_rings[iq]; 3084 1.3.2.2 martin 3085 1.3.2.2 martin igc_free_transmit_buffers(txr); 3086 1.3.2.2 martin } 3087 1.3.2.2 martin } 3088 1.3.2.2 martin 3089 1.3.2.2 martin /********************************************************************* 3090 1.3.2.2 martin * 3091 1.3.2.2 martin * Free transmit ring related data structures. 3092 1.3.2.2 martin * 3093 1.3.2.2 martin **********************************************************************/ 3094 1.3.2.2 martin static void 3095 1.3.2.2 martin igc_free_transmit_buffers(struct tx_ring *txr) 3096 1.3.2.2 martin { 3097 1.3.2.2 martin struct igc_softc *sc = txr->sc; 3098 1.3.2.2 martin 3099 1.3.2.2 martin if (txr->tx_buffers == NULL) 3100 1.3.2.2 martin return; 3101 1.3.2.2 martin 3102 1.3.2.2 martin igc_withdraw_transmit_packets(txr, true); 3103 1.3.2.2 martin 3104 1.3.2.2 martin kmem_free(txr->tx_buffers, 3105 1.3.2.2 martin sc->num_tx_desc * sizeof(struct igc_tx_buf)); 3106 1.3.2.2 martin txr->tx_buffers = NULL; 3107 1.3.2.2 martin txr->txtag = NULL; 3108 1.3.2.2 martin 3109 1.3.2.2 martin pcq_destroy(txr->txr_interq); 3110 1.3.2.2 martin mutex_destroy(&txr->txr_lock); 3111 1.3.2.2 martin } 3112 1.3.2.2 martin 3113 1.3.2.2 martin /********************************************************************* 3114 1.3.2.2 martin * 3115 1.3.2.2 martin * Withdraw transmit packets. 3116 1.3.2.2 martin * 3117 1.3.2.2 martin **********************************************************************/ 3118 1.3.2.2 martin static void 3119 1.3.2.2 martin igc_withdraw_transmit_packets(struct tx_ring *txr, bool destroy) 3120 1.3.2.2 martin { 3121 1.3.2.2 martin struct igc_softc *sc = txr->sc; 3122 1.3.2.2 martin struct igc_queue *q = txr->txr_igcq; 3123 1.3.2.2 martin 3124 1.3.2.2 martin mutex_enter(&txr->txr_lock); 3125 1.3.2.2 martin 3126 1.3.2.2 martin for (int id = 0; id < sc->num_tx_desc; id++) { 3127 1.3.2.2 martin union igc_adv_tx_desc *txdesc = &txr->tx_base[id]; 3128 1.3.2.2 martin 3129 1.3.2.2 martin igc_txdesc_sync(txr, id, 3130 1.3.2.2 martin BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE); 3131 1.3.2.2 martin txdesc->read.buffer_addr = 0; 3132 1.3.2.2 martin txdesc->read.cmd_type_len = 0; 3133 1.3.2.2 martin txdesc->read.olinfo_status = 0; 3134 1.3.2.2 martin igc_txdesc_sync(txr, id, 3135 1.3.2.2 martin BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); 3136 1.3.2.2 martin 3137 1.3.2.2 martin struct igc_tx_buf *txbuf = &txr->tx_buffers[id]; 3138 1.3.2.2 martin bus_dmamap_t map = txbuf->map; 3139 1.3.2.2 martin 3140 1.3.2.2 martin if (map != NULL && map->dm_nsegs > 0) { 3141 1.3.2.2 martin bus_dmamap_sync(txr->txdma.dma_tag, map, 3142 1.3.2.2 martin 0, map->dm_mapsize, BUS_DMASYNC_POSTWRITE); 3143 1.3.2.2 martin bus_dmamap_unload(txr->txdma.dma_tag, map); 3144 1.3.2.2 martin } 3145 1.3.2.2 martin if (txbuf->m_head != NULL) { 3146 1.3.2.2 martin m_freem(txbuf->m_head); 3147 1.3.2.2 martin txbuf->m_head = NULL; 3148 1.3.2.2 martin } 3149 1.3.2.2 martin if (map != NULL && destroy) { 3150 1.3.2.2 martin bus_dmamap_destroy(txr->txdma.dma_tag, map); 3151 1.3.2.2 martin txbuf->map = NULL; 3152 1.3.2.2 martin } 3153 1.3.2.2 martin txbuf->eop_index = -1; 3154 1.3.2.2 martin 3155 1.3.2.2 martin txr->next_avail_desc = 0; 3156 1.3.2.2 martin txr->next_to_clean = 0; 3157 1.3.2.2 martin } 3158 1.3.2.2 martin 3159 1.3.2.2 martin struct mbuf *m; 3160 1.3.2.2 martin while ((m = pcq_get(txr->txr_interq)) != NULL) { 3161 1.3.2.2 martin IGC_QUEUE_EVENT(q, tx_pcq_drop, 1); 3162 1.3.2.2 martin m_freem(m); 3163 1.3.2.2 martin } 3164 1.3.2.2 martin 3165 1.3.2.2 martin mutex_exit(&txr->txr_lock); 3166 1.3.2.2 martin } 3167 1.3.2.2 martin 3168 1.3.2.2 martin 3169 1.3.2.2 martin /********************************************************************* 3170 1.3.2.2 martin * 3171 1.3.2.2 martin * Advanced Context Descriptor setup for VLAN, CSUM or TSO 3172 1.3.2.2 martin * 3173 1.3.2.2 martin **********************************************************************/ 3174 1.3.2.2 martin 3175 1.3.2.2 martin static int 3176 1.3.2.2 martin igc_tx_ctx_setup(struct tx_ring *txr, struct mbuf *mp, int prod, 3177 1.3.2.2 martin uint32_t *cmd_type_len, uint32_t *olinfo_status) 3178 1.3.2.2 martin { 3179 1.3.2.2 martin struct ether_vlan_header *evl; 3180 1.3.2.2 martin uint32_t type_tucmd_mlhl = 0; 3181 1.3.2.2 martin uint32_t vlan_macip_lens = 0; 3182 1.3.2.2 martin uint32_t ehlen, iphlen; 3183 1.3.2.2 martin uint16_t ehtype; 3184 1.3.2.2 martin int off = 0; 3185 1.3.2.2 martin 3186 1.3.2.2 martin const int csum_flags = mp->m_pkthdr.csum_flags; 3187 1.3.2.2 martin 3188 1.3.2.2 martin /* First check if TSO is to be used */ 3189 1.3.2.2 martin if ((csum_flags & (M_CSUM_TSOv4 | M_CSUM_TSOv6)) != 0) { 3190 1.3.2.2 martin return igc_tso_setup(txr, mp, prod, cmd_type_len, 3191 1.3.2.2 martin olinfo_status); 3192 1.3.2.2 martin } 3193 1.3.2.2 martin 3194 1.3.2.2 martin const bool v4 = (csum_flags & 3195 1.3.2.2 martin (M_CSUM_IPv4 | M_CSUM_TCPv4 | M_CSUM_UDPv4)) != 0; 3196 1.3.2.2 martin const bool v6 = (csum_flags & (M_CSUM_UDPv6 | M_CSUM_TCPv6)) != 0; 3197 1.3.2.2 martin 3198 1.3.2.2 martin /* Indicate the whole packet as payload when not doing TSO */ 3199 1.3.2.2 martin *olinfo_status |= mp->m_pkthdr.len << IGC_ADVTXD_PAYLEN_SHIFT; 3200 1.3.2.2 martin 3201 1.3.2.2 martin /* 3202 1.3.2.2 martin * In advanced descriptors the vlan tag must 3203 1.3.2.2 martin * be placed into the context descriptor. Hence 3204 1.3.2.2 martin * we need to make one even if not doing offloads. 3205 1.3.2.2 martin */ 3206 1.3.2.2 martin #if NVLAN > 0 3207 1.3.2.2 martin if (vlan_has_tag(mp)) { 3208 1.3.2.2 martin vlan_macip_lens |= (uint32_t)vlan_get_tag(mp) 3209 1.3.2.2 martin << IGC_ADVTXD_VLAN_SHIFT; 3210 1.3.2.2 martin off = 1; 3211 1.3.2.2 martin } else 3212 1.3.2.2 martin #endif 3213 1.3.2.2 martin if (!v4 && !v6) 3214 1.3.2.2 martin return 0; 3215 1.3.2.2 martin 3216 1.3.2.2 martin KASSERT(mp->m_len >= sizeof(struct ether_header)); 3217 1.3.2.2 martin evl = mtod(mp, struct ether_vlan_header *); 3218 1.3.2.2 martin if (evl->evl_encap_proto == htons(ETHERTYPE_VLAN)) { 3219 1.3.2.2 martin KASSERT(mp->m_len >= sizeof(struct ether_vlan_header)); 3220 1.3.2.2 martin ehlen = ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN; 3221 1.3.2.2 martin ehtype = evl->evl_proto; 3222 1.3.2.2 martin } else { 3223 1.3.2.2 martin ehlen = ETHER_HDR_LEN; 3224 1.3.2.2 martin ehtype = evl->evl_encap_proto; 3225 1.3.2.2 martin } 3226 1.3.2.2 martin 3227 1.3.2.2 martin vlan_macip_lens |= ehlen << IGC_ADVTXD_MACLEN_SHIFT; 3228 1.3.2.2 martin 3229 1.3.2.2 martin #ifdef IGC_DEBUG 3230 1.3.2.2 martin /* 3231 1.3.2.2 martin * For checksum offloading, L3 headers are not mandatory. 3232 1.3.2.2 martin * We use these only for consistency checks. 3233 1.3.2.2 martin */ 3234 1.3.2.2 martin struct ip *ip; 3235 1.3.2.2 martin struct ip6_hdr *ip6; 3236 1.3.2.2 martin uint8_t ipproto; 3237 1.3.2.2 martin char *l3d; 3238 1.3.2.2 martin 3239 1.3.2.2 martin if (mp->m_len == ehlen && mp->m_next != NULL) 3240 1.3.2.2 martin l3d = mtod(mp->m_next, char *); 3241 1.3.2.2 martin else 3242 1.3.2.2 martin l3d = mtod(mp, char *) + ehlen; 3243 1.3.2.2 martin #endif 3244 1.3.2.2 martin 3245 1.3.2.2 martin switch (ntohs(ehtype)) { 3246 1.3.2.2 martin case ETHERTYPE_IP: 3247 1.3.2.2 martin iphlen = M_CSUM_DATA_IPv4_IPHL(mp->m_pkthdr.csum_data); 3248 1.3.2.2 martin type_tucmd_mlhl |= IGC_ADVTXD_TUCMD_IPV4; 3249 1.3.2.2 martin 3250 1.3.2.2 martin if ((csum_flags & M_CSUM_IPv4) != 0) { 3251 1.3.2.2 martin *olinfo_status |= IGC_TXD_POPTS_IXSM << 8; 3252 1.3.2.2 martin off = 1; 3253 1.3.2.2 martin } 3254 1.3.2.2 martin #ifdef IGC_DEBUG 3255 1.3.2.2 martin KASSERT(!v6); 3256 1.3.2.2 martin ip = (void *)l3d; 3257 1.3.2.2 martin ipproto = ip->ip_p; 3258 1.3.2.2 martin KASSERT(iphlen == ip->ip_hl << 2); 3259 1.3.2.2 martin KASSERT((mp->m_pkthdr.csum_flags & M_CSUM_IPv4) == 0 || 3260 1.3.2.2 martin ip->ip_sum == 0); 3261 1.3.2.2 martin #endif 3262 1.3.2.2 martin break; 3263 1.3.2.2 martin case ETHERTYPE_IPV6: 3264 1.3.2.2 martin iphlen = M_CSUM_DATA_IPv6_IPHL(mp->m_pkthdr.csum_data); 3265 1.3.2.2 martin type_tucmd_mlhl |= IGC_ADVTXD_TUCMD_IPV6; 3266 1.3.2.2 martin #ifdef IGC_DEBUG 3267 1.3.2.2 martin KASSERT(!v4); 3268 1.3.2.2 martin ip6 = (void *)l3d; 3269 1.3.2.2 martin ipproto = ip6->ip6_nxt; /* XXX */ 3270 1.3.2.2 martin KASSERT(iphlen == sizeof(struct ip6_hdr)); 3271 1.3.2.2 martin #endif 3272 1.3.2.2 martin break; 3273 1.3.2.2 martin default: 3274 1.3.2.2 martin /* 3275 1.3.2.2 martin * Unknown L3 protocol. Clear L3 header length and proceed for 3276 1.3.2.2 martin * LAN as done by Linux driver. 3277 1.3.2.2 martin */ 3278 1.3.2.2 martin iphlen = 0; 3279 1.3.2.2 martin #ifdef IGC_DEBUG 3280 1.3.2.2 martin KASSERT(!v4 && !v6); 3281 1.3.2.2 martin ipproto = 0; 3282 1.3.2.2 martin #endif 3283 1.3.2.2 martin break; 3284 1.3.2.2 martin } 3285 1.3.2.2 martin 3286 1.3.2.2 martin vlan_macip_lens |= iphlen; 3287 1.3.2.2 martin 3288 1.3.2.2 martin const bool tcp = (csum_flags & (M_CSUM_TCPv4 | M_CSUM_TCPv6)) != 0; 3289 1.3.2.2 martin const bool udp = (csum_flags & (M_CSUM_UDPv4 | M_CSUM_UDPv6)) != 0; 3290 1.3.2.2 martin 3291 1.3.2.2 martin if (tcp) { 3292 1.3.2.2 martin #ifdef IGC_DEBUG 3293 1.3.2.2 martin KASSERTMSG(ipproto == IPPROTO_TCP, "ipproto = %d", ipproto); 3294 1.3.2.2 martin #endif 3295 1.3.2.2 martin type_tucmd_mlhl |= IGC_ADVTXD_TUCMD_L4T_TCP; 3296 1.3.2.2 martin *olinfo_status |= IGC_TXD_POPTS_TXSM << 8; 3297 1.3.2.2 martin off = 1; 3298 1.3.2.2 martin } else if (udp) { 3299 1.3.2.2 martin #ifdef IGC_DEBUG 3300 1.3.2.2 martin KASSERTMSG(ipproto == IPPROTO_UDP, "ipproto = %d", ipproto); 3301 1.3.2.2 martin #endif 3302 1.3.2.2 martin type_tucmd_mlhl |= IGC_ADVTXD_TUCMD_L4T_UDP; 3303 1.3.2.2 martin *olinfo_status |= IGC_TXD_POPTS_TXSM << 8; 3304 1.3.2.2 martin off = 1; 3305 1.3.2.2 martin } 3306 1.3.2.2 martin 3307 1.3.2.2 martin if (off == 0) 3308 1.3.2.2 martin return 0; 3309 1.3.2.2 martin 3310 1.3.2.2 martin type_tucmd_mlhl |= IGC_ADVTXD_DCMD_DEXT | IGC_ADVTXD_DTYP_CTXT; 3311 1.3.2.2 martin 3312 1.3.2.2 martin /* Now ready a context descriptor */ 3313 1.3.2.2 martin struct igc_adv_tx_context_desc *txdesc = 3314 1.3.2.2 martin (struct igc_adv_tx_context_desc *)&txr->tx_base[prod]; 3315 1.3.2.2 martin 3316 1.3.2.2 martin /* Now copy bits into descriptor */ 3317 1.3.2.2 martin igc_txdesc_sync(txr, prod, 3318 1.3.2.2 martin BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE); 3319 1.3.2.2 martin htolem32(&txdesc->vlan_macip_lens, vlan_macip_lens); 3320 1.3.2.2 martin htolem32(&txdesc->type_tucmd_mlhl, type_tucmd_mlhl); 3321 1.3.2.2 martin htolem32(&txdesc->seqnum_seed, 0); 3322 1.3.2.2 martin htolem32(&txdesc->mss_l4len_idx, 0); 3323 1.3.2.2 martin igc_txdesc_sync(txr, prod, 3324 1.3.2.2 martin BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); 3325 1.3.2.2 martin 3326 1.3.2.2 martin return 1; 3327 1.3.2.2 martin } 3328 1.3.2.2 martin 3329 1.3.2.2 martin /********************************************************************* 3330 1.3.2.2 martin * 3331 1.3.2.2 martin * Advanced Context Descriptor setup for TSO 3332 1.3.2.2 martin * 3333 1.3.2.2 martin * XXX XXXRO 3334 1.3.2.2 martin * Not working. Some packets are sent with correct csums, but 3335 1.3.2.2 martin * others aren't. th->th_sum may be adjusted. 3336 1.3.2.2 martin * 3337 1.3.2.2 martin **********************************************************************/ 3338 1.3.2.2 martin 3339 1.3.2.2 martin static int 3340 1.3.2.2 martin igc_tso_setup(struct tx_ring *txr, struct mbuf *mp, int prod, 3341 1.3.2.2 martin uint32_t *cmd_type_len, uint32_t *olinfo_status) 3342 1.3.2.2 martin { 3343 1.3.2.2 martin #if 1 /* notyet */ 3344 1.3.2.2 martin return 0; 3345 1.3.2.2 martin #else 3346 1.3.2.2 martin struct ether_vlan_header *evl; 3347 1.3.2.2 martin struct ip *ip; 3348 1.3.2.2 martin struct ip6_hdr *ip6; 3349 1.3.2.2 martin struct tcphdr *th; 3350 1.3.2.2 martin uint32_t type_tucmd_mlhl = 0; 3351 1.3.2.2 martin uint32_t vlan_macip_lens = 0; 3352 1.3.2.2 martin uint32_t mss_l4len_idx = 0; 3353 1.3.2.2 martin uint32_t ehlen, iphlen, tcphlen, paylen; 3354 1.3.2.2 martin uint16_t ehtype; 3355 1.3.2.2 martin 3356 1.3.2.2 martin /* 3357 1.3.2.2 martin * In advanced descriptors the vlan tag must 3358 1.3.2.2 martin * be placed into the context descriptor. Hence 3359 1.3.2.2 martin * we need to make one even if not doing offloads. 3360 1.3.2.2 martin */ 3361 1.3.2.2 martin #if NVLAN > 0 3362 1.3.2.2 martin if (vlan_has_tag(mp)) { 3363 1.3.2.2 martin vlan_macip_lens |= (uint32_t)vlan_get_tag(mp) 3364 1.3.2.2 martin << IGC_ADVTXD_VLAN_SHIFT; 3365 1.3.2.2 martin } 3366 1.3.2.2 martin #endif 3367 1.3.2.2 martin 3368 1.3.2.2 martin KASSERT(mp->m_len >= sizeof(struct ether_header)); 3369 1.3.2.2 martin evl = mtod(mp, struct ether_vlan_header *); 3370 1.3.2.2 martin if (evl->evl_encap_proto == htons(ETHERTYPE_VLAN)) { 3371 1.3.2.2 martin KASSERT(mp->m_len >= sizeof(struct ether_vlan_header)); 3372 1.3.2.2 martin ehlen = ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN; 3373 1.3.2.2 martin ehtype = evl->evl_proto; 3374 1.3.2.2 martin } else { 3375 1.3.2.2 martin ehlen = ETHER_HDR_LEN; 3376 1.3.2.2 martin ehtype = evl->evl_encap_proto; 3377 1.3.2.2 martin } 3378 1.3.2.2 martin 3379 1.3.2.2 martin vlan_macip_lens |= ehlen << IGC_ADVTXD_MACLEN_SHIFT; 3380 1.3.2.2 martin 3381 1.3.2.2 martin switch (ntohs(ehtype)) { 3382 1.3.2.2 martin case ETHERTYPE_IP: 3383 1.3.2.2 martin iphlen = M_CSUM_DATA_IPv4_IPHL(mp->m_pkthdr.csum_data); 3384 1.3.2.2 martin type_tucmd_mlhl |= IGC_ADVTXD_TUCMD_IPV4; 3385 1.3.2.2 martin *olinfo_status |= IGC_TXD_POPTS_IXSM << 8; 3386 1.3.2.2 martin 3387 1.3.2.2 martin KASSERT(mp->m_len >= ehlen + sizeof(*ip)); 3388 1.3.2.2 martin ip = (void *)(mtod(mp, char *) + ehlen); 3389 1.3.2.2 martin ip->ip_len = 0; 3390 1.3.2.2 martin KASSERT(iphlen == ip->ip_hl << 2); 3391 1.3.2.2 martin KASSERT(ip->ip_sum == 0); 3392 1.3.2.2 martin KASSERT(ip->ip_p == IPPROTO_TCP); 3393 1.3.2.2 martin 3394 1.3.2.2 martin KASSERT(mp->m_len >= ehlen + iphlen + sizeof(*th)); 3395 1.3.2.2 martin th = (void *)((char *)ip + iphlen); 3396 1.3.2.2 martin th->th_sum = in_cksum_phdr(ip->ip_src.s_addr, ip->ip_dst.s_addr, 3397 1.3.2.2 martin htons(IPPROTO_TCP)); 3398 1.3.2.2 martin break; 3399 1.3.2.2 martin case ETHERTYPE_IPV6: 3400 1.3.2.2 martin iphlen = M_CSUM_DATA_IPv6_IPHL(mp->m_pkthdr.csum_data); 3401 1.3.2.2 martin type_tucmd_mlhl |= IGC_ADVTXD_TUCMD_IPV6; 3402 1.3.2.2 martin 3403 1.3.2.2 martin KASSERT(mp->m_len >= ehlen + sizeof(*ip6)); 3404 1.3.2.2 martin ip6 = (void *)(mtod(mp, char *) + ehlen); 3405 1.3.2.2 martin ip6->ip6_plen = 0; 3406 1.3.2.2 martin KASSERT(iphlen == sizeof(struct ip6_hdr)); 3407 1.3.2.2 martin KASSERT(ip6->ip6_nxt == IPPROTO_TCP); 3408 1.3.2.2 martin 3409 1.3.2.2 martin KASSERT(mp->m_len >= ehlen + iphlen + sizeof(*th)); 3410 1.3.2.2 martin th = (void *)((char *)ip6 + iphlen); 3411 1.3.2.2 martin tcphlen = th->th_off << 2; 3412 1.3.2.2 martin paylen = mp->m_pkthdr.len - ehlen - iphlen - tcphlen; 3413 1.3.2.2 martin th->th_sum = in6_cksum_phdr(&ip6->ip6_src, &ip6->ip6_dst, 0, 3414 1.3.2.2 martin htonl(IPPROTO_TCP)); 3415 1.3.2.2 martin break; 3416 1.3.2.2 martin default: 3417 1.3.2.2 martin panic("%s", __func__); 3418 1.3.2.2 martin } 3419 1.3.2.2 martin 3420 1.3.2.2 martin tcphlen = th->th_off << 2; 3421 1.3.2.2 martin paylen = mp->m_pkthdr.len - ehlen - iphlen - tcphlen; 3422 1.3.2.2 martin 3423 1.3.2.2 martin vlan_macip_lens |= iphlen; 3424 1.3.2.2 martin 3425 1.3.2.2 martin type_tucmd_mlhl |= IGC_ADVTXD_DCMD_DEXT | IGC_ADVTXD_DTYP_CTXT; 3426 1.3.2.2 martin type_tucmd_mlhl |= IGC_ADVTXD_TUCMD_L4T_TCP; 3427 1.3.2.2 martin 3428 1.3.2.2 martin mss_l4len_idx |= mp->m_pkthdr.segsz << IGC_ADVTXD_MSS_SHIFT; 3429 1.3.2.2 martin mss_l4len_idx |= tcphlen << IGC_ADVTXD_L4LEN_SHIFT; 3430 1.3.2.2 martin 3431 1.3.2.2 martin /* Now ready a context descriptor */ 3432 1.3.2.2 martin struct igc_adv_tx_context_desc *txdesc = 3433 1.3.2.2 martin (struct igc_adv_tx_context_desc *)&txr->tx_base[prod]; 3434 1.3.2.2 martin 3435 1.3.2.2 martin /* Now copy bits into descriptor */ 3436 1.3.2.2 martin igc_txdesc_sync(txr, prod, 3437 1.3.2.2 martin BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE); 3438 1.3.2.2 martin htolem32(&txdesc->vlan_macip_lens, vlan_macip_lens); 3439 1.3.2.2 martin htolem32(&txdesc->type_tucmd_mlhl, type_tucmd_mlhl); 3440 1.3.2.2 martin htolem32(&txdesc->seqnum_seed, 0); 3441 1.3.2.2 martin htolem32(&txdesc->mss_l4len_idx, mss_l4len_idx); 3442 1.3.2.2 martin igc_txdesc_sync(txr, prod, 3443 1.3.2.2 martin BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); 3444 1.3.2.2 martin 3445 1.3.2.2 martin *cmd_type_len |= IGC_ADVTXD_DCMD_TSE; 3446 1.3.2.2 martin *olinfo_status |= IGC_TXD_POPTS_TXSM << 8; 3447 1.3.2.2 martin *olinfo_status |= paylen << IGC_ADVTXD_PAYLEN_SHIFT; 3448 1.3.2.2 martin 3449 1.3.2.2 martin return 1; 3450 1.3.2.2 martin #endif /* notyet */ 3451 1.3.2.2 martin } 3452 1.3.2.2 martin 3453 1.3.2.2 martin /********************************************************************* 3454 1.3.2.2 martin * 3455 1.3.2.2 martin * Allocate memory for rx_buffer structures. Since we use one 3456 1.3.2.2 martin * rx_buffer per received packet, the maximum number of rx_buffer's 3457 1.3.2.2 martin * that we'll need is equal to the number of receive descriptors 3458 1.3.2.2 martin * that we've allocated. 3459 1.3.2.2 martin * 3460 1.3.2.2 martin **********************************************************************/ 3461 1.3.2.2 martin static int 3462 1.3.2.2 martin igc_allocate_receive_buffers(struct rx_ring *rxr) 3463 1.3.2.2 martin { 3464 1.3.2.2 martin struct igc_softc *sc = rxr->sc; 3465 1.3.2.2 martin int error; 3466 1.3.2.2 martin 3467 1.3.2.2 martin rxr->rx_buffers = 3468 1.3.2.2 martin kmem_zalloc(sc->num_rx_desc * sizeof(struct igc_rx_buf), KM_SLEEP); 3469 1.3.2.2 martin 3470 1.3.2.2 martin for (int id = 0; id < sc->num_rx_desc; id++) { 3471 1.3.2.2 martin struct igc_rx_buf *rxbuf = &rxr->rx_buffers[id]; 3472 1.3.2.2 martin 3473 1.3.2.2 martin error = bus_dmamap_create(rxr->rxdma.dma_tag, MCLBYTES, 1, 3474 1.3.2.2 martin MCLBYTES, 0, BUS_DMA_WAITOK, &rxbuf->map); 3475 1.3.2.2 martin if (error) { 3476 1.3.2.2 martin aprint_error_dev(sc->sc_dev, 3477 1.3.2.2 martin "unable to create RX DMA map\n"); 3478 1.3.2.2 martin goto fail; 3479 1.3.2.2 martin } 3480 1.3.2.2 martin } 3481 1.3.2.2 martin bus_dmamap_sync(rxr->rxdma.dma_tag, rxr->rxdma.dma_map, 0, 3482 1.3.2.2 martin rxr->rxdma.dma_map->dm_mapsize, 3483 1.3.2.2 martin BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); 3484 1.3.2.2 martin 3485 1.3.2.2 martin return 0; 3486 1.3.2.2 martin fail: 3487 1.3.2.2 martin return error; 3488 1.3.2.2 martin } 3489 1.3.2.2 martin 3490 1.3.2.2 martin /********************************************************************* 3491 1.3.2.2 martin * 3492 1.3.2.2 martin * Allocate and initialize receive structures. 3493 1.3.2.2 martin * 3494 1.3.2.2 martin **********************************************************************/ 3495 1.3.2.2 martin static int 3496 1.3.2.2 martin igc_setup_receive_structures(struct igc_softc *sc) 3497 1.3.2.2 martin { 3498 1.3.2.2 martin 3499 1.3.2.2 martin for (int iq = 0; iq < sc->sc_nqueues; iq++) { 3500 1.3.2.2 martin struct rx_ring *rxr = &sc->rx_rings[iq]; 3501 1.3.2.2 martin 3502 1.3.2.2 martin if (igc_setup_receive_ring(rxr)) 3503 1.3.2.2 martin goto fail; 3504 1.3.2.2 martin } 3505 1.3.2.2 martin 3506 1.3.2.2 martin return 0; 3507 1.3.2.2 martin fail: 3508 1.3.2.2 martin igc_free_receive_structures(sc); 3509 1.3.2.2 martin return ENOBUFS; 3510 1.3.2.2 martin } 3511 1.3.2.2 martin 3512 1.3.2.2 martin /********************************************************************* 3513 1.3.2.2 martin * 3514 1.3.2.2 martin * Initialize a receive ring and its buffers. 3515 1.3.2.2 martin * 3516 1.3.2.2 martin **********************************************************************/ 3517 1.3.2.2 martin static int 3518 1.3.2.2 martin igc_setup_receive_ring(struct rx_ring *rxr) 3519 1.3.2.2 martin { 3520 1.3.2.2 martin struct igc_softc *sc = rxr->sc; 3521 1.3.2.2 martin const int rsize = roundup2( 3522 1.3.2.2 martin sc->num_rx_desc * sizeof(union igc_adv_rx_desc), IGC_DBA_ALIGN); 3523 1.3.2.2 martin 3524 1.3.2.2 martin /* Clear the ring contents. */ 3525 1.3.2.2 martin memset(rxr->rx_base, 0, rsize); 3526 1.3.2.2 martin 3527 1.3.2.2 martin if (igc_allocate_receive_buffers(rxr)) 3528 1.3.2.2 martin return ENOMEM; 3529 1.3.2.2 martin 3530 1.3.2.2 martin /* Setup our descriptor indices. */ 3531 1.3.2.2 martin rxr->next_to_check = 0; 3532 1.3.2.2 martin rxr->last_desc_filled = 0; 3533 1.3.2.2 martin 3534 1.3.2.2 martin mutex_init(&rxr->rxr_lock, MUTEX_DEFAULT, IPL_NET); 3535 1.3.2.2 martin 3536 1.3.2.2 martin return 0; 3537 1.3.2.2 martin } 3538 1.3.2.2 martin 3539 1.3.2.2 martin /********************************************************************* 3540 1.3.2.2 martin * 3541 1.3.2.2 martin * Enable receive unit. 3542 1.3.2.2 martin * 3543 1.3.2.2 martin **********************************************************************/ 3544 1.3.2.2 martin static void 3545 1.3.2.2 martin igc_initialize_receive_unit(struct igc_softc *sc) 3546 1.3.2.2 martin { 3547 1.3.2.2 martin struct ifnet *ifp = &sc->sc_ec.ec_if; 3548 1.3.2.2 martin struct igc_hw *hw = &sc->hw; 3549 1.3.2.2 martin uint32_t rctl, rxcsum, srrctl; 3550 1.3.2.2 martin 3551 1.3.2.2 martin DPRINTF(RX, "called\n"); 3552 1.3.2.2 martin 3553 1.3.2.2 martin /* 3554 1.3.2.2 martin * Make sure receives are disabled while setting 3555 1.3.2.2 martin * up the descriptor ring. 3556 1.3.2.2 martin */ 3557 1.3.2.2 martin rctl = IGC_READ_REG(hw, IGC_RCTL); 3558 1.3.2.2 martin IGC_WRITE_REG(hw, IGC_RCTL, rctl & ~IGC_RCTL_EN); 3559 1.3.2.2 martin 3560 1.3.2.2 martin /* Setup the Receive Control Register */ 3561 1.3.2.2 martin rctl &= ~(3 << IGC_RCTL_MO_SHIFT); 3562 1.3.2.2 martin rctl |= IGC_RCTL_EN | IGC_RCTL_BAM | IGC_RCTL_LBM_NO | 3563 1.3.2.2 martin IGC_RCTL_RDMTS_HALF | (hw->mac.mc_filter_type << IGC_RCTL_MO_SHIFT); 3564 1.3.2.2 martin 3565 1.3.2.2 martin #if 1 3566 1.3.2.2 martin /* Do not store bad packets */ 3567 1.3.2.2 martin rctl &= ~IGC_RCTL_SBP; 3568 1.3.2.2 martin #else 3569 1.3.2.2 martin /* for debug */ 3570 1.3.2.2 martin rctl |= IGC_RCTL_SBP; 3571 1.3.2.2 martin #endif 3572 1.3.2.2 martin 3573 1.3.2.2 martin /* Enable Long Packet receive */ 3574 1.3.2.2 martin if (sc->hw.mac.max_frame_size > ETHER_MAX_LEN) 3575 1.3.2.2 martin rctl |= IGC_RCTL_LPE; 3576 1.3.2.2 martin else 3577 1.3.2.2 martin rctl &= ~IGC_RCTL_LPE; 3578 1.3.2.2 martin 3579 1.3.2.2 martin /* Strip the CRC */ 3580 1.3.2.2 martin rctl |= IGC_RCTL_SECRC; 3581 1.3.2.2 martin 3582 1.3.2.2 martin /* 3583 1.3.2.2 martin * Set the interrupt throttling rate. Value is calculated 3584 1.3.2.2 martin * as DEFAULT_ITR = 1/(MAX_INTS_PER_SEC * 256ns) 3585 1.3.2.2 martin * 3586 1.3.2.2 martin * XXX Sync with Linux, especially for jumbo MTU or TSO. 3587 1.3.2.2 martin * XXX Shouldn't be here? 3588 1.3.2.2 martin */ 3589 1.3.2.2 martin IGC_WRITE_REG(hw, IGC_ITR, DEFAULT_ITR); 3590 1.3.2.2 martin 3591 1.3.2.2 martin rxcsum = IGC_READ_REG(hw, IGC_RXCSUM); 3592 1.3.2.2 martin rxcsum &= ~(IGC_RXCSUM_IPOFL | IGC_RXCSUM_TUOFL | IGC_RXCSUM_PCSD); 3593 1.3.2.2 martin if (ifp->if_capenable & IFCAP_CSUM_IPv4_Rx) 3594 1.3.2.2 martin rxcsum |= IGC_RXCSUM_IPOFL; 3595 1.3.2.2 martin if (ifp->if_capenable & (IFCAP_CSUM_TCPv4_Rx | IFCAP_CSUM_UDPv4_Rx | 3596 1.3.2.2 martin IFCAP_CSUM_TCPv6_Rx | IFCAP_CSUM_UDPv6_Rx)) 3597 1.3.2.2 martin rxcsum |= IGC_RXCSUM_TUOFL; 3598 1.3.2.2 martin if (sc->sc_nqueues > 1) 3599 1.3.2.2 martin rxcsum |= IGC_RXCSUM_PCSD; 3600 1.3.2.2 martin IGC_WRITE_REG(hw, IGC_RXCSUM, rxcsum); 3601 1.3.2.2 martin 3602 1.3.2.2 martin if (sc->sc_nqueues > 1) 3603 1.3.2.2 martin igc_initialize_rss_mapping(sc); 3604 1.3.2.2 martin 3605 1.3.2.2 martin srrctl = 0; 3606 1.3.2.2 martin #if 0 3607 1.3.2.2 martin srrctl |= 4096 >> IGC_SRRCTL_BSIZEPKT_SHIFT; 3608 1.3.2.2 martin rctl |= IGC_RCTL_SZ_4096 | IGC_RCTL_BSEX; 3609 1.3.2.2 martin #else 3610 1.3.2.2 martin srrctl |= 2048 >> IGC_SRRCTL_BSIZEPKT_SHIFT; 3611 1.3.2.2 martin rctl |= IGC_RCTL_SZ_2048; 3612 1.3.2.2 martin #endif 3613 1.3.2.2 martin 3614 1.3.2.2 martin /* 3615 1.3.2.2 martin * If TX flow control is disabled and there's > 1 queue defined, 3616 1.3.2.2 martin * enable DROP. 3617 1.3.2.2 martin * 3618 1.3.2.2 martin * This drops frames rather than hanging the RX MAC for all queues. 3619 1.3.2.2 martin */ 3620 1.3.2.2 martin if (sc->sc_nqueues > 1 && 3621 1.3.2.2 martin (sc->fc == igc_fc_none || sc->fc == igc_fc_rx_pause)) 3622 1.3.2.2 martin srrctl |= IGC_SRRCTL_DROP_EN; 3623 1.3.2.2 martin 3624 1.3.2.2 martin /* Setup the Base and Length of the RX descriptor rings. */ 3625 1.3.2.2 martin for (int iq = 0; iq < sc->sc_nqueues; iq++) { 3626 1.3.2.2 martin struct rx_ring *rxr = &sc->rx_rings[iq]; 3627 1.3.2.2 martin const uint64_t bus_addr = 3628 1.3.2.2 martin rxr->rxdma.dma_map->dm_segs[0].ds_addr; 3629 1.3.2.2 martin 3630 1.3.2.2 martin IGC_WRITE_REG(hw, IGC_RXDCTL(iq), 0); 3631 1.3.2.2 martin 3632 1.3.2.2 martin srrctl |= IGC_SRRCTL_DESCTYPE_ADV_ONEBUF; 3633 1.3.2.2 martin 3634 1.3.2.2 martin IGC_WRITE_REG(hw, IGC_RDLEN(iq), 3635 1.3.2.2 martin sc->num_rx_desc * sizeof(union igc_adv_rx_desc)); 3636 1.3.2.2 martin IGC_WRITE_REG(hw, IGC_RDBAH(iq), (uint32_t)(bus_addr >> 32)); 3637 1.3.2.2 martin IGC_WRITE_REG(hw, IGC_RDBAL(iq), (uint32_t)bus_addr); 3638 1.3.2.2 martin IGC_WRITE_REG(hw, IGC_SRRCTL(iq), srrctl); 3639 1.3.2.2 martin 3640 1.3.2.2 martin /* Setup the Head and Tail Descriptor Pointers */ 3641 1.3.2.2 martin IGC_WRITE_REG(hw, IGC_RDH(iq), 0); 3642 1.3.2.2 martin IGC_WRITE_REG(hw, IGC_RDT(iq), 0 /* XXX rxr->last_desc_filled */); 3643 1.3.2.2 martin 3644 1.3.2.2 martin /* Enable this Queue */ 3645 1.3.2.2 martin uint32_t rxdctl = IGC_READ_REG(hw, IGC_RXDCTL(iq)); 3646 1.3.2.2 martin rxdctl |= IGC_RXDCTL_QUEUE_ENABLE; 3647 1.3.2.2 martin rxdctl &= 0xFFF00000; 3648 1.3.2.2 martin rxdctl |= IGC_RX_PTHRESH; 3649 1.3.2.2 martin rxdctl |= IGC_RX_HTHRESH << 8; 3650 1.3.2.2 martin rxdctl |= IGC_RX_WTHRESH << 16; 3651 1.3.2.2 martin IGC_WRITE_REG(hw, IGC_RXDCTL(iq), rxdctl); 3652 1.3.2.2 martin } 3653 1.3.2.2 martin 3654 1.3.2.2 martin /* Make sure VLAN Filters are off */ 3655 1.3.2.2 martin rctl &= ~IGC_RCTL_VFE; 3656 1.3.2.2 martin 3657 1.3.2.2 martin /* Write out the settings */ 3658 1.3.2.2 martin IGC_WRITE_REG(hw, IGC_RCTL, rctl); 3659 1.3.2.2 martin } 3660 1.3.2.2 martin 3661 1.3.2.2 martin /********************************************************************* 3662 1.3.2.2 martin * 3663 1.3.2.2 martin * Free all receive rings. 3664 1.3.2.2 martin * 3665 1.3.2.2 martin **********************************************************************/ 3666 1.3.2.2 martin static void 3667 1.3.2.2 martin igc_free_receive_structures(struct igc_softc *sc) 3668 1.3.2.2 martin { 3669 1.3.2.2 martin 3670 1.3.2.2 martin for (int iq = 0; iq < sc->sc_nqueues; iq++) { 3671 1.3.2.2 martin struct rx_ring *rxr = &sc->rx_rings[iq]; 3672 1.3.2.2 martin 3673 1.3.2.2 martin igc_free_receive_buffers(rxr); 3674 1.3.2.2 martin } 3675 1.3.2.2 martin } 3676 1.3.2.2 martin 3677 1.3.2.2 martin /********************************************************************* 3678 1.3.2.2 martin * 3679 1.3.2.2 martin * Free receive ring data structures 3680 1.3.2.2 martin * 3681 1.3.2.2 martin **********************************************************************/ 3682 1.3.2.2 martin static void 3683 1.3.2.2 martin igc_free_receive_buffers(struct rx_ring *rxr) 3684 1.3.2.2 martin { 3685 1.3.2.2 martin struct igc_softc *sc = rxr->sc; 3686 1.3.2.2 martin 3687 1.3.2.2 martin if (rxr->rx_buffers != NULL) { 3688 1.3.2.2 martin for (int id = 0; id < sc->num_rx_desc; id++) { 3689 1.3.2.2 martin struct igc_rx_buf *rxbuf = &rxr->rx_buffers[id]; 3690 1.3.2.2 martin bus_dmamap_t map = rxbuf->map; 3691 1.3.2.2 martin 3692 1.3.2.2 martin if (rxbuf->buf != NULL) { 3693 1.3.2.2 martin bus_dmamap_sync(rxr->rxdma.dma_tag, map, 3694 1.3.2.2 martin 0, map->dm_mapsize, BUS_DMASYNC_POSTREAD); 3695 1.3.2.2 martin bus_dmamap_unload(rxr->rxdma.dma_tag, map); 3696 1.3.2.2 martin m_freem(rxbuf->buf); 3697 1.3.2.2 martin rxbuf->buf = NULL; 3698 1.3.2.2 martin } 3699 1.3.2.2 martin bus_dmamap_destroy(rxr->rxdma.dma_tag, map); 3700 1.3.2.2 martin rxbuf->map = NULL; 3701 1.3.2.2 martin } 3702 1.3.2.2 martin kmem_free(rxr->rx_buffers, 3703 1.3.2.2 martin sc->num_rx_desc * sizeof(struct igc_rx_buf)); 3704 1.3.2.2 martin rxr->rx_buffers = NULL; 3705 1.3.2.2 martin } 3706 1.3.2.2 martin 3707 1.3.2.2 martin mutex_destroy(&rxr->rxr_lock); 3708 1.3.2.2 martin } 3709 1.3.2.2 martin 3710 1.3.2.2 martin /********************************************************************* 3711 1.3.2.2 martin * 3712 1.3.2.2 martin * Clear status registers in all RX descriptors. 3713 1.3.2.2 martin * 3714 1.3.2.2 martin **********************************************************************/ 3715 1.3.2.2 martin static void 3716 1.3.2.2 martin igc_clear_receive_status(struct rx_ring *rxr) 3717 1.3.2.2 martin { 3718 1.3.2.2 martin struct igc_softc *sc = rxr->sc; 3719 1.3.2.2 martin 3720 1.3.2.2 martin mutex_enter(&rxr->rxr_lock); 3721 1.3.2.2 martin 3722 1.3.2.2 martin for (int id = 0; id < sc->num_rx_desc; id++) { 3723 1.3.2.2 martin union igc_adv_rx_desc *rxdesc = &rxr->rx_base[id]; 3724 1.3.2.2 martin 3725 1.3.2.2 martin igc_rxdesc_sync(rxr, id, 3726 1.3.2.2 martin BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE); 3727 1.3.2.2 martin rxdesc->wb.upper.status_error = 0; 3728 1.3.2.2 martin igc_rxdesc_sync(rxr, id, 3729 1.3.2.2 martin BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); 3730 1.3.2.2 martin } 3731 1.3.2.2 martin 3732 1.3.2.2 martin mutex_exit(&rxr->rxr_lock); 3733 1.3.2.2 martin } 3734 1.3.2.2 martin 3735 1.3.2.2 martin /* 3736 1.3.2.2 martin * Initialise the RSS mapping for NICs that support multiple transmit/ 3737 1.3.2.2 martin * receive rings. 3738 1.3.2.2 martin */ 3739 1.3.2.2 martin static void 3740 1.3.2.2 martin igc_initialize_rss_mapping(struct igc_softc *sc) 3741 1.3.2.2 martin { 3742 1.3.2.2 martin struct igc_hw *hw = &sc->hw; 3743 1.3.2.2 martin 3744 1.3.2.2 martin /* 3745 1.3.2.2 martin * The redirection table controls which destination 3746 1.3.2.2 martin * queue each bucket redirects traffic to. 3747 1.3.2.2 martin * Each DWORD represents four queues, with the LSB 3748 1.3.2.2 martin * being the first queue in the DWORD. 3749 1.3.2.2 martin * 3750 1.3.2.2 martin * This just allocates buckets to queues using round-robin 3751 1.3.2.2 martin * allocation. 3752 1.3.2.2 martin * 3753 1.3.2.2 martin * NOTE: It Just Happens to line up with the default 3754 1.3.2.2 martin * RSS allocation method. 3755 1.3.2.2 martin */ 3756 1.3.2.2 martin 3757 1.3.2.2 martin /* Warning FM follows */ 3758 1.3.2.2 martin uint32_t reta = 0; 3759 1.3.2.2 martin for (int i = 0; i < 128; i++) { 3760 1.3.2.2 martin const int shift = 0; /* XXXRO */ 3761 1.3.2.2 martin int queue_id = i % sc->sc_nqueues; 3762 1.3.2.2 martin /* Adjust if required */ 3763 1.3.2.2 martin queue_id <<= shift; 3764 1.3.2.2 martin 3765 1.3.2.2 martin /* 3766 1.3.2.2 martin * The low 8 bits are for hash value (n+0); 3767 1.3.2.2 martin * The next 8 bits are for hash value (n+1), etc. 3768 1.3.2.2 martin */ 3769 1.3.2.2 martin reta >>= 8; 3770 1.3.2.2 martin reta |= ((uint32_t)queue_id) << 24; 3771 1.3.2.2 martin if ((i & 3) == 3) { 3772 1.3.2.2 martin IGC_WRITE_REG(hw, IGC_RETA(i >> 2), reta); 3773 1.3.2.2 martin reta = 0; 3774 1.3.2.2 martin } 3775 1.3.2.2 martin } 3776 1.3.2.2 martin 3777 1.3.2.2 martin /* 3778 1.3.2.2 martin * MRQC: Multiple Receive Queues Command 3779 1.3.2.2 martin * Set queuing to RSS control, number depends on the device. 3780 1.3.2.2 martin */ 3781 1.3.2.2 martin 3782 1.3.2.2 martin /* Set up random bits */ 3783 1.3.2.2 martin uint32_t rss_key[RSS_KEYSIZE / sizeof(uint32_t)]; 3784 1.3.2.2 martin rss_getkey((uint8_t *)rss_key); 3785 1.3.2.2 martin 3786 1.3.2.2 martin /* Now fill our hash function seeds */ 3787 1.3.2.2 martin for (int i = 0; i < __arraycount(rss_key); i++) 3788 1.3.2.2 martin IGC_WRITE_REG_ARRAY(hw, IGC_RSSRK(0), i, rss_key[i]); 3789 1.3.2.2 martin 3790 1.3.2.2 martin /* 3791 1.3.2.2 martin * Configure the RSS fields to hash upon. 3792 1.3.2.2 martin */ 3793 1.3.2.2 martin uint32_t mrqc = IGC_MRQC_ENABLE_RSS_4Q; 3794 1.3.2.2 martin mrqc |= IGC_MRQC_RSS_FIELD_IPV4 | IGC_MRQC_RSS_FIELD_IPV4_TCP; 3795 1.3.2.2 martin mrqc |= IGC_MRQC_RSS_FIELD_IPV6 | IGC_MRQC_RSS_FIELD_IPV6_TCP; 3796 1.3.2.2 martin mrqc |= IGC_MRQC_RSS_FIELD_IPV6_TCP_EX; 3797 1.3.2.2 martin 3798 1.3.2.2 martin IGC_WRITE_REG(hw, IGC_MRQC, mrqc); 3799 1.3.2.2 martin } 3800 1.3.2.2 martin 3801 1.3.2.2 martin /* 3802 1.3.2.2 martin * igc_get_hw_control sets the {CTRL_EXT|FWSM}:DRV_LOAD bit. 3803 1.3.2.2 martin * For ASF and Pass Through versions of f/w this means 3804 1.3.2.2 martin * that the driver is loaded. For AMT version type f/w 3805 1.3.2.2 martin * this means that the network i/f is open. 3806 1.3.2.2 martin */ 3807 1.3.2.2 martin static void 3808 1.3.2.2 martin igc_get_hw_control(struct igc_softc *sc) 3809 1.3.2.2 martin { 3810 1.3.2.2 martin const uint32_t ctrl_ext = IGC_READ_REG(&sc->hw, IGC_CTRL_EXT); 3811 1.3.2.2 martin 3812 1.3.2.2 martin IGC_WRITE_REG(&sc->hw, IGC_CTRL_EXT, ctrl_ext | IGC_CTRL_EXT_DRV_LOAD); 3813 1.3.2.2 martin } 3814 1.3.2.2 martin 3815 1.3.2.2 martin /* 3816 1.3.2.2 martin * igc_release_hw_control resets {CTRL_EXT|FWSM}:DRV_LOAD bit. 3817 1.3.2.2 martin * For ASF and Pass Through versions of f/w this means that 3818 1.3.2.2 martin * the driver is no longer loaded. For AMT versions of the 3819 1.3.2.2 martin * f/w this means that the network i/f is closed. 3820 1.3.2.2 martin */ 3821 1.3.2.2 martin static void 3822 1.3.2.2 martin igc_release_hw_control(struct igc_softc *sc) 3823 1.3.2.2 martin { 3824 1.3.2.2 martin const uint32_t ctrl_ext = IGC_READ_REG(&sc->hw, IGC_CTRL_EXT); 3825 1.3.2.2 martin 3826 1.3.2.2 martin IGC_WRITE_REG(&sc->hw, IGC_CTRL_EXT, ctrl_ext & ~IGC_CTRL_EXT_DRV_LOAD); 3827 1.3.2.2 martin } 3828 1.3.2.2 martin 3829 1.3.2.2 martin static int 3830 1.3.2.2 martin igc_is_valid_ether_addr(uint8_t *addr) 3831 1.3.2.2 martin { 3832 1.3.2.2 martin const char zero_addr[6] = { 0, 0, 0, 0, 0, 0 }; 3833 1.3.2.2 martin 3834 1.3.2.2 martin if ((addr[0] & 1) || !bcmp(addr, zero_addr, ETHER_ADDR_LEN)) 3835 1.3.2.2 martin return 0; 3836 1.3.2.2 martin 3837 1.3.2.2 martin return 1; 3838 1.3.2.2 martin } 3839 1.3.2.2 martin 3840 1.3.2.2 martin static void 3841 1.3.2.2 martin igc_print_devinfo(struct igc_softc *sc) 3842 1.3.2.2 martin { 3843 1.3.2.2 martin device_t dev = sc->sc_dev; 3844 1.3.2.2 martin struct igc_hw *hw = &sc->hw; 3845 1.3.2.2 martin struct igc_phy_info *phy = &hw->phy; 3846 1.3.2.2 martin u_int oui, model, rev; 3847 1.3.2.4 martin uint16_t id1, id2, nvm_ver, phy_ver, etk_lo, etk_hi; 3848 1.3.2.2 martin char descr[MII_MAX_DESCR_LEN]; 3849 1.3.2.2 martin 3850 1.3.2.2 martin /* Print PHY Info */ 3851 1.3.2.2 martin id1 = phy->id >> 16; 3852 1.3.2.2 martin /* The revision field in phy->id is cleard and it's in phy->revision */ 3853 1.3.2.2 martin id2 = (phy->id & 0xfff0) | phy->revision; 3854 1.3.2.2 martin oui = MII_OUI(id1, id2); 3855 1.3.2.2 martin model = MII_MODEL(id2); 3856 1.3.2.2 martin rev = MII_REV(id2); 3857 1.3.2.2 martin mii_get_descr(descr, sizeof(descr), oui, model); 3858 1.3.2.2 martin if (descr[0]) 3859 1.3.2.4 martin aprint_normal_dev(dev, "PHY: %s, rev. %d", 3860 1.3.2.2 martin descr, rev); 3861 1.3.2.2 martin else 3862 1.3.2.2 martin aprint_normal_dev(dev, 3863 1.3.2.4 martin "PHY OUI 0x%06x, model 0x%04x, rev. %d", 3864 1.3.2.2 martin oui, model, rev); 3865 1.3.2.2 martin 3866 1.3.2.4 martin /* PHY FW version */ 3867 1.3.2.4 martin phy->ops.read_reg(hw, 0x1e, &phy_ver); 3868 1.3.2.4 martin aprint_normal(", PHY FW version 0x%04hx\n", phy_ver); 3869 1.3.2.4 martin 3870 1.3.2.4 martin /* NVM version */ 3871 1.3.2.2 martin hw->nvm.ops.read(hw, NVM_VERSION, 1, &nvm_ver); 3872 1.3.2.2 martin 3873 1.3.2.4 martin /* EtrackID */ 3874 1.3.2.4 martin hw->nvm.ops.read(hw, NVM_ETKID_LO, 1, &etk_lo); 3875 1.3.2.4 martin hw->nvm.ops.read(hw, NVM_ETKID_HI, 1, &etk_hi); 3876 1.3.2.2 martin 3877 1.3.2.4 martin aprint_normal_dev(dev, 3878 1.3.2.4 martin "NVM image version %x.%02x, EtrackID %04hx%04hx\n", 3879 1.3.2.2 martin (nvm_ver & NVM_VERSION_MAJOR) >> NVM_VERSION_MAJOR_SHIFT, 3880 1.3.2.4 martin nvm_ver & NVM_VERSION_MINOR, etk_hi, etk_lo); 3881 1.3.2.2 martin } 3882
Indexes created Tue Sep 30 20:09:53 GMT 2025