ixgbe.c revision 1.320
1 /* $NetBSD: ixgbe.c,v 1.320 2022/06/02 16:56:22 msaitoh Exp $ */ 2 3 /****************************************************************************** 4 5 Copyright (c) 2001-2017, Intel Corporation 6 All rights reserved. 7 8 Redistribution and use in source and binary forms, with or without 9 modification, are permitted provided that the following conditions are met: 10 11 1. Redistributions of source code must retain the above copyright notice, 12 this list of conditions and the following disclaimer. 13 14 2. Redistributions in binary form must reproduce the above copyright 15 notice, this list of conditions and the following disclaimer in the 16 documentation and/or other materials provided with the distribution. 17 18 3. Neither the name of the Intel Corporation nor the names of its 19 contributors may be used to endorse or promote products derived from 20 this software without specific prior written permission. 21 22 THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 23 AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 24 IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 25 ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE 26 LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 27 CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 28 SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 29 INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 30 CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 31 ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 32 POSSIBILITY OF SUCH DAMAGE. 33 34 ******************************************************************************/ 35 /*$FreeBSD: head/sys/dev/ixgbe/if_ix.c 331224 2018-03-19 20:55:05Z erj $*/ 36 37 /* 38 * Copyright (c) 2011 The NetBSD Foundation, Inc. 39 * All rights reserved. 40 * 41 * This code is derived from software contributed to The NetBSD Foundation 42 * by Coyote Point Systems, Inc. 43 * 44 * Redistribution and use in source and binary forms, with or without 45 * modification, are permitted provided that the following conditions 46 * are met: 47 * 1. Redistributions of source code must retain the above copyright 48 * notice, this list of conditions and the following disclaimer. 49 * 2. Redistributions in binary form must reproduce the above copyright 50 * notice, this list of conditions and the following disclaimer in the 51 * documentation and/or other materials provided with the distribution. 52 * 53 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS 54 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED 55 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 56 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS 57 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 58 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 59 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 60 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 61 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 62 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 63 * POSSIBILITY OF SUCH DAMAGE. 64 */ 65 66 #include <sys/cdefs.h> 67 __KERNEL_RCSID(0, "$NetBSD: ixgbe.c,v 1.320 2022/06/02 16:56:22 msaitoh Exp $"); 68 69 #ifdef _KERNEL_OPT 70 #include "opt_inet.h" 71 #include "opt_inet6.h" 72 #include "opt_net_mpsafe.h" 73 #endif 74 75 #include "ixgbe.h" 76 #include "ixgbe_phy.h" 77 #include "ixgbe_sriov.h" 78 79 #include <sys/cprng.h> 80 #include <dev/mii/mii.h> 81 #include <dev/mii/miivar.h> 82 83 /************************************************************************ 84 * Driver version 85 ************************************************************************/ 86 static const char ixgbe_driver_version[] = "4.0.1-k"; 87 /* XXX NetBSD: + 3.3.24 */ 88 89 /************************************************************************ 90 * PCI Device ID Table 91 * 92 * Used by probe to select devices to load on 93 * Last field stores an index into ixgbe_strings 94 * Last entry must be all 0s 95 * 96 * { Vendor ID, Device ID, SubVendor ID, SubDevice ID, String Index } 97 ************************************************************************/ 98 static const ixgbe_vendor_info_t ixgbe_vendor_info_array[] = 99 { 100 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598AF_DUAL_PORT, 0, 0, 0}, 101 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598AF_SINGLE_PORT, 0, 0, 0}, 102 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598EB_CX4, 0, 0, 0}, 103 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598AT, 0, 0, 0}, 104 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598AT2, 0, 0, 0}, 105 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598, 0, 0, 0}, 106 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598_BX, 0, 0, 0}, 107 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598_DA_DUAL_PORT, 0, 0, 0}, 108 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598_CX4_DUAL_PORT, 0, 0, 0}, 109 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598EB_XF_LR, 0, 0, 0}, 110 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598_SR_DUAL_PORT_EM, 0, 0, 0}, 111 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598EB_SFP_LOM, 0, 0, 0}, 112 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_KR, 0, 0, 0}, 113 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_KX4, 0, 0, 0}, 114 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_KX4_MEZZ, 0, 0, 0}, 115 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_SFP, 0, 0, 0}, 116 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_SFP_EM, 0, 0, 0}, 117 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_XAUI_LOM, 0, 0, 0}, 118 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_CX4, 0, 0, 0}, 119 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_T3_LOM, 0, 0, 0}, 120 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_COMBO_BACKPLANE, 0, 0, 0}, 121 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_BACKPLANE_FCOE, 0, 0, 0}, 122 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_SFP_SF2, 0, 0, 0}, 123 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_SFP_FCOE, 0, 0, 0}, 124 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599EN_SFP, 0, 0, 0}, 125 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_SFP_SF_QP, 0, 0, 0}, 126 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_QSFP_SF_QP, 0, 0, 0}, 127 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_X540T, 0, 0, 0}, 128 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_X540T1, 0, 0, 0}, 129 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_X550T, 0, 0, 0}, 130 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_X550T1, 0, 0, 0}, 131 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_X550EM_X_KR, 0, 0, 0}, 132 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_X550EM_X_KX4, 0, 0, 0}, 133 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_X550EM_X_10G_T, 0, 0, 0}, 134 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_X550EM_X_1G_T, 0, 0, 0}, 135 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_X550EM_X_SFP, 0, 0, 0}, 136 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_X550EM_X_XFI, 0, 0, 0}, 137 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_X550EM_A_KR, 0, 0, 0}, 138 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_X550EM_A_KR_L, 0, 0, 0}, 139 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_X550EM_A_QSFP, 0, 0, 0}, 140 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_X550EM_A_QSFP_N, 0, 0, 0}, 141 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_X550EM_A_SFP, 0, 0, 0}, 142 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_X550EM_A_SFP_N, 0, 0, 0}, 143 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_X550EM_A_SGMII, 0, 0, 0}, 144 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_X550EM_A_SGMII_L, 0, 0, 0}, 145 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_X550EM_A_10G_T, 0, 0, 0}, 146 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_X550EM_A_1G_T, 0, 0, 0}, 147 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_X550EM_A_1G_T_L, 0, 0, 0}, 148 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_X540_BYPASS, 0, 0, 0}, 149 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_BYPASS, 0, 0, 0}, 150 /* required last entry */ 151 {0, 0, 0, 0, 0} 152 }; 153 154 /************************************************************************ 155 * Table of branding strings 156 ************************************************************************/ 157 static const char *ixgbe_strings[] = { 158 "Intel(R) PRO/10GbE PCI-Express Network Driver" 159 }; 160 161 /************************************************************************ 162 * Function prototypes 163 ************************************************************************/ 164 static int ixgbe_probe(device_t, cfdata_t, void *); 165 static void ixgbe_quirks(struct adapter *); 166 static void ixgbe_attach(device_t, device_t, void *); 167 static int ixgbe_detach(device_t, int); 168 #if 0 169 static int ixgbe_shutdown(device_t); 170 #endif 171 static bool ixgbe_suspend(device_t, const pmf_qual_t *); 172 static bool ixgbe_resume(device_t, const pmf_qual_t *); 173 static int ixgbe_ifflags_cb(struct ethercom *); 174 static int ixgbe_ioctl(struct ifnet *, u_long, void *); 175 static int ixgbe_init(struct ifnet *); 176 static void ixgbe_init_locked(struct adapter *); 177 static void ixgbe_ifstop(struct ifnet *, int); 178 static void ixgbe_stop_locked(void *); 179 static void ixgbe_init_device_features(struct adapter *); 180 static int ixgbe_check_fan_failure(struct adapter *, u32, bool); 181 static void ixgbe_add_media_types(struct adapter *); 182 static void ixgbe_media_status(struct ifnet *, struct ifmediareq *); 183 static int ixgbe_media_change(struct ifnet *); 184 static int ixgbe_allocate_pci_resources(struct adapter *, 185 const struct pci_attach_args *); 186 static void ixgbe_free_deferred_handlers(struct adapter *); 187 static void ixgbe_get_slot_info(struct adapter *); 188 static int ixgbe_allocate_msix(struct adapter *, 189 const struct pci_attach_args *); 190 static int ixgbe_allocate_legacy(struct adapter *, 191 const struct pci_attach_args *); 192 static int ixgbe_configure_interrupts(struct adapter *); 193 static void ixgbe_free_pciintr_resources(struct adapter *); 194 static void ixgbe_free_pci_resources(struct adapter *); 195 static void ixgbe_local_timer(void *); 196 static void ixgbe_handle_timer(struct work *, void *); 197 static void ixgbe_recovery_mode_timer(void *); 198 static void ixgbe_handle_recovery_mode_timer(struct work *, void *); 199 static int ixgbe_setup_interface(device_t, struct adapter *); 200 static void ixgbe_config_gpie(struct adapter *); 201 static void ixgbe_config_dmac(struct adapter *); 202 static void ixgbe_config_delay_values(struct adapter *); 203 static void ixgbe_schedule_admin_tasklet(struct adapter *); 204 static void ixgbe_config_link(struct adapter *); 205 static void ixgbe_check_wol_support(struct adapter *); 206 static int ixgbe_setup_low_power_mode(struct adapter *); 207 #if 0 208 static void ixgbe_rearm_queues(struct adapter *, u64); 209 #endif 210 211 static void ixgbe_initialize_transmit_units(struct adapter *); 212 static void ixgbe_initialize_receive_units(struct adapter *); 213 static void ixgbe_enable_rx_drop(struct adapter *); 214 static void ixgbe_disable_rx_drop(struct adapter *); 215 static void ixgbe_initialize_rss_mapping(struct adapter *); 216 217 static void ixgbe_enable_intr(struct adapter *); 218 static void ixgbe_disable_intr(struct adapter *); 219 static void ixgbe_update_stats_counters(struct adapter *); 220 static void ixgbe_set_rxfilter(struct adapter *); 221 static void ixgbe_update_link_status(struct adapter *); 222 static void ixgbe_set_ivar(struct adapter *, u8, u8, s8); 223 static void ixgbe_configure_ivars(struct adapter *); 224 static u8 * ixgbe_mc_array_itr(struct ixgbe_hw *, u8 **, u32 *); 225 static void ixgbe_eitr_write(struct adapter *, uint32_t, uint32_t); 226 227 static void ixgbe_setup_vlan_hw_tagging(struct adapter *); 228 static void ixgbe_setup_vlan_hw_support(struct adapter *); 229 static int ixgbe_vlan_cb(struct ethercom *, uint16_t, bool); 230 static int ixgbe_register_vlan(struct adapter *, u16); 231 static int ixgbe_unregister_vlan(struct adapter *, u16); 232 233 static void ixgbe_add_device_sysctls(struct adapter *); 234 static void ixgbe_add_hw_stats(struct adapter *); 235 static void ixgbe_clear_evcnt(struct adapter *); 236 static int ixgbe_set_flowcntl(struct adapter *, int); 237 static int ixgbe_set_advertise(struct adapter *, int); 238 static int ixgbe_get_default_advertise(struct adapter *); 239 240 /* Sysctl handlers */ 241 static int ixgbe_sysctl_flowcntl(SYSCTLFN_PROTO); 242 static int ixgbe_sysctl_advertise(SYSCTLFN_PROTO); 243 static int ixgbe_sysctl_interrupt_rate_handler(SYSCTLFN_PROTO); 244 static int ixgbe_sysctl_dmac(SYSCTLFN_PROTO); 245 static int ixgbe_sysctl_phy_temp(SYSCTLFN_PROTO); 246 static int ixgbe_sysctl_phy_overtemp_occurred(SYSCTLFN_PROTO); 247 #ifdef IXGBE_DEBUG 248 static int ixgbe_sysctl_power_state(SYSCTLFN_PROTO); 249 static int ixgbe_sysctl_print_rss_config(SYSCTLFN_PROTO); 250 #endif 251 static int ixgbe_sysctl_next_to_check_handler(SYSCTLFN_PROTO); 252 static int ixgbe_sysctl_next_to_refresh_handler(SYSCTLFN_PROTO); 253 static int ixgbe_sysctl_rdh_handler(SYSCTLFN_PROTO); 254 static int ixgbe_sysctl_rdt_handler(SYSCTLFN_PROTO); 255 static int ixgbe_sysctl_tdt_handler(SYSCTLFN_PROTO); 256 static int ixgbe_sysctl_tdh_handler(SYSCTLFN_PROTO); 257 static int ixgbe_sysctl_eee_state(SYSCTLFN_PROTO); 258 static int ixgbe_sysctl_debug(SYSCTLFN_PROTO); 259 static int ixgbe_sysctl_rx_copy_len(SYSCTLFN_PROTO); 260 static int ixgbe_sysctl_tx_process_limit(SYSCTLFN_PROTO); 261 static int ixgbe_sysctl_rx_process_limit(SYSCTLFN_PROTO); 262 static int ixgbe_sysctl_wol_enable(SYSCTLFN_PROTO); 263 static int ixgbe_sysctl_wufc(SYSCTLFN_PROTO); 264 265 /* Interrupt functions */ 266 static int ixgbe_msix_que(void *); 267 static int ixgbe_msix_admin(void *); 268 static void ixgbe_intr_admin_common(struct adapter *, u32, u32 *); 269 static int ixgbe_legacy_irq(void *); 270 271 /* Event handlers running on workqueue */ 272 static void ixgbe_handle_que(void *); 273 static void ixgbe_handle_link(void *); 274 static void ixgbe_handle_msf(void *); 275 static void ixgbe_handle_mod(void *, bool); 276 static void ixgbe_handle_phy(void *); 277 278 /* Deferred workqueue handlers */ 279 static void ixgbe_handle_admin(struct work *, void *); 280 static void ixgbe_handle_que_work(struct work *, void *); 281 282 static const ixgbe_vendor_info_t *ixgbe_lookup(const struct pci_attach_args *); 283 284 /************************************************************************ 285 * NetBSD Device Interface Entry Points 286 ************************************************************************/ 287 CFATTACH_DECL3_NEW(ixg, sizeof(struct adapter), 288 ixgbe_probe, ixgbe_attach, ixgbe_detach, NULL, NULL, NULL, 289 DVF_DETACH_SHUTDOWN); 290 291 #if 0 292 devclass_t ix_devclass; 293 DRIVER_MODULE(ix, pci, ix_driver, ix_devclass, 0, 0); 294 295 MODULE_DEPEND(ix, pci, 1, 1, 1); 296 MODULE_DEPEND(ix, ether, 1, 1, 1); 297 #ifdef DEV_NETMAP 298 MODULE_DEPEND(ix, netmap, 1, 1, 1); 299 #endif 300 #endif 301 302 /* 303 * TUNEABLE PARAMETERS: 304 */ 305 306 /* 307 * AIM: Adaptive Interrupt Moderation 308 * which means that the interrupt rate 309 * is varied over time based on the 310 * traffic for that interrupt vector 311 */ 312 static bool ixgbe_enable_aim = true; 313 #define SYSCTL_INT(_a1, _a2, _a3, _a4, _a5, _a6, _a7) 314 SYSCTL_INT(_hw_ix, OID_AUTO, enable_aim, CTLFLAG_RDTUN, &ixgbe_enable_aim, 0, 315 "Enable adaptive interrupt moderation"); 316 317 static int ixgbe_max_interrupt_rate = (4000000 / IXGBE_LOW_LATENCY); 318 SYSCTL_INT(_hw_ix, OID_AUTO, max_interrupt_rate, CTLFLAG_RDTUN, 319 &ixgbe_max_interrupt_rate, 0, "Maximum interrupts per second"); 320 321 /* How many packets rxeof tries to clean at a time */ 322 static int ixgbe_rx_process_limit = 256; 323 SYSCTL_INT(_hw_ix, OID_AUTO, rx_process_limit, CTLFLAG_RDTUN, 324 &ixgbe_rx_process_limit, 0, "Maximum number of received packets to process at a time, -1 means unlimited"); 325 326 /* How many packets txeof tries to clean at a time */ 327 static int ixgbe_tx_process_limit = 256; 328 SYSCTL_INT(_hw_ix, OID_AUTO, tx_process_limit, CTLFLAG_RDTUN, 329 &ixgbe_tx_process_limit, 0, 330 "Maximum number of sent packets to process at a time, -1 means unlimited"); 331 332 /* Flow control setting, default to full */ 333 static int ixgbe_flow_control = ixgbe_fc_full; 334 SYSCTL_INT(_hw_ix, OID_AUTO, flow_control, CTLFLAG_RDTUN, 335 &ixgbe_flow_control, 0, "Default flow control used for all adapters"); 336 337 /* Which packet processing uses workqueue or softint */ 338 static bool ixgbe_txrx_workqueue = false; 339 340 /* 341 * Smart speed setting, default to on 342 * this only works as a compile option 343 * right now as its during attach, set 344 * this to 'ixgbe_smart_speed_off' to 345 * disable. 346 */ 347 static int ixgbe_smart_speed = ixgbe_smart_speed_on; 348 349 /* 350 * MSI-X should be the default for best performance, 351 * but this allows it to be forced off for testing. 352 */ 353 static int ixgbe_enable_msix = 1; 354 SYSCTL_INT(_hw_ix, OID_AUTO, enable_msix, CTLFLAG_RDTUN, &ixgbe_enable_msix, 0, 355 "Enable MSI-X interrupts"); 356 357 /* 358 * Number of Queues, can be set to 0, 359 * it then autoconfigures based on the 360 * number of cpus with a max of 8. This 361 * can be overridden manually here. 362 */ 363 static int ixgbe_num_queues = 0; 364 SYSCTL_INT(_hw_ix, OID_AUTO, num_queues, CTLFLAG_RDTUN, &ixgbe_num_queues, 0, 365 "Number of queues to configure, 0 indicates autoconfigure"); 366 367 /* 368 * Number of TX descriptors per ring, 369 * setting higher than RX as this seems 370 * the better performing choice. 371 */ 372 static int ixgbe_txd = PERFORM_TXD; 373 SYSCTL_INT(_hw_ix, OID_AUTO, txd, CTLFLAG_RDTUN, &ixgbe_txd, 0, 374 "Number of transmit descriptors per queue"); 375 376 /* Number of RX descriptors per ring */ 377 static int ixgbe_rxd = PERFORM_RXD; 378 SYSCTL_INT(_hw_ix, OID_AUTO, rxd, CTLFLAG_RDTUN, &ixgbe_rxd, 0, 379 "Number of receive descriptors per queue"); 380 381 /* 382 * Defining this on will allow the use 383 * of unsupported SFP+ modules, note that 384 * doing so you are on your own :) 385 */ 386 static int allow_unsupported_sfp = false; 387 #define TUNABLE_INT(__x, __y) 388 TUNABLE_INT("hw.ix.unsupported_sfp", &allow_unsupported_sfp); 389 390 /* 391 * Not sure if Flow Director is fully baked, 392 * so we'll default to turning it off. 393 */ 394 static int ixgbe_enable_fdir = 0; 395 SYSCTL_INT(_hw_ix, OID_AUTO, enable_fdir, CTLFLAG_RDTUN, &ixgbe_enable_fdir, 0, 396 "Enable Flow Director"); 397 398 /* Legacy Transmit (single queue) */ 399 static int ixgbe_enable_legacy_tx = 0; 400 SYSCTL_INT(_hw_ix, OID_AUTO, enable_legacy_tx, CTLFLAG_RDTUN, 401 &ixgbe_enable_legacy_tx, 0, "Enable Legacy TX flow"); 402 403 /* Receive-Side Scaling */ 404 static int ixgbe_enable_rss = 1; 405 SYSCTL_INT(_hw_ix, OID_AUTO, enable_rss, CTLFLAG_RDTUN, &ixgbe_enable_rss, 0, 406 "Enable Receive-Side Scaling (RSS)"); 407 408 #if 0 409 static int (*ixgbe_start_locked)(struct ifnet *, struct tx_ring *); 410 static int (*ixgbe_ring_empty)(struct ifnet *, pcq_t *); 411 #endif 412 413 #ifdef NET_MPSAFE 414 #define IXGBE_MPSAFE 1 415 #define IXGBE_CALLOUT_FLAGS CALLOUT_MPSAFE 416 #define IXGBE_SOFTINT_FLAGS SOFTINT_MPSAFE 417 #define IXGBE_WORKQUEUE_FLAGS WQ_PERCPU | WQ_MPSAFE 418 #define IXGBE_TASKLET_WQ_FLAGS WQ_MPSAFE 419 #else 420 #define IXGBE_CALLOUT_FLAGS 0 421 #define IXGBE_SOFTINT_FLAGS 0 422 #define IXGBE_WORKQUEUE_FLAGS WQ_PERCPU 423 #define IXGBE_TASKLET_WQ_FLAGS 0 424 #endif 425 #define IXGBE_WORKQUEUE_PRI PRI_SOFTNET 426 427 /* Interval between reports of errors */ 428 static const struct timeval ixgbe_errlog_intrvl = { 60, 0 }; /* 60s */ 429 430 /************************************************************************ 431 * ixgbe_initialize_rss_mapping 432 ************************************************************************/ 433 static void 434 ixgbe_initialize_rss_mapping(struct adapter *adapter) 435 { 436 struct ixgbe_hw *hw = &adapter->hw; 437 u32 reta = 0, mrqc, rss_key[10]; 438 int queue_id, table_size, index_mult; 439 int i, j; 440 u32 rss_hash_config; 441 442 /* force use default RSS key. */ 443 #ifdef __NetBSD__ 444 rss_getkey((uint8_t *) &rss_key); 445 #else 446 if (adapter->feat_en & IXGBE_FEATURE_RSS) { 447 /* Fetch the configured RSS key */ 448 rss_getkey((uint8_t *) &rss_key); 449 } else { 450 /* set up random bits */ 451 cprng_fast(&rss_key, sizeof(rss_key)); 452 } 453 #endif 454 455 /* Set multiplier for RETA setup and table size based on MAC */ 456 index_mult = 0x1; 457 table_size = 128; 458 switch (adapter->hw.mac.type) { 459 case ixgbe_mac_82598EB: 460 index_mult = 0x11; 461 break; 462 case ixgbe_mac_X550: 463 case ixgbe_mac_X550EM_x: 464 case ixgbe_mac_X550EM_a: 465 table_size = 512; 466 break; 467 default: 468 break; 469 } 470 471 /* Set up the redirection table */ 472 for (i = 0, j = 0; i < table_size; i++, j++) { 473 if (j == adapter->num_queues) 474 j = 0; 475 476 if (adapter->feat_en & IXGBE_FEATURE_RSS) { 477 /* 478 * Fetch the RSS bucket id for the given indirection 479 * entry. Cap it at the number of configured buckets 480 * (which is num_queues.) 481 */ 482 queue_id = rss_get_indirection_to_bucket(i); 483 queue_id = queue_id % adapter->num_queues; 484 } else 485 queue_id = (j * index_mult); 486 487 /* 488 * The low 8 bits are for hash value (n+0); 489 * The next 8 bits are for hash value (n+1), etc. 490 */ 491 reta = reta >> 8; 492 reta = reta | (((uint32_t) queue_id) << 24); 493 if ((i & 3) == 3) { 494 if (i < 128) 495 IXGBE_WRITE_REG(hw, IXGBE_RETA(i >> 2), reta); 496 else 497 IXGBE_WRITE_REG(hw, IXGBE_ERETA((i >> 2) - 32), 498 reta); 499 reta = 0; 500 } 501 } 502 503 /* Now fill our hash function seeds */ 504 for (i = 0; i < 10; i++) 505 IXGBE_WRITE_REG(hw, IXGBE_RSSRK(i), rss_key[i]); 506 507 /* Perform hash on these packet types */ 508 if (adapter->feat_en & IXGBE_FEATURE_RSS) 509 rss_hash_config = rss_gethashconfig(); 510 else { 511 /* 512 * Disable UDP - IP fragments aren't currently being handled 513 * and so we end up with a mix of 2-tuple and 4-tuple 514 * traffic. 515 */ 516 rss_hash_config = RSS_HASHTYPE_RSS_IPV4 517 | RSS_HASHTYPE_RSS_TCP_IPV4 518 | RSS_HASHTYPE_RSS_IPV6 519 | RSS_HASHTYPE_RSS_TCP_IPV6 520 | RSS_HASHTYPE_RSS_IPV6_EX 521 | RSS_HASHTYPE_RSS_TCP_IPV6_EX; 522 } 523 524 mrqc = IXGBE_MRQC_RSSEN; 525 if (rss_hash_config & RSS_HASHTYPE_RSS_IPV4) 526 mrqc |= IXGBE_MRQC_RSS_FIELD_IPV4; 527 if (rss_hash_config & RSS_HASHTYPE_RSS_TCP_IPV4) 528 mrqc |= IXGBE_MRQC_RSS_FIELD_IPV4_TCP; 529 if (rss_hash_config & RSS_HASHTYPE_RSS_IPV6) 530 mrqc |= IXGBE_MRQC_RSS_FIELD_IPV6; 531 if (rss_hash_config & RSS_HASHTYPE_RSS_TCP_IPV6) 532 mrqc |= IXGBE_MRQC_RSS_FIELD_IPV6_TCP; 533 if (rss_hash_config & RSS_HASHTYPE_RSS_IPV6_EX) 534 mrqc |= IXGBE_MRQC_RSS_FIELD_IPV6_EX; 535 if (rss_hash_config & RSS_HASHTYPE_RSS_TCP_IPV6_EX) 536 mrqc |= IXGBE_MRQC_RSS_FIELD_IPV6_EX_TCP; 537 if (rss_hash_config & RSS_HASHTYPE_RSS_UDP_IPV4) 538 mrqc |= IXGBE_MRQC_RSS_FIELD_IPV4_UDP; 539 if (rss_hash_config & RSS_HASHTYPE_RSS_UDP_IPV6) 540 mrqc |= IXGBE_MRQC_RSS_FIELD_IPV6_UDP; 541 if (rss_hash_config & RSS_HASHTYPE_RSS_UDP_IPV6_EX) 542 mrqc |= IXGBE_MRQC_RSS_FIELD_IPV6_EX_UDP; 543 mrqc |= ixgbe_get_mrqc(adapter->iov_mode); 544 IXGBE_WRITE_REG(hw, IXGBE_MRQC, mrqc); 545 } /* ixgbe_initialize_rss_mapping */ 546 547 /************************************************************************ 548 * ixgbe_initialize_receive_units - Setup receive registers and features. 549 ************************************************************************/ 550 #define BSIZEPKT_ROUNDUP ((1<<IXGBE_SRRCTL_BSIZEPKT_SHIFT)-1) 551 552 static void 553 ixgbe_initialize_receive_units(struct adapter *adapter) 554 { 555 struct rx_ring *rxr = adapter->rx_rings; 556 struct ixgbe_hw *hw = &adapter->hw; 557 struct ifnet *ifp = adapter->ifp; 558 int i, j; 559 u32 bufsz, fctrl, srrctl, rxcsum; 560 u32 hlreg; 561 562 /* 563 * Make sure receives are disabled while 564 * setting up the descriptor ring 565 */ 566 ixgbe_disable_rx(hw); 567 568 /* Enable broadcasts */ 569 fctrl = IXGBE_READ_REG(hw, IXGBE_FCTRL); 570 fctrl |= IXGBE_FCTRL_BAM; 571 if (adapter->hw.mac.type == ixgbe_mac_82598EB) { 572 fctrl |= IXGBE_FCTRL_DPF; 573 fctrl |= IXGBE_FCTRL_PMCF; 574 } 575 IXGBE_WRITE_REG(hw, IXGBE_FCTRL, fctrl); 576 577 /* Set for Jumbo Frames? */ 578 hlreg = IXGBE_READ_REG(hw, IXGBE_HLREG0); 579 if (ifp->if_mtu > ETHERMTU) 580 hlreg |= IXGBE_HLREG0_JUMBOEN; 581 else 582 hlreg &= ~IXGBE_HLREG0_JUMBOEN; 583 584 #ifdef DEV_NETMAP 585 /* CRC stripping is conditional in Netmap */ 586 if ((adapter->feat_en & IXGBE_FEATURE_NETMAP) && 587 (ifp->if_capenable & IFCAP_NETMAP) && 588 !ix_crcstrip) 589 hlreg &= ~IXGBE_HLREG0_RXCRCSTRP; 590 else 591 #endif /* DEV_NETMAP */ 592 hlreg |= IXGBE_HLREG0_RXCRCSTRP; 593 594 IXGBE_WRITE_REG(hw, IXGBE_HLREG0, hlreg); 595 596 bufsz = (adapter->rx_mbuf_sz + BSIZEPKT_ROUNDUP) >> 597 IXGBE_SRRCTL_BSIZEPKT_SHIFT; 598 599 for (i = 0; i < adapter->num_queues; i++, rxr++) { 600 u64 rdba = rxr->rxdma.dma_paddr; 601 u32 reg; 602 int regnum = i / 4; /* 1 register per 4 queues */ 603 int regshift = i % 4; /* 4 bits per 1 queue */ 604 j = rxr->me; 605 606 /* Setup the Base and Length of the Rx Descriptor Ring */ 607 IXGBE_WRITE_REG(hw, IXGBE_RDBAL(j), 608 (rdba & 0x00000000ffffffffULL)); 609 IXGBE_WRITE_REG(hw, IXGBE_RDBAH(j), (rdba >> 32)); 610 IXGBE_WRITE_REG(hw, IXGBE_RDLEN(j), 611 adapter->num_rx_desc * sizeof(union ixgbe_adv_rx_desc)); 612 613 /* Set up the SRRCTL register */ 614 srrctl = IXGBE_READ_REG(hw, IXGBE_SRRCTL(j)); 615 srrctl &= ~IXGBE_SRRCTL_BSIZEHDR_MASK; 616 srrctl &= ~IXGBE_SRRCTL_BSIZEPKT_MASK; 617 srrctl |= bufsz; 618 srrctl |= IXGBE_SRRCTL_DESCTYPE_ADV_ONEBUF; 619 620 /* Set RQSMR (Receive Queue Statistic Mapping) register */ 621 reg = IXGBE_READ_REG(hw, IXGBE_RQSMR(regnum)); 622 reg &= ~(0x000000ffUL << (regshift * 8)); 623 reg |= i << (regshift * 8); 624 IXGBE_WRITE_REG(hw, IXGBE_RQSMR(regnum), reg); 625 626 /* 627 * Set DROP_EN iff we have no flow control and >1 queue. 628 * Note that srrctl was cleared shortly before during reset, 629 * so we do not need to clear the bit, but do it just in case 630 * this code is moved elsewhere. 631 */ 632 if ((adapter->num_queues > 1) && 633 (adapter->hw.fc.requested_mode == ixgbe_fc_none)) 634 srrctl |= IXGBE_SRRCTL_DROP_EN; 635 else 636 srrctl &= ~IXGBE_SRRCTL_DROP_EN; 637 638 IXGBE_WRITE_REG(hw, IXGBE_SRRCTL(j), srrctl); 639 640 /* Setup the HW Rx Head and Tail Descriptor Pointers */ 641 IXGBE_WRITE_REG(hw, IXGBE_RDH(j), 0); 642 IXGBE_WRITE_REG(hw, IXGBE_RDT(j), 0); 643 644 /* Set the driver rx tail address */ 645 rxr->tail = IXGBE_RDT(rxr->me); 646 } 647 648 if (adapter->hw.mac.type != ixgbe_mac_82598EB) { 649 u32 psrtype = IXGBE_PSRTYPE_TCPHDR 650 | IXGBE_PSRTYPE_UDPHDR 651 | IXGBE_PSRTYPE_IPV4HDR 652 | IXGBE_PSRTYPE_IPV6HDR; 653 IXGBE_WRITE_REG(hw, IXGBE_PSRTYPE(0), psrtype); 654 } 655 656 rxcsum = IXGBE_READ_REG(hw, IXGBE_RXCSUM); 657 658 ixgbe_initialize_rss_mapping(adapter); 659 660 if (adapter->num_queues > 1) { 661 /* RSS and RX IPP Checksum are mutually exclusive */ 662 rxcsum |= IXGBE_RXCSUM_PCSD; 663 } 664 665 if (ifp->if_capenable & IFCAP_RXCSUM) 666 rxcsum |= IXGBE_RXCSUM_PCSD; 667 668 /* This is useful for calculating UDP/IP fragment checksums */ 669 if (!(rxcsum & IXGBE_RXCSUM_PCSD)) 670 rxcsum |= IXGBE_RXCSUM_IPPCSE; 671 672 IXGBE_WRITE_REG(hw, IXGBE_RXCSUM, rxcsum); 673 674 } /* ixgbe_initialize_receive_units */ 675 676 /************************************************************************ 677 * ixgbe_initialize_transmit_units - Enable transmit units. 678 ************************************************************************/ 679 static void 680 ixgbe_initialize_transmit_units(struct adapter *adapter) 681 { 682 struct tx_ring *txr = adapter->tx_rings; 683 struct ixgbe_hw *hw = &adapter->hw; 684 int i; 685 686 INIT_DEBUGOUT("ixgbe_initialize_transmit_units"); 687 688 /* Setup the Base and Length of the Tx Descriptor Ring */ 689 for (i = 0; i < adapter->num_queues; i++, txr++) { 690 u64 tdba = txr->txdma.dma_paddr; 691 u32 txctrl = 0; 692 u32 tqsmreg, reg; 693 int regnum = i / 4; /* 1 register per 4 queues */ 694 int regshift = i % 4; /* 4 bits per 1 queue */ 695 int j = txr->me; 696 697 IXGBE_WRITE_REG(hw, IXGBE_TDBAL(j), 698 (tdba & 0x00000000ffffffffULL)); 699 IXGBE_WRITE_REG(hw, IXGBE_TDBAH(j), (tdba >> 32)); 700 IXGBE_WRITE_REG(hw, IXGBE_TDLEN(j), 701 adapter->num_tx_desc * sizeof(union ixgbe_adv_tx_desc)); 702 703 /* 704 * Set TQSMR (Transmit Queue Statistic Mapping) register. 705 * Register location is different between 82598 and others. 706 */ 707 if (adapter->hw.mac.type == ixgbe_mac_82598EB) 708 tqsmreg = IXGBE_TQSMR(regnum); 709 else 710 tqsmreg = IXGBE_TQSM(regnum); 711 reg = IXGBE_READ_REG(hw, tqsmreg); 712 reg &= ~(0x000000ffUL << (regshift * 8)); 713 reg |= i << (regshift * 8); 714 IXGBE_WRITE_REG(hw, tqsmreg, reg); 715 716 /* Setup the HW Tx Head and Tail descriptor pointers */ 717 IXGBE_WRITE_REG(hw, IXGBE_TDH(j), 0); 718 IXGBE_WRITE_REG(hw, IXGBE_TDT(j), 0); 719 720 /* Cache the tail address */ 721 txr->tail = IXGBE_TDT(j); 722 723 txr->txr_no_space = false; 724 725 /* Disable Head Writeback */ 726 /* 727 * Note: for X550 series devices, these registers are actually 728 * prefixed with TPH_ instead of DCA_, but the addresses and 729 * fields remain the same. 730 */ 731 switch (hw->mac.type) { 732 case ixgbe_mac_82598EB: 733 txctrl = IXGBE_READ_REG(hw, IXGBE_DCA_TXCTRL(j)); 734 break; 735 default: 736 txctrl = IXGBE_READ_REG(hw, IXGBE_DCA_TXCTRL_82599(j)); 737 break; 738 } 739 txctrl &= ~IXGBE_DCA_TXCTRL_DESC_WRO_EN; 740 switch (hw->mac.type) { 741 case ixgbe_mac_82598EB: 742 IXGBE_WRITE_REG(hw, IXGBE_DCA_TXCTRL(j), txctrl); 743 break; 744 default: 745 IXGBE_WRITE_REG(hw, IXGBE_DCA_TXCTRL_82599(j), txctrl); 746 break; 747 } 748 749 } 750 751 if (hw->mac.type != ixgbe_mac_82598EB) { 752 u32 dmatxctl, rttdcs; 753 754 dmatxctl = IXGBE_READ_REG(hw, IXGBE_DMATXCTL); 755 dmatxctl |= IXGBE_DMATXCTL_TE; 756 IXGBE_WRITE_REG(hw, IXGBE_DMATXCTL, dmatxctl); 757 /* Disable arbiter to set MTQC */ 758 rttdcs = IXGBE_READ_REG(hw, IXGBE_RTTDCS); 759 rttdcs |= IXGBE_RTTDCS_ARBDIS; 760 IXGBE_WRITE_REG(hw, IXGBE_RTTDCS, rttdcs); 761 IXGBE_WRITE_REG(hw, IXGBE_MTQC, 762 ixgbe_get_mtqc(adapter->iov_mode)); 763 rttdcs &= ~IXGBE_RTTDCS_ARBDIS; 764 IXGBE_WRITE_REG(hw, IXGBE_RTTDCS, rttdcs); 765 } 766 767 return; 768 } /* ixgbe_initialize_transmit_units */ 769 770 static void 771 ixgbe_quirks(struct adapter *adapter) 772 { 773 device_t dev = adapter->dev; 774 struct ixgbe_hw *hw = &adapter->hw; 775 const char *vendor, *product; 776 777 if (hw->device_id == IXGBE_DEV_ID_X550EM_A_SFP_N) { 778 /* 779 * Quirk for inverted logic of SFP+'s MOD_ABS on GIGABYTE 780 * MA10-ST0. 781 */ 782 vendor = pmf_get_platform("system-vendor"); 783 product = pmf_get_platform("system-product"); 784 785 if ((vendor == NULL) || (product == NULL)) 786 return; 787 788 if ((strcmp(vendor, "GIGABYTE") == 0) && 789 (strcmp(product, "MA10-ST0") == 0)) { 790 aprint_verbose_dev(dev, 791 "Enable SFP+ MOD_ABS inverse quirk\n"); 792 adapter->hw.quirks |= IXGBE_QUIRK_MOD_ABS_INVERT; 793 } 794 } 795 } 796 797 /************************************************************************ 798 * ixgbe_attach - Device initialization routine 799 * 800 * Called when the driver is being loaded. 801 * Identifies the type of hardware, allocates all resources 802 * and initializes the hardware. 803 * 804 * return 0 on success, positive on failure 805 ************************************************************************/ 806 static void 807 ixgbe_attach(device_t parent, device_t dev, void *aux) 808 { 809 struct adapter *adapter; 810 struct ixgbe_hw *hw; 811 int error = -1; 812 u32 ctrl_ext; 813 u16 high, low, nvmreg; 814 pcireg_t id, subid; 815 const ixgbe_vendor_info_t *ent; 816 struct pci_attach_args *pa = aux; 817 bool unsupported_sfp = false; 818 const char *str; 819 char wqname[MAXCOMLEN]; 820 char buf[256]; 821 822 INIT_DEBUGOUT("ixgbe_attach: begin"); 823 824 /* Allocate, clear, and link in our adapter structure */ 825 adapter = device_private(dev); 826 adapter->hw.back = adapter; 827 adapter->dev = dev; 828 hw = &adapter->hw; 829 adapter->osdep.pc = pa->pa_pc; 830 adapter->osdep.tag = pa->pa_tag; 831 if (pci_dma64_available(pa)) 832 adapter->osdep.dmat = pa->pa_dmat64; 833 else 834 adapter->osdep.dmat = pa->pa_dmat; 835 adapter->osdep.attached = false; 836 adapter->osdep.detaching = false; 837 838 ent = ixgbe_lookup(pa); 839 840 KASSERT(ent != NULL); 841 842 aprint_normal(": %s, Version - %s\n", 843 ixgbe_strings[ent->index], ixgbe_driver_version); 844 845 /* Core Lock Init */ 846 IXGBE_CORE_LOCK_INIT(adapter, device_xname(dev)); 847 848 /* Set up the timer callout and workqueue */ 849 callout_init(&adapter->timer, IXGBE_CALLOUT_FLAGS); 850 snprintf(wqname, sizeof(wqname), "%s-timer", device_xname(dev)); 851 error = workqueue_create(&adapter->timer_wq, wqname, 852 ixgbe_handle_timer, adapter, IXGBE_WORKQUEUE_PRI, IPL_NET, 853 IXGBE_TASKLET_WQ_FLAGS); 854 if (error) { 855 aprint_error_dev(dev, 856 "could not create timer workqueue (%d)\n", error); 857 goto err_out; 858 } 859 860 /* Determine hardware revision */ 861 id = pci_conf_read(pa->pa_pc, pa->pa_tag, PCI_ID_REG); 862 subid = pci_conf_read(pa->pa_pc, pa->pa_tag, PCI_SUBSYS_ID_REG); 863 864 hw->vendor_id = PCI_VENDOR(id); 865 hw->device_id = PCI_PRODUCT(id); 866 hw->revision_id = 867 PCI_REVISION(pci_conf_read(pa->pa_pc, pa->pa_tag, PCI_CLASS_REG)); 868 hw->subsystem_vendor_id = PCI_SUBSYS_VENDOR(subid); 869 hw->subsystem_device_id = PCI_SUBSYS_ID(subid); 870 871 /* Set quirk flags */ 872 ixgbe_quirks(adapter); 873 874 /* 875 * Make sure BUSMASTER is set 876 */ 877 ixgbe_pci_enable_busmaster(pa->pa_pc, pa->pa_tag); 878 879 /* Do base PCI setup - map BAR0 */ 880 if (ixgbe_allocate_pci_resources(adapter, pa)) { 881 aprint_error_dev(dev, "Allocation of PCI resources failed\n"); 882 error = ENXIO; 883 goto err_out; 884 } 885 886 /* let hardware know driver is loaded */ 887 ctrl_ext = IXGBE_READ_REG(hw, IXGBE_CTRL_EXT); 888 ctrl_ext |= IXGBE_CTRL_EXT_DRV_LOAD; 889 IXGBE_WRITE_REG(hw, IXGBE_CTRL_EXT, ctrl_ext); 890 891 /* 892 * Initialize the shared code 893 */ 894 if (ixgbe_init_shared_code(hw) != 0) { 895 aprint_error_dev(dev, 896 "Unable to initialize the shared code\n"); 897 error = ENXIO; 898 goto err_out; 899 } 900 901 switch (hw->mac.type) { 902 case ixgbe_mac_82598EB: 903 str = "82598EB"; 904 break; 905 case ixgbe_mac_82599EB: 906 str = "82599EB"; 907 break; 908 case ixgbe_mac_X540: 909 str = "X540"; 910 break; 911 case ixgbe_mac_X550: 912 str = "X550"; 913 break; 914 case ixgbe_mac_X550EM_x: 915 str = "X550EM X"; 916 break; 917 case ixgbe_mac_X550EM_a: 918 str = "X550EM A"; 919 break; 920 default: 921 str = "Unknown"; 922 break; 923 } 924 aprint_normal_dev(dev, "device %s\n", str); 925 926 hw->allow_unsupported_sfp = allow_unsupported_sfp; 927 928 /* Pick up the 82599 settings */ 929 if (hw->mac.type != ixgbe_mac_82598EB) 930 hw->phy.smart_speed = ixgbe_smart_speed; 931 932 /* Set the right number of segments */ 933 KASSERT(IXGBE_82599_SCATTER_MAX >= IXGBE_SCATTER_DEFAULT); 934 adapter->num_segs = IXGBE_SCATTER_DEFAULT; 935 936 /* Ensure SW/FW semaphore is free */ 937 ixgbe_init_swfw_semaphore(hw); 938 939 hw->mac.ops.set_lan_id(hw); 940 ixgbe_init_device_features(adapter); 941 942 if (ixgbe_configure_interrupts(adapter)) { 943 error = ENXIO; 944 goto err_out; 945 } 946 947 /* Allocate multicast array memory. */ 948 adapter->mta = malloc(sizeof(*adapter->mta) * 949 MAX_NUM_MULTICAST_ADDRESSES, M_DEVBUF, M_WAITOK); 950 951 /* Enable WoL (if supported) */ 952 ixgbe_check_wol_support(adapter); 953 954 /* Register for VLAN events */ 955 ether_set_vlan_cb(&adapter->osdep.ec, ixgbe_vlan_cb); 956 957 /* Verify adapter fan is still functional (if applicable) */ 958 if (adapter->feat_en & IXGBE_FEATURE_FAN_FAIL) { 959 u32 esdp = IXGBE_READ_REG(hw, IXGBE_ESDP); 960 ixgbe_check_fan_failure(adapter, esdp, FALSE); 961 } 962 963 /* Set an initial default flow control value */ 964 hw->fc.requested_mode = ixgbe_flow_control; 965 966 /* Do descriptor calc and sanity checks */ 967 if (((ixgbe_txd * sizeof(union ixgbe_adv_tx_desc)) % DBA_ALIGN) != 0 || 968 ixgbe_txd < MIN_TXD || ixgbe_txd > MAX_TXD) { 969 aprint_error_dev(dev, "TXD config issue, using default!\n"); 970 adapter->num_tx_desc = DEFAULT_TXD; 971 } else 972 adapter->num_tx_desc = ixgbe_txd; 973 974 if (((ixgbe_rxd * sizeof(union ixgbe_adv_rx_desc)) % DBA_ALIGN) != 0 || 975 ixgbe_rxd < MIN_RXD || ixgbe_rxd > MAX_RXD) { 976 aprint_error_dev(dev, "RXD config issue, using default!\n"); 977 adapter->num_rx_desc = DEFAULT_RXD; 978 } else 979 adapter->num_rx_desc = ixgbe_rxd; 980 981 /* Sysctls for limiting the amount of work done in the taskqueues */ 982 adapter->rx_process_limit 983 = (ixgbe_rx_process_limit <= adapter->num_rx_desc) 984 ? ixgbe_rx_process_limit : adapter->num_rx_desc; 985 adapter->tx_process_limit 986 = (ixgbe_tx_process_limit <= adapter->num_tx_desc) 987 ? ixgbe_tx_process_limit : adapter->num_tx_desc; 988 989 /* Set default high limit of copying mbuf in rxeof */ 990 adapter->rx_copy_len = IXGBE_RX_COPY_LEN_MAX; 991 992 /* Allocate our TX/RX Queues */ 993 if (ixgbe_allocate_queues(adapter)) { 994 error = ENOMEM; 995 goto err_out; 996 } 997 998 hw->phy.reset_if_overtemp = TRUE; 999 error = ixgbe_reset_hw(hw); 1000 hw->phy.reset_if_overtemp = FALSE; 1001 if (error == IXGBE_ERR_SFP_NOT_PRESENT) 1002 error = IXGBE_SUCCESS; 1003 else if (error == IXGBE_ERR_SFP_NOT_SUPPORTED) { 1004 aprint_error_dev(dev, "Unsupported SFP+ module detected!\n"); 1005 unsupported_sfp = true; 1006 error = IXGBE_SUCCESS; 1007 } else if (error) { 1008 aprint_error_dev(dev, 1009 "Hardware initialization failed(error = %d)\n", error); 1010 error = EIO; 1011 goto err_late; 1012 } 1013 1014 /* Make sure we have a good EEPROM before we read from it */ 1015 if (ixgbe_validate_eeprom_checksum(&adapter->hw, NULL) < 0) { 1016 aprint_error_dev(dev, "The EEPROM Checksum Is Not Valid\n"); 1017 error = EIO; 1018 goto err_late; 1019 } 1020 1021 aprint_normal("%s:", device_xname(dev)); 1022 /* NVM Image Version */ 1023 high = low = 0; 1024 switch (hw->mac.type) { 1025 case ixgbe_mac_82598EB: 1026 /* 1027 * Print version from the dev starter version (0x29). The 1028 * location is the same as newer device's IXGBE_NVM_MAP_VER. 1029 */ 1030 hw->eeprom.ops.read(hw, IXGBE_NVM_MAP_VER, &nvmreg); 1031 if (nvmreg == 0xffff) 1032 break; 1033 high = (nvmreg >> 12) & 0x0f; 1034 low = (nvmreg >> 4) & 0xff; 1035 id = nvmreg & 0x0f; 1036 /* 1037 * The following output might not be correct. Some 82598 cards 1038 * have 0x1070 or 0x2090. 82598 spec update notes about 2.9.0. 1039 */ 1040 aprint_normal(" NVM Image Version %u.%u.%u,", high, low, id); 1041 break; 1042 case ixgbe_mac_X540: 1043 case ixgbe_mac_X550EM_a: 1044 hw->eeprom.ops.read(hw, IXGBE_NVM_IMAGE_VER, &nvmreg); 1045 if (nvmreg == 0xffff) 1046 break; 1047 high = (nvmreg >> 12) & 0x0f; 1048 low = (nvmreg >> 4) & 0xff; 1049 id = nvmreg & 0x0f; 1050 aprint_normal(" NVM Image Version %u.", high); 1051 if (hw->mac.type == ixgbe_mac_X540) 1052 str = "%x"; 1053 else 1054 str = "%02x"; 1055 aprint_normal(str, low); 1056 aprint_normal(" ID 0x%x,", id); 1057 break; 1058 case ixgbe_mac_X550EM_x: 1059 case ixgbe_mac_X550: 1060 hw->eeprom.ops.read(hw, IXGBE_NVM_IMAGE_VER, &nvmreg); 1061 if (nvmreg == 0xffff) 1062 break; 1063 high = (nvmreg >> 12) & 0x0f; 1064 low = nvmreg & 0xff; 1065 aprint_normal(" NVM Image Version %u.%02x,", high, low); 1066 break; 1067 default: 1068 break; 1069 } 1070 hw->eeprom.nvm_image_ver_high = high; 1071 hw->eeprom.nvm_image_ver_low = low; 1072 1073 /* PHY firmware revision */ 1074 switch (hw->mac.type) { 1075 case ixgbe_mac_X540: 1076 case ixgbe_mac_X550: 1077 hw->eeprom.ops.read(hw, IXGBE_PHYFW_REV, &nvmreg); 1078 if (nvmreg == 0xffff) 1079 break; 1080 high = (nvmreg >> 12) & 0x0f; 1081 low = (nvmreg >> 4) & 0xff; 1082 id = nvmreg & 0x000f; 1083 aprint_normal(" PHY FW Revision %u.", high); 1084 if (hw->mac.type == ixgbe_mac_X540) 1085 str = "%x"; 1086 else 1087 str = "%02x"; 1088 aprint_normal(str, low); 1089 aprint_normal(" ID 0x%x,", id); 1090 break; 1091 default: 1092 break; 1093 } 1094 1095 /* NVM Map version & OEM NVM Image version */ 1096 switch (hw->mac.type) { 1097 case ixgbe_mac_X550: 1098 case ixgbe_mac_X550EM_x: 1099 case ixgbe_mac_X550EM_a: 1100 hw->eeprom.ops.read(hw, IXGBE_NVM_MAP_VER, &nvmreg); 1101 if (nvmreg != 0xffff) { 1102 high = (nvmreg >> 12) & 0x0f; 1103 low = nvmreg & 0x00ff; 1104 aprint_normal(" NVM Map version %u.%02x,", high, low); 1105 } 1106 hw->eeprom.ops.read(hw, IXGBE_OEM_NVM_IMAGE_VER, &nvmreg); 1107 if (nvmreg != 0xffff) { 1108 high = (nvmreg >> 12) & 0x0f; 1109 low = nvmreg & 0x00ff; 1110 aprint_verbose(" OEM NVM Image version %u.%02x,", high, 1111 low); 1112 } 1113 break; 1114 default: 1115 break; 1116 } 1117 1118 /* Print the ETrackID */ 1119 hw->eeprom.ops.read(hw, IXGBE_ETRACKID_H, &high); 1120 hw->eeprom.ops.read(hw, IXGBE_ETRACKID_L, &low); 1121 aprint_normal(" ETrackID %08x\n", ((uint32_t)high << 16) | low); 1122 1123 /* Printed Board Assembly number */ 1124 error = ixgbe_read_pba_string(hw, buf, IXGBE_PBANUM_LENGTH); 1125 aprint_normal_dev(dev, "PBA number %s\n", error ? "unknown" : buf); 1126 1127 if (adapter->feat_en & IXGBE_FEATURE_MSIX) { 1128 error = ixgbe_allocate_msix(adapter, pa); 1129 if (error) { 1130 /* Free allocated queue structures first */ 1131 ixgbe_free_queues(adapter); 1132 1133 /* Fallback to legacy interrupt */ 1134 if (adapter->feat_cap & IXGBE_FEATURE_MSI) 1135 adapter->feat_en |= IXGBE_FEATURE_MSI; 1136 adapter->num_queues = 1; 1137 1138 /* Allocate our TX/RX Queues again */ 1139 if (ixgbe_allocate_queues(adapter)) { 1140 error = ENOMEM; 1141 goto err_out; 1142 } 1143 } 1144 } 1145 1146 /* Recovery mode */ 1147 switch (adapter->hw.mac.type) { 1148 case ixgbe_mac_X550: 1149 case ixgbe_mac_X550EM_x: 1150 case ixgbe_mac_X550EM_a: 1151 /* >= 2.00 */ 1152 if (hw->eeprom.nvm_image_ver_high >= 2) { 1153 adapter->feat_cap |= IXGBE_FEATURE_RECOVERY_MODE; 1154 adapter->feat_en |= IXGBE_FEATURE_RECOVERY_MODE; 1155 } 1156 break; 1157 default: 1158 break; 1159 } 1160 1161 if ((adapter->feat_en & IXGBE_FEATURE_MSIX) == 0) 1162 error = ixgbe_allocate_legacy(adapter, pa); 1163 if (error) 1164 goto err_late; 1165 1166 /* Tasklets for Link, SFP, Multispeed Fiber and Flow Director */ 1167 mutex_init(&(adapter)->admin_mtx, MUTEX_DEFAULT, IPL_NET); 1168 snprintf(wqname, sizeof(wqname), "%s-admin", device_xname(dev)); 1169 error = workqueue_create(&adapter->admin_wq, wqname, 1170 ixgbe_handle_admin, adapter, IXGBE_WORKQUEUE_PRI, IPL_NET, 1171 IXGBE_TASKLET_WQ_FLAGS); 1172 if (error) { 1173 aprint_error_dev(dev, 1174 "could not create admin workqueue (%d)\n", error); 1175 goto err_out; 1176 } 1177 1178 error = ixgbe_start_hw(hw); 1179 switch (error) { 1180 case IXGBE_ERR_EEPROM_VERSION: 1181 aprint_error_dev(dev, 1182 "This device is a pre-production adapter/" 1183 "LOM. Please be aware there may be issues associated " 1184 "with your hardware.\nIf you are experiencing problems " 1185 "please contact your Intel or hardware representative " 1186 "who provided you with this hardware.\n"); 1187 break; 1188 default: 1189 break; 1190 } 1191 1192 /* Setup OS specific network interface */ 1193 if (ixgbe_setup_interface(dev, adapter) != 0) 1194 goto err_late; 1195 1196 /* 1197 * Print PHY ID only for copper PHY. On device which has SFP(+) cage 1198 * and a module is inserted, phy.id is not MII PHY id but SFF 8024 ID. 1199 */ 1200 if (hw->phy.media_type == ixgbe_media_type_copper) { 1201 uint16_t id1, id2; 1202 int oui, model, rev; 1203 char descr[MII_MAX_DESCR_LEN]; 1204 1205 id1 = hw->phy.id >> 16; 1206 id2 = hw->phy.id & 0xffff; 1207 oui = MII_OUI(id1, id2); 1208 model = MII_MODEL(id2); 1209 rev = MII_REV(id2); 1210 mii_get_descr(descr, sizeof(descr), oui, model); 1211 if (descr[0]) 1212 aprint_normal_dev(dev, "PHY: %s, rev. %d\n", 1213 descr, rev); 1214 else 1215 aprint_normal_dev(dev, 1216 "PHY OUI 0x%06x, model 0x%04x, rev. %d\n", 1217 oui, model, rev); 1218 } 1219 1220 /* Enable EEE power saving */ 1221 if (adapter->feat_cap & IXGBE_FEATURE_EEE) 1222 hw->mac.ops.setup_eee(hw, 1223 adapter->feat_en & IXGBE_FEATURE_EEE); 1224 1225 /* Enable power to the phy. */ 1226 if (!unsupported_sfp) { 1227 /* Enable the optics for 82599 SFP+ fiber */ 1228 ixgbe_enable_tx_laser(hw); 1229 1230 /* 1231 * XXX Currently, ixgbe_set_phy_power() supports only copper 1232 * PHY, so it's not required to test with !unsupported_sfp. 1233 */ 1234 ixgbe_set_phy_power(hw, TRUE); 1235 } 1236 1237 /* Initialize statistics */ 1238 ixgbe_update_stats_counters(adapter); 1239 1240 /* Check PCIE slot type/speed/width */ 1241 ixgbe_get_slot_info(adapter); 1242 1243 /* 1244 * Do time init and sysctl init here, but 1245 * only on the first port of a bypass adapter. 1246 */ 1247 ixgbe_bypass_init(adapter); 1248 1249 /* Set an initial dmac value */ 1250 adapter->dmac = 0; 1251 /* Set initial advertised speeds (if applicable) */ 1252 adapter->advertise = ixgbe_get_default_advertise(adapter); 1253 1254 if (adapter->feat_cap & IXGBE_FEATURE_SRIOV) 1255 ixgbe_define_iov_schemas(dev, &error); 1256 1257 /* Add sysctls */ 1258 ixgbe_add_device_sysctls(adapter); 1259 ixgbe_add_hw_stats(adapter); 1260 1261 /* For Netmap */ 1262 adapter->init_locked = ixgbe_init_locked; 1263 adapter->stop_locked = ixgbe_stop_locked; 1264 1265 if (adapter->feat_en & IXGBE_FEATURE_NETMAP) 1266 ixgbe_netmap_attach(adapter); 1267 1268 snprintb(buf, sizeof(buf), IXGBE_FEATURE_FLAGS, adapter->feat_cap); 1269 aprint_verbose_dev(dev, "feature cap %s\n", buf); 1270 snprintb(buf, sizeof(buf), IXGBE_FEATURE_FLAGS, adapter->feat_en); 1271 aprint_verbose_dev(dev, "feature ena %s\n", buf); 1272 1273 if (pmf_device_register(dev, ixgbe_suspend, ixgbe_resume)) 1274 pmf_class_network_register(dev, adapter->ifp); 1275 else 1276 aprint_error_dev(dev, "couldn't establish power handler\n"); 1277 1278 /* Init recovery mode timer and state variable */ 1279 if (adapter->feat_en & IXGBE_FEATURE_RECOVERY_MODE) { 1280 adapter->recovery_mode = 0; 1281 1282 /* Set up the timer callout */ 1283 callout_init(&adapter->recovery_mode_timer, 1284 IXGBE_CALLOUT_FLAGS); 1285 snprintf(wqname, sizeof(wqname), "%s-recovery", 1286 device_xname(dev)); 1287 error = workqueue_create(&adapter->recovery_mode_timer_wq, 1288 wqname, ixgbe_handle_recovery_mode_timer, adapter, 1289 IXGBE_WORKQUEUE_PRI, IPL_NET, IXGBE_TASKLET_WQ_FLAGS); 1290 if (error) { 1291 aprint_error_dev(dev, "could not create " 1292 "recovery_mode_timer workqueue (%d)\n", error); 1293 goto err_out; 1294 } 1295 1296 /* Start the task */ 1297 callout_reset(&adapter->recovery_mode_timer, hz, 1298 ixgbe_recovery_mode_timer, adapter); 1299 } 1300 1301 INIT_DEBUGOUT("ixgbe_attach: end"); 1302 adapter->osdep.attached = true; 1303 1304 return; 1305 1306 err_late: 1307 ixgbe_free_queues(adapter); 1308 err_out: 1309 ctrl_ext = IXGBE_READ_REG(&adapter->hw, IXGBE_CTRL_EXT); 1310 ctrl_ext &= ~IXGBE_CTRL_EXT_DRV_LOAD; 1311 IXGBE_WRITE_REG(&adapter->hw, IXGBE_CTRL_EXT, ctrl_ext); 1312 ixgbe_free_deferred_handlers(adapter); 1313 ixgbe_free_pci_resources(adapter); 1314 if (adapter->mta != NULL) 1315 free(adapter->mta, M_DEVBUF); 1316 mutex_destroy(&(adapter)->admin_mtx); /* XXX appropriate order? */ 1317 IXGBE_CORE_LOCK_DESTROY(adapter); 1318 1319 return; 1320 } /* ixgbe_attach */ 1321 1322 /************************************************************************ 1323 * ixgbe_check_wol_support 1324 * 1325 * Checks whether the adapter's ports are capable of 1326 * Wake On LAN by reading the adapter's NVM. 1327 * 1328 * Sets each port's hw->wol_enabled value depending 1329 * on the value read here. 1330 ************************************************************************/ 1331 static void 1332 ixgbe_check_wol_support(struct adapter *adapter) 1333 { 1334 struct ixgbe_hw *hw = &adapter->hw; 1335 u16 dev_caps = 0; 1336 1337 /* Find out WoL support for port */ 1338 adapter->wol_support = hw->wol_enabled = 0; 1339 ixgbe_get_device_caps(hw, &dev_caps); 1340 if ((dev_caps & IXGBE_DEVICE_CAPS_WOL_PORT0_1) || 1341 ((dev_caps & IXGBE_DEVICE_CAPS_WOL_PORT0) && 1342 hw->bus.func == 0)) 1343 adapter->wol_support = hw->wol_enabled = 1; 1344 1345 /* Save initial wake up filter configuration */ 1346 adapter->wufc = IXGBE_READ_REG(hw, IXGBE_WUFC); 1347 1348 return; 1349 } /* ixgbe_check_wol_support */ 1350 1351 /************************************************************************ 1352 * ixgbe_setup_interface 1353 * 1354 * Setup networking device structure and register an interface. 1355 ************************************************************************/ 1356 static int 1357 ixgbe_setup_interface(device_t dev, struct adapter *adapter) 1358 { 1359 struct ethercom *ec = &adapter->osdep.ec; 1360 struct ifnet *ifp; 1361 1362 INIT_DEBUGOUT("ixgbe_setup_interface: begin"); 1363 1364 ifp = adapter->ifp = &ec->ec_if; 1365 strlcpy(ifp->if_xname, device_xname(dev), IFNAMSIZ); 1366 ifp->if_baudrate = IF_Gbps(10); 1367 ifp->if_init = ixgbe_init; 1368 ifp->if_stop = ixgbe_ifstop; 1369 ifp->if_softc = adapter; 1370 ifp->if_flags = IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST; 1371 #ifdef IXGBE_MPSAFE 1372 ifp->if_extflags = IFEF_MPSAFE; 1373 #endif 1374 ifp->if_ioctl = ixgbe_ioctl; 1375 #if __FreeBSD_version >= 1100045 1376 /* TSO parameters */ 1377 ifp->if_hw_tsomax = 65518; 1378 ifp->if_hw_tsomaxsegcount = IXGBE_82599_SCATTER; 1379 ifp->if_hw_tsomaxsegsize = 2048; 1380 #endif 1381 if (adapter->feat_en & IXGBE_FEATURE_LEGACY_TX) { 1382 #if 0 1383 ixgbe_start_locked = ixgbe_legacy_start_locked; 1384 #endif 1385 } else { 1386 ifp->if_transmit = ixgbe_mq_start; 1387 #if 0 1388 ixgbe_start_locked = ixgbe_mq_start_locked; 1389 #endif 1390 } 1391 ifp->if_start = ixgbe_legacy_start; 1392 IFQ_SET_MAXLEN(&ifp->if_snd, adapter->num_tx_desc - 2); 1393 IFQ_SET_READY(&ifp->if_snd); 1394 1395 if_initialize(ifp); 1396 adapter->ipq = if_percpuq_create(&adapter->osdep.ec.ec_if); 1397 ether_ifattach(ifp, adapter->hw.mac.addr); 1398 aprint_normal_dev(dev, "Ethernet address %s\n", 1399 ether_sprintf(adapter->hw.mac.addr)); 1400 /* 1401 * We use per TX queue softint, so if_deferred_start_init() isn't 1402 * used. 1403 */ 1404 ether_set_ifflags_cb(ec, ixgbe_ifflags_cb); 1405 1406 adapter->max_frame_size = ifp->if_mtu + ETHER_HDR_LEN + ETHER_CRC_LEN; 1407 1408 /* 1409 * Tell the upper layer(s) we support long frames. 1410 */ 1411 ifp->if_hdrlen = sizeof(struct ether_vlan_header); 1412 1413 /* Set capability flags */ 1414 ifp->if_capabilities |= IFCAP_RXCSUM 1415 | IFCAP_TXCSUM 1416 | IFCAP_TSOv4 1417 | IFCAP_TSOv6; 1418 ifp->if_capenable = 0; 1419 1420 ec->ec_capabilities |= ETHERCAP_VLAN_HWTAGGING 1421 | ETHERCAP_VLAN_HWCSUM 1422 | ETHERCAP_JUMBO_MTU 1423 | ETHERCAP_VLAN_MTU; 1424 1425 /* Enable the above capabilities by default */ 1426 ec->ec_capenable = ec->ec_capabilities; 1427 1428 /* 1429 * Don't turn this on by default, if vlans are 1430 * created on another pseudo device (eg. lagg) 1431 * then vlan events are not passed thru, breaking 1432 * operation, but with HW FILTER off it works. If 1433 * using vlans directly on the ixgbe driver you can 1434 * enable this and get full hardware tag filtering. 1435 */ 1436 ec->ec_capabilities |= ETHERCAP_VLAN_HWFILTER; 1437 1438 /* 1439 * Specify the media types supported by this adapter and register 1440 * callbacks to update media and link information 1441 */ 1442 ec->ec_ifmedia = &adapter->media; 1443 ifmedia_init_with_lock(&adapter->media, IFM_IMASK, ixgbe_media_change, 1444 ixgbe_media_status, &adapter->core_mtx); 1445 1446 adapter->phy_layer = ixgbe_get_supported_physical_layer(&adapter->hw); 1447 ixgbe_add_media_types(adapter); 1448 1449 /* Set autoselect media by default */ 1450 ifmedia_set(&adapter->media, IFM_ETHER | IFM_AUTO); 1451 1452 if_register(ifp); 1453 1454 return (0); 1455 } /* ixgbe_setup_interface */ 1456 1457 /************************************************************************ 1458 * ixgbe_add_media_types 1459 ************************************************************************/ 1460 static void 1461 ixgbe_add_media_types(struct adapter *adapter) 1462 { 1463 struct ixgbe_hw *hw = &adapter->hw; 1464 u64 layer; 1465 1466 layer = adapter->phy_layer; 1467 1468 #define ADD(mm, dd) \ 1469 ifmedia_add(&adapter->media, IFM_ETHER | (mm), (dd), NULL); 1470 1471 ADD(IFM_NONE, 0); 1472 1473 /* Media types with matching NetBSD media defines */ 1474 if (layer & IXGBE_PHYSICAL_LAYER_10GBASE_T) 1475 ADD(IFM_10G_T | IFM_FDX, 0); 1476 if (layer & IXGBE_PHYSICAL_LAYER_1000BASE_T) 1477 ADD(IFM_1000_T | IFM_FDX, 0); 1478 if (layer & IXGBE_PHYSICAL_LAYER_100BASE_TX) 1479 ADD(IFM_100_TX | IFM_FDX, 0); 1480 if (layer & IXGBE_PHYSICAL_LAYER_10BASE_T) 1481 ADD(IFM_10_T | IFM_FDX, 0); 1482 1483 if (layer & IXGBE_PHYSICAL_LAYER_SFP_PLUS_CU || 1484 layer & IXGBE_PHYSICAL_LAYER_SFP_ACTIVE_DA) 1485 ADD(IFM_10G_TWINAX | IFM_FDX, 0); 1486 1487 if (layer & IXGBE_PHYSICAL_LAYER_10GBASE_LR) { 1488 ADD(IFM_10G_LR | IFM_FDX, 0); 1489 if (hw->phy.multispeed_fiber) 1490 ADD(IFM_1000_LX | IFM_FDX, 0); 1491 } 1492 if (layer & IXGBE_PHYSICAL_LAYER_10GBASE_SR) { 1493 ADD(IFM_10G_SR | IFM_FDX, 0); 1494 if (hw->phy.multispeed_fiber) 1495 ADD(IFM_1000_SX | IFM_FDX, 0); 1496 } else if (layer & IXGBE_PHYSICAL_LAYER_1000BASE_SX) 1497 ADD(IFM_1000_SX | IFM_FDX, 0); 1498 if (layer & IXGBE_PHYSICAL_LAYER_10GBASE_CX4) 1499 ADD(IFM_10G_CX4 | IFM_FDX, 0); 1500 1501 if (layer & IXGBE_PHYSICAL_LAYER_10GBASE_KR) 1502 ADD(IFM_10G_KR | IFM_FDX, 0); 1503 if (layer & IXGBE_PHYSICAL_LAYER_10GBASE_KX4) 1504 ADD(IFM_10G_KX4 | IFM_FDX, 0); 1505 if (layer & IXGBE_PHYSICAL_LAYER_1000BASE_KX) 1506 ADD(IFM_1000_KX | IFM_FDX, 0); 1507 if (layer & IXGBE_PHYSICAL_LAYER_2500BASE_KX) 1508 ADD(IFM_2500_KX | IFM_FDX, 0); 1509 if (layer & IXGBE_PHYSICAL_LAYER_2500BASE_T) 1510 ADD(IFM_2500_T | IFM_FDX, 0); 1511 if (layer & IXGBE_PHYSICAL_LAYER_5GBASE_T) 1512 ADD(IFM_5000_T | IFM_FDX, 0); 1513 if (layer & IXGBE_PHYSICAL_LAYER_1000BASE_BX) 1514 ADD(IFM_1000_BX10 | IFM_FDX, 0); 1515 /* XXX no ifmedia_set? */ 1516 1517 ADD(IFM_AUTO, 0); 1518 1519 #undef ADD 1520 } /* ixgbe_add_media_types */ 1521 1522 /************************************************************************ 1523 * ixgbe_is_sfp 1524 ************************************************************************/ 1525 static inline bool 1526 ixgbe_is_sfp(struct ixgbe_hw *hw) 1527 { 1528 switch (hw->mac.type) { 1529 case ixgbe_mac_82598EB: 1530 if (hw->phy.type == ixgbe_phy_nl) 1531 return (TRUE); 1532 return (FALSE); 1533 case ixgbe_mac_82599EB: 1534 case ixgbe_mac_X550EM_x: 1535 case ixgbe_mac_X550EM_a: 1536 switch (hw->mac.ops.get_media_type(hw)) { 1537 case ixgbe_media_type_fiber: 1538 case ixgbe_media_type_fiber_qsfp: 1539 return (TRUE); 1540 default: 1541 return (FALSE); 1542 } 1543 default: 1544 return (FALSE); 1545 } 1546 } /* ixgbe_is_sfp */ 1547 1548 static void 1549 ixgbe_schedule_admin_tasklet(struct adapter *adapter) 1550 { 1551 1552 KASSERT(mutex_owned(&adapter->admin_mtx)); 1553 1554 if (__predict_true(adapter->osdep.detaching == false)) { 1555 if (adapter->admin_pending == 0) 1556 workqueue_enqueue(adapter->admin_wq, 1557 &adapter->admin_wc, NULL); 1558 adapter->admin_pending = 1; 1559 } 1560 } 1561 1562 /************************************************************************ 1563 * ixgbe_config_link 1564 ************************************************************************/ 1565 static void 1566 ixgbe_config_link(struct adapter *adapter) 1567 { 1568 struct ixgbe_hw *hw = &adapter->hw; 1569 u32 autoneg, err = 0; 1570 u32 task_requests = 0; 1571 bool sfp, negotiate = false; 1572 1573 sfp = ixgbe_is_sfp(hw); 1574 1575 if (sfp) { 1576 if (hw->phy.multispeed_fiber) { 1577 ixgbe_enable_tx_laser(hw); 1578 task_requests |= IXGBE_REQUEST_TASK_MSF_WOI; 1579 } 1580 task_requests |= IXGBE_REQUEST_TASK_MOD_WOI; 1581 1582 mutex_enter(&adapter->admin_mtx); 1583 adapter->task_requests |= task_requests; 1584 ixgbe_schedule_admin_tasklet(adapter); 1585 mutex_exit(&adapter->admin_mtx); 1586 } else { 1587 struct ifmedia *ifm = &adapter->media; 1588 1589 if (hw->mac.ops.check_link) 1590 err = ixgbe_check_link(hw, &adapter->link_speed, 1591 &adapter->link_up, FALSE); 1592 if (err) 1593 return; 1594 1595 /* 1596 * Check if it's the first call. If it's the first call, 1597 * get value for auto negotiation. 1598 */ 1599 autoneg = hw->phy.autoneg_advertised; 1600 if ((IFM_SUBTYPE(ifm->ifm_cur->ifm_media) != IFM_NONE) 1601 && ((!autoneg) && (hw->mac.ops.get_link_capabilities))) 1602 err = hw->mac.ops.get_link_capabilities(hw, &autoneg, 1603 &negotiate); 1604 if (err) 1605 return; 1606 if (hw->mac.ops.setup_link) 1607 err = hw->mac.ops.setup_link(hw, autoneg, 1608 adapter->link_up); 1609 } 1610 } /* ixgbe_config_link */ 1611 1612 /************************************************************************ 1613 * ixgbe_update_stats_counters - Update board statistics counters. 1614 ************************************************************************/ 1615 static void 1616 ixgbe_update_stats_counters(struct adapter *adapter) 1617 { 1618 struct ifnet *ifp = adapter->ifp; 1619 struct ixgbe_hw *hw = &adapter->hw; 1620 struct ixgbe_hw_stats *stats = &adapter->stats.pf; 1621 u32 missed_rx = 0, bprc, lxontxc, lxofftxc; 1622 u64 total, total_missed_rx = 0; 1623 uint64_t crcerrs, illerrc, rlec, ruc, rfc, roc, rjc; 1624 unsigned int queue_counters; 1625 int i; 1626 1627 IXGBE_EVC_REGADD2(hw, stats, IXGBE_CRCERRS, crcerrs); 1628 IXGBE_EVC_REGADD2(hw, stats, IXGBE_ILLERRC, illerrc); 1629 1630 IXGBE_EVC_REGADD(hw, stats, IXGBE_ERRBC, errbc); 1631 IXGBE_EVC_REGADD(hw, stats, IXGBE_MSPDC, mspdc); 1632 if (hw->mac.type >= ixgbe_mac_X550) 1633 IXGBE_EVC_REGADD(hw, stats, IXGBE_MBSDC, mbsdc); 1634 1635 /* 16 registers exist */ 1636 queue_counters = uimin(__arraycount(stats->qprc), adapter->num_queues); 1637 for (i = 0; i < queue_counters; i++) { 1638 IXGBE_EVC_REGADD(hw, stats, IXGBE_QPRC(i), qprc[i]); 1639 IXGBE_EVC_REGADD(hw, stats, IXGBE_QPTC(i), qptc[i]); 1640 if (hw->mac.type >= ixgbe_mac_82599EB) 1641 IXGBE_EVC_REGADD(hw, stats, IXGBE_QPRDC(i), qprdc[i]); 1642 } 1643 1644 /* 8 registers exist */ 1645 for (i = 0; i < IXGBE_TC_COUNTER_NUM; i++) { 1646 uint32_t mp; 1647 1648 /* MPC */ 1649 mp = IXGBE_READ_REG(hw, IXGBE_MPC(i)); 1650 /* global total per queue */ 1651 IXGBE_EVC_ADD(&stats->mpc[i], mp); 1652 /* running comprehensive total for stats display */ 1653 total_missed_rx += mp; 1654 1655 if (hw->mac.type == ixgbe_mac_82598EB) 1656 IXGBE_EVC_REGADD(hw, stats, IXGBE_RNBC(i), rnbc[i]); 1657 1658 IXGBE_EVC_REGADD(hw, stats, IXGBE_PXONTXC(i), pxontxc[i]); 1659 IXGBE_EVC_REGADD(hw, stats, IXGBE_PXOFFTXC(i), pxofftxc[i]); 1660 if (hw->mac.type >= ixgbe_mac_82599EB) { 1661 IXGBE_EVC_REGADD(hw, stats, 1662 IXGBE_PXONRXCNT(i), pxonrxc[i]); 1663 IXGBE_EVC_REGADD(hw, stats, 1664 IXGBE_PXOFFRXCNT(i), pxoffrxc[i]); 1665 IXGBE_EVC_REGADD(hw, stats, 1666 IXGBE_PXON2OFFCNT(i), pxon2offc[i]); 1667 } else { 1668 IXGBE_EVC_REGADD(hw, stats, 1669 IXGBE_PXONRXC(i), pxonrxc[i]); 1670 IXGBE_EVC_REGADD(hw, stats, 1671 IXGBE_PXOFFRXC(i), pxoffrxc[i]); 1672 } 1673 } 1674 IXGBE_EVC_ADD(&stats->mpctotal, total_missed_rx); 1675 1676 /* Document says M[LR]FC are valid when link is up and 10Gbps */ 1677 if ((adapter->link_active == LINK_STATE_UP) 1678 && (adapter->link_speed == IXGBE_LINK_SPEED_10GB_FULL)) { 1679 IXGBE_EVC_REGADD(hw, stats, IXGBE_MLFC, mlfc); 1680 IXGBE_EVC_REGADD(hw, stats, IXGBE_MRFC, mrfc); 1681 } 1682 IXGBE_EVC_REGADD2(hw, stats, IXGBE_RLEC, rlec); 1683 1684 /* Hardware workaround, gprc counts missed packets */ 1685 IXGBE_EVC_ADD(&stats->gprc, 1686 IXGBE_READ_REG(hw, IXGBE_GPRC) - missed_rx); 1687 1688 IXGBE_EVC_REGADD2(hw, stats, IXGBE_LXONTXC, lxontxc); 1689 IXGBE_EVC_REGADD2(hw, stats, IXGBE_LXOFFTXC, lxofftxc); 1690 total = lxontxc + lxofftxc; 1691 1692 if (hw->mac.type != ixgbe_mac_82598EB) { 1693 IXGBE_EVC_ADD(&stats->gorc, IXGBE_READ_REG(hw, IXGBE_GORCL) + 1694 ((u64)IXGBE_READ_REG(hw, IXGBE_GORCH) << 32)); 1695 IXGBE_EVC_ADD(&stats->gotc, IXGBE_READ_REG(hw, IXGBE_GOTCL) + 1696 ((u64)IXGBE_READ_REG(hw, IXGBE_GOTCH) << 32) 1697 - total * ETHER_MIN_LEN); 1698 IXGBE_EVC_ADD(&stats->tor, IXGBE_READ_REG(hw, IXGBE_TORL) + 1699 ((u64)IXGBE_READ_REG(hw, IXGBE_TORH) << 32)); 1700 IXGBE_EVC_REGADD(hw, stats, IXGBE_LXONRXCNT, lxonrxc); 1701 IXGBE_EVC_REGADD(hw, stats, IXGBE_LXOFFRXCNT, lxoffrxc); 1702 } else { 1703 IXGBE_EVC_REGADD(hw, stats, IXGBE_LXONRXC, lxonrxc); 1704 IXGBE_EVC_REGADD(hw, stats, IXGBE_LXOFFRXC, lxoffrxc); 1705 /* 82598 only has a counter in the high register */ 1706 IXGBE_EVC_REGADD(hw, stats, IXGBE_GORCH, gorc); 1707 IXGBE_EVC_ADD(&stats->gotc, IXGBE_READ_REG(hw, IXGBE_GOTCH) 1708 - total * ETHER_MIN_LEN); 1709 IXGBE_EVC_REGADD(hw, stats, IXGBE_TORH, tor); 1710 } 1711 1712 /* 1713 * Workaround: mprc hardware is incorrectly counting 1714 * broadcasts, so for now we subtract those. 1715 */ 1716 IXGBE_EVC_REGADD2(hw, stats, IXGBE_BPRC, bprc); 1717 IXGBE_EVC_ADD(&stats->mprc, IXGBE_READ_REG(hw, IXGBE_MPRC) 1718 - ((hw->mac.type == ixgbe_mac_82598EB) ? bprc : 0)); 1719 1720 IXGBE_EVC_REGADD(hw, stats, IXGBE_PRC64, prc64); 1721 IXGBE_EVC_REGADD(hw, stats, IXGBE_PRC127, prc127); 1722 IXGBE_EVC_REGADD(hw, stats, IXGBE_PRC255, prc255); 1723 IXGBE_EVC_REGADD(hw, stats, IXGBE_PRC511, prc511); 1724 IXGBE_EVC_REGADD(hw, stats, IXGBE_PRC1023, prc1023); 1725 IXGBE_EVC_REGADD(hw, stats, IXGBE_PRC1522, prc1522); 1726 1727 IXGBE_EVC_ADD(&stats->gptc, IXGBE_READ_REG(hw, IXGBE_GPTC) - total); 1728 IXGBE_EVC_ADD(&stats->mptc, IXGBE_READ_REG(hw, IXGBE_MPTC) - total); 1729 IXGBE_EVC_ADD(&stats->ptc64, IXGBE_READ_REG(hw, IXGBE_PTC64) - total); 1730 1731 IXGBE_EVC_REGADD2(hw, stats, IXGBE_RUC, ruc); 1732 IXGBE_EVC_REGADD2(hw, stats, IXGBE_RFC, rfc); 1733 IXGBE_EVC_REGADD2(hw, stats, IXGBE_ROC, roc); 1734 IXGBE_EVC_REGADD2(hw, stats, IXGBE_RJC, rjc); 1735 1736 IXGBE_EVC_REGADD(hw, stats, IXGBE_MNGPRC, mngprc); 1737 IXGBE_EVC_REGADD(hw, stats, IXGBE_MNGPDC, mngpdc); 1738 IXGBE_EVC_REGADD(hw, stats, IXGBE_MNGPTC, mngptc); 1739 IXGBE_EVC_REGADD(hw, stats, IXGBE_TPR, tpr); 1740 IXGBE_EVC_REGADD(hw, stats, IXGBE_TPT, tpt); 1741 IXGBE_EVC_REGADD(hw, stats, IXGBE_PTC127, ptc127); 1742 IXGBE_EVC_REGADD(hw, stats, IXGBE_PTC255, ptc255); 1743 IXGBE_EVC_REGADD(hw, stats, IXGBE_PTC511, ptc511); 1744 IXGBE_EVC_REGADD(hw, stats, IXGBE_PTC1023, ptc1023); 1745 IXGBE_EVC_REGADD(hw, stats, IXGBE_PTC1522, ptc1522); 1746 IXGBE_EVC_REGADD(hw, stats, IXGBE_BPTC, bptc); 1747 IXGBE_EVC_REGADD(hw, stats, IXGBE_XEC, xec); 1748 IXGBE_EVC_REGADD(hw, stats, IXGBE_FCCRC, fccrc); 1749 IXGBE_EVC_REGADD(hw, stats, IXGBE_FCLAST, fclast); 1750 /* Only read FCOE on 82599 */ 1751 if (hw->mac.type != ixgbe_mac_82598EB) { 1752 IXGBE_EVC_REGADD(hw, stats, IXGBE_FCOERPDC, fcoerpdc); 1753 IXGBE_EVC_REGADD(hw, stats, IXGBE_FCOEPRC, fcoeprc); 1754 IXGBE_EVC_REGADD(hw, stats, IXGBE_FCOEPTC, fcoeptc); 1755 IXGBE_EVC_REGADD(hw, stats, IXGBE_FCOEDWRC, fcoedwrc); 1756 IXGBE_EVC_REGADD(hw, stats, IXGBE_FCOEDWTC, fcoedwtc); 1757 } 1758 1759 /* 1760 * Fill out the OS statistics structure. Only RX errors are required 1761 * here because all TX counters are incremented in the TX path and 1762 * normal RX counters are prepared in ether_input(). 1763 */ 1764 net_stat_ref_t nsr = IF_STAT_GETREF(ifp); 1765 if_statadd_ref(nsr, if_iqdrops, total_missed_rx); 1766 1767 /* 1768 * Aggregate following types of errors as RX errors: 1769 * - CRC error count, 1770 * - illegal byte error count, 1771 * - length error count, 1772 * - undersized packets count, 1773 * - fragmented packets count, 1774 * - oversized packets count, 1775 * - jabber count. 1776 */ 1777 if_statadd_ref(nsr, if_ierrors, 1778 crcerrs + illerrc + rlec + ruc + rfc + roc + rjc); 1779 1780 IF_STAT_PUTREF(ifp); 1781 } /* ixgbe_update_stats_counters */ 1782 1783 /************************************************************************ 1784 * ixgbe_add_hw_stats 1785 * 1786 * Add sysctl variables, one per statistic, to the system. 1787 ************************************************************************/ 1788 static void 1789 ixgbe_add_hw_stats(struct adapter *adapter) 1790 { 1791 device_t dev = adapter->dev; 1792 const struct sysctlnode *rnode, *cnode; 1793 struct sysctllog **log = &adapter->sysctllog; 1794 struct tx_ring *txr = adapter->tx_rings; 1795 struct rx_ring *rxr = adapter->rx_rings; 1796 struct ixgbe_hw *hw = &adapter->hw; 1797 struct ixgbe_hw_stats *stats = &adapter->stats.pf; 1798 const char *xname = device_xname(dev); 1799 int i; 1800 1801 /* Driver Statistics */ 1802 evcnt_attach_dynamic(&adapter->efbig_tx_dma_setup, EVCNT_TYPE_MISC, 1803 NULL, xname, "Driver tx dma soft fail EFBIG"); 1804 evcnt_attach_dynamic(&adapter->mbuf_defrag_failed, EVCNT_TYPE_MISC, 1805 NULL, xname, "m_defrag() failed"); 1806 evcnt_attach_dynamic(&adapter->efbig2_tx_dma_setup, EVCNT_TYPE_MISC, 1807 NULL, xname, "Driver tx dma hard fail EFBIG"); 1808 evcnt_attach_dynamic(&adapter->einval_tx_dma_setup, EVCNT_TYPE_MISC, 1809 NULL, xname, "Driver tx dma hard fail EINVAL"); 1810 evcnt_attach_dynamic(&adapter->other_tx_dma_setup, EVCNT_TYPE_MISC, 1811 NULL, xname, "Driver tx dma hard fail other"); 1812 evcnt_attach_dynamic(&adapter->eagain_tx_dma_setup, EVCNT_TYPE_MISC, 1813 NULL, xname, "Driver tx dma soft fail EAGAIN"); 1814 evcnt_attach_dynamic(&adapter->enomem_tx_dma_setup, EVCNT_TYPE_MISC, 1815 NULL, xname, "Driver tx dma soft fail ENOMEM"); 1816 evcnt_attach_dynamic(&adapter->watchdog_events, EVCNT_TYPE_MISC, 1817 NULL, xname, "Watchdog timeouts"); 1818 evcnt_attach_dynamic(&adapter->tso_err, EVCNT_TYPE_MISC, 1819 NULL, xname, "TSO errors"); 1820 evcnt_attach_dynamic(&adapter->admin_irqev, EVCNT_TYPE_INTR, 1821 NULL, xname, "Admin MSI-X IRQ Handled"); 1822 evcnt_attach_dynamic(&adapter->link_workev, EVCNT_TYPE_INTR, 1823 NULL, xname, "Link event"); 1824 evcnt_attach_dynamic(&adapter->mod_workev, EVCNT_TYPE_INTR, 1825 NULL, xname, "SFP+ module event"); 1826 evcnt_attach_dynamic(&adapter->msf_workev, EVCNT_TYPE_INTR, 1827 NULL, xname, "Multispeed event"); 1828 evcnt_attach_dynamic(&adapter->phy_workev, EVCNT_TYPE_INTR, 1829 NULL, xname, "External PHY event"); 1830 1831 /* Max number of traffic class is 8 */ 1832 KASSERT(IXGBE_DCB_MAX_TRAFFIC_CLASS == 8); 1833 for (i = 0; i < IXGBE_TC_COUNTER_NUM; i++) { 1834 snprintf(adapter->tcs[i].evnamebuf, 1835 sizeof(adapter->tcs[i].evnamebuf), "%s tc%d", xname, i); 1836 if (i < __arraycount(stats->mpc)) { 1837 evcnt_attach_dynamic(&stats->mpc[i], 1838 EVCNT_TYPE_MISC, NULL, adapter->tcs[i].evnamebuf, 1839 "RX Missed Packet Count"); 1840 if (hw->mac.type == ixgbe_mac_82598EB) 1841 evcnt_attach_dynamic(&stats->rnbc[i], 1842 EVCNT_TYPE_MISC, NULL, 1843 adapter->tcs[i].evnamebuf, 1844 "Receive No Buffers"); 1845 } 1846 if (i < __arraycount(stats->pxontxc)) { 1847 evcnt_attach_dynamic(&stats->pxontxc[i], 1848 EVCNT_TYPE_MISC, NULL, adapter->tcs[i].evnamebuf, 1849 "pxontxc"); 1850 evcnt_attach_dynamic(&stats->pxonrxc[i], 1851 EVCNT_TYPE_MISC, NULL, adapter->tcs[i].evnamebuf, 1852 "pxonrxc"); 1853 evcnt_attach_dynamic(&stats->pxofftxc[i], 1854 EVCNT_TYPE_MISC, NULL, adapter->tcs[i].evnamebuf, 1855 "pxofftxc"); 1856 evcnt_attach_dynamic(&stats->pxoffrxc[i], 1857 EVCNT_TYPE_MISC, NULL, adapter->tcs[i].evnamebuf, 1858 "pxoffrxc"); 1859 if (hw->mac.type >= ixgbe_mac_82599EB) 1860 evcnt_attach_dynamic(&stats->pxon2offc[i], 1861 EVCNT_TYPE_MISC, NULL, 1862 adapter->tcs[i].evnamebuf, 1863 "pxon2offc"); 1864 } 1865 } 1866 1867 for (i = 0; i < adapter->num_queues; i++, rxr++, txr++) { 1868 #ifdef LRO 1869 struct lro_ctrl *lro = &rxr->lro; 1870 #endif /* LRO */ 1871 1872 snprintf(adapter->queues[i].evnamebuf, 1873 sizeof(adapter->queues[i].evnamebuf), "%s q%d", xname, i); 1874 snprintf(adapter->queues[i].namebuf, 1875 sizeof(adapter->queues[i].namebuf), "q%d", i); 1876 1877 if ((rnode = ixgbe_sysctl_instance(adapter)) == NULL) { 1878 aprint_error_dev(dev, 1879 "could not create sysctl root\n"); 1880 break; 1881 } 1882 1883 if (sysctl_createv(log, 0, &rnode, &rnode, 1884 0, CTLTYPE_NODE, 1885 adapter->queues[i].namebuf, SYSCTL_DESCR("Queue Name"), 1886 NULL, 0, NULL, 0, CTL_CREATE, CTL_EOL) != 0) 1887 break; 1888 1889 if (sysctl_createv(log, 0, &rnode, &cnode, 1890 CTLFLAG_READWRITE, CTLTYPE_INT, 1891 "interrupt_rate", SYSCTL_DESCR("Interrupt Rate"), 1892 ixgbe_sysctl_interrupt_rate_handler, 0, 1893 (void *)&adapter->queues[i], 0, CTL_CREATE, CTL_EOL) != 0) 1894 break; 1895 1896 if (sysctl_createv(log, 0, &rnode, &cnode, 1897 CTLFLAG_READONLY, CTLTYPE_INT, 1898 "txd_head", SYSCTL_DESCR("Transmit Descriptor Head"), 1899 ixgbe_sysctl_tdh_handler, 0, (void *)txr, 1900 0, CTL_CREATE, CTL_EOL) != 0) 1901 break; 1902 1903 if (sysctl_createv(log, 0, &rnode, &cnode, 1904 CTLFLAG_READONLY, CTLTYPE_INT, 1905 "txd_tail", SYSCTL_DESCR("Transmit Descriptor Tail"), 1906 ixgbe_sysctl_tdt_handler, 0, (void *)txr, 1907 0, CTL_CREATE, CTL_EOL) != 0) 1908 break; 1909 1910 evcnt_attach_dynamic(&adapter->queues[i].irqs, EVCNT_TYPE_INTR, 1911 NULL, adapter->queues[i].evnamebuf, "IRQs on queue"); 1912 evcnt_attach_dynamic(&adapter->queues[i].handleq, 1913 EVCNT_TYPE_MISC, NULL, adapter->queues[i].evnamebuf, 1914 "Handled queue in softint"); 1915 evcnt_attach_dynamic(&adapter->queues[i].req, EVCNT_TYPE_MISC, 1916 NULL, adapter->queues[i].evnamebuf, "Requeued in softint"); 1917 evcnt_attach_dynamic(&txr->tso_tx, EVCNT_TYPE_MISC, 1918 NULL, adapter->queues[i].evnamebuf, "TSO"); 1919 evcnt_attach_dynamic(&txr->no_desc_avail, EVCNT_TYPE_MISC, 1920 NULL, adapter->queues[i].evnamebuf, 1921 "TX Queue No Descriptor Available"); 1922 evcnt_attach_dynamic(&txr->total_packets, EVCNT_TYPE_MISC, 1923 NULL, adapter->queues[i].evnamebuf, 1924 "Queue Packets Transmitted"); 1925 #ifndef IXGBE_LEGACY_TX 1926 evcnt_attach_dynamic(&txr->pcq_drops, EVCNT_TYPE_MISC, 1927 NULL, adapter->queues[i].evnamebuf, 1928 "Packets dropped in pcq"); 1929 #endif 1930 1931 if (sysctl_createv(log, 0, &rnode, &cnode, 1932 CTLFLAG_READONLY, CTLTYPE_INT, "rxd_nxck", 1933 SYSCTL_DESCR("Receive Descriptor next to check"), 1934 ixgbe_sysctl_next_to_check_handler, 0, (void *)rxr, 0, 1935 CTL_CREATE, CTL_EOL) != 0) 1936 break; 1937 1938 if (sysctl_createv(log, 0, &rnode, &cnode, 1939 CTLFLAG_READONLY, CTLTYPE_INT, "rxd_nxrf", 1940 SYSCTL_DESCR("Receive Descriptor next to refresh"), 1941 ixgbe_sysctl_next_to_refresh_handler, 0, (void *)rxr, 0, 1942 CTL_CREATE, CTL_EOL) != 0) 1943 break; 1944 1945 if (sysctl_createv(log, 0, &rnode, &cnode, 1946 CTLFLAG_READONLY, CTLTYPE_INT, "rxd_head", 1947 SYSCTL_DESCR("Receive Descriptor Head"), 1948 ixgbe_sysctl_rdh_handler, 0, (void *)rxr, 0, 1949 CTL_CREATE, CTL_EOL) != 0) 1950 break; 1951 1952 if (sysctl_createv(log, 0, &rnode, &cnode, 1953 CTLFLAG_READONLY, CTLTYPE_INT, "rxd_tail", 1954 SYSCTL_DESCR("Receive Descriptor Tail"), 1955 ixgbe_sysctl_rdt_handler, 0, (void *)rxr, 0, 1956 CTL_CREATE, CTL_EOL) != 0) 1957 break; 1958 1959 if (i < __arraycount(stats->qprc)) { 1960 evcnt_attach_dynamic(&stats->qprc[i], EVCNT_TYPE_MISC, 1961 NULL, adapter->queues[i].evnamebuf, "qprc"); 1962 evcnt_attach_dynamic(&stats->qptc[i], EVCNT_TYPE_MISC, 1963 NULL, adapter->queues[i].evnamebuf, "qptc"); 1964 evcnt_attach_dynamic(&stats->qbrc[i], EVCNT_TYPE_MISC, 1965 NULL, adapter->queues[i].evnamebuf, "qbrc"); 1966 evcnt_attach_dynamic(&stats->qbtc[i], EVCNT_TYPE_MISC, 1967 NULL, adapter->queues[i].evnamebuf, "qbtc"); 1968 if (hw->mac.type >= ixgbe_mac_82599EB) 1969 evcnt_attach_dynamic(&stats->qprdc[i], 1970 EVCNT_TYPE_MISC, NULL, 1971 adapter->queues[i].evnamebuf, "qprdc"); 1972 } 1973 1974 evcnt_attach_dynamic(&rxr->rx_packets, EVCNT_TYPE_MISC, 1975 NULL, adapter->queues[i].evnamebuf, 1976 "Queue Packets Received"); 1977 evcnt_attach_dynamic(&rxr->rx_bytes, EVCNT_TYPE_MISC, 1978 NULL, adapter->queues[i].evnamebuf, 1979 "Queue Bytes Received"); 1980 evcnt_attach_dynamic(&rxr->rx_copies, EVCNT_TYPE_MISC, 1981 NULL, adapter->queues[i].evnamebuf, "Copied RX Frames"); 1982 evcnt_attach_dynamic(&rxr->no_mbuf, EVCNT_TYPE_MISC, 1983 NULL, adapter->queues[i].evnamebuf, "Rx no mbuf"); 1984 evcnt_attach_dynamic(&rxr->rx_discarded, EVCNT_TYPE_MISC, 1985 NULL, adapter->queues[i].evnamebuf, "Rx discarded"); 1986 #ifdef LRO 1987 SYSCTL_ADD_INT(ctx, queue_list, OID_AUTO, "lro_queued", 1988 CTLFLAG_RD, &lro->lro_queued, 0, 1989 "LRO Queued"); 1990 SYSCTL_ADD_INT(ctx, queue_list, OID_AUTO, "lro_flushed", 1991 CTLFLAG_RD, &lro->lro_flushed, 0, 1992 "LRO Flushed"); 1993 #endif /* LRO */ 1994 } 1995 1996 /* MAC stats get their own sub node */ 1997 1998 snprintf(stats->namebuf, 1999 sizeof(stats->namebuf), "%s MAC Statistics", xname); 2000 2001 evcnt_attach_dynamic(&stats->ipcs, EVCNT_TYPE_MISC, NULL, 2002 stats->namebuf, "rx csum offload - IP"); 2003 evcnt_attach_dynamic(&stats->l4cs, EVCNT_TYPE_MISC, NULL, 2004 stats->namebuf, "rx csum offload - L4"); 2005 evcnt_attach_dynamic(&stats->ipcs_bad, EVCNT_TYPE_MISC, NULL, 2006 stats->namebuf, "rx csum offload - IP bad"); 2007 evcnt_attach_dynamic(&stats->l4cs_bad, EVCNT_TYPE_MISC, NULL, 2008 stats->namebuf, "rx csum offload - L4 bad"); 2009 evcnt_attach_dynamic(&stats->intzero, EVCNT_TYPE_MISC, NULL, 2010 stats->namebuf, "Interrupt conditions zero"); 2011 evcnt_attach_dynamic(&stats->legint, EVCNT_TYPE_MISC, NULL, 2012 stats->namebuf, "Legacy interrupts"); 2013 2014 evcnt_attach_dynamic(&stats->crcerrs, EVCNT_TYPE_MISC, NULL, 2015 stats->namebuf, "CRC Errors"); 2016 evcnt_attach_dynamic(&stats->illerrc, EVCNT_TYPE_MISC, NULL, 2017 stats->namebuf, "Illegal Byte Errors"); 2018 evcnt_attach_dynamic(&stats->errbc, EVCNT_TYPE_MISC, NULL, 2019 stats->namebuf, "Byte Errors"); 2020 evcnt_attach_dynamic(&stats->mspdc, EVCNT_TYPE_MISC, NULL, 2021 stats->namebuf, "MAC Short Packets Discarded"); 2022 if (hw->mac.type >= ixgbe_mac_X550) 2023 evcnt_attach_dynamic(&stats->mbsdc, EVCNT_TYPE_MISC, NULL, 2024 stats->namebuf, "Bad SFD"); 2025 evcnt_attach_dynamic(&stats->mpctotal, EVCNT_TYPE_MISC, NULL, 2026 stats->namebuf, "Total Packets Missed"); 2027 evcnt_attach_dynamic(&stats->mlfc, EVCNT_TYPE_MISC, NULL, 2028 stats->namebuf, "MAC Local Faults"); 2029 evcnt_attach_dynamic(&stats->mrfc, EVCNT_TYPE_MISC, NULL, 2030 stats->namebuf, "MAC Remote Faults"); 2031 evcnt_attach_dynamic(&stats->rlec, EVCNT_TYPE_MISC, NULL, 2032 stats->namebuf, "Receive Length Errors"); 2033 evcnt_attach_dynamic(&stats->lxontxc, EVCNT_TYPE_MISC, NULL, 2034 stats->namebuf, "Link XON Transmitted"); 2035 evcnt_attach_dynamic(&stats->lxonrxc, EVCNT_TYPE_MISC, NULL, 2036 stats->namebuf, "Link XON Received"); 2037 evcnt_attach_dynamic(&stats->lxofftxc, EVCNT_TYPE_MISC, NULL, 2038 stats->namebuf, "Link XOFF Transmitted"); 2039 evcnt_attach_dynamic(&stats->lxoffrxc, EVCNT_TYPE_MISC, NULL, 2040 stats->namebuf, "Link XOFF Received"); 2041 2042 /* Packet Reception Stats */ 2043 evcnt_attach_dynamic(&stats->tor, EVCNT_TYPE_MISC, NULL, 2044 stats->namebuf, "Total Octets Received"); 2045 evcnt_attach_dynamic(&stats->gorc, EVCNT_TYPE_MISC, NULL, 2046 stats->namebuf, "Good Octets Received"); 2047 evcnt_attach_dynamic(&stats->tpr, EVCNT_TYPE_MISC, NULL, 2048 stats->namebuf, "Total Packets Received"); 2049 evcnt_attach_dynamic(&stats->gprc, EVCNT_TYPE_MISC, NULL, 2050 stats->namebuf, "Good Packets Received"); 2051 evcnt_attach_dynamic(&stats->mprc, EVCNT_TYPE_MISC, NULL, 2052 stats->namebuf, "Multicast Packets Received"); 2053 evcnt_attach_dynamic(&stats->bprc, EVCNT_TYPE_MISC, NULL, 2054 stats->namebuf, "Broadcast Packets Received"); 2055 evcnt_attach_dynamic(&stats->prc64, EVCNT_TYPE_MISC, NULL, 2056 stats->namebuf, "64 byte frames received "); 2057 evcnt_attach_dynamic(&stats->prc127, EVCNT_TYPE_MISC, NULL, 2058 stats->namebuf, "65-127 byte frames received"); 2059 evcnt_attach_dynamic(&stats->prc255, EVCNT_TYPE_MISC, NULL, 2060 stats->namebuf, "128-255 byte frames received"); 2061 evcnt_attach_dynamic(&stats->prc511, EVCNT_TYPE_MISC, NULL, 2062 stats->namebuf, "256-511 byte frames received"); 2063 evcnt_attach_dynamic(&stats->prc1023, EVCNT_TYPE_MISC, NULL, 2064 stats->namebuf, "512-1023 byte frames received"); 2065 evcnt_attach_dynamic(&stats->prc1522, EVCNT_TYPE_MISC, NULL, 2066 stats->namebuf, "1023-1522 byte frames received"); 2067 evcnt_attach_dynamic(&stats->ruc, EVCNT_TYPE_MISC, NULL, 2068 stats->namebuf, "Receive Undersized"); 2069 evcnt_attach_dynamic(&stats->rfc, EVCNT_TYPE_MISC, NULL, 2070 stats->namebuf, "Fragmented Packets Received "); 2071 evcnt_attach_dynamic(&stats->roc, EVCNT_TYPE_MISC, NULL, 2072 stats->namebuf, "Oversized Packets Received"); 2073 evcnt_attach_dynamic(&stats->rjc, EVCNT_TYPE_MISC, NULL, 2074 stats->namebuf, "Received Jabber"); 2075 evcnt_attach_dynamic(&stats->mngprc, EVCNT_TYPE_MISC, NULL, 2076 stats->namebuf, "Management Packets Received"); 2077 evcnt_attach_dynamic(&stats->mngpdc, EVCNT_TYPE_MISC, NULL, 2078 stats->namebuf, "Management Packets Dropped"); 2079 evcnt_attach_dynamic(&stats->xec, EVCNT_TYPE_MISC, NULL, 2080 stats->namebuf, "Checksum Errors"); 2081 2082 /* Packet Transmission Stats */ 2083 evcnt_attach_dynamic(&stats->gotc, EVCNT_TYPE_MISC, NULL, 2084 stats->namebuf, "Good Octets Transmitted"); 2085 evcnt_attach_dynamic(&stats->tpt, EVCNT_TYPE_MISC, NULL, 2086 stats->namebuf, "Total Packets Transmitted"); 2087 evcnt_attach_dynamic(&stats->gptc, EVCNT_TYPE_MISC, NULL, 2088 stats->namebuf, "Good Packets Transmitted"); 2089 evcnt_attach_dynamic(&stats->bptc, EVCNT_TYPE_MISC, NULL, 2090 stats->namebuf, "Broadcast Packets Transmitted"); 2091 evcnt_attach_dynamic(&stats->mptc, EVCNT_TYPE_MISC, NULL, 2092 stats->namebuf, "Multicast Packets Transmitted"); 2093 evcnt_attach_dynamic(&stats->mngptc, EVCNT_TYPE_MISC, NULL, 2094 stats->namebuf, "Management Packets Transmitted"); 2095 evcnt_attach_dynamic(&stats->ptc64, EVCNT_TYPE_MISC, NULL, 2096 stats->namebuf, "64 byte frames transmitted "); 2097 evcnt_attach_dynamic(&stats->ptc127, EVCNT_TYPE_MISC, NULL, 2098 stats->namebuf, "65-127 byte frames transmitted"); 2099 evcnt_attach_dynamic(&stats->ptc255, EVCNT_TYPE_MISC, NULL, 2100 stats->namebuf, "128-255 byte frames transmitted"); 2101 evcnt_attach_dynamic(&stats->ptc511, EVCNT_TYPE_MISC, NULL, 2102 stats->namebuf, "256-511 byte frames transmitted"); 2103 evcnt_attach_dynamic(&stats->ptc1023, EVCNT_TYPE_MISC, NULL, 2104 stats->namebuf, "512-1023 byte frames transmitted"); 2105 evcnt_attach_dynamic(&stats->ptc1522, EVCNT_TYPE_MISC, NULL, 2106 stats->namebuf, "1024-1522 byte frames transmitted"); 2107 } /* ixgbe_add_hw_stats */ 2108 2109 static void 2110 ixgbe_clear_evcnt(struct adapter *adapter) 2111 { 2112 struct tx_ring *txr = adapter->tx_rings; 2113 struct rx_ring *rxr = adapter->rx_rings; 2114 struct ixgbe_hw *hw = &adapter->hw; 2115 struct ixgbe_hw_stats *stats = &adapter->stats.pf; 2116 int i; 2117 2118 IXGBE_EVC_STORE(&adapter->efbig_tx_dma_setup, 0); 2119 IXGBE_EVC_STORE(&adapter->mbuf_defrag_failed, 0); 2120 IXGBE_EVC_STORE(&adapter->efbig2_tx_dma_setup, 0); 2121 IXGBE_EVC_STORE(&adapter->einval_tx_dma_setup, 0); 2122 IXGBE_EVC_STORE(&adapter->other_tx_dma_setup, 0); 2123 IXGBE_EVC_STORE(&adapter->eagain_tx_dma_setup, 0); 2124 IXGBE_EVC_STORE(&adapter->enomem_tx_dma_setup, 0); 2125 IXGBE_EVC_STORE(&adapter->tso_err, 0); 2126 IXGBE_EVC_STORE(&adapter->watchdog_events, 0); 2127 IXGBE_EVC_STORE(&adapter->admin_irqev, 0); 2128 IXGBE_EVC_STORE(&adapter->link_workev, 0); 2129 IXGBE_EVC_STORE(&adapter->mod_workev, 0); 2130 IXGBE_EVC_STORE(&adapter->msf_workev, 0); 2131 IXGBE_EVC_STORE(&adapter->phy_workev, 0); 2132 2133 for (i = 0; i < IXGBE_TC_COUNTER_NUM; i++) { 2134 if (i < __arraycount(stats->mpc)) { 2135 IXGBE_EVC_STORE(&stats->mpc[i], 0); 2136 if (hw->mac.type == ixgbe_mac_82598EB) 2137 IXGBE_EVC_STORE(&stats->rnbc[i], 0); 2138 } 2139 if (i < __arraycount(stats->pxontxc)) { 2140 IXGBE_EVC_STORE(&stats->pxontxc[i], 0); 2141 IXGBE_EVC_STORE(&stats->pxonrxc[i], 0); 2142 IXGBE_EVC_STORE(&stats->pxofftxc[i], 0); 2143 IXGBE_EVC_STORE(&stats->pxoffrxc[i], 0); 2144 if (hw->mac.type >= ixgbe_mac_82599EB) 2145 IXGBE_EVC_STORE(&stats->pxon2offc[i], 0); 2146 } 2147 } 2148 2149 txr = adapter->tx_rings; 2150 for (i = 0; i < adapter->num_queues; i++, rxr++, txr++) { 2151 IXGBE_EVC_STORE(&adapter->queues[i].irqs, 0); 2152 IXGBE_EVC_STORE(&adapter->queues[i].handleq, 0); 2153 IXGBE_EVC_STORE(&adapter->queues[i].req, 0); 2154 IXGBE_EVC_STORE(&txr->no_desc_avail, 0); 2155 IXGBE_EVC_STORE(&txr->total_packets, 0); 2156 IXGBE_EVC_STORE(&txr->tso_tx, 0); 2157 #ifndef IXGBE_LEGACY_TX 2158 IXGBE_EVC_STORE(&txr->pcq_drops, 0); 2159 #endif 2160 txr->q_efbig_tx_dma_setup = 0; 2161 txr->q_mbuf_defrag_failed = 0; 2162 txr->q_efbig2_tx_dma_setup = 0; 2163 txr->q_einval_tx_dma_setup = 0; 2164 txr->q_other_tx_dma_setup = 0; 2165 txr->q_eagain_tx_dma_setup = 0; 2166 txr->q_enomem_tx_dma_setup = 0; 2167 txr->q_tso_err = 0; 2168 2169 if (i < __arraycount(stats->qprc)) { 2170 IXGBE_EVC_STORE(&stats->qprc[i], 0); 2171 IXGBE_EVC_STORE(&stats->qptc[i], 0); 2172 IXGBE_EVC_STORE(&stats->qbrc[i], 0); 2173 IXGBE_EVC_STORE(&stats->qbtc[i], 0); 2174 if (hw->mac.type >= ixgbe_mac_82599EB) 2175 IXGBE_EVC_STORE(&stats->qprdc[i], 0); 2176 } 2177 2178 IXGBE_EVC_STORE(&rxr->rx_packets, 0); 2179 IXGBE_EVC_STORE(&rxr->rx_bytes, 0); 2180 IXGBE_EVC_STORE(&rxr->rx_copies, 0); 2181 IXGBE_EVC_STORE(&rxr->no_mbuf, 0); 2182 IXGBE_EVC_STORE(&rxr->rx_discarded, 0); 2183 } 2184 IXGBE_EVC_STORE(&stats->ipcs, 0); 2185 IXGBE_EVC_STORE(&stats->l4cs, 0); 2186 IXGBE_EVC_STORE(&stats->ipcs_bad, 0); 2187 IXGBE_EVC_STORE(&stats->l4cs_bad, 0); 2188 IXGBE_EVC_STORE(&stats->intzero, 0); 2189 IXGBE_EVC_STORE(&stats->legint, 0); 2190 IXGBE_EVC_STORE(&stats->crcerrs, 0); 2191 IXGBE_EVC_STORE(&stats->illerrc, 0); 2192 IXGBE_EVC_STORE(&stats->errbc, 0); 2193 IXGBE_EVC_STORE(&stats->mspdc, 0); 2194 if (hw->mac.type >= ixgbe_mac_X550) 2195 IXGBE_EVC_STORE(&stats->mbsdc, 0); 2196 IXGBE_EVC_STORE(&stats->mpctotal, 0); 2197 IXGBE_EVC_STORE(&stats->mlfc, 0); 2198 IXGBE_EVC_STORE(&stats->mrfc, 0); 2199 IXGBE_EVC_STORE(&stats->rlec, 0); 2200 IXGBE_EVC_STORE(&stats->lxontxc, 0); 2201 IXGBE_EVC_STORE(&stats->lxonrxc, 0); 2202 IXGBE_EVC_STORE(&stats->lxofftxc, 0); 2203 IXGBE_EVC_STORE(&stats->lxoffrxc, 0); 2204 2205 /* Packet Reception Stats */ 2206 IXGBE_EVC_STORE(&stats->tor, 0); 2207 IXGBE_EVC_STORE(&stats->gorc, 0); 2208 IXGBE_EVC_STORE(&stats->tpr, 0); 2209 IXGBE_EVC_STORE(&stats->gprc, 0); 2210 IXGBE_EVC_STORE(&stats->mprc, 0); 2211 IXGBE_EVC_STORE(&stats->bprc, 0); 2212 IXGBE_EVC_STORE(&stats->prc64, 0); 2213 IXGBE_EVC_STORE(&stats->prc127, 0); 2214 IXGBE_EVC_STORE(&stats->prc255, 0); 2215 IXGBE_EVC_STORE(&stats->prc511, 0); 2216 IXGBE_EVC_STORE(&stats->prc1023, 0); 2217 IXGBE_EVC_STORE(&stats->prc1522, 0); 2218 IXGBE_EVC_STORE(&stats->ruc, 0); 2219 IXGBE_EVC_STORE(&stats->rfc, 0); 2220 IXGBE_EVC_STORE(&stats->roc, 0); 2221 IXGBE_EVC_STORE(&stats->rjc, 0); 2222 IXGBE_EVC_STORE(&stats->mngprc, 0); 2223 IXGBE_EVC_STORE(&stats->mngpdc, 0); 2224 IXGBE_EVC_STORE(&stats->xec, 0); 2225 2226 /* Packet Transmission Stats */ 2227 IXGBE_EVC_STORE(&stats->gotc, 0); 2228 IXGBE_EVC_STORE(&stats->tpt, 0); 2229 IXGBE_EVC_STORE(&stats->gptc, 0); 2230 IXGBE_EVC_STORE(&stats->bptc, 0); 2231 IXGBE_EVC_STORE(&stats->mptc, 0); 2232 IXGBE_EVC_STORE(&stats->mngptc, 0); 2233 IXGBE_EVC_STORE(&stats->ptc64, 0); 2234 IXGBE_EVC_STORE(&stats->ptc127, 0); 2235 IXGBE_EVC_STORE(&stats->ptc255, 0); 2236 IXGBE_EVC_STORE(&stats->ptc511, 0); 2237 IXGBE_EVC_STORE(&stats->ptc1023, 0); 2238 IXGBE_EVC_STORE(&stats->ptc1522, 0); 2239 } 2240 2241 /************************************************************************ 2242 * ixgbe_sysctl_tdh_handler - Transmit Descriptor Head handler function 2243 * 2244 * Retrieves the TDH value from the hardware 2245 ************************************************************************/ 2246 static int 2247 ixgbe_sysctl_tdh_handler(SYSCTLFN_ARGS) 2248 { 2249 struct sysctlnode node = *rnode; 2250 struct tx_ring *txr = (struct tx_ring *)node.sysctl_data; 2251 struct adapter *adapter; 2252 uint32_t val; 2253 2254 if (!txr) 2255 return (0); 2256 2257 adapter = txr->adapter; 2258 if (ixgbe_fw_recovery_mode_swflag(adapter)) 2259 return (EPERM); 2260 2261 val = IXGBE_READ_REG(&adapter->hw, IXGBE_TDH(txr->me)); 2262 node.sysctl_data = &val; 2263 return sysctl_lookup(SYSCTLFN_CALL(&node)); 2264 } /* ixgbe_sysctl_tdh_handler */ 2265 2266 /************************************************************************ 2267 * ixgbe_sysctl_tdt_handler - Transmit Descriptor Tail handler function 2268 * 2269 * Retrieves the TDT value from the hardware 2270 ************************************************************************/ 2271 static int 2272 ixgbe_sysctl_tdt_handler(SYSCTLFN_ARGS) 2273 { 2274 struct sysctlnode node = *rnode; 2275 struct tx_ring *txr = (struct tx_ring *)node.sysctl_data; 2276 struct adapter *adapter; 2277 uint32_t val; 2278 2279 if (!txr) 2280 return (0); 2281 2282 adapter = txr->adapter; 2283 if (ixgbe_fw_recovery_mode_swflag(adapter)) 2284 return (EPERM); 2285 2286 val = IXGBE_READ_REG(&adapter->hw, IXGBE_TDT(txr->me)); 2287 node.sysctl_data = &val; 2288 return sysctl_lookup(SYSCTLFN_CALL(&node)); 2289 } /* ixgbe_sysctl_tdt_handler */ 2290 2291 /************************************************************************ 2292 * ixgbe_sysctl_next_to_check_handler - Receive Descriptor next to check 2293 * handler function 2294 * 2295 * Retrieves the next_to_check value 2296 ************************************************************************/ 2297 static int 2298 ixgbe_sysctl_next_to_check_handler(SYSCTLFN_ARGS) 2299 { 2300 struct sysctlnode node = *rnode; 2301 struct rx_ring *rxr = (struct rx_ring *)node.sysctl_data; 2302 struct adapter *adapter; 2303 uint32_t val; 2304 2305 if (!rxr) 2306 return (0); 2307 2308 adapter = rxr->adapter; 2309 if (ixgbe_fw_recovery_mode_swflag(adapter)) 2310 return (EPERM); 2311 2312 val = rxr->next_to_check; 2313 node.sysctl_data = &val; 2314 return sysctl_lookup(SYSCTLFN_CALL(&node)); 2315 } /* ixgbe_sysctl_next_to_check_handler */ 2316 2317 /************************************************************************ 2318 * ixgbe_sysctl_next_to_refresh_handler - Receive Descriptor next to check 2319 * handler function 2320 * 2321 * Retrieves the next_to_refresh value 2322 ************************************************************************/ 2323 static int 2324 ixgbe_sysctl_next_to_refresh_handler(SYSCTLFN_ARGS) 2325 { 2326 struct sysctlnode node = *rnode; 2327 struct rx_ring *rxr = (struct rx_ring *)node.sysctl_data; 2328 struct adapter *adapter; 2329 uint32_t val; 2330 2331 if (!rxr) 2332 return (0); 2333 2334 adapter = rxr->adapter; 2335 if (ixgbe_fw_recovery_mode_swflag(adapter)) 2336 return (EPERM); 2337 2338 val = rxr->next_to_refresh; 2339 node.sysctl_data = &val; 2340 return sysctl_lookup(SYSCTLFN_CALL(&node)); 2341 } /* ixgbe_sysctl_next_to_refresh_handler */ 2342 2343 /************************************************************************ 2344 * ixgbe_sysctl_rdh_handler - Receive Descriptor Head handler function 2345 * 2346 * Retrieves the RDH value from the hardware 2347 ************************************************************************/ 2348 static int 2349 ixgbe_sysctl_rdh_handler(SYSCTLFN_ARGS) 2350 { 2351 struct sysctlnode node = *rnode; 2352 struct rx_ring *rxr = (struct rx_ring *)node.sysctl_data; 2353 struct adapter *adapter; 2354 uint32_t val; 2355 2356 if (!rxr) 2357 return (0); 2358 2359 adapter = rxr->adapter; 2360 if (ixgbe_fw_recovery_mode_swflag(adapter)) 2361 return (EPERM); 2362 2363 val = IXGBE_READ_REG(&adapter->hw, IXGBE_RDH(rxr->me)); 2364 node.sysctl_data = &val; 2365 return sysctl_lookup(SYSCTLFN_CALL(&node)); 2366 } /* ixgbe_sysctl_rdh_handler */ 2367 2368 /************************************************************************ 2369 * ixgbe_sysctl_rdt_handler - Receive Descriptor Tail handler function 2370 * 2371 * Retrieves the RDT value from the hardware 2372 ************************************************************************/ 2373 static int 2374 ixgbe_sysctl_rdt_handler(SYSCTLFN_ARGS) 2375 { 2376 struct sysctlnode node = *rnode; 2377 struct rx_ring *rxr = (struct rx_ring *)node.sysctl_data; 2378 struct adapter *adapter; 2379 uint32_t val; 2380 2381 if (!rxr) 2382 return (0); 2383 2384 adapter = rxr->adapter; 2385 if (ixgbe_fw_recovery_mode_swflag(adapter)) 2386 return (EPERM); 2387 2388 val = IXGBE_READ_REG(&adapter->hw, IXGBE_RDT(rxr->me)); 2389 node.sysctl_data = &val; 2390 return sysctl_lookup(SYSCTLFN_CALL(&node)); 2391 } /* ixgbe_sysctl_rdt_handler */ 2392 2393 static int 2394 ixgbe_vlan_cb(struct ethercom *ec, uint16_t vid, bool set) 2395 { 2396 struct ifnet *ifp = &ec->ec_if; 2397 struct adapter *adapter = ifp->if_softc; 2398 int rv; 2399 2400 if (set) 2401 rv = ixgbe_register_vlan(adapter, vid); 2402 else 2403 rv = ixgbe_unregister_vlan(adapter, vid); 2404 2405 if (rv != 0) 2406 return rv; 2407 2408 /* 2409 * Control VLAN HW tagging when ec_nvlan is changed from 1 to 0 2410 * or 0 to 1. 2411 */ 2412 if ((set && (ec->ec_nvlans == 1)) || (!set && (ec->ec_nvlans == 0))) 2413 ixgbe_setup_vlan_hw_tagging(adapter); 2414 2415 return rv; 2416 } 2417 2418 /************************************************************************ 2419 * ixgbe_register_vlan 2420 * 2421 * Run via vlan config EVENT, it enables us to use the 2422 * HW Filter table since we can get the vlan id. This 2423 * just creates the entry in the soft version of the 2424 * VFTA, init will repopulate the real table. 2425 ************************************************************************/ 2426 static int 2427 ixgbe_register_vlan(struct adapter *adapter, u16 vtag) 2428 { 2429 u16 index, bit; 2430 int error; 2431 2432 if ((vtag == 0) || (vtag > 4095)) /* Invalid */ 2433 return EINVAL; 2434 2435 IXGBE_CORE_LOCK(adapter); 2436 index = (vtag >> 5) & 0x7F; 2437 bit = vtag & 0x1F; 2438 adapter->shadow_vfta[index] |= ((u32)1 << bit); 2439 error = adapter->hw.mac.ops.set_vfta(&adapter->hw, vtag, 0, true, 2440 true); 2441 IXGBE_CORE_UNLOCK(adapter); 2442 if (error != 0) 2443 error = EACCES; 2444 2445 return error; 2446 } /* ixgbe_register_vlan */ 2447 2448 /************************************************************************ 2449 * ixgbe_unregister_vlan 2450 * 2451 * Run via vlan unconfig EVENT, remove our entry in the soft vfta. 2452 ************************************************************************/ 2453 static int 2454 ixgbe_unregister_vlan(struct adapter *adapter, u16 vtag) 2455 { 2456 u16 index, bit; 2457 int error; 2458 2459 if ((vtag == 0) || (vtag > 4095)) /* Invalid */ 2460 return EINVAL; 2461 2462 IXGBE_CORE_LOCK(adapter); 2463 index = (vtag >> 5) & 0x7F; 2464 bit = vtag & 0x1F; 2465 adapter->shadow_vfta[index] &= ~((u32)1 << bit); 2466 error = adapter->hw.mac.ops.set_vfta(&adapter->hw, vtag, 0, false, 2467 true); 2468 IXGBE_CORE_UNLOCK(adapter); 2469 if (error != 0) 2470 error = EACCES; 2471 2472 return error; 2473 } /* ixgbe_unregister_vlan */ 2474 2475 static void 2476 ixgbe_setup_vlan_hw_tagging(struct adapter *adapter) 2477 { 2478 struct ethercom *ec = &adapter->osdep.ec; 2479 struct ixgbe_hw *hw = &adapter->hw; 2480 struct rx_ring *rxr; 2481 u32 ctrl; 2482 int i; 2483 bool hwtagging; 2484 2485 /* Enable HW tagging only if any vlan is attached */ 2486 hwtagging = (ec->ec_capenable & ETHERCAP_VLAN_HWTAGGING) 2487 && VLAN_ATTACHED(ec); 2488 2489 /* Setup the queues for vlans */ 2490 for (i = 0; i < adapter->num_queues; i++) { 2491 rxr = &adapter->rx_rings[i]; 2492 /* 2493 * On 82599 and later, the VLAN enable is per/queue in RXDCTL. 2494 */ 2495 if (hw->mac.type != ixgbe_mac_82598EB) { 2496 ctrl = IXGBE_READ_REG(hw, IXGBE_RXDCTL(rxr->me)); 2497 if (hwtagging) 2498 ctrl |= IXGBE_RXDCTL_VME; 2499 else 2500 ctrl &= ~IXGBE_RXDCTL_VME; 2501 IXGBE_WRITE_REG(hw, IXGBE_RXDCTL(rxr->me), ctrl); 2502 } 2503 rxr->vtag_strip = hwtagging ? TRUE : FALSE; 2504 } 2505 2506 /* VLAN hw tagging for 82598 */ 2507 if (hw->mac.type == ixgbe_mac_82598EB) { 2508 ctrl = IXGBE_READ_REG(hw, IXGBE_VLNCTRL); 2509 if (hwtagging) 2510 ctrl |= IXGBE_VLNCTRL_VME; 2511 else 2512 ctrl &= ~IXGBE_VLNCTRL_VME; 2513 IXGBE_WRITE_REG(hw, IXGBE_VLNCTRL, ctrl); 2514 } 2515 } /* ixgbe_setup_vlan_hw_tagging */ 2516 2517 static void 2518 ixgbe_setup_vlan_hw_support(struct adapter *adapter) 2519 { 2520 struct ethercom *ec = &adapter->osdep.ec; 2521 struct ixgbe_hw *hw = &adapter->hw; 2522 int i; 2523 u32 ctrl; 2524 struct vlanid_list *vlanidp; 2525 2526 /* 2527 * This function is called from both if_init and ifflags_cb() 2528 * on NetBSD. 2529 */ 2530 2531 /* 2532 * Part 1: 2533 * Setup VLAN HW tagging 2534 */ 2535 ixgbe_setup_vlan_hw_tagging(adapter); 2536 2537 /* 2538 * Part 2: 2539 * Setup VLAN HW filter 2540 */ 2541 /* Cleanup shadow_vfta */ 2542 for (i = 0; i < IXGBE_VFTA_SIZE; i++) 2543 adapter->shadow_vfta[i] = 0; 2544 /* Generate shadow_vfta from ec_vids */ 2545 ETHER_LOCK(ec); 2546 SIMPLEQ_FOREACH(vlanidp, &ec->ec_vids, vid_list) { 2547 uint32_t idx; 2548 2549 idx = vlanidp->vid / 32; 2550 KASSERT(idx < IXGBE_VFTA_SIZE); 2551 adapter->shadow_vfta[idx] |= (u32)1 << (vlanidp->vid % 32); 2552 } 2553 ETHER_UNLOCK(ec); 2554 for (i = 0; i < IXGBE_VFTA_SIZE; i++) 2555 IXGBE_WRITE_REG(hw, IXGBE_VFTA(i), adapter->shadow_vfta[i]); 2556 2557 ctrl = IXGBE_READ_REG(hw, IXGBE_VLNCTRL); 2558 /* Enable the Filter Table if enabled */ 2559 if (ec->ec_capenable & ETHERCAP_VLAN_HWFILTER) 2560 ctrl |= IXGBE_VLNCTRL_VFE; 2561 else 2562 ctrl &= ~IXGBE_VLNCTRL_VFE; 2563 IXGBE_WRITE_REG(hw, IXGBE_VLNCTRL, ctrl); 2564 } /* ixgbe_setup_vlan_hw_support */ 2565 2566 /************************************************************************ 2567 * ixgbe_get_slot_info 2568 * 2569 * Get the width and transaction speed of 2570 * the slot this adapter is plugged into. 2571 ************************************************************************/ 2572 static void 2573 ixgbe_get_slot_info(struct adapter *adapter) 2574 { 2575 device_t dev = adapter->dev; 2576 struct ixgbe_hw *hw = &adapter->hw; 2577 u32 offset; 2578 u16 link; 2579 int bus_info_valid = TRUE; 2580 2581 /* Some devices are behind an internal bridge */ 2582 switch (hw->device_id) { 2583 case IXGBE_DEV_ID_82599_SFP_SF_QP: 2584 case IXGBE_DEV_ID_82599_QSFP_SF_QP: 2585 goto get_parent_info; 2586 default: 2587 break; 2588 } 2589 2590 ixgbe_get_bus_info(hw); 2591 2592 /* 2593 * Some devices don't use PCI-E, but there is no need 2594 * to display "Unknown" for bus speed and width. 2595 */ 2596 switch (hw->mac.type) { 2597 case ixgbe_mac_X550EM_x: 2598 case ixgbe_mac_X550EM_a: 2599 return; 2600 default: 2601 goto display; 2602 } 2603 2604 get_parent_info: 2605 /* 2606 * For the Quad port adapter we need to parse back 2607 * up the PCI tree to find the speed of the expansion 2608 * slot into which this adapter is plugged. A bit more work. 2609 */ 2610 dev = device_parent(device_parent(dev)); 2611 #if 0 2612 #ifdef IXGBE_DEBUG 2613 device_printf(dev, "parent pcib = %x,%x,%x\n", pci_get_bus(dev), 2614 pci_get_slot(dev), pci_get_function(dev)); 2615 #endif 2616 dev = device_parent(device_parent(dev)); 2617 #ifdef IXGBE_DEBUG 2618 device_printf(dev, "slot pcib = %x,%x,%x\n", pci_get_bus(dev), 2619 pci_get_slot(dev), pci_get_function(dev)); 2620 #endif 2621 #endif 2622 /* Now get the PCI Express Capabilities offset */ 2623 if (pci_get_capability(adapter->osdep.pc, adapter->osdep.tag, 2624 PCI_CAP_PCIEXPRESS, &offset, NULL)) { 2625 /* 2626 * Hmm...can't get PCI-Express capabilities. 2627 * Falling back to default method. 2628 */ 2629 bus_info_valid = FALSE; 2630 ixgbe_get_bus_info(hw); 2631 goto display; 2632 } 2633 /* ...and read the Link Status Register */ 2634 link = pci_conf_read(adapter->osdep.pc, adapter->osdep.tag, 2635 offset + PCIE_LCSR) >> 16; 2636 ixgbe_set_pci_config_data_generic(hw, link); 2637 2638 display: 2639 device_printf(dev, "PCI Express Bus: Speed %s Width %s\n", 2640 ((hw->bus.speed == ixgbe_bus_speed_8000) ? "8.0GT/s" : 2641 (hw->bus.speed == ixgbe_bus_speed_5000) ? "5.0GT/s" : 2642 (hw->bus.speed == ixgbe_bus_speed_2500) ? "2.5GT/s" : 2643 "Unknown"), 2644 ((hw->bus.width == ixgbe_bus_width_pcie_x8) ? "x8" : 2645 (hw->bus.width == ixgbe_bus_width_pcie_x4) ? "x4" : 2646 (hw->bus.width == ixgbe_bus_width_pcie_x1) ? "x1" : 2647 "Unknown")); 2648 2649 if (bus_info_valid) { 2650 if ((hw->device_id != IXGBE_DEV_ID_82599_SFP_SF_QP) && 2651 ((hw->bus.width <= ixgbe_bus_width_pcie_x4) && 2652 (hw->bus.speed == ixgbe_bus_speed_2500))) { 2653 device_printf(dev, "PCI-Express bandwidth available" 2654 " for this card\n is not sufficient for" 2655 " optimal performance.\n"); 2656 device_printf(dev, "For optimal performance a x8 " 2657 "PCIE, or x4 PCIE Gen2 slot is required.\n"); 2658 } 2659 if ((hw->device_id == IXGBE_DEV_ID_82599_SFP_SF_QP) && 2660 ((hw->bus.width <= ixgbe_bus_width_pcie_x8) && 2661 (hw->bus.speed < ixgbe_bus_speed_8000))) { 2662 device_printf(dev, "PCI-Express bandwidth available" 2663 " for this card\n is not sufficient for" 2664 " optimal performance.\n"); 2665 device_printf(dev, "For optimal performance a x8 " 2666 "PCIE Gen3 slot is required.\n"); 2667 } 2668 } else 2669 device_printf(dev, 2670 "Unable to determine slot speed/width. The speed/width " 2671 "reported are that of the internal switch.\n"); 2672 2673 return; 2674 } /* ixgbe_get_slot_info */ 2675 2676 /************************************************************************ 2677 * ixgbe_enable_queue - MSI-X Interrupt Handlers and Tasklets 2678 ************************************************************************/ 2679 static inline void 2680 ixgbe_enable_queue(struct adapter *adapter, u32 vector) 2681 { 2682 struct ixgbe_hw *hw = &adapter->hw; 2683 struct ix_queue *que = &adapter->queues[vector]; 2684 u64 queue = 1ULL << vector; 2685 u32 mask; 2686 2687 mutex_enter(&que->dc_mtx); 2688 if (que->disabled_count > 0 && --que->disabled_count > 0) 2689 goto out; 2690 2691 if (hw->mac.type == ixgbe_mac_82598EB) { 2692 mask = (IXGBE_EIMS_RTX_QUEUE & queue); 2693 IXGBE_WRITE_REG(hw, IXGBE_EIMS, mask); 2694 } else { 2695 mask = (queue & 0xFFFFFFFF); 2696 if (mask) 2697 IXGBE_WRITE_REG(hw, IXGBE_EIMS_EX(0), mask); 2698 mask = (queue >> 32); 2699 if (mask) 2700 IXGBE_WRITE_REG(hw, IXGBE_EIMS_EX(1), mask); 2701 } 2702 out: 2703 mutex_exit(&que->dc_mtx); 2704 } /* ixgbe_enable_queue */ 2705 2706 /************************************************************************ 2707 * ixgbe_disable_queue_internal 2708 ************************************************************************/ 2709 static inline void 2710 ixgbe_disable_queue_internal(struct adapter *adapter, u32 vector, bool nestok) 2711 { 2712 struct ixgbe_hw *hw = &adapter->hw; 2713 struct ix_queue *que = &adapter->queues[vector]; 2714 u64 queue = 1ULL << vector; 2715 u32 mask; 2716 2717 mutex_enter(&que->dc_mtx); 2718 2719 if (que->disabled_count > 0) { 2720 if (nestok) 2721 que->disabled_count++; 2722 goto out; 2723 } 2724 que->disabled_count++; 2725 2726 if (hw->mac.type == ixgbe_mac_82598EB) { 2727 mask = (IXGBE_EIMS_RTX_QUEUE & queue); 2728 IXGBE_WRITE_REG(hw, IXGBE_EIMC, mask); 2729 } else { 2730 mask = (queue & 0xFFFFFFFF); 2731 if (mask) 2732 IXGBE_WRITE_REG(hw, IXGBE_EIMC_EX(0), mask); 2733 mask = (queue >> 32); 2734 if (mask) 2735 IXGBE_WRITE_REG(hw, IXGBE_EIMC_EX(1), mask); 2736 } 2737 out: 2738 mutex_exit(&que->dc_mtx); 2739 } /* ixgbe_disable_queue_internal */ 2740 2741 /************************************************************************ 2742 * ixgbe_disable_queue 2743 ************************************************************************/ 2744 static inline void 2745 ixgbe_disable_queue(struct adapter *adapter, u32 vector) 2746 { 2747 2748 ixgbe_disable_queue_internal(adapter, vector, true); 2749 } /* ixgbe_disable_queue */ 2750 2751 /************************************************************************ 2752 * ixgbe_sched_handle_que - schedule deferred packet processing 2753 ************************************************************************/ 2754 static inline void 2755 ixgbe_sched_handle_que(struct adapter *adapter, struct ix_queue *que) 2756 { 2757 2758 if (que->txrx_use_workqueue) { 2759 /* 2760 * adapter->que_wq is bound to each CPU instead of 2761 * each NIC queue to reduce workqueue kthread. As we 2762 * should consider about interrupt affinity in this 2763 * function, the workqueue kthread must be WQ_PERCPU. 2764 * If create WQ_PERCPU workqueue kthread for each NIC 2765 * queue, that number of created workqueue kthread is 2766 * (number of used NIC queue) * (number of CPUs) = 2767 * (number of CPUs) ^ 2 most often. 2768 * 2769 * The same NIC queue's interrupts are avoided by 2770 * masking the queue's interrupt. And different 2771 * NIC queue's interrupts use different struct work 2772 * (que->wq_cookie). So, "enqueued flag" to avoid 2773 * twice workqueue_enqueue() is not required . 2774 */ 2775 workqueue_enqueue(adapter->que_wq, &que->wq_cookie, curcpu()); 2776 } else 2777 softint_schedule(que->que_si); 2778 } 2779 2780 /************************************************************************ 2781 * ixgbe_msix_que - MSI-X Queue Interrupt Service routine 2782 ************************************************************************/ 2783 static int 2784 ixgbe_msix_que(void *arg) 2785 { 2786 struct ix_queue *que = arg; 2787 struct adapter *adapter = que->adapter; 2788 struct ifnet *ifp = adapter->ifp; 2789 struct tx_ring *txr = que->txr; 2790 struct rx_ring *rxr = que->rxr; 2791 u32 newitr = 0; 2792 2793 /* Protect against spurious interrupts */ 2794 if ((ifp->if_flags & IFF_RUNNING) == 0) 2795 return 0; 2796 2797 ixgbe_disable_queue(adapter, que->msix); 2798 IXGBE_EVC_ADD(&que->irqs, 1); 2799 2800 /* 2801 * Don't change "que->txrx_use_workqueue" from this point to avoid 2802 * flip-flopping softint/workqueue mode in one deferred processing. 2803 */ 2804 que->txrx_use_workqueue = adapter->txrx_use_workqueue; 2805 2806 IXGBE_TX_LOCK(txr); 2807 ixgbe_txeof(txr); 2808 IXGBE_TX_UNLOCK(txr); 2809 2810 /* Do AIM now? */ 2811 2812 if (adapter->enable_aim == false) 2813 goto no_calc; 2814 /* 2815 * Do Adaptive Interrupt Moderation: 2816 * - Write out last calculated setting 2817 * - Calculate based on average size over 2818 * the last interval. 2819 */ 2820 if (que->eitr_setting) 2821 ixgbe_eitr_write(adapter, que->msix, que->eitr_setting); 2822 2823 que->eitr_setting = 0; 2824 2825 /* Idle, do nothing */ 2826 if ((txr->bytes == 0) && (rxr->bytes == 0)) 2827 goto no_calc; 2828 2829 if ((txr->bytes) && (txr->packets)) 2830 newitr = txr->bytes/txr->packets; 2831 if ((rxr->bytes) && (rxr->packets)) 2832 newitr = uimax(newitr, (rxr->bytes / rxr->packets)); 2833 newitr += 24; /* account for hardware frame, crc */ 2834 2835 /* set an upper boundary */ 2836 newitr = uimin(newitr, 3000); 2837 2838 /* Be nice to the mid range */ 2839 if ((newitr > 300) && (newitr < 1200)) 2840 newitr = (newitr / 3); 2841 else 2842 newitr = (newitr / 2); 2843 2844 /* 2845 * When RSC is used, ITR interval must be larger than RSC_DELAY. 2846 * Currently, we use 2us for RSC_DELAY. The minimum value is always 2847 * greater than 2us on 100M (and 10M?(not documented)), but it's not 2848 * on 1G and higher. 2849 */ 2850 if ((adapter->link_speed != IXGBE_LINK_SPEED_100_FULL) 2851 && (adapter->link_speed != IXGBE_LINK_SPEED_10_FULL)) 2852 if (newitr < IXGBE_MIN_RSC_EITR_10G1G) 2853 newitr = IXGBE_MIN_RSC_EITR_10G1G; 2854 2855 /* save for next interrupt */ 2856 que->eitr_setting = newitr; 2857 2858 /* Reset state */ 2859 txr->bytes = 0; 2860 txr->packets = 0; 2861 rxr->bytes = 0; 2862 rxr->packets = 0; 2863 2864 no_calc: 2865 ixgbe_sched_handle_que(adapter, que); 2866 2867 return 1; 2868 } /* ixgbe_msix_que */ 2869 2870 /************************************************************************ 2871 * ixgbe_media_status - Media Ioctl callback 2872 * 2873 * Called whenever the user queries the status of 2874 * the interface using ifconfig. 2875 ************************************************************************/ 2876 static void 2877 ixgbe_media_status(struct ifnet *ifp, struct ifmediareq *ifmr) 2878 { 2879 struct adapter *adapter = ifp->if_softc; 2880 struct ixgbe_hw *hw = &adapter->hw; 2881 int layer; 2882 2883 INIT_DEBUGOUT("ixgbe_media_status: begin"); 2884 ixgbe_update_link_status(adapter); 2885 2886 ifmr->ifm_status = IFM_AVALID; 2887 ifmr->ifm_active = IFM_ETHER; 2888 2889 if (adapter->link_active != LINK_STATE_UP) { 2890 ifmr->ifm_active |= IFM_NONE; 2891 return; 2892 } 2893 2894 ifmr->ifm_status |= IFM_ACTIVE; 2895 layer = adapter->phy_layer; 2896 2897 if (layer & IXGBE_PHYSICAL_LAYER_10GBASE_T || 2898 layer & IXGBE_PHYSICAL_LAYER_5GBASE_T || 2899 layer & IXGBE_PHYSICAL_LAYER_2500BASE_T || 2900 layer & IXGBE_PHYSICAL_LAYER_1000BASE_T || 2901 layer & IXGBE_PHYSICAL_LAYER_100BASE_TX || 2902 layer & IXGBE_PHYSICAL_LAYER_10BASE_T) 2903 switch (adapter->link_speed) { 2904 case IXGBE_LINK_SPEED_10GB_FULL: 2905 ifmr->ifm_active |= IFM_10G_T | IFM_FDX; 2906 break; 2907 case IXGBE_LINK_SPEED_5GB_FULL: 2908 ifmr->ifm_active |= IFM_5000_T | IFM_FDX; 2909 break; 2910 case IXGBE_LINK_SPEED_2_5GB_FULL: 2911 ifmr->ifm_active |= IFM_2500_T | IFM_FDX; 2912 break; 2913 case IXGBE_LINK_SPEED_1GB_FULL: 2914 ifmr->ifm_active |= IFM_1000_T | IFM_FDX; 2915 break; 2916 case IXGBE_LINK_SPEED_100_FULL: 2917 ifmr->ifm_active |= IFM_100_TX | IFM_FDX; 2918 break; 2919 case IXGBE_LINK_SPEED_10_FULL: 2920 ifmr->ifm_active |= IFM_10_T | IFM_FDX; 2921 break; 2922 } 2923 if (layer & IXGBE_PHYSICAL_LAYER_SFP_PLUS_CU || 2924 layer & IXGBE_PHYSICAL_LAYER_SFP_ACTIVE_DA) 2925 switch (adapter->link_speed) { 2926 case IXGBE_LINK_SPEED_10GB_FULL: 2927 ifmr->ifm_active |= IFM_10G_TWINAX | IFM_FDX; 2928 break; 2929 } 2930 if (layer & IXGBE_PHYSICAL_LAYER_10GBASE_LR) 2931 switch (adapter->link_speed) { 2932 case IXGBE_LINK_SPEED_10GB_FULL: 2933 ifmr->ifm_active |= IFM_10G_LR | IFM_FDX; 2934 break; 2935 case IXGBE_LINK_SPEED_1GB_FULL: 2936 ifmr->ifm_active |= IFM_1000_LX | IFM_FDX; 2937 break; 2938 } 2939 if (layer & IXGBE_PHYSICAL_LAYER_10GBASE_LRM) 2940 switch (adapter->link_speed) { 2941 case IXGBE_LINK_SPEED_10GB_FULL: 2942 ifmr->ifm_active |= IFM_10G_LRM | IFM_FDX; 2943 break; 2944 case IXGBE_LINK_SPEED_1GB_FULL: 2945 ifmr->ifm_active |= IFM_1000_LX | IFM_FDX; 2946 break; 2947 } 2948 if (layer & IXGBE_PHYSICAL_LAYER_10GBASE_SR || 2949 layer & IXGBE_PHYSICAL_LAYER_1000BASE_SX) 2950 switch (adapter->link_speed) { 2951 case IXGBE_LINK_SPEED_10GB_FULL: 2952 ifmr->ifm_active |= IFM_10G_SR | IFM_FDX; 2953 break; 2954 case IXGBE_LINK_SPEED_1GB_FULL: 2955 ifmr->ifm_active |= IFM_1000_SX | IFM_FDX; 2956 break; 2957 } 2958 if (layer & IXGBE_PHYSICAL_LAYER_10GBASE_CX4) 2959 switch (adapter->link_speed) { 2960 case IXGBE_LINK_SPEED_10GB_FULL: 2961 ifmr->ifm_active |= IFM_10G_CX4 | IFM_FDX; 2962 break; 2963 } 2964 /* 2965 * XXX: These need to use the proper media types once 2966 * they're added. 2967 */ 2968 if (layer & IXGBE_PHYSICAL_LAYER_10GBASE_KR) 2969 switch (adapter->link_speed) { 2970 case IXGBE_LINK_SPEED_10GB_FULL: 2971 ifmr->ifm_active |= IFM_10G_KR | IFM_FDX; 2972 break; 2973 case IXGBE_LINK_SPEED_2_5GB_FULL: 2974 ifmr->ifm_active |= IFM_2500_KX | IFM_FDX; 2975 break; 2976 case IXGBE_LINK_SPEED_1GB_FULL: 2977 ifmr->ifm_active |= IFM_1000_KX | IFM_FDX; 2978 break; 2979 } 2980 else if (layer & IXGBE_PHYSICAL_LAYER_10GBASE_KX4 || 2981 layer & IXGBE_PHYSICAL_LAYER_2500BASE_KX || 2982 layer & IXGBE_PHYSICAL_LAYER_1000BASE_KX) 2983 switch (adapter->link_speed) { 2984 case IXGBE_LINK_SPEED_10GB_FULL: 2985 ifmr->ifm_active |= IFM_10G_KX4 | IFM_FDX; 2986 break; 2987 case IXGBE_LINK_SPEED_2_5GB_FULL: 2988 ifmr->ifm_active |= IFM_2500_KX | IFM_FDX; 2989 break; 2990 case IXGBE_LINK_SPEED_1GB_FULL: 2991 ifmr->ifm_active |= IFM_1000_KX | IFM_FDX; 2992 break; 2993 } 2994 2995 /* If nothing is recognized... */ 2996 #if 0 2997 if (IFM_SUBTYPE(ifmr->ifm_active) == 0) 2998 ifmr->ifm_active |= IFM_UNKNOWN; 2999 #endif 3000 3001 ifp->if_baudrate = ifmedia_baudrate(ifmr->ifm_active); 3002 3003 /* Display current flow control setting used on link */ 3004 if (hw->fc.current_mode == ixgbe_fc_rx_pause || 3005 hw->fc.current_mode == ixgbe_fc_full) 3006 ifmr->ifm_active |= IFM_ETH_RXPAUSE; 3007 if (hw->fc.current_mode == ixgbe_fc_tx_pause || 3008 hw->fc.current_mode == ixgbe_fc_full) 3009 ifmr->ifm_active |= IFM_ETH_TXPAUSE; 3010 3011 return; 3012 } /* ixgbe_media_status */ 3013 3014 /************************************************************************ 3015 * ixgbe_media_change - Media Ioctl callback 3016 * 3017 * Called when the user changes speed/duplex using 3018 * media/mediopt option with ifconfig. 3019 ************************************************************************/ 3020 static int 3021 ixgbe_media_change(struct ifnet *ifp) 3022 { 3023 struct adapter *adapter = ifp->if_softc; 3024 struct ifmedia *ifm = &adapter->media; 3025 struct ixgbe_hw *hw = &adapter->hw; 3026 ixgbe_link_speed speed = 0; 3027 ixgbe_link_speed link_caps = 0; 3028 bool negotiate = false; 3029 s32 err = IXGBE_NOT_IMPLEMENTED; 3030 3031 INIT_DEBUGOUT("ixgbe_media_change: begin"); 3032 3033 if (IFM_TYPE(ifm->ifm_media) != IFM_ETHER) 3034 return (EINVAL); 3035 3036 if (hw->phy.media_type == ixgbe_media_type_backplane) 3037 return (EPERM); 3038 3039 /* 3040 * We don't actually need to check against the supported 3041 * media types of the adapter; ifmedia will take care of 3042 * that for us. 3043 */ 3044 switch (IFM_SUBTYPE(ifm->ifm_media)) { 3045 case IFM_AUTO: 3046 err = hw->mac.ops.get_link_capabilities(hw, &link_caps, 3047 &negotiate); 3048 if (err != IXGBE_SUCCESS) { 3049 device_printf(adapter->dev, "Unable to determine " 3050 "supported advertise speeds\n"); 3051 return (ENODEV); 3052 } 3053 speed |= link_caps; 3054 break; 3055 case IFM_10G_T: 3056 case IFM_10G_LRM: 3057 case IFM_10G_LR: 3058 case IFM_10G_TWINAX: 3059 case IFM_10G_SR: 3060 case IFM_10G_CX4: 3061 case IFM_10G_KR: 3062 case IFM_10G_KX4: 3063 speed |= IXGBE_LINK_SPEED_10GB_FULL; 3064 break; 3065 case IFM_5000_T: 3066 speed |= IXGBE_LINK_SPEED_5GB_FULL; 3067 break; 3068 case IFM_2500_T: 3069 case IFM_2500_KX: 3070 speed |= IXGBE_LINK_SPEED_2_5GB_FULL; 3071 break; 3072 case IFM_1000_T: 3073 case IFM_1000_LX: 3074 case IFM_1000_SX: 3075 case IFM_1000_KX: 3076 speed |= IXGBE_LINK_SPEED_1GB_FULL; 3077 break; 3078 case IFM_100_TX: 3079 speed |= IXGBE_LINK_SPEED_100_FULL; 3080 break; 3081 case IFM_10_T: 3082 speed |= IXGBE_LINK_SPEED_10_FULL; 3083 break; 3084 case IFM_NONE: 3085 break; 3086 default: 3087 goto invalid; 3088 } 3089 3090 hw->mac.autotry_restart = TRUE; 3091 hw->mac.ops.setup_link(hw, speed, TRUE); 3092 adapter->advertise = 0; 3093 if (IFM_SUBTYPE(ifm->ifm_media) != IFM_AUTO) { 3094 if ((speed & IXGBE_LINK_SPEED_10GB_FULL) != 0) 3095 adapter->advertise |= 1 << 2; 3096 if ((speed & IXGBE_LINK_SPEED_1GB_FULL) != 0) 3097 adapter->advertise |= 1 << 1; 3098 if ((speed & IXGBE_LINK_SPEED_100_FULL) != 0) 3099 adapter->advertise |= 1 << 0; 3100 if ((speed & IXGBE_LINK_SPEED_10_FULL) != 0) 3101 adapter->advertise |= 1 << 3; 3102 if ((speed & IXGBE_LINK_SPEED_2_5GB_FULL) != 0) 3103 adapter->advertise |= 1 << 4; 3104 if ((speed & IXGBE_LINK_SPEED_5GB_FULL) != 0) 3105 adapter->advertise |= 1 << 5; 3106 } 3107 3108 return (0); 3109 3110 invalid: 3111 device_printf(adapter->dev, "Invalid media type!\n"); 3112 3113 return (EINVAL); 3114 } /* ixgbe_media_change */ 3115 3116 /************************************************************************ 3117 * ixgbe_msix_admin - Link status change ISR (MSI-X) 3118 ************************************************************************/ 3119 static int 3120 ixgbe_msix_admin(void *arg) 3121 { 3122 struct adapter *adapter = arg; 3123 struct ixgbe_hw *hw = &adapter->hw; 3124 u32 eicr; 3125 u32 eims_orig; 3126 u32 eims_disable = 0; 3127 3128 IXGBE_EVC_ADD(&adapter->admin_irqev, 1); 3129 3130 eims_orig = IXGBE_READ_REG(hw, IXGBE_EIMS); 3131 /* Pause other interrupts */ 3132 IXGBE_WRITE_REG(hw, IXGBE_EIMC, IXGBE_MSIX_OTHER_CLEAR_MASK); 3133 3134 /* 3135 * First get the cause. 3136 * 3137 * The specifications of 82598, 82599, X540 and X550 say EICS register 3138 * is write only. However, Linux says it is a workaround for silicon 3139 * errata to read EICS instead of EICR to get interrupt cause. 3140 * At least, reading EICR clears lower 16bits of EIMS on 82598. 3141 */ 3142 eicr = IXGBE_READ_REG(hw, IXGBE_EICS); 3143 /* Be sure the queue bits are not cleared */ 3144 eicr &= ~IXGBE_EICR_RTX_QUEUE; 3145 /* Clear all OTHER interrupts with write */ 3146 IXGBE_WRITE_REG(hw, IXGBE_EICR, eicr); 3147 3148 ixgbe_intr_admin_common(adapter, eicr, &eims_disable); 3149 3150 /* Re-enable some OTHER interrupts */ 3151 IXGBE_WRITE_REG(hw, IXGBE_EIMS, eims_orig & ~eims_disable); 3152 3153 return 1; 3154 } /* ixgbe_msix_admin */ 3155 3156 static void 3157 ixgbe_intr_admin_common(struct adapter *adapter, u32 eicr, u32 *eims_disable) 3158 { 3159 struct ixgbe_hw *hw = &adapter->hw; 3160 u32 task_requests = 0; 3161 s32 retval; 3162 3163 /* Link status change */ 3164 if (eicr & IXGBE_EICR_LSC) { 3165 task_requests |= IXGBE_REQUEST_TASK_LSC; 3166 *eims_disable |= IXGBE_EIMS_LSC; 3167 } 3168 3169 if (ixgbe_is_sfp(hw)) { 3170 u32 eicr_mask; 3171 3172 /* Pluggable optics-related interrupt */ 3173 if (hw->mac.type >= ixgbe_mac_X540) 3174 eicr_mask = IXGBE_EICR_GPI_SDP0_X540; 3175 else 3176 eicr_mask = IXGBE_EICR_GPI_SDP2_BY_MAC(hw); 3177 3178 /* 3179 * An interrupt might not arrive when a module is inserted. 3180 * When an link status change interrupt occurred and the driver 3181 * still regard SFP as unplugged, issue the module softint 3182 * and then issue LSC interrupt. 3183 */ 3184 if ((eicr & eicr_mask) 3185 || ((hw->phy.sfp_type == ixgbe_sfp_type_not_present) 3186 && (eicr & IXGBE_EICR_LSC))) { 3187 task_requests |= IXGBE_REQUEST_TASK_MOD; 3188 *eims_disable |= IXGBE_EIMS_LSC; 3189 } 3190 3191 if ((hw->mac.type == ixgbe_mac_82599EB) && 3192 (eicr & IXGBE_EICR_GPI_SDP1_BY_MAC(hw))) { 3193 task_requests |= IXGBE_REQUEST_TASK_MSF; 3194 *eims_disable |= IXGBE_EIMS_GPI_SDP1_BY_MAC(hw); 3195 } 3196 } 3197 3198 if (adapter->hw.mac.type != ixgbe_mac_82598EB) { 3199 #ifdef IXGBE_FDIR 3200 if ((adapter->feat_en & IXGBE_FEATURE_FDIR) && 3201 (eicr & IXGBE_EICR_FLOW_DIR)) { 3202 if (!atomic_cas_uint(&adapter->fdir_reinit, 0, 1)) { 3203 task_requests |= IXGBE_REQUEST_TASK_FDIR; 3204 /* Disable the interrupt */ 3205 *eims_disable |= IXGBE_EIMS_FLOW_DIR; 3206 } 3207 } 3208 #endif 3209 3210 if (eicr & IXGBE_EICR_ECC) { 3211 if (ratecheck(&adapter->lasterr_time, 3212 &ixgbe_errlog_intrvl)) 3213 device_printf(adapter->dev, 3214 "CRITICAL: ECC ERROR!! Please Reboot!!\n"); 3215 } 3216 3217 /* Check for over temp condition */ 3218 if (adapter->feat_en & IXGBE_FEATURE_TEMP_SENSOR) { 3219 switch (adapter->hw.mac.type) { 3220 case ixgbe_mac_X550EM_a: 3221 if (!(eicr & IXGBE_EICR_GPI_SDP0_X550EM_a)) 3222 break; 3223 retval = hw->phy.ops.check_overtemp(hw); 3224 if (retval != IXGBE_ERR_OVERTEMP) 3225 break; 3226 if (ratecheck(&adapter->lasterr_time, 3227 &ixgbe_errlog_intrvl)) { 3228 device_printf(adapter->dev, 3229 "CRITICAL: OVER TEMP!! " 3230 "PHY IS SHUT DOWN!!\n"); 3231 device_printf(adapter->dev, 3232 "System shutdown required!\n"); 3233 } 3234 break; 3235 default: 3236 if (!(eicr & IXGBE_EICR_TS)) 3237 break; 3238 retval = hw->phy.ops.check_overtemp(hw); 3239 if (retval != IXGBE_ERR_OVERTEMP) 3240 break; 3241 if (ratecheck(&adapter->lasterr_time, 3242 &ixgbe_errlog_intrvl)) { 3243 device_printf(adapter->dev, 3244 "CRITICAL: OVER TEMP!! " 3245 "PHY IS SHUT DOWN!!\n"); 3246 device_printf(adapter->dev, 3247 "System shutdown required!\n"); 3248 } 3249 break; 3250 } 3251 } 3252 3253 /* Check for VF message */ 3254 if ((adapter->feat_en & IXGBE_FEATURE_SRIOV) && 3255 (eicr & IXGBE_EICR_MAILBOX)) { 3256 task_requests |= IXGBE_REQUEST_TASK_MBX; 3257 *eims_disable |= IXGBE_EIMS_MAILBOX; 3258 } 3259 } 3260 3261 /* Check for fan failure */ 3262 if (adapter->feat_en & IXGBE_FEATURE_FAN_FAIL) 3263 ixgbe_check_fan_failure(adapter, eicr, true); 3264 3265 /* External PHY interrupt */ 3266 if ((hw->phy.type == ixgbe_phy_x550em_ext_t) && 3267 (eicr & IXGBE_EICR_GPI_SDP0_X540)) { 3268 task_requests |= IXGBE_REQUEST_TASK_PHY; 3269 *eims_disable |= IXGBE_EICR_GPI_SDP0_X540; 3270 } 3271 3272 if (task_requests != 0) { 3273 mutex_enter(&adapter->admin_mtx); 3274 adapter->task_requests |= task_requests; 3275 ixgbe_schedule_admin_tasklet(adapter); 3276 mutex_exit(&adapter->admin_mtx); 3277 } 3278 } 3279 3280 static void 3281 ixgbe_eitr_write(struct adapter *adapter, uint32_t index, uint32_t itr) 3282 { 3283 3284 if (adapter->hw.mac.type == ixgbe_mac_82598EB) 3285 itr |= itr << 16; 3286 else 3287 itr |= IXGBE_EITR_CNT_WDIS; 3288 3289 IXGBE_WRITE_REG(&adapter->hw, IXGBE_EITR(index), itr); 3290 } 3291 3292 3293 /************************************************************************ 3294 * ixgbe_sysctl_interrupt_rate_handler 3295 ************************************************************************/ 3296 static int 3297 ixgbe_sysctl_interrupt_rate_handler(SYSCTLFN_ARGS) 3298 { 3299 struct sysctlnode node = *rnode; 3300 struct ix_queue *que = (struct ix_queue *)node.sysctl_data; 3301 struct adapter *adapter; 3302 uint32_t reg, usec, rate; 3303 int error; 3304 3305 if (que == NULL) 3306 return 0; 3307 3308 adapter = que->adapter; 3309 if (ixgbe_fw_recovery_mode_swflag(adapter)) 3310 return (EPERM); 3311 3312 reg = IXGBE_READ_REG(&adapter->hw, IXGBE_EITR(que->msix)); 3313 usec = ((reg & 0x0FF8) >> 3); 3314 if (usec > 0) 3315 rate = 500000 / usec; 3316 else 3317 rate = 0; 3318 node.sysctl_data = &rate; 3319 error = sysctl_lookup(SYSCTLFN_CALL(&node)); 3320 if (error || newp == NULL) 3321 return error; 3322 reg &= ~0xfff; /* default, no limitation */ 3323 if (rate > 0 && rate < 500000) { 3324 if (rate < 1000) 3325 rate = 1000; 3326 reg |= ((4000000 / rate) & 0xff8); 3327 /* 3328 * When RSC is used, ITR interval must be larger than 3329 * RSC_DELAY. Currently, we use 2us for RSC_DELAY. 3330 * The minimum value is always greater than 2us on 100M 3331 * (and 10M?(not documented)), but it's not on 1G and higher. 3332 */ 3333 if ((adapter->link_speed != IXGBE_LINK_SPEED_100_FULL) 3334 && (adapter->link_speed != IXGBE_LINK_SPEED_10_FULL)) { 3335 if ((adapter->num_queues > 1) 3336 && (reg < IXGBE_MIN_RSC_EITR_10G1G)) 3337 return EINVAL; 3338 } 3339 ixgbe_max_interrupt_rate = rate; 3340 } else 3341 ixgbe_max_interrupt_rate = 0; 3342 ixgbe_eitr_write(adapter, que->msix, reg); 3343 3344 return (0); 3345 } /* ixgbe_sysctl_interrupt_rate_handler */ 3346 3347 const struct sysctlnode * 3348 ixgbe_sysctl_instance(struct adapter *adapter) 3349 { 3350 const char *dvname; 3351 struct sysctllog **log; 3352 int rc; 3353 const struct sysctlnode *rnode; 3354 3355 if (adapter->sysctltop != NULL) 3356 return adapter->sysctltop; 3357 3358 log = &adapter->sysctllog; 3359 dvname = device_xname(adapter->dev); 3360 3361 if ((rc = sysctl_createv(log, 0, NULL, &rnode, 3362 0, CTLTYPE_NODE, dvname, 3363 SYSCTL_DESCR("ixgbe information and settings"), 3364 NULL, 0, NULL, 0, CTL_HW, CTL_CREATE, CTL_EOL)) != 0) 3365 goto err; 3366 3367 return rnode; 3368 err: 3369 device_printf(adapter->dev, 3370 "%s: sysctl_createv failed, rc = %d\n", __func__, rc); 3371 return NULL; 3372 } 3373 3374 /************************************************************************ 3375 * ixgbe_add_device_sysctls 3376 ************************************************************************/ 3377 static void 3378 ixgbe_add_device_sysctls(struct adapter *adapter) 3379 { 3380 device_t dev = adapter->dev; 3381 struct ixgbe_hw *hw = &adapter->hw; 3382 struct sysctllog **log; 3383 const struct sysctlnode *rnode, *cnode; 3384 3385 log = &adapter->sysctllog; 3386 3387 if ((rnode = ixgbe_sysctl_instance(adapter)) == NULL) { 3388 aprint_error_dev(dev, "could not create sysctl root\n"); 3389 return; 3390 } 3391 3392 if (sysctl_createv(log, 0, &rnode, &cnode, 3393 CTLFLAG_READWRITE, CTLTYPE_INT, 3394 "debug", SYSCTL_DESCR("Debug Info"), 3395 ixgbe_sysctl_debug, 0, (void *)adapter, 0, CTL_CREATE, CTL_EOL) 3396 != 0) 3397 aprint_error_dev(dev, "could not create sysctl\n"); 3398 3399 if (sysctl_createv(log, 0, &rnode, &cnode, 3400 CTLFLAG_READWRITE, CTLTYPE_INT, 3401 "rx_copy_len", SYSCTL_DESCR("RX Copy Length"), 3402 ixgbe_sysctl_rx_copy_len, 0, 3403 (void *)adapter, 0, CTL_CREATE, CTL_EOL) != 0) 3404 aprint_error_dev(dev, "could not create sysctl\n"); 3405 3406 if (sysctl_createv(log, 0, &rnode, &cnode, 3407 CTLFLAG_READONLY, CTLTYPE_INT, 3408 "num_tx_desc", SYSCTL_DESCR("Number of TX descriptors"), 3409 NULL, 0, &adapter->num_tx_desc, 0, CTL_CREATE, CTL_EOL) != 0) 3410 aprint_error_dev(dev, "could not create sysctl\n"); 3411 3412 if (sysctl_createv(log, 0, &rnode, &cnode, 3413 CTLFLAG_READONLY, CTLTYPE_INT, 3414 "num_rx_desc", SYSCTL_DESCR("Number of RX descriptors"), 3415 NULL, 0, &adapter->num_rx_desc, 0, CTL_CREATE, CTL_EOL) != 0) 3416 aprint_error_dev(dev, "could not create sysctl\n"); 3417 3418 if (sysctl_createv(log, 0, &rnode, &cnode, 3419 CTLFLAG_READWRITE, CTLTYPE_INT, "rx_process_limit", 3420 SYSCTL_DESCR("max number of RX packets to process"), 3421 ixgbe_sysctl_rx_process_limit, 0, (void *)adapter, 0, CTL_CREATE, 3422 CTL_EOL) != 0) 3423 aprint_error_dev(dev, "could not create sysctl\n"); 3424 3425 if (sysctl_createv(log, 0, &rnode, &cnode, 3426 CTLFLAG_READWRITE, CTLTYPE_INT, "tx_process_limit", 3427 SYSCTL_DESCR("max number of TX packets to process"), 3428 ixgbe_sysctl_tx_process_limit, 0, (void *)adapter, 0, CTL_CREATE, 3429 CTL_EOL) != 0) 3430 aprint_error_dev(dev, "could not create sysctl\n"); 3431 3432 if (sysctl_createv(log, 0, &rnode, &cnode, 3433 CTLFLAG_READONLY, CTLTYPE_INT, 3434 "num_queues", SYSCTL_DESCR("Number of queues"), 3435 NULL, 0, &adapter->num_queues, 0, CTL_CREATE, CTL_EOL) != 0) 3436 aprint_error_dev(dev, "could not create sysctl\n"); 3437 3438 /* Sysctls for all devices */ 3439 if (sysctl_createv(log, 0, &rnode, &cnode, CTLFLAG_READWRITE, 3440 CTLTYPE_INT, "fc", SYSCTL_DESCR(IXGBE_SYSCTL_DESC_SET_FC), 3441 ixgbe_sysctl_flowcntl, 0, (void *)adapter, 0, CTL_CREATE, 3442 CTL_EOL) != 0) 3443 aprint_error_dev(dev, "could not create sysctl\n"); 3444 3445 adapter->enable_aim = ixgbe_enable_aim; 3446 if (sysctl_createv(log, 0, &rnode, &cnode, CTLFLAG_READWRITE, 3447 CTLTYPE_BOOL, "enable_aim", SYSCTL_DESCR("Interrupt Moderation"), 3448 NULL, 0, &adapter->enable_aim, 0, CTL_CREATE, CTL_EOL) != 0) 3449 aprint_error_dev(dev, "could not create sysctl\n"); 3450 3451 if (sysctl_createv(log, 0, &rnode, &cnode, 3452 CTLFLAG_READWRITE, CTLTYPE_INT, 3453 "advertise_speed", SYSCTL_DESCR(IXGBE_SYSCTL_DESC_ADV_SPEED), 3454 ixgbe_sysctl_advertise, 0, (void *)adapter, 0, CTL_CREATE, 3455 CTL_EOL) != 0) 3456 aprint_error_dev(dev, "could not create sysctl\n"); 3457 3458 /* 3459 * If each "que->txrx_use_workqueue" is changed in sysctl handler, 3460 * it causesflip-flopping softint/workqueue mode in one deferred 3461 * processing. Therefore, preempt_disable()/preempt_enable() are 3462 * required in ixgbe_sched_handle_que() to avoid 3463 * KASSERT(ixgbe_sched_handle_que()) in softint_schedule(). 3464 * I think changing "que->txrx_use_workqueue" in interrupt handler 3465 * is lighter than doing preempt_disable()/preempt_enable() in every 3466 * ixgbe_sched_handle_que(). 3467 */ 3468 adapter->txrx_use_workqueue = ixgbe_txrx_workqueue; 3469 if (sysctl_createv(log, 0, &rnode, &cnode, CTLFLAG_READWRITE, 3470 CTLTYPE_BOOL, "txrx_workqueue", 3471 SYSCTL_DESCR("Use workqueue for packet processing"), 3472 NULL, 0, &adapter->txrx_use_workqueue, 0, CTL_CREATE, 3473 CTL_EOL) != 0) 3474 aprint_error_dev(dev, "could not create sysctl\n"); 3475 3476 #ifdef IXGBE_DEBUG 3477 /* testing sysctls (for all devices) */ 3478 if (sysctl_createv(log, 0, &rnode, &cnode, CTLFLAG_READWRITE, 3479 CTLTYPE_INT, "power_state", SYSCTL_DESCR("PCI Power State"), 3480 ixgbe_sysctl_power_state, 0, (void *)adapter, 0, CTL_CREATE, 3481 CTL_EOL) != 0) 3482 aprint_error_dev(dev, "could not create sysctl\n"); 3483 3484 if (sysctl_createv(log, 0, &rnode, &cnode, CTLFLAG_READONLY, 3485 CTLTYPE_STRING, "print_rss_config", 3486 SYSCTL_DESCR("Prints RSS Configuration"), 3487 ixgbe_sysctl_print_rss_config, 0, (void *)adapter, 0, CTL_CREATE, 3488 CTL_EOL) != 0) 3489 aprint_error_dev(dev, "could not create sysctl\n"); 3490 #endif 3491 /* for X550 series devices */ 3492 if (hw->mac.type >= ixgbe_mac_X550) 3493 if (sysctl_createv(log, 0, &rnode, &cnode, CTLFLAG_READWRITE, 3494 CTLTYPE_INT, "dmac", SYSCTL_DESCR("DMA Coalesce"), 3495 ixgbe_sysctl_dmac, 0, (void *)adapter, 0, CTL_CREATE, 3496 CTL_EOL) != 0) 3497 aprint_error_dev(dev, "could not create sysctl\n"); 3498 3499 /* for WoL-capable devices */ 3500 if (adapter->wol_support) { 3501 if (sysctl_createv(log, 0, &rnode, &cnode, CTLFLAG_READWRITE, 3502 CTLTYPE_BOOL, "wol_enable", 3503 SYSCTL_DESCR("Enable/Disable Wake on LAN"), 3504 ixgbe_sysctl_wol_enable, 0, (void *)adapter, 0, CTL_CREATE, 3505 CTL_EOL) != 0) 3506 aprint_error_dev(dev, "could not create sysctl\n"); 3507 3508 if (sysctl_createv(log, 0, &rnode, &cnode, CTLFLAG_READWRITE, 3509 CTLTYPE_INT, "wufc", 3510 SYSCTL_DESCR("Enable/Disable Wake Up Filters"), 3511 ixgbe_sysctl_wufc, 0, (void *)adapter, 0, CTL_CREATE, 3512 CTL_EOL) != 0) 3513 aprint_error_dev(dev, "could not create sysctl\n"); 3514 } 3515 3516 /* for X552/X557-AT devices */ 3517 if (hw->device_id == IXGBE_DEV_ID_X550EM_X_10G_T) { 3518 const struct sysctlnode *phy_node; 3519 3520 if (sysctl_createv(log, 0, &rnode, &phy_node, 0, CTLTYPE_NODE, 3521 "phy", SYSCTL_DESCR("External PHY sysctls"), 3522 NULL, 0, NULL, 0, CTL_CREATE, CTL_EOL) != 0) { 3523 aprint_error_dev(dev, "could not create sysctl\n"); 3524 return; 3525 } 3526 3527 if (sysctl_createv(log, 0, &phy_node, &cnode, CTLFLAG_READONLY, 3528 CTLTYPE_INT, "temp", 3529 SYSCTL_DESCR("Current External PHY Temperature (Celsius)"), 3530 ixgbe_sysctl_phy_temp, 0, (void *)adapter, 0, CTL_CREATE, 3531 CTL_EOL) != 0) 3532 aprint_error_dev(dev, "could not create sysctl\n"); 3533 3534 if (sysctl_createv(log, 0, &phy_node, &cnode, CTLFLAG_READONLY, 3535 CTLTYPE_INT, "overtemp_occurred", 3536 SYSCTL_DESCR( 3537 "External PHY High Temperature Event Occurred"), 3538 ixgbe_sysctl_phy_overtemp_occurred, 0, (void *)adapter, 0, 3539 CTL_CREATE, CTL_EOL) != 0) 3540 aprint_error_dev(dev, "could not create sysctl\n"); 3541 } 3542 3543 if ((hw->mac.type == ixgbe_mac_X550EM_a) 3544 && (hw->phy.type == ixgbe_phy_fw)) 3545 if (sysctl_createv(log, 0, &rnode, &cnode, CTLFLAG_READWRITE, 3546 CTLTYPE_BOOL, "force_10_100_autonego", 3547 SYSCTL_DESCR("Force autonego on 10M and 100M"), 3548 NULL, 0, &hw->phy.force_10_100_autonego, 0, 3549 CTL_CREATE, CTL_EOL) != 0) 3550 aprint_error_dev(dev, "could not create sysctl\n"); 3551 3552 if (adapter->feat_cap & IXGBE_FEATURE_EEE) { 3553 if (sysctl_createv(log, 0, &rnode, &cnode, CTLFLAG_READWRITE, 3554 CTLTYPE_INT, "eee_state", 3555 SYSCTL_DESCR("EEE Power Save State"), 3556 ixgbe_sysctl_eee_state, 0, (void *)adapter, 0, CTL_CREATE, 3557 CTL_EOL) != 0) 3558 aprint_error_dev(dev, "could not create sysctl\n"); 3559 } 3560 } /* ixgbe_add_device_sysctls */ 3561 3562 /************************************************************************ 3563 * ixgbe_allocate_pci_resources 3564 ************************************************************************/ 3565 static int 3566 ixgbe_allocate_pci_resources(struct adapter *adapter, 3567 const struct pci_attach_args *pa) 3568 { 3569 pcireg_t memtype, csr; 3570 device_t dev = adapter->dev; 3571 bus_addr_t addr; 3572 int flags; 3573 3574 memtype = pci_mapreg_type(pa->pa_pc, pa->pa_tag, PCI_BAR(0)); 3575 switch (memtype) { 3576 case PCI_MAPREG_TYPE_MEM | PCI_MAPREG_MEM_TYPE_32BIT: 3577 case PCI_MAPREG_TYPE_MEM | PCI_MAPREG_MEM_TYPE_64BIT: 3578 adapter->osdep.mem_bus_space_tag = pa->pa_memt; 3579 if (pci_mapreg_info(pa->pa_pc, pa->pa_tag, PCI_BAR(0), 3580 memtype, &addr, &adapter->osdep.mem_size, &flags) != 0) 3581 goto map_err; 3582 if ((flags & BUS_SPACE_MAP_PREFETCHABLE) != 0) { 3583 aprint_normal_dev(dev, "clearing prefetchable bit\n"); 3584 flags &= ~BUS_SPACE_MAP_PREFETCHABLE; 3585 } 3586 if (bus_space_map(adapter->osdep.mem_bus_space_tag, addr, 3587 adapter->osdep.mem_size, flags, 3588 &adapter->osdep.mem_bus_space_handle) != 0) { 3589 map_err: 3590 adapter->osdep.mem_size = 0; 3591 aprint_error_dev(dev, "unable to map BAR0\n"); 3592 return ENXIO; 3593 } 3594 /* 3595 * Enable address decoding for memory range in case BIOS or 3596 * UEFI don't set it. 3597 */ 3598 csr = pci_conf_read(pa->pa_pc, pa->pa_tag, 3599 PCI_COMMAND_STATUS_REG); 3600 csr |= PCI_COMMAND_MEM_ENABLE; 3601 pci_conf_write(pa->pa_pc, pa->pa_tag, PCI_COMMAND_STATUS_REG, 3602 csr); 3603 break; 3604 default: 3605 aprint_error_dev(dev, "unexpected type on BAR0\n"); 3606 return ENXIO; 3607 } 3608 3609 return (0); 3610 } /* ixgbe_allocate_pci_resources */ 3611 3612 static void 3613 ixgbe_free_deferred_handlers(struct adapter *adapter) 3614 { 3615 struct ix_queue *que = adapter->queues; 3616 struct tx_ring *txr = adapter->tx_rings; 3617 int i; 3618 3619 for (i = 0; i < adapter->num_queues; i++, que++, txr++) { 3620 if (!(adapter->feat_en & IXGBE_FEATURE_LEGACY_TX)) { 3621 if (txr->txr_si != NULL) 3622 softint_disestablish(txr->txr_si); 3623 } 3624 if (que->que_si != NULL) 3625 softint_disestablish(que->que_si); 3626 } 3627 if (adapter->txr_wq != NULL) 3628 workqueue_destroy(adapter->txr_wq); 3629 if (adapter->txr_wq_enqueued != NULL) 3630 percpu_free(adapter->txr_wq_enqueued, sizeof(u_int)); 3631 if (adapter->que_wq != NULL) 3632 workqueue_destroy(adapter->que_wq); 3633 3634 if (adapter->admin_wq != NULL) { 3635 workqueue_destroy(adapter->admin_wq); 3636 adapter->admin_wq = NULL; 3637 } 3638 if (adapter->timer_wq != NULL) { 3639 workqueue_destroy(adapter->timer_wq); 3640 adapter->timer_wq = NULL; 3641 } 3642 if (adapter->recovery_mode_timer_wq != NULL) { 3643 /* 3644 * ixgbe_ifstop() doesn't call the workqueue_wait() for 3645 * the recovery_mode_timer workqueue, so call it here. 3646 */ 3647 workqueue_wait(adapter->recovery_mode_timer_wq, 3648 &adapter->recovery_mode_timer_wc); 3649 atomic_store_relaxed(&adapter->recovery_mode_timer_pending, 0); 3650 workqueue_destroy(adapter->recovery_mode_timer_wq); 3651 adapter->recovery_mode_timer_wq = NULL; 3652 } 3653 } /* ixgbe_free_deferred_handlers */ 3654 3655 /************************************************************************ 3656 * ixgbe_detach - Device removal routine 3657 * 3658 * Called when the driver is being removed. 3659 * Stops the adapter and deallocates all the resources 3660 * that were allocated for driver operation. 3661 * 3662 * return 0 on success, positive on failure 3663 ************************************************************************/ 3664 static int 3665 ixgbe_detach(device_t dev, int flags) 3666 { 3667 struct adapter *adapter = device_private(dev); 3668 struct rx_ring *rxr = adapter->rx_rings; 3669 struct tx_ring *txr = adapter->tx_rings; 3670 struct ixgbe_hw *hw = &adapter->hw; 3671 struct ixgbe_hw_stats *stats = &adapter->stats.pf; 3672 u32 ctrl_ext; 3673 int i; 3674 3675 INIT_DEBUGOUT("ixgbe_detach: begin"); 3676 if (adapter->osdep.attached == false) 3677 return 0; 3678 3679 if (ixgbe_pci_iov_detach(dev) != 0) { 3680 device_printf(dev, "SR-IOV in use; detach first.\n"); 3681 return (EBUSY); 3682 } 3683 3684 if (VLAN_ATTACHED(&adapter->osdep.ec) && 3685 (flags & (DETACH_SHUTDOWN | DETACH_FORCE)) == 0) { 3686 aprint_error_dev(dev, "VLANs in use, detach first\n"); 3687 return (EBUSY); 3688 } 3689 3690 ether_ifdetach(adapter->ifp); 3691 3692 adapter->osdep.detaching = true; 3693 /* 3694 * Stop the interface. ixgbe_setup_low_power_mode() calls 3695 * ixgbe_ifstop(), so it's not required to call ixgbe_ifstop() 3696 * directly. 3697 */ 3698 ixgbe_setup_low_power_mode(adapter); 3699 3700 callout_halt(&adapter->timer, NULL); 3701 if (adapter->feat_en & IXGBE_FEATURE_RECOVERY_MODE) 3702 callout_halt(&adapter->recovery_mode_timer, NULL); 3703 3704 workqueue_wait(adapter->admin_wq, &adapter->admin_wc); 3705 atomic_store_relaxed(&adapter->admin_pending, 0); 3706 workqueue_wait(adapter->timer_wq, &adapter->timer_wc); 3707 atomic_store_relaxed(&adapter->timer_pending, 0); 3708 3709 pmf_device_deregister(dev); 3710 3711 ixgbe_free_deferred_handlers(adapter); 3712 3713 /* let hardware know driver is unloading */ 3714 ctrl_ext = IXGBE_READ_REG(&adapter->hw, IXGBE_CTRL_EXT); 3715 ctrl_ext &= ~IXGBE_CTRL_EXT_DRV_LOAD; 3716 IXGBE_WRITE_REG(&adapter->hw, IXGBE_CTRL_EXT, ctrl_ext); 3717 3718 if (adapter->feat_en & IXGBE_FEATURE_NETMAP) 3719 netmap_detach(adapter->ifp); 3720 3721 ixgbe_free_pci_resources(adapter); 3722 #if 0 /* XXX the NetBSD port is probably missing something here */ 3723 bus_generic_detach(dev); 3724 #endif 3725 if_detach(adapter->ifp); 3726 ifmedia_fini(&adapter->media); 3727 if_percpuq_destroy(adapter->ipq); 3728 3729 sysctl_teardown(&adapter->sysctllog); 3730 evcnt_detach(&adapter->efbig_tx_dma_setup); 3731 evcnt_detach(&adapter->mbuf_defrag_failed); 3732 evcnt_detach(&adapter->efbig2_tx_dma_setup); 3733 evcnt_detach(&adapter->einval_tx_dma_setup); 3734 evcnt_detach(&adapter->other_tx_dma_setup); 3735 evcnt_detach(&adapter->eagain_tx_dma_setup); 3736 evcnt_detach(&adapter->enomem_tx_dma_setup); 3737 evcnt_detach(&adapter->watchdog_events); 3738 evcnt_detach(&adapter->tso_err); 3739 evcnt_detach(&adapter->admin_irqev); 3740 evcnt_detach(&adapter->link_workev); 3741 evcnt_detach(&adapter->mod_workev); 3742 evcnt_detach(&adapter->msf_workev); 3743 evcnt_detach(&adapter->phy_workev); 3744 3745 for (i = 0; i < IXGBE_TC_COUNTER_NUM; i++) { 3746 if (i < __arraycount(stats->mpc)) { 3747 evcnt_detach(&stats->mpc[i]); 3748 if (hw->mac.type == ixgbe_mac_82598EB) 3749 evcnt_detach(&stats->rnbc[i]); 3750 } 3751 if (i < __arraycount(stats->pxontxc)) { 3752 evcnt_detach(&stats->pxontxc[i]); 3753 evcnt_detach(&stats->pxonrxc[i]); 3754 evcnt_detach(&stats->pxofftxc[i]); 3755 evcnt_detach(&stats->pxoffrxc[i]); 3756 if (hw->mac.type >= ixgbe_mac_82599EB) 3757 evcnt_detach(&stats->pxon2offc[i]); 3758 } 3759 } 3760 3761 txr = adapter->tx_rings; 3762 for (i = 0; i < adapter->num_queues; i++, rxr++, txr++) { 3763 evcnt_detach(&adapter->queues[i].irqs); 3764 evcnt_detach(&adapter->queues[i].handleq); 3765 evcnt_detach(&adapter->queues[i].req); 3766 evcnt_detach(&txr->no_desc_avail); 3767 evcnt_detach(&txr->total_packets); 3768 evcnt_detach(&txr->tso_tx); 3769 #ifndef IXGBE_LEGACY_TX 3770 evcnt_detach(&txr->pcq_drops); 3771 #endif 3772 3773 if (i < __arraycount(stats->qprc)) { 3774 evcnt_detach(&stats->qprc[i]); 3775 evcnt_detach(&stats->qptc[i]); 3776 evcnt_detach(&stats->qbrc[i]); 3777 evcnt_detach(&stats->qbtc[i]); 3778 if (hw->mac.type >= ixgbe_mac_82599EB) 3779 evcnt_detach(&stats->qprdc[i]); 3780 } 3781 3782 evcnt_detach(&rxr->rx_packets); 3783 evcnt_detach(&rxr->rx_bytes); 3784 evcnt_detach(&rxr->rx_copies); 3785 evcnt_detach(&rxr->no_mbuf); 3786 evcnt_detach(&rxr->rx_discarded); 3787 } 3788 evcnt_detach(&stats->ipcs); 3789 evcnt_detach(&stats->l4cs); 3790 evcnt_detach(&stats->ipcs_bad); 3791 evcnt_detach(&stats->l4cs_bad); 3792 evcnt_detach(&stats->intzero); 3793 evcnt_detach(&stats->legint); 3794 evcnt_detach(&stats->crcerrs); 3795 evcnt_detach(&stats->illerrc); 3796 evcnt_detach(&stats->errbc); 3797 evcnt_detach(&stats->mspdc); 3798 if (hw->mac.type >= ixgbe_mac_X550) 3799 evcnt_detach(&stats->mbsdc); 3800 evcnt_detach(&stats->mpctotal); 3801 evcnt_detach(&stats->mlfc); 3802 evcnt_detach(&stats->mrfc); 3803 evcnt_detach(&stats->rlec); 3804 evcnt_detach(&stats->lxontxc); 3805 evcnt_detach(&stats->lxonrxc); 3806 evcnt_detach(&stats->lxofftxc); 3807 evcnt_detach(&stats->lxoffrxc); 3808 3809 /* Packet Reception Stats */ 3810 evcnt_detach(&stats->tor); 3811 evcnt_detach(&stats->gorc); 3812 evcnt_detach(&stats->tpr); 3813 evcnt_detach(&stats->gprc); 3814 evcnt_detach(&stats->mprc); 3815 evcnt_detach(&stats->bprc); 3816 evcnt_detach(&stats->prc64); 3817 evcnt_detach(&stats->prc127); 3818 evcnt_detach(&stats->prc255); 3819 evcnt_detach(&stats->prc511); 3820 evcnt_detach(&stats->prc1023); 3821 evcnt_detach(&stats->prc1522); 3822 evcnt_detach(&stats->ruc); 3823 evcnt_detach(&stats->rfc); 3824 evcnt_detach(&stats->roc); 3825 evcnt_detach(&stats->rjc); 3826 evcnt_detach(&stats->mngprc); 3827 evcnt_detach(&stats->mngpdc); 3828 evcnt_detach(&stats->xec); 3829 3830 /* Packet Transmission Stats */ 3831 evcnt_detach(&stats->gotc); 3832 evcnt_detach(&stats->tpt); 3833 evcnt_detach(&stats->gptc); 3834 evcnt_detach(&stats->bptc); 3835 evcnt_detach(&stats->mptc); 3836 evcnt_detach(&stats->mngptc); 3837 evcnt_detach(&stats->ptc64); 3838 evcnt_detach(&stats->ptc127); 3839 evcnt_detach(&stats->ptc255); 3840 evcnt_detach(&stats->ptc511); 3841 evcnt_detach(&stats->ptc1023); 3842 evcnt_detach(&stats->ptc1522); 3843 3844 ixgbe_free_queues(adapter); 3845 free(adapter->mta, M_DEVBUF); 3846 3847 mutex_destroy(&adapter->admin_mtx); /* XXX appropriate order? */ 3848 IXGBE_CORE_LOCK_DESTROY(adapter); 3849 3850 return (0); 3851 } /* ixgbe_detach */ 3852 3853 /************************************************************************ 3854 * ixgbe_setup_low_power_mode - LPLU/WoL preparation 3855 * 3856 * Prepare the adapter/port for LPLU and/or WoL 3857 ************************************************************************/ 3858 static int 3859 ixgbe_setup_low_power_mode(struct adapter *adapter) 3860 { 3861 struct ixgbe_hw *hw = &adapter->hw; 3862 device_t dev = adapter->dev; 3863 struct ifnet *ifp = adapter->ifp; 3864 s32 error = 0; 3865 3866 /* Limit power management flow to X550EM baseT */ 3867 if (hw->device_id == IXGBE_DEV_ID_X550EM_X_10G_T && 3868 hw->phy.ops.enter_lplu) { 3869 /* X550EM baseT adapters need a special LPLU flow */ 3870 hw->phy.reset_disable = true; 3871 ixgbe_ifstop(ifp, 1); 3872 error = hw->phy.ops.enter_lplu(hw); 3873 if (error) 3874 device_printf(dev, 3875 "Error entering LPLU: %d\n", error); 3876 hw->phy.reset_disable = false; 3877 } else { 3878 /* Just stop for other adapters */ 3879 ixgbe_ifstop(ifp, 1); 3880 } 3881 3882 IXGBE_CORE_LOCK(adapter); 3883 3884 if (!hw->wol_enabled) { 3885 ixgbe_set_phy_power(hw, FALSE); 3886 IXGBE_WRITE_REG(hw, IXGBE_WUFC, 0); 3887 IXGBE_WRITE_REG(hw, IXGBE_WUC, 0); 3888 } else { 3889 /* Turn off support for APM wakeup. (Using ACPI instead) */ 3890 IXGBE_WRITE_REG(hw, IXGBE_GRC_BY_MAC(hw), 3891 IXGBE_READ_REG(hw, IXGBE_GRC_BY_MAC(hw)) & ~(u32)2); 3892 3893 /* 3894 * Clear Wake Up Status register to prevent any previous wakeup 3895 * events from waking us up immediately after we suspend. 3896 */ 3897 IXGBE_WRITE_REG(hw, IXGBE_WUS, 0xffffffff); 3898 3899 /* 3900 * Program the Wakeup Filter Control register with user filter 3901 * settings 3902 */ 3903 IXGBE_WRITE_REG(hw, IXGBE_WUFC, adapter->wufc); 3904 3905 /* Enable wakeups and power management in Wakeup Control */ 3906 IXGBE_WRITE_REG(hw, IXGBE_WUC, 3907 IXGBE_WUC_WKEN | IXGBE_WUC_PME_EN); 3908 } 3909 3910 IXGBE_CORE_UNLOCK(adapter); 3911 3912 return error; 3913 } /* ixgbe_setup_low_power_mode */ 3914 3915 /************************************************************************ 3916 * ixgbe_shutdown - Shutdown entry point 3917 ************************************************************************/ 3918 #if 0 /* XXX NetBSD ought to register something like this through pmf(9) */ 3919 static int 3920 ixgbe_shutdown(device_t dev) 3921 { 3922 struct adapter *adapter = device_private(dev); 3923 int error = 0; 3924 3925 INIT_DEBUGOUT("ixgbe_shutdown: begin"); 3926 3927 error = ixgbe_setup_low_power_mode(adapter); 3928 3929 return (error); 3930 } /* ixgbe_shutdown */ 3931 #endif 3932 3933 /************************************************************************ 3934 * ixgbe_suspend 3935 * 3936 * From D0 to D3 3937 ************************************************************************/ 3938 static bool 3939 ixgbe_suspend(device_t dev, const pmf_qual_t *qual) 3940 { 3941 struct adapter *adapter = device_private(dev); 3942 int error = 0; 3943 3944 INIT_DEBUGOUT("ixgbe_suspend: begin"); 3945 3946 error = ixgbe_setup_low_power_mode(adapter); 3947 3948 return (error); 3949 } /* ixgbe_suspend */ 3950 3951 /************************************************************************ 3952 * ixgbe_resume 3953 * 3954 * From D3 to D0 3955 ************************************************************************/ 3956 static bool 3957 ixgbe_resume(device_t dev, const pmf_qual_t *qual) 3958 { 3959 struct adapter *adapter = device_private(dev); 3960 struct ifnet *ifp = adapter->ifp; 3961 struct ixgbe_hw *hw = &adapter->hw; 3962 u32 wus; 3963 3964 INIT_DEBUGOUT("ixgbe_resume: begin"); 3965 3966 IXGBE_CORE_LOCK(adapter); 3967 3968 /* Read & clear WUS register */ 3969 wus = IXGBE_READ_REG(hw, IXGBE_WUS); 3970 if (wus) 3971 device_printf(dev, "Woken up by (WUS): %#010x\n", 3972 IXGBE_READ_REG(hw, IXGBE_WUS)); 3973 IXGBE_WRITE_REG(hw, IXGBE_WUS, 0xffffffff); 3974 /* And clear WUFC until next low-power transition */ 3975 IXGBE_WRITE_REG(hw, IXGBE_WUFC, 0); 3976 3977 /* 3978 * Required after D3->D0 transition; 3979 * will re-advertise all previous advertised speeds 3980 */ 3981 if (ifp->if_flags & IFF_UP) 3982 ixgbe_init_locked(adapter); 3983 3984 IXGBE_CORE_UNLOCK(adapter); 3985 3986 return true; 3987 } /* ixgbe_resume */ 3988 3989 /* 3990 * Set the various hardware offload abilities. 3991 * 3992 * This takes the ifnet's if_capenable flags (e.g. set by the user using 3993 * ifconfig) and indicates to the OS via the ifnet's if_hwassist field what 3994 * mbuf offload flags the driver will understand. 3995 */ 3996 static void 3997 ixgbe_set_if_hwassist(struct adapter *adapter) 3998 { 3999 /* XXX */ 4000 } 4001 4002 /************************************************************************ 4003 * ixgbe_init_locked - Init entry point 4004 * 4005 * Used in two ways: It is used by the stack as an init 4006 * entry point in network interface structure. It is also 4007 * used by the driver as a hw/sw initialization routine to 4008 * get to a consistent state. 4009 * 4010 * return 0 on success, positive on failure 4011 ************************************************************************/ 4012 static void 4013 ixgbe_init_locked(struct adapter *adapter) 4014 { 4015 struct ifnet *ifp = adapter->ifp; 4016 device_t dev = adapter->dev; 4017 struct ixgbe_hw *hw = &adapter->hw; 4018 struct ix_queue *que; 4019 struct tx_ring *txr; 4020 struct rx_ring *rxr; 4021 u32 txdctl, mhadd; 4022 u32 rxdctl, rxctrl; 4023 u32 ctrl_ext; 4024 bool unsupported_sfp = false; 4025 int i, j, error; 4026 4027 /* XXX check IFF_UP and IFF_RUNNING, power-saving state! */ 4028 4029 KASSERT(mutex_owned(&adapter->core_mtx)); 4030 INIT_DEBUGOUT("ixgbe_init_locked: begin"); 4031 4032 hw->need_unsupported_sfp_recovery = false; 4033 hw->adapter_stopped = FALSE; 4034 ixgbe_stop_adapter(hw); 4035 callout_stop(&adapter->timer); 4036 if (adapter->feat_en & IXGBE_FEATURE_RECOVERY_MODE) 4037 callout_stop(&adapter->recovery_mode_timer); 4038 for (i = 0, que = adapter->queues; i < adapter->num_queues; i++, que++) 4039 que->disabled_count = 0; 4040 4041 /* XXX I moved this here from the SIOCSIFMTU case in ixgbe_ioctl(). */ 4042 adapter->max_frame_size = 4043 ifp->if_mtu + ETHER_HDR_LEN + ETHER_CRC_LEN; 4044 4045 /* Queue indices may change with IOV mode */ 4046 ixgbe_align_all_queue_indices(adapter); 4047 4048 /* reprogram the RAR[0] in case user changed it. */ 4049 ixgbe_set_rar(hw, 0, hw->mac.addr, adapter->pool, IXGBE_RAH_AV); 4050 4051 /* Get the latest mac address, User can use a LAA */ 4052 memcpy(hw->mac.addr, CLLADDR(ifp->if_sadl), 4053 IXGBE_ETH_LENGTH_OF_ADDRESS); 4054 ixgbe_set_rar(hw, 0, hw->mac.addr, adapter->pool, 1); 4055 hw->addr_ctrl.rar_used_count = 1; 4056 4057 /* Set hardware offload abilities from ifnet flags */ 4058 ixgbe_set_if_hwassist(adapter); 4059 4060 /* Prepare transmit descriptors and buffers */ 4061 if (ixgbe_setup_transmit_structures(adapter)) { 4062 device_printf(dev, "Could not setup transmit structures\n"); 4063 ixgbe_stop_locked(adapter); 4064 return; 4065 } 4066 4067 ixgbe_init_hw(hw); 4068 4069 ixgbe_initialize_iov(adapter); 4070 4071 ixgbe_initialize_transmit_units(adapter); 4072 4073 /* Setup Multicast table */ 4074 ixgbe_set_rxfilter(adapter); 4075 4076 /* Use fixed buffer size, even for jumbo frames */ 4077 adapter->rx_mbuf_sz = MCLBYTES; 4078 4079 /* Prepare receive descriptors and buffers */ 4080 error = ixgbe_setup_receive_structures(adapter); 4081 if (error) { 4082 device_printf(dev, 4083 "Could not setup receive structures (err = %d)\n", error); 4084 ixgbe_stop_locked(adapter); 4085 return; 4086 } 4087 4088 /* Configure RX settings */ 4089 ixgbe_initialize_receive_units(adapter); 4090 4091 /* Initialize variable holding task enqueue requests interrupts */ 4092 adapter->task_requests = 0; 4093 4094 /* Enable SDP & MSI-X interrupts based on adapter */ 4095 ixgbe_config_gpie(adapter); 4096 4097 /* Set MTU size */ 4098 if (ifp->if_mtu > ETHERMTU) { 4099 /* aka IXGBE_MAXFRS on 82599 and newer */ 4100 mhadd = IXGBE_READ_REG(hw, IXGBE_MHADD); 4101 mhadd &= ~IXGBE_MHADD_MFS_MASK; 4102 mhadd |= adapter->max_frame_size << IXGBE_MHADD_MFS_SHIFT; 4103 IXGBE_WRITE_REG(hw, IXGBE_MHADD, mhadd); 4104 } 4105 4106 /* Now enable all the queues */ 4107 for (i = 0; i < adapter->num_queues; i++) { 4108 txr = &adapter->tx_rings[i]; 4109 txdctl = IXGBE_READ_REG(hw, IXGBE_TXDCTL(txr->me)); 4110 txdctl |= IXGBE_TXDCTL_ENABLE; 4111 /* Set WTHRESH to 8, burst writeback */ 4112 txdctl |= IXGBE_TX_WTHRESH << IXGBE_TXDCTL_WTHRESH_SHIFT; 4113 /* 4114 * When the internal queue falls below PTHRESH (32), 4115 * start prefetching as long as there are at least 4116 * HTHRESH (1) buffers ready. The values are taken 4117 * from the Intel linux driver 3.8.21. 4118 * Prefetching enables tx line rate even with 1 queue. 4119 */ 4120 txdctl |= (32 << 0) | (1 << 8); 4121 IXGBE_WRITE_REG(hw, IXGBE_TXDCTL(txr->me), txdctl); 4122 } 4123 4124 for (i = 0; i < adapter->num_queues; i++) { 4125 rxr = &adapter->rx_rings[i]; 4126 rxdctl = IXGBE_READ_REG(hw, IXGBE_RXDCTL(rxr->me)); 4127 if (hw->mac.type == ixgbe_mac_82598EB) { 4128 /* 4129 * PTHRESH = 21 4130 * HTHRESH = 4 4131 * WTHRESH = 8 4132 */ 4133 rxdctl &= ~0x3FFFFF; 4134 rxdctl |= 0x080420; 4135 } 4136 rxdctl |= IXGBE_RXDCTL_ENABLE; 4137 IXGBE_WRITE_REG(hw, IXGBE_RXDCTL(rxr->me), rxdctl); 4138 for (j = 0; j < 10; j++) { 4139 if (IXGBE_READ_REG(hw, IXGBE_RXDCTL(rxr->me)) & 4140 IXGBE_RXDCTL_ENABLE) 4141 break; 4142 else 4143 msec_delay(1); 4144 } 4145 IXGBE_WRITE_BARRIER(hw); 4146 4147 /* 4148 * In netmap mode, we must preserve the buffers made 4149 * available to userspace before the if_init() 4150 * (this is true by default on the TX side, because 4151 * init makes all buffers available to userspace). 4152 * 4153 * netmap_reset() and the device specific routines 4154 * (e.g. ixgbe_setup_receive_rings()) map these 4155 * buffers at the end of the NIC ring, so here we 4156 * must set the RDT (tail) register to make sure 4157 * they are not overwritten. 4158 * 4159 * In this driver the NIC ring starts at RDH = 0, 4160 * RDT points to the last slot available for reception (?), 4161 * so RDT = num_rx_desc - 1 means the whole ring is available. 4162 */ 4163 #ifdef DEV_NETMAP 4164 if ((adapter->feat_en & IXGBE_FEATURE_NETMAP) && 4165 (ifp->if_capenable & IFCAP_NETMAP)) { 4166 struct netmap_adapter *na = NA(adapter->ifp); 4167 struct netmap_kring *kring = na->rx_rings[i]; 4168 int t = na->num_rx_desc - 1 - nm_kr_rxspace(kring); 4169 4170 IXGBE_WRITE_REG(hw, IXGBE_RDT(rxr->me), t); 4171 } else 4172 #endif /* DEV_NETMAP */ 4173 IXGBE_WRITE_REG(hw, IXGBE_RDT(rxr->me), 4174 adapter->num_rx_desc - 1); 4175 } 4176 4177 /* Enable Receive engine */ 4178 rxctrl = IXGBE_READ_REG(hw, IXGBE_RXCTRL); 4179 if (hw->mac.type == ixgbe_mac_82598EB) 4180 rxctrl |= IXGBE_RXCTRL_DMBYPS; 4181 rxctrl |= IXGBE_RXCTRL_RXEN; 4182 ixgbe_enable_rx_dma(hw, rxctrl); 4183 4184 callout_reset(&adapter->timer, hz, ixgbe_local_timer, adapter); 4185 atomic_store_relaxed(&adapter->timer_pending, 0); 4186 if (adapter->feat_en & IXGBE_FEATURE_RECOVERY_MODE) 4187 callout_reset(&adapter->recovery_mode_timer, hz, 4188 ixgbe_recovery_mode_timer, adapter); 4189 4190 /* Set up MSI/MSI-X routing */ 4191 if (adapter->feat_en & IXGBE_FEATURE_MSIX) { 4192 ixgbe_configure_ivars(adapter); 4193 /* Set up auto-mask */ 4194 if (hw->mac.type == ixgbe_mac_82598EB) 4195 IXGBE_WRITE_REG(hw, IXGBE_EIAM, IXGBE_EICS_RTX_QUEUE); 4196 else { 4197 IXGBE_WRITE_REG(hw, IXGBE_EIAM_EX(0), 0xFFFFFFFF); 4198 IXGBE_WRITE_REG(hw, IXGBE_EIAM_EX(1), 0xFFFFFFFF); 4199 } 4200 } else { /* Simple settings for Legacy/MSI */ 4201 ixgbe_set_ivar(adapter, 0, 0, 0); 4202 ixgbe_set_ivar(adapter, 0, 0, 1); 4203 IXGBE_WRITE_REG(hw, IXGBE_EIAM, IXGBE_EICS_RTX_QUEUE); 4204 } 4205 4206 ixgbe_init_fdir(adapter); 4207 4208 /* 4209 * Check on any SFP devices that 4210 * need to be kick-started 4211 */ 4212 if (hw->phy.type == ixgbe_phy_none) { 4213 error = hw->phy.ops.identify(hw); 4214 if (error == IXGBE_ERR_SFP_NOT_SUPPORTED) 4215 unsupported_sfp = true; 4216 } else if (hw->phy.type == ixgbe_phy_sfp_unsupported) 4217 unsupported_sfp = true; 4218 4219 if (unsupported_sfp) 4220 device_printf(dev, 4221 "Unsupported SFP+ module type was detected.\n"); 4222 4223 /* Set moderation on the Link interrupt */ 4224 ixgbe_eitr_write(adapter, adapter->vector, IXGBE_LINK_ITR); 4225 4226 /* Enable EEE power saving */ 4227 if (adapter->feat_cap & IXGBE_FEATURE_EEE) 4228 hw->mac.ops.setup_eee(hw, 4229 adapter->feat_en & IXGBE_FEATURE_EEE); 4230 4231 /* Enable power to the phy. */ 4232 if (!unsupported_sfp) { 4233 ixgbe_set_phy_power(hw, TRUE); 4234 4235 /* Config/Enable Link */ 4236 ixgbe_config_link(adapter); 4237 } 4238 4239 /* Hardware Packet Buffer & Flow Control setup */ 4240 ixgbe_config_delay_values(adapter); 4241 4242 /* Initialize the FC settings */ 4243 ixgbe_start_hw(hw); 4244 4245 /* Set up VLAN support and filter */ 4246 ixgbe_setup_vlan_hw_support(adapter); 4247 4248 /* Setup DMA Coalescing */ 4249 ixgbe_config_dmac(adapter); 4250 4251 /* OK to schedule workqueues. */ 4252 adapter->schedule_wqs_ok = true; 4253 4254 /* And now turn on interrupts */ 4255 ixgbe_enable_intr(adapter); 4256 4257 /* Enable the use of the MBX by the VF's */ 4258 if (adapter->feat_en & IXGBE_FEATURE_SRIOV) { 4259 ctrl_ext = IXGBE_READ_REG(hw, IXGBE_CTRL_EXT); 4260 ctrl_ext |= IXGBE_CTRL_EXT_PFRSTD; 4261 IXGBE_WRITE_REG(hw, IXGBE_CTRL_EXT, ctrl_ext); 4262 } 4263 4264 /* Update saved flags. See ixgbe_ifflags_cb() */ 4265 adapter->if_flags = ifp->if_flags; 4266 adapter->ec_capenable = adapter->osdep.ec.ec_capenable; 4267 4268 /* Now inform the stack we're ready */ 4269 ifp->if_flags |= IFF_RUNNING; 4270 4271 return; 4272 } /* ixgbe_init_locked */ 4273 4274 /************************************************************************ 4275 * ixgbe_init 4276 ************************************************************************/ 4277 static int 4278 ixgbe_init(struct ifnet *ifp) 4279 { 4280 struct adapter *adapter = ifp->if_softc; 4281 4282 IXGBE_CORE_LOCK(adapter); 4283 ixgbe_init_locked(adapter); 4284 IXGBE_CORE_UNLOCK(adapter); 4285 4286 return 0; /* XXX ixgbe_init_locked cannot fail? really? */ 4287 } /* ixgbe_init */ 4288 4289 /************************************************************************ 4290 * ixgbe_set_ivar 4291 * 4292 * Setup the correct IVAR register for a particular MSI-X interrupt 4293 * (yes this is all very magic and confusing :) 4294 * - entry is the register array entry 4295 * - vector is the MSI-X vector for this queue 4296 * - type is RX/TX/MISC 4297 ************************************************************************/ 4298 static void 4299 ixgbe_set_ivar(struct adapter *adapter, u8 entry, u8 vector, s8 type) 4300 { 4301 struct ixgbe_hw *hw = &adapter->hw; 4302 u32 ivar, index; 4303 4304 vector |= IXGBE_IVAR_ALLOC_VAL; 4305 4306 switch (hw->mac.type) { 4307 case ixgbe_mac_82598EB: 4308 if (type == -1) 4309 entry = IXGBE_IVAR_OTHER_CAUSES_INDEX; 4310 else 4311 entry += (type * 64); 4312 index = (entry >> 2) & 0x1F; 4313 ivar = IXGBE_READ_REG(hw, IXGBE_IVAR(index)); 4314 ivar &= ~(0xffUL << (8 * (entry & 0x3))); 4315 ivar |= ((u32)vector << (8 * (entry & 0x3))); 4316 IXGBE_WRITE_REG(&adapter->hw, IXGBE_IVAR(index), ivar); 4317 break; 4318 case ixgbe_mac_82599EB: 4319 case ixgbe_mac_X540: 4320 case ixgbe_mac_X550: 4321 case ixgbe_mac_X550EM_x: 4322 case ixgbe_mac_X550EM_a: 4323 if (type == -1) { /* MISC IVAR */ 4324 index = (entry & 1) * 8; 4325 ivar = IXGBE_READ_REG(hw, IXGBE_IVAR_MISC); 4326 ivar &= ~(0xffUL << index); 4327 ivar |= ((u32)vector << index); 4328 IXGBE_WRITE_REG(hw, IXGBE_IVAR_MISC, ivar); 4329 } else { /* RX/TX IVARS */ 4330 index = (16 * (entry & 1)) + (8 * type); 4331 ivar = IXGBE_READ_REG(hw, IXGBE_IVAR(entry >> 1)); 4332 ivar &= ~(0xffUL << index); 4333 ivar |= ((u32)vector << index); 4334 IXGBE_WRITE_REG(hw, IXGBE_IVAR(entry >> 1), ivar); 4335 } 4336 break; 4337 default: 4338 break; 4339 } 4340 } /* ixgbe_set_ivar */ 4341 4342 /************************************************************************ 4343 * ixgbe_configure_ivars 4344 ************************************************************************/ 4345 static void 4346 ixgbe_configure_ivars(struct adapter *adapter) 4347 { 4348 struct ix_queue *que = adapter->queues; 4349 u32 newitr; 4350 4351 if (ixgbe_max_interrupt_rate > 0) 4352 newitr = (4000000 / ixgbe_max_interrupt_rate) & 0x0FF8; 4353 else { 4354 /* 4355 * Disable DMA coalescing if interrupt moderation is 4356 * disabled. 4357 */ 4358 adapter->dmac = 0; 4359 newitr = 0; 4360 } 4361 4362 for (int i = 0; i < adapter->num_queues; i++, que++) { 4363 struct rx_ring *rxr = &adapter->rx_rings[i]; 4364 struct tx_ring *txr = &adapter->tx_rings[i]; 4365 /* First the RX queue entry */ 4366 ixgbe_set_ivar(adapter, rxr->me, que->msix, 0); 4367 /* ... and the TX */ 4368 ixgbe_set_ivar(adapter, txr->me, que->msix, 1); 4369 /* Set an Initial EITR value */ 4370 ixgbe_eitr_write(adapter, que->msix, newitr); 4371 /* 4372 * To eliminate influence of the previous state. 4373 * At this point, Tx/Rx interrupt handler 4374 * (ixgbe_msix_que()) cannot be called, so both 4375 * IXGBE_TX_LOCK and IXGBE_RX_LOCK are not required. 4376 */ 4377 que->eitr_setting = 0; 4378 } 4379 4380 /* For the Link interrupt */ 4381 ixgbe_set_ivar(adapter, 1, adapter->vector, -1); 4382 } /* ixgbe_configure_ivars */ 4383 4384 /************************************************************************ 4385 * ixgbe_config_gpie 4386 ************************************************************************/ 4387 static void 4388 ixgbe_config_gpie(struct adapter *adapter) 4389 { 4390 struct ixgbe_hw *hw = &adapter->hw; 4391 u32 gpie; 4392 4393 gpie = IXGBE_READ_REG(hw, IXGBE_GPIE); 4394 4395 if (adapter->feat_en & IXGBE_FEATURE_MSIX) { 4396 /* Enable Enhanced MSI-X mode */ 4397 gpie |= IXGBE_GPIE_MSIX_MODE 4398 | IXGBE_GPIE_EIAME 4399 | IXGBE_GPIE_PBA_SUPPORT 4400 | IXGBE_GPIE_OCD; 4401 } 4402 4403 /* Fan Failure Interrupt */ 4404 if (adapter->feat_en & IXGBE_FEATURE_FAN_FAIL) 4405 gpie |= IXGBE_SDP1_GPIEN; 4406 4407 /* Thermal Sensor Interrupt */ 4408 if (adapter->feat_en & IXGBE_FEATURE_TEMP_SENSOR) 4409 gpie |= IXGBE_SDP0_GPIEN_X540; 4410 4411 /* Link detection */ 4412 switch (hw->mac.type) { 4413 case ixgbe_mac_82599EB: 4414 gpie |= IXGBE_SDP1_GPIEN | IXGBE_SDP2_GPIEN; 4415 break; 4416 case ixgbe_mac_X550EM_x: 4417 case ixgbe_mac_X550EM_a: 4418 gpie |= IXGBE_SDP0_GPIEN_X540; 4419 break; 4420 default: 4421 break; 4422 } 4423 4424 IXGBE_WRITE_REG(hw, IXGBE_GPIE, gpie); 4425 4426 } /* ixgbe_config_gpie */ 4427 4428 /************************************************************************ 4429 * ixgbe_config_delay_values 4430 * 4431 * Requires adapter->max_frame_size to be set. 4432 ************************************************************************/ 4433 static void 4434 ixgbe_config_delay_values(struct adapter *adapter) 4435 { 4436 struct ixgbe_hw *hw = &adapter->hw; 4437 u32 rxpb, frame, size, tmp; 4438 4439 frame = adapter->max_frame_size; 4440 4441 /* Calculate High Water */ 4442 switch (hw->mac.type) { 4443 case ixgbe_mac_X540: 4444 case ixgbe_mac_X550: 4445 case ixgbe_mac_X550EM_x: 4446 case ixgbe_mac_X550EM_a: 4447 tmp = IXGBE_DV_X540(frame, frame); 4448 break; 4449 default: 4450 tmp = IXGBE_DV(frame, frame); 4451 break; 4452 } 4453 size = IXGBE_BT2KB(tmp); 4454 rxpb = IXGBE_READ_REG(hw, IXGBE_RXPBSIZE(0)) >> 10; 4455 hw->fc.high_water[0] = rxpb - size; 4456 4457 /* Now calculate Low Water */ 4458 switch (hw->mac.type) { 4459 case ixgbe_mac_X540: 4460 case ixgbe_mac_X550: 4461 case ixgbe_mac_X550EM_x: 4462 case ixgbe_mac_X550EM_a: 4463 tmp = IXGBE_LOW_DV_X540(frame); 4464 break; 4465 default: 4466 tmp = IXGBE_LOW_DV(frame); 4467 break; 4468 } 4469 hw->fc.low_water[0] = IXGBE_BT2KB(tmp); 4470 4471 hw->fc.pause_time = IXGBE_FC_PAUSE; 4472 hw->fc.send_xon = TRUE; 4473 } /* ixgbe_config_delay_values */ 4474 4475 /************************************************************************ 4476 * ixgbe_set_rxfilter - Multicast Update 4477 * 4478 * Called whenever multicast address list is updated. 4479 ************************************************************************/ 4480 static void 4481 ixgbe_set_rxfilter(struct adapter *adapter) 4482 { 4483 struct ixgbe_mc_addr *mta; 4484 struct ifnet *ifp = adapter->ifp; 4485 u8 *update_ptr; 4486 int mcnt = 0; 4487 u32 fctrl; 4488 struct ethercom *ec = &adapter->osdep.ec; 4489 struct ether_multi *enm; 4490 struct ether_multistep step; 4491 4492 KASSERT(mutex_owned(&adapter->core_mtx)); 4493 IOCTL_DEBUGOUT("ixgbe_set_rxfilter: begin"); 4494 4495 mta = adapter->mta; 4496 bzero(mta, sizeof(*mta) * MAX_NUM_MULTICAST_ADDRESSES); 4497 4498 ETHER_LOCK(ec); 4499 ec->ec_flags &= ~ETHER_F_ALLMULTI; 4500 ETHER_FIRST_MULTI(step, ec, enm); 4501 while (enm != NULL) { 4502 if ((mcnt == MAX_NUM_MULTICAST_ADDRESSES) || 4503 (memcmp(enm->enm_addrlo, enm->enm_addrhi, 4504 ETHER_ADDR_LEN) != 0)) { 4505 ec->ec_flags |= ETHER_F_ALLMULTI; 4506 break; 4507 } 4508 bcopy(enm->enm_addrlo, 4509 mta[mcnt].addr, IXGBE_ETH_LENGTH_OF_ADDRESS); 4510 mta[mcnt].vmdq = adapter->pool; 4511 mcnt++; 4512 ETHER_NEXT_MULTI(step, enm); 4513 } 4514 4515 fctrl = IXGBE_READ_REG(&adapter->hw, IXGBE_FCTRL); 4516 if (ifp->if_flags & IFF_PROMISC) 4517 fctrl |= (IXGBE_FCTRL_UPE | IXGBE_FCTRL_MPE); 4518 else if (ec->ec_flags & ETHER_F_ALLMULTI) { 4519 fctrl |= IXGBE_FCTRL_MPE; 4520 fctrl &= ~IXGBE_FCTRL_UPE; 4521 } else 4522 fctrl &= ~(IXGBE_FCTRL_UPE | IXGBE_FCTRL_MPE); 4523 4524 IXGBE_WRITE_REG(&adapter->hw, IXGBE_FCTRL, fctrl); 4525 4526 /* Update multicast filter entries only when it's not ALLMULTI */ 4527 if ((ec->ec_flags & ETHER_F_ALLMULTI) == 0) { 4528 ETHER_UNLOCK(ec); 4529 update_ptr = (u8 *)mta; 4530 ixgbe_update_mc_addr_list(&adapter->hw, update_ptr, mcnt, 4531 ixgbe_mc_array_itr, TRUE); 4532 } else 4533 ETHER_UNLOCK(ec); 4534 } /* ixgbe_set_rxfilter */ 4535 4536 /************************************************************************ 4537 * ixgbe_mc_array_itr 4538 * 4539 * An iterator function needed by the multicast shared code. 4540 * It feeds the shared code routine the addresses in the 4541 * array of ixgbe_set_rxfilter() one by one. 4542 ************************************************************************/ 4543 static u8 * 4544 ixgbe_mc_array_itr(struct ixgbe_hw *hw, u8 **update_ptr, u32 *vmdq) 4545 { 4546 struct ixgbe_mc_addr *mta; 4547 4548 mta = (struct ixgbe_mc_addr *)*update_ptr; 4549 *vmdq = mta->vmdq; 4550 4551 *update_ptr = (u8*)(mta + 1); 4552 4553 return (mta->addr); 4554 } /* ixgbe_mc_array_itr */ 4555 4556 /************************************************************************ 4557 * ixgbe_local_timer - Timer routine 4558 * 4559 * Checks for link status, updates statistics, 4560 * and runs the watchdog check. 4561 ************************************************************************/ 4562 static void 4563 ixgbe_local_timer(void *arg) 4564 { 4565 struct adapter *adapter = arg; 4566 4567 if (adapter->schedule_wqs_ok) { 4568 if (atomic_cas_uint(&adapter->timer_pending, 0, 1) == 0) 4569 workqueue_enqueue(adapter->timer_wq, 4570 &adapter->timer_wc, NULL); 4571 } 4572 } 4573 4574 static void 4575 ixgbe_handle_timer(struct work *wk, void *context) 4576 { 4577 struct adapter *adapter = context; 4578 struct ixgbe_hw *hw = &adapter->hw; 4579 device_t dev = adapter->dev; 4580 struct ix_queue *que = adapter->queues; 4581 u64 queues = 0; 4582 u64 v0, v1, v2, v3, v4, v5, v6, v7; 4583 int hung = 0; 4584 int i; 4585 4586 IXGBE_CORE_LOCK(adapter); 4587 4588 /* Check for pluggable optics */ 4589 if (ixgbe_is_sfp(hw)) { 4590 bool sched_mod_task = false; 4591 4592 if (hw->mac.type == ixgbe_mac_82598EB) { 4593 /* 4594 * On 82598EB, SFP+'s MOD_ABS pin is not connected to 4595 * any GPIO(SDP). So just schedule TASK_MOD. 4596 */ 4597 sched_mod_task = true; 4598 } else { 4599 bool was_full, is_full; 4600 4601 was_full = 4602 hw->phy.sfp_type != ixgbe_sfp_type_not_present; 4603 is_full = ixgbe_sfp_cage_full(hw); 4604 4605 /* Do probe if cage state changed */ 4606 if (was_full ^ is_full) 4607 sched_mod_task = true; 4608 } 4609 if (sched_mod_task) { 4610 mutex_enter(&adapter->admin_mtx); 4611 adapter->task_requests |= IXGBE_REQUEST_TASK_MOD_WOI; 4612 ixgbe_schedule_admin_tasklet(adapter); 4613 mutex_exit(&adapter->admin_mtx); 4614 } 4615 } 4616 4617 ixgbe_update_link_status(adapter); 4618 ixgbe_update_stats_counters(adapter); 4619 4620 /* Update some event counters */ 4621 v0 = v1 = v2 = v3 = v4 = v5 = v6 = v7 = 0; 4622 que = adapter->queues; 4623 for (i = 0; i < adapter->num_queues; i++, que++) { 4624 struct tx_ring *txr = que->txr; 4625 4626 v0 += txr->q_efbig_tx_dma_setup; 4627 v1 += txr->q_mbuf_defrag_failed; 4628 v2 += txr->q_efbig2_tx_dma_setup; 4629 v3 += txr->q_einval_tx_dma_setup; 4630 v4 += txr->q_other_tx_dma_setup; 4631 v5 += txr->q_eagain_tx_dma_setup; 4632 v6 += txr->q_enomem_tx_dma_setup; 4633 v7 += txr->q_tso_err; 4634 } 4635 IXGBE_EVC_STORE(&adapter->efbig_tx_dma_setup, v0); 4636 IXGBE_EVC_STORE(&adapter->mbuf_defrag_failed, v1); 4637 IXGBE_EVC_STORE(&adapter->efbig2_tx_dma_setup, v2); 4638 IXGBE_EVC_STORE(&adapter->einval_tx_dma_setup, v3); 4639 IXGBE_EVC_STORE(&adapter->other_tx_dma_setup, v4); 4640 IXGBE_EVC_STORE(&adapter->eagain_tx_dma_setup, v5); 4641 IXGBE_EVC_STORE(&adapter->enomem_tx_dma_setup, v6); 4642 IXGBE_EVC_STORE(&adapter->tso_err, v7); 4643 4644 /* 4645 * Check the TX queues status 4646 * - mark hung queues so we don't schedule on them 4647 * - watchdog only if all queues show hung 4648 */ 4649 que = adapter->queues; 4650 for (i = 0; i < adapter->num_queues; i++, que++) { 4651 /* Keep track of queues with work for soft irq */ 4652 if (que->txr->busy) 4653 queues |= 1ULL << que->me; 4654 /* 4655 * Each time txeof runs without cleaning, but there 4656 * are uncleaned descriptors it increments busy. If 4657 * we get to the MAX we declare it hung. 4658 */ 4659 if (que->busy == IXGBE_QUEUE_HUNG) { 4660 ++hung; 4661 /* Mark the queue as inactive */ 4662 adapter->active_queues &= ~(1ULL << que->me); 4663 continue; 4664 } else { 4665 /* Check if we've come back from hung */ 4666 if ((adapter->active_queues & (1ULL << que->me)) == 0) 4667 adapter->active_queues |= 1ULL << que->me; 4668 } 4669 if (que->busy >= IXGBE_MAX_TX_BUSY) { 4670 device_printf(dev, 4671 "Warning queue %d appears to be hung!\n", i); 4672 que->txr->busy = IXGBE_QUEUE_HUNG; 4673 ++hung; 4674 } 4675 } 4676 4677 /* Only truly watchdog if all queues show hung */ 4678 if (hung == adapter->num_queues) 4679 goto watchdog; 4680 #if 0 /* XXX Avoid unexpectedly disabling interrupt forever (PR#53294) */ 4681 else if (queues != 0) { /* Force an IRQ on queues with work */ 4682 que = adapter->queues; 4683 for (i = 0; i < adapter->num_queues; i++, que++) { 4684 mutex_enter(&que->dc_mtx); 4685 if (que->disabled_count == 0) 4686 ixgbe_rearm_queues(adapter, 4687 queues & ((u64)1 << i)); 4688 mutex_exit(&que->dc_mtx); 4689 } 4690 } 4691 #endif 4692 4693 atomic_store_relaxed(&adapter->timer_pending, 0); 4694 IXGBE_CORE_UNLOCK(adapter); 4695 callout_reset(&adapter->timer, hz, ixgbe_local_timer, adapter); 4696 return; 4697 4698 watchdog: 4699 device_printf(adapter->dev, "Watchdog timeout -- resetting\n"); 4700 adapter->ifp->if_flags &= ~IFF_RUNNING; 4701 IXGBE_EVC_ADD(&adapter->watchdog_events, 1); 4702 ixgbe_init_locked(adapter); 4703 IXGBE_CORE_UNLOCK(adapter); 4704 } /* ixgbe_handle_timer */ 4705 4706 /************************************************************************ 4707 * ixgbe_recovery_mode_timer - Recovery mode timer routine 4708 ************************************************************************/ 4709 static void 4710 ixgbe_recovery_mode_timer(void *arg) 4711 { 4712 struct adapter *adapter = arg; 4713 4714 if (__predict_true(adapter->osdep.detaching == false)) { 4715 if (atomic_cas_uint(&adapter->recovery_mode_timer_pending, 4716 0, 1) == 0) { 4717 workqueue_enqueue(adapter->recovery_mode_timer_wq, 4718 &adapter->recovery_mode_timer_wc, NULL); 4719 } 4720 } 4721 } 4722 4723 static void 4724 ixgbe_handle_recovery_mode_timer(struct work *wk, void *context) 4725 { 4726 struct adapter *adapter = context; 4727 struct ixgbe_hw *hw = &adapter->hw; 4728 4729 IXGBE_CORE_LOCK(adapter); 4730 if (ixgbe_fw_recovery_mode(hw)) { 4731 if (atomic_cas_uint(&adapter->recovery_mode, 0, 1) == 0) { 4732 /* Firmware error detected, entering recovery mode */ 4733 device_printf(adapter->dev, 4734 "Firmware recovery mode detected. Limiting " 4735 "functionality. Refer to the Intel(R) Ethernet " 4736 "Adapters and Devices User Guide for details on " 4737 "firmware recovery mode.\n"); 4738 4739 if (hw->adapter_stopped == FALSE) 4740 ixgbe_stop_locked(adapter); 4741 } 4742 } else 4743 atomic_cas_uint(&adapter->recovery_mode, 1, 0); 4744 4745 atomic_store_relaxed(&adapter->recovery_mode_timer_pending, 0); 4746 callout_reset(&adapter->recovery_mode_timer, hz, 4747 ixgbe_recovery_mode_timer, adapter); 4748 IXGBE_CORE_UNLOCK(adapter); 4749 } /* ixgbe_handle_recovery_mode_timer */ 4750 4751 /************************************************************************ 4752 * ixgbe_handle_mod - Tasklet for SFP module interrupts 4753 * bool int_en: true if it's called when the interrupt is enabled. 4754 ************************************************************************/ 4755 static void 4756 ixgbe_handle_mod(void *context, bool int_en) 4757 { 4758 struct adapter *adapter = context; 4759 struct ixgbe_hw *hw = &adapter->hw; 4760 device_t dev = adapter->dev; 4761 enum ixgbe_sfp_type last_sfp_type; 4762 u32 err; 4763 bool last_unsupported_sfp_recovery; 4764 4765 KASSERT(mutex_owned(&adapter->core_mtx)); 4766 4767 last_sfp_type = hw->phy.sfp_type; 4768 last_unsupported_sfp_recovery = hw->need_unsupported_sfp_recovery; 4769 IXGBE_EVC_ADD(&adapter->mod_workev, 1); 4770 if (adapter->hw.need_crosstalk_fix) { 4771 if ((hw->mac.type != ixgbe_mac_82598EB) && 4772 !ixgbe_sfp_cage_full(hw)) 4773 goto out; 4774 } 4775 4776 err = hw->phy.ops.identify_sfp(hw); 4777 if (err == IXGBE_ERR_SFP_NOT_SUPPORTED) { 4778 if (last_unsupported_sfp_recovery == false) 4779 device_printf(dev, 4780 "Unsupported SFP+ module type was detected.\n"); 4781 goto out; 4782 } 4783 4784 if (hw->need_unsupported_sfp_recovery) { 4785 device_printf(dev, "Recovering from unsupported SFP\n"); 4786 /* 4787 * We could recover the status by calling setup_sfp(), 4788 * setup_link() and some others. It's complex and might not 4789 * work correctly on some unknown cases. To avoid such type of 4790 * problem, call ixgbe_init_locked(). It's simple and safe 4791 * approach. 4792 */ 4793 ixgbe_init_locked(adapter); 4794 } else if ((hw->phy.sfp_type != ixgbe_sfp_type_not_present) && 4795 (hw->phy.sfp_type != last_sfp_type)) { 4796 /* A module is inserted and changed. */ 4797 4798 if (hw->mac.type == ixgbe_mac_82598EB) 4799 err = hw->phy.ops.reset(hw); 4800 else { 4801 err = hw->mac.ops.setup_sfp(hw); 4802 hw->phy.sfp_setup_needed = FALSE; 4803 } 4804 if (err == IXGBE_ERR_SFP_NOT_SUPPORTED) { 4805 device_printf(dev, 4806 "Setup failure - unsupported SFP+ module type.\n"); 4807 goto out; 4808 } 4809 } 4810 4811 out: 4812 /* get_supported_phy_layer will call hw->phy.ops.identify_sfp() */ 4813 adapter->phy_layer = ixgbe_get_supported_physical_layer(hw); 4814 4815 /* Adjust media types shown in ifconfig */ 4816 IXGBE_CORE_UNLOCK(adapter); 4817 ifmedia_removeall(&adapter->media); 4818 ixgbe_add_media_types(adapter); 4819 ifmedia_set(&adapter->media, IFM_ETHER | IFM_AUTO); 4820 IXGBE_CORE_LOCK(adapter); 4821 4822 /* 4823 * Don't schedule MSF event if the chip is 82598. 82598 doesn't support 4824 * MSF. At least, calling ixgbe_handle_msf on 82598 DA makes the link 4825 * flap because the function calls setup_link(). 4826 */ 4827 if (hw->mac.type != ixgbe_mac_82598EB) { 4828 mutex_enter(&adapter->admin_mtx); 4829 if (int_en) 4830 adapter->task_requests |= IXGBE_REQUEST_TASK_MSF; 4831 else 4832 adapter->task_requests |= IXGBE_REQUEST_TASK_MSF_WOI; 4833 mutex_exit(&adapter->admin_mtx); 4834 } 4835 4836 /* 4837 * Don't call ixgbe_schedule_admin_tasklet() because we are on 4838 * the workqueue now. 4839 */ 4840 } /* ixgbe_handle_mod */ 4841 4842 4843 /************************************************************************ 4844 * ixgbe_handle_msf - Tasklet for MSF (multispeed fiber) interrupts 4845 ************************************************************************/ 4846 static void 4847 ixgbe_handle_msf(void *context) 4848 { 4849 struct adapter *adapter = context; 4850 struct ixgbe_hw *hw = &adapter->hw; 4851 u32 autoneg; 4852 bool negotiate; 4853 4854 KASSERT(mutex_owned(&adapter->core_mtx)); 4855 4856 IXGBE_EVC_ADD(&adapter->msf_workev, 1); 4857 4858 autoneg = hw->phy.autoneg_advertised; 4859 if ((!autoneg) && (hw->mac.ops.get_link_capabilities)) 4860 hw->mac.ops.get_link_capabilities(hw, &autoneg, &negotiate); 4861 if (hw->mac.ops.setup_link) 4862 hw->mac.ops.setup_link(hw, autoneg, TRUE); 4863 } /* ixgbe_handle_msf */ 4864 4865 /************************************************************************ 4866 * ixgbe_handle_phy - Tasklet for external PHY interrupts 4867 ************************************************************************/ 4868 static void 4869 ixgbe_handle_phy(void *context) 4870 { 4871 struct adapter *adapter = context; 4872 struct ixgbe_hw *hw = &adapter->hw; 4873 int error; 4874 4875 KASSERT(mutex_owned(&adapter->core_mtx)); 4876 4877 IXGBE_EVC_ADD(&adapter->phy_workev, 1); 4878 error = hw->phy.ops.handle_lasi(hw); 4879 if (error == IXGBE_ERR_OVERTEMP) 4880 device_printf(adapter->dev, 4881 "CRITICAL: EXTERNAL PHY OVER TEMP!! " 4882 " PHY will downshift to lower power state!\n"); 4883 else if (error) 4884 device_printf(adapter->dev, 4885 "Error handling LASI interrupt: %d\n", error); 4886 } /* ixgbe_handle_phy */ 4887 4888 static void 4889 ixgbe_handle_admin(struct work *wk, void *context) 4890 { 4891 struct adapter *adapter = context; 4892 struct ifnet *ifp = adapter->ifp; 4893 struct ixgbe_hw *hw = &adapter->hw; 4894 u32 task_requests; 4895 u32 eims_enable = 0; 4896 4897 mutex_enter(&adapter->admin_mtx); 4898 adapter->admin_pending = 0; 4899 task_requests = adapter->task_requests; 4900 adapter->task_requests = 0; 4901 mutex_exit(&adapter->admin_mtx); 4902 4903 /* 4904 * Hold the IFNET_LOCK across this entire call. This will 4905 * prevent additional changes to adapter->phy_layer 4906 * and serialize calls to this tasklet. We cannot hold the 4907 * CORE_LOCK while calling into the ifmedia functions as 4908 * they call ifmedia_lock() and the lock is CORE_LOCK. 4909 */ 4910 IFNET_LOCK(ifp); 4911 IXGBE_CORE_LOCK(adapter); 4912 if ((task_requests & IXGBE_REQUEST_TASK_LSC) != 0) { 4913 ixgbe_handle_link(adapter); 4914 eims_enable |= IXGBE_EIMS_LSC; 4915 } 4916 if ((task_requests & IXGBE_REQUEST_TASK_MOD_WOI) != 0) 4917 ixgbe_handle_mod(adapter, false); 4918 if ((task_requests & IXGBE_REQUEST_TASK_MOD) != 0) { 4919 ixgbe_handle_mod(adapter, true); 4920 if (hw->mac.type >= ixgbe_mac_X540) 4921 eims_enable |= IXGBE_EICR_GPI_SDP0_X540; 4922 else 4923 eims_enable |= IXGBE_EICR_GPI_SDP2_BY_MAC(hw); 4924 } 4925 if ((task_requests 4926 & (IXGBE_REQUEST_TASK_MSF_WOI | IXGBE_REQUEST_TASK_MSF)) != 0) { 4927 ixgbe_handle_msf(adapter); 4928 if (((task_requests & IXGBE_REQUEST_TASK_MSF) != 0) && 4929 (hw->mac.type == ixgbe_mac_82599EB)) 4930 eims_enable |= IXGBE_EIMS_GPI_SDP1_BY_MAC(hw); 4931 } 4932 if ((task_requests & IXGBE_REQUEST_TASK_PHY) != 0) { 4933 ixgbe_handle_phy(adapter); 4934 eims_enable |= IXGBE_EICR_GPI_SDP0_X540; 4935 } 4936 if ((task_requests & IXGBE_REQUEST_TASK_FDIR) != 0) { 4937 ixgbe_reinit_fdir(adapter); 4938 eims_enable |= IXGBE_EIMS_FLOW_DIR; 4939 } 4940 #if 0 /* notyet */ 4941 if ((task_requests & IXGBE_REQUEST_TASK_MBX) != 0) { 4942 ixgbe_handle_mbx(adapter); 4943 eims_enable |= IXGBE_EIMS_MAILBOX; 4944 } 4945 #endif 4946 IXGBE_WRITE_REG(hw, IXGBE_EIMS, eims_enable); 4947 4948 IXGBE_CORE_UNLOCK(adapter); 4949 IFNET_UNLOCK(ifp); 4950 } /* ixgbe_handle_admin */ 4951 4952 static void 4953 ixgbe_ifstop(struct ifnet *ifp, int disable) 4954 { 4955 struct adapter *adapter = ifp->if_softc; 4956 4957 IXGBE_CORE_LOCK(adapter); 4958 ixgbe_stop_locked(adapter); 4959 IXGBE_CORE_UNLOCK(adapter); 4960 4961 workqueue_wait(adapter->timer_wq, &adapter->timer_wc); 4962 atomic_store_relaxed(&adapter->timer_pending, 0); 4963 } 4964 4965 /************************************************************************ 4966 * ixgbe_stop_locked - Stop the hardware 4967 * 4968 * Disables all traffic on the adapter by issuing a 4969 * global reset on the MAC and deallocates TX/RX buffers. 4970 ************************************************************************/ 4971 static void 4972 ixgbe_stop_locked(void *arg) 4973 { 4974 struct ifnet *ifp; 4975 struct adapter *adapter = arg; 4976 struct ixgbe_hw *hw = &adapter->hw; 4977 4978 ifp = adapter->ifp; 4979 4980 KASSERT(mutex_owned(&adapter->core_mtx)); 4981 4982 INIT_DEBUGOUT("ixgbe_stop_locked: begin\n"); 4983 ixgbe_disable_intr(adapter); 4984 callout_stop(&adapter->timer); 4985 4986 /* Don't schedule workqueues. */ 4987 adapter->schedule_wqs_ok = false; 4988 4989 /* Let the stack know...*/ 4990 ifp->if_flags &= ~IFF_RUNNING; 4991 4992 ixgbe_reset_hw(hw); 4993 hw->adapter_stopped = FALSE; 4994 ixgbe_stop_adapter(hw); 4995 if (hw->mac.type == ixgbe_mac_82599EB) 4996 ixgbe_stop_mac_link_on_d3_82599(hw); 4997 /* Turn off the laser - noop with no optics */ 4998 ixgbe_disable_tx_laser(hw); 4999 5000 /* Update the stack */ 5001 adapter->link_up = FALSE; 5002 ixgbe_update_link_status(adapter); 5003 5004 /* reprogram the RAR[0] in case user changed it. */ 5005 ixgbe_set_rar(&adapter->hw, 0, adapter->hw.mac.addr, 0, IXGBE_RAH_AV); 5006 5007 return; 5008 } /* ixgbe_stop_locked */ 5009 5010 /************************************************************************ 5011 * ixgbe_update_link_status - Update OS on link state 5012 * 5013 * Note: Only updates the OS on the cached link state. 5014 * The real check of the hardware only happens with 5015 * a link interrupt. 5016 ************************************************************************/ 5017 static void 5018 ixgbe_update_link_status(struct adapter *adapter) 5019 { 5020 struct ifnet *ifp = adapter->ifp; 5021 device_t dev = adapter->dev; 5022 struct ixgbe_hw *hw = &adapter->hw; 5023 5024 KASSERT(mutex_owned(&adapter->core_mtx)); 5025 5026 if (adapter->link_up) { 5027 if (adapter->link_active != LINK_STATE_UP) { 5028 /* 5029 * To eliminate influence of the previous state 5030 * in the same way as ixgbe_init_locked(). 5031 */ 5032 struct ix_queue *que = adapter->queues; 5033 for (int i = 0; i < adapter->num_queues; i++, que++) 5034 que->eitr_setting = 0; 5035 5036 if (adapter->link_speed == IXGBE_LINK_SPEED_10GB_FULL){ 5037 /* 5038 * Discard count for both MAC Local Fault and 5039 * Remote Fault because those registers are 5040 * valid only when the link speed is up and 5041 * 10Gbps. 5042 */ 5043 IXGBE_READ_REG(hw, IXGBE_MLFC); 5044 IXGBE_READ_REG(hw, IXGBE_MRFC); 5045 } 5046 5047 if (bootverbose) { 5048 const char *bpsmsg; 5049 5050 switch (adapter->link_speed) { 5051 case IXGBE_LINK_SPEED_10GB_FULL: 5052 bpsmsg = "10 Gbps"; 5053 break; 5054 case IXGBE_LINK_SPEED_5GB_FULL: 5055 bpsmsg = "5 Gbps"; 5056 break; 5057 case IXGBE_LINK_SPEED_2_5GB_FULL: 5058 bpsmsg = "2.5 Gbps"; 5059 break; 5060 case IXGBE_LINK_SPEED_1GB_FULL: 5061 bpsmsg = "1 Gbps"; 5062 break; 5063 case IXGBE_LINK_SPEED_100_FULL: 5064 bpsmsg = "100 Mbps"; 5065 break; 5066 case IXGBE_LINK_SPEED_10_FULL: 5067 bpsmsg = "10 Mbps"; 5068 break; 5069 default: 5070 bpsmsg = "unknown speed"; 5071 break; 5072 } 5073 device_printf(dev, "Link is up %s %s \n", 5074 bpsmsg, "Full Duplex"); 5075 } 5076 adapter->link_active = LINK_STATE_UP; 5077 /* Update any Flow Control changes */ 5078 ixgbe_fc_enable(&adapter->hw); 5079 /* Update DMA coalescing config */ 5080 ixgbe_config_dmac(adapter); 5081 if_link_state_change(ifp, LINK_STATE_UP); 5082 5083 if (adapter->feat_en & IXGBE_FEATURE_SRIOV) 5084 ixgbe_ping_all_vfs(adapter); 5085 } 5086 } else { 5087 /* 5088 * Do it when link active changes to DOWN. i.e. 5089 * a) LINK_STATE_UNKNOWN -> LINK_STATE_DOWN 5090 * b) LINK_STATE_UP -> LINK_STATE_DOWN 5091 */ 5092 if (adapter->link_active != LINK_STATE_DOWN) { 5093 if (bootverbose) 5094 device_printf(dev, "Link is Down\n"); 5095 if_link_state_change(ifp, LINK_STATE_DOWN); 5096 adapter->link_active = LINK_STATE_DOWN; 5097 if (adapter->feat_en & IXGBE_FEATURE_SRIOV) 5098 ixgbe_ping_all_vfs(adapter); 5099 ixgbe_drain_all(adapter); 5100 } 5101 } 5102 } /* ixgbe_update_link_status */ 5103 5104 /************************************************************************ 5105 * ixgbe_config_dmac - Configure DMA Coalescing 5106 ************************************************************************/ 5107 static void 5108 ixgbe_config_dmac(struct adapter *adapter) 5109 { 5110 struct ixgbe_hw *hw = &adapter->hw; 5111 struct ixgbe_dmac_config *dcfg = &hw->mac.dmac_config; 5112 5113 if (hw->mac.type < ixgbe_mac_X550 || !hw->mac.ops.dmac_config) 5114 return; 5115 5116 if (dcfg->watchdog_timer ^ adapter->dmac || 5117 dcfg->link_speed ^ adapter->link_speed) { 5118 dcfg->watchdog_timer = adapter->dmac; 5119 dcfg->fcoe_en = false; 5120 dcfg->link_speed = adapter->link_speed; 5121 dcfg->num_tcs = 1; 5122 5123 INIT_DEBUGOUT2("dmac settings: watchdog %d, link speed %d\n", 5124 dcfg->watchdog_timer, dcfg->link_speed); 5125 5126 hw->mac.ops.dmac_config(hw); 5127 } 5128 } /* ixgbe_config_dmac */ 5129 5130 /************************************************************************ 5131 * ixgbe_enable_intr 5132 ************************************************************************/ 5133 static void 5134 ixgbe_enable_intr(struct adapter *adapter) 5135 { 5136 struct ixgbe_hw *hw = &adapter->hw; 5137 struct ix_queue *que = adapter->queues; 5138 u32 mask, fwsm; 5139 5140 mask = (IXGBE_EIMS_ENABLE_MASK & ~IXGBE_EIMS_RTX_QUEUE); 5141 5142 switch (adapter->hw.mac.type) { 5143 case ixgbe_mac_82599EB: 5144 mask |= IXGBE_EIMS_ECC; 5145 /* Temperature sensor on some adapters */ 5146 mask |= IXGBE_EIMS_GPI_SDP0; 5147 /* SFP+ (RX_LOS_N & MOD_ABS_N) */ 5148 mask |= IXGBE_EIMS_GPI_SDP1; 5149 mask |= IXGBE_EIMS_GPI_SDP2; 5150 break; 5151 case ixgbe_mac_X540: 5152 /* Detect if Thermal Sensor is enabled */ 5153 fwsm = IXGBE_READ_REG(hw, IXGBE_FWSM); 5154 if (fwsm & IXGBE_FWSM_TS_ENABLED) 5155 mask |= IXGBE_EIMS_TS; 5156 mask |= IXGBE_EIMS_ECC; 5157 break; 5158 case ixgbe_mac_X550: 5159 /* MAC thermal sensor is automatically enabled */ 5160 mask |= IXGBE_EIMS_TS; 5161 mask |= IXGBE_EIMS_ECC; 5162 break; 5163 case ixgbe_mac_X550EM_x: 5164 case ixgbe_mac_X550EM_a: 5165 /* Some devices use SDP0 for important information */ 5166 if (hw->device_id == IXGBE_DEV_ID_X550EM_X_SFP || 5167 hw->device_id == IXGBE_DEV_ID_X550EM_A_SFP || 5168 hw->device_id == IXGBE_DEV_ID_X550EM_A_SFP_N || 5169 hw->device_id == IXGBE_DEV_ID_X550EM_X_10G_T) 5170 mask |= IXGBE_EIMS_GPI_SDP0_BY_MAC(hw); 5171 if (hw->phy.type == ixgbe_phy_x550em_ext_t) 5172 mask |= IXGBE_EICR_GPI_SDP0_X540; 5173 mask |= IXGBE_EIMS_ECC; 5174 break; 5175 default: 5176 break; 5177 } 5178 5179 /* Enable Fan Failure detection */ 5180 if (adapter->feat_en & IXGBE_FEATURE_FAN_FAIL) 5181 mask |= IXGBE_EIMS_GPI_SDP1; 5182 /* Enable SR-IOV */ 5183 if (adapter->feat_en & IXGBE_FEATURE_SRIOV) 5184 mask |= IXGBE_EIMS_MAILBOX; 5185 /* Enable Flow Director */ 5186 if (adapter->feat_en & IXGBE_FEATURE_FDIR) 5187 mask |= IXGBE_EIMS_FLOW_DIR; 5188 5189 IXGBE_WRITE_REG(hw, IXGBE_EIMS, mask); 5190 5191 /* With MSI-X we use auto clear */ 5192 if ((adapter->feat_en & IXGBE_FEATURE_MSIX) != 0) { 5193 /* 5194 * We use auto clear for RTX_QUEUE only. Don't use other 5195 * interrupts (e.g. link interrupt). BTW, we don't use 5196 * TCP_TIMER interrupt itself. 5197 */ 5198 IXGBE_WRITE_REG(hw, IXGBE_EIAC, IXGBE_EIMS_RTX_QUEUE); 5199 } 5200 5201 /* 5202 * Now enable all queues, this is done separately to 5203 * allow for handling the extended (beyond 32) MSI-X 5204 * vectors that can be used by 82599 5205 */ 5206 for (int i = 0; i < adapter->num_queues; i++, que++) 5207 ixgbe_enable_queue(adapter, que->msix); 5208 5209 IXGBE_WRITE_FLUSH(hw); 5210 5211 } /* ixgbe_enable_intr */ 5212 5213 /************************************************************************ 5214 * ixgbe_disable_intr_internal 5215 ************************************************************************/ 5216 static void 5217 ixgbe_disable_intr_internal(struct adapter *adapter, bool nestok) 5218 { 5219 struct ix_queue *que = adapter->queues; 5220 5221 /* disable interrupts other than queues */ 5222 IXGBE_WRITE_REG(&adapter->hw, IXGBE_EIMC, ~IXGBE_EIMC_RTX_QUEUE); 5223 5224 if ((adapter->feat_en & IXGBE_FEATURE_MSIX) != 0) 5225 IXGBE_WRITE_REG(&adapter->hw, IXGBE_EIAC, 0); 5226 5227 for (int i = 0; i < adapter->num_queues; i++, que++) 5228 ixgbe_disable_queue_internal(adapter, que->msix, nestok); 5229 5230 IXGBE_WRITE_FLUSH(&adapter->hw); 5231 5232 } /* ixgbe_do_disable_intr_internal */ 5233 5234 /************************************************************************ 5235 * ixgbe_disable_intr 5236 ************************************************************************/ 5237 static void 5238 ixgbe_disable_intr(struct adapter *adapter) 5239 { 5240 5241 ixgbe_disable_intr_internal(adapter, true); 5242 } /* ixgbe_disable_intr */ 5243 5244 /************************************************************************ 5245 * ixgbe_ensure_disabled_intr 5246 ************************************************************************/ 5247 void 5248 ixgbe_ensure_disabled_intr(struct adapter *adapter) 5249 { 5250 5251 ixgbe_disable_intr_internal(adapter, false); 5252 } /* ixgbe_ensure_disabled_intr */ 5253 5254 /************************************************************************ 5255 * ixgbe_legacy_irq - Legacy Interrupt Service routine 5256 ************************************************************************/ 5257 static int 5258 ixgbe_legacy_irq(void *arg) 5259 { 5260 struct ix_queue *que = arg; 5261 struct adapter *adapter = que->adapter; 5262 struct ixgbe_hw *hw = &adapter->hw; 5263 struct ifnet *ifp = adapter->ifp; 5264 struct tx_ring *txr = adapter->tx_rings; 5265 u32 eicr; 5266 u32 eims_orig; 5267 u32 eims_enable = 0; 5268 u32 eims_disable = 0; 5269 5270 eims_orig = IXGBE_READ_REG(hw, IXGBE_EIMS); 5271 /* 5272 * Silicon errata #26 on 82598. Disable all interrupts before reading 5273 * EICR. 5274 */ 5275 IXGBE_WRITE_REG(hw, IXGBE_EIMC, IXGBE_IRQ_CLEAR_MASK); 5276 5277 /* Read and clear EICR */ 5278 eicr = IXGBE_READ_REG(hw, IXGBE_EICR); 5279 5280 if (eicr == 0) { 5281 IXGBE_EVC_ADD(&adapter->stats.pf.intzero, 1); 5282 IXGBE_WRITE_REG(hw, IXGBE_EIMS, eims_orig); 5283 return 0; 5284 } 5285 IXGBE_EVC_ADD(&adapter->stats.pf.legint, 1); 5286 5287 /* Queue (0) intr */ 5288 if (((ifp->if_flags & IFF_RUNNING) != 0) && 5289 (eicr & IXGBE_EIMC_RTX_QUEUE) != 0) { 5290 IXGBE_EVC_ADD(&que->irqs, 1); 5291 5292 /* 5293 * The same as ixgbe_msix_que() about 5294 * "que->txrx_use_workqueue". 5295 */ 5296 que->txrx_use_workqueue = adapter->txrx_use_workqueue; 5297 5298 IXGBE_TX_LOCK(txr); 5299 ixgbe_txeof(txr); 5300 #ifdef notyet 5301 if (!ixgbe_ring_empty(ifp, txr->br)) 5302 ixgbe_start_locked(ifp, txr); 5303 #endif 5304 IXGBE_TX_UNLOCK(txr); 5305 5306 IXGBE_EVC_ADD(&que->req, 1); 5307 ixgbe_sched_handle_que(adapter, que); 5308 /* Disable queue 0 interrupt */ 5309 eims_disable |= 1UL << 0; 5310 } else 5311 eims_enable |= eims_orig & IXGBE_EIMC_RTX_QUEUE; 5312 5313 ixgbe_intr_admin_common(adapter, eicr, &eims_disable); 5314 5315 /* Re-enable some interrupts */ 5316 IXGBE_WRITE_REG(hw, IXGBE_EIMS, 5317 (eims_orig & ~eims_disable) | eims_enable); 5318 5319 return 1; 5320 } /* ixgbe_legacy_irq */ 5321 5322 /************************************************************************ 5323 * ixgbe_free_pciintr_resources 5324 ************************************************************************/ 5325 static void 5326 ixgbe_free_pciintr_resources(struct adapter *adapter) 5327 { 5328 struct ix_queue *que = adapter->queues; 5329 int rid; 5330 5331 /* 5332 * Release all msix queue resources: 5333 */ 5334 for (int i = 0; i < adapter->num_queues; i++, que++) { 5335 if (que->res != NULL) { 5336 pci_intr_disestablish(adapter->osdep.pc, 5337 adapter->osdep.ihs[i]); 5338 adapter->osdep.ihs[i] = NULL; 5339 } 5340 } 5341 5342 /* Clean the Legacy or Link interrupt last */ 5343 if (adapter->vector) /* we are doing MSIX */ 5344 rid = adapter->vector; 5345 else 5346 rid = 0; 5347 5348 if (adapter->osdep.ihs[rid] != NULL) { 5349 pci_intr_disestablish(adapter->osdep.pc, 5350 adapter->osdep.ihs[rid]); 5351 adapter->osdep.ihs[rid] = NULL; 5352 } 5353 5354 if (adapter->osdep.intrs != NULL) { 5355 pci_intr_release(adapter->osdep.pc, adapter->osdep.intrs, 5356 adapter->osdep.nintrs); 5357 adapter->osdep.intrs = NULL; 5358 } 5359 } /* ixgbe_free_pciintr_resources */ 5360 5361 /************************************************************************ 5362 * ixgbe_free_pci_resources 5363 ************************************************************************/ 5364 static void 5365 ixgbe_free_pci_resources(struct adapter *adapter) 5366 { 5367 5368 ixgbe_free_pciintr_resources(adapter); 5369 5370 if (adapter->osdep.mem_size != 0) { 5371 bus_space_unmap(adapter->osdep.mem_bus_space_tag, 5372 adapter->osdep.mem_bus_space_handle, 5373 adapter->osdep.mem_size); 5374 } 5375 } /* ixgbe_free_pci_resources */ 5376 5377 /************************************************************************ 5378 * ixgbe_sysctl_flowcntl 5379 * 5380 * SYSCTL wrapper around setting Flow Control 5381 ************************************************************************/ 5382 static int 5383 ixgbe_sysctl_flowcntl(SYSCTLFN_ARGS) 5384 { 5385 struct sysctlnode node = *rnode; 5386 struct adapter *adapter = (struct adapter *)node.sysctl_data; 5387 int error, fc; 5388 5389 if (ixgbe_fw_recovery_mode_swflag(adapter)) 5390 return (EPERM); 5391 5392 fc = adapter->hw.fc.current_mode; 5393 node.sysctl_data = &fc; 5394 error = sysctl_lookup(SYSCTLFN_CALL(&node)); 5395 if (error != 0 || newp == NULL) 5396 return error; 5397 5398 /* Don't bother if it's not changed */ 5399 if (fc == adapter->hw.fc.current_mode) 5400 return (0); 5401 5402 return ixgbe_set_flowcntl(adapter, fc); 5403 } /* ixgbe_sysctl_flowcntl */ 5404 5405 /************************************************************************ 5406 * ixgbe_set_flowcntl - Set flow control 5407 * 5408 * Flow control values: 5409 * 0 - off 5410 * 1 - rx pause 5411 * 2 - tx pause 5412 * 3 - full 5413 ************************************************************************/ 5414 static int 5415 ixgbe_set_flowcntl(struct adapter *adapter, int fc) 5416 { 5417 switch (fc) { 5418 case ixgbe_fc_rx_pause: 5419 case ixgbe_fc_tx_pause: 5420 case ixgbe_fc_full: 5421 adapter->hw.fc.requested_mode = fc; 5422 if (adapter->num_queues > 1) 5423 ixgbe_disable_rx_drop(adapter); 5424 break; 5425 case ixgbe_fc_none: 5426 adapter->hw.fc.requested_mode = ixgbe_fc_none; 5427 if (adapter->num_queues > 1) 5428 ixgbe_enable_rx_drop(adapter); 5429 break; 5430 default: 5431 return (EINVAL); 5432 } 5433 5434 #if 0 /* XXX NetBSD */ 5435 /* Don't autoneg if forcing a value */ 5436 adapter->hw.fc.disable_fc_autoneg = TRUE; 5437 #endif 5438 ixgbe_fc_enable(&adapter->hw); 5439 5440 return (0); 5441 } /* ixgbe_set_flowcntl */ 5442 5443 /************************************************************************ 5444 * ixgbe_enable_rx_drop 5445 * 5446 * Enable the hardware to drop packets when the buffer is 5447 * full. This is useful with multiqueue, so that no single 5448 * queue being full stalls the entire RX engine. We only 5449 * enable this when Multiqueue is enabled AND Flow Control 5450 * is disabled. 5451 ************************************************************************/ 5452 static void 5453 ixgbe_enable_rx_drop(struct adapter *adapter) 5454 { 5455 struct ixgbe_hw *hw = &adapter->hw; 5456 struct rx_ring *rxr; 5457 u32 srrctl; 5458 5459 for (int i = 0; i < adapter->num_queues; i++) { 5460 rxr = &adapter->rx_rings[i]; 5461 srrctl = IXGBE_READ_REG(hw, IXGBE_SRRCTL(rxr->me)); 5462 srrctl |= IXGBE_SRRCTL_DROP_EN; 5463 IXGBE_WRITE_REG(hw, IXGBE_SRRCTL(rxr->me), srrctl); 5464 } 5465 5466 /* enable drop for each vf */ 5467 for (int i = 0; i < adapter->num_vfs; i++) { 5468 IXGBE_WRITE_REG(hw, IXGBE_QDE, 5469 (IXGBE_QDE_WRITE | (i << IXGBE_QDE_IDX_SHIFT) | 5470 IXGBE_QDE_ENABLE)); 5471 } 5472 } /* ixgbe_enable_rx_drop */ 5473 5474 /************************************************************************ 5475 * ixgbe_disable_rx_drop 5476 ************************************************************************/ 5477 static void 5478 ixgbe_disable_rx_drop(struct adapter *adapter) 5479 { 5480 struct ixgbe_hw *hw = &adapter->hw; 5481 struct rx_ring *rxr; 5482 u32 srrctl; 5483 5484 for (int i = 0; i < adapter->num_queues; i++) { 5485 rxr = &adapter->rx_rings[i]; 5486 srrctl = IXGBE_READ_REG(hw, IXGBE_SRRCTL(rxr->me)); 5487 srrctl &= ~IXGBE_SRRCTL_DROP_EN; 5488 IXGBE_WRITE_REG(hw, IXGBE_SRRCTL(rxr->me), srrctl); 5489 } 5490 5491 /* disable drop for each vf */ 5492 for (int i = 0; i < adapter->num_vfs; i++) { 5493 IXGBE_WRITE_REG(hw, IXGBE_QDE, 5494 (IXGBE_QDE_WRITE | (i << IXGBE_QDE_IDX_SHIFT))); 5495 } 5496 } /* ixgbe_disable_rx_drop */ 5497 5498 /************************************************************************ 5499 * ixgbe_sysctl_advertise 5500 * 5501 * SYSCTL wrapper around setting advertised speed 5502 ************************************************************************/ 5503 static int 5504 ixgbe_sysctl_advertise(SYSCTLFN_ARGS) 5505 { 5506 struct sysctlnode node = *rnode; 5507 struct adapter *adapter = (struct adapter *)node.sysctl_data; 5508 int error = 0, advertise; 5509 5510 if (ixgbe_fw_recovery_mode_swflag(adapter)) 5511 return (EPERM); 5512 5513 advertise = adapter->advertise; 5514 node.sysctl_data = &advertise; 5515 error = sysctl_lookup(SYSCTLFN_CALL(&node)); 5516 if (error != 0 || newp == NULL) 5517 return error; 5518 5519 return ixgbe_set_advertise(adapter, advertise); 5520 } /* ixgbe_sysctl_advertise */ 5521 5522 /************************************************************************ 5523 * ixgbe_set_advertise - Control advertised link speed 5524 * 5525 * Flags: 5526 * 0x00 - Default (all capable link speed) 5527 * 0x1 - advertise 100 Mb 5528 * 0x2 - advertise 1G 5529 * 0x4 - advertise 10G 5530 * 0x8 - advertise 10 Mb (yes, Mb) 5531 * 0x10 - advertise 2.5G 5532 * 0x20 - advertise 5G 5533 ************************************************************************/ 5534 static int 5535 ixgbe_set_advertise(struct adapter *adapter, int advertise) 5536 { 5537 device_t dev; 5538 struct ixgbe_hw *hw; 5539 ixgbe_link_speed speed = 0; 5540 ixgbe_link_speed link_caps = 0; 5541 s32 err = IXGBE_NOT_IMPLEMENTED; 5542 bool negotiate = FALSE; 5543 5544 /* Checks to validate new value */ 5545 if (adapter->advertise == advertise) /* no change */ 5546 return (0); 5547 5548 dev = adapter->dev; 5549 hw = &adapter->hw; 5550 5551 /* No speed changes for backplane media */ 5552 if (hw->phy.media_type == ixgbe_media_type_backplane) 5553 return (ENODEV); 5554 5555 if (!((hw->phy.media_type == ixgbe_media_type_copper) || 5556 (hw->phy.multispeed_fiber))) { 5557 device_printf(dev, 5558 "Advertised speed can only be set on copper or " 5559 "multispeed fiber media types.\n"); 5560 return (EINVAL); 5561 } 5562 5563 if (advertise < 0x0 || advertise > 0x3f) { 5564 device_printf(dev, "Invalid advertised speed; " 5565 "valid modes are 0x0 through 0x3f\n"); 5566 return (EINVAL); 5567 } 5568 5569 if (hw->mac.ops.get_link_capabilities) { 5570 err = hw->mac.ops.get_link_capabilities(hw, &link_caps, 5571 &negotiate); 5572 if (err != IXGBE_SUCCESS) { 5573 device_printf(dev, "Unable to determine supported " 5574 "advertise speeds\n"); 5575 return (ENODEV); 5576 } 5577 } 5578 5579 /* Set new value and report new advertised mode */ 5580 if (advertise & 0x1) { 5581 if (!(link_caps & IXGBE_LINK_SPEED_100_FULL)) { 5582 device_printf(dev, "Interface does not support 100Mb " 5583 "advertised speed\n"); 5584 return (EINVAL); 5585 } 5586 speed |= IXGBE_LINK_SPEED_100_FULL; 5587 } 5588 if (advertise & 0x2) { 5589 if (!(link_caps & IXGBE_LINK_SPEED_1GB_FULL)) { 5590 device_printf(dev, "Interface does not support 1Gb " 5591 "advertised speed\n"); 5592 return (EINVAL); 5593 } 5594 speed |= IXGBE_LINK_SPEED_1GB_FULL; 5595 } 5596 if (advertise & 0x4) { 5597 if (!(link_caps & IXGBE_LINK_SPEED_10GB_FULL)) { 5598 device_printf(dev, "Interface does not support 10Gb " 5599 "advertised speed\n"); 5600 return (EINVAL); 5601 } 5602 speed |= IXGBE_LINK_SPEED_10GB_FULL; 5603 } 5604 if (advertise & 0x8) { 5605 if (!(link_caps & IXGBE_LINK_SPEED_10_FULL)) { 5606 device_printf(dev, "Interface does not support 10Mb " 5607 "advertised speed\n"); 5608 return (EINVAL); 5609 } 5610 speed |= IXGBE_LINK_SPEED_10_FULL; 5611 } 5612 if (advertise & 0x10) { 5613 if (!(link_caps & IXGBE_LINK_SPEED_2_5GB_FULL)) { 5614 device_printf(dev, "Interface does not support 2.5Gb " 5615 "advertised speed\n"); 5616 return (EINVAL); 5617 } 5618 speed |= IXGBE_LINK_SPEED_2_5GB_FULL; 5619 } 5620 if (advertise & 0x20) { 5621 if (!(link_caps & IXGBE_LINK_SPEED_5GB_FULL)) { 5622 device_printf(dev, "Interface does not support 5Gb " 5623 "advertised speed\n"); 5624 return (EINVAL); 5625 } 5626 speed |= IXGBE_LINK_SPEED_5GB_FULL; 5627 } 5628 if (advertise == 0) 5629 speed = link_caps; /* All capable link speed */ 5630 5631 hw->mac.autotry_restart = TRUE; 5632 hw->mac.ops.setup_link(hw, speed, TRUE); 5633 adapter->advertise = advertise; 5634 5635 return (0); 5636 } /* ixgbe_set_advertise */ 5637 5638 /************************************************************************ 5639 * ixgbe_get_default_advertise - Get default advertised speed settings 5640 * 5641 * Formatted for sysctl usage. 5642 * Flags: 5643 * 0x1 - advertise 100 Mb 5644 * 0x2 - advertise 1G 5645 * 0x4 - advertise 10G 5646 * 0x8 - advertise 10 Mb (yes, Mb) 5647 * 0x10 - advertise 2.5G 5648 * 0x20 - advertise 5G 5649 ************************************************************************/ 5650 static int 5651 ixgbe_get_default_advertise(struct adapter *adapter) 5652 { 5653 struct ixgbe_hw *hw = &adapter->hw; 5654 int speed; 5655 ixgbe_link_speed link_caps = 0; 5656 s32 err; 5657 bool negotiate = FALSE; 5658 5659 /* 5660 * Advertised speed means nothing unless it's copper or 5661 * multi-speed fiber 5662 */ 5663 if (!(hw->phy.media_type == ixgbe_media_type_copper) && 5664 !(hw->phy.multispeed_fiber)) 5665 return (0); 5666 5667 err = hw->mac.ops.get_link_capabilities(hw, &link_caps, &negotiate); 5668 if (err != IXGBE_SUCCESS) 5669 return (0); 5670 5671 speed = 5672 ((link_caps & IXGBE_LINK_SPEED_10GB_FULL) ? 0x4 : 0) | 5673 ((link_caps & IXGBE_LINK_SPEED_5GB_FULL) ? 0x20 : 0) | 5674 ((link_caps & IXGBE_LINK_SPEED_2_5GB_FULL) ? 0x10 : 0) | 5675 ((link_caps & IXGBE_LINK_SPEED_1GB_FULL) ? 0x2 : 0) | 5676 ((link_caps & IXGBE_LINK_SPEED_100_FULL) ? 0x1 : 0) | 5677 ((link_caps & IXGBE_LINK_SPEED_10_FULL) ? 0x8 : 0); 5678 5679 return speed; 5680 } /* ixgbe_get_default_advertise */ 5681 5682 /************************************************************************ 5683 * ixgbe_sysctl_dmac - Manage DMA Coalescing 5684 * 5685 * Control values: 5686 * 0/1 - off / on (use default value of 1000) 5687 * 5688 * Legal timer values are: 5689 * 50,100,250,500,1000,2000,5000,10000 5690 * 5691 * Turning off interrupt moderation will also turn this off. 5692 ************************************************************************/ 5693 static int 5694 ixgbe_sysctl_dmac(SYSCTLFN_ARGS) 5695 { 5696 struct sysctlnode node = *rnode; 5697 struct adapter *adapter = (struct adapter *)node.sysctl_data; 5698 struct ifnet *ifp = adapter->ifp; 5699 int error; 5700 int newval; 5701 5702 if (ixgbe_fw_recovery_mode_swflag(adapter)) 5703 return (EPERM); 5704 5705 newval = adapter->dmac; 5706 node.sysctl_data = &newval; 5707 error = sysctl_lookup(SYSCTLFN_CALL(&node)); 5708 if ((error) || (newp == NULL)) 5709 return (error); 5710 5711 switch (newval) { 5712 case 0: 5713 /* Disabled */ 5714 adapter->dmac = 0; 5715 break; 5716 case 1: 5717 /* Enable and use default */ 5718 adapter->dmac = 1000; 5719 break; 5720 case 50: 5721 case 100: 5722 case 250: 5723 case 500: 5724 case 1000: 5725 case 2000: 5726 case 5000: 5727 case 10000: 5728 /* Legal values - allow */ 5729 adapter->dmac = newval; 5730 break; 5731 default: 5732 /* Do nothing, illegal value */ 5733 return (EINVAL); 5734 } 5735 5736 /* Re-initialize hardware if it's already running */ 5737 if (ifp->if_flags & IFF_RUNNING) 5738 if_init(ifp); 5739 5740 return (0); 5741 } 5742 5743 #ifdef IXGBE_DEBUG 5744 /************************************************************************ 5745 * ixgbe_sysctl_power_state 5746 * 5747 * Sysctl to test power states 5748 * Values: 5749 * 0 - set device to D0 5750 * 3 - set device to D3 5751 * (none) - get current device power state 5752 ************************************************************************/ 5753 static int 5754 ixgbe_sysctl_power_state(SYSCTLFN_ARGS) 5755 { 5756 #ifdef notyet 5757 struct sysctlnode node = *rnode; 5758 struct adapter *adapter = (struct adapter *)node.sysctl_data; 5759 device_t dev = adapter->dev; 5760 int curr_ps, new_ps, error = 0; 5761 5762 if (ixgbe_fw_recovery_mode_swflag(adapter)) 5763 return (EPERM); 5764 5765 curr_ps = new_ps = pci_get_powerstate(dev); 5766 5767 error = sysctl_lookup(SYSCTLFN_CALL(&node)); 5768 if ((error) || (req->newp == NULL)) 5769 return (error); 5770 5771 if (new_ps == curr_ps) 5772 return (0); 5773 5774 if (new_ps == 3 && curr_ps == 0) 5775 error = DEVICE_SUSPEND(dev); 5776 else if (new_ps == 0 && curr_ps == 3) 5777 error = DEVICE_RESUME(dev); 5778 else 5779 return (EINVAL); 5780 5781 device_printf(dev, "New state: %d\n", pci_get_powerstate(dev)); 5782 5783 return (error); 5784 #else 5785 return 0; 5786 #endif 5787 } /* ixgbe_sysctl_power_state */ 5788 #endif 5789 5790 /************************************************************************ 5791 * ixgbe_sysctl_wol_enable 5792 * 5793 * Sysctl to enable/disable the WoL capability, 5794 * if supported by the adapter. 5795 * 5796 * Values: 5797 * 0 - disabled 5798 * 1 - enabled 5799 ************************************************************************/ 5800 static int 5801 ixgbe_sysctl_wol_enable(SYSCTLFN_ARGS) 5802 { 5803 struct sysctlnode node = *rnode; 5804 struct adapter *adapter = (struct adapter *)node.sysctl_data; 5805 struct ixgbe_hw *hw = &adapter->hw; 5806 bool new_wol_enabled; 5807 int error = 0; 5808 5809 /* 5810 * It's not required to check recovery mode because this function never 5811 * touches hardware. 5812 */ 5813 new_wol_enabled = hw->wol_enabled; 5814 node.sysctl_data = &new_wol_enabled; 5815 error = sysctl_lookup(SYSCTLFN_CALL(&node)); 5816 if ((error) || (newp == NULL)) 5817 return (error); 5818 if (new_wol_enabled == hw->wol_enabled) 5819 return (0); 5820 5821 if (new_wol_enabled && !adapter->wol_support) 5822 return (ENODEV); 5823 else 5824 hw->wol_enabled = new_wol_enabled; 5825 5826 return (0); 5827 } /* ixgbe_sysctl_wol_enable */ 5828 5829 /************************************************************************ 5830 * ixgbe_sysctl_wufc - Wake Up Filter Control 5831 * 5832 * Sysctl to enable/disable the types of packets that the 5833 * adapter will wake up on upon receipt. 5834 * Flags: 5835 * 0x1 - Link Status Change 5836 * 0x2 - Magic Packet 5837 * 0x4 - Direct Exact 5838 * 0x8 - Directed Multicast 5839 * 0x10 - Broadcast 5840 * 0x20 - ARP/IPv4 Request Packet 5841 * 0x40 - Direct IPv4 Packet 5842 * 0x80 - Direct IPv6 Packet 5843 * 5844 * Settings not listed above will cause the sysctl to return an error. 5845 ************************************************************************/ 5846 static int 5847 ixgbe_sysctl_wufc(SYSCTLFN_ARGS) 5848 { 5849 struct sysctlnode node = *rnode; 5850 struct adapter *adapter = (struct adapter *)node.sysctl_data; 5851 int error = 0; 5852 u32 new_wufc; 5853 5854 /* 5855 * It's not required to check recovery mode because this function never 5856 * touches hardware. 5857 */ 5858 new_wufc = adapter->wufc; 5859 node.sysctl_data = &new_wufc; 5860 error = sysctl_lookup(SYSCTLFN_CALL(&node)); 5861 if ((error) || (newp == NULL)) 5862 return (error); 5863 if (new_wufc == adapter->wufc) 5864 return (0); 5865 5866 if (new_wufc & 0xffffff00) 5867 return (EINVAL); 5868 5869 new_wufc &= 0xff; 5870 new_wufc |= (0xffffff & adapter->wufc); 5871 adapter->wufc = new_wufc; 5872 5873 return (0); 5874 } /* ixgbe_sysctl_wufc */ 5875 5876 #ifdef IXGBE_DEBUG 5877 /************************************************************************ 5878 * ixgbe_sysctl_print_rss_config 5879 ************************************************************************/ 5880 static int 5881 ixgbe_sysctl_print_rss_config(SYSCTLFN_ARGS) 5882 { 5883 #ifdef notyet 5884 struct sysctlnode node = *rnode; 5885 struct adapter *adapter = (struct adapter *)node.sysctl_data; 5886 struct ixgbe_hw *hw = &adapter->hw; 5887 device_t dev = adapter->dev; 5888 struct sbuf *buf; 5889 int error = 0, reta_size; 5890 u32 reg; 5891 5892 if (ixgbe_fw_recovery_mode_swflag(adapter)) 5893 return (EPERM); 5894 5895 buf = sbuf_new_for_sysctl(NULL, NULL, 128, req); 5896 if (!buf) { 5897 device_printf(dev, "Could not allocate sbuf for output.\n"); 5898 return (ENOMEM); 5899 } 5900 5901 // TODO: use sbufs to make a string to print out 5902 /* Set multiplier for RETA setup and table size based on MAC */ 5903 switch (adapter->hw.mac.type) { 5904 case ixgbe_mac_X550: 5905 case ixgbe_mac_X550EM_x: 5906 case ixgbe_mac_X550EM_a: 5907 reta_size = 128; 5908 break; 5909 default: 5910 reta_size = 32; 5911 break; 5912 } 5913 5914 /* Print out the redirection table */ 5915 sbuf_cat(buf, "\n"); 5916 for (int i = 0; i < reta_size; i++) { 5917 if (i < 32) { 5918 reg = IXGBE_READ_REG(hw, IXGBE_RETA(i)); 5919 sbuf_printf(buf, "RETA(%2d): 0x%08x\n", i, reg); 5920 } else { 5921 reg = IXGBE_READ_REG(hw, IXGBE_ERETA(i - 32)); 5922 sbuf_printf(buf, "ERETA(%2d): 0x%08x\n", i - 32, reg); 5923 } 5924 } 5925 5926 // TODO: print more config 5927 5928 error = sbuf_finish(buf); 5929 if (error) 5930 device_printf(dev, "Error finishing sbuf: %d\n", error); 5931 5932 sbuf_delete(buf); 5933 #endif 5934 return (0); 5935 } /* ixgbe_sysctl_print_rss_config */ 5936 #endif /* IXGBE_DEBUG */ 5937 5938 /************************************************************************ 5939 * ixgbe_sysctl_phy_temp - Retrieve temperature of PHY 5940 * 5941 * For X552/X557-AT devices using an external PHY 5942 ************************************************************************/ 5943 static int 5944 ixgbe_sysctl_phy_temp(SYSCTLFN_ARGS) 5945 { 5946 struct sysctlnode node = *rnode; 5947 struct adapter *adapter = (struct adapter *)node.sysctl_data; 5948 struct ixgbe_hw *hw = &adapter->hw; 5949 int val; 5950 u16 reg; 5951 int error; 5952 5953 if (ixgbe_fw_recovery_mode_swflag(adapter)) 5954 return (EPERM); 5955 5956 if (hw->device_id != IXGBE_DEV_ID_X550EM_X_10G_T) { 5957 device_printf(adapter->dev, 5958 "Device has no supported external thermal sensor.\n"); 5959 return (ENODEV); 5960 } 5961 5962 if (hw->phy.ops.read_reg(hw, IXGBE_PHY_CURRENT_TEMP, 5963 IXGBE_MDIO_VENDOR_SPECIFIC_1_DEV_TYPE, ®)) { 5964 device_printf(adapter->dev, 5965 "Error reading from PHY's current temperature register\n"); 5966 return (EAGAIN); 5967 } 5968 5969 node.sysctl_data = &val; 5970 5971 /* Shift temp for output */ 5972 val = reg >> 8; 5973 5974 error = sysctl_lookup(SYSCTLFN_CALL(&node)); 5975 if ((error) || (newp == NULL)) 5976 return (error); 5977 5978 return (0); 5979 } /* ixgbe_sysctl_phy_temp */ 5980 5981 /************************************************************************ 5982 * ixgbe_sysctl_phy_overtemp_occurred 5983 * 5984 * Reports (directly from the PHY) whether the current PHY 5985 * temperature is over the overtemp threshold. 5986 ************************************************************************/ 5987 static int 5988 ixgbe_sysctl_phy_overtemp_occurred(SYSCTLFN_ARGS) 5989 { 5990 struct sysctlnode node = *rnode; 5991 struct adapter *adapter = (struct adapter *)node.sysctl_data; 5992 struct ixgbe_hw *hw = &adapter->hw; 5993 int val, error; 5994 u16 reg; 5995 5996 if (ixgbe_fw_recovery_mode_swflag(adapter)) 5997 return (EPERM); 5998 5999 if (hw->device_id != IXGBE_DEV_ID_X550EM_X_10G_T) { 6000 device_printf(adapter->dev, 6001 "Device has no supported external thermal sensor.\n"); 6002 return (ENODEV); 6003 } 6004 6005 if (hw->phy.ops.read_reg(hw, IXGBE_PHY_OVERTEMP_STATUS, 6006 IXGBE_MDIO_VENDOR_SPECIFIC_1_DEV_TYPE, ®)) { 6007 device_printf(adapter->dev, 6008 "Error reading from PHY's temperature status register\n"); 6009 return (EAGAIN); 6010 } 6011 6012 node.sysctl_data = &val; 6013 6014 /* Get occurrence bit */ 6015 val = !!(reg & 0x4000); 6016 6017 error = sysctl_lookup(SYSCTLFN_CALL(&node)); 6018 if ((error) || (newp == NULL)) 6019 return (error); 6020 6021 return (0); 6022 } /* ixgbe_sysctl_phy_overtemp_occurred */ 6023 6024 /************************************************************************ 6025 * ixgbe_sysctl_eee_state 6026 * 6027 * Sysctl to set EEE power saving feature 6028 * Values: 6029 * 0 - disable EEE 6030 * 1 - enable EEE 6031 * (none) - get current device EEE state 6032 ************************************************************************/ 6033 static int 6034 ixgbe_sysctl_eee_state(SYSCTLFN_ARGS) 6035 { 6036 struct sysctlnode node = *rnode; 6037 struct adapter *adapter = (struct adapter *)node.sysctl_data; 6038 struct ifnet *ifp = adapter->ifp; 6039 device_t dev = adapter->dev; 6040 int curr_eee, new_eee, error = 0; 6041 s32 retval; 6042 6043 if (ixgbe_fw_recovery_mode_swflag(adapter)) 6044 return (EPERM); 6045 6046 curr_eee = new_eee = !!(adapter->feat_en & IXGBE_FEATURE_EEE); 6047 node.sysctl_data = &new_eee; 6048 error = sysctl_lookup(SYSCTLFN_CALL(&node)); 6049 if ((error) || (newp == NULL)) 6050 return (error); 6051 6052 /* Nothing to do */ 6053 if (new_eee == curr_eee) 6054 return (0); 6055 6056 /* Not supported */ 6057 if (!(adapter->feat_cap & IXGBE_FEATURE_EEE)) 6058 return (EINVAL); 6059 6060 /* Bounds checking */ 6061 if ((new_eee < 0) || (new_eee > 1)) 6062 return (EINVAL); 6063 6064 retval = ixgbe_setup_eee(&adapter->hw, new_eee); 6065 if (retval) { 6066 device_printf(dev, "Error in EEE setup: 0x%08X\n", retval); 6067 return (EINVAL); 6068 } 6069 6070 /* Restart auto-neg */ 6071 if_init(ifp); 6072 6073 device_printf(dev, "New EEE state: %d\n", new_eee); 6074 6075 /* Cache new value */ 6076 if (new_eee) 6077 adapter->feat_en |= IXGBE_FEATURE_EEE; 6078 else 6079 adapter->feat_en &= ~IXGBE_FEATURE_EEE; 6080 6081 return (error); 6082 } /* ixgbe_sysctl_eee_state */ 6083 6084 #define PRINTQS(adapter, regname) \ 6085 do { \ 6086 struct ixgbe_hw *_hw = &(adapter)->hw; \ 6087 int _i; \ 6088 \ 6089 printf("%s: %s", device_xname((adapter)->dev), #regname); \ 6090 for (_i = 0; _i < (adapter)->num_queues; _i++) { \ 6091 printf((_i == 0) ? "\t" : " "); \ 6092 printf("%08x", IXGBE_READ_REG(_hw, \ 6093 IXGBE_##regname(_i))); \ 6094 } \ 6095 printf("\n"); \ 6096 } while (0) 6097 6098 /************************************************************************ 6099 * ixgbe_print_debug_info 6100 * 6101 * Called only when em_display_debug_stats is enabled. 6102 * Provides a way to take a look at important statistics 6103 * maintained by the driver and hardware. 6104 ************************************************************************/ 6105 static void 6106 ixgbe_print_debug_info(struct adapter *adapter) 6107 { 6108 device_t dev = adapter->dev; 6109 struct ixgbe_hw *hw = &adapter->hw; 6110 int table_size; 6111 int i; 6112 6113 switch (adapter->hw.mac.type) { 6114 case ixgbe_mac_X550: 6115 case ixgbe_mac_X550EM_x: 6116 case ixgbe_mac_X550EM_a: 6117 table_size = 128; 6118 break; 6119 default: 6120 table_size = 32; 6121 break; 6122 } 6123 6124 device_printf(dev, "[E]RETA:\n"); 6125 for (i = 0; i < table_size; i++) { 6126 if (i < 32) 6127 printf("%02x: %08x\n", i, IXGBE_READ_REG(hw, 6128 IXGBE_RETA(i))); 6129 else 6130 printf("%02x: %08x\n", i, IXGBE_READ_REG(hw, 6131 IXGBE_ERETA(i - 32))); 6132 } 6133 6134 device_printf(dev, "queue:"); 6135 for (i = 0; i < adapter->num_queues; i++) { 6136 printf((i == 0) ? "\t" : " "); 6137 printf("%8d", i); 6138 } 6139 printf("\n"); 6140 PRINTQS(adapter, RDBAL); 6141 PRINTQS(adapter, RDBAH); 6142 PRINTQS(adapter, RDLEN); 6143 PRINTQS(adapter, SRRCTL); 6144 PRINTQS(adapter, RDH); 6145 PRINTQS(adapter, RDT); 6146 PRINTQS(adapter, RXDCTL); 6147 6148 device_printf(dev, "RQSMR:"); 6149 for (i = 0; i < adapter->num_queues / 4; i++) { 6150 printf((i == 0) ? "\t" : " "); 6151 printf("%08x", IXGBE_READ_REG(hw, IXGBE_RQSMR(i))); 6152 } 6153 printf("\n"); 6154 6155 device_printf(dev, "disabled_count:"); 6156 for (i = 0; i < adapter->num_queues; i++) { 6157 printf((i == 0) ? "\t" : " "); 6158 printf("%8d", adapter->queues[i].disabled_count); 6159 } 6160 printf("\n"); 6161 6162 device_printf(dev, "EIMS:\t%08x\n", IXGBE_READ_REG(hw, IXGBE_EIMS)); 6163 if (hw->mac.type != ixgbe_mac_82598EB) { 6164 device_printf(dev, "EIMS_EX(0):\t%08x\n", 6165 IXGBE_READ_REG(hw, IXGBE_EIMS_EX(0))); 6166 device_printf(dev, "EIMS_EX(1):\t%08x\n", 6167 IXGBE_READ_REG(hw, IXGBE_EIMS_EX(1))); 6168 } 6169 device_printf(dev, "EIAM:\t%08x\n", IXGBE_READ_REG(hw, IXGBE_EIAM)); 6170 device_printf(dev, "EIAC:\t%08x\n", IXGBE_READ_REG(hw, IXGBE_EIAC)); 6171 } /* ixgbe_print_debug_info */ 6172 6173 /************************************************************************ 6174 * ixgbe_sysctl_debug 6175 ************************************************************************/ 6176 static int 6177 ixgbe_sysctl_debug(SYSCTLFN_ARGS) 6178 { 6179 struct sysctlnode node = *rnode; 6180 struct adapter *adapter = (struct adapter *)node.sysctl_data; 6181 int error, result = 0; 6182 6183 if (ixgbe_fw_recovery_mode_swflag(adapter)) 6184 return (EPERM); 6185 6186 node.sysctl_data = &result; 6187 error = sysctl_lookup(SYSCTLFN_CALL(&node)); 6188 6189 if (error || newp == NULL) 6190 return error; 6191 6192 if (result == 1) 6193 ixgbe_print_debug_info(adapter); 6194 6195 return 0; 6196 } /* ixgbe_sysctl_debug */ 6197 6198 /************************************************************************ 6199 * ixgbe_sysctl_rx_copy_len 6200 ************************************************************************/ 6201 static int 6202 ixgbe_sysctl_rx_copy_len(SYSCTLFN_ARGS) 6203 { 6204 struct sysctlnode node = *rnode; 6205 struct adapter *adapter = (struct adapter *)node.sysctl_data; 6206 int error; 6207 int result = adapter->rx_copy_len; 6208 6209 node.sysctl_data = &result; 6210 error = sysctl_lookup(SYSCTLFN_CALL(&node)); 6211 6212 if (error || newp == NULL) 6213 return error; 6214 6215 if ((result < 0) || (result > IXGBE_RX_COPY_LEN_MAX)) 6216 return EINVAL; 6217 6218 adapter->rx_copy_len = result; 6219 6220 return 0; 6221 } /* ixgbe_sysctl_rx_copy_len */ 6222 6223 /************************************************************************ 6224 * ixgbe_sysctl_tx_process_limit 6225 ************************************************************************/ 6226 static int 6227 ixgbe_sysctl_tx_process_limit(SYSCTLFN_ARGS) 6228 { 6229 struct sysctlnode node = *rnode; 6230 struct adapter *adapter = (struct adapter *)node.sysctl_data; 6231 int error; 6232 int result = adapter->tx_process_limit; 6233 6234 node.sysctl_data = &result; 6235 error = sysctl_lookup(SYSCTLFN_CALL(&node)); 6236 6237 if (error || newp == NULL) 6238 return error; 6239 6240 if ((result <= 0) || (result > adapter->num_tx_desc)) 6241 return EINVAL; 6242 6243 adapter->tx_process_limit = result; 6244 6245 return 0; 6246 } /* ixgbe_sysctl_tx_process_limit */ 6247 6248 /************************************************************************ 6249 * ixgbe_sysctl_rx_process_limit 6250 ************************************************************************/ 6251 static int 6252 ixgbe_sysctl_rx_process_limit(SYSCTLFN_ARGS) 6253 { 6254 struct sysctlnode node = *rnode; 6255 struct adapter *adapter = (struct adapter *)node.sysctl_data; 6256 int error; 6257 int result = adapter->rx_process_limit; 6258 6259 node.sysctl_data = &result; 6260 error = sysctl_lookup(SYSCTLFN_CALL(&node)); 6261 6262 if (error || newp == NULL) 6263 return error; 6264 6265 if ((result <= 0) || (result > adapter->num_rx_desc)) 6266 return EINVAL; 6267 6268 adapter->rx_process_limit = result; 6269 6270 return 0; 6271 } /* ixgbe_sysctl_rx_process_limit */ 6272 6273 /************************************************************************ 6274 * ixgbe_init_device_features 6275 ************************************************************************/ 6276 static void 6277 ixgbe_init_device_features(struct adapter *adapter) 6278 { 6279 adapter->feat_cap = IXGBE_FEATURE_NETMAP 6280 | IXGBE_FEATURE_RSS 6281 | IXGBE_FEATURE_MSI 6282 | IXGBE_FEATURE_MSIX 6283 | IXGBE_FEATURE_LEGACY_IRQ 6284 | IXGBE_FEATURE_LEGACY_TX; 6285 6286 /* Set capabilities first... */ 6287 switch (adapter->hw.mac.type) { 6288 case ixgbe_mac_82598EB: 6289 if (adapter->hw.device_id == IXGBE_DEV_ID_82598AT) 6290 adapter->feat_cap |= IXGBE_FEATURE_FAN_FAIL; 6291 break; 6292 case ixgbe_mac_X540: 6293 adapter->feat_cap |= IXGBE_FEATURE_SRIOV; 6294 adapter->feat_cap |= IXGBE_FEATURE_FDIR; 6295 if ((adapter->hw.device_id == IXGBE_DEV_ID_X540_BYPASS) && 6296 (adapter->hw.bus.func == 0)) 6297 adapter->feat_cap |= IXGBE_FEATURE_BYPASS; 6298 break; 6299 case ixgbe_mac_X550: 6300 /* 6301 * IXGBE_FEATURE_RECOVERY_MODE will be set after reading 6302 * NVM Image version. 6303 */ 6304 adapter->feat_cap |= IXGBE_FEATURE_TEMP_SENSOR; 6305 adapter->feat_cap |= IXGBE_FEATURE_SRIOV; 6306 adapter->feat_cap |= IXGBE_FEATURE_FDIR; 6307 break; 6308 case ixgbe_mac_X550EM_x: 6309 /* 6310 * IXGBE_FEATURE_RECOVERY_MODE will be set after reading 6311 * NVM Image version. 6312 */ 6313 adapter->feat_cap |= IXGBE_FEATURE_SRIOV; 6314 adapter->feat_cap |= IXGBE_FEATURE_FDIR; 6315 break; 6316 case ixgbe_mac_X550EM_a: 6317 /* 6318 * IXGBE_FEATURE_RECOVERY_MODE will be set after reading 6319 * NVM Image version. 6320 */ 6321 adapter->feat_cap |= IXGBE_FEATURE_SRIOV; 6322 adapter->feat_cap |= IXGBE_FEATURE_FDIR; 6323 adapter->feat_cap &= ~IXGBE_FEATURE_LEGACY_IRQ; 6324 if ((adapter->hw.device_id == IXGBE_DEV_ID_X550EM_A_1G_T) || 6325 (adapter->hw.device_id == IXGBE_DEV_ID_X550EM_A_1G_T_L)) { 6326 adapter->feat_cap |= IXGBE_FEATURE_TEMP_SENSOR; 6327 adapter->feat_cap |= IXGBE_FEATURE_EEE; 6328 } 6329 break; 6330 case ixgbe_mac_82599EB: 6331 adapter->feat_cap |= IXGBE_FEATURE_SRIOV; 6332 adapter->feat_cap |= IXGBE_FEATURE_FDIR; 6333 if ((adapter->hw.device_id == IXGBE_DEV_ID_82599_BYPASS) && 6334 (adapter->hw.bus.func == 0)) 6335 adapter->feat_cap |= IXGBE_FEATURE_BYPASS; 6336 if (adapter->hw.device_id == IXGBE_DEV_ID_82599_QSFP_SF_QP) 6337 adapter->feat_cap &= ~IXGBE_FEATURE_LEGACY_IRQ; 6338 break; 6339 default: 6340 break; 6341 } 6342 6343 /* Enabled by default... */ 6344 /* Fan failure detection */ 6345 if (adapter->feat_cap & IXGBE_FEATURE_FAN_FAIL) 6346 adapter->feat_en |= IXGBE_FEATURE_FAN_FAIL; 6347 /* Netmap */ 6348 if (adapter->feat_cap & IXGBE_FEATURE_NETMAP) 6349 adapter->feat_en |= IXGBE_FEATURE_NETMAP; 6350 /* EEE */ 6351 if (adapter->feat_cap & IXGBE_FEATURE_EEE) 6352 adapter->feat_en |= IXGBE_FEATURE_EEE; 6353 /* Thermal Sensor */ 6354 if (adapter->feat_cap & IXGBE_FEATURE_TEMP_SENSOR) 6355 adapter->feat_en |= IXGBE_FEATURE_TEMP_SENSOR; 6356 /* 6357 * Recovery mode: 6358 * NetBSD: IXGBE_FEATURE_RECOVERY_MODE will be controlled after reading 6359 * NVM Image version. 6360 */ 6361 6362 /* Enabled via global sysctl... */ 6363 /* Flow Director */ 6364 if (ixgbe_enable_fdir) { 6365 if (adapter->feat_cap & IXGBE_FEATURE_FDIR) 6366 adapter->feat_en |= IXGBE_FEATURE_FDIR; 6367 else 6368 device_printf(adapter->dev, "Device does not support " 6369 "Flow Director. Leaving disabled."); 6370 } 6371 /* Legacy (single queue) transmit */ 6372 if ((adapter->feat_cap & IXGBE_FEATURE_LEGACY_TX) && 6373 ixgbe_enable_legacy_tx) 6374 adapter->feat_en |= IXGBE_FEATURE_LEGACY_TX; 6375 /* 6376 * Message Signal Interrupts - Extended (MSI-X) 6377 * Normal MSI is only enabled if MSI-X calls fail. 6378 */ 6379 if (!ixgbe_enable_msix) 6380 adapter->feat_cap &= ~IXGBE_FEATURE_MSIX; 6381 /* Receive-Side Scaling (RSS) */ 6382 if ((adapter->feat_cap & IXGBE_FEATURE_RSS) && ixgbe_enable_rss) 6383 adapter->feat_en |= IXGBE_FEATURE_RSS; 6384 6385 /* Disable features with unmet dependencies... */ 6386 /* No MSI-X */ 6387 if (!(adapter->feat_cap & IXGBE_FEATURE_MSIX)) { 6388 adapter->feat_cap &= ~IXGBE_FEATURE_RSS; 6389 adapter->feat_cap &= ~IXGBE_FEATURE_SRIOV; 6390 adapter->feat_en &= ~IXGBE_FEATURE_RSS; 6391 adapter->feat_en &= ~IXGBE_FEATURE_SRIOV; 6392 } 6393 } /* ixgbe_init_device_features */ 6394 6395 /************************************************************************ 6396 * ixgbe_probe - Device identification routine 6397 * 6398 * Determines if the driver should be loaded on 6399 * adapter based on its PCI vendor/device ID. 6400 * 6401 * return BUS_PROBE_DEFAULT on success, positive on failure 6402 ************************************************************************/ 6403 static int 6404 ixgbe_probe(device_t dev, cfdata_t cf, void *aux) 6405 { 6406 const struct pci_attach_args *pa = aux; 6407 6408 return (ixgbe_lookup(pa) != NULL) ? 1 : 0; 6409 } 6410 6411 static const ixgbe_vendor_info_t * 6412 ixgbe_lookup(const struct pci_attach_args *pa) 6413 { 6414 const ixgbe_vendor_info_t *ent; 6415 pcireg_t subid; 6416 6417 INIT_DEBUGOUT("ixgbe_lookup: begin"); 6418 6419 if (PCI_VENDOR(pa->pa_id) != IXGBE_INTEL_VENDOR_ID) 6420 return NULL; 6421 6422 subid = pci_conf_read(pa->pa_pc, pa->pa_tag, PCI_SUBSYS_ID_REG); 6423 6424 for (ent = ixgbe_vendor_info_array; ent->vendor_id != 0; ent++) { 6425 if ((PCI_VENDOR(pa->pa_id) == ent->vendor_id) && 6426 (PCI_PRODUCT(pa->pa_id) == ent->device_id) && 6427 ((PCI_SUBSYS_VENDOR(subid) == ent->subvendor_id) || 6428 (ent->subvendor_id == 0)) && 6429 ((PCI_SUBSYS_ID(subid) == ent->subdevice_id) || 6430 (ent->subdevice_id == 0))) { 6431 return ent; 6432 } 6433 } 6434 return NULL; 6435 } 6436 6437 static int 6438 ixgbe_ifflags_cb(struct ethercom *ec) 6439 { 6440 struct ifnet *ifp = &ec->ec_if; 6441 struct adapter *adapter = ifp->if_softc; 6442 u_short change; 6443 int rv = 0; 6444 6445 IXGBE_CORE_LOCK(adapter); 6446 6447 change = ifp->if_flags ^ adapter->if_flags; 6448 if (change != 0) 6449 adapter->if_flags = ifp->if_flags; 6450 6451 if ((change & ~(IFF_CANTCHANGE | IFF_DEBUG)) != 0) { 6452 rv = ENETRESET; 6453 goto out; 6454 } else if ((change & IFF_PROMISC) != 0) 6455 ixgbe_set_rxfilter(adapter); 6456 6457 /* Check for ec_capenable. */ 6458 change = ec->ec_capenable ^ adapter->ec_capenable; 6459 adapter->ec_capenable = ec->ec_capenable; 6460 if ((change & ~(ETHERCAP_VLAN_MTU | ETHERCAP_VLAN_HWTAGGING 6461 | ETHERCAP_VLAN_HWFILTER)) != 0) { 6462 rv = ENETRESET; 6463 goto out; 6464 } 6465 6466 /* 6467 * Special handling is not required for ETHERCAP_VLAN_MTU. 6468 * MAXFRS(MHADD) does not include the 4bytes of the VLAN header. 6469 */ 6470 6471 /* Set up VLAN support and filter */ 6472 if ((change & (ETHERCAP_VLAN_HWTAGGING | ETHERCAP_VLAN_HWFILTER)) != 0) 6473 ixgbe_setup_vlan_hw_support(adapter); 6474 6475 out: 6476 IXGBE_CORE_UNLOCK(adapter); 6477 6478 return rv; 6479 } 6480 6481 /************************************************************************ 6482 * ixgbe_ioctl - Ioctl entry point 6483 * 6484 * Called when the user wants to configure the interface. 6485 * 6486 * return 0 on success, positive on failure 6487 ************************************************************************/ 6488 static int 6489 ixgbe_ioctl(struct ifnet *ifp, u_long command, void *data) 6490 { 6491 struct adapter *adapter = ifp->if_softc; 6492 struct ixgbe_hw *hw = &adapter->hw; 6493 struct ifcapreq *ifcr = data; 6494 struct ifreq *ifr = data; 6495 int error = 0; 6496 int l4csum_en; 6497 const int l4csum = IFCAP_CSUM_TCPv4_Rx | IFCAP_CSUM_UDPv4_Rx | 6498 IFCAP_CSUM_TCPv6_Rx | IFCAP_CSUM_UDPv6_Rx; 6499 6500 if (ixgbe_fw_recovery_mode_swflag(adapter)) 6501 return (EPERM); 6502 6503 switch (command) { 6504 case SIOCSIFFLAGS: 6505 IOCTL_DEBUGOUT("ioctl: SIOCSIFFLAGS (Set Interface Flags)"); 6506 break; 6507 case SIOCADDMULTI: 6508 case SIOCDELMULTI: 6509 IOCTL_DEBUGOUT("ioctl: SIOC(ADD|DEL)MULTI"); 6510 break; 6511 case SIOCSIFMEDIA: 6512 case SIOCGIFMEDIA: 6513 IOCTL_DEBUGOUT("ioctl: SIOCxIFMEDIA (Get/Set Interface Media)"); 6514 break; 6515 case SIOCSIFCAP: 6516 IOCTL_DEBUGOUT("ioctl: SIOCSIFCAP (Set Capabilities)"); 6517 break; 6518 case SIOCSIFMTU: 6519 IOCTL_DEBUGOUT("ioctl: SIOCSIFMTU (Set Interface MTU)"); 6520 break; 6521 #ifdef __NetBSD__ 6522 case SIOCINITIFADDR: 6523 IOCTL_DEBUGOUT("ioctl: SIOCINITIFADDR"); 6524 break; 6525 case SIOCGIFFLAGS: 6526 IOCTL_DEBUGOUT("ioctl: SIOCGIFFLAGS"); 6527 break; 6528 case SIOCGIFAFLAG_IN: 6529 IOCTL_DEBUGOUT("ioctl: SIOCGIFAFLAG_IN"); 6530 break; 6531 case SIOCGIFADDR: 6532 IOCTL_DEBUGOUT("ioctl: SIOCGIFADDR"); 6533 break; 6534 case SIOCGIFMTU: 6535 IOCTL_DEBUGOUT("ioctl: SIOCGIFMTU (Get Interface MTU)"); 6536 break; 6537 case SIOCGIFCAP: 6538 IOCTL_DEBUGOUT("ioctl: SIOCGIFCAP (Get IF cap)"); 6539 break; 6540 case SIOCGETHERCAP: 6541 IOCTL_DEBUGOUT("ioctl: SIOCGETHERCAP (Get ethercap)"); 6542 break; 6543 case SIOCGLIFADDR: 6544 IOCTL_DEBUGOUT("ioctl: SIOCGLIFADDR (Get Interface addr)"); 6545 break; 6546 case SIOCZIFDATA: 6547 IOCTL_DEBUGOUT("ioctl: SIOCZIFDATA (Zero counter)"); 6548 hw->mac.ops.clear_hw_cntrs(hw); 6549 ixgbe_clear_evcnt(adapter); 6550 break; 6551 case SIOCAIFADDR: 6552 IOCTL_DEBUGOUT("ioctl: SIOCAIFADDR (add/chg IF alias)"); 6553 break; 6554 #endif 6555 default: 6556 IOCTL_DEBUGOUT1("ioctl: UNKNOWN (0x%X)", (int)command); 6557 break; 6558 } 6559 6560 switch (command) { 6561 case SIOCGI2C: 6562 { 6563 struct ixgbe_i2c_req i2c; 6564 6565 IOCTL_DEBUGOUT("ioctl: SIOCGI2C (Get I2C Data)"); 6566 error = copyin(ifr->ifr_data, &i2c, sizeof(i2c)); 6567 if (error != 0) 6568 break; 6569 if (i2c.dev_addr != 0xA0 && i2c.dev_addr != 0xA2) { 6570 error = EINVAL; 6571 break; 6572 } 6573 if (i2c.len > sizeof(i2c.data)) { 6574 error = EINVAL; 6575 break; 6576 } 6577 6578 hw->phy.ops.read_i2c_byte(hw, i2c.offset, 6579 i2c.dev_addr, i2c.data); 6580 error = copyout(&i2c, ifr->ifr_data, sizeof(i2c)); 6581 break; 6582 } 6583 case SIOCSIFCAP: 6584 /* Layer-4 Rx checksum offload has to be turned on and 6585 * off as a unit. 6586 */ 6587 l4csum_en = ifcr->ifcr_capenable & l4csum; 6588 if (l4csum_en != l4csum && l4csum_en != 0) 6589 return EINVAL; 6590 /*FALLTHROUGH*/ 6591 case SIOCADDMULTI: 6592 case SIOCDELMULTI: 6593 case SIOCSIFFLAGS: 6594 case SIOCSIFMTU: 6595 default: 6596 if ((error = ether_ioctl(ifp, command, data)) != ENETRESET) 6597 return error; 6598 if ((ifp->if_flags & IFF_RUNNING) == 0) 6599 ; 6600 else if (command == SIOCSIFCAP || command == SIOCSIFMTU) { 6601 IXGBE_CORE_LOCK(adapter); 6602 if ((ifp->if_flags & IFF_RUNNING) != 0) 6603 ixgbe_init_locked(adapter); 6604 ixgbe_recalculate_max_frame(adapter); 6605 IXGBE_CORE_UNLOCK(adapter); 6606 } else if (command == SIOCADDMULTI || command == SIOCDELMULTI) { 6607 /* 6608 * Multicast list has changed; set the hardware filter 6609 * accordingly. 6610 */ 6611 IXGBE_CORE_LOCK(adapter); 6612 ixgbe_disable_intr(adapter); 6613 ixgbe_set_rxfilter(adapter); 6614 ixgbe_enable_intr(adapter); 6615 IXGBE_CORE_UNLOCK(adapter); 6616 } 6617 return 0; 6618 } 6619 6620 return error; 6621 } /* ixgbe_ioctl */ 6622 6623 /************************************************************************ 6624 * ixgbe_check_fan_failure 6625 ************************************************************************/ 6626 static int 6627 ixgbe_check_fan_failure(struct adapter *adapter, u32 reg, bool in_interrupt) 6628 { 6629 u32 mask; 6630 6631 mask = (in_interrupt) ? IXGBE_EICR_GPI_SDP1_BY_MAC(&adapter->hw) : 6632 IXGBE_ESDP_SDP1; 6633 6634 if ((reg & mask) == 0) 6635 return IXGBE_SUCCESS; 6636 6637 /* 6638 * Use ratecheck() just in case interrupt occur frequently. 6639 * When EXPX9501AT's fan stopped, interrupt occurred only once, 6640 * an red LED on the board turned on and link never up until 6641 * power off. 6642 */ 6643 if (ratecheck(&adapter->lasterr_time, &ixgbe_errlog_intrvl)) 6644 device_printf(adapter->dev, 6645 "\nCRITICAL: FAN FAILURE!! REPLACE IMMEDIATELY!!\n"); 6646 6647 return IXGBE_ERR_FAN_FAILURE; 6648 } /* ixgbe_check_fan_failure */ 6649 6650 /************************************************************************ 6651 * ixgbe_handle_que 6652 ************************************************************************/ 6653 static void 6654 ixgbe_handle_que(void *context) 6655 { 6656 struct ix_queue *que = context; 6657 struct adapter *adapter = que->adapter; 6658 struct tx_ring *txr = que->txr; 6659 struct ifnet *ifp = adapter->ifp; 6660 bool more = false; 6661 6662 IXGBE_EVC_ADD(&que->handleq, 1); 6663 6664 if (ifp->if_flags & IFF_RUNNING) { 6665 more = ixgbe_rxeof(que); 6666 IXGBE_TX_LOCK(txr); 6667 more |= ixgbe_txeof(txr); 6668 if (!(adapter->feat_en & IXGBE_FEATURE_LEGACY_TX)) 6669 if (!ixgbe_mq_ring_empty(ifp, txr->txr_interq)) 6670 ixgbe_mq_start_locked(ifp, txr); 6671 /* Only for queue 0 */ 6672 /* NetBSD still needs this for CBQ */ 6673 if ((&adapter->queues[0] == que) 6674 && (!ixgbe_legacy_ring_empty(ifp, NULL))) 6675 ixgbe_legacy_start_locked(ifp, txr); 6676 IXGBE_TX_UNLOCK(txr); 6677 } 6678 6679 if (more) { 6680 IXGBE_EVC_ADD(&que->req, 1); 6681 ixgbe_sched_handle_que(adapter, que); 6682 } else if (que->res != NULL) { 6683 /* MSIX: Re-enable this interrupt */ 6684 ixgbe_enable_queue(adapter, que->msix); 6685 } else { 6686 /* INTx or MSI */ 6687 ixgbe_enable_queue(adapter, 0); 6688 } 6689 6690 return; 6691 } /* ixgbe_handle_que */ 6692 6693 /************************************************************************ 6694 * ixgbe_handle_que_work 6695 ************************************************************************/ 6696 static void 6697 ixgbe_handle_que_work(struct work *wk, void *context) 6698 { 6699 struct ix_queue *que = container_of(wk, struct ix_queue, wq_cookie); 6700 6701 /* 6702 * "enqueued flag" is not required here. 6703 * See ixgbe_msix_que(). 6704 */ 6705 ixgbe_handle_que(que); 6706 } 6707 6708 /************************************************************************ 6709 * ixgbe_allocate_legacy - Setup the Legacy or MSI Interrupt handler 6710 ************************************************************************/ 6711 static int 6712 ixgbe_allocate_legacy(struct adapter *adapter, 6713 const struct pci_attach_args *pa) 6714 { 6715 device_t dev = adapter->dev; 6716 struct ix_queue *que = adapter->queues; 6717 struct tx_ring *txr = adapter->tx_rings; 6718 int counts[PCI_INTR_TYPE_SIZE]; 6719 pci_intr_type_t intr_type, max_type; 6720 char intrbuf[PCI_INTRSTR_LEN]; 6721 char wqname[MAXCOMLEN]; 6722 const char *intrstr = NULL; 6723 int defertx_error = 0, error; 6724 6725 /* We allocate a single interrupt resource */ 6726 max_type = PCI_INTR_TYPE_MSI; 6727 counts[PCI_INTR_TYPE_MSIX] = 0; 6728 counts[PCI_INTR_TYPE_MSI] = 6729 (adapter->feat_en & IXGBE_FEATURE_MSI) ? 1 : 0; 6730 /* Check not feat_en but feat_cap to fallback to INTx */ 6731 counts[PCI_INTR_TYPE_INTX] = 6732 (adapter->feat_cap & IXGBE_FEATURE_LEGACY_IRQ) ? 1 : 0; 6733 6734 alloc_retry: 6735 if (pci_intr_alloc(pa, &adapter->osdep.intrs, counts, max_type) != 0) { 6736 aprint_error_dev(dev, "couldn't alloc interrupt\n"); 6737 return ENXIO; 6738 } 6739 adapter->osdep.nintrs = 1; 6740 intrstr = pci_intr_string(adapter->osdep.pc, adapter->osdep.intrs[0], 6741 intrbuf, sizeof(intrbuf)); 6742 adapter->osdep.ihs[0] = pci_intr_establish_xname(adapter->osdep.pc, 6743 adapter->osdep.intrs[0], IPL_NET, ixgbe_legacy_irq, que, 6744 device_xname(dev)); 6745 intr_type = pci_intr_type(adapter->osdep.pc, adapter->osdep.intrs[0]); 6746 if (adapter->osdep.ihs[0] == NULL) { 6747 aprint_error_dev(dev,"unable to establish %s\n", 6748 (intr_type == PCI_INTR_TYPE_MSI) ? "MSI" : "INTx"); 6749 pci_intr_release(adapter->osdep.pc, adapter->osdep.intrs, 1); 6750 adapter->osdep.intrs = NULL; 6751 switch (intr_type) { 6752 case PCI_INTR_TYPE_MSI: 6753 /* The next try is for INTx: Disable MSI */ 6754 max_type = PCI_INTR_TYPE_INTX; 6755 counts[PCI_INTR_TYPE_INTX] = 1; 6756 adapter->feat_en &= ~IXGBE_FEATURE_MSI; 6757 if (adapter->feat_cap & IXGBE_FEATURE_LEGACY_IRQ) { 6758 adapter->feat_en |= IXGBE_FEATURE_LEGACY_IRQ; 6759 goto alloc_retry; 6760 } else 6761 break; 6762 case PCI_INTR_TYPE_INTX: 6763 default: 6764 /* See below */ 6765 break; 6766 } 6767 } 6768 if (intr_type == PCI_INTR_TYPE_INTX) { 6769 adapter->feat_en &= ~IXGBE_FEATURE_MSI; 6770 adapter->feat_en |= IXGBE_FEATURE_LEGACY_IRQ; 6771 } 6772 if (adapter->osdep.ihs[0] == NULL) { 6773 aprint_error_dev(dev, 6774 "couldn't establish interrupt%s%s\n", 6775 intrstr ? " at " : "", intrstr ? intrstr : ""); 6776 pci_intr_release(adapter->osdep.pc, adapter->osdep.intrs, 1); 6777 adapter->osdep.intrs = NULL; 6778 return ENXIO; 6779 } 6780 aprint_normal_dev(dev, "interrupting at %s\n", intrstr); 6781 /* 6782 * Try allocating a fast interrupt and the associated deferred 6783 * processing contexts. 6784 */ 6785 if (!(adapter->feat_en & IXGBE_FEATURE_LEGACY_TX)) { 6786 txr->txr_si = 6787 softint_establish(SOFTINT_NET | IXGBE_SOFTINT_FLAGS, 6788 ixgbe_deferred_mq_start, txr); 6789 6790 snprintf(wqname, sizeof(wqname), "%sdeferTx", 6791 device_xname(dev)); 6792 defertx_error = workqueue_create(&adapter->txr_wq, wqname, 6793 ixgbe_deferred_mq_start_work, adapter, IXGBE_WORKQUEUE_PRI, 6794 IPL_NET, IXGBE_WORKQUEUE_FLAGS); 6795 adapter->txr_wq_enqueued = percpu_alloc(sizeof(u_int)); 6796 } 6797 que->que_si = softint_establish(SOFTINT_NET | IXGBE_SOFTINT_FLAGS, 6798 ixgbe_handle_que, que); 6799 snprintf(wqname, sizeof(wqname), "%sTxRx", device_xname(dev)); 6800 error = workqueue_create(&adapter->que_wq, wqname, 6801 ixgbe_handle_que_work, adapter, IXGBE_WORKQUEUE_PRI, IPL_NET, 6802 IXGBE_WORKQUEUE_FLAGS); 6803 6804 if ((!(adapter->feat_en & IXGBE_FEATURE_LEGACY_TX) 6805 && ((txr->txr_si == NULL) || defertx_error != 0)) 6806 || (que->que_si == NULL) || error != 0) { 6807 aprint_error_dev(dev, 6808 "could not establish software interrupts\n"); 6809 6810 return ENXIO; 6811 } 6812 /* For simplicity in the handlers */ 6813 adapter->active_queues = IXGBE_EIMS_ENABLE_MASK; 6814 6815 return (0); 6816 } /* ixgbe_allocate_legacy */ 6817 6818 /************************************************************************ 6819 * ixgbe_allocate_msix - Setup MSI-X Interrupt resources and handlers 6820 ************************************************************************/ 6821 static int 6822 ixgbe_allocate_msix(struct adapter *adapter, const struct pci_attach_args *pa) 6823 { 6824 device_t dev = adapter->dev; 6825 struct ix_queue *que = adapter->queues; 6826 struct tx_ring *txr = adapter->tx_rings; 6827 pci_chipset_tag_t pc; 6828 char intrbuf[PCI_INTRSTR_LEN]; 6829 char intr_xname[32]; 6830 char wqname[MAXCOMLEN]; 6831 const char *intrstr = NULL; 6832 int error, vector = 0; 6833 int cpu_id = 0; 6834 kcpuset_t *affinity; 6835 #ifdef RSS 6836 unsigned int rss_buckets = 0; 6837 kcpuset_t cpu_mask; 6838 #endif 6839 6840 pc = adapter->osdep.pc; 6841 #ifdef RSS 6842 /* 6843 * If we're doing RSS, the number of queues needs to 6844 * match the number of RSS buckets that are configured. 6845 * 6846 * + If there's more queues than RSS buckets, we'll end 6847 * up with queues that get no traffic. 6848 * 6849 * + If there's more RSS buckets than queues, we'll end 6850 * up having multiple RSS buckets map to the same queue, 6851 * so there'll be some contention. 6852 */ 6853 rss_buckets = rss_getnumbuckets(); 6854 if ((adapter->feat_en & IXGBE_FEATURE_RSS) && 6855 (adapter->num_queues != rss_buckets)) { 6856 device_printf(dev, 6857 "%s: number of queues (%d) != number of RSS buckets (%d)" 6858 "; performance will be impacted.\n", 6859 __func__, adapter->num_queues, rss_buckets); 6860 } 6861 #endif 6862 6863 adapter->osdep.nintrs = adapter->num_queues + 1; 6864 if (pci_msix_alloc_exact(pa, &adapter->osdep.intrs, 6865 adapter->osdep.nintrs) != 0) { 6866 aprint_error_dev(dev, 6867 "failed to allocate MSI-X interrupt\n"); 6868 adapter->feat_en &= ~IXGBE_FEATURE_MSIX; 6869 return (ENXIO); 6870 } 6871 6872 kcpuset_create(&affinity, false); 6873 for (int i = 0; i < adapter->num_queues; i++, vector++, que++, txr++) { 6874 snprintf(intr_xname, sizeof(intr_xname), "%s TXRX%d", 6875 device_xname(dev), i); 6876 intrstr = pci_intr_string(pc, adapter->osdep.intrs[i], intrbuf, 6877 sizeof(intrbuf)); 6878 #ifdef IXGBE_MPSAFE 6879 pci_intr_setattr(pc, &adapter->osdep.intrs[i], PCI_INTR_MPSAFE, 6880 true); 6881 #endif 6882 /* Set the handler function */ 6883 que->res = adapter->osdep.ihs[i] = pci_intr_establish_xname(pc, 6884 adapter->osdep.intrs[i], IPL_NET, ixgbe_msix_que, que, 6885 intr_xname); 6886 if (que->res == NULL) { 6887 aprint_error_dev(dev, 6888 "Failed to register QUE handler\n"); 6889 error = ENXIO; 6890 goto err_out; 6891 } 6892 que->msix = vector; 6893 adapter->active_queues |= 1ULL << que->msix; 6894 6895 if (adapter->feat_en & IXGBE_FEATURE_RSS) { 6896 #ifdef RSS 6897 /* 6898 * The queue ID is used as the RSS layer bucket ID. 6899 * We look up the queue ID -> RSS CPU ID and select 6900 * that. 6901 */ 6902 cpu_id = rss_getcpu(i % rss_getnumbuckets()); 6903 CPU_SETOF(cpu_id, &cpu_mask); 6904 #endif 6905 } else { 6906 /* 6907 * Bind the MSI-X vector, and thus the 6908 * rings to the corresponding CPU. 6909 * 6910 * This just happens to match the default RSS 6911 * round-robin bucket -> queue -> CPU allocation. 6912 */ 6913 if (adapter->num_queues > 1) 6914 cpu_id = i; 6915 } 6916 /* Round-robin affinity */ 6917 kcpuset_zero(affinity); 6918 kcpuset_set(affinity, cpu_id % ncpu); 6919 error = interrupt_distribute(adapter->osdep.ihs[i], affinity, 6920 NULL); 6921 aprint_normal_dev(dev, "for TX/RX, interrupting at %s", 6922 intrstr); 6923 if (error == 0) { 6924 #if 1 /* def IXGBE_DEBUG */ 6925 #ifdef RSS 6926 aprintf_normal(", bound RSS bucket %d to CPU %d", i, 6927 cpu_id % ncpu); 6928 #else 6929 aprint_normal(", bound queue %d to cpu %d", i, 6930 cpu_id % ncpu); 6931 #endif 6932 #endif /* IXGBE_DEBUG */ 6933 } 6934 aprint_normal("\n"); 6935 6936 if (!(adapter->feat_en & IXGBE_FEATURE_LEGACY_TX)) { 6937 txr->txr_si = softint_establish( 6938 SOFTINT_NET | IXGBE_SOFTINT_FLAGS, 6939 ixgbe_deferred_mq_start, txr); 6940 if (txr->txr_si == NULL) { 6941 aprint_error_dev(dev, 6942 "couldn't establish software interrupt\n"); 6943 error = ENXIO; 6944 goto err_out; 6945 } 6946 } 6947 que->que_si 6948 = softint_establish(SOFTINT_NET | IXGBE_SOFTINT_FLAGS, 6949 ixgbe_handle_que, que); 6950 if (que->que_si == NULL) { 6951 aprint_error_dev(dev, 6952 "couldn't establish software interrupt\n"); 6953 error = ENXIO; 6954 goto err_out; 6955 } 6956 } 6957 snprintf(wqname, sizeof(wqname), "%sdeferTx", device_xname(dev)); 6958 error = workqueue_create(&adapter->txr_wq, wqname, 6959 ixgbe_deferred_mq_start_work, adapter, IXGBE_WORKQUEUE_PRI, IPL_NET, 6960 IXGBE_WORKQUEUE_FLAGS); 6961 if (error) { 6962 aprint_error_dev(dev, 6963 "couldn't create workqueue for deferred Tx\n"); 6964 goto err_out; 6965 } 6966 adapter->txr_wq_enqueued = percpu_alloc(sizeof(u_int)); 6967 6968 snprintf(wqname, sizeof(wqname), "%sTxRx", device_xname(dev)); 6969 error = workqueue_create(&adapter->que_wq, wqname, 6970 ixgbe_handle_que_work, adapter, IXGBE_WORKQUEUE_PRI, IPL_NET, 6971 IXGBE_WORKQUEUE_FLAGS); 6972 if (error) { 6973 aprint_error_dev(dev, "couldn't create workqueue for Tx/Rx\n"); 6974 goto err_out; 6975 } 6976 6977 /* and Link */ 6978 cpu_id++; 6979 snprintf(intr_xname, sizeof(intr_xname), "%s link", device_xname(dev)); 6980 adapter->vector = vector; 6981 intrstr = pci_intr_string(pc, adapter->osdep.intrs[vector], intrbuf, 6982 sizeof(intrbuf)); 6983 #ifdef IXGBE_MPSAFE 6984 pci_intr_setattr(pc, &adapter->osdep.intrs[vector], PCI_INTR_MPSAFE, 6985 true); 6986 #endif 6987 /* Set the link handler function */ 6988 adapter->osdep.ihs[vector] = pci_intr_establish_xname(pc, 6989 adapter->osdep.intrs[vector], IPL_NET, ixgbe_msix_admin, adapter, 6990 intr_xname); 6991 if (adapter->osdep.ihs[vector] == NULL) { 6992 aprint_error_dev(dev, "Failed to register LINK handler\n"); 6993 error = ENXIO; 6994 goto err_out; 6995 } 6996 /* Round-robin affinity */ 6997 kcpuset_zero(affinity); 6998 kcpuset_set(affinity, cpu_id % ncpu); 6999 error = interrupt_distribute(adapter->osdep.ihs[vector], affinity, 7000 NULL); 7001 7002 aprint_normal_dev(dev, 7003 "for link, interrupting at %s", intrstr); 7004 if (error == 0) 7005 aprint_normal(", affinity to cpu %d\n", cpu_id % ncpu); 7006 else 7007 aprint_normal("\n"); 7008 7009 kcpuset_destroy(affinity); 7010 aprint_normal_dev(dev, 7011 "Using MSI-X interrupts with %d vectors\n", vector + 1); 7012 7013 return (0); 7014 7015 err_out: 7016 kcpuset_destroy(affinity); 7017 ixgbe_free_deferred_handlers(adapter); 7018 ixgbe_free_pciintr_resources(adapter); 7019 return (error); 7020 } /* ixgbe_allocate_msix */ 7021 7022 /************************************************************************ 7023 * ixgbe_configure_interrupts 7024 * 7025 * Setup MSI-X, MSI, or legacy interrupts (in that order). 7026 * This will also depend on user settings. 7027 ************************************************************************/ 7028 static int 7029 ixgbe_configure_interrupts(struct adapter *adapter) 7030 { 7031 device_t dev = adapter->dev; 7032 struct ixgbe_mac_info *mac = &adapter->hw.mac; 7033 int want, queues, msgs; 7034 7035 /* Default to 1 queue if MSI-X setup fails */ 7036 adapter->num_queues = 1; 7037 7038 /* Override by tuneable */ 7039 if (!(adapter->feat_cap & IXGBE_FEATURE_MSIX)) 7040 goto msi; 7041 7042 /* 7043 * NetBSD only: Use single vector MSI when number of CPU is 1 to save 7044 * interrupt slot. 7045 */ 7046 if (ncpu == 1) 7047 goto msi; 7048 7049 /* First try MSI-X */ 7050 msgs = pci_msix_count(adapter->osdep.pc, adapter->osdep.tag); 7051 msgs = MIN(msgs, IXG_MAX_NINTR); 7052 if (msgs < 2) 7053 goto msi; 7054 7055 adapter->feat_en |= IXGBE_FEATURE_MSIX; 7056 7057 /* Figure out a reasonable auto config value */ 7058 queues = (ncpu > (msgs - 1)) ? (msgs - 1) : ncpu; 7059 7060 #ifdef RSS 7061 /* If we're doing RSS, clamp at the number of RSS buckets */ 7062 if (adapter->feat_en & IXGBE_FEATURE_RSS) 7063 queues = uimin(queues, rss_getnumbuckets()); 7064 #endif 7065 if (ixgbe_num_queues > queues) { 7066 aprint_error_dev(adapter->dev, 7067 "ixgbe_num_queues (%d) is too large, " 7068 "using reduced amount (%d).\n", ixgbe_num_queues, queues); 7069 ixgbe_num_queues = queues; 7070 } 7071 7072 if (ixgbe_num_queues != 0) 7073 queues = ixgbe_num_queues; 7074 else 7075 queues = uimin(queues, 7076 uimin(mac->max_tx_queues, mac->max_rx_queues)); 7077 7078 /* reflect correct sysctl value */ 7079 ixgbe_num_queues = queues; 7080 7081 /* 7082 * Want one vector (RX/TX pair) per queue 7083 * plus an additional for Link. 7084 */ 7085 want = queues + 1; 7086 if (msgs >= want) 7087 msgs = want; 7088 else { 7089 aprint_error_dev(dev, "MSI-X Configuration Problem, " 7090 "%d vectors but %d queues wanted!\n", msgs, want); 7091 goto msi; 7092 } 7093 adapter->num_queues = queues; 7094 adapter->feat_en |= IXGBE_FEATURE_MSIX; 7095 return (0); 7096 7097 /* 7098 * MSI-X allocation failed or provided us with 7099 * less vectors than needed. Free MSI-X resources 7100 * and we'll try enabling MSI. 7101 */ 7102 msi: 7103 /* Without MSI-X, some features are no longer supported */ 7104 adapter->feat_cap &= ~IXGBE_FEATURE_RSS; 7105 adapter->feat_en &= ~IXGBE_FEATURE_RSS; 7106 adapter->feat_cap &= ~IXGBE_FEATURE_SRIOV; 7107 adapter->feat_en &= ~IXGBE_FEATURE_SRIOV; 7108 7109 msgs = pci_msi_count(adapter->osdep.pc, adapter->osdep.tag); 7110 adapter->feat_en &= ~IXGBE_FEATURE_MSIX; 7111 if (msgs > 1) 7112 msgs = 1; 7113 if (msgs != 0) { 7114 msgs = 1; 7115 adapter->feat_en |= IXGBE_FEATURE_MSI; 7116 return (0); 7117 } 7118 7119 if (!(adapter->feat_cap & IXGBE_FEATURE_LEGACY_IRQ)) { 7120 aprint_error_dev(dev, 7121 "Device does not support legacy interrupts.\n"); 7122 return 1; 7123 } 7124 7125 adapter->feat_en |= IXGBE_FEATURE_LEGACY_IRQ; 7126 7127 return (0); 7128 } /* ixgbe_configure_interrupts */ 7129 7130 7131 /************************************************************************ 7132 * ixgbe_handle_link - Tasklet for MSI-X Link interrupts 7133 * 7134 * Done outside of interrupt context since the driver might sleep 7135 ************************************************************************/ 7136 static void 7137 ixgbe_handle_link(void *context) 7138 { 7139 struct adapter *adapter = context; 7140 struct ixgbe_hw *hw = &adapter->hw; 7141 7142 KASSERT(mutex_owned(&adapter->core_mtx)); 7143 7144 IXGBE_EVC_ADD(&adapter->link_workev, 1); 7145 ixgbe_check_link(hw, &adapter->link_speed, &adapter->link_up, 0); 7146 ixgbe_update_link_status(adapter); 7147 7148 /* Re-enable link interrupts */ 7149 IXGBE_WRITE_REG(hw, IXGBE_EIMS, IXGBE_EIMS_LSC); 7150 } /* ixgbe_handle_link */ 7151 7152 #if 0 7153 /************************************************************************ 7154 * ixgbe_rearm_queues 7155 ************************************************************************/ 7156 static __inline void 7157 ixgbe_rearm_queues(struct adapter *adapter, u64 queues) 7158 { 7159 u32 mask; 7160 7161 switch (adapter->hw.mac.type) { 7162 case ixgbe_mac_82598EB: 7163 mask = (IXGBE_EIMS_RTX_QUEUE & queues); 7164 IXGBE_WRITE_REG(&adapter->hw, IXGBE_EICS, mask); 7165 break; 7166 case ixgbe_mac_82599EB: 7167 case ixgbe_mac_X540: 7168 case ixgbe_mac_X550: 7169 case ixgbe_mac_X550EM_x: 7170 case ixgbe_mac_X550EM_a: 7171 mask = (queues & 0xFFFFFFFF); 7172 IXGBE_WRITE_REG(&adapter->hw, IXGBE_EICS_EX(0), mask); 7173 mask = (queues >> 32); 7174 IXGBE_WRITE_REG(&adapter->hw, IXGBE_EICS_EX(1), mask); 7175 break; 7176 default: 7177 break; 7178 } 7179 } /* ixgbe_rearm_queues */ 7180 #endif 7181
Indexes created Tue Sep 30 06:09:59 GMT 2025