ixgbe.c revision 1.309
1 /* $NetBSD: ixgbe.c,v 1.309 2022/03/10 03:57:33 msaitoh Exp $ */ 2 3 /****************************************************************************** 4 5 Copyright (c) 2001-2017, Intel Corporation 6 All rights reserved. 7 8 Redistribution and use in source and binary forms, with or without 9 modification, are permitted provided that the following conditions are met: 10 11 1. Redistributions of source code must retain the above copyright notice, 12 this list of conditions and the following disclaimer. 13 14 2. Redistributions in binary form must reproduce the above copyright 15 notice, this list of conditions and the following disclaimer in the 16 documentation and/or other materials provided with the distribution. 17 18 3. Neither the name of the Intel Corporation nor the names of its 19 contributors may be used to endorse or promote products derived from 20 this software without specific prior written permission. 21 22 THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 23 AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 24 IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 25 ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE 26 LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 27 CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 28 SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 29 INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 30 CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 31 ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 32 POSSIBILITY OF SUCH DAMAGE. 33 34 ******************************************************************************/ 35 /*$FreeBSD: head/sys/dev/ixgbe/if_ix.c 331224 2018-03-19 20:55:05Z erj $*/ 36 37 /* 38 * Copyright (c) 2011 The NetBSD Foundation, Inc. 39 * All rights reserved. 40 * 41 * This code is derived from software contributed to The NetBSD Foundation 42 * by Coyote Point Systems, Inc. 43 * 44 * Redistribution and use in source and binary forms, with or without 45 * modification, are permitted provided that the following conditions 46 * are met: 47 * 1. Redistributions of source code must retain the above copyright 48 * notice, this list of conditions and the following disclaimer. 49 * 2. Redistributions in binary form must reproduce the above copyright 50 * notice, this list of conditions and the following disclaimer in the 51 * documentation and/or other materials provided with the distribution. 52 * 53 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS 54 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED 55 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 56 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS 57 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 58 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 59 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 60 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 61 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 62 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 63 * POSSIBILITY OF SUCH DAMAGE. 64 */ 65 66 #include <sys/cdefs.h> 67 __KERNEL_RCSID(0, "$NetBSD: ixgbe.c,v 1.309 2022/03/10 03:57:33 msaitoh Exp $"); 68 69 #ifdef _KERNEL_OPT 70 #include "opt_inet.h" 71 #include "opt_inet6.h" 72 #include "opt_net_mpsafe.h" 73 #endif 74 75 #include "ixgbe.h" 76 #include "ixgbe_phy.h" 77 #include "ixgbe_sriov.h" 78 79 #include <sys/cprng.h> 80 #include <dev/mii/mii.h> 81 #include <dev/mii/miivar.h> 82 83 /************************************************************************ 84 * Driver version 85 ************************************************************************/ 86 static const char ixgbe_driver_version[] = "4.0.1-k"; 87 /* XXX NetBSD: + 3.3.24 */ 88 89 /************************************************************************ 90 * PCI Device ID Table 91 * 92 * Used by probe to select devices to load on 93 * Last field stores an index into ixgbe_strings 94 * Last entry must be all 0s 95 * 96 * { Vendor ID, Device ID, SubVendor ID, SubDevice ID, String Index } 97 ************************************************************************/ 98 static const ixgbe_vendor_info_t ixgbe_vendor_info_array[] = 99 { 100 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598AF_DUAL_PORT, 0, 0, 0}, 101 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598AF_SINGLE_PORT, 0, 0, 0}, 102 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598EB_CX4, 0, 0, 0}, 103 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598AT, 0, 0, 0}, 104 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598AT2, 0, 0, 0}, 105 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598, 0, 0, 0}, 106 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598_BX, 0, 0, 0}, 107 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598_DA_DUAL_PORT, 0, 0, 0}, 108 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598_CX4_DUAL_PORT, 0, 0, 0}, 109 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598EB_XF_LR, 0, 0, 0}, 110 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598_SR_DUAL_PORT_EM, 0, 0, 0}, 111 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598EB_SFP_LOM, 0, 0, 0}, 112 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_KR, 0, 0, 0}, 113 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_KX4, 0, 0, 0}, 114 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_KX4_MEZZ, 0, 0, 0}, 115 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_SFP, 0, 0, 0}, 116 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_SFP_EM, 0, 0, 0}, 117 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_XAUI_LOM, 0, 0, 0}, 118 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_CX4, 0, 0, 0}, 119 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_T3_LOM, 0, 0, 0}, 120 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_COMBO_BACKPLANE, 0, 0, 0}, 121 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_BACKPLANE_FCOE, 0, 0, 0}, 122 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_SFP_SF2, 0, 0, 0}, 123 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_SFP_FCOE, 0, 0, 0}, 124 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599EN_SFP, 0, 0, 0}, 125 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_SFP_SF_QP, 0, 0, 0}, 126 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_QSFP_SF_QP, 0, 0, 0}, 127 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_X540T, 0, 0, 0}, 128 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_X540T1, 0, 0, 0}, 129 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_X550T, 0, 0, 0}, 130 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_X550T1, 0, 0, 0}, 131 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_X550EM_X_KR, 0, 0, 0}, 132 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_X550EM_X_KX4, 0, 0, 0}, 133 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_X550EM_X_10G_T, 0, 0, 0}, 134 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_X550EM_X_1G_T, 0, 0, 0}, 135 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_X550EM_X_SFP, 0, 0, 0}, 136 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_X550EM_X_XFI, 0, 0, 0}, 137 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_X550EM_A_KR, 0, 0, 0}, 138 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_X550EM_A_KR_L, 0, 0, 0}, 139 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_X550EM_A_QSFP, 0, 0, 0}, 140 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_X550EM_A_QSFP_N, 0, 0, 0}, 141 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_X550EM_A_SFP, 0, 0, 0}, 142 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_X550EM_A_SFP_N, 0, 0, 0}, 143 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_X550EM_A_SGMII, 0, 0, 0}, 144 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_X550EM_A_SGMII_L, 0, 0, 0}, 145 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_X550EM_A_10G_T, 0, 0, 0}, 146 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_X550EM_A_1G_T, 0, 0, 0}, 147 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_X550EM_A_1G_T_L, 0, 0, 0}, 148 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_X540_BYPASS, 0, 0, 0}, 149 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_BYPASS, 0, 0, 0}, 150 /* required last entry */ 151 {0, 0, 0, 0, 0} 152 }; 153 154 /************************************************************************ 155 * Table of branding strings 156 ************************************************************************/ 157 static const char *ixgbe_strings[] = { 158 "Intel(R) PRO/10GbE PCI-Express Network Driver" 159 }; 160 161 /************************************************************************ 162 * Function prototypes 163 ************************************************************************/ 164 static int ixgbe_probe(device_t, cfdata_t, void *); 165 static void ixgbe_quirks(struct adapter *); 166 static void ixgbe_attach(device_t, device_t, void *); 167 static int ixgbe_detach(device_t, int); 168 #if 0 169 static int ixgbe_shutdown(device_t); 170 #endif 171 static bool ixgbe_suspend(device_t, const pmf_qual_t *); 172 static bool ixgbe_resume(device_t, const pmf_qual_t *); 173 static int ixgbe_ifflags_cb(struct ethercom *); 174 static int ixgbe_ioctl(struct ifnet *, u_long, void *); 175 static int ixgbe_init(struct ifnet *); 176 static void ixgbe_init_locked(struct adapter *); 177 static void ixgbe_ifstop(struct ifnet *, int); 178 static void ixgbe_stop_locked(void *); 179 static void ixgbe_init_device_features(struct adapter *); 180 static int ixgbe_check_fan_failure(struct adapter *, u32, bool); 181 static void ixgbe_add_media_types(struct adapter *); 182 static void ixgbe_media_status(struct ifnet *, struct ifmediareq *); 183 static int ixgbe_media_change(struct ifnet *); 184 static int ixgbe_allocate_pci_resources(struct adapter *, 185 const struct pci_attach_args *); 186 static void ixgbe_free_deferred_handlers(struct adapter *); 187 static void ixgbe_get_slot_info(struct adapter *); 188 static int ixgbe_allocate_msix(struct adapter *, 189 const struct pci_attach_args *); 190 static int ixgbe_allocate_legacy(struct adapter *, 191 const struct pci_attach_args *); 192 static int ixgbe_configure_interrupts(struct adapter *); 193 static void ixgbe_free_pciintr_resources(struct adapter *); 194 static void ixgbe_free_pci_resources(struct adapter *); 195 static void ixgbe_local_timer(void *); 196 static void ixgbe_handle_timer(struct work *, void *); 197 static void ixgbe_recovery_mode_timer(void *); 198 static void ixgbe_handle_recovery_mode_timer(struct work *, void *); 199 static int ixgbe_setup_interface(device_t, struct adapter *); 200 static void ixgbe_config_gpie(struct adapter *); 201 static void ixgbe_config_dmac(struct adapter *); 202 static void ixgbe_config_delay_values(struct adapter *); 203 static void ixgbe_schedule_admin_tasklet(struct adapter *); 204 static void ixgbe_config_link(struct adapter *); 205 static void ixgbe_check_wol_support(struct adapter *); 206 static int ixgbe_setup_low_power_mode(struct adapter *); 207 #if 0 208 static void ixgbe_rearm_queues(struct adapter *, u64); 209 #endif 210 211 static void ixgbe_initialize_transmit_units(struct adapter *); 212 static void ixgbe_initialize_receive_units(struct adapter *); 213 static void ixgbe_enable_rx_drop(struct adapter *); 214 static void ixgbe_disable_rx_drop(struct adapter *); 215 static void ixgbe_initialize_rss_mapping(struct adapter *); 216 217 static void ixgbe_enable_intr(struct adapter *); 218 static void ixgbe_disable_intr(struct adapter *); 219 static void ixgbe_update_stats_counters(struct adapter *); 220 static void ixgbe_set_rxfilter(struct adapter *); 221 static void ixgbe_update_link_status(struct adapter *); 222 static void ixgbe_set_ivar(struct adapter *, u8, u8, s8); 223 static void ixgbe_configure_ivars(struct adapter *); 224 static u8 * ixgbe_mc_array_itr(struct ixgbe_hw *, u8 **, u32 *); 225 static void ixgbe_eitr_write(struct adapter *, uint32_t, uint32_t); 226 227 static void ixgbe_setup_vlan_hw_tagging(struct adapter *); 228 static void ixgbe_setup_vlan_hw_support(struct adapter *); 229 static int ixgbe_vlan_cb(struct ethercom *, uint16_t, bool); 230 static int ixgbe_register_vlan(struct adapter *, u16); 231 static int ixgbe_unregister_vlan(struct adapter *, u16); 232 233 static void ixgbe_add_device_sysctls(struct adapter *); 234 static void ixgbe_add_hw_stats(struct adapter *); 235 static void ixgbe_clear_evcnt(struct adapter *); 236 static int ixgbe_set_flowcntl(struct adapter *, int); 237 static int ixgbe_set_advertise(struct adapter *, int); 238 static int ixgbe_get_default_advertise(struct adapter *); 239 240 /* Sysctl handlers */ 241 static void ixgbe_set_sysctl_value(struct adapter *, const char *, 242 const char *, int *, int); 243 static int ixgbe_sysctl_flowcntl(SYSCTLFN_PROTO); 244 static int ixgbe_sysctl_advertise(SYSCTLFN_PROTO); 245 static int ixgbe_sysctl_interrupt_rate_handler(SYSCTLFN_PROTO); 246 static int ixgbe_sysctl_dmac(SYSCTLFN_PROTO); 247 static int ixgbe_sysctl_phy_temp(SYSCTLFN_PROTO); 248 static int ixgbe_sysctl_phy_overtemp_occurred(SYSCTLFN_PROTO); 249 #ifdef IXGBE_DEBUG 250 static int ixgbe_sysctl_power_state(SYSCTLFN_PROTO); 251 static int ixgbe_sysctl_print_rss_config(SYSCTLFN_PROTO); 252 #endif 253 static int ixgbe_sysctl_next_to_check_handler(SYSCTLFN_PROTO); 254 static int ixgbe_sysctl_next_to_refresh_handler(SYSCTLFN_PROTO); 255 static int ixgbe_sysctl_rdh_handler(SYSCTLFN_PROTO); 256 static int ixgbe_sysctl_rdt_handler(SYSCTLFN_PROTO); 257 static int ixgbe_sysctl_tdt_handler(SYSCTLFN_PROTO); 258 static int ixgbe_sysctl_tdh_handler(SYSCTLFN_PROTO); 259 static int ixgbe_sysctl_eee_state(SYSCTLFN_PROTO); 260 static int ixgbe_sysctl_debug(SYSCTLFN_PROTO); 261 static int ixgbe_sysctl_rx_copy_len(SYSCTLFN_PROTO); 262 static int ixgbe_sysctl_wol_enable(SYSCTLFN_PROTO); 263 static int ixgbe_sysctl_wufc(SYSCTLFN_PROTO); 264 265 /* Interrupt functions */ 266 static int ixgbe_msix_que(void *); 267 static int ixgbe_msix_admin(void *); 268 static void ixgbe_intr_admin_common(struct adapter *, u32, u32 *); 269 static int ixgbe_legacy_irq(void *); 270 271 /* Event handlers running on workqueue */ 272 static void ixgbe_handle_que(void *); 273 static void ixgbe_handle_link(void *); 274 static void ixgbe_handle_msf(void *); 275 static void ixgbe_handle_mod(void *, bool); 276 static void ixgbe_handle_phy(void *); 277 278 /* Deferred workqueue handlers */ 279 static void ixgbe_handle_admin(struct work *, void *); 280 static void ixgbe_handle_que_work(struct work *, void *); 281 282 static const ixgbe_vendor_info_t *ixgbe_lookup(const struct pci_attach_args *); 283 284 /************************************************************************ 285 * NetBSD Device Interface Entry Points 286 ************************************************************************/ 287 CFATTACH_DECL3_NEW(ixg, sizeof(struct adapter), 288 ixgbe_probe, ixgbe_attach, ixgbe_detach, NULL, NULL, NULL, 289 DVF_DETACH_SHUTDOWN); 290 291 #if 0 292 devclass_t ix_devclass; 293 DRIVER_MODULE(ix, pci, ix_driver, ix_devclass, 0, 0); 294 295 MODULE_DEPEND(ix, pci, 1, 1, 1); 296 MODULE_DEPEND(ix, ether, 1, 1, 1); 297 #ifdef DEV_NETMAP 298 MODULE_DEPEND(ix, netmap, 1, 1, 1); 299 #endif 300 #endif 301 302 /* 303 * TUNEABLE PARAMETERS: 304 */ 305 306 /* 307 * AIM: Adaptive Interrupt Moderation 308 * which means that the interrupt rate 309 * is varied over time based on the 310 * traffic for that interrupt vector 311 */ 312 static bool ixgbe_enable_aim = true; 313 #define SYSCTL_INT(_a1, _a2, _a3, _a4, _a5, _a6, _a7) 314 SYSCTL_INT(_hw_ix, OID_AUTO, enable_aim, CTLFLAG_RDTUN, &ixgbe_enable_aim, 0, 315 "Enable adaptive interrupt moderation"); 316 317 static int ixgbe_max_interrupt_rate = (4000000 / IXGBE_LOW_LATENCY); 318 SYSCTL_INT(_hw_ix, OID_AUTO, max_interrupt_rate, CTLFLAG_RDTUN, 319 &ixgbe_max_interrupt_rate, 0, "Maximum interrupts per second"); 320 321 /* How many packets rxeof tries to clean at a time */ 322 static int ixgbe_rx_process_limit = 256; 323 SYSCTL_INT(_hw_ix, OID_AUTO, rx_process_limit, CTLFLAG_RDTUN, 324 &ixgbe_rx_process_limit, 0, "Maximum number of received packets to process at a time, -1 means unlimited"); 325 326 /* How many packets txeof tries to clean at a time */ 327 static int ixgbe_tx_process_limit = 256; 328 SYSCTL_INT(_hw_ix, OID_AUTO, tx_process_limit, CTLFLAG_RDTUN, 329 &ixgbe_tx_process_limit, 0, 330 "Maximum number of sent packets to process at a time, -1 means unlimited"); 331 332 /* Flow control setting, default to full */ 333 static int ixgbe_flow_control = ixgbe_fc_full; 334 SYSCTL_INT(_hw_ix, OID_AUTO, flow_control, CTLFLAG_RDTUN, 335 &ixgbe_flow_control, 0, "Default flow control used for all adapters"); 336 337 /* Which packet processing uses workqueue or softint */ 338 static bool ixgbe_txrx_workqueue = false; 339 340 /* 341 * Smart speed setting, default to on 342 * this only works as a compile option 343 * right now as its during attach, set 344 * this to 'ixgbe_smart_speed_off' to 345 * disable. 346 */ 347 static int ixgbe_smart_speed = ixgbe_smart_speed_on; 348 349 /* 350 * MSI-X should be the default for best performance, 351 * but this allows it to be forced off for testing. 352 */ 353 static int ixgbe_enable_msix = 1; 354 SYSCTL_INT(_hw_ix, OID_AUTO, enable_msix, CTLFLAG_RDTUN, &ixgbe_enable_msix, 0, 355 "Enable MSI-X interrupts"); 356 357 /* 358 * Number of Queues, can be set to 0, 359 * it then autoconfigures based on the 360 * number of cpus with a max of 8. This 361 * can be overridden manually here. 362 */ 363 static int ixgbe_num_queues = 0; 364 SYSCTL_INT(_hw_ix, OID_AUTO, num_queues, CTLFLAG_RDTUN, &ixgbe_num_queues, 0, 365 "Number of queues to configure, 0 indicates autoconfigure"); 366 367 /* 368 * Number of TX descriptors per ring, 369 * setting higher than RX as this seems 370 * the better performing choice. 371 */ 372 static int ixgbe_txd = PERFORM_TXD; 373 SYSCTL_INT(_hw_ix, OID_AUTO, txd, CTLFLAG_RDTUN, &ixgbe_txd, 0, 374 "Number of transmit descriptors per queue"); 375 376 /* Number of RX descriptors per ring */ 377 static int ixgbe_rxd = PERFORM_RXD; 378 SYSCTL_INT(_hw_ix, OID_AUTO, rxd, CTLFLAG_RDTUN, &ixgbe_rxd, 0, 379 "Number of receive descriptors per queue"); 380 381 /* 382 * Defining this on will allow the use 383 * of unsupported SFP+ modules, note that 384 * doing so you are on your own :) 385 */ 386 static int allow_unsupported_sfp = false; 387 #define TUNABLE_INT(__x, __y) 388 TUNABLE_INT("hw.ix.unsupported_sfp", &allow_unsupported_sfp); 389 390 /* 391 * Not sure if Flow Director is fully baked, 392 * so we'll default to turning it off. 393 */ 394 static int ixgbe_enable_fdir = 0; 395 SYSCTL_INT(_hw_ix, OID_AUTO, enable_fdir, CTLFLAG_RDTUN, &ixgbe_enable_fdir, 0, 396 "Enable Flow Director"); 397 398 /* Legacy Transmit (single queue) */ 399 static int ixgbe_enable_legacy_tx = 0; 400 SYSCTL_INT(_hw_ix, OID_AUTO, enable_legacy_tx, CTLFLAG_RDTUN, 401 &ixgbe_enable_legacy_tx, 0, "Enable Legacy TX flow"); 402 403 /* Receive-Side Scaling */ 404 static int ixgbe_enable_rss = 1; 405 SYSCTL_INT(_hw_ix, OID_AUTO, enable_rss, CTLFLAG_RDTUN, &ixgbe_enable_rss, 0, 406 "Enable Receive-Side Scaling (RSS)"); 407 408 #if 0 409 static int (*ixgbe_start_locked)(struct ifnet *, struct tx_ring *); 410 static int (*ixgbe_ring_empty)(struct ifnet *, pcq_t *); 411 #endif 412 413 #ifdef NET_MPSAFE 414 #define IXGBE_MPSAFE 1 415 #define IXGBE_CALLOUT_FLAGS CALLOUT_MPSAFE 416 #define IXGBE_SOFTINT_FLAGS SOFTINT_MPSAFE 417 #define IXGBE_WORKQUEUE_FLAGS WQ_PERCPU | WQ_MPSAFE 418 #define IXGBE_TASKLET_WQ_FLAGS WQ_MPSAFE 419 #else 420 #define IXGBE_CALLOUT_FLAGS 0 421 #define IXGBE_SOFTINT_FLAGS 0 422 #define IXGBE_WORKQUEUE_FLAGS WQ_PERCPU 423 #define IXGBE_TASKLET_WQ_FLAGS 0 424 #endif 425 #define IXGBE_WORKQUEUE_PRI PRI_SOFTNET 426 427 /************************************************************************ 428 * ixgbe_initialize_rss_mapping 429 ************************************************************************/ 430 static void 431 ixgbe_initialize_rss_mapping(struct adapter *adapter) 432 { 433 struct ixgbe_hw *hw = &adapter->hw; 434 u32 reta = 0, mrqc, rss_key[10]; 435 int queue_id, table_size, index_mult; 436 int i, j; 437 u32 rss_hash_config; 438 439 /* force use default RSS key. */ 440 #ifdef __NetBSD__ 441 rss_getkey((uint8_t *) &rss_key); 442 #else 443 if (adapter->feat_en & IXGBE_FEATURE_RSS) { 444 /* Fetch the configured RSS key */ 445 rss_getkey((uint8_t *) &rss_key); 446 } else { 447 /* set up random bits */ 448 cprng_fast(&rss_key, sizeof(rss_key)); 449 } 450 #endif 451 452 /* Set multiplier for RETA setup and table size based on MAC */ 453 index_mult = 0x1; 454 table_size = 128; 455 switch (adapter->hw.mac.type) { 456 case ixgbe_mac_82598EB: 457 index_mult = 0x11; 458 break; 459 case ixgbe_mac_X550: 460 case ixgbe_mac_X550EM_x: 461 case ixgbe_mac_X550EM_a: 462 table_size = 512; 463 break; 464 default: 465 break; 466 } 467 468 /* Set up the redirection table */ 469 for (i = 0, j = 0; i < table_size; i++, j++) { 470 if (j == adapter->num_queues) 471 j = 0; 472 473 if (adapter->feat_en & IXGBE_FEATURE_RSS) { 474 /* 475 * Fetch the RSS bucket id for the given indirection 476 * entry. Cap it at the number of configured buckets 477 * (which is num_queues.) 478 */ 479 queue_id = rss_get_indirection_to_bucket(i); 480 queue_id = queue_id % adapter->num_queues; 481 } else 482 queue_id = (j * index_mult); 483 484 /* 485 * The low 8 bits are for hash value (n+0); 486 * The next 8 bits are for hash value (n+1), etc. 487 */ 488 reta = reta >> 8; 489 reta = reta | (((uint32_t) queue_id) << 24); 490 if ((i & 3) == 3) { 491 if (i < 128) 492 IXGBE_WRITE_REG(hw, IXGBE_RETA(i >> 2), reta); 493 else 494 IXGBE_WRITE_REG(hw, IXGBE_ERETA((i >> 2) - 32), 495 reta); 496 reta = 0; 497 } 498 } 499 500 /* Now fill our hash function seeds */ 501 for (i = 0; i < 10; i++) 502 IXGBE_WRITE_REG(hw, IXGBE_RSSRK(i), rss_key[i]); 503 504 /* Perform hash on these packet types */ 505 if (adapter->feat_en & IXGBE_FEATURE_RSS) 506 rss_hash_config = rss_gethashconfig(); 507 else { 508 /* 509 * Disable UDP - IP fragments aren't currently being handled 510 * and so we end up with a mix of 2-tuple and 4-tuple 511 * traffic. 512 */ 513 rss_hash_config = RSS_HASHTYPE_RSS_IPV4 514 | RSS_HASHTYPE_RSS_TCP_IPV4 515 | RSS_HASHTYPE_RSS_IPV6 516 | RSS_HASHTYPE_RSS_TCP_IPV6 517 | RSS_HASHTYPE_RSS_IPV6_EX 518 | RSS_HASHTYPE_RSS_TCP_IPV6_EX; 519 } 520 521 mrqc = IXGBE_MRQC_RSSEN; 522 if (rss_hash_config & RSS_HASHTYPE_RSS_IPV4) 523 mrqc |= IXGBE_MRQC_RSS_FIELD_IPV4; 524 if (rss_hash_config & RSS_HASHTYPE_RSS_TCP_IPV4) 525 mrqc |= IXGBE_MRQC_RSS_FIELD_IPV4_TCP; 526 if (rss_hash_config & RSS_HASHTYPE_RSS_IPV6) 527 mrqc |= IXGBE_MRQC_RSS_FIELD_IPV6; 528 if (rss_hash_config & RSS_HASHTYPE_RSS_TCP_IPV6) 529 mrqc |= IXGBE_MRQC_RSS_FIELD_IPV6_TCP; 530 if (rss_hash_config & RSS_HASHTYPE_RSS_IPV6_EX) 531 mrqc |= IXGBE_MRQC_RSS_FIELD_IPV6_EX; 532 if (rss_hash_config & RSS_HASHTYPE_RSS_TCP_IPV6_EX) 533 mrqc |= IXGBE_MRQC_RSS_FIELD_IPV6_EX_TCP; 534 if (rss_hash_config & RSS_HASHTYPE_RSS_UDP_IPV4) 535 mrqc |= IXGBE_MRQC_RSS_FIELD_IPV4_UDP; 536 if (rss_hash_config & RSS_HASHTYPE_RSS_UDP_IPV6) 537 mrqc |= IXGBE_MRQC_RSS_FIELD_IPV6_UDP; 538 if (rss_hash_config & RSS_HASHTYPE_RSS_UDP_IPV6_EX) 539 mrqc |= IXGBE_MRQC_RSS_FIELD_IPV6_EX_UDP; 540 mrqc |= ixgbe_get_mrqc(adapter->iov_mode); 541 IXGBE_WRITE_REG(hw, IXGBE_MRQC, mrqc); 542 } /* ixgbe_initialize_rss_mapping */ 543 544 /************************************************************************ 545 * ixgbe_initialize_receive_units - Setup receive registers and features. 546 ************************************************************************/ 547 #define BSIZEPKT_ROUNDUP ((1<<IXGBE_SRRCTL_BSIZEPKT_SHIFT)-1) 548 549 static void 550 ixgbe_initialize_receive_units(struct adapter *adapter) 551 { 552 struct rx_ring *rxr = adapter->rx_rings; 553 struct ixgbe_hw *hw = &adapter->hw; 554 struct ifnet *ifp = adapter->ifp; 555 int i, j; 556 u32 bufsz, fctrl, srrctl, rxcsum; 557 u32 hlreg; 558 559 /* 560 * Make sure receives are disabled while 561 * setting up the descriptor ring 562 */ 563 ixgbe_disable_rx(hw); 564 565 /* Enable broadcasts */ 566 fctrl = IXGBE_READ_REG(hw, IXGBE_FCTRL); 567 fctrl |= IXGBE_FCTRL_BAM; 568 if (adapter->hw.mac.type == ixgbe_mac_82598EB) { 569 fctrl |= IXGBE_FCTRL_DPF; 570 fctrl |= IXGBE_FCTRL_PMCF; 571 } 572 IXGBE_WRITE_REG(hw, IXGBE_FCTRL, fctrl); 573 574 /* Set for Jumbo Frames? */ 575 hlreg = IXGBE_READ_REG(hw, IXGBE_HLREG0); 576 if (ifp->if_mtu > ETHERMTU) 577 hlreg |= IXGBE_HLREG0_JUMBOEN; 578 else 579 hlreg &= ~IXGBE_HLREG0_JUMBOEN; 580 581 #ifdef DEV_NETMAP 582 /* CRC stripping is conditional in Netmap */ 583 if ((adapter->feat_en & IXGBE_FEATURE_NETMAP) && 584 (ifp->if_capenable & IFCAP_NETMAP) && 585 !ix_crcstrip) 586 hlreg &= ~IXGBE_HLREG0_RXCRCSTRP; 587 else 588 #endif /* DEV_NETMAP */ 589 hlreg |= IXGBE_HLREG0_RXCRCSTRP; 590 591 IXGBE_WRITE_REG(hw, IXGBE_HLREG0, hlreg); 592 593 bufsz = (adapter->rx_mbuf_sz + BSIZEPKT_ROUNDUP) >> 594 IXGBE_SRRCTL_BSIZEPKT_SHIFT; 595 596 for (i = 0; i < adapter->num_queues; i++, rxr++) { 597 u64 rdba = rxr->rxdma.dma_paddr; 598 u32 reg; 599 int regnum = i / 4; /* 1 register per 4 queues */ 600 int regshift = i % 4; /* 4 bits per 1 queue */ 601 j = rxr->me; 602 603 /* Setup the Base and Length of the Rx Descriptor Ring */ 604 IXGBE_WRITE_REG(hw, IXGBE_RDBAL(j), 605 (rdba & 0x00000000ffffffffULL)); 606 IXGBE_WRITE_REG(hw, IXGBE_RDBAH(j), (rdba >> 32)); 607 IXGBE_WRITE_REG(hw, IXGBE_RDLEN(j), 608 adapter->num_rx_desc * sizeof(union ixgbe_adv_rx_desc)); 609 610 /* Set up the SRRCTL register */ 611 srrctl = IXGBE_READ_REG(hw, IXGBE_SRRCTL(j)); 612 srrctl &= ~IXGBE_SRRCTL_BSIZEHDR_MASK; 613 srrctl &= ~IXGBE_SRRCTL_BSIZEPKT_MASK; 614 srrctl |= bufsz; 615 srrctl |= IXGBE_SRRCTL_DESCTYPE_ADV_ONEBUF; 616 617 /* Set RQSMR (Receive Queue Statistic Mapping) register */ 618 reg = IXGBE_READ_REG(hw, IXGBE_RQSMR(regnum)); 619 reg &= ~(0x000000ffUL << (regshift * 8)); 620 reg |= i << (regshift * 8); 621 IXGBE_WRITE_REG(hw, IXGBE_RQSMR(regnum), reg); 622 623 /* 624 * Set DROP_EN iff we have no flow control and >1 queue. 625 * Note that srrctl was cleared shortly before during reset, 626 * so we do not need to clear the bit, but do it just in case 627 * this code is moved elsewhere. 628 */ 629 if (adapter->num_queues > 1 && 630 adapter->hw.fc.requested_mode == ixgbe_fc_none) { 631 srrctl |= IXGBE_SRRCTL_DROP_EN; 632 } else { 633 srrctl &= ~IXGBE_SRRCTL_DROP_EN; 634 } 635 636 IXGBE_WRITE_REG(hw, IXGBE_SRRCTL(j), srrctl); 637 638 /* Setup the HW Rx Head and Tail Descriptor Pointers */ 639 IXGBE_WRITE_REG(hw, IXGBE_RDH(j), 0); 640 IXGBE_WRITE_REG(hw, IXGBE_RDT(j), 0); 641 642 /* Set the driver rx tail address */ 643 rxr->tail = IXGBE_RDT(rxr->me); 644 } 645 646 if (adapter->hw.mac.type != ixgbe_mac_82598EB) { 647 u32 psrtype = IXGBE_PSRTYPE_TCPHDR 648 | IXGBE_PSRTYPE_UDPHDR 649 | IXGBE_PSRTYPE_IPV4HDR 650 | IXGBE_PSRTYPE_IPV6HDR; 651 IXGBE_WRITE_REG(hw, IXGBE_PSRTYPE(0), psrtype); 652 } 653 654 rxcsum = IXGBE_READ_REG(hw, IXGBE_RXCSUM); 655 656 ixgbe_initialize_rss_mapping(adapter); 657 658 if (adapter->num_queues > 1) { 659 /* RSS and RX IPP Checksum are mutually exclusive */ 660 rxcsum |= IXGBE_RXCSUM_PCSD; 661 } 662 663 if (ifp->if_capenable & IFCAP_RXCSUM) 664 rxcsum |= IXGBE_RXCSUM_PCSD; 665 666 /* This is useful for calculating UDP/IP fragment checksums */ 667 if (!(rxcsum & IXGBE_RXCSUM_PCSD)) 668 rxcsum |= IXGBE_RXCSUM_IPPCSE; 669 670 IXGBE_WRITE_REG(hw, IXGBE_RXCSUM, rxcsum); 671 672 } /* ixgbe_initialize_receive_units */ 673 674 /************************************************************************ 675 * ixgbe_initialize_transmit_units - Enable transmit units. 676 ************************************************************************/ 677 static void 678 ixgbe_initialize_transmit_units(struct adapter *adapter) 679 { 680 struct tx_ring *txr = adapter->tx_rings; 681 struct ixgbe_hw *hw = &adapter->hw; 682 int i; 683 684 INIT_DEBUGOUT("ixgbe_initialize_transmit_units"); 685 686 /* Setup the Base and Length of the Tx Descriptor Ring */ 687 for (i = 0; i < adapter->num_queues; i++, txr++) { 688 u64 tdba = txr->txdma.dma_paddr; 689 u32 txctrl = 0; 690 u32 tqsmreg, reg; 691 int regnum = i / 4; /* 1 register per 4 queues */ 692 int regshift = i % 4; /* 4 bits per 1 queue */ 693 int j = txr->me; 694 695 IXGBE_WRITE_REG(hw, IXGBE_TDBAL(j), 696 (tdba & 0x00000000ffffffffULL)); 697 IXGBE_WRITE_REG(hw, IXGBE_TDBAH(j), (tdba >> 32)); 698 IXGBE_WRITE_REG(hw, IXGBE_TDLEN(j), 699 adapter->num_tx_desc * sizeof(union ixgbe_adv_tx_desc)); 700 701 /* 702 * Set TQSMR (Transmit Queue Statistic Mapping) register. 703 * Register location is different between 82598 and others. 704 */ 705 if (adapter->hw.mac.type == ixgbe_mac_82598EB) 706 tqsmreg = IXGBE_TQSMR(regnum); 707 else 708 tqsmreg = IXGBE_TQSM(regnum); 709 reg = IXGBE_READ_REG(hw, tqsmreg); 710 reg &= ~(0x000000ffUL << (regshift * 8)); 711 reg |= i << (regshift * 8); 712 IXGBE_WRITE_REG(hw, tqsmreg, reg); 713 714 /* Setup the HW Tx Head and Tail descriptor pointers */ 715 IXGBE_WRITE_REG(hw, IXGBE_TDH(j), 0); 716 IXGBE_WRITE_REG(hw, IXGBE_TDT(j), 0); 717 718 /* Cache the tail address */ 719 txr->tail = IXGBE_TDT(j); 720 721 txr->txr_no_space = false; 722 723 /* Disable Head Writeback */ 724 /* 725 * Note: for X550 series devices, these registers are actually 726 * prefixed with TPH_ instead of DCA_, but the addresses and 727 * fields remain the same. 728 */ 729 switch (hw->mac.type) { 730 case ixgbe_mac_82598EB: 731 txctrl = IXGBE_READ_REG(hw, IXGBE_DCA_TXCTRL(j)); 732 break; 733 default: 734 txctrl = IXGBE_READ_REG(hw, IXGBE_DCA_TXCTRL_82599(j)); 735 break; 736 } 737 txctrl &= ~IXGBE_DCA_TXCTRL_DESC_WRO_EN; 738 switch (hw->mac.type) { 739 case ixgbe_mac_82598EB: 740 IXGBE_WRITE_REG(hw, IXGBE_DCA_TXCTRL(j), txctrl); 741 break; 742 default: 743 IXGBE_WRITE_REG(hw, IXGBE_DCA_TXCTRL_82599(j), txctrl); 744 break; 745 } 746 747 } 748 749 if (hw->mac.type != ixgbe_mac_82598EB) { 750 u32 dmatxctl, rttdcs; 751 752 dmatxctl = IXGBE_READ_REG(hw, IXGBE_DMATXCTL); 753 dmatxctl |= IXGBE_DMATXCTL_TE; 754 IXGBE_WRITE_REG(hw, IXGBE_DMATXCTL, dmatxctl); 755 /* Disable arbiter to set MTQC */ 756 rttdcs = IXGBE_READ_REG(hw, IXGBE_RTTDCS); 757 rttdcs |= IXGBE_RTTDCS_ARBDIS; 758 IXGBE_WRITE_REG(hw, IXGBE_RTTDCS, rttdcs); 759 IXGBE_WRITE_REG(hw, IXGBE_MTQC, 760 ixgbe_get_mtqc(adapter->iov_mode)); 761 rttdcs &= ~IXGBE_RTTDCS_ARBDIS; 762 IXGBE_WRITE_REG(hw, IXGBE_RTTDCS, rttdcs); 763 } 764 765 return; 766 } /* ixgbe_initialize_transmit_units */ 767 768 static void 769 ixgbe_quirks(struct adapter *adapter) 770 { 771 device_t dev = adapter->dev; 772 struct ixgbe_hw *hw = &adapter->hw; 773 const char *vendor, *product; 774 775 if (hw->device_id == IXGBE_DEV_ID_X550EM_A_SFP_N) { 776 /* 777 * Quirk for inverted logic of SFP+'s MOD_ABS on GIGABYTE 778 * MA10-ST0. 779 */ 780 vendor = pmf_get_platform("system-vendor"); 781 product = pmf_get_platform("system-product"); 782 783 if ((vendor == NULL) || (product == NULL)) 784 return; 785 786 if ((strcmp(vendor, "GIGABYTE") == 0) && 787 (strcmp(product, "MA10-ST0") == 0)) { 788 aprint_verbose_dev(dev, 789 "Enable SFP+ MOD_ABS inverse quirk\n"); 790 adapter->hw.quirks |= IXGBE_QUIRK_MOD_ABS_INVERT; 791 } 792 } 793 } 794 795 /************************************************************************ 796 * ixgbe_attach - Device initialization routine 797 * 798 * Called when the driver is being loaded. 799 * Identifies the type of hardware, allocates all resources 800 * and initializes the hardware. 801 * 802 * return 0 on success, positive on failure 803 ************************************************************************/ 804 static void 805 ixgbe_attach(device_t parent, device_t dev, void *aux) 806 { 807 struct adapter *adapter; 808 struct ixgbe_hw *hw; 809 int error = -1; 810 u32 ctrl_ext; 811 u16 high, low, nvmreg; 812 pcireg_t id, subid; 813 const ixgbe_vendor_info_t *ent; 814 struct pci_attach_args *pa = aux; 815 bool unsupported_sfp = false; 816 const char *str; 817 char wqname[MAXCOMLEN]; 818 char buf[256]; 819 820 INIT_DEBUGOUT("ixgbe_attach: begin"); 821 822 /* Allocate, clear, and link in our adapter structure */ 823 adapter = device_private(dev); 824 adapter->hw.back = adapter; 825 adapter->dev = dev; 826 hw = &adapter->hw; 827 adapter->osdep.pc = pa->pa_pc; 828 adapter->osdep.tag = pa->pa_tag; 829 if (pci_dma64_available(pa)) 830 adapter->osdep.dmat = pa->pa_dmat64; 831 else 832 adapter->osdep.dmat = pa->pa_dmat; 833 adapter->osdep.attached = false; 834 adapter->osdep.detaching = false; 835 836 ent = ixgbe_lookup(pa); 837 838 KASSERT(ent != NULL); 839 840 aprint_normal(": %s, Version - %s\n", 841 ixgbe_strings[ent->index], ixgbe_driver_version); 842 843 /* Core Lock Init */ 844 IXGBE_CORE_LOCK_INIT(adapter, device_xname(dev)); 845 846 /* Set up the timer callout and workqueue */ 847 callout_init(&adapter->timer, IXGBE_CALLOUT_FLAGS); 848 snprintf(wqname, sizeof(wqname), "%s-timer", device_xname(dev)); 849 error = workqueue_create(&adapter->timer_wq, wqname, 850 ixgbe_handle_timer, adapter, IXGBE_WORKQUEUE_PRI, IPL_NET, 851 IXGBE_TASKLET_WQ_FLAGS); 852 if (error) { 853 aprint_error_dev(dev, 854 "could not create timer workqueue (%d)\n", error); 855 goto err_out; 856 } 857 858 /* Determine hardware revision */ 859 id = pci_conf_read(pa->pa_pc, pa->pa_tag, PCI_ID_REG); 860 subid = pci_conf_read(pa->pa_pc, pa->pa_tag, PCI_SUBSYS_ID_REG); 861 862 hw->vendor_id = PCI_VENDOR(id); 863 hw->device_id = PCI_PRODUCT(id); 864 hw->revision_id = 865 PCI_REVISION(pci_conf_read(pa->pa_pc, pa->pa_tag, PCI_CLASS_REG)); 866 hw->subsystem_vendor_id = PCI_SUBSYS_VENDOR(subid); 867 hw->subsystem_device_id = PCI_SUBSYS_ID(subid); 868 869 /* Set quirk flags */ 870 ixgbe_quirks(adapter); 871 872 /* 873 * Make sure BUSMASTER is set 874 */ 875 ixgbe_pci_enable_busmaster(pa->pa_pc, pa->pa_tag); 876 877 /* Do base PCI setup - map BAR0 */ 878 if (ixgbe_allocate_pci_resources(adapter, pa)) { 879 aprint_error_dev(dev, "Allocation of PCI resources failed\n"); 880 error = ENXIO; 881 goto err_out; 882 } 883 884 /* let hardware know driver is loaded */ 885 ctrl_ext = IXGBE_READ_REG(hw, IXGBE_CTRL_EXT); 886 ctrl_ext |= IXGBE_CTRL_EXT_DRV_LOAD; 887 IXGBE_WRITE_REG(hw, IXGBE_CTRL_EXT, ctrl_ext); 888 889 /* 890 * Initialize the shared code 891 */ 892 if (ixgbe_init_shared_code(hw) != 0) { 893 aprint_error_dev(dev, "Unable to initialize the shared code\n"); 894 error = ENXIO; 895 goto err_out; 896 } 897 898 switch (hw->mac.type) { 899 case ixgbe_mac_82598EB: 900 str = "82598EB"; 901 break; 902 case ixgbe_mac_82599EB: 903 str = "82599EB"; 904 break; 905 case ixgbe_mac_X540: 906 str = "X540"; 907 break; 908 case ixgbe_mac_X550: 909 str = "X550"; 910 break; 911 case ixgbe_mac_X550EM_x: 912 str = "X550EM X"; 913 break; 914 case ixgbe_mac_X550EM_a: 915 str = "X550EM A"; 916 break; 917 default: 918 str = "Unknown"; 919 break; 920 } 921 aprint_normal_dev(dev, "device %s\n", str); 922 923 hw->allow_unsupported_sfp = allow_unsupported_sfp; 924 925 /* Pick up the 82599 settings */ 926 if (hw->mac.type != ixgbe_mac_82598EB) 927 hw->phy.smart_speed = ixgbe_smart_speed; 928 929 /* Set the right number of segments */ 930 KASSERT(IXGBE_82599_SCATTER_MAX >= IXGBE_SCATTER_DEFAULT); 931 adapter->num_segs = IXGBE_SCATTER_DEFAULT; 932 933 /* Ensure SW/FW semaphore is free */ 934 ixgbe_init_swfw_semaphore(hw); 935 936 hw->mac.ops.set_lan_id(hw); 937 ixgbe_init_device_features(adapter); 938 939 if (ixgbe_configure_interrupts(adapter)) { 940 error = ENXIO; 941 goto err_out; 942 } 943 944 /* Allocate multicast array memory. */ 945 adapter->mta = malloc(sizeof(*adapter->mta) * 946 MAX_NUM_MULTICAST_ADDRESSES, M_DEVBUF, M_WAITOK); 947 948 /* Enable WoL (if supported) */ 949 ixgbe_check_wol_support(adapter); 950 951 /* Register for VLAN events */ 952 ether_set_vlan_cb(&adapter->osdep.ec, ixgbe_vlan_cb); 953 954 /* Verify adapter fan is still functional (if applicable) */ 955 if (adapter->feat_en & IXGBE_FEATURE_FAN_FAIL) { 956 u32 esdp = IXGBE_READ_REG(hw, IXGBE_ESDP); 957 ixgbe_check_fan_failure(adapter, esdp, FALSE); 958 } 959 960 /* Set an initial default flow control value */ 961 hw->fc.requested_mode = ixgbe_flow_control; 962 963 /* Sysctls for limiting the amount of work done in the taskqueues */ 964 ixgbe_set_sysctl_value(adapter, "rx_processing_limit", 965 "max number of rx packets to process", 966 &adapter->rx_process_limit, ixgbe_rx_process_limit); 967 968 ixgbe_set_sysctl_value(adapter, "tx_processing_limit", 969 "max number of tx packets to process", 970 &adapter->tx_process_limit, ixgbe_tx_process_limit); 971 972 /* Do descriptor calc and sanity checks */ 973 if (((ixgbe_txd * sizeof(union ixgbe_adv_tx_desc)) % DBA_ALIGN) != 0 || 974 ixgbe_txd < MIN_TXD || ixgbe_txd > MAX_TXD) { 975 aprint_error_dev(dev, "TXD config issue, using default!\n"); 976 adapter->num_tx_desc = DEFAULT_TXD; 977 } else 978 adapter->num_tx_desc = ixgbe_txd; 979 980 if (((ixgbe_rxd * sizeof(union ixgbe_adv_rx_desc)) % DBA_ALIGN) != 0 || 981 ixgbe_rxd < MIN_RXD || ixgbe_rxd > MAX_RXD) { 982 aprint_error_dev(dev, "RXD config issue, using default!\n"); 983 adapter->num_rx_desc = DEFAULT_RXD; 984 } else 985 adapter->num_rx_desc = ixgbe_rxd; 986 987 /* Set default high limit of copying mbuf in rxeof */ 988 adapter->rx_copy_len = IXGBE_RX_COPY_LEN_MAX; 989 990 /* Allocate our TX/RX Queues */ 991 if (ixgbe_allocate_queues(adapter)) { 992 error = ENOMEM; 993 goto err_out; 994 } 995 996 hw->phy.reset_if_overtemp = TRUE; 997 error = ixgbe_reset_hw(hw); 998 hw->phy.reset_if_overtemp = FALSE; 999 if (error == IXGBE_ERR_SFP_NOT_PRESENT) 1000 error = IXGBE_SUCCESS; 1001 else if (error == IXGBE_ERR_SFP_NOT_SUPPORTED) { 1002 aprint_error_dev(dev, "Unsupported SFP+ module detected!\n"); 1003 unsupported_sfp = true; 1004 error = IXGBE_SUCCESS; 1005 } else if (error) { 1006 aprint_error_dev(dev, 1007 "Hardware initialization failed(error = %d)\n", error); 1008 error = EIO; 1009 goto err_late; 1010 } 1011 1012 /* Make sure we have a good EEPROM before we read from it */ 1013 if (ixgbe_validate_eeprom_checksum(&adapter->hw, NULL) < 0) { 1014 aprint_error_dev(dev, "The EEPROM Checksum Is Not Valid\n"); 1015 error = EIO; 1016 goto err_late; 1017 } 1018 1019 aprint_normal("%s:", device_xname(dev)); 1020 /* NVM Image Version */ 1021 high = low = 0; 1022 switch (hw->mac.type) { 1023 case ixgbe_mac_82598EB: 1024 /* 1025 * Print version from the dev starter version (0x29). The 1026 * location is the same as newer device's IXGBE_NVM_MAP_VER. 1027 */ 1028 hw->eeprom.ops.read(hw, IXGBE_NVM_MAP_VER, &nvmreg); 1029 if (nvmreg == 0xffff) 1030 break; 1031 high = (nvmreg >> 12) & 0x0f; 1032 low = (nvmreg >> 4) & 0xff; 1033 id = nvmreg & 0x0f; 1034 /* 1035 * The following output might not be correct. Some 82598 cards 1036 * have 0x1070 or 0x2090. 82598 spec update notes about 2.9.0. 1037 */ 1038 aprint_normal(" NVM Image Version %u.%u.%u,", high, low, id); 1039 break; 1040 case ixgbe_mac_X540: 1041 case ixgbe_mac_X550EM_a: 1042 hw->eeprom.ops.read(hw, IXGBE_NVM_IMAGE_VER, &nvmreg); 1043 if (nvmreg == 0xffff) 1044 break; 1045 high = (nvmreg >> 12) & 0x0f; 1046 low = (nvmreg >> 4) & 0xff; 1047 id = nvmreg & 0x0f; 1048 aprint_normal(" NVM Image Version %u.", high); 1049 if (hw->mac.type == ixgbe_mac_X540) 1050 str = "%x"; 1051 else 1052 str = "%02x"; 1053 aprint_normal(str, low); 1054 aprint_normal(" ID 0x%x,", id); 1055 break; 1056 case ixgbe_mac_X550EM_x: 1057 case ixgbe_mac_X550: 1058 hw->eeprom.ops.read(hw, IXGBE_NVM_IMAGE_VER, &nvmreg); 1059 if (nvmreg == 0xffff) 1060 break; 1061 high = (nvmreg >> 12) & 0x0f; 1062 low = nvmreg & 0xff; 1063 aprint_normal(" NVM Image Version %u.%02x,", high, low); 1064 break; 1065 default: 1066 break; 1067 } 1068 hw->eeprom.nvm_image_ver_high = high; 1069 hw->eeprom.nvm_image_ver_low = low; 1070 1071 /* PHY firmware revision */ 1072 switch (hw->mac.type) { 1073 case ixgbe_mac_X540: 1074 case ixgbe_mac_X550: 1075 hw->eeprom.ops.read(hw, IXGBE_PHYFW_REV, &nvmreg); 1076 if (nvmreg == 0xffff) 1077 break; 1078 high = (nvmreg >> 12) & 0x0f; 1079 low = (nvmreg >> 4) & 0xff; 1080 id = nvmreg & 0x000f; 1081 aprint_normal(" PHY FW Revision %u.", high); 1082 if (hw->mac.type == ixgbe_mac_X540) 1083 str = "%x"; 1084 else 1085 str = "%02x"; 1086 aprint_normal(str, low); 1087 aprint_normal(" ID 0x%x,", id); 1088 break; 1089 default: 1090 break; 1091 } 1092 1093 /* NVM Map version & OEM NVM Image version */ 1094 switch (hw->mac.type) { 1095 case ixgbe_mac_X550: 1096 case ixgbe_mac_X550EM_x: 1097 case ixgbe_mac_X550EM_a: 1098 hw->eeprom.ops.read(hw, IXGBE_NVM_MAP_VER, &nvmreg); 1099 if (nvmreg != 0xffff) { 1100 high = (nvmreg >> 12) & 0x0f; 1101 low = nvmreg & 0x00ff; 1102 aprint_normal(" NVM Map version %u.%02x,", high, low); 1103 } 1104 hw->eeprom.ops.read(hw, IXGBE_OEM_NVM_IMAGE_VER, &nvmreg); 1105 if (nvmreg != 0xffff) { 1106 high = (nvmreg >> 12) & 0x0f; 1107 low = nvmreg & 0x00ff; 1108 aprint_verbose(" OEM NVM Image version %u.%02x,", high, 1109 low); 1110 } 1111 break; 1112 default: 1113 break; 1114 } 1115 1116 /* Print the ETrackID */ 1117 hw->eeprom.ops.read(hw, IXGBE_ETRACKID_H, &high); 1118 hw->eeprom.ops.read(hw, IXGBE_ETRACKID_L, &low); 1119 aprint_normal(" ETrackID %08x\n", ((uint32_t)high << 16) | low); 1120 1121 /* Printed Board Assembly number */ 1122 error = ixgbe_read_pba_string(hw, buf, IXGBE_PBANUM_LENGTH); 1123 aprint_normal_dev(dev, "PBA number %s\n", error ? "unknown" : buf); 1124 1125 if (adapter->feat_en & IXGBE_FEATURE_MSIX) { 1126 error = ixgbe_allocate_msix(adapter, pa); 1127 if (error) { 1128 /* Free allocated queue structures first */ 1129 ixgbe_free_queues(adapter); 1130 1131 /* Fallback to legacy interrupt */ 1132 adapter->feat_en &= ~IXGBE_FEATURE_MSIX; 1133 if (adapter->feat_cap & IXGBE_FEATURE_MSI) 1134 adapter->feat_en |= IXGBE_FEATURE_MSI; 1135 adapter->num_queues = 1; 1136 1137 /* Allocate our TX/RX Queues again */ 1138 if (ixgbe_allocate_queues(adapter)) { 1139 error = ENOMEM; 1140 goto err_out; 1141 } 1142 } 1143 } 1144 1145 /* Recovery mode */ 1146 switch (adapter->hw.mac.type) { 1147 case ixgbe_mac_X550: 1148 case ixgbe_mac_X550EM_x: 1149 case ixgbe_mac_X550EM_a: 1150 /* >= 2.00 */ 1151 if (hw->eeprom.nvm_image_ver_high >= 2) { 1152 adapter->feat_cap |= IXGBE_FEATURE_RECOVERY_MODE; 1153 adapter->feat_en |= IXGBE_FEATURE_RECOVERY_MODE; 1154 } 1155 break; 1156 default: 1157 break; 1158 } 1159 1160 if ((adapter->feat_en & IXGBE_FEATURE_MSIX) == 0) 1161 error = ixgbe_allocate_legacy(adapter, pa); 1162 if (error) 1163 goto err_late; 1164 1165 /* Tasklets for Link, SFP, Multispeed Fiber and Flow Director */ 1166 mutex_init(&(adapter)->admin_mtx, MUTEX_DEFAULT, IPL_NET); 1167 snprintf(wqname, sizeof(wqname), "%s-admin", device_xname(dev)); 1168 error = workqueue_create(&adapter->admin_wq, wqname, 1169 ixgbe_handle_admin, adapter, IXGBE_WORKQUEUE_PRI, IPL_NET, 1170 IXGBE_TASKLET_WQ_FLAGS); 1171 if (error) { 1172 aprint_error_dev(dev, 1173 "could not create admin workqueue (%d)\n", error); 1174 goto err_out; 1175 } 1176 1177 error = ixgbe_start_hw(hw); 1178 switch (error) { 1179 case IXGBE_ERR_EEPROM_VERSION: 1180 aprint_error_dev(dev, "This device is a pre-production adapter/" 1181 "LOM. Please be aware there may be issues associated " 1182 "with your hardware.\nIf you are experiencing problems " 1183 "please contact your Intel or hardware representative " 1184 "who provided you with this hardware.\n"); 1185 break; 1186 default: 1187 break; 1188 } 1189 1190 /* Setup OS specific network interface */ 1191 if (ixgbe_setup_interface(dev, adapter) != 0) 1192 goto err_late; 1193 1194 /* 1195 * Print PHY ID only for copper PHY. On device which has SFP(+) cage 1196 * and a module is inserted, phy.id is not MII PHY id but SFF 8024 ID. 1197 */ 1198 if (hw->phy.media_type == ixgbe_media_type_copper) { 1199 uint16_t id1, id2; 1200 int oui, model, rev; 1201 char descr[MII_MAX_DESCR_LEN]; 1202 1203 id1 = hw->phy.id >> 16; 1204 id2 = hw->phy.id & 0xffff; 1205 oui = MII_OUI(id1, id2); 1206 model = MII_MODEL(id2); 1207 rev = MII_REV(id2); 1208 mii_get_descr(descr, sizeof(descr), oui, model); 1209 if (descr[0]) 1210 aprint_normal_dev(dev, "PHY: %s, rev. %d\n", 1211 descr, rev); 1212 else 1213 aprint_normal_dev(dev, 1214 "PHY OUI 0x%06x, model 0x%04x, rev. %d\n", 1215 oui, model, rev); 1216 } 1217 1218 /* Enable EEE power saving */ 1219 if (adapter->feat_cap & IXGBE_FEATURE_EEE) 1220 hw->mac.ops.setup_eee(hw, 1221 adapter->feat_en & IXGBE_FEATURE_EEE); 1222 1223 /* Enable power to the phy. */ 1224 if (!unsupported_sfp) { 1225 /* Enable the optics for 82599 SFP+ fiber */ 1226 ixgbe_enable_tx_laser(hw); 1227 1228 /* 1229 * XXX Currently, ixgbe_set_phy_power() supports only copper 1230 * PHY, so it's not required to test with !unsupported_sfp. 1231 */ 1232 ixgbe_set_phy_power(hw, TRUE); 1233 } 1234 1235 /* Initialize statistics */ 1236 ixgbe_update_stats_counters(adapter); 1237 1238 /* Check PCIE slot type/speed/width */ 1239 ixgbe_get_slot_info(adapter); 1240 1241 /* 1242 * Do time init and sysctl init here, but 1243 * only on the first port of a bypass adapter. 1244 */ 1245 ixgbe_bypass_init(adapter); 1246 1247 /* Set an initial dmac value */ 1248 adapter->dmac = 0; 1249 /* Set initial advertised speeds (if applicable) */ 1250 adapter->advertise = ixgbe_get_default_advertise(adapter); 1251 1252 if (adapter->feat_cap & IXGBE_FEATURE_SRIOV) 1253 ixgbe_define_iov_schemas(dev, &error); 1254 1255 /* Add sysctls */ 1256 ixgbe_add_device_sysctls(adapter); 1257 ixgbe_add_hw_stats(adapter); 1258 1259 /* For Netmap */ 1260 adapter->init_locked = ixgbe_init_locked; 1261 adapter->stop_locked = ixgbe_stop_locked; 1262 1263 if (adapter->feat_en & IXGBE_FEATURE_NETMAP) 1264 ixgbe_netmap_attach(adapter); 1265 1266 snprintb(buf, sizeof(buf), IXGBE_FEATURE_FLAGS, adapter->feat_cap); 1267 aprint_verbose_dev(dev, "feature cap %s\n", buf); 1268 snprintb(buf, sizeof(buf), IXGBE_FEATURE_FLAGS, adapter->feat_en); 1269 aprint_verbose_dev(dev, "feature ena %s\n", buf); 1270 1271 if (pmf_device_register(dev, ixgbe_suspend, ixgbe_resume)) 1272 pmf_class_network_register(dev, adapter->ifp); 1273 else 1274 aprint_error_dev(dev, "couldn't establish power handler\n"); 1275 1276 /* Init recovery mode timer and state variable */ 1277 if (adapter->feat_en & IXGBE_FEATURE_RECOVERY_MODE) { 1278 adapter->recovery_mode = 0; 1279 1280 /* Set up the timer callout */ 1281 callout_init(&adapter->recovery_mode_timer, 1282 IXGBE_CALLOUT_FLAGS); 1283 snprintf(wqname, sizeof(wqname), "%s-recovery", 1284 device_xname(dev)); 1285 error = workqueue_create(&adapter->recovery_mode_timer_wq, 1286 wqname, ixgbe_handle_recovery_mode_timer, adapter, 1287 IXGBE_WORKQUEUE_PRI, IPL_NET, IXGBE_TASKLET_WQ_FLAGS); 1288 if (error) { 1289 aprint_error_dev(dev, "could not create " 1290 "recovery_mode_timer workqueue (%d)\n", error); 1291 goto err_out; 1292 } 1293 1294 /* Start the task */ 1295 callout_reset(&adapter->recovery_mode_timer, hz, 1296 ixgbe_recovery_mode_timer, adapter); 1297 } 1298 1299 INIT_DEBUGOUT("ixgbe_attach: end"); 1300 adapter->osdep.attached = true; 1301 1302 return; 1303 1304 err_late: 1305 ixgbe_free_queues(adapter); 1306 err_out: 1307 ctrl_ext = IXGBE_READ_REG(&adapter->hw, IXGBE_CTRL_EXT); 1308 ctrl_ext &= ~IXGBE_CTRL_EXT_DRV_LOAD; 1309 IXGBE_WRITE_REG(&adapter->hw, IXGBE_CTRL_EXT, ctrl_ext); 1310 ixgbe_free_deferred_handlers(adapter); 1311 ixgbe_free_pci_resources(adapter); 1312 if (adapter->mta != NULL) 1313 free(adapter->mta, M_DEVBUF); 1314 mutex_destroy(&(adapter)->admin_mtx); /* XXX appropriate order? */ 1315 IXGBE_CORE_LOCK_DESTROY(adapter); 1316 1317 return; 1318 } /* ixgbe_attach */ 1319 1320 /************************************************************************ 1321 * ixgbe_check_wol_support 1322 * 1323 * Checks whether the adapter's ports are capable of 1324 * Wake On LAN by reading the adapter's NVM. 1325 * 1326 * Sets each port's hw->wol_enabled value depending 1327 * on the value read here. 1328 ************************************************************************/ 1329 static void 1330 ixgbe_check_wol_support(struct adapter *adapter) 1331 { 1332 struct ixgbe_hw *hw = &adapter->hw; 1333 u16 dev_caps = 0; 1334 1335 /* Find out WoL support for port */ 1336 adapter->wol_support = hw->wol_enabled = 0; 1337 ixgbe_get_device_caps(hw, &dev_caps); 1338 if ((dev_caps & IXGBE_DEVICE_CAPS_WOL_PORT0_1) || 1339 ((dev_caps & IXGBE_DEVICE_CAPS_WOL_PORT0) && 1340 hw->bus.func == 0)) 1341 adapter->wol_support = hw->wol_enabled = 1; 1342 1343 /* Save initial wake up filter configuration */ 1344 adapter->wufc = IXGBE_READ_REG(hw, IXGBE_WUFC); 1345 1346 return; 1347 } /* ixgbe_check_wol_support */ 1348 1349 /************************************************************************ 1350 * ixgbe_setup_interface 1351 * 1352 * Setup networking device structure and register an interface. 1353 ************************************************************************/ 1354 static int 1355 ixgbe_setup_interface(device_t dev, struct adapter *adapter) 1356 { 1357 struct ethercom *ec = &adapter->osdep.ec; 1358 struct ifnet *ifp; 1359 1360 INIT_DEBUGOUT("ixgbe_setup_interface: begin"); 1361 1362 ifp = adapter->ifp = &ec->ec_if; 1363 strlcpy(ifp->if_xname, device_xname(dev), IFNAMSIZ); 1364 ifp->if_baudrate = IF_Gbps(10); 1365 ifp->if_init = ixgbe_init; 1366 ifp->if_stop = ixgbe_ifstop; 1367 ifp->if_softc = adapter; 1368 ifp->if_flags = IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST; 1369 #ifdef IXGBE_MPSAFE 1370 ifp->if_extflags = IFEF_MPSAFE; 1371 #endif 1372 ifp->if_ioctl = ixgbe_ioctl; 1373 #if __FreeBSD_version >= 1100045 1374 /* TSO parameters */ 1375 ifp->if_hw_tsomax = 65518; 1376 ifp->if_hw_tsomaxsegcount = IXGBE_82599_SCATTER; 1377 ifp->if_hw_tsomaxsegsize = 2048; 1378 #endif 1379 if (adapter->feat_en & IXGBE_FEATURE_LEGACY_TX) { 1380 #if 0 1381 ixgbe_start_locked = ixgbe_legacy_start_locked; 1382 #endif 1383 } else { 1384 ifp->if_transmit = ixgbe_mq_start; 1385 #if 0 1386 ixgbe_start_locked = ixgbe_mq_start_locked; 1387 #endif 1388 } 1389 ifp->if_start = ixgbe_legacy_start; 1390 IFQ_SET_MAXLEN(&ifp->if_snd, adapter->num_tx_desc - 2); 1391 IFQ_SET_READY(&ifp->if_snd); 1392 1393 if_initialize(ifp); 1394 adapter->ipq = if_percpuq_create(&adapter->osdep.ec.ec_if); 1395 ether_ifattach(ifp, adapter->hw.mac.addr); 1396 aprint_normal_dev(dev, "Ethernet address %s\n", 1397 ether_sprintf(adapter->hw.mac.addr)); 1398 /* 1399 * We use per TX queue softint, so if_deferred_start_init() isn't 1400 * used. 1401 */ 1402 ether_set_ifflags_cb(ec, ixgbe_ifflags_cb); 1403 1404 adapter->max_frame_size = ifp->if_mtu + ETHER_HDR_LEN + ETHER_CRC_LEN; 1405 1406 /* 1407 * Tell the upper layer(s) we support long frames. 1408 */ 1409 ifp->if_hdrlen = sizeof(struct ether_vlan_header); 1410 1411 /* Set capability flags */ 1412 ifp->if_capabilities |= IFCAP_RXCSUM 1413 | IFCAP_TXCSUM 1414 | IFCAP_TSOv4 1415 | IFCAP_TSOv6; 1416 ifp->if_capenable = 0; 1417 1418 ec->ec_capabilities |= ETHERCAP_VLAN_HWTAGGING 1419 | ETHERCAP_VLAN_HWCSUM 1420 | ETHERCAP_JUMBO_MTU 1421 | ETHERCAP_VLAN_MTU; 1422 1423 /* Enable the above capabilities by default */ 1424 ec->ec_capenable = ec->ec_capabilities; 1425 1426 /* 1427 * Don't turn this on by default, if vlans are 1428 * created on another pseudo device (eg. lagg) 1429 * then vlan events are not passed thru, breaking 1430 * operation, but with HW FILTER off it works. If 1431 * using vlans directly on the ixgbe driver you can 1432 * enable this and get full hardware tag filtering. 1433 */ 1434 ec->ec_capabilities |= ETHERCAP_VLAN_HWFILTER; 1435 1436 /* 1437 * Specify the media types supported by this adapter and register 1438 * callbacks to update media and link information 1439 */ 1440 ec->ec_ifmedia = &adapter->media; 1441 ifmedia_init_with_lock(&adapter->media, IFM_IMASK, ixgbe_media_change, 1442 ixgbe_media_status, &adapter->core_mtx); 1443 1444 adapter->phy_layer = ixgbe_get_supported_physical_layer(&adapter->hw); 1445 ixgbe_add_media_types(adapter); 1446 1447 /* Set autoselect media by default */ 1448 ifmedia_set(&adapter->media, IFM_ETHER | IFM_AUTO); 1449 1450 if_register(ifp); 1451 1452 return (0); 1453 } /* ixgbe_setup_interface */ 1454 1455 /************************************************************************ 1456 * ixgbe_add_media_types 1457 ************************************************************************/ 1458 static void 1459 ixgbe_add_media_types(struct adapter *adapter) 1460 { 1461 struct ixgbe_hw *hw = &adapter->hw; 1462 u64 layer; 1463 1464 layer = adapter->phy_layer; 1465 1466 #define ADD(mm, dd) \ 1467 ifmedia_add(&adapter->media, IFM_ETHER | (mm), (dd), NULL); 1468 1469 ADD(IFM_NONE, 0); 1470 1471 /* Media types with matching NetBSD media defines */ 1472 if (layer & IXGBE_PHYSICAL_LAYER_10GBASE_T) { 1473 ADD(IFM_10G_T | IFM_FDX, 0); 1474 } 1475 if (layer & IXGBE_PHYSICAL_LAYER_1000BASE_T) { 1476 ADD(IFM_1000_T | IFM_FDX, 0); 1477 } 1478 if (layer & IXGBE_PHYSICAL_LAYER_100BASE_TX) { 1479 ADD(IFM_100_TX | IFM_FDX, 0); 1480 } 1481 if (layer & IXGBE_PHYSICAL_LAYER_10BASE_T) { 1482 ADD(IFM_10_T | IFM_FDX, 0); 1483 } 1484 1485 if (layer & IXGBE_PHYSICAL_LAYER_SFP_PLUS_CU || 1486 layer & IXGBE_PHYSICAL_LAYER_SFP_ACTIVE_DA) { 1487 ADD(IFM_10G_TWINAX | IFM_FDX, 0); 1488 } 1489 1490 if (layer & IXGBE_PHYSICAL_LAYER_10GBASE_LR) { 1491 ADD(IFM_10G_LR | IFM_FDX, 0); 1492 if (hw->phy.multispeed_fiber) { 1493 ADD(IFM_1000_LX | IFM_FDX, 0); 1494 } 1495 } 1496 if (layer & IXGBE_PHYSICAL_LAYER_10GBASE_SR) { 1497 ADD(IFM_10G_SR | IFM_FDX, 0); 1498 if (hw->phy.multispeed_fiber) { 1499 ADD(IFM_1000_SX | IFM_FDX, 0); 1500 } 1501 } else if (layer & IXGBE_PHYSICAL_LAYER_1000BASE_SX) { 1502 ADD(IFM_1000_SX | IFM_FDX, 0); 1503 } 1504 if (layer & IXGBE_PHYSICAL_LAYER_10GBASE_CX4) { 1505 ADD(IFM_10G_CX4 | IFM_FDX, 0); 1506 } 1507 1508 if (layer & IXGBE_PHYSICAL_LAYER_10GBASE_KR) { 1509 ADD(IFM_10G_KR | IFM_FDX, 0); 1510 } 1511 if (layer & IXGBE_PHYSICAL_LAYER_10GBASE_KX4) { 1512 ADD(IFM_10G_KX4 | IFM_FDX, 0); 1513 } 1514 if (layer & IXGBE_PHYSICAL_LAYER_1000BASE_KX) { 1515 ADD(IFM_1000_KX | IFM_FDX, 0); 1516 } 1517 if (layer & IXGBE_PHYSICAL_LAYER_2500BASE_KX) { 1518 ADD(IFM_2500_KX | IFM_FDX, 0); 1519 } 1520 if (layer & IXGBE_PHYSICAL_LAYER_2500BASE_T) { 1521 ADD(IFM_2500_T | IFM_FDX, 0); 1522 } 1523 if (layer & IXGBE_PHYSICAL_LAYER_5GBASE_T) { 1524 ADD(IFM_5000_T | IFM_FDX, 0); 1525 } 1526 if (layer & IXGBE_PHYSICAL_LAYER_1000BASE_BX) 1527 ADD(IFM_1000_BX10 | IFM_FDX, 0); 1528 /* XXX no ifmedia_set? */ 1529 1530 ADD(IFM_AUTO, 0); 1531 1532 #undef ADD 1533 } /* ixgbe_add_media_types */ 1534 1535 /************************************************************************ 1536 * ixgbe_is_sfp 1537 ************************************************************************/ 1538 static inline bool 1539 ixgbe_is_sfp(struct ixgbe_hw *hw) 1540 { 1541 switch (hw->mac.type) { 1542 case ixgbe_mac_82598EB: 1543 if (hw->phy.type == ixgbe_phy_nl) 1544 return (TRUE); 1545 return (FALSE); 1546 case ixgbe_mac_82599EB: 1547 case ixgbe_mac_X550EM_x: 1548 case ixgbe_mac_X550EM_a: 1549 switch (hw->mac.ops.get_media_type(hw)) { 1550 case ixgbe_media_type_fiber: 1551 case ixgbe_media_type_fiber_qsfp: 1552 return (TRUE); 1553 default: 1554 return (FALSE); 1555 } 1556 default: 1557 return (FALSE); 1558 } 1559 } /* ixgbe_is_sfp */ 1560 1561 static void 1562 ixgbe_schedule_admin_tasklet(struct adapter *adapter) 1563 { 1564 1565 KASSERT(mutex_owned(&adapter->admin_mtx)); 1566 1567 if (__predict_true(adapter->osdep.detaching == false)) { 1568 if (adapter->admin_pending == 0) 1569 workqueue_enqueue(adapter->admin_wq, 1570 &adapter->admin_wc, NULL); 1571 adapter->admin_pending = 1; 1572 } 1573 } 1574 1575 /************************************************************************ 1576 * ixgbe_config_link 1577 ************************************************************************/ 1578 static void 1579 ixgbe_config_link(struct adapter *adapter) 1580 { 1581 struct ixgbe_hw *hw = &adapter->hw; 1582 u32 autoneg, err = 0; 1583 u32 task_requests = 0; 1584 bool sfp, negotiate = false; 1585 1586 sfp = ixgbe_is_sfp(hw); 1587 1588 if (sfp) { 1589 if (hw->phy.multispeed_fiber) { 1590 ixgbe_enable_tx_laser(hw); 1591 task_requests |= IXGBE_REQUEST_TASK_MSF_WOI; 1592 } 1593 task_requests |= IXGBE_REQUEST_TASK_MOD_WOI; 1594 1595 mutex_enter(&adapter->admin_mtx); 1596 adapter->task_requests |= task_requests; 1597 ixgbe_schedule_admin_tasklet(adapter); 1598 mutex_exit(&adapter->admin_mtx); 1599 } else { 1600 struct ifmedia *ifm = &adapter->media; 1601 1602 if (hw->mac.ops.check_link) 1603 err = ixgbe_check_link(hw, &adapter->link_speed, 1604 &adapter->link_up, FALSE); 1605 if (err) 1606 return; 1607 1608 /* 1609 * Check if it's the first call. If it's the first call, 1610 * get value for auto negotiation. 1611 */ 1612 autoneg = hw->phy.autoneg_advertised; 1613 if ((IFM_SUBTYPE(ifm->ifm_cur->ifm_media) != IFM_NONE) 1614 && ((!autoneg) && (hw->mac.ops.get_link_capabilities))) 1615 err = hw->mac.ops.get_link_capabilities(hw, &autoneg, 1616 &negotiate); 1617 if (err) 1618 return; 1619 if (hw->mac.ops.setup_link) 1620 err = hw->mac.ops.setup_link(hw, autoneg, 1621 adapter->link_up); 1622 } 1623 1624 } /* ixgbe_config_link */ 1625 1626 /************************************************************************ 1627 * ixgbe_update_stats_counters - Update board statistics counters. 1628 ************************************************************************/ 1629 static void 1630 ixgbe_update_stats_counters(struct adapter *adapter) 1631 { 1632 struct ifnet *ifp = adapter->ifp; 1633 struct ixgbe_hw *hw = &adapter->hw; 1634 struct ixgbe_hw_stats *stats = &adapter->stats.pf; 1635 u32 missed_rx = 0, bprc, lxontxc, lxofftxc; 1636 u64 total, total_missed_rx = 0; 1637 uint64_t crcerrs, illerrc, rlec, ruc, rfc, roc, rjc; 1638 unsigned int queue_counters; 1639 int i; 1640 1641 IXGBE_EVC_REGADD2(hw, stats, IXGBE_CRCERRS, crcerrs); 1642 IXGBE_EVC_REGADD2(hw, stats, IXGBE_ILLERRC, illerrc); 1643 1644 IXGBE_EVC_REGADD(hw, stats, IXGBE_ERRBC, errbc); 1645 IXGBE_EVC_REGADD(hw, stats, IXGBE_MSPDC, mspdc); 1646 if (hw->mac.type >= ixgbe_mac_X550) 1647 IXGBE_EVC_REGADD(hw, stats, IXGBE_MBSDC, mbsdc); 1648 1649 /* 16 registers exist */ 1650 queue_counters = uimin(__arraycount(stats->qprc), adapter->num_queues); 1651 for (i = 0; i < queue_counters; i++) { 1652 IXGBE_EVC_REGADD(hw, stats, IXGBE_QPRC(i), qprc[i]); 1653 IXGBE_EVC_REGADD(hw, stats, IXGBE_QPTC(i), qptc[i]); 1654 if (hw->mac.type >= ixgbe_mac_82599EB) 1655 IXGBE_EVC_REGADD(hw, stats, IXGBE_QPRDC(i), qprdc[i]); 1656 } 1657 1658 /* 8 registers exist */ 1659 for (i = 0; i < IXGBE_TC_COUNTER_NUM; i++) { 1660 uint32_t mp; 1661 1662 /* MPC */ 1663 mp = IXGBE_READ_REG(hw, IXGBE_MPC(i)); 1664 /* global total per queue */ 1665 IXGBE_EVC_ADD(&stats->mpc[i], mp); 1666 /* running comprehensive total for stats display */ 1667 total_missed_rx += mp; 1668 1669 if (hw->mac.type == ixgbe_mac_82598EB) 1670 IXGBE_EVC_REGADD(hw, stats, IXGBE_RNBC(i), rnbc[i]); 1671 1672 IXGBE_EVC_REGADD(hw, stats, IXGBE_PXONTXC(i), pxontxc[i]); 1673 IXGBE_EVC_REGADD(hw, stats, IXGBE_PXOFFTXC(i), pxofftxc[i]); 1674 if (hw->mac.type >= ixgbe_mac_82599EB) { 1675 IXGBE_EVC_REGADD(hw, stats, IXGBE_PXONRXCNT(i), pxonrxc[i]); 1676 IXGBE_EVC_REGADD(hw, stats, IXGBE_PXOFFRXCNT(i), pxoffrxc[i]); 1677 IXGBE_EVC_REGADD(hw, stats, IXGBE_PXON2OFFCNT(i), 1678 pxon2offc[i]); 1679 } else { 1680 IXGBE_EVC_REGADD(hw, stats, IXGBE_PXONRXC(i), pxonrxc[i]); 1681 IXGBE_EVC_REGADD(hw, stats, IXGBE_PXOFFRXC(i), pxoffrxc[i]); 1682 } 1683 } 1684 IXGBE_EVC_ADD(&stats->mpctotal, total_missed_rx); 1685 1686 /* Document says M[LR]FC are valid when link is up and 10Gbps */ 1687 if ((adapter->link_active == LINK_STATE_UP) 1688 && (adapter->link_speed == IXGBE_LINK_SPEED_10GB_FULL)) { 1689 IXGBE_EVC_REGADD(hw, stats, IXGBE_MLFC, mlfc); 1690 IXGBE_EVC_REGADD(hw, stats, IXGBE_MRFC, mrfc); 1691 } 1692 IXGBE_EVC_REGADD2(hw, stats, IXGBE_RLEC, rlec); 1693 1694 /* Hardware workaround, gprc counts missed packets */ 1695 IXGBE_EVC_ADD(&stats->gprc, 1696 IXGBE_READ_REG(hw, IXGBE_GPRC) - missed_rx); 1697 1698 IXGBE_EVC_REGADD2(hw, stats, IXGBE_LXONTXC, lxontxc); 1699 IXGBE_EVC_REGADD2(hw, stats, IXGBE_LXOFFTXC, lxofftxc); 1700 total = lxontxc + lxofftxc; 1701 1702 if (hw->mac.type != ixgbe_mac_82598EB) { 1703 IXGBE_EVC_ADD(&stats->gorc, IXGBE_READ_REG(hw, IXGBE_GORCL) + 1704 ((u64)IXGBE_READ_REG(hw, IXGBE_GORCH) << 32)); 1705 IXGBE_EVC_ADD(&stats->gotc, IXGBE_READ_REG(hw, IXGBE_GOTCL) + 1706 ((u64)IXGBE_READ_REG(hw, IXGBE_GOTCH) << 32) 1707 - total * ETHER_MIN_LEN); 1708 IXGBE_EVC_ADD(&stats->tor, IXGBE_READ_REG(hw, IXGBE_TORL) + 1709 ((u64)IXGBE_READ_REG(hw, IXGBE_TORH) << 32)); 1710 IXGBE_EVC_REGADD(hw, stats, IXGBE_LXONRXCNT, lxonrxc); 1711 IXGBE_EVC_REGADD(hw, stats, IXGBE_LXOFFRXCNT, lxoffrxc); 1712 } else { 1713 IXGBE_EVC_REGADD(hw, stats, IXGBE_LXONRXC, lxonrxc); 1714 IXGBE_EVC_REGADD(hw, stats, IXGBE_LXOFFRXC, lxoffrxc); 1715 /* 82598 only has a counter in the high register */ 1716 IXGBE_EVC_REGADD(hw, stats, IXGBE_GORCH, gorc); 1717 IXGBE_EVC_ADD(&stats->gotc, IXGBE_READ_REG(hw, IXGBE_GOTCH) 1718 - total * ETHER_MIN_LEN); 1719 IXGBE_EVC_REGADD(hw, stats, IXGBE_TORH, tor); 1720 } 1721 1722 /* 1723 * Workaround: mprc hardware is incorrectly counting 1724 * broadcasts, so for now we subtract those. 1725 */ 1726 IXGBE_EVC_REGADD2(hw, stats, IXGBE_BPRC, bprc); 1727 IXGBE_EVC_ADD(&stats->mprc, IXGBE_READ_REG(hw, IXGBE_MPRC) 1728 - ((hw->mac.type == ixgbe_mac_82598EB) ? bprc : 0)); 1729 1730 IXGBE_EVC_REGADD(hw, stats, IXGBE_PRC64, prc64); 1731 IXGBE_EVC_REGADD(hw, stats, IXGBE_PRC127, prc127); 1732 IXGBE_EVC_REGADD(hw, stats, IXGBE_PRC255, prc255); 1733 IXGBE_EVC_REGADD(hw, stats, IXGBE_PRC511, prc511); 1734 IXGBE_EVC_REGADD(hw, stats, IXGBE_PRC1023, prc1023); 1735 IXGBE_EVC_REGADD(hw, stats, IXGBE_PRC1522, prc1522); 1736 1737 IXGBE_EVC_ADD(&stats->gptc, IXGBE_READ_REG(hw, IXGBE_GPTC) - total); 1738 IXGBE_EVC_ADD(&stats->mptc, IXGBE_READ_REG(hw, IXGBE_MPTC) - total); 1739 IXGBE_EVC_ADD(&stats->ptc64, IXGBE_READ_REG(hw, IXGBE_PTC64) - total); 1740 1741 IXGBE_EVC_REGADD2(hw, stats, IXGBE_RUC, ruc); 1742 IXGBE_EVC_REGADD2(hw, stats, IXGBE_RFC, rfc); 1743 IXGBE_EVC_REGADD2(hw, stats, IXGBE_ROC, roc); 1744 IXGBE_EVC_REGADD2(hw, stats, IXGBE_RJC, rjc); 1745 1746 IXGBE_EVC_REGADD(hw, stats, IXGBE_MNGPRC, mngprc); 1747 IXGBE_EVC_REGADD(hw, stats, IXGBE_MNGPDC, mngpdc); 1748 IXGBE_EVC_REGADD(hw, stats, IXGBE_MNGPTC, mngptc); 1749 IXGBE_EVC_REGADD(hw, stats, IXGBE_TPR, tpr); 1750 IXGBE_EVC_REGADD(hw, stats, IXGBE_TPT, tpt); 1751 IXGBE_EVC_REGADD(hw, stats, IXGBE_PTC127, ptc127); 1752 IXGBE_EVC_REGADD(hw, stats, IXGBE_PTC255, ptc255); 1753 IXGBE_EVC_REGADD(hw, stats, IXGBE_PTC511, ptc511); 1754 IXGBE_EVC_REGADD(hw, stats, IXGBE_PTC1023, ptc1023); 1755 IXGBE_EVC_REGADD(hw, stats, IXGBE_PTC1522, ptc1522); 1756 IXGBE_EVC_REGADD(hw, stats, IXGBE_BPTC, bptc); 1757 IXGBE_EVC_REGADD(hw, stats, IXGBE_XEC, xec); 1758 IXGBE_EVC_REGADD(hw, stats, IXGBE_FCCRC, fccrc); 1759 IXGBE_EVC_REGADD(hw, stats, IXGBE_FCLAST, fclast); 1760 /* Only read FCOE on 82599 */ 1761 if (hw->mac.type != ixgbe_mac_82598EB) { 1762 IXGBE_EVC_REGADD(hw, stats, IXGBE_FCOERPDC, fcoerpdc); 1763 IXGBE_EVC_REGADD(hw, stats, IXGBE_FCOEPRC, fcoeprc); 1764 IXGBE_EVC_REGADD(hw, stats, IXGBE_FCOEPTC, fcoeptc); 1765 IXGBE_EVC_REGADD(hw, stats, IXGBE_FCOEDWRC, fcoedwrc); 1766 IXGBE_EVC_REGADD(hw, stats, IXGBE_FCOEDWTC, fcoedwtc); 1767 } 1768 1769 /* 1770 * Fill out the OS statistics structure. Only RX errors are required 1771 * here because all TX counters are incremented in the TX path and 1772 * normal RX counters are prepared in ether_input(). 1773 */ 1774 net_stat_ref_t nsr = IF_STAT_GETREF(ifp); 1775 if_statadd_ref(nsr, if_iqdrops, total_missed_rx); 1776 1777 /* 1778 * Aggregate following types of errors as RX errors: 1779 * - CRC error count, 1780 * - illegal byte error count, 1781 * - length error count, 1782 * - undersized packets count, 1783 * - fragmented packets count, 1784 * - oversized packets count, 1785 * - jabber count. 1786 */ 1787 if_statadd_ref(nsr, if_ierrors, 1788 crcerrs + illerrc + rlec + ruc + rfc + roc + rjc); 1789 1790 IF_STAT_PUTREF(ifp); 1791 } /* ixgbe_update_stats_counters */ 1792 1793 /************************************************************************ 1794 * ixgbe_add_hw_stats 1795 * 1796 * Add sysctl variables, one per statistic, to the system. 1797 ************************************************************************/ 1798 static void 1799 ixgbe_add_hw_stats(struct adapter *adapter) 1800 { 1801 device_t dev = adapter->dev; 1802 const struct sysctlnode *rnode, *cnode; 1803 struct sysctllog **log = &adapter->sysctllog; 1804 struct tx_ring *txr = adapter->tx_rings; 1805 struct rx_ring *rxr = adapter->rx_rings; 1806 struct ixgbe_hw *hw = &adapter->hw; 1807 struct ixgbe_hw_stats *stats = &adapter->stats.pf; 1808 const char *xname = device_xname(dev); 1809 int i; 1810 1811 /* Driver Statistics */ 1812 evcnt_attach_dynamic(&adapter->efbig_tx_dma_setup, EVCNT_TYPE_MISC, 1813 NULL, xname, "Driver tx dma soft fail EFBIG"); 1814 evcnt_attach_dynamic(&adapter->mbuf_defrag_failed, EVCNT_TYPE_MISC, 1815 NULL, xname, "m_defrag() failed"); 1816 evcnt_attach_dynamic(&adapter->efbig2_tx_dma_setup, EVCNT_TYPE_MISC, 1817 NULL, xname, "Driver tx dma hard fail EFBIG"); 1818 evcnt_attach_dynamic(&adapter->einval_tx_dma_setup, EVCNT_TYPE_MISC, 1819 NULL, xname, "Driver tx dma hard fail EINVAL"); 1820 evcnt_attach_dynamic(&adapter->other_tx_dma_setup, EVCNT_TYPE_MISC, 1821 NULL, xname, "Driver tx dma hard fail other"); 1822 evcnt_attach_dynamic(&adapter->eagain_tx_dma_setup, EVCNT_TYPE_MISC, 1823 NULL, xname, "Driver tx dma soft fail EAGAIN"); 1824 evcnt_attach_dynamic(&adapter->enomem_tx_dma_setup, EVCNT_TYPE_MISC, 1825 NULL, xname, "Driver tx dma soft fail ENOMEM"); 1826 evcnt_attach_dynamic(&adapter->watchdog_events, EVCNT_TYPE_MISC, 1827 NULL, xname, "Watchdog timeouts"); 1828 evcnt_attach_dynamic(&adapter->tso_err, EVCNT_TYPE_MISC, 1829 NULL, xname, "TSO errors"); 1830 evcnt_attach_dynamic(&adapter->admin_irqev, EVCNT_TYPE_INTR, 1831 NULL, xname, "Admin MSI-X IRQ Handled"); 1832 evcnt_attach_dynamic(&adapter->link_workev, EVCNT_TYPE_INTR, 1833 NULL, xname, "Link event"); 1834 evcnt_attach_dynamic(&adapter->mod_workev, EVCNT_TYPE_INTR, 1835 NULL, xname, "SFP+ module event"); 1836 evcnt_attach_dynamic(&adapter->msf_workev, EVCNT_TYPE_INTR, 1837 NULL, xname, "Multispeed event"); 1838 evcnt_attach_dynamic(&adapter->phy_workev, EVCNT_TYPE_INTR, 1839 NULL, xname, "External PHY event"); 1840 1841 /* Max number of traffic class is 8 */ 1842 KASSERT(IXGBE_DCB_MAX_TRAFFIC_CLASS == 8); 1843 for (i = 0; i < IXGBE_TC_COUNTER_NUM; i++) { 1844 snprintf(adapter->tcs[i].evnamebuf, 1845 sizeof(adapter->tcs[i].evnamebuf), "%s tc%d", 1846 xname, i); 1847 if (i < __arraycount(stats->mpc)) { 1848 evcnt_attach_dynamic(&stats->mpc[i], 1849 EVCNT_TYPE_MISC, NULL, adapter->tcs[i].evnamebuf, 1850 "RX Missed Packet Count"); 1851 if (hw->mac.type == ixgbe_mac_82598EB) 1852 evcnt_attach_dynamic(&stats->rnbc[i], 1853 EVCNT_TYPE_MISC, NULL, 1854 adapter->tcs[i].evnamebuf, 1855 "Receive No Buffers"); 1856 } 1857 if (i < __arraycount(stats->pxontxc)) { 1858 evcnt_attach_dynamic(&stats->pxontxc[i], 1859 EVCNT_TYPE_MISC, NULL, adapter->tcs[i].evnamebuf, 1860 "pxontxc"); 1861 evcnt_attach_dynamic(&stats->pxonrxc[i], 1862 EVCNT_TYPE_MISC, NULL, adapter->tcs[i].evnamebuf, 1863 "pxonrxc"); 1864 evcnt_attach_dynamic(&stats->pxofftxc[i], 1865 EVCNT_TYPE_MISC, NULL, adapter->tcs[i].evnamebuf, 1866 "pxofftxc"); 1867 evcnt_attach_dynamic(&stats->pxoffrxc[i], 1868 EVCNT_TYPE_MISC, NULL, adapter->tcs[i].evnamebuf, 1869 "pxoffrxc"); 1870 if (hw->mac.type >= ixgbe_mac_82599EB) 1871 evcnt_attach_dynamic(&stats->pxon2offc[i], 1872 EVCNT_TYPE_MISC, NULL, 1873 adapter->tcs[i].evnamebuf, 1874 "pxon2offc"); 1875 } 1876 } 1877 1878 for (i = 0; i < adapter->num_queues; i++, rxr++, txr++) { 1879 #ifdef LRO 1880 struct lro_ctrl *lro = &rxr->lro; 1881 #endif /* LRO */ 1882 1883 snprintf(adapter->queues[i].evnamebuf, 1884 sizeof(adapter->queues[i].evnamebuf), "%s q%d", 1885 xname, i); 1886 snprintf(adapter->queues[i].namebuf, 1887 sizeof(adapter->queues[i].namebuf), "q%d", i); 1888 1889 if ((rnode = ixgbe_sysctl_instance(adapter)) == NULL) { 1890 aprint_error_dev(dev, "could not create sysctl root\n"); 1891 break; 1892 } 1893 1894 if (sysctl_createv(log, 0, &rnode, &rnode, 1895 0, CTLTYPE_NODE, 1896 adapter->queues[i].namebuf, SYSCTL_DESCR("Queue Name"), 1897 NULL, 0, NULL, 0, CTL_CREATE, CTL_EOL) != 0) 1898 break; 1899 1900 if (sysctl_createv(log, 0, &rnode, &cnode, 1901 CTLFLAG_READWRITE, CTLTYPE_INT, 1902 "interrupt_rate", SYSCTL_DESCR("Interrupt Rate"), 1903 ixgbe_sysctl_interrupt_rate_handler, 0, 1904 (void *)&adapter->queues[i], 0, CTL_CREATE, CTL_EOL) != 0) 1905 break; 1906 1907 if (sysctl_createv(log, 0, &rnode, &cnode, 1908 CTLFLAG_READONLY, CTLTYPE_INT, 1909 "txd_head", SYSCTL_DESCR("Transmit Descriptor Head"), 1910 ixgbe_sysctl_tdh_handler, 0, (void *)txr, 1911 0, CTL_CREATE, CTL_EOL) != 0) 1912 break; 1913 1914 if (sysctl_createv(log, 0, &rnode, &cnode, 1915 CTLFLAG_READONLY, CTLTYPE_INT, 1916 "txd_tail", SYSCTL_DESCR("Transmit Descriptor Tail"), 1917 ixgbe_sysctl_tdt_handler, 0, (void *)txr, 1918 0, CTL_CREATE, CTL_EOL) != 0) 1919 break; 1920 1921 evcnt_attach_dynamic(&adapter->queues[i].irqs, EVCNT_TYPE_INTR, 1922 NULL, adapter->queues[i].evnamebuf, "IRQs on queue"); 1923 evcnt_attach_dynamic(&adapter->queues[i].handleq, 1924 EVCNT_TYPE_MISC, NULL, adapter->queues[i].evnamebuf, 1925 "Handled queue in softint"); 1926 evcnt_attach_dynamic(&adapter->queues[i].req, EVCNT_TYPE_MISC, 1927 NULL, adapter->queues[i].evnamebuf, "Requeued in softint"); 1928 evcnt_attach_dynamic(&txr->tso_tx, EVCNT_TYPE_MISC, 1929 NULL, adapter->queues[i].evnamebuf, "TSO"); 1930 evcnt_attach_dynamic(&txr->no_desc_avail, EVCNT_TYPE_MISC, 1931 NULL, adapter->queues[i].evnamebuf, 1932 "TX Queue No Descriptor Available"); 1933 evcnt_attach_dynamic(&txr->total_packets, EVCNT_TYPE_MISC, 1934 NULL, adapter->queues[i].evnamebuf, 1935 "Queue Packets Transmitted"); 1936 #ifndef IXGBE_LEGACY_TX 1937 evcnt_attach_dynamic(&txr->pcq_drops, EVCNT_TYPE_MISC, 1938 NULL, adapter->queues[i].evnamebuf, 1939 "Packets dropped in pcq"); 1940 #endif 1941 1942 if (sysctl_createv(log, 0, &rnode, &cnode, 1943 CTLFLAG_READONLY, CTLTYPE_INT, "rxd_nxck", 1944 SYSCTL_DESCR("Receive Descriptor next to check"), 1945 ixgbe_sysctl_next_to_check_handler, 0, (void *)rxr, 0, 1946 CTL_CREATE, CTL_EOL) != 0) 1947 break; 1948 1949 if (sysctl_createv(log, 0, &rnode, &cnode, 1950 CTLFLAG_READONLY, CTLTYPE_INT, "rxd_nxrf", 1951 SYSCTL_DESCR("Receive Descriptor next to refresh"), 1952 ixgbe_sysctl_next_to_refresh_handler, 0, (void *)rxr, 0, 1953 CTL_CREATE, CTL_EOL) != 0) 1954 break; 1955 1956 if (sysctl_createv(log, 0, &rnode, &cnode, 1957 CTLFLAG_READONLY, CTLTYPE_INT, "rxd_head", 1958 SYSCTL_DESCR("Receive Descriptor Head"), 1959 ixgbe_sysctl_rdh_handler, 0, (void *)rxr, 0, 1960 CTL_CREATE, CTL_EOL) != 0) 1961 break; 1962 1963 if (sysctl_createv(log, 0, &rnode, &cnode, 1964 CTLFLAG_READONLY, CTLTYPE_INT, "rxd_tail", 1965 SYSCTL_DESCR("Receive Descriptor Tail"), 1966 ixgbe_sysctl_rdt_handler, 0, (void *)rxr, 0, 1967 CTL_CREATE, CTL_EOL) != 0) 1968 break; 1969 1970 if (i < __arraycount(stats->qprc)) { 1971 evcnt_attach_dynamic(&stats->qprc[i], EVCNT_TYPE_MISC, 1972 NULL, adapter->queues[i].evnamebuf, "qprc"); 1973 evcnt_attach_dynamic(&stats->qptc[i], EVCNT_TYPE_MISC, 1974 NULL, adapter->queues[i].evnamebuf, "qptc"); 1975 evcnt_attach_dynamic(&stats->qbrc[i], EVCNT_TYPE_MISC, 1976 NULL, adapter->queues[i].evnamebuf, "qbrc"); 1977 evcnt_attach_dynamic(&stats->qbtc[i], EVCNT_TYPE_MISC, 1978 NULL, adapter->queues[i].evnamebuf, "qbtc"); 1979 if (hw->mac.type >= ixgbe_mac_82599EB) 1980 evcnt_attach_dynamic(&stats->qprdc[i], 1981 EVCNT_TYPE_MISC, NULL, 1982 adapter->queues[i].evnamebuf, "qprdc"); 1983 } 1984 1985 evcnt_attach_dynamic(&rxr->rx_packets, EVCNT_TYPE_MISC, 1986 NULL, adapter->queues[i].evnamebuf, 1987 "Queue Packets Received"); 1988 evcnt_attach_dynamic(&rxr->rx_bytes, EVCNT_TYPE_MISC, 1989 NULL, adapter->queues[i].evnamebuf, 1990 "Queue Bytes Received"); 1991 evcnt_attach_dynamic(&rxr->rx_copies, EVCNT_TYPE_MISC, 1992 NULL, adapter->queues[i].evnamebuf, "Copied RX Frames"); 1993 evcnt_attach_dynamic(&rxr->no_mbuf, EVCNT_TYPE_MISC, 1994 NULL, adapter->queues[i].evnamebuf, "Rx no mbuf"); 1995 evcnt_attach_dynamic(&rxr->rx_discarded, EVCNT_TYPE_MISC, 1996 NULL, adapter->queues[i].evnamebuf, "Rx discarded"); 1997 #ifdef LRO 1998 SYSCTL_ADD_INT(ctx, queue_list, OID_AUTO, "lro_queued", 1999 CTLFLAG_RD, &lro->lro_queued, 0, 2000 "LRO Queued"); 2001 SYSCTL_ADD_INT(ctx, queue_list, OID_AUTO, "lro_flushed", 2002 CTLFLAG_RD, &lro->lro_flushed, 0, 2003 "LRO Flushed"); 2004 #endif /* LRO */ 2005 } 2006 2007 /* MAC stats get their own sub node */ 2008 2009 snprintf(stats->namebuf, 2010 sizeof(stats->namebuf), "%s MAC Statistics", xname); 2011 2012 evcnt_attach_dynamic(&stats->ipcs, EVCNT_TYPE_MISC, NULL, 2013 stats->namebuf, "rx csum offload - IP"); 2014 evcnt_attach_dynamic(&stats->l4cs, EVCNT_TYPE_MISC, NULL, 2015 stats->namebuf, "rx csum offload - L4"); 2016 evcnt_attach_dynamic(&stats->ipcs_bad, EVCNT_TYPE_MISC, NULL, 2017 stats->namebuf, "rx csum offload - IP bad"); 2018 evcnt_attach_dynamic(&stats->l4cs_bad, EVCNT_TYPE_MISC, NULL, 2019 stats->namebuf, "rx csum offload - L4 bad"); 2020 evcnt_attach_dynamic(&stats->intzero, EVCNT_TYPE_MISC, NULL, 2021 stats->namebuf, "Interrupt conditions zero"); 2022 evcnt_attach_dynamic(&stats->legint, EVCNT_TYPE_MISC, NULL, 2023 stats->namebuf, "Legacy interrupts"); 2024 2025 evcnt_attach_dynamic(&stats->crcerrs, EVCNT_TYPE_MISC, NULL, 2026 stats->namebuf, "CRC Errors"); 2027 evcnt_attach_dynamic(&stats->illerrc, EVCNT_TYPE_MISC, NULL, 2028 stats->namebuf, "Illegal Byte Errors"); 2029 evcnt_attach_dynamic(&stats->errbc, EVCNT_TYPE_MISC, NULL, 2030 stats->namebuf, "Byte Errors"); 2031 evcnt_attach_dynamic(&stats->mspdc, EVCNT_TYPE_MISC, NULL, 2032 stats->namebuf, "MAC Short Packets Discarded"); 2033 if (hw->mac.type >= ixgbe_mac_X550) 2034 evcnt_attach_dynamic(&stats->mbsdc, EVCNT_TYPE_MISC, NULL, 2035 stats->namebuf, "Bad SFD"); 2036 evcnt_attach_dynamic(&stats->mpctotal, EVCNT_TYPE_MISC, NULL, 2037 stats->namebuf, "Total Packets Missed"); 2038 evcnt_attach_dynamic(&stats->mlfc, EVCNT_TYPE_MISC, NULL, 2039 stats->namebuf, "MAC Local Faults"); 2040 evcnt_attach_dynamic(&stats->mrfc, EVCNT_TYPE_MISC, NULL, 2041 stats->namebuf, "MAC Remote Faults"); 2042 evcnt_attach_dynamic(&stats->rlec, EVCNT_TYPE_MISC, NULL, 2043 stats->namebuf, "Receive Length Errors"); 2044 evcnt_attach_dynamic(&stats->lxontxc, EVCNT_TYPE_MISC, NULL, 2045 stats->namebuf, "Link XON Transmitted"); 2046 evcnt_attach_dynamic(&stats->lxonrxc, EVCNT_TYPE_MISC, NULL, 2047 stats->namebuf, "Link XON Received"); 2048 evcnt_attach_dynamic(&stats->lxofftxc, EVCNT_TYPE_MISC, NULL, 2049 stats->namebuf, "Link XOFF Transmitted"); 2050 evcnt_attach_dynamic(&stats->lxoffrxc, EVCNT_TYPE_MISC, NULL, 2051 stats->namebuf, "Link XOFF Received"); 2052 2053 /* Packet Reception Stats */ 2054 evcnt_attach_dynamic(&stats->tor, EVCNT_TYPE_MISC, NULL, 2055 stats->namebuf, "Total Octets Received"); 2056 evcnt_attach_dynamic(&stats->gorc, EVCNT_TYPE_MISC, NULL, 2057 stats->namebuf, "Good Octets Received"); 2058 evcnt_attach_dynamic(&stats->tpr, EVCNT_TYPE_MISC, NULL, 2059 stats->namebuf, "Total Packets Received"); 2060 evcnt_attach_dynamic(&stats->gprc, EVCNT_TYPE_MISC, NULL, 2061 stats->namebuf, "Good Packets Received"); 2062 evcnt_attach_dynamic(&stats->mprc, EVCNT_TYPE_MISC, NULL, 2063 stats->namebuf, "Multicast Packets Received"); 2064 evcnt_attach_dynamic(&stats->bprc, EVCNT_TYPE_MISC, NULL, 2065 stats->namebuf, "Broadcast Packets Received"); 2066 evcnt_attach_dynamic(&stats->prc64, EVCNT_TYPE_MISC, NULL, 2067 stats->namebuf, "64 byte frames received "); 2068 evcnt_attach_dynamic(&stats->prc127, EVCNT_TYPE_MISC, NULL, 2069 stats->namebuf, "65-127 byte frames received"); 2070 evcnt_attach_dynamic(&stats->prc255, EVCNT_TYPE_MISC, NULL, 2071 stats->namebuf, "128-255 byte frames received"); 2072 evcnt_attach_dynamic(&stats->prc511, EVCNT_TYPE_MISC, NULL, 2073 stats->namebuf, "256-511 byte frames received"); 2074 evcnt_attach_dynamic(&stats->prc1023, EVCNT_TYPE_MISC, NULL, 2075 stats->namebuf, "512-1023 byte frames received"); 2076 evcnt_attach_dynamic(&stats->prc1522, EVCNT_TYPE_MISC, NULL, 2077 stats->namebuf, "1023-1522 byte frames received"); 2078 evcnt_attach_dynamic(&stats->ruc, EVCNT_TYPE_MISC, NULL, 2079 stats->namebuf, "Receive Undersized"); 2080 evcnt_attach_dynamic(&stats->rfc, EVCNT_TYPE_MISC, NULL, 2081 stats->namebuf, "Fragmented Packets Received "); 2082 evcnt_attach_dynamic(&stats->roc, EVCNT_TYPE_MISC, NULL, 2083 stats->namebuf, "Oversized Packets Received"); 2084 evcnt_attach_dynamic(&stats->rjc, EVCNT_TYPE_MISC, NULL, 2085 stats->namebuf, "Received Jabber"); 2086 evcnt_attach_dynamic(&stats->mngprc, EVCNT_TYPE_MISC, NULL, 2087 stats->namebuf, "Management Packets Received"); 2088 evcnt_attach_dynamic(&stats->mngpdc, EVCNT_TYPE_MISC, NULL, 2089 stats->namebuf, "Management Packets Dropped"); 2090 evcnt_attach_dynamic(&stats->xec, EVCNT_TYPE_MISC, NULL, 2091 stats->namebuf, "Checksum Errors"); 2092 2093 /* Packet Transmission Stats */ 2094 evcnt_attach_dynamic(&stats->gotc, EVCNT_TYPE_MISC, NULL, 2095 stats->namebuf, "Good Octets Transmitted"); 2096 evcnt_attach_dynamic(&stats->tpt, EVCNT_TYPE_MISC, NULL, 2097 stats->namebuf, "Total Packets Transmitted"); 2098 evcnt_attach_dynamic(&stats->gptc, EVCNT_TYPE_MISC, NULL, 2099 stats->namebuf, "Good Packets Transmitted"); 2100 evcnt_attach_dynamic(&stats->bptc, EVCNT_TYPE_MISC, NULL, 2101 stats->namebuf, "Broadcast Packets Transmitted"); 2102 evcnt_attach_dynamic(&stats->mptc, EVCNT_TYPE_MISC, NULL, 2103 stats->namebuf, "Multicast Packets Transmitted"); 2104 evcnt_attach_dynamic(&stats->mngptc, EVCNT_TYPE_MISC, NULL, 2105 stats->namebuf, "Management Packets Transmitted"); 2106 evcnt_attach_dynamic(&stats->ptc64, EVCNT_TYPE_MISC, NULL, 2107 stats->namebuf, "64 byte frames transmitted "); 2108 evcnt_attach_dynamic(&stats->ptc127, EVCNT_TYPE_MISC, NULL, 2109 stats->namebuf, "65-127 byte frames transmitted"); 2110 evcnt_attach_dynamic(&stats->ptc255, EVCNT_TYPE_MISC, NULL, 2111 stats->namebuf, "128-255 byte frames transmitted"); 2112 evcnt_attach_dynamic(&stats->ptc511, EVCNT_TYPE_MISC, NULL, 2113 stats->namebuf, "256-511 byte frames transmitted"); 2114 evcnt_attach_dynamic(&stats->ptc1023, EVCNT_TYPE_MISC, NULL, 2115 stats->namebuf, "512-1023 byte frames transmitted"); 2116 evcnt_attach_dynamic(&stats->ptc1522, EVCNT_TYPE_MISC, NULL, 2117 stats->namebuf, "1024-1522 byte frames transmitted"); 2118 } /* ixgbe_add_hw_stats */ 2119 2120 static void 2121 ixgbe_clear_evcnt(struct adapter *adapter) 2122 { 2123 struct tx_ring *txr = adapter->tx_rings; 2124 struct rx_ring *rxr = adapter->rx_rings; 2125 struct ixgbe_hw *hw = &adapter->hw; 2126 struct ixgbe_hw_stats *stats = &adapter->stats.pf; 2127 int i; 2128 2129 IXGBE_EVC_STORE(&adapter->efbig_tx_dma_setup, 0); 2130 IXGBE_EVC_STORE(&adapter->mbuf_defrag_failed, 0); 2131 IXGBE_EVC_STORE(&adapter->efbig2_tx_dma_setup, 0); 2132 IXGBE_EVC_STORE(&adapter->einval_tx_dma_setup, 0); 2133 IXGBE_EVC_STORE(&adapter->other_tx_dma_setup, 0); 2134 IXGBE_EVC_STORE(&adapter->eagain_tx_dma_setup, 0); 2135 IXGBE_EVC_STORE(&adapter->enomem_tx_dma_setup, 0); 2136 IXGBE_EVC_STORE(&adapter->tso_err, 0); 2137 IXGBE_EVC_STORE(&adapter->watchdog_events, 0); 2138 IXGBE_EVC_STORE(&adapter->admin_irqev, 0); 2139 IXGBE_EVC_STORE(&adapter->link_workev, 0); 2140 IXGBE_EVC_STORE(&adapter->mod_workev, 0); 2141 IXGBE_EVC_STORE(&adapter->msf_workev, 0); 2142 IXGBE_EVC_STORE(&adapter->phy_workev, 0); 2143 2144 for (i = 0; i < IXGBE_TC_COUNTER_NUM; i++) { 2145 if (i < __arraycount(stats->mpc)) { 2146 IXGBE_EVC_STORE(&stats->mpc[i], 0); 2147 if (hw->mac.type == ixgbe_mac_82598EB) 2148 IXGBE_EVC_STORE(&stats->rnbc[i], 0); 2149 } 2150 if (i < __arraycount(stats->pxontxc)) { 2151 IXGBE_EVC_STORE(&stats->pxontxc[i], 0); 2152 IXGBE_EVC_STORE(&stats->pxonrxc[i], 0); 2153 IXGBE_EVC_STORE(&stats->pxofftxc[i], 0); 2154 IXGBE_EVC_STORE(&stats->pxoffrxc[i], 0); 2155 if (hw->mac.type >= ixgbe_mac_82599EB) 2156 IXGBE_EVC_STORE(&stats->pxon2offc[i], 0); 2157 } 2158 } 2159 2160 txr = adapter->tx_rings; 2161 for (i = 0; i < adapter->num_queues; i++, rxr++, txr++) { 2162 IXGBE_EVC_STORE(&adapter->queues[i].irqs, 0); 2163 IXGBE_EVC_STORE(&adapter->queues[i].handleq, 0); 2164 IXGBE_EVC_STORE(&adapter->queues[i].req, 0); 2165 IXGBE_EVC_STORE(&txr->no_desc_avail, 0); 2166 IXGBE_EVC_STORE(&txr->total_packets, 0); 2167 IXGBE_EVC_STORE(&txr->tso_tx, 0); 2168 #ifndef IXGBE_LEGACY_TX 2169 IXGBE_EVC_STORE(&txr->pcq_drops, 0); 2170 #endif 2171 txr->q_efbig_tx_dma_setup = 0; 2172 txr->q_mbuf_defrag_failed = 0; 2173 txr->q_efbig2_tx_dma_setup = 0; 2174 txr->q_einval_tx_dma_setup = 0; 2175 txr->q_other_tx_dma_setup = 0; 2176 txr->q_eagain_tx_dma_setup = 0; 2177 txr->q_enomem_tx_dma_setup = 0; 2178 txr->q_tso_err = 0; 2179 2180 if (i < __arraycount(stats->qprc)) { 2181 IXGBE_EVC_STORE(&stats->qprc[i], 0); 2182 IXGBE_EVC_STORE(&stats->qptc[i], 0); 2183 IXGBE_EVC_STORE(&stats->qbrc[i], 0); 2184 IXGBE_EVC_STORE(&stats->qbtc[i], 0); 2185 if (hw->mac.type >= ixgbe_mac_82599EB) 2186 IXGBE_EVC_STORE(&stats->qprdc[i], 0); 2187 } 2188 2189 IXGBE_EVC_STORE(&rxr->rx_packets, 0); 2190 IXGBE_EVC_STORE(&rxr->rx_bytes, 0); 2191 IXGBE_EVC_STORE(&rxr->rx_copies, 0); 2192 IXGBE_EVC_STORE(&rxr->no_mbuf, 0); 2193 IXGBE_EVC_STORE(&rxr->rx_discarded, 0); 2194 } 2195 IXGBE_EVC_STORE(&stats->ipcs, 0); 2196 IXGBE_EVC_STORE(&stats->l4cs, 0); 2197 IXGBE_EVC_STORE(&stats->ipcs_bad, 0); 2198 IXGBE_EVC_STORE(&stats->l4cs_bad, 0); 2199 IXGBE_EVC_STORE(&stats->intzero, 0); 2200 IXGBE_EVC_STORE(&stats->legint, 0); 2201 IXGBE_EVC_STORE(&stats->crcerrs, 0); 2202 IXGBE_EVC_STORE(&stats->illerrc, 0); 2203 IXGBE_EVC_STORE(&stats->errbc, 0); 2204 IXGBE_EVC_STORE(&stats->mspdc, 0); 2205 if (hw->mac.type >= ixgbe_mac_X550) 2206 IXGBE_EVC_STORE(&stats->mbsdc, 0); 2207 IXGBE_EVC_STORE(&stats->mpctotal, 0); 2208 IXGBE_EVC_STORE(&stats->mlfc, 0); 2209 IXGBE_EVC_STORE(&stats->mrfc, 0); 2210 IXGBE_EVC_STORE(&stats->rlec, 0); 2211 IXGBE_EVC_STORE(&stats->lxontxc, 0); 2212 IXGBE_EVC_STORE(&stats->lxonrxc, 0); 2213 IXGBE_EVC_STORE(&stats->lxofftxc, 0); 2214 IXGBE_EVC_STORE(&stats->lxoffrxc, 0); 2215 2216 /* Packet Reception Stats */ 2217 IXGBE_EVC_STORE(&stats->tor, 0); 2218 IXGBE_EVC_STORE(&stats->gorc, 0); 2219 IXGBE_EVC_STORE(&stats->tpr, 0); 2220 IXGBE_EVC_STORE(&stats->gprc, 0); 2221 IXGBE_EVC_STORE(&stats->mprc, 0); 2222 IXGBE_EVC_STORE(&stats->bprc, 0); 2223 IXGBE_EVC_STORE(&stats->prc64, 0); 2224 IXGBE_EVC_STORE(&stats->prc127, 0); 2225 IXGBE_EVC_STORE(&stats->prc255, 0); 2226 IXGBE_EVC_STORE(&stats->prc511, 0); 2227 IXGBE_EVC_STORE(&stats->prc1023, 0); 2228 IXGBE_EVC_STORE(&stats->prc1522, 0); 2229 IXGBE_EVC_STORE(&stats->ruc, 0); 2230 IXGBE_EVC_STORE(&stats->rfc, 0); 2231 IXGBE_EVC_STORE(&stats->roc, 0); 2232 IXGBE_EVC_STORE(&stats->rjc, 0); 2233 IXGBE_EVC_STORE(&stats->mngprc, 0); 2234 IXGBE_EVC_STORE(&stats->mngpdc, 0); 2235 IXGBE_EVC_STORE(&stats->xec, 0); 2236 2237 /* Packet Transmission Stats */ 2238 IXGBE_EVC_STORE(&stats->gotc, 0); 2239 IXGBE_EVC_STORE(&stats->tpt, 0); 2240 IXGBE_EVC_STORE(&stats->gptc, 0); 2241 IXGBE_EVC_STORE(&stats->bptc, 0); 2242 IXGBE_EVC_STORE(&stats->mptc, 0); 2243 IXGBE_EVC_STORE(&stats->mngptc, 0); 2244 IXGBE_EVC_STORE(&stats->ptc64, 0); 2245 IXGBE_EVC_STORE(&stats->ptc127, 0); 2246 IXGBE_EVC_STORE(&stats->ptc255, 0); 2247 IXGBE_EVC_STORE(&stats->ptc511, 0); 2248 IXGBE_EVC_STORE(&stats->ptc1023, 0); 2249 IXGBE_EVC_STORE(&stats->ptc1522, 0); 2250 } 2251 2252 /************************************************************************ 2253 * ixgbe_sysctl_tdh_handler - Transmit Descriptor Head handler function 2254 * 2255 * Retrieves the TDH value from the hardware 2256 ************************************************************************/ 2257 static int 2258 ixgbe_sysctl_tdh_handler(SYSCTLFN_ARGS) 2259 { 2260 struct sysctlnode node = *rnode; 2261 struct tx_ring *txr = (struct tx_ring *)node.sysctl_data; 2262 struct adapter *adapter; 2263 uint32_t val; 2264 2265 if (!txr) 2266 return (0); 2267 2268 adapter = txr->adapter; 2269 if (ixgbe_fw_recovery_mode_swflag(adapter)) 2270 return (EPERM); 2271 2272 val = IXGBE_READ_REG(&adapter->hw, IXGBE_TDH(txr->me)); 2273 node.sysctl_data = &val; 2274 return sysctl_lookup(SYSCTLFN_CALL(&node)); 2275 } /* ixgbe_sysctl_tdh_handler */ 2276 2277 /************************************************************************ 2278 * ixgbe_sysctl_tdt_handler - Transmit Descriptor Tail handler function 2279 * 2280 * Retrieves the TDT value from the hardware 2281 ************************************************************************/ 2282 static int 2283 ixgbe_sysctl_tdt_handler(SYSCTLFN_ARGS) 2284 { 2285 struct sysctlnode node = *rnode; 2286 struct tx_ring *txr = (struct tx_ring *)node.sysctl_data; 2287 struct adapter *adapter; 2288 uint32_t val; 2289 2290 if (!txr) 2291 return (0); 2292 2293 adapter = txr->adapter; 2294 if (ixgbe_fw_recovery_mode_swflag(adapter)) 2295 return (EPERM); 2296 2297 val = IXGBE_READ_REG(&adapter->hw, IXGBE_TDT(txr->me)); 2298 node.sysctl_data = &val; 2299 return sysctl_lookup(SYSCTLFN_CALL(&node)); 2300 } /* ixgbe_sysctl_tdt_handler */ 2301 2302 /************************************************************************ 2303 * ixgbe_sysctl_next_to_check_handler - Receive Descriptor next to check 2304 * handler function 2305 * 2306 * Retrieves the next_to_check value 2307 ************************************************************************/ 2308 static int 2309 ixgbe_sysctl_next_to_check_handler(SYSCTLFN_ARGS) 2310 { 2311 struct sysctlnode node = *rnode; 2312 struct rx_ring *rxr = (struct rx_ring *)node.sysctl_data; 2313 struct adapter *adapter; 2314 uint32_t val; 2315 2316 if (!rxr) 2317 return (0); 2318 2319 adapter = rxr->adapter; 2320 if (ixgbe_fw_recovery_mode_swflag(adapter)) 2321 return (EPERM); 2322 2323 val = rxr->next_to_check; 2324 node.sysctl_data = &val; 2325 return sysctl_lookup(SYSCTLFN_CALL(&node)); 2326 } /* ixgbe_sysctl_next_to_check_handler */ 2327 2328 /************************************************************************ 2329 * ixgbe_sysctl_next_to_refresh_handler - Receive Descriptor next to check 2330 * handler function 2331 * 2332 * Retrieves the next_to_refresh value 2333 ************************************************************************/ 2334 static int 2335 ixgbe_sysctl_next_to_refresh_handler(SYSCTLFN_ARGS) 2336 { 2337 struct sysctlnode node = *rnode; 2338 struct rx_ring *rxr = (struct rx_ring *)node.sysctl_data; 2339 struct adapter *adapter; 2340 uint32_t val; 2341 2342 if (!rxr) 2343 return (0); 2344 2345 adapter = rxr->adapter; 2346 if (ixgbe_fw_recovery_mode_swflag(adapter)) 2347 return (EPERM); 2348 2349 val = rxr->next_to_refresh; 2350 node.sysctl_data = &val; 2351 return sysctl_lookup(SYSCTLFN_CALL(&node)); 2352 } /* ixgbe_sysctl_next_to_refresh_handler */ 2353 2354 /************************************************************************ 2355 * ixgbe_sysctl_rdh_handler - Receive Descriptor Head handler function 2356 * 2357 * Retrieves the RDH value from the hardware 2358 ************************************************************************/ 2359 static int 2360 ixgbe_sysctl_rdh_handler(SYSCTLFN_ARGS) 2361 { 2362 struct sysctlnode node = *rnode; 2363 struct rx_ring *rxr = (struct rx_ring *)node.sysctl_data; 2364 struct adapter *adapter; 2365 uint32_t val; 2366 2367 if (!rxr) 2368 return (0); 2369 2370 adapter = rxr->adapter; 2371 if (ixgbe_fw_recovery_mode_swflag(adapter)) 2372 return (EPERM); 2373 2374 val = IXGBE_READ_REG(&adapter->hw, IXGBE_RDH(rxr->me)); 2375 node.sysctl_data = &val; 2376 return sysctl_lookup(SYSCTLFN_CALL(&node)); 2377 } /* ixgbe_sysctl_rdh_handler */ 2378 2379 /************************************************************************ 2380 * ixgbe_sysctl_rdt_handler - Receive Descriptor Tail handler function 2381 * 2382 * Retrieves the RDT value from the hardware 2383 ************************************************************************/ 2384 static int 2385 ixgbe_sysctl_rdt_handler(SYSCTLFN_ARGS) 2386 { 2387 struct sysctlnode node = *rnode; 2388 struct rx_ring *rxr = (struct rx_ring *)node.sysctl_data; 2389 struct adapter *adapter; 2390 uint32_t val; 2391 2392 if (!rxr) 2393 return (0); 2394 2395 adapter = rxr->adapter; 2396 if (ixgbe_fw_recovery_mode_swflag(adapter)) 2397 return (EPERM); 2398 2399 val = IXGBE_READ_REG(&adapter->hw, IXGBE_RDT(rxr->me)); 2400 node.sysctl_data = &val; 2401 return sysctl_lookup(SYSCTLFN_CALL(&node)); 2402 } /* ixgbe_sysctl_rdt_handler */ 2403 2404 static int 2405 ixgbe_vlan_cb(struct ethercom *ec, uint16_t vid, bool set) 2406 { 2407 struct ifnet *ifp = &ec->ec_if; 2408 struct adapter *adapter = ifp->if_softc; 2409 int rv; 2410 2411 if (set) 2412 rv = ixgbe_register_vlan(adapter, vid); 2413 else 2414 rv = ixgbe_unregister_vlan(adapter, vid); 2415 2416 if (rv != 0) 2417 return rv; 2418 2419 /* 2420 * Control VLAN HW tagging when ec_nvlan is changed from 1 to 0 2421 * or 0 to 1. 2422 */ 2423 if ((set && (ec->ec_nvlans == 1)) || (!set && (ec->ec_nvlans == 0))) 2424 ixgbe_setup_vlan_hw_tagging(adapter); 2425 2426 return rv; 2427 } 2428 2429 /************************************************************************ 2430 * ixgbe_register_vlan 2431 * 2432 * Run via vlan config EVENT, it enables us to use the 2433 * HW Filter table since we can get the vlan id. This 2434 * just creates the entry in the soft version of the 2435 * VFTA, init will repopulate the real table. 2436 ************************************************************************/ 2437 static int 2438 ixgbe_register_vlan(struct adapter *adapter, u16 vtag) 2439 { 2440 u16 index, bit; 2441 int error; 2442 2443 if ((vtag == 0) || (vtag > 4095)) /* Invalid */ 2444 return EINVAL; 2445 2446 IXGBE_CORE_LOCK(adapter); 2447 index = (vtag >> 5) & 0x7F; 2448 bit = vtag & 0x1F; 2449 adapter->shadow_vfta[index] |= ((u32)1 << bit); 2450 error = adapter->hw.mac.ops.set_vfta(&adapter->hw, vtag, 0, true, 2451 true); 2452 IXGBE_CORE_UNLOCK(adapter); 2453 if (error != 0) 2454 error = EACCES; 2455 2456 return error; 2457 } /* ixgbe_register_vlan */ 2458 2459 /************************************************************************ 2460 * ixgbe_unregister_vlan 2461 * 2462 * Run via vlan unconfig EVENT, remove our entry in the soft vfta. 2463 ************************************************************************/ 2464 static int 2465 ixgbe_unregister_vlan(struct adapter *adapter, u16 vtag) 2466 { 2467 u16 index, bit; 2468 int error; 2469 2470 if ((vtag == 0) || (vtag > 4095)) /* Invalid */ 2471 return EINVAL; 2472 2473 IXGBE_CORE_LOCK(adapter); 2474 index = (vtag >> 5) & 0x7F; 2475 bit = vtag & 0x1F; 2476 adapter->shadow_vfta[index] &= ~((u32)1 << bit); 2477 error = adapter->hw.mac.ops.set_vfta(&adapter->hw, vtag, 0, false, 2478 true); 2479 IXGBE_CORE_UNLOCK(adapter); 2480 if (error != 0) 2481 error = EACCES; 2482 2483 return error; 2484 } /* ixgbe_unregister_vlan */ 2485 2486 static void 2487 ixgbe_setup_vlan_hw_tagging(struct adapter *adapter) 2488 { 2489 struct ethercom *ec = &adapter->osdep.ec; 2490 struct ixgbe_hw *hw = &adapter->hw; 2491 struct rx_ring *rxr; 2492 u32 ctrl; 2493 int i; 2494 bool hwtagging; 2495 2496 /* Enable HW tagging only if any vlan is attached */ 2497 hwtagging = (ec->ec_capenable & ETHERCAP_VLAN_HWTAGGING) 2498 && VLAN_ATTACHED(ec); 2499 2500 /* Setup the queues for vlans */ 2501 for (i = 0; i < adapter->num_queues; i++) { 2502 rxr = &adapter->rx_rings[i]; 2503 /* 2504 * On 82599 and later, the VLAN enable is per/queue in RXDCTL. 2505 */ 2506 if (hw->mac.type != ixgbe_mac_82598EB) { 2507 ctrl = IXGBE_READ_REG(hw, IXGBE_RXDCTL(rxr->me)); 2508 if (hwtagging) 2509 ctrl |= IXGBE_RXDCTL_VME; 2510 else 2511 ctrl &= ~IXGBE_RXDCTL_VME; 2512 IXGBE_WRITE_REG(hw, IXGBE_RXDCTL(rxr->me), ctrl); 2513 } 2514 rxr->vtag_strip = hwtagging ? TRUE : FALSE; 2515 } 2516 2517 /* VLAN hw tagging for 82598 */ 2518 if (hw->mac.type == ixgbe_mac_82598EB) { 2519 ctrl = IXGBE_READ_REG(hw, IXGBE_VLNCTRL); 2520 if (hwtagging) 2521 ctrl |= IXGBE_VLNCTRL_VME; 2522 else 2523 ctrl &= ~IXGBE_VLNCTRL_VME; 2524 IXGBE_WRITE_REG(hw, IXGBE_VLNCTRL, ctrl); 2525 } 2526 } /* ixgbe_setup_vlan_hw_tagging */ 2527 2528 static void 2529 ixgbe_setup_vlan_hw_support(struct adapter *adapter) 2530 { 2531 struct ethercom *ec = &adapter->osdep.ec; 2532 struct ixgbe_hw *hw = &adapter->hw; 2533 int i; 2534 u32 ctrl; 2535 struct vlanid_list *vlanidp; 2536 2537 /* 2538 * This function is called from both if_init and ifflags_cb() 2539 * on NetBSD. 2540 */ 2541 2542 /* 2543 * Part 1: 2544 * Setup VLAN HW tagging 2545 */ 2546 ixgbe_setup_vlan_hw_tagging(adapter); 2547 2548 /* 2549 * Part 2: 2550 * Setup VLAN HW filter 2551 */ 2552 /* Cleanup shadow_vfta */ 2553 for (i = 0; i < IXGBE_VFTA_SIZE; i++) 2554 adapter->shadow_vfta[i] = 0; 2555 /* Generate shadow_vfta from ec_vids */ 2556 ETHER_LOCK(ec); 2557 SIMPLEQ_FOREACH(vlanidp, &ec->ec_vids, vid_list) { 2558 uint32_t idx; 2559 2560 idx = vlanidp->vid / 32; 2561 KASSERT(idx < IXGBE_VFTA_SIZE); 2562 adapter->shadow_vfta[idx] |= (u32)1 << (vlanidp->vid % 32); 2563 } 2564 ETHER_UNLOCK(ec); 2565 for (i = 0; i < IXGBE_VFTA_SIZE; i++) 2566 IXGBE_WRITE_REG(hw, IXGBE_VFTA(i), adapter->shadow_vfta[i]); 2567 2568 ctrl = IXGBE_READ_REG(hw, IXGBE_VLNCTRL); 2569 /* Enable the Filter Table if enabled */ 2570 if (ec->ec_capenable & ETHERCAP_VLAN_HWFILTER) 2571 ctrl |= IXGBE_VLNCTRL_VFE; 2572 else 2573 ctrl &= ~IXGBE_VLNCTRL_VFE; 2574 IXGBE_WRITE_REG(hw, IXGBE_VLNCTRL, ctrl); 2575 } /* ixgbe_setup_vlan_hw_support */ 2576 2577 /************************************************************************ 2578 * ixgbe_get_slot_info 2579 * 2580 * Get the width and transaction speed of 2581 * the slot this adapter is plugged into. 2582 ************************************************************************/ 2583 static void 2584 ixgbe_get_slot_info(struct adapter *adapter) 2585 { 2586 device_t dev = adapter->dev; 2587 struct ixgbe_hw *hw = &adapter->hw; 2588 u32 offset; 2589 u16 link; 2590 int bus_info_valid = TRUE; 2591 2592 /* Some devices are behind an internal bridge */ 2593 switch (hw->device_id) { 2594 case IXGBE_DEV_ID_82599_SFP_SF_QP: 2595 case IXGBE_DEV_ID_82599_QSFP_SF_QP: 2596 goto get_parent_info; 2597 default: 2598 break; 2599 } 2600 2601 ixgbe_get_bus_info(hw); 2602 2603 /* 2604 * Some devices don't use PCI-E, but there is no need 2605 * to display "Unknown" for bus speed and width. 2606 */ 2607 switch (hw->mac.type) { 2608 case ixgbe_mac_X550EM_x: 2609 case ixgbe_mac_X550EM_a: 2610 return; 2611 default: 2612 goto display; 2613 } 2614 2615 get_parent_info: 2616 /* 2617 * For the Quad port adapter we need to parse back 2618 * up the PCI tree to find the speed of the expansion 2619 * slot into which this adapter is plugged. A bit more work. 2620 */ 2621 dev = device_parent(device_parent(dev)); 2622 #if 0 2623 #ifdef IXGBE_DEBUG 2624 device_printf(dev, "parent pcib = %x,%x,%x\n", pci_get_bus(dev), 2625 pci_get_slot(dev), pci_get_function(dev)); 2626 #endif 2627 dev = device_parent(device_parent(dev)); 2628 #ifdef IXGBE_DEBUG 2629 device_printf(dev, "slot pcib = %x,%x,%x\n", pci_get_bus(dev), 2630 pci_get_slot(dev), pci_get_function(dev)); 2631 #endif 2632 #endif 2633 /* Now get the PCI Express Capabilities offset */ 2634 if (pci_get_capability(adapter->osdep.pc, adapter->osdep.tag, 2635 PCI_CAP_PCIEXPRESS, &offset, NULL)) { 2636 /* 2637 * Hmm...can't get PCI-Express capabilities. 2638 * Falling back to default method. 2639 */ 2640 bus_info_valid = FALSE; 2641 ixgbe_get_bus_info(hw); 2642 goto display; 2643 } 2644 /* ...and read the Link Status Register */ 2645 link = pci_conf_read(adapter->osdep.pc, adapter->osdep.tag, 2646 offset + PCIE_LCSR) >> 16; 2647 ixgbe_set_pci_config_data_generic(hw, link); 2648 2649 display: 2650 device_printf(dev, "PCI Express Bus: Speed %s Width %s\n", 2651 ((hw->bus.speed == ixgbe_bus_speed_8000) ? "8.0GT/s" : 2652 (hw->bus.speed == ixgbe_bus_speed_5000) ? "5.0GT/s" : 2653 (hw->bus.speed == ixgbe_bus_speed_2500) ? "2.5GT/s" : 2654 "Unknown"), 2655 ((hw->bus.width == ixgbe_bus_width_pcie_x8) ? "x8" : 2656 (hw->bus.width == ixgbe_bus_width_pcie_x4) ? "x4" : 2657 (hw->bus.width == ixgbe_bus_width_pcie_x1) ? "x1" : 2658 "Unknown")); 2659 2660 if (bus_info_valid) { 2661 if ((hw->device_id != IXGBE_DEV_ID_82599_SFP_SF_QP) && 2662 ((hw->bus.width <= ixgbe_bus_width_pcie_x4) && 2663 (hw->bus.speed == ixgbe_bus_speed_2500))) { 2664 device_printf(dev, "PCI-Express bandwidth available" 2665 " for this card\n is not sufficient for" 2666 " optimal performance.\n"); 2667 device_printf(dev, "For optimal performance a x8 " 2668 "PCIE, or x4 PCIE Gen2 slot is required.\n"); 2669 } 2670 if ((hw->device_id == IXGBE_DEV_ID_82599_SFP_SF_QP) && 2671 ((hw->bus.width <= ixgbe_bus_width_pcie_x8) && 2672 (hw->bus.speed < ixgbe_bus_speed_8000))) { 2673 device_printf(dev, "PCI-Express bandwidth available" 2674 " for this card\n is not sufficient for" 2675 " optimal performance.\n"); 2676 device_printf(dev, "For optimal performance a x8 " 2677 "PCIE Gen3 slot is required.\n"); 2678 } 2679 } else 2680 device_printf(dev, "Unable to determine slot speed/width. The speed/width reported are that of the internal switch.\n"); 2681 2682 return; 2683 } /* ixgbe_get_slot_info */ 2684 2685 /************************************************************************ 2686 * ixgbe_enable_queue - MSI-X Interrupt Handlers and Tasklets 2687 ************************************************************************/ 2688 static inline void 2689 ixgbe_enable_queue(struct adapter *adapter, u32 vector) 2690 { 2691 struct ixgbe_hw *hw = &adapter->hw; 2692 struct ix_queue *que = &adapter->queues[vector]; 2693 u64 queue = 1ULL << vector; 2694 u32 mask; 2695 2696 mutex_enter(&que->dc_mtx); 2697 if (que->disabled_count > 0 && --que->disabled_count > 0) 2698 goto out; 2699 2700 if (hw->mac.type == ixgbe_mac_82598EB) { 2701 mask = (IXGBE_EIMS_RTX_QUEUE & queue); 2702 IXGBE_WRITE_REG(hw, IXGBE_EIMS, mask); 2703 } else { 2704 mask = (queue & 0xFFFFFFFF); 2705 if (mask) 2706 IXGBE_WRITE_REG(hw, IXGBE_EIMS_EX(0), mask); 2707 mask = (queue >> 32); 2708 if (mask) 2709 IXGBE_WRITE_REG(hw, IXGBE_EIMS_EX(1), mask); 2710 } 2711 out: 2712 mutex_exit(&que->dc_mtx); 2713 } /* ixgbe_enable_queue */ 2714 2715 /************************************************************************ 2716 * ixgbe_disable_queue_internal 2717 ************************************************************************/ 2718 static inline void 2719 ixgbe_disable_queue_internal(struct adapter *adapter, u32 vector, bool nestok) 2720 { 2721 struct ixgbe_hw *hw = &adapter->hw; 2722 struct ix_queue *que = &adapter->queues[vector]; 2723 u64 queue = 1ULL << vector; 2724 u32 mask; 2725 2726 mutex_enter(&que->dc_mtx); 2727 2728 if (que->disabled_count > 0) { 2729 if (nestok) 2730 que->disabled_count++; 2731 goto out; 2732 } 2733 que->disabled_count++; 2734 2735 if (hw->mac.type == ixgbe_mac_82598EB) { 2736 mask = (IXGBE_EIMS_RTX_QUEUE & queue); 2737 IXGBE_WRITE_REG(hw, IXGBE_EIMC, mask); 2738 } else { 2739 mask = (queue & 0xFFFFFFFF); 2740 if (mask) 2741 IXGBE_WRITE_REG(hw, IXGBE_EIMC_EX(0), mask); 2742 mask = (queue >> 32); 2743 if (mask) 2744 IXGBE_WRITE_REG(hw, IXGBE_EIMC_EX(1), mask); 2745 } 2746 out: 2747 mutex_exit(&que->dc_mtx); 2748 } /* ixgbe_disable_queue_internal */ 2749 2750 /************************************************************************ 2751 * ixgbe_disable_queue 2752 ************************************************************************/ 2753 static inline void 2754 ixgbe_disable_queue(struct adapter *adapter, u32 vector) 2755 { 2756 2757 ixgbe_disable_queue_internal(adapter, vector, true); 2758 } /* ixgbe_disable_queue */ 2759 2760 /************************************************************************ 2761 * ixgbe_sched_handle_que - schedule deferred packet processing 2762 ************************************************************************/ 2763 static inline void 2764 ixgbe_sched_handle_que(struct adapter *adapter, struct ix_queue *que) 2765 { 2766 2767 if (que->txrx_use_workqueue) { 2768 /* 2769 * adapter->que_wq is bound to each CPU instead of 2770 * each NIC queue to reduce workqueue kthread. As we 2771 * should consider about interrupt affinity in this 2772 * function, the workqueue kthread must be WQ_PERCPU. 2773 * If create WQ_PERCPU workqueue kthread for each NIC 2774 * queue, that number of created workqueue kthread is 2775 * (number of used NIC queue) * (number of CPUs) = 2776 * (number of CPUs) ^ 2 most often. 2777 * 2778 * The same NIC queue's interrupts are avoided by 2779 * masking the queue's interrupt. And different 2780 * NIC queue's interrupts use different struct work 2781 * (que->wq_cookie). So, "enqueued flag" to avoid 2782 * twice workqueue_enqueue() is not required . 2783 */ 2784 workqueue_enqueue(adapter->que_wq, &que->wq_cookie, curcpu()); 2785 } else { 2786 softint_schedule(que->que_si); 2787 } 2788 } 2789 2790 /************************************************************************ 2791 * ixgbe_msix_que - MSI-X Queue Interrupt Service routine 2792 ************************************************************************/ 2793 static int 2794 ixgbe_msix_que(void *arg) 2795 { 2796 struct ix_queue *que = arg; 2797 struct adapter *adapter = que->adapter; 2798 struct ifnet *ifp = adapter->ifp; 2799 struct tx_ring *txr = que->txr; 2800 struct rx_ring *rxr = que->rxr; 2801 bool more; 2802 u32 newitr = 0; 2803 2804 /* Protect against spurious interrupts */ 2805 if ((ifp->if_flags & IFF_RUNNING) == 0) 2806 return 0; 2807 2808 ixgbe_disable_queue(adapter, que->msix); 2809 IXGBE_EVC_ADD(&que->irqs, 1); 2810 2811 /* 2812 * Don't change "que->txrx_use_workqueue" from this point to avoid 2813 * flip-flopping softint/workqueue mode in one deferred processing. 2814 */ 2815 que->txrx_use_workqueue = adapter->txrx_use_workqueue; 2816 2817 #ifdef __NetBSD__ 2818 /* Don't run ixgbe_rxeof in interrupt context */ 2819 more = true; 2820 #else 2821 more = ixgbe_rxeof(que); 2822 #endif 2823 2824 IXGBE_TX_LOCK(txr); 2825 ixgbe_txeof(txr); 2826 IXGBE_TX_UNLOCK(txr); 2827 2828 /* Do AIM now? */ 2829 2830 if (adapter->enable_aim == false) 2831 goto no_calc; 2832 /* 2833 * Do Adaptive Interrupt Moderation: 2834 * - Write out last calculated setting 2835 * - Calculate based on average size over 2836 * the last interval. 2837 */ 2838 if (que->eitr_setting) 2839 ixgbe_eitr_write(adapter, que->msix, que->eitr_setting); 2840 2841 que->eitr_setting = 0; 2842 2843 /* Idle, do nothing */ 2844 if ((txr->bytes == 0) && (rxr->bytes == 0)) 2845 goto no_calc; 2846 2847 if ((txr->bytes) && (txr->packets)) 2848 newitr = txr->bytes/txr->packets; 2849 if ((rxr->bytes) && (rxr->packets)) 2850 newitr = uimax(newitr, (rxr->bytes / rxr->packets)); 2851 newitr += 24; /* account for hardware frame, crc */ 2852 2853 /* set an upper boundary */ 2854 newitr = uimin(newitr, 3000); 2855 2856 /* Be nice to the mid range */ 2857 if ((newitr > 300) && (newitr < 1200)) 2858 newitr = (newitr / 3); 2859 else 2860 newitr = (newitr / 2); 2861 2862 /* 2863 * When RSC is used, ITR interval must be larger than RSC_DELAY. 2864 * Currently, we use 2us for RSC_DELAY. The minimum value is always 2865 * greater than 2us on 100M (and 10M?(not documented)), but it's not 2866 * on 1G and higher. 2867 */ 2868 if ((adapter->link_speed != IXGBE_LINK_SPEED_100_FULL) 2869 && (adapter->link_speed != IXGBE_LINK_SPEED_10_FULL)) { 2870 if (newitr < IXGBE_MIN_RSC_EITR_10G1G) 2871 newitr = IXGBE_MIN_RSC_EITR_10G1G; 2872 } 2873 2874 /* save for next interrupt */ 2875 que->eitr_setting = newitr; 2876 2877 /* Reset state */ 2878 txr->bytes = 0; 2879 txr->packets = 0; 2880 rxr->bytes = 0; 2881 rxr->packets = 0; 2882 2883 no_calc: 2884 if (more) 2885 ixgbe_sched_handle_que(adapter, que); 2886 else 2887 ixgbe_enable_queue(adapter, que->msix); 2888 2889 return 1; 2890 } /* ixgbe_msix_que */ 2891 2892 /************************************************************************ 2893 * ixgbe_media_status - Media Ioctl callback 2894 * 2895 * Called whenever the user queries the status of 2896 * the interface using ifconfig. 2897 ************************************************************************/ 2898 static void 2899 ixgbe_media_status(struct ifnet *ifp, struct ifmediareq *ifmr) 2900 { 2901 struct adapter *adapter = ifp->if_softc; 2902 struct ixgbe_hw *hw = &adapter->hw; 2903 int layer; 2904 2905 INIT_DEBUGOUT("ixgbe_media_status: begin"); 2906 ixgbe_update_link_status(adapter); 2907 2908 ifmr->ifm_status = IFM_AVALID; 2909 ifmr->ifm_active = IFM_ETHER; 2910 2911 if (adapter->link_active != LINK_STATE_UP) { 2912 ifmr->ifm_active |= IFM_NONE; 2913 return; 2914 } 2915 2916 ifmr->ifm_status |= IFM_ACTIVE; 2917 layer = adapter->phy_layer; 2918 2919 if (layer & IXGBE_PHYSICAL_LAYER_10GBASE_T || 2920 layer & IXGBE_PHYSICAL_LAYER_5GBASE_T || 2921 layer & IXGBE_PHYSICAL_LAYER_2500BASE_T || 2922 layer & IXGBE_PHYSICAL_LAYER_1000BASE_T || 2923 layer & IXGBE_PHYSICAL_LAYER_100BASE_TX || 2924 layer & IXGBE_PHYSICAL_LAYER_10BASE_T) 2925 switch (adapter->link_speed) { 2926 case IXGBE_LINK_SPEED_10GB_FULL: 2927 ifmr->ifm_active |= IFM_10G_T | IFM_FDX; 2928 break; 2929 case IXGBE_LINK_SPEED_5GB_FULL: 2930 ifmr->ifm_active |= IFM_5000_T | IFM_FDX; 2931 break; 2932 case IXGBE_LINK_SPEED_2_5GB_FULL: 2933 ifmr->ifm_active |= IFM_2500_T | IFM_FDX; 2934 break; 2935 case IXGBE_LINK_SPEED_1GB_FULL: 2936 ifmr->ifm_active |= IFM_1000_T | IFM_FDX; 2937 break; 2938 case IXGBE_LINK_SPEED_100_FULL: 2939 ifmr->ifm_active |= IFM_100_TX | IFM_FDX; 2940 break; 2941 case IXGBE_LINK_SPEED_10_FULL: 2942 ifmr->ifm_active |= IFM_10_T | IFM_FDX; 2943 break; 2944 } 2945 if (layer & IXGBE_PHYSICAL_LAYER_SFP_PLUS_CU || 2946 layer & IXGBE_PHYSICAL_LAYER_SFP_ACTIVE_DA) 2947 switch (adapter->link_speed) { 2948 case IXGBE_LINK_SPEED_10GB_FULL: 2949 ifmr->ifm_active |= IFM_10G_TWINAX | IFM_FDX; 2950 break; 2951 } 2952 if (layer & IXGBE_PHYSICAL_LAYER_10GBASE_LR) 2953 switch (adapter->link_speed) { 2954 case IXGBE_LINK_SPEED_10GB_FULL: 2955 ifmr->ifm_active |= IFM_10G_LR | IFM_FDX; 2956 break; 2957 case IXGBE_LINK_SPEED_1GB_FULL: 2958 ifmr->ifm_active |= IFM_1000_LX | IFM_FDX; 2959 break; 2960 } 2961 if (layer & IXGBE_PHYSICAL_LAYER_10GBASE_LRM) 2962 switch (adapter->link_speed) { 2963 case IXGBE_LINK_SPEED_10GB_FULL: 2964 ifmr->ifm_active |= IFM_10G_LRM | IFM_FDX; 2965 break; 2966 case IXGBE_LINK_SPEED_1GB_FULL: 2967 ifmr->ifm_active |= IFM_1000_LX | IFM_FDX; 2968 break; 2969 } 2970 if (layer & IXGBE_PHYSICAL_LAYER_10GBASE_SR || 2971 layer & IXGBE_PHYSICAL_LAYER_1000BASE_SX) 2972 switch (adapter->link_speed) { 2973 case IXGBE_LINK_SPEED_10GB_FULL: 2974 ifmr->ifm_active |= IFM_10G_SR | IFM_FDX; 2975 break; 2976 case IXGBE_LINK_SPEED_1GB_FULL: 2977 ifmr->ifm_active |= IFM_1000_SX | IFM_FDX; 2978 break; 2979 } 2980 if (layer & IXGBE_PHYSICAL_LAYER_10GBASE_CX4) 2981 switch (adapter->link_speed) { 2982 case IXGBE_LINK_SPEED_10GB_FULL: 2983 ifmr->ifm_active |= IFM_10G_CX4 | IFM_FDX; 2984 break; 2985 } 2986 /* 2987 * XXX: These need to use the proper media types once 2988 * they're added. 2989 */ 2990 if (layer & IXGBE_PHYSICAL_LAYER_10GBASE_KR) 2991 switch (adapter->link_speed) { 2992 case IXGBE_LINK_SPEED_10GB_FULL: 2993 ifmr->ifm_active |= IFM_10G_KR | IFM_FDX; 2994 break; 2995 case IXGBE_LINK_SPEED_2_5GB_FULL: 2996 ifmr->ifm_active |= IFM_2500_KX | IFM_FDX; 2997 break; 2998 case IXGBE_LINK_SPEED_1GB_FULL: 2999 ifmr->ifm_active |= IFM_1000_KX | IFM_FDX; 3000 break; 3001 } 3002 else if (layer & IXGBE_PHYSICAL_LAYER_10GBASE_KX4 || 3003 layer & IXGBE_PHYSICAL_LAYER_2500BASE_KX || 3004 layer & IXGBE_PHYSICAL_LAYER_1000BASE_KX) 3005 switch (adapter->link_speed) { 3006 case IXGBE_LINK_SPEED_10GB_FULL: 3007 ifmr->ifm_active |= IFM_10G_KX4 | IFM_FDX; 3008 break; 3009 case IXGBE_LINK_SPEED_2_5GB_FULL: 3010 ifmr->ifm_active |= IFM_2500_KX | IFM_FDX; 3011 break; 3012 case IXGBE_LINK_SPEED_1GB_FULL: 3013 ifmr->ifm_active |= IFM_1000_KX | IFM_FDX; 3014 break; 3015 } 3016 3017 /* If nothing is recognized... */ 3018 #if 0 3019 if (IFM_SUBTYPE(ifmr->ifm_active) == 0) 3020 ifmr->ifm_active |= IFM_UNKNOWN; 3021 #endif 3022 3023 ifp->if_baudrate = ifmedia_baudrate(ifmr->ifm_active); 3024 3025 /* Display current flow control setting used on link */ 3026 if (hw->fc.current_mode == ixgbe_fc_rx_pause || 3027 hw->fc.current_mode == ixgbe_fc_full) 3028 ifmr->ifm_active |= IFM_ETH_RXPAUSE; 3029 if (hw->fc.current_mode == ixgbe_fc_tx_pause || 3030 hw->fc.current_mode == ixgbe_fc_full) 3031 ifmr->ifm_active |= IFM_ETH_TXPAUSE; 3032 3033 return; 3034 } /* ixgbe_media_status */ 3035 3036 /************************************************************************ 3037 * ixgbe_media_change - Media Ioctl callback 3038 * 3039 * Called when the user changes speed/duplex using 3040 * media/mediopt option with ifconfig. 3041 ************************************************************************/ 3042 static int 3043 ixgbe_media_change(struct ifnet *ifp) 3044 { 3045 struct adapter *adapter = ifp->if_softc; 3046 struct ifmedia *ifm = &adapter->media; 3047 struct ixgbe_hw *hw = &adapter->hw; 3048 ixgbe_link_speed speed = 0; 3049 ixgbe_link_speed link_caps = 0; 3050 bool negotiate = false; 3051 s32 err = IXGBE_NOT_IMPLEMENTED; 3052 3053 INIT_DEBUGOUT("ixgbe_media_change: begin"); 3054 3055 if (IFM_TYPE(ifm->ifm_media) != IFM_ETHER) 3056 return (EINVAL); 3057 3058 if (hw->phy.media_type == ixgbe_media_type_backplane) 3059 return (EPERM); 3060 3061 /* 3062 * We don't actually need to check against the supported 3063 * media types of the adapter; ifmedia will take care of 3064 * that for us. 3065 */ 3066 switch (IFM_SUBTYPE(ifm->ifm_media)) { 3067 case IFM_AUTO: 3068 err = hw->mac.ops.get_link_capabilities(hw, &link_caps, 3069 &negotiate); 3070 if (err != IXGBE_SUCCESS) { 3071 device_printf(adapter->dev, "Unable to determine " 3072 "supported advertise speeds\n"); 3073 return (ENODEV); 3074 } 3075 speed |= link_caps; 3076 break; 3077 case IFM_10G_T: 3078 case IFM_10G_LRM: 3079 case IFM_10G_LR: 3080 case IFM_10G_TWINAX: 3081 case IFM_10G_SR: 3082 case IFM_10G_CX4: 3083 case IFM_10G_KR: 3084 case IFM_10G_KX4: 3085 speed |= IXGBE_LINK_SPEED_10GB_FULL; 3086 break; 3087 case IFM_5000_T: 3088 speed |= IXGBE_LINK_SPEED_5GB_FULL; 3089 break; 3090 case IFM_2500_T: 3091 case IFM_2500_KX: 3092 speed |= IXGBE_LINK_SPEED_2_5GB_FULL; 3093 break; 3094 case IFM_1000_T: 3095 case IFM_1000_LX: 3096 case IFM_1000_SX: 3097 case IFM_1000_KX: 3098 speed |= IXGBE_LINK_SPEED_1GB_FULL; 3099 break; 3100 case IFM_100_TX: 3101 speed |= IXGBE_LINK_SPEED_100_FULL; 3102 break; 3103 case IFM_10_T: 3104 speed |= IXGBE_LINK_SPEED_10_FULL; 3105 break; 3106 case IFM_NONE: 3107 break; 3108 default: 3109 goto invalid; 3110 } 3111 3112 hw->mac.autotry_restart = TRUE; 3113 hw->mac.ops.setup_link(hw, speed, TRUE); 3114 adapter->advertise = 0; 3115 if (IFM_SUBTYPE(ifm->ifm_media) != IFM_AUTO) { 3116 if ((speed & IXGBE_LINK_SPEED_10GB_FULL) != 0) 3117 adapter->advertise |= 1 << 2; 3118 if ((speed & IXGBE_LINK_SPEED_1GB_FULL) != 0) 3119 adapter->advertise |= 1 << 1; 3120 if ((speed & IXGBE_LINK_SPEED_100_FULL) != 0) 3121 adapter->advertise |= 1 << 0; 3122 if ((speed & IXGBE_LINK_SPEED_10_FULL) != 0) 3123 adapter->advertise |= 1 << 3; 3124 if ((speed & IXGBE_LINK_SPEED_2_5GB_FULL) != 0) 3125 adapter->advertise |= 1 << 4; 3126 if ((speed & IXGBE_LINK_SPEED_5GB_FULL) != 0) 3127 adapter->advertise |= 1 << 5; 3128 } 3129 3130 return (0); 3131 3132 invalid: 3133 device_printf(adapter->dev, "Invalid media type!\n"); 3134 3135 return (EINVAL); 3136 } /* ixgbe_media_change */ 3137 3138 /************************************************************************ 3139 * ixgbe_msix_admin - Link status change ISR (MSI/MSI-X) 3140 ************************************************************************/ 3141 static int 3142 ixgbe_msix_admin(void *arg) 3143 { 3144 struct adapter *adapter = arg; 3145 struct ixgbe_hw *hw = &adapter->hw; 3146 u32 eicr; 3147 u32 eims_orig; 3148 u32 eims_disable = 0; 3149 3150 IXGBE_EVC_ADD(&adapter->admin_irqev, 1); 3151 3152 eims_orig = IXGBE_READ_REG(hw, IXGBE_EIMS); 3153 /* Pause other interrupts */ 3154 IXGBE_WRITE_REG(hw, IXGBE_EIMC, IXGBE_MSIX_OTHER_CLEAR_MASK); 3155 3156 /* 3157 * First get the cause. 3158 * 3159 * The specifications of 82598, 82599, X540 and X550 say EICS register 3160 * is write only. However, Linux says it is a workaround for silicon 3161 * errata to read EICS instead of EICR to get interrupt cause. 3162 * At least, reading EICR clears lower 16bits of EIMS on 82598. 3163 */ 3164 eicr = IXGBE_READ_REG(hw, IXGBE_EICS); 3165 /* Be sure the queue bits are not cleared */ 3166 eicr &= ~IXGBE_EICR_RTX_QUEUE; 3167 /* Clear all OTHER interrupts with write */ 3168 IXGBE_WRITE_REG(hw, IXGBE_EICR, eicr); 3169 3170 ixgbe_intr_admin_common(adapter, eicr, &eims_disable); 3171 3172 /* Re-enable some OTHER interrupts */ 3173 IXGBE_WRITE_REG(hw, IXGBE_EIMS, eims_orig & ~eims_disable); 3174 3175 return 1; 3176 } /* ixgbe_msix_admin */ 3177 3178 static void 3179 ixgbe_intr_admin_common(struct adapter *adapter, u32 eicr, u32 *eims_disable) 3180 { 3181 struct ixgbe_hw *hw = &adapter->hw; 3182 u32 eicr_mask; 3183 u32 task_requests = 0; 3184 s32 retval; 3185 3186 /* Link status change */ 3187 if (eicr & IXGBE_EICR_LSC) { 3188 task_requests |= IXGBE_REQUEST_TASK_LSC; 3189 *eims_disable |= IXGBE_EIMS_LSC; 3190 } 3191 3192 if (ixgbe_is_sfp(hw)) { 3193 /* Pluggable optics-related interrupt */ 3194 if (hw->mac.type >= ixgbe_mac_X540) 3195 eicr_mask = IXGBE_EICR_GPI_SDP0_X540; 3196 else 3197 eicr_mask = IXGBE_EICR_GPI_SDP2_BY_MAC(hw); 3198 3199 /* 3200 * An interrupt might not arrive when a module is inserted. 3201 * When an link status change interrupt occurred and the driver 3202 * still regard SFP as unplugged, issue the module softint 3203 * and then issue LSC interrupt. 3204 */ 3205 if ((eicr & eicr_mask) 3206 || ((hw->phy.sfp_type == ixgbe_sfp_type_not_present) 3207 && (eicr & IXGBE_EICR_LSC))) { 3208 task_requests |= IXGBE_REQUEST_TASK_MOD; 3209 *eims_disable |= IXGBE_EIMS_LSC; 3210 } 3211 3212 if ((hw->mac.type == ixgbe_mac_82599EB) && 3213 (eicr & IXGBE_EICR_GPI_SDP1_BY_MAC(hw))) { 3214 task_requests |= IXGBE_REQUEST_TASK_MSF; 3215 *eims_disable |= IXGBE_EIMS_GPI_SDP1_BY_MAC(hw); 3216 } 3217 } 3218 3219 if (adapter->hw.mac.type != ixgbe_mac_82598EB) { 3220 if ((adapter->feat_en & IXGBE_FEATURE_FDIR) && 3221 (eicr & IXGBE_EICR_FLOW_DIR)) { 3222 if (!atomic_cas_uint(&adapter->fdir_reinit, 0, 1)) { 3223 task_requests |= IXGBE_REQUEST_TASK_FDIR; 3224 /* Disable the interrupt */ 3225 *eims_disable |= IXGBE_EIMS_FLOW_DIR; 3226 } 3227 } 3228 3229 if (eicr & IXGBE_EICR_ECC) { 3230 device_printf(adapter->dev, 3231 "CRITICAL: ECC ERROR!! Please Reboot!!\n"); 3232 /* Disable interrupt to prevent log spam */ 3233 *eims_disable |= IXGBE_EICR_ECC; 3234 } 3235 3236 /* Check for over temp condition */ 3237 if (adapter->feat_en & IXGBE_FEATURE_TEMP_SENSOR) { 3238 switch (adapter->hw.mac.type) { 3239 case ixgbe_mac_X550EM_a: 3240 if (!(eicr & IXGBE_EICR_GPI_SDP0_X550EM_a)) 3241 break; 3242 /* Disable interrupt to prevent log spam */ 3243 *eims_disable |= IXGBE_EICR_GPI_SDP0_X550EM_a; 3244 3245 retval = hw->phy.ops.check_overtemp(hw); 3246 if (retval != IXGBE_ERR_OVERTEMP) 3247 break; 3248 device_printf(adapter->dev, 3249 "CRITICAL: OVER TEMP!! " 3250 "PHY IS SHUT DOWN!!\n"); 3251 device_printf(adapter->dev, 3252 "System shutdown required!\n"); 3253 break; 3254 default: 3255 if (!(eicr & IXGBE_EICR_TS)) 3256 break; 3257 /* Disable interrupt to prevent log spam */ 3258 *eims_disable |= IXGBE_EIMS_TS; 3259 3260 retval = hw->phy.ops.check_overtemp(hw); 3261 if (retval != IXGBE_ERR_OVERTEMP) 3262 break; 3263 device_printf(adapter->dev, 3264 "CRITICAL: OVER TEMP!! " 3265 "PHY IS SHUT DOWN!!\n"); 3266 device_printf(adapter->dev, 3267 "System shutdown required!\n"); 3268 break; 3269 } 3270 } 3271 3272 /* Check for VF message */ 3273 if ((adapter->feat_en & IXGBE_FEATURE_SRIOV) && 3274 (eicr & IXGBE_EICR_MAILBOX)) { 3275 task_requests |= IXGBE_REQUEST_TASK_MBX; 3276 *eims_disable |= IXGBE_EIMS_MAILBOX; 3277 } 3278 } 3279 3280 /* Check for fan failure */ 3281 if (adapter->feat_en & IXGBE_FEATURE_FAN_FAIL) { 3282 retval = ixgbe_check_fan_failure(adapter, eicr, true); 3283 if (retval == IXGBE_ERR_FAN_FAILURE) { 3284 /* Disable interrupt to prevent log spam */ 3285 *eims_disable |= IXGBE_EIMS_GPI_SDP1_BY_MAC(hw); 3286 } 3287 } 3288 3289 /* External PHY interrupt */ 3290 if ((hw->phy.type == ixgbe_phy_x550em_ext_t) && 3291 (eicr & IXGBE_EICR_GPI_SDP0_X540)) { 3292 task_requests |= IXGBE_REQUEST_TASK_PHY; 3293 *eims_disable |= IXGBE_EICR_GPI_SDP0_X540; 3294 } 3295 3296 if (task_requests != 0) { 3297 mutex_enter(&adapter->admin_mtx); 3298 adapter->task_requests |= task_requests; 3299 ixgbe_schedule_admin_tasklet(adapter); 3300 mutex_exit(&adapter->admin_mtx); 3301 } 3302 3303 } 3304 3305 static void 3306 ixgbe_eitr_write(struct adapter *adapter, uint32_t index, uint32_t itr) 3307 { 3308 3309 if (adapter->hw.mac.type == ixgbe_mac_82598EB) 3310 itr |= itr << 16; 3311 else 3312 itr |= IXGBE_EITR_CNT_WDIS; 3313 3314 IXGBE_WRITE_REG(&adapter->hw, IXGBE_EITR(index), itr); 3315 } 3316 3317 3318 /************************************************************************ 3319 * ixgbe_sysctl_interrupt_rate_handler 3320 ************************************************************************/ 3321 static int 3322 ixgbe_sysctl_interrupt_rate_handler(SYSCTLFN_ARGS) 3323 { 3324 struct sysctlnode node = *rnode; 3325 struct ix_queue *que = (struct ix_queue *)node.sysctl_data; 3326 struct adapter *adapter; 3327 uint32_t reg, usec, rate; 3328 int error; 3329 3330 if (que == NULL) 3331 return 0; 3332 3333 adapter = que->adapter; 3334 if (ixgbe_fw_recovery_mode_swflag(adapter)) 3335 return (EPERM); 3336 3337 reg = IXGBE_READ_REG(&adapter->hw, IXGBE_EITR(que->msix)); 3338 usec = ((reg & 0x0FF8) >> 3); 3339 if (usec > 0) 3340 rate = 500000 / usec; 3341 else 3342 rate = 0; 3343 node.sysctl_data = &rate; 3344 error = sysctl_lookup(SYSCTLFN_CALL(&node)); 3345 if (error || newp == NULL) 3346 return error; 3347 reg &= ~0xfff; /* default, no limitation */ 3348 if (rate > 0 && rate < 500000) { 3349 if (rate < 1000) 3350 rate = 1000; 3351 reg |= ((4000000 / rate) & 0xff8); 3352 /* 3353 * When RSC is used, ITR interval must be larger than 3354 * RSC_DELAY. Currently, we use 2us for RSC_DELAY. 3355 * The minimum value is always greater than 2us on 100M 3356 * (and 10M?(not documented)), but it's not on 1G and higher. 3357 */ 3358 if ((adapter->link_speed != IXGBE_LINK_SPEED_100_FULL) 3359 && (adapter->link_speed != IXGBE_LINK_SPEED_10_FULL)) { 3360 if ((adapter->num_queues > 1) 3361 && (reg < IXGBE_MIN_RSC_EITR_10G1G)) 3362 return EINVAL; 3363 } 3364 ixgbe_max_interrupt_rate = rate; 3365 } else 3366 ixgbe_max_interrupt_rate = 0; 3367 ixgbe_eitr_write(adapter, que->msix, reg); 3368 3369 return (0); 3370 } /* ixgbe_sysctl_interrupt_rate_handler */ 3371 3372 const struct sysctlnode * 3373 ixgbe_sysctl_instance(struct adapter *adapter) 3374 { 3375 const char *dvname; 3376 struct sysctllog **log; 3377 int rc; 3378 const struct sysctlnode *rnode; 3379 3380 if (adapter->sysctltop != NULL) 3381 return adapter->sysctltop; 3382 3383 log = &adapter->sysctllog; 3384 dvname = device_xname(adapter->dev); 3385 3386 if ((rc = sysctl_createv(log, 0, NULL, &rnode, 3387 0, CTLTYPE_NODE, dvname, 3388 SYSCTL_DESCR("ixgbe information and settings"), 3389 NULL, 0, NULL, 0, CTL_HW, CTL_CREATE, CTL_EOL)) != 0) 3390 goto err; 3391 3392 return rnode; 3393 err: 3394 device_printf(adapter->dev, 3395 "%s: sysctl_createv failed, rc = %d\n", __func__, rc); 3396 return NULL; 3397 } 3398 3399 /************************************************************************ 3400 * ixgbe_add_device_sysctls 3401 ************************************************************************/ 3402 static void 3403 ixgbe_add_device_sysctls(struct adapter *adapter) 3404 { 3405 device_t dev = adapter->dev; 3406 struct ixgbe_hw *hw = &adapter->hw; 3407 struct sysctllog **log; 3408 const struct sysctlnode *rnode, *cnode; 3409 3410 log = &adapter->sysctllog; 3411 3412 if ((rnode = ixgbe_sysctl_instance(adapter)) == NULL) { 3413 aprint_error_dev(dev, "could not create sysctl root\n"); 3414 return; 3415 } 3416 3417 if (sysctl_createv(log, 0, &rnode, &cnode, 3418 CTLFLAG_READWRITE, CTLTYPE_INT, 3419 "debug", SYSCTL_DESCR("Debug Info"), 3420 ixgbe_sysctl_debug, 0, (void *)adapter, 0, CTL_CREATE, CTL_EOL) 3421 != 0) 3422 aprint_error_dev(dev, "could not create sysctl\n"); 3423 3424 if (sysctl_createv(log, 0, &rnode, &cnode, 3425 CTLFLAG_READWRITE, CTLTYPE_INT, 3426 "rx_copy_len", SYSCTL_DESCR("RX Copy Length"), 3427 ixgbe_sysctl_rx_copy_len, 0, 3428 (void *)adapter, 0, CTL_CREATE, CTL_EOL) != 0) 3429 aprint_error_dev(dev, "could not create sysctl\n"); 3430 3431 if (sysctl_createv(log, 0, &rnode, &cnode, 3432 CTLFLAG_READONLY, CTLTYPE_INT, 3433 "num_rx_desc", SYSCTL_DESCR("Number of rx descriptors"), 3434 NULL, 0, &adapter->num_rx_desc, 0, CTL_CREATE, CTL_EOL) != 0) 3435 aprint_error_dev(dev, "could not create sysctl\n"); 3436 3437 if (sysctl_createv(log, 0, &rnode, &cnode, 3438 CTLFLAG_READONLY, CTLTYPE_INT, 3439 "num_queues", SYSCTL_DESCR("Number of queues"), 3440 NULL, 0, &adapter->num_queues, 0, CTL_CREATE, CTL_EOL) != 0) 3441 aprint_error_dev(dev, "could not create sysctl\n"); 3442 3443 /* Sysctls for all devices */ 3444 if (sysctl_createv(log, 0, &rnode, &cnode, CTLFLAG_READWRITE, 3445 CTLTYPE_INT, "fc", SYSCTL_DESCR(IXGBE_SYSCTL_DESC_SET_FC), 3446 ixgbe_sysctl_flowcntl, 0, (void *)adapter, 0, CTL_CREATE, 3447 CTL_EOL) != 0) 3448 aprint_error_dev(dev, "could not create sysctl\n"); 3449 3450 adapter->enable_aim = ixgbe_enable_aim; 3451 if (sysctl_createv(log, 0, &rnode, &cnode, CTLFLAG_READWRITE, 3452 CTLTYPE_BOOL, "enable_aim", SYSCTL_DESCR("Interrupt Moderation"), 3453 NULL, 0, &adapter->enable_aim, 0, CTL_CREATE, CTL_EOL) != 0) 3454 aprint_error_dev(dev, "could not create sysctl\n"); 3455 3456 if (sysctl_createv(log, 0, &rnode, &cnode, 3457 CTLFLAG_READWRITE, CTLTYPE_INT, 3458 "advertise_speed", SYSCTL_DESCR(IXGBE_SYSCTL_DESC_ADV_SPEED), 3459 ixgbe_sysctl_advertise, 0, (void *)adapter, 0, CTL_CREATE, 3460 CTL_EOL) != 0) 3461 aprint_error_dev(dev, "could not create sysctl\n"); 3462 3463 /* 3464 * If each "que->txrx_use_workqueue" is changed in sysctl handler, 3465 * it causesflip-flopping softint/workqueue mode in one deferred 3466 * processing. Therefore, preempt_disable()/preempt_enable() are 3467 * required in ixgbe_sched_handle_que() to avoid 3468 * KASSERT(ixgbe_sched_handle_que()) in softint_schedule(). 3469 * I think changing "que->txrx_use_workqueue" in interrupt handler 3470 * is lighter than doing preempt_disable()/preempt_enable() in every 3471 * ixgbe_sched_handle_que(). 3472 */ 3473 adapter->txrx_use_workqueue = ixgbe_txrx_workqueue; 3474 if (sysctl_createv(log, 0, &rnode, &cnode, CTLFLAG_READWRITE, 3475 CTLTYPE_BOOL, "txrx_workqueue", 3476 SYSCTL_DESCR("Use workqueue for packet processing"), 3477 NULL, 0, &adapter->txrx_use_workqueue, 0, CTL_CREATE, 3478 CTL_EOL) != 0) 3479 aprint_error_dev(dev, "could not create sysctl\n"); 3480 3481 #ifdef IXGBE_DEBUG 3482 /* testing sysctls (for all devices) */ 3483 if (sysctl_createv(log, 0, &rnode, &cnode, CTLFLAG_READWRITE, 3484 CTLTYPE_INT, "power_state", SYSCTL_DESCR("PCI Power State"), 3485 ixgbe_sysctl_power_state, 0, (void *)adapter, 0, CTL_CREATE, 3486 CTL_EOL) != 0) 3487 aprint_error_dev(dev, "could not create sysctl\n"); 3488 3489 if (sysctl_createv(log, 0, &rnode, &cnode, CTLFLAG_READONLY, 3490 CTLTYPE_STRING, "print_rss_config", 3491 SYSCTL_DESCR("Prints RSS Configuration"), 3492 ixgbe_sysctl_print_rss_config, 0, (void *)adapter, 0, CTL_CREATE, 3493 CTL_EOL) != 0) 3494 aprint_error_dev(dev, "could not create sysctl\n"); 3495 #endif 3496 /* for X550 series devices */ 3497 if (hw->mac.type >= ixgbe_mac_X550) 3498 if (sysctl_createv(log, 0, &rnode, &cnode, CTLFLAG_READWRITE, 3499 CTLTYPE_INT, "dmac", SYSCTL_DESCR("DMA Coalesce"), 3500 ixgbe_sysctl_dmac, 0, (void *)adapter, 0, CTL_CREATE, 3501 CTL_EOL) != 0) 3502 aprint_error_dev(dev, "could not create sysctl\n"); 3503 3504 /* for WoL-capable devices */ 3505 if (adapter->wol_support) { 3506 if (sysctl_createv(log, 0, &rnode, &cnode, CTLFLAG_READWRITE, 3507 CTLTYPE_BOOL, "wol_enable", 3508 SYSCTL_DESCR("Enable/Disable Wake on LAN"), 3509 ixgbe_sysctl_wol_enable, 0, (void *)adapter, 0, CTL_CREATE, 3510 CTL_EOL) != 0) 3511 aprint_error_dev(dev, "could not create sysctl\n"); 3512 3513 if (sysctl_createv(log, 0, &rnode, &cnode, CTLFLAG_READWRITE, 3514 CTLTYPE_INT, "wufc", 3515 SYSCTL_DESCR("Enable/Disable Wake Up Filters"), 3516 ixgbe_sysctl_wufc, 0, (void *)adapter, 0, CTL_CREATE, 3517 CTL_EOL) != 0) 3518 aprint_error_dev(dev, "could not create sysctl\n"); 3519 } 3520 3521 /* for X552/X557-AT devices */ 3522 if (hw->device_id == IXGBE_DEV_ID_X550EM_X_10G_T) { 3523 const struct sysctlnode *phy_node; 3524 3525 if (sysctl_createv(log, 0, &rnode, &phy_node, 0, CTLTYPE_NODE, 3526 "phy", SYSCTL_DESCR("External PHY sysctls"), 3527 NULL, 0, NULL, 0, CTL_CREATE, CTL_EOL) != 0) { 3528 aprint_error_dev(dev, "could not create sysctl\n"); 3529 return; 3530 } 3531 3532 if (sysctl_createv(log, 0, &phy_node, &cnode, CTLFLAG_READONLY, 3533 CTLTYPE_INT, "temp", 3534 SYSCTL_DESCR("Current External PHY Temperature (Celsius)"), 3535 ixgbe_sysctl_phy_temp, 0, (void *)adapter, 0, CTL_CREATE, 3536 CTL_EOL) != 0) 3537 aprint_error_dev(dev, "could not create sysctl\n"); 3538 3539 if (sysctl_createv(log, 0, &phy_node, &cnode, CTLFLAG_READONLY, 3540 CTLTYPE_INT, "overtemp_occurred", 3541 SYSCTL_DESCR( 3542 "External PHY High Temperature Event Occurred"), 3543 ixgbe_sysctl_phy_overtemp_occurred, 0, (void *)adapter, 0, 3544 CTL_CREATE, CTL_EOL) != 0) 3545 aprint_error_dev(dev, "could not create sysctl\n"); 3546 } 3547 3548 if ((hw->mac.type == ixgbe_mac_X550EM_a) 3549 && (hw->phy.type == ixgbe_phy_fw)) 3550 if (sysctl_createv(log, 0, &rnode, &cnode, CTLFLAG_READWRITE, 3551 CTLTYPE_BOOL, "force_10_100_autonego", 3552 SYSCTL_DESCR("Force autonego on 10M and 100M"), 3553 NULL, 0, &hw->phy.force_10_100_autonego, 0, 3554 CTL_CREATE, CTL_EOL) != 0) 3555 aprint_error_dev(dev, "could not create sysctl\n"); 3556 3557 if (adapter->feat_cap & IXGBE_FEATURE_EEE) { 3558 if (sysctl_createv(log, 0, &rnode, &cnode, CTLFLAG_READWRITE, 3559 CTLTYPE_INT, "eee_state", 3560 SYSCTL_DESCR("EEE Power Save State"), 3561 ixgbe_sysctl_eee_state, 0, (void *)adapter, 0, CTL_CREATE, 3562 CTL_EOL) != 0) 3563 aprint_error_dev(dev, "could not create sysctl\n"); 3564 } 3565 } /* ixgbe_add_device_sysctls */ 3566 3567 /************************************************************************ 3568 * ixgbe_allocate_pci_resources 3569 ************************************************************************/ 3570 static int 3571 ixgbe_allocate_pci_resources(struct adapter *adapter, 3572 const struct pci_attach_args *pa) 3573 { 3574 pcireg_t memtype, csr; 3575 device_t dev = adapter->dev; 3576 bus_addr_t addr; 3577 int flags; 3578 3579 memtype = pci_mapreg_type(pa->pa_pc, pa->pa_tag, PCI_BAR(0)); 3580 switch (memtype) { 3581 case PCI_MAPREG_TYPE_MEM | PCI_MAPREG_MEM_TYPE_32BIT: 3582 case PCI_MAPREG_TYPE_MEM | PCI_MAPREG_MEM_TYPE_64BIT: 3583 adapter->osdep.mem_bus_space_tag = pa->pa_memt; 3584 if (pci_mapreg_info(pa->pa_pc, pa->pa_tag, PCI_BAR(0), 3585 memtype, &addr, &adapter->osdep.mem_size, &flags) != 0) 3586 goto map_err; 3587 if ((flags & BUS_SPACE_MAP_PREFETCHABLE) != 0) { 3588 aprint_normal_dev(dev, "clearing prefetchable bit\n"); 3589 flags &= ~BUS_SPACE_MAP_PREFETCHABLE; 3590 } 3591 if (bus_space_map(adapter->osdep.mem_bus_space_tag, addr, 3592 adapter->osdep.mem_size, flags, 3593 &adapter->osdep.mem_bus_space_handle) != 0) { 3594 map_err: 3595 adapter->osdep.mem_size = 0; 3596 aprint_error_dev(dev, "unable to map BAR0\n"); 3597 return ENXIO; 3598 } 3599 /* 3600 * Enable address decoding for memory range in case BIOS or 3601 * UEFI don't set it. 3602 */ 3603 csr = pci_conf_read(pa->pa_pc, pa->pa_tag, 3604 PCI_COMMAND_STATUS_REG); 3605 csr |= PCI_COMMAND_MEM_ENABLE; 3606 pci_conf_write(pa->pa_pc, pa->pa_tag, PCI_COMMAND_STATUS_REG, 3607 csr); 3608 break; 3609 default: 3610 aprint_error_dev(dev, "unexpected type on BAR0\n"); 3611 return ENXIO; 3612 } 3613 3614 return (0); 3615 } /* ixgbe_allocate_pci_resources */ 3616 3617 static void 3618 ixgbe_free_deferred_handlers(struct adapter *adapter) 3619 { 3620 struct ix_queue *que = adapter->queues; 3621 struct tx_ring *txr = adapter->tx_rings; 3622 int i; 3623 3624 for (i = 0; i < adapter->num_queues; i++, que++, txr++) { 3625 if (!(adapter->feat_en & IXGBE_FEATURE_LEGACY_TX)) { 3626 if (txr->txr_si != NULL) 3627 softint_disestablish(txr->txr_si); 3628 } 3629 if (que->que_si != NULL) 3630 softint_disestablish(que->que_si); 3631 } 3632 if (adapter->txr_wq != NULL) 3633 workqueue_destroy(adapter->txr_wq); 3634 if (adapter->txr_wq_enqueued != NULL) 3635 percpu_free(adapter->txr_wq_enqueued, sizeof(u_int)); 3636 if (adapter->que_wq != NULL) 3637 workqueue_destroy(adapter->que_wq); 3638 3639 if (adapter->admin_wq != NULL) { 3640 workqueue_destroy(adapter->admin_wq); 3641 adapter->admin_wq = NULL; 3642 } 3643 if (adapter->timer_wq != NULL) { 3644 workqueue_destroy(adapter->timer_wq); 3645 adapter->timer_wq = NULL; 3646 } 3647 if (adapter->recovery_mode_timer_wq != NULL) { 3648 /* 3649 * ixgbe_ifstop() doesn't call the workqueue_wait() for 3650 * the recovery_mode_timer workqueue, so call it here. 3651 */ 3652 workqueue_wait(adapter->recovery_mode_timer_wq, 3653 &adapter->recovery_mode_timer_wc); 3654 atomic_store_relaxed(&adapter->recovery_mode_timer_pending, 0); 3655 workqueue_destroy(adapter->recovery_mode_timer_wq); 3656 adapter->recovery_mode_timer_wq = NULL; 3657 } 3658 } /* ixgbe_free_deferred_handlers */ 3659 3660 /************************************************************************ 3661 * ixgbe_detach - Device removal routine 3662 * 3663 * Called when the driver is being removed. 3664 * Stops the adapter and deallocates all the resources 3665 * that were allocated for driver operation. 3666 * 3667 * return 0 on success, positive on failure 3668 ************************************************************************/ 3669 static int 3670 ixgbe_detach(device_t dev, int flags) 3671 { 3672 struct adapter *adapter = device_private(dev); 3673 struct rx_ring *rxr = adapter->rx_rings; 3674 struct tx_ring *txr = adapter->tx_rings; 3675 struct ixgbe_hw *hw = &adapter->hw; 3676 struct ixgbe_hw_stats *stats = &adapter->stats.pf; 3677 u32 ctrl_ext; 3678 int i; 3679 3680 INIT_DEBUGOUT("ixgbe_detach: begin"); 3681 if (adapter->osdep.attached == false) 3682 return 0; 3683 3684 if (ixgbe_pci_iov_detach(dev) != 0) { 3685 device_printf(dev, "SR-IOV in use; detach first.\n"); 3686 return (EBUSY); 3687 } 3688 3689 if (VLAN_ATTACHED(&adapter->osdep.ec) && 3690 (flags & (DETACH_SHUTDOWN | DETACH_FORCE)) == 0) { 3691 aprint_error_dev(dev, "VLANs in use, detach first\n"); 3692 return (EBUSY); 3693 } 3694 3695 ether_ifdetach(adapter->ifp); 3696 3697 adapter->osdep.detaching = true; 3698 /* 3699 * Stop the interface. ixgbe_setup_low_power_mode() calls 3700 * ixgbe_ifstop(), so it's not required to call ixgbe_ifstop() 3701 * directly. 3702 */ 3703 ixgbe_setup_low_power_mode(adapter); 3704 3705 callout_halt(&adapter->timer, NULL); 3706 if (adapter->feat_en & IXGBE_FEATURE_RECOVERY_MODE) 3707 callout_halt(&adapter->recovery_mode_timer, NULL); 3708 3709 workqueue_wait(adapter->admin_wq, &adapter->admin_wc); 3710 atomic_store_relaxed(&adapter->admin_pending, 0); 3711 workqueue_wait(adapter->timer_wq, &adapter->timer_wc); 3712 atomic_store_relaxed(&adapter->timer_pending, 0); 3713 3714 pmf_device_deregister(dev); 3715 3716 ixgbe_free_deferred_handlers(adapter); 3717 3718 /* let hardware know driver is unloading */ 3719 ctrl_ext = IXGBE_READ_REG(&adapter->hw, IXGBE_CTRL_EXT); 3720 ctrl_ext &= ~IXGBE_CTRL_EXT_DRV_LOAD; 3721 IXGBE_WRITE_REG(&adapter->hw, IXGBE_CTRL_EXT, ctrl_ext); 3722 3723 if (adapter->feat_en & IXGBE_FEATURE_NETMAP) 3724 netmap_detach(adapter->ifp); 3725 3726 ixgbe_free_pci_resources(adapter); 3727 #if 0 /* XXX the NetBSD port is probably missing something here */ 3728 bus_generic_detach(dev); 3729 #endif 3730 if_detach(adapter->ifp); 3731 ifmedia_fini(&adapter->media); 3732 if_percpuq_destroy(adapter->ipq); 3733 3734 sysctl_teardown(&adapter->sysctllog); 3735 evcnt_detach(&adapter->efbig_tx_dma_setup); 3736 evcnt_detach(&adapter->mbuf_defrag_failed); 3737 evcnt_detach(&adapter->efbig2_tx_dma_setup); 3738 evcnt_detach(&adapter->einval_tx_dma_setup); 3739 evcnt_detach(&adapter->other_tx_dma_setup); 3740 evcnt_detach(&adapter->eagain_tx_dma_setup); 3741 evcnt_detach(&adapter->enomem_tx_dma_setup); 3742 evcnt_detach(&adapter->watchdog_events); 3743 evcnt_detach(&adapter->tso_err); 3744 evcnt_detach(&adapter->admin_irqev); 3745 evcnt_detach(&adapter->link_workev); 3746 evcnt_detach(&adapter->mod_workev); 3747 evcnt_detach(&adapter->msf_workev); 3748 evcnt_detach(&adapter->phy_workev); 3749 3750 for (i = 0; i < IXGBE_TC_COUNTER_NUM; i++) { 3751 if (i < __arraycount(stats->mpc)) { 3752 evcnt_detach(&stats->mpc[i]); 3753 if (hw->mac.type == ixgbe_mac_82598EB) 3754 evcnt_detach(&stats->rnbc[i]); 3755 } 3756 if (i < __arraycount(stats->pxontxc)) { 3757 evcnt_detach(&stats->pxontxc[i]); 3758 evcnt_detach(&stats->pxonrxc[i]); 3759 evcnt_detach(&stats->pxofftxc[i]); 3760 evcnt_detach(&stats->pxoffrxc[i]); 3761 if (hw->mac.type >= ixgbe_mac_82599EB) 3762 evcnt_detach(&stats->pxon2offc[i]); 3763 } 3764 } 3765 3766 txr = adapter->tx_rings; 3767 for (i = 0; i < adapter->num_queues; i++, rxr++, txr++) { 3768 evcnt_detach(&adapter->queues[i].irqs); 3769 evcnt_detach(&adapter->queues[i].handleq); 3770 evcnt_detach(&adapter->queues[i].req); 3771 evcnt_detach(&txr->no_desc_avail); 3772 evcnt_detach(&txr->total_packets); 3773 evcnt_detach(&txr->tso_tx); 3774 #ifndef IXGBE_LEGACY_TX 3775 evcnt_detach(&txr->pcq_drops); 3776 #endif 3777 3778 if (i < __arraycount(stats->qprc)) { 3779 evcnt_detach(&stats->qprc[i]); 3780 evcnt_detach(&stats->qptc[i]); 3781 evcnt_detach(&stats->qbrc[i]); 3782 evcnt_detach(&stats->qbtc[i]); 3783 if (hw->mac.type >= ixgbe_mac_82599EB) 3784 evcnt_detach(&stats->qprdc[i]); 3785 } 3786 3787 evcnt_detach(&rxr->rx_packets); 3788 evcnt_detach(&rxr->rx_bytes); 3789 evcnt_detach(&rxr->rx_copies); 3790 evcnt_detach(&rxr->no_mbuf); 3791 evcnt_detach(&rxr->rx_discarded); 3792 } 3793 evcnt_detach(&stats->ipcs); 3794 evcnt_detach(&stats->l4cs); 3795 evcnt_detach(&stats->ipcs_bad); 3796 evcnt_detach(&stats->l4cs_bad); 3797 evcnt_detach(&stats->intzero); 3798 evcnt_detach(&stats->legint); 3799 evcnt_detach(&stats->crcerrs); 3800 evcnt_detach(&stats->illerrc); 3801 evcnt_detach(&stats->errbc); 3802 evcnt_detach(&stats->mspdc); 3803 if (hw->mac.type >= ixgbe_mac_X550) 3804 evcnt_detach(&stats->mbsdc); 3805 evcnt_detach(&stats->mpctotal); 3806 evcnt_detach(&stats->mlfc); 3807 evcnt_detach(&stats->mrfc); 3808 evcnt_detach(&stats->rlec); 3809 evcnt_detach(&stats->lxontxc); 3810 evcnt_detach(&stats->lxonrxc); 3811 evcnt_detach(&stats->lxofftxc); 3812 evcnt_detach(&stats->lxoffrxc); 3813 3814 /* Packet Reception Stats */ 3815 evcnt_detach(&stats->tor); 3816 evcnt_detach(&stats->gorc); 3817 evcnt_detach(&stats->tpr); 3818 evcnt_detach(&stats->gprc); 3819 evcnt_detach(&stats->mprc); 3820 evcnt_detach(&stats->bprc); 3821 evcnt_detach(&stats->prc64); 3822 evcnt_detach(&stats->prc127); 3823 evcnt_detach(&stats->prc255); 3824 evcnt_detach(&stats->prc511); 3825 evcnt_detach(&stats->prc1023); 3826 evcnt_detach(&stats->prc1522); 3827 evcnt_detach(&stats->ruc); 3828 evcnt_detach(&stats->rfc); 3829 evcnt_detach(&stats->roc); 3830 evcnt_detach(&stats->rjc); 3831 evcnt_detach(&stats->mngprc); 3832 evcnt_detach(&stats->mngpdc); 3833 evcnt_detach(&stats->xec); 3834 3835 /* Packet Transmission Stats */ 3836 evcnt_detach(&stats->gotc); 3837 evcnt_detach(&stats->tpt); 3838 evcnt_detach(&stats->gptc); 3839 evcnt_detach(&stats->bptc); 3840 evcnt_detach(&stats->mptc); 3841 evcnt_detach(&stats->mngptc); 3842 evcnt_detach(&stats->ptc64); 3843 evcnt_detach(&stats->ptc127); 3844 evcnt_detach(&stats->ptc255); 3845 evcnt_detach(&stats->ptc511); 3846 evcnt_detach(&stats->ptc1023); 3847 evcnt_detach(&stats->ptc1522); 3848 3849 ixgbe_free_queues(adapter); 3850 free(adapter->mta, M_DEVBUF); 3851 3852 mutex_destroy(&adapter->admin_mtx); /* XXX appropriate order? */ 3853 IXGBE_CORE_LOCK_DESTROY(adapter); 3854 3855 return (0); 3856 } /* ixgbe_detach */ 3857 3858 /************************************************************************ 3859 * ixgbe_setup_low_power_mode - LPLU/WoL preparation 3860 * 3861 * Prepare the adapter/port for LPLU and/or WoL 3862 ************************************************************************/ 3863 static int 3864 ixgbe_setup_low_power_mode(struct adapter *adapter) 3865 { 3866 struct ixgbe_hw *hw = &adapter->hw; 3867 device_t dev = adapter->dev; 3868 struct ifnet *ifp = adapter->ifp; 3869 s32 error = 0; 3870 3871 /* Limit power management flow to X550EM baseT */ 3872 if (hw->device_id == IXGBE_DEV_ID_X550EM_X_10G_T && 3873 hw->phy.ops.enter_lplu) { 3874 /* X550EM baseT adapters need a special LPLU flow */ 3875 hw->phy.reset_disable = true; 3876 ixgbe_ifstop(ifp, 1); 3877 error = hw->phy.ops.enter_lplu(hw); 3878 if (error) 3879 device_printf(dev, 3880 "Error entering LPLU: %d\n", error); 3881 hw->phy.reset_disable = false; 3882 } else { 3883 /* Just stop for other adapters */ 3884 ixgbe_ifstop(ifp, 1); 3885 } 3886 3887 IXGBE_CORE_LOCK(adapter); 3888 3889 if (!hw->wol_enabled) { 3890 ixgbe_set_phy_power(hw, FALSE); 3891 IXGBE_WRITE_REG(hw, IXGBE_WUFC, 0); 3892 IXGBE_WRITE_REG(hw, IXGBE_WUC, 0); 3893 } else { 3894 /* Turn off support for APM wakeup. (Using ACPI instead) */ 3895 IXGBE_WRITE_REG(hw, IXGBE_GRC_BY_MAC(hw), 3896 IXGBE_READ_REG(hw, IXGBE_GRC_BY_MAC(hw)) & ~(u32)2); 3897 3898 /* 3899 * Clear Wake Up Status register to prevent any previous wakeup 3900 * events from waking us up immediately after we suspend. 3901 */ 3902 IXGBE_WRITE_REG(hw, IXGBE_WUS, 0xffffffff); 3903 3904 /* 3905 * Program the Wakeup Filter Control register with user filter 3906 * settings 3907 */ 3908 IXGBE_WRITE_REG(hw, IXGBE_WUFC, adapter->wufc); 3909 3910 /* Enable wakeups and power management in Wakeup Control */ 3911 IXGBE_WRITE_REG(hw, IXGBE_WUC, 3912 IXGBE_WUC_WKEN | IXGBE_WUC_PME_EN); 3913 3914 } 3915 3916 IXGBE_CORE_UNLOCK(adapter); 3917 3918 return error; 3919 } /* ixgbe_setup_low_power_mode */ 3920 3921 /************************************************************************ 3922 * ixgbe_shutdown - Shutdown entry point 3923 ************************************************************************/ 3924 #if 0 /* XXX NetBSD ought to register something like this through pmf(9) */ 3925 static int 3926 ixgbe_shutdown(device_t dev) 3927 { 3928 struct adapter *adapter = device_private(dev); 3929 int error = 0; 3930 3931 INIT_DEBUGOUT("ixgbe_shutdown: begin"); 3932 3933 error = ixgbe_setup_low_power_mode(adapter); 3934 3935 return (error); 3936 } /* ixgbe_shutdown */ 3937 #endif 3938 3939 /************************************************************************ 3940 * ixgbe_suspend 3941 * 3942 * From D0 to D3 3943 ************************************************************************/ 3944 static bool 3945 ixgbe_suspend(device_t dev, const pmf_qual_t *qual) 3946 { 3947 struct adapter *adapter = device_private(dev); 3948 int error = 0; 3949 3950 INIT_DEBUGOUT("ixgbe_suspend: begin"); 3951 3952 error = ixgbe_setup_low_power_mode(adapter); 3953 3954 return (error); 3955 } /* ixgbe_suspend */ 3956 3957 /************************************************************************ 3958 * ixgbe_resume 3959 * 3960 * From D3 to D0 3961 ************************************************************************/ 3962 static bool 3963 ixgbe_resume(device_t dev, const pmf_qual_t *qual) 3964 { 3965 struct adapter *adapter = device_private(dev); 3966 struct ifnet *ifp = adapter->ifp; 3967 struct ixgbe_hw *hw = &adapter->hw; 3968 u32 wus; 3969 3970 INIT_DEBUGOUT("ixgbe_resume: begin"); 3971 3972 IXGBE_CORE_LOCK(adapter); 3973 3974 /* Read & clear WUS register */ 3975 wus = IXGBE_READ_REG(hw, IXGBE_WUS); 3976 if (wus) 3977 device_printf(dev, "Woken up by (WUS): %#010x\n", 3978 IXGBE_READ_REG(hw, IXGBE_WUS)); 3979 IXGBE_WRITE_REG(hw, IXGBE_WUS, 0xffffffff); 3980 /* And clear WUFC until next low-power transition */ 3981 IXGBE_WRITE_REG(hw, IXGBE_WUFC, 0); 3982 3983 /* 3984 * Required after D3->D0 transition; 3985 * will re-advertise all previous advertised speeds 3986 */ 3987 if (ifp->if_flags & IFF_UP) 3988 ixgbe_init_locked(adapter); 3989 3990 IXGBE_CORE_UNLOCK(adapter); 3991 3992 return true; 3993 } /* ixgbe_resume */ 3994 3995 /* 3996 * Set the various hardware offload abilities. 3997 * 3998 * This takes the ifnet's if_capenable flags (e.g. set by the user using 3999 * ifconfig) and indicates to the OS via the ifnet's if_hwassist field what 4000 * mbuf offload flags the driver will understand. 4001 */ 4002 static void 4003 ixgbe_set_if_hwassist(struct adapter *adapter) 4004 { 4005 /* XXX */ 4006 } 4007 4008 /************************************************************************ 4009 * ixgbe_init_locked - Init entry point 4010 * 4011 * Used in two ways: It is used by the stack as an init 4012 * entry point in network interface structure. It is also 4013 * used by the driver as a hw/sw initialization routine to 4014 * get to a consistent state. 4015 * 4016 * return 0 on success, positive on failure 4017 ************************************************************************/ 4018 static void 4019 ixgbe_init_locked(struct adapter *adapter) 4020 { 4021 struct ifnet *ifp = adapter->ifp; 4022 device_t dev = adapter->dev; 4023 struct ixgbe_hw *hw = &adapter->hw; 4024 struct ix_queue *que; 4025 struct tx_ring *txr; 4026 struct rx_ring *rxr; 4027 u32 txdctl, mhadd; 4028 u32 rxdctl, rxctrl; 4029 u32 ctrl_ext; 4030 bool unsupported_sfp = false; 4031 int i, j, error; 4032 4033 /* XXX check IFF_UP and IFF_RUNNING, power-saving state! */ 4034 4035 KASSERT(mutex_owned(&adapter->core_mtx)); 4036 INIT_DEBUGOUT("ixgbe_init_locked: begin"); 4037 4038 hw->need_unsupported_sfp_recovery = false; 4039 hw->adapter_stopped = FALSE; 4040 ixgbe_stop_adapter(hw); 4041 callout_stop(&adapter->timer); 4042 if (adapter->feat_en & IXGBE_FEATURE_RECOVERY_MODE) 4043 callout_stop(&adapter->recovery_mode_timer); 4044 for (i = 0, que = adapter->queues; i < adapter->num_queues; i++, que++) 4045 que->disabled_count = 0; 4046 4047 /* XXX I moved this here from the SIOCSIFMTU case in ixgbe_ioctl(). */ 4048 adapter->max_frame_size = 4049 ifp->if_mtu + ETHER_HDR_LEN + ETHER_CRC_LEN; 4050 4051 /* Queue indices may change with IOV mode */ 4052 ixgbe_align_all_queue_indices(adapter); 4053 4054 /* reprogram the RAR[0] in case user changed it. */ 4055 ixgbe_set_rar(hw, 0, hw->mac.addr, adapter->pool, IXGBE_RAH_AV); 4056 4057 /* Get the latest mac address, User can use a LAA */ 4058 memcpy(hw->mac.addr, CLLADDR(ifp->if_sadl), 4059 IXGBE_ETH_LENGTH_OF_ADDRESS); 4060 ixgbe_set_rar(hw, 0, hw->mac.addr, adapter->pool, 1); 4061 hw->addr_ctrl.rar_used_count = 1; 4062 4063 /* Set hardware offload abilities from ifnet flags */ 4064 ixgbe_set_if_hwassist(adapter); 4065 4066 /* Prepare transmit descriptors and buffers */ 4067 if (ixgbe_setup_transmit_structures(adapter)) { 4068 device_printf(dev, "Could not setup transmit structures\n"); 4069 ixgbe_stop_locked(adapter); 4070 return; 4071 } 4072 4073 ixgbe_init_hw(hw); 4074 4075 ixgbe_initialize_iov(adapter); 4076 4077 ixgbe_initialize_transmit_units(adapter); 4078 4079 /* Setup Multicast table */ 4080 ixgbe_set_rxfilter(adapter); 4081 4082 /* Use fixed buffer size, even for jumbo frames */ 4083 adapter->rx_mbuf_sz = MCLBYTES; 4084 4085 /* Prepare receive descriptors and buffers */ 4086 error = ixgbe_setup_receive_structures(adapter); 4087 if (error) { 4088 device_printf(dev, 4089 "Could not setup receive structures (err = %d)\n", error); 4090 ixgbe_stop_locked(adapter); 4091 return; 4092 } 4093 4094 /* Configure RX settings */ 4095 ixgbe_initialize_receive_units(adapter); 4096 4097 /* Initialize variable holding task enqueue requests interrupts */ 4098 adapter->task_requests = 0; 4099 4100 /* Enable SDP & MSI-X interrupts based on adapter */ 4101 ixgbe_config_gpie(adapter); 4102 4103 /* Set MTU size */ 4104 if (ifp->if_mtu > ETHERMTU) { 4105 /* aka IXGBE_MAXFRS on 82599 and newer */ 4106 mhadd = IXGBE_READ_REG(hw, IXGBE_MHADD); 4107 mhadd &= ~IXGBE_MHADD_MFS_MASK; 4108 mhadd |= adapter->max_frame_size << IXGBE_MHADD_MFS_SHIFT; 4109 IXGBE_WRITE_REG(hw, IXGBE_MHADD, mhadd); 4110 } 4111 4112 /* Now enable all the queues */ 4113 for (i = 0; i < adapter->num_queues; i++) { 4114 txr = &adapter->tx_rings[i]; 4115 txdctl = IXGBE_READ_REG(hw, IXGBE_TXDCTL(txr->me)); 4116 txdctl |= IXGBE_TXDCTL_ENABLE; 4117 /* Set WTHRESH to 8, burst writeback */ 4118 txdctl |= IXGBE_TX_WTHRESH << IXGBE_TXDCTL_WTHRESH_SHIFT; 4119 /* 4120 * When the internal queue falls below PTHRESH (32), 4121 * start prefetching as long as there are at least 4122 * HTHRESH (1) buffers ready. The values are taken 4123 * from the Intel linux driver 3.8.21. 4124 * Prefetching enables tx line rate even with 1 queue. 4125 */ 4126 txdctl |= (32 << 0) | (1 << 8); 4127 IXGBE_WRITE_REG(hw, IXGBE_TXDCTL(txr->me), txdctl); 4128 } 4129 4130 for (i = 0; i < adapter->num_queues; i++) { 4131 rxr = &adapter->rx_rings[i]; 4132 rxdctl = IXGBE_READ_REG(hw, IXGBE_RXDCTL(rxr->me)); 4133 if (hw->mac.type == ixgbe_mac_82598EB) { 4134 /* 4135 * PTHRESH = 21 4136 * HTHRESH = 4 4137 * WTHRESH = 8 4138 */ 4139 rxdctl &= ~0x3FFFFF; 4140 rxdctl |= 0x080420; 4141 } 4142 rxdctl |= IXGBE_RXDCTL_ENABLE; 4143 IXGBE_WRITE_REG(hw, IXGBE_RXDCTL(rxr->me), rxdctl); 4144 for (j = 0; j < 10; j++) { 4145 if (IXGBE_READ_REG(hw, IXGBE_RXDCTL(rxr->me)) & 4146 IXGBE_RXDCTL_ENABLE) 4147 break; 4148 else 4149 msec_delay(1); 4150 } 4151 IXGBE_WRITE_BARRIER(hw); 4152 4153 /* 4154 * In netmap mode, we must preserve the buffers made 4155 * available to userspace before the if_init() 4156 * (this is true by default on the TX side, because 4157 * init makes all buffers available to userspace). 4158 * 4159 * netmap_reset() and the device specific routines 4160 * (e.g. ixgbe_setup_receive_rings()) map these 4161 * buffers at the end of the NIC ring, so here we 4162 * must set the RDT (tail) register to make sure 4163 * they are not overwritten. 4164 * 4165 * In this driver the NIC ring starts at RDH = 0, 4166 * RDT points to the last slot available for reception (?), 4167 * so RDT = num_rx_desc - 1 means the whole ring is available. 4168 */ 4169 #ifdef DEV_NETMAP 4170 if ((adapter->feat_en & IXGBE_FEATURE_NETMAP) && 4171 (ifp->if_capenable & IFCAP_NETMAP)) { 4172 struct netmap_adapter *na = NA(adapter->ifp); 4173 struct netmap_kring *kring = na->rx_rings[i]; 4174 int t = na->num_rx_desc - 1 - nm_kr_rxspace(kring); 4175 4176 IXGBE_WRITE_REG(hw, IXGBE_RDT(rxr->me), t); 4177 } else 4178 #endif /* DEV_NETMAP */ 4179 IXGBE_WRITE_REG(hw, IXGBE_RDT(rxr->me), 4180 adapter->num_rx_desc - 1); 4181 } 4182 4183 /* Enable Receive engine */ 4184 rxctrl = IXGBE_READ_REG(hw, IXGBE_RXCTRL); 4185 if (hw->mac.type == ixgbe_mac_82598EB) 4186 rxctrl |= IXGBE_RXCTRL_DMBYPS; 4187 rxctrl |= IXGBE_RXCTRL_RXEN; 4188 ixgbe_enable_rx_dma(hw, rxctrl); 4189 4190 callout_reset(&adapter->timer, hz, ixgbe_local_timer, adapter); 4191 atomic_store_relaxed(&adapter->timer_pending, 0); 4192 if (adapter->feat_en & IXGBE_FEATURE_RECOVERY_MODE) 4193 callout_reset(&adapter->recovery_mode_timer, hz, 4194 ixgbe_recovery_mode_timer, adapter); 4195 4196 /* Set up MSI/MSI-X routing */ 4197 if (adapter->feat_en & IXGBE_FEATURE_MSIX) { 4198 ixgbe_configure_ivars(adapter); 4199 /* Set up auto-mask */ 4200 if (hw->mac.type == ixgbe_mac_82598EB) 4201 IXGBE_WRITE_REG(hw, IXGBE_EIAM, IXGBE_EICS_RTX_QUEUE); 4202 else { 4203 IXGBE_WRITE_REG(hw, IXGBE_EIAM_EX(0), 0xFFFFFFFF); 4204 IXGBE_WRITE_REG(hw, IXGBE_EIAM_EX(1), 0xFFFFFFFF); 4205 } 4206 } else { /* Simple settings for Legacy/MSI */ 4207 ixgbe_set_ivar(adapter, 0, 0, 0); 4208 ixgbe_set_ivar(adapter, 0, 0, 1); 4209 IXGBE_WRITE_REG(hw, IXGBE_EIAM, IXGBE_EICS_RTX_QUEUE); 4210 } 4211 4212 ixgbe_init_fdir(adapter); 4213 4214 /* 4215 * Check on any SFP devices that 4216 * need to be kick-started 4217 */ 4218 if (hw->phy.type == ixgbe_phy_none) { 4219 error = hw->phy.ops.identify(hw); 4220 if (error == IXGBE_ERR_SFP_NOT_SUPPORTED) 4221 unsupported_sfp = true; 4222 } else if (hw->phy.type == ixgbe_phy_sfp_unsupported) 4223 unsupported_sfp = true; 4224 4225 if (unsupported_sfp) 4226 device_printf(dev, 4227 "Unsupported SFP+ module type was detected.\n"); 4228 4229 /* Set moderation on the Link interrupt */ 4230 ixgbe_eitr_write(adapter, adapter->vector, IXGBE_LINK_ITR); 4231 4232 /* Enable EEE power saving */ 4233 if (adapter->feat_cap & IXGBE_FEATURE_EEE) 4234 hw->mac.ops.setup_eee(hw, 4235 adapter->feat_en & IXGBE_FEATURE_EEE); 4236 4237 /* Enable power to the phy. */ 4238 if (!unsupported_sfp) { 4239 ixgbe_set_phy_power(hw, TRUE); 4240 4241 /* Config/Enable Link */ 4242 ixgbe_config_link(adapter); 4243 } 4244 4245 /* Hardware Packet Buffer & Flow Control setup */ 4246 ixgbe_config_delay_values(adapter); 4247 4248 /* Initialize the FC settings */ 4249 ixgbe_start_hw(hw); 4250 4251 /* Set up VLAN support and filter */ 4252 ixgbe_setup_vlan_hw_support(adapter); 4253 4254 /* Setup DMA Coalescing */ 4255 ixgbe_config_dmac(adapter); 4256 4257 /* OK to schedule workqueues. */ 4258 adapter->schedule_wqs_ok = true; 4259 4260 /* And now turn on interrupts */ 4261 ixgbe_enable_intr(adapter); 4262 4263 /* Enable the use of the MBX by the VF's */ 4264 if (adapter->feat_en & IXGBE_FEATURE_SRIOV) { 4265 ctrl_ext = IXGBE_READ_REG(hw, IXGBE_CTRL_EXT); 4266 ctrl_ext |= IXGBE_CTRL_EXT_PFRSTD; 4267 IXGBE_WRITE_REG(hw, IXGBE_CTRL_EXT, ctrl_ext); 4268 } 4269 4270 /* Update saved flags. See ixgbe_ifflags_cb() */ 4271 adapter->if_flags = ifp->if_flags; 4272 adapter->ec_capenable = adapter->osdep.ec.ec_capenable; 4273 4274 /* Now inform the stack we're ready */ 4275 ifp->if_flags |= IFF_RUNNING; 4276 4277 return; 4278 } /* ixgbe_init_locked */ 4279 4280 /************************************************************************ 4281 * ixgbe_init 4282 ************************************************************************/ 4283 static int 4284 ixgbe_init(struct ifnet *ifp) 4285 { 4286 struct adapter *adapter = ifp->if_softc; 4287 4288 IXGBE_CORE_LOCK(adapter); 4289 ixgbe_init_locked(adapter); 4290 IXGBE_CORE_UNLOCK(adapter); 4291 4292 return 0; /* XXX ixgbe_init_locked cannot fail? really? */ 4293 } /* ixgbe_init */ 4294 4295 /************************************************************************ 4296 * ixgbe_set_ivar 4297 * 4298 * Setup the correct IVAR register for a particular MSI-X interrupt 4299 * (yes this is all very magic and confusing :) 4300 * - entry is the register array entry 4301 * - vector is the MSI-X vector for this queue 4302 * - type is RX/TX/MISC 4303 ************************************************************************/ 4304 static void 4305 ixgbe_set_ivar(struct adapter *adapter, u8 entry, u8 vector, s8 type) 4306 { 4307 struct ixgbe_hw *hw = &adapter->hw; 4308 u32 ivar, index; 4309 4310 vector |= IXGBE_IVAR_ALLOC_VAL; 4311 4312 switch (hw->mac.type) { 4313 case ixgbe_mac_82598EB: 4314 if (type == -1) 4315 entry = IXGBE_IVAR_OTHER_CAUSES_INDEX; 4316 else 4317 entry += (type * 64); 4318 index = (entry >> 2) & 0x1F; 4319 ivar = IXGBE_READ_REG(hw, IXGBE_IVAR(index)); 4320 ivar &= ~(0xffUL << (8 * (entry & 0x3))); 4321 ivar |= ((u32)vector << (8 * (entry & 0x3))); 4322 IXGBE_WRITE_REG(&adapter->hw, IXGBE_IVAR(index), ivar); 4323 break; 4324 case ixgbe_mac_82599EB: 4325 case ixgbe_mac_X540: 4326 case ixgbe_mac_X550: 4327 case ixgbe_mac_X550EM_x: 4328 case ixgbe_mac_X550EM_a: 4329 if (type == -1) { /* MISC IVAR */ 4330 index = (entry & 1) * 8; 4331 ivar = IXGBE_READ_REG(hw, IXGBE_IVAR_MISC); 4332 ivar &= ~(0xffUL << index); 4333 ivar |= ((u32)vector << index); 4334 IXGBE_WRITE_REG(hw, IXGBE_IVAR_MISC, ivar); 4335 } else { /* RX/TX IVARS */ 4336 index = (16 * (entry & 1)) + (8 * type); 4337 ivar = IXGBE_READ_REG(hw, IXGBE_IVAR(entry >> 1)); 4338 ivar &= ~(0xffUL << index); 4339 ivar |= ((u32)vector << index); 4340 IXGBE_WRITE_REG(hw, IXGBE_IVAR(entry >> 1), ivar); 4341 } 4342 break; 4343 default: 4344 break; 4345 } 4346 } /* ixgbe_set_ivar */ 4347 4348 /************************************************************************ 4349 * ixgbe_configure_ivars 4350 ************************************************************************/ 4351 static void 4352 ixgbe_configure_ivars(struct adapter *adapter) 4353 { 4354 struct ix_queue *que = adapter->queues; 4355 u32 newitr; 4356 4357 if (ixgbe_max_interrupt_rate > 0) 4358 newitr = (4000000 / ixgbe_max_interrupt_rate) & 0x0FF8; 4359 else { 4360 /* 4361 * Disable DMA coalescing if interrupt moderation is 4362 * disabled. 4363 */ 4364 adapter->dmac = 0; 4365 newitr = 0; 4366 } 4367 4368 for (int i = 0; i < adapter->num_queues; i++, que++) { 4369 struct rx_ring *rxr = &adapter->rx_rings[i]; 4370 struct tx_ring *txr = &adapter->tx_rings[i]; 4371 /* First the RX queue entry */ 4372 ixgbe_set_ivar(adapter, rxr->me, que->msix, 0); 4373 /* ... and the TX */ 4374 ixgbe_set_ivar(adapter, txr->me, que->msix, 1); 4375 /* Set an Initial EITR value */ 4376 ixgbe_eitr_write(adapter, que->msix, newitr); 4377 /* 4378 * To eliminate influence of the previous state. 4379 * At this point, Tx/Rx interrupt handler 4380 * (ixgbe_msix_que()) cannot be called, so both 4381 * IXGBE_TX_LOCK and IXGBE_RX_LOCK are not required. 4382 */ 4383 que->eitr_setting = 0; 4384 } 4385 4386 /* For the Link interrupt */ 4387 ixgbe_set_ivar(adapter, 1, adapter->vector, -1); 4388 } /* ixgbe_configure_ivars */ 4389 4390 /************************************************************************ 4391 * ixgbe_config_gpie 4392 ************************************************************************/ 4393 static void 4394 ixgbe_config_gpie(struct adapter *adapter) 4395 { 4396 struct ixgbe_hw *hw = &adapter->hw; 4397 u32 gpie; 4398 4399 gpie = IXGBE_READ_REG(hw, IXGBE_GPIE); 4400 4401 if (adapter->feat_en & IXGBE_FEATURE_MSIX) { 4402 /* Enable Enhanced MSI-X mode */ 4403 gpie |= IXGBE_GPIE_MSIX_MODE 4404 | IXGBE_GPIE_EIAME 4405 | IXGBE_GPIE_PBA_SUPPORT 4406 | IXGBE_GPIE_OCD; 4407 } 4408 4409 /* Fan Failure Interrupt */ 4410 if (adapter->feat_en & IXGBE_FEATURE_FAN_FAIL) 4411 gpie |= IXGBE_SDP1_GPIEN; 4412 4413 /* Thermal Sensor Interrupt */ 4414 if (adapter->feat_en & IXGBE_FEATURE_TEMP_SENSOR) 4415 gpie |= IXGBE_SDP0_GPIEN_X540; 4416 4417 /* Link detection */ 4418 switch (hw->mac.type) { 4419 case ixgbe_mac_82599EB: 4420 gpie |= IXGBE_SDP1_GPIEN | IXGBE_SDP2_GPIEN; 4421 break; 4422 case ixgbe_mac_X550EM_x: 4423 case ixgbe_mac_X550EM_a: 4424 gpie |= IXGBE_SDP0_GPIEN_X540; 4425 break; 4426 default: 4427 break; 4428 } 4429 4430 IXGBE_WRITE_REG(hw, IXGBE_GPIE, gpie); 4431 4432 } /* ixgbe_config_gpie */ 4433 4434 /************************************************************************ 4435 * ixgbe_config_delay_values 4436 * 4437 * Requires adapter->max_frame_size to be set. 4438 ************************************************************************/ 4439 static void 4440 ixgbe_config_delay_values(struct adapter *adapter) 4441 { 4442 struct ixgbe_hw *hw = &adapter->hw; 4443 u32 rxpb, frame, size, tmp; 4444 4445 frame = adapter->max_frame_size; 4446 4447 /* Calculate High Water */ 4448 switch (hw->mac.type) { 4449 case ixgbe_mac_X540: 4450 case ixgbe_mac_X550: 4451 case ixgbe_mac_X550EM_x: 4452 case ixgbe_mac_X550EM_a: 4453 tmp = IXGBE_DV_X540(frame, frame); 4454 break; 4455 default: 4456 tmp = IXGBE_DV(frame, frame); 4457 break; 4458 } 4459 size = IXGBE_BT2KB(tmp); 4460 rxpb = IXGBE_READ_REG(hw, IXGBE_RXPBSIZE(0)) >> 10; 4461 hw->fc.high_water[0] = rxpb - size; 4462 4463 /* Now calculate Low Water */ 4464 switch (hw->mac.type) { 4465 case ixgbe_mac_X540: 4466 case ixgbe_mac_X550: 4467 case ixgbe_mac_X550EM_x: 4468 case ixgbe_mac_X550EM_a: 4469 tmp = IXGBE_LOW_DV_X540(frame); 4470 break; 4471 default: 4472 tmp = IXGBE_LOW_DV(frame); 4473 break; 4474 } 4475 hw->fc.low_water[0] = IXGBE_BT2KB(tmp); 4476 4477 hw->fc.pause_time = IXGBE_FC_PAUSE; 4478 hw->fc.send_xon = TRUE; 4479 } /* ixgbe_config_delay_values */ 4480 4481 /************************************************************************ 4482 * ixgbe_set_rxfilter - Multicast Update 4483 * 4484 * Called whenever multicast address list is updated. 4485 ************************************************************************/ 4486 static void 4487 ixgbe_set_rxfilter(struct adapter *adapter) 4488 { 4489 struct ixgbe_mc_addr *mta; 4490 struct ifnet *ifp = adapter->ifp; 4491 u8 *update_ptr; 4492 int mcnt = 0; 4493 u32 fctrl; 4494 struct ethercom *ec = &adapter->osdep.ec; 4495 struct ether_multi *enm; 4496 struct ether_multistep step; 4497 4498 KASSERT(mutex_owned(&adapter->core_mtx)); 4499 IOCTL_DEBUGOUT("ixgbe_set_rxfilter: begin"); 4500 4501 mta = adapter->mta; 4502 bzero(mta, sizeof(*mta) * MAX_NUM_MULTICAST_ADDRESSES); 4503 4504 ETHER_LOCK(ec); 4505 ec->ec_flags &= ~ETHER_F_ALLMULTI; 4506 ETHER_FIRST_MULTI(step, ec, enm); 4507 while (enm != NULL) { 4508 if ((mcnt == MAX_NUM_MULTICAST_ADDRESSES) || 4509 (memcmp(enm->enm_addrlo, enm->enm_addrhi, 4510 ETHER_ADDR_LEN) != 0)) { 4511 ec->ec_flags |= ETHER_F_ALLMULTI; 4512 break; 4513 } 4514 bcopy(enm->enm_addrlo, 4515 mta[mcnt].addr, IXGBE_ETH_LENGTH_OF_ADDRESS); 4516 mta[mcnt].vmdq = adapter->pool; 4517 mcnt++; 4518 ETHER_NEXT_MULTI(step, enm); 4519 } 4520 4521 fctrl = IXGBE_READ_REG(&adapter->hw, IXGBE_FCTRL); 4522 if (ifp->if_flags & IFF_PROMISC) 4523 fctrl |= (IXGBE_FCTRL_UPE | IXGBE_FCTRL_MPE); 4524 else if (ec->ec_flags & ETHER_F_ALLMULTI) { 4525 fctrl |= IXGBE_FCTRL_MPE; 4526 fctrl &= ~IXGBE_FCTRL_UPE; 4527 } else 4528 fctrl &= ~(IXGBE_FCTRL_UPE | IXGBE_FCTRL_MPE); 4529 4530 IXGBE_WRITE_REG(&adapter->hw, IXGBE_FCTRL, fctrl); 4531 4532 /* Update multicast filter entries only when it's not ALLMULTI */ 4533 if ((ec->ec_flags & ETHER_F_ALLMULTI) == 0) { 4534 ETHER_UNLOCK(ec); 4535 update_ptr = (u8 *)mta; 4536 ixgbe_update_mc_addr_list(&adapter->hw, update_ptr, mcnt, 4537 ixgbe_mc_array_itr, TRUE); 4538 } else 4539 ETHER_UNLOCK(ec); 4540 } /* ixgbe_set_rxfilter */ 4541 4542 /************************************************************************ 4543 * ixgbe_mc_array_itr 4544 * 4545 * An iterator function needed by the multicast shared code. 4546 * It feeds the shared code routine the addresses in the 4547 * array of ixgbe_set_rxfilter() one by one. 4548 ************************************************************************/ 4549 static u8 * 4550 ixgbe_mc_array_itr(struct ixgbe_hw *hw, u8 **update_ptr, u32 *vmdq) 4551 { 4552 struct ixgbe_mc_addr *mta; 4553 4554 mta = (struct ixgbe_mc_addr *)*update_ptr; 4555 *vmdq = mta->vmdq; 4556 4557 *update_ptr = (u8*)(mta + 1); 4558 4559 return (mta->addr); 4560 } /* ixgbe_mc_array_itr */ 4561 4562 /************************************************************************ 4563 * ixgbe_local_timer - Timer routine 4564 * 4565 * Checks for link status, updates statistics, 4566 * and runs the watchdog check. 4567 ************************************************************************/ 4568 static void 4569 ixgbe_local_timer(void *arg) 4570 { 4571 struct adapter *adapter = arg; 4572 4573 if (adapter->schedule_wqs_ok) { 4574 if (atomic_cas_uint(&adapter->timer_pending, 0, 1) == 0) 4575 workqueue_enqueue(adapter->timer_wq, 4576 &adapter->timer_wc, NULL); 4577 } 4578 } 4579 4580 static void 4581 ixgbe_handle_timer(struct work *wk, void *context) 4582 { 4583 struct adapter *adapter = context; 4584 struct ixgbe_hw *hw = &adapter->hw; 4585 device_t dev = adapter->dev; 4586 struct ix_queue *que = adapter->queues; 4587 u64 queues = 0; 4588 u64 v0, v1, v2, v3, v4, v5, v6, v7; 4589 int hung = 0; 4590 int i; 4591 4592 IXGBE_CORE_LOCK(adapter); 4593 4594 /* Check for pluggable optics */ 4595 if (ixgbe_is_sfp(hw)) { 4596 bool sched_mod_task = false; 4597 4598 if (hw->mac.type == ixgbe_mac_82598EB) { 4599 /* 4600 * On 82598EB, SFP+'s MOD_ABS pin is not connected to 4601 * any GPIO(SDP). So just schedule TASK_MOD. 4602 */ 4603 sched_mod_task = true; 4604 } else { 4605 bool was_full, is_full; 4606 4607 was_full = 4608 hw->phy.sfp_type != ixgbe_sfp_type_not_present; 4609 is_full = ixgbe_sfp_cage_full(hw); 4610 4611 /* Do probe if cage state changed */ 4612 if (was_full ^ is_full) 4613 sched_mod_task = true; 4614 } 4615 if (sched_mod_task) { 4616 mutex_enter(&adapter->admin_mtx); 4617 adapter->task_requests |= IXGBE_REQUEST_TASK_MOD_WOI; 4618 ixgbe_schedule_admin_tasklet(adapter); 4619 mutex_exit(&adapter->admin_mtx); 4620 } 4621 } 4622 4623 ixgbe_update_link_status(adapter); 4624 ixgbe_update_stats_counters(adapter); 4625 4626 /* Update some event counters */ 4627 v0 = v1 = v2 = v3 = v4 = v5 = v6 = v7 = 0; 4628 que = adapter->queues; 4629 for (i = 0; i < adapter->num_queues; i++, que++) { 4630 struct tx_ring *txr = que->txr; 4631 4632 v0 += txr->q_efbig_tx_dma_setup; 4633 v1 += txr->q_mbuf_defrag_failed; 4634 v2 += txr->q_efbig2_tx_dma_setup; 4635 v3 += txr->q_einval_tx_dma_setup; 4636 v4 += txr->q_other_tx_dma_setup; 4637 v5 += txr->q_eagain_tx_dma_setup; 4638 v6 += txr->q_enomem_tx_dma_setup; 4639 v7 += txr->q_tso_err; 4640 } 4641 IXGBE_EVC_STORE(&adapter->efbig_tx_dma_setup, v0); 4642 IXGBE_EVC_STORE(&adapter->mbuf_defrag_failed, v1); 4643 IXGBE_EVC_STORE(&adapter->efbig2_tx_dma_setup, v2); 4644 IXGBE_EVC_STORE(&adapter->einval_tx_dma_setup, v3); 4645 IXGBE_EVC_STORE(&adapter->other_tx_dma_setup, v4); 4646 IXGBE_EVC_STORE(&adapter->eagain_tx_dma_setup, v5); 4647 IXGBE_EVC_STORE(&adapter->enomem_tx_dma_setup, v6); 4648 IXGBE_EVC_STORE(&adapter->tso_err, v7); 4649 4650 /* 4651 * Check the TX queues status 4652 * - mark hung queues so we don't schedule on them 4653 * - watchdog only if all queues show hung 4654 */ 4655 que = adapter->queues; 4656 for (i = 0; i < adapter->num_queues; i++, que++) { 4657 /* Keep track of queues with work for soft irq */ 4658 if (que->txr->busy) 4659 queues |= 1ULL << que->me; 4660 /* 4661 * Each time txeof runs without cleaning, but there 4662 * are uncleaned descriptors it increments busy. If 4663 * we get to the MAX we declare it hung. 4664 */ 4665 if (que->busy == IXGBE_QUEUE_HUNG) { 4666 ++hung; 4667 /* Mark the queue as inactive */ 4668 adapter->active_queues &= ~(1ULL << que->me); 4669 continue; 4670 } else { 4671 /* Check if we've come back from hung */ 4672 if ((adapter->active_queues & (1ULL << que->me)) == 0) 4673 adapter->active_queues |= 1ULL << que->me; 4674 } 4675 if (que->busy >= IXGBE_MAX_TX_BUSY) { 4676 device_printf(dev, 4677 "Warning queue %d appears to be hung!\n", i); 4678 que->txr->busy = IXGBE_QUEUE_HUNG; 4679 ++hung; 4680 } 4681 } 4682 4683 /* Only truly watchdog if all queues show hung */ 4684 if (hung == adapter->num_queues) 4685 goto watchdog; 4686 #if 0 /* XXX Avoid unexpectedly disabling interrupt forever (PR#53294) */ 4687 else if (queues != 0) { /* Force an IRQ on queues with work */ 4688 que = adapter->queues; 4689 for (i = 0; i < adapter->num_queues; i++, que++) { 4690 mutex_enter(&que->dc_mtx); 4691 if (que->disabled_count == 0) 4692 ixgbe_rearm_queues(adapter, 4693 queues & ((u64)1 << i)); 4694 mutex_exit(&que->dc_mtx); 4695 } 4696 } 4697 #endif 4698 4699 atomic_store_relaxed(&adapter->timer_pending, 0); 4700 IXGBE_CORE_UNLOCK(adapter); 4701 callout_reset(&adapter->timer, hz, ixgbe_local_timer, adapter); 4702 return; 4703 4704 watchdog: 4705 device_printf(adapter->dev, "Watchdog timeout -- resetting\n"); 4706 adapter->ifp->if_flags &= ~IFF_RUNNING; 4707 IXGBE_EVC_ADD(&adapter->watchdog_events, 1); 4708 ixgbe_init_locked(adapter); 4709 IXGBE_CORE_UNLOCK(adapter); 4710 } /* ixgbe_handle_timer */ 4711 4712 /************************************************************************ 4713 * ixgbe_recovery_mode_timer - Recovery mode timer routine 4714 ************************************************************************/ 4715 static void 4716 ixgbe_recovery_mode_timer(void *arg) 4717 { 4718 struct adapter *adapter = arg; 4719 4720 if (__predict_true(adapter->osdep.detaching == false)) { 4721 if (atomic_cas_uint(&adapter->recovery_mode_timer_pending, 4722 0, 1) == 0) { 4723 workqueue_enqueue(adapter->recovery_mode_timer_wq, 4724 &adapter->recovery_mode_timer_wc, NULL); 4725 } 4726 } 4727 } 4728 4729 static void 4730 ixgbe_handle_recovery_mode_timer(struct work *wk, void *context) 4731 { 4732 struct adapter *adapter = context; 4733 struct ixgbe_hw *hw = &adapter->hw; 4734 4735 IXGBE_CORE_LOCK(adapter); 4736 if (ixgbe_fw_recovery_mode(hw)) { 4737 if (atomic_cas_uint(&adapter->recovery_mode, 0, 1)) { 4738 /* Firmware error detected, entering recovery mode */ 4739 device_printf(adapter->dev, "Firmware recovery mode detected. Limiting functionality. Refer to the Intel(R) Ethernet Adapters and Devices User Guide for details on firmware recovery mode.\n"); 4740 4741 if (hw->adapter_stopped == FALSE) 4742 ixgbe_stop_locked(adapter); 4743 } 4744 } else 4745 atomic_cas_uint(&adapter->recovery_mode, 1, 0); 4746 4747 atomic_store_relaxed(&adapter->recovery_mode_timer_pending, 0); 4748 callout_reset(&adapter->recovery_mode_timer, hz, 4749 ixgbe_recovery_mode_timer, adapter); 4750 IXGBE_CORE_UNLOCK(adapter); 4751 } /* ixgbe_handle_recovery_mode_timer */ 4752 4753 /************************************************************************ 4754 * ixgbe_handle_mod - Tasklet for SFP module interrupts 4755 * bool int_en: true if it's called when the interrupt is enabled. 4756 ************************************************************************/ 4757 static void 4758 ixgbe_handle_mod(void *context, bool int_en) 4759 { 4760 struct adapter *adapter = context; 4761 struct ixgbe_hw *hw = &adapter->hw; 4762 device_t dev = adapter->dev; 4763 enum ixgbe_sfp_type last_sfp_type; 4764 u32 err; 4765 bool last_unsupported_sfp_recovery; 4766 4767 KASSERT(mutex_owned(&adapter->core_mtx)); 4768 4769 last_sfp_type = hw->phy.sfp_type; 4770 last_unsupported_sfp_recovery = hw->need_unsupported_sfp_recovery; 4771 IXGBE_EVC_ADD(&adapter->mod_workev, 1); 4772 if (adapter->hw.need_crosstalk_fix) { 4773 if ((hw->mac.type != ixgbe_mac_82598EB) && 4774 !ixgbe_sfp_cage_full(hw)) 4775 goto out; 4776 } 4777 4778 err = hw->phy.ops.identify_sfp(hw); 4779 if (err == IXGBE_ERR_SFP_NOT_SUPPORTED) { 4780 if (last_unsupported_sfp_recovery == false) 4781 device_printf(dev, 4782 "Unsupported SFP+ module type was detected.\n"); 4783 goto out; 4784 } 4785 4786 if (hw->need_unsupported_sfp_recovery) { 4787 device_printf(dev, "Recovering from unsupported SFP\n"); 4788 /* 4789 * We could recover the status by calling setup_sfp(), 4790 * setup_link() and some others. It's complex and might not 4791 * work correctly on some unknown cases. To avoid such type of 4792 * problem, call ixgbe_init_locked(). It's simple and safe 4793 * approach. 4794 */ 4795 ixgbe_init_locked(adapter); 4796 } else if ((hw->phy.sfp_type != ixgbe_sfp_type_not_present) && 4797 (hw->phy.sfp_type != last_sfp_type)) { 4798 /* A module is inserted and changed. */ 4799 4800 if (hw->mac.type == ixgbe_mac_82598EB) 4801 err = hw->phy.ops.reset(hw); 4802 else { 4803 err = hw->mac.ops.setup_sfp(hw); 4804 hw->phy.sfp_setup_needed = FALSE; 4805 } 4806 if (err == IXGBE_ERR_SFP_NOT_SUPPORTED) { 4807 device_printf(dev, 4808 "Setup failure - unsupported SFP+ module type.\n"); 4809 goto out; 4810 } 4811 } 4812 4813 out: 4814 /* get_supported_phy_layer will call hw->phy.ops.identify_sfp() */ 4815 adapter->phy_layer = ixgbe_get_supported_physical_layer(hw); 4816 4817 /* Adjust media types shown in ifconfig */ 4818 IXGBE_CORE_UNLOCK(adapter); 4819 ifmedia_removeall(&adapter->media); 4820 ixgbe_add_media_types(adapter); 4821 ifmedia_set(&adapter->media, IFM_ETHER | IFM_AUTO); 4822 IXGBE_CORE_LOCK(adapter); 4823 4824 /* 4825 * Don't schedule MSF event if the chip is 82598. 82598 doesn't support 4826 * MSF. At least, calling ixgbe_handle_msf on 82598 DA makes the link 4827 * flap because the function calls setup_link(). 4828 */ 4829 if (hw->mac.type != ixgbe_mac_82598EB) { 4830 mutex_enter(&adapter->admin_mtx); 4831 if (int_en) 4832 adapter->task_requests |= IXGBE_REQUEST_TASK_MSF; 4833 else 4834 adapter->task_requests |= IXGBE_REQUEST_TASK_MSF_WOI; 4835 mutex_exit(&adapter->admin_mtx); 4836 } 4837 4838 /* 4839 * Don't call ixgbe_schedule_admin_tasklet() because we are on 4840 * the workqueue now. 4841 */ 4842 } /* ixgbe_handle_mod */ 4843 4844 4845 /************************************************************************ 4846 * ixgbe_handle_msf - Tasklet for MSF (multispeed fiber) interrupts 4847 ************************************************************************/ 4848 static void 4849 ixgbe_handle_msf(void *context) 4850 { 4851 struct adapter *adapter = context; 4852 struct ixgbe_hw *hw = &adapter->hw; 4853 u32 autoneg; 4854 bool negotiate; 4855 4856 KASSERT(mutex_owned(&adapter->core_mtx)); 4857 4858 IXGBE_EVC_ADD(&adapter->msf_workev, 1); 4859 4860 autoneg = hw->phy.autoneg_advertised; 4861 if ((!autoneg) && (hw->mac.ops.get_link_capabilities)) 4862 hw->mac.ops.get_link_capabilities(hw, &autoneg, &negotiate); 4863 if (hw->mac.ops.setup_link) 4864 hw->mac.ops.setup_link(hw, autoneg, TRUE); 4865 } /* ixgbe_handle_msf */ 4866 4867 /************************************************************************ 4868 * ixgbe_handle_phy - Tasklet for external PHY interrupts 4869 ************************************************************************/ 4870 static void 4871 ixgbe_handle_phy(void *context) 4872 { 4873 struct adapter *adapter = context; 4874 struct ixgbe_hw *hw = &adapter->hw; 4875 int error; 4876 4877 KASSERT(mutex_owned(&adapter->core_mtx)); 4878 4879 IXGBE_EVC_ADD(&adapter->phy_workev, 1); 4880 error = hw->phy.ops.handle_lasi(hw); 4881 if (error == IXGBE_ERR_OVERTEMP) 4882 device_printf(adapter->dev, 4883 "CRITICAL: EXTERNAL PHY OVER TEMP!! " 4884 " PHY will downshift to lower power state!\n"); 4885 else if (error) 4886 device_printf(adapter->dev, 4887 "Error handling LASI interrupt: %d\n", error); 4888 } /* ixgbe_handle_phy */ 4889 4890 static void 4891 ixgbe_handle_admin(struct work *wk, void *context) 4892 { 4893 struct adapter *adapter = context; 4894 struct ifnet *ifp = adapter->ifp; 4895 struct ixgbe_hw *hw = &adapter->hw; 4896 u32 task_requests; 4897 u32 eims_enable = 0; 4898 4899 mutex_enter(&adapter->admin_mtx); 4900 adapter->admin_pending = 0; 4901 task_requests = adapter->task_requests; 4902 adapter->task_requests = 0; 4903 mutex_exit(&adapter->admin_mtx); 4904 4905 /* 4906 * Hold the IFNET_LOCK across this entire call. This will 4907 * prevent additional changes to adapter->phy_layer 4908 * and serialize calls to this tasklet. We cannot hold the 4909 * CORE_LOCK while calling into the ifmedia functions as 4910 * they call ifmedia_lock() and the lock is CORE_LOCK. 4911 */ 4912 IFNET_LOCK(ifp); 4913 IXGBE_CORE_LOCK(adapter); 4914 if ((task_requests & IXGBE_REQUEST_TASK_LSC) != 0) { 4915 ixgbe_handle_link(adapter); 4916 eims_enable |= IXGBE_EIMS_LSC; 4917 } 4918 if ((task_requests & IXGBE_REQUEST_TASK_MOD_WOI) != 0) { 4919 ixgbe_handle_mod(adapter, false); 4920 } 4921 if ((task_requests & IXGBE_REQUEST_TASK_MOD) != 0) { 4922 ixgbe_handle_mod(adapter, true); 4923 if (hw->mac.type >= ixgbe_mac_X540) 4924 eims_enable |= IXGBE_EICR_GPI_SDP0_X540; 4925 else 4926 eims_enable |= IXGBE_EICR_GPI_SDP2_BY_MAC(hw); 4927 } 4928 if ((task_requests 4929 & (IXGBE_REQUEST_TASK_MSF_WOI | IXGBE_REQUEST_TASK_MSF)) != 0) { 4930 ixgbe_handle_msf(adapter); 4931 if (((task_requests & IXGBE_REQUEST_TASK_MSF) != 0) && 4932 (hw->mac.type == ixgbe_mac_82599EB)) 4933 eims_enable |= IXGBE_EIMS_GPI_SDP1_BY_MAC(hw); 4934 } 4935 if ((task_requests & IXGBE_REQUEST_TASK_PHY) != 0) { 4936 ixgbe_handle_phy(adapter); 4937 eims_enable |= IXGBE_EICR_GPI_SDP0_X540; 4938 } 4939 if ((task_requests & IXGBE_REQUEST_TASK_FDIR) != 0) { 4940 ixgbe_reinit_fdir(adapter); 4941 eims_enable |= IXGBE_EIMS_FLOW_DIR; 4942 } 4943 #if 0 /* notyet */ 4944 if ((task_requests & IXGBE_REQUEST_TASK_MBX) != 0) { 4945 ixgbe_handle_mbx(adapter); 4946 eims_enable |= IXGBE_EIMS_MAILBOX; 4947 } 4948 #endif 4949 IXGBE_WRITE_REG(hw, IXGBE_EIMS, eims_enable); 4950 4951 IXGBE_CORE_UNLOCK(adapter); 4952 IFNET_UNLOCK(ifp); 4953 } /* ixgbe_handle_admin */ 4954 4955 static void 4956 ixgbe_ifstop(struct ifnet *ifp, int disable) 4957 { 4958 struct adapter *adapter = ifp->if_softc; 4959 4960 IXGBE_CORE_LOCK(adapter); 4961 ixgbe_stop_locked(adapter); 4962 IXGBE_CORE_UNLOCK(adapter); 4963 4964 workqueue_wait(adapter->timer_wq, &adapter->timer_wc); 4965 atomic_store_relaxed(&adapter->timer_pending, 0); 4966 } 4967 4968 /************************************************************************ 4969 * ixgbe_stop_locked - Stop the hardware 4970 * 4971 * Disables all traffic on the adapter by issuing a 4972 * global reset on the MAC and deallocates TX/RX buffers. 4973 ************************************************************************/ 4974 static void 4975 ixgbe_stop_locked(void *arg) 4976 { 4977 struct ifnet *ifp; 4978 struct adapter *adapter = arg; 4979 struct ixgbe_hw *hw = &adapter->hw; 4980 4981 ifp = adapter->ifp; 4982 4983 KASSERT(mutex_owned(&adapter->core_mtx)); 4984 4985 INIT_DEBUGOUT("ixgbe_stop_locked: begin\n"); 4986 ixgbe_disable_intr(adapter); 4987 callout_stop(&adapter->timer); 4988 4989 /* Don't schedule workqueues. */ 4990 adapter->schedule_wqs_ok = false; 4991 4992 /* Let the stack know...*/ 4993 ifp->if_flags &= ~IFF_RUNNING; 4994 4995 ixgbe_reset_hw(hw); 4996 hw->adapter_stopped = FALSE; 4997 ixgbe_stop_adapter(hw); 4998 if (hw->mac.type == ixgbe_mac_82599EB) 4999 ixgbe_stop_mac_link_on_d3_82599(hw); 5000 /* Turn off the laser - noop with no optics */ 5001 ixgbe_disable_tx_laser(hw); 5002 5003 /* Update the stack */ 5004 adapter->link_up = FALSE; 5005 ixgbe_update_link_status(adapter); 5006 5007 /* reprogram the RAR[0] in case user changed it. */ 5008 ixgbe_set_rar(&adapter->hw, 0, adapter->hw.mac.addr, 0, IXGBE_RAH_AV); 5009 5010 return; 5011 } /* ixgbe_stop_locked */ 5012 5013 /************************************************************************ 5014 * ixgbe_update_link_status - Update OS on link state 5015 * 5016 * Note: Only updates the OS on the cached link state. 5017 * The real check of the hardware only happens with 5018 * a link interrupt. 5019 ************************************************************************/ 5020 static void 5021 ixgbe_update_link_status(struct adapter *adapter) 5022 { 5023 struct ifnet *ifp = adapter->ifp; 5024 device_t dev = adapter->dev; 5025 struct ixgbe_hw *hw = &adapter->hw; 5026 5027 KASSERT(mutex_owned(&adapter->core_mtx)); 5028 5029 if (adapter->link_up) { 5030 if (adapter->link_active != LINK_STATE_UP) { 5031 /* 5032 * To eliminate influence of the previous state 5033 * in the same way as ixgbe_init_locked(). 5034 */ 5035 struct ix_queue *que = adapter->queues; 5036 for (int i = 0; i < adapter->num_queues; i++, que++) 5037 que->eitr_setting = 0; 5038 5039 if (adapter->link_speed == IXGBE_LINK_SPEED_10GB_FULL){ 5040 /* 5041 * Discard count for both MAC Local Fault and 5042 * Remote Fault because those registers are 5043 * valid only when the link speed is up and 5044 * 10Gbps. 5045 */ 5046 IXGBE_READ_REG(hw, IXGBE_MLFC); 5047 IXGBE_READ_REG(hw, IXGBE_MRFC); 5048 } 5049 5050 if (bootverbose) { 5051 const char *bpsmsg; 5052 5053 switch (adapter->link_speed) { 5054 case IXGBE_LINK_SPEED_10GB_FULL: 5055 bpsmsg = "10 Gbps"; 5056 break; 5057 case IXGBE_LINK_SPEED_5GB_FULL: 5058 bpsmsg = "5 Gbps"; 5059 break; 5060 case IXGBE_LINK_SPEED_2_5GB_FULL: 5061 bpsmsg = "2.5 Gbps"; 5062 break; 5063 case IXGBE_LINK_SPEED_1GB_FULL: 5064 bpsmsg = "1 Gbps"; 5065 break; 5066 case IXGBE_LINK_SPEED_100_FULL: 5067 bpsmsg = "100 Mbps"; 5068 break; 5069 case IXGBE_LINK_SPEED_10_FULL: 5070 bpsmsg = "10 Mbps"; 5071 break; 5072 default: 5073 bpsmsg = "unknown speed"; 5074 break; 5075 } 5076 device_printf(dev, "Link is up %s %s \n", 5077 bpsmsg, "Full Duplex"); 5078 } 5079 adapter->link_active = LINK_STATE_UP; 5080 /* Update any Flow Control changes */ 5081 ixgbe_fc_enable(&adapter->hw); 5082 /* Update DMA coalescing config */ 5083 ixgbe_config_dmac(adapter); 5084 if_link_state_change(ifp, LINK_STATE_UP); 5085 5086 if (adapter->feat_en & IXGBE_FEATURE_SRIOV) 5087 ixgbe_ping_all_vfs(adapter); 5088 } 5089 } else { 5090 /* 5091 * Do it when link active changes to DOWN. i.e. 5092 * a) LINK_STATE_UNKNOWN -> LINK_STATE_DOWN 5093 * b) LINK_STATE_UP -> LINK_STATE_DOWN 5094 */ 5095 if (adapter->link_active != LINK_STATE_DOWN) { 5096 if (bootverbose) 5097 device_printf(dev, "Link is Down\n"); 5098 if_link_state_change(ifp, LINK_STATE_DOWN); 5099 adapter->link_active = LINK_STATE_DOWN; 5100 if (adapter->feat_en & IXGBE_FEATURE_SRIOV) 5101 ixgbe_ping_all_vfs(adapter); 5102 ixgbe_drain_all(adapter); 5103 } 5104 } 5105 } /* ixgbe_update_link_status */ 5106 5107 /************************************************************************ 5108 * ixgbe_config_dmac - Configure DMA Coalescing 5109 ************************************************************************/ 5110 static void 5111 ixgbe_config_dmac(struct adapter *adapter) 5112 { 5113 struct ixgbe_hw *hw = &adapter->hw; 5114 struct ixgbe_dmac_config *dcfg = &hw->mac.dmac_config; 5115 5116 if (hw->mac.type < ixgbe_mac_X550 || !hw->mac.ops.dmac_config) 5117 return; 5118 5119 if (dcfg->watchdog_timer ^ adapter->dmac || 5120 dcfg->link_speed ^ adapter->link_speed) { 5121 dcfg->watchdog_timer = adapter->dmac; 5122 dcfg->fcoe_en = false; 5123 dcfg->link_speed = adapter->link_speed; 5124 dcfg->num_tcs = 1; 5125 5126 INIT_DEBUGOUT2("dmac settings: watchdog %d, link speed %d\n", 5127 dcfg->watchdog_timer, dcfg->link_speed); 5128 5129 hw->mac.ops.dmac_config(hw); 5130 } 5131 } /* ixgbe_config_dmac */ 5132 5133 /************************************************************************ 5134 * ixgbe_enable_intr 5135 ************************************************************************/ 5136 static void 5137 ixgbe_enable_intr(struct adapter *adapter) 5138 { 5139 struct ixgbe_hw *hw = &adapter->hw; 5140 struct ix_queue *que = adapter->queues; 5141 u32 mask, fwsm; 5142 5143 mask = (IXGBE_EIMS_ENABLE_MASK & ~IXGBE_EIMS_RTX_QUEUE); 5144 5145 switch (adapter->hw.mac.type) { 5146 case ixgbe_mac_82599EB: 5147 mask |= IXGBE_EIMS_ECC; 5148 /* Temperature sensor on some adapters */ 5149 mask |= IXGBE_EIMS_GPI_SDP0; 5150 /* SFP+ (RX_LOS_N & MOD_ABS_N) */ 5151 mask |= IXGBE_EIMS_GPI_SDP1; 5152 mask |= IXGBE_EIMS_GPI_SDP2; 5153 break; 5154 case ixgbe_mac_X540: 5155 /* Detect if Thermal Sensor is enabled */ 5156 fwsm = IXGBE_READ_REG(hw, IXGBE_FWSM); 5157 if (fwsm & IXGBE_FWSM_TS_ENABLED) 5158 mask |= IXGBE_EIMS_TS; 5159 mask |= IXGBE_EIMS_ECC; 5160 break; 5161 case ixgbe_mac_X550: 5162 /* MAC thermal sensor is automatically enabled */ 5163 mask |= IXGBE_EIMS_TS; 5164 mask |= IXGBE_EIMS_ECC; 5165 break; 5166 case ixgbe_mac_X550EM_x: 5167 case ixgbe_mac_X550EM_a: 5168 /* Some devices use SDP0 for important information */ 5169 if (hw->device_id == IXGBE_DEV_ID_X550EM_X_SFP || 5170 hw->device_id == IXGBE_DEV_ID_X550EM_A_SFP || 5171 hw->device_id == IXGBE_DEV_ID_X550EM_A_SFP_N || 5172 hw->device_id == IXGBE_DEV_ID_X550EM_X_10G_T) 5173 mask |= IXGBE_EIMS_GPI_SDP0_BY_MAC(hw); 5174 if (hw->phy.type == ixgbe_phy_x550em_ext_t) 5175 mask |= IXGBE_EICR_GPI_SDP0_X540; 5176 mask |= IXGBE_EIMS_ECC; 5177 break; 5178 default: 5179 break; 5180 } 5181 5182 /* Enable Fan Failure detection */ 5183 if (adapter->feat_en & IXGBE_FEATURE_FAN_FAIL) 5184 mask |= IXGBE_EIMS_GPI_SDP1; 5185 /* Enable SR-IOV */ 5186 if (adapter->feat_en & IXGBE_FEATURE_SRIOV) 5187 mask |= IXGBE_EIMS_MAILBOX; 5188 /* Enable Flow Director */ 5189 if (adapter->feat_en & IXGBE_FEATURE_FDIR) 5190 mask |= IXGBE_EIMS_FLOW_DIR; 5191 5192 IXGBE_WRITE_REG(hw, IXGBE_EIMS, mask); 5193 5194 /* With MSI-X we use auto clear */ 5195 if (adapter->msix_mem) { 5196 /* 5197 * We use auto clear for RTX_QUEUE only. Don't use other 5198 * interrupts (e.g. link interrupt). BTW, we don't use 5199 * TCP_TIMER interrupt itself. 5200 */ 5201 IXGBE_WRITE_REG(hw, IXGBE_EIAC, IXGBE_EIMS_RTX_QUEUE); 5202 } 5203 5204 /* 5205 * Now enable all queues, this is done separately to 5206 * allow for handling the extended (beyond 32) MSI-X 5207 * vectors that can be used by 82599 5208 */ 5209 for (int i = 0; i < adapter->num_queues; i++, que++) 5210 ixgbe_enable_queue(adapter, que->msix); 5211 5212 IXGBE_WRITE_FLUSH(hw); 5213 5214 } /* ixgbe_enable_intr */ 5215 5216 /************************************************************************ 5217 * ixgbe_disable_intr_internal 5218 ************************************************************************/ 5219 static void 5220 ixgbe_disable_intr_internal(struct adapter *adapter, bool nestok) 5221 { 5222 struct ix_queue *que = adapter->queues; 5223 5224 /* disable interrupts other than queues */ 5225 IXGBE_WRITE_REG(&adapter->hw, IXGBE_EIMC, ~IXGBE_EIMC_RTX_QUEUE); 5226 5227 if (adapter->msix_mem) 5228 IXGBE_WRITE_REG(&adapter->hw, IXGBE_EIAC, 0); 5229 5230 for (int i = 0; i < adapter->num_queues; i++, que++) 5231 ixgbe_disable_queue_internal(adapter, que->msix, nestok); 5232 5233 IXGBE_WRITE_FLUSH(&adapter->hw); 5234 5235 } /* ixgbe_do_disable_intr_internal */ 5236 5237 /************************************************************************ 5238 * ixgbe_disable_intr 5239 ************************************************************************/ 5240 static void 5241 ixgbe_disable_intr(struct adapter *adapter) 5242 { 5243 5244 ixgbe_disable_intr_internal(adapter, true); 5245 } /* ixgbe_disable_intr */ 5246 5247 /************************************************************************ 5248 * ixgbe_ensure_disabled_intr 5249 ************************************************************************/ 5250 void 5251 ixgbe_ensure_disabled_intr(struct adapter *adapter) 5252 { 5253 5254 ixgbe_disable_intr_internal(adapter, false); 5255 } /* ixgbe_ensure_disabled_intr */ 5256 5257 /************************************************************************ 5258 * ixgbe_legacy_irq - Legacy Interrupt Service routine 5259 ************************************************************************/ 5260 static int 5261 ixgbe_legacy_irq(void *arg) 5262 { 5263 struct ix_queue *que = arg; 5264 struct adapter *adapter = que->adapter; 5265 struct ixgbe_hw *hw = &adapter->hw; 5266 struct ifnet *ifp = adapter->ifp; 5267 struct tx_ring *txr = adapter->tx_rings; 5268 u32 eicr; 5269 u32 eims_orig; 5270 u32 eims_enable = 0; 5271 u32 eims_disable = 0; 5272 5273 eims_orig = IXGBE_READ_REG(hw, IXGBE_EIMS); 5274 /* 5275 * Silicon errata #26 on 82598. Disable all interrupts before reading 5276 * EICR. 5277 */ 5278 IXGBE_WRITE_REG(hw, IXGBE_EIMC, IXGBE_IRQ_CLEAR_MASK); 5279 5280 /* Read and clear EICR */ 5281 eicr = IXGBE_READ_REG(hw, IXGBE_EICR); 5282 5283 if (eicr == 0) { 5284 IXGBE_EVC_ADD(&adapter->stats.pf.intzero, 1); 5285 IXGBE_WRITE_REG(hw, IXGBE_EIMS, eims_orig); 5286 return 0; 5287 } 5288 IXGBE_EVC_ADD(&adapter->stats.pf.legint, 1); 5289 5290 /* Queue (0) intr */ 5291 if (((ifp->if_flags & IFF_RUNNING) != 0) && 5292 (eicr & IXGBE_EIMC_RTX_QUEUE) != 0) { 5293 IXGBE_EVC_ADD(&que->irqs, 1); 5294 5295 /* 5296 * The same as ixgbe_msix_que() about 5297 * "que->txrx_use_workqueue". 5298 */ 5299 que->txrx_use_workqueue = adapter->txrx_use_workqueue; 5300 5301 IXGBE_TX_LOCK(txr); 5302 ixgbe_txeof(txr); 5303 #ifdef notyet 5304 if (!ixgbe_ring_empty(ifp, txr->br)) 5305 ixgbe_start_locked(ifp, txr); 5306 #endif 5307 IXGBE_TX_UNLOCK(txr); 5308 5309 IXGBE_EVC_ADD(&que->req, 1); 5310 ixgbe_sched_handle_que(adapter, que); 5311 /* Disable queue 0 interrupt */ 5312 eims_disable |= 1UL << 0; 5313 5314 } else 5315 eims_enable |= IXGBE_EIMC_RTX_QUEUE; 5316 5317 ixgbe_intr_admin_common(adapter, eicr, &eims_disable); 5318 5319 /* Re-enable some interrupts */ 5320 IXGBE_WRITE_REG(hw, IXGBE_EIMS, 5321 (eims_orig & ~eims_disable) | eims_enable); 5322 5323 return 1; 5324 } /* ixgbe_legacy_irq */ 5325 5326 /************************************************************************ 5327 * ixgbe_free_pciintr_resources 5328 ************************************************************************/ 5329 static void 5330 ixgbe_free_pciintr_resources(struct adapter *adapter) 5331 { 5332 struct ix_queue *que = adapter->queues; 5333 int rid; 5334 5335 /* 5336 * Release all msix queue resources: 5337 */ 5338 for (int i = 0; i < adapter->num_queues; i++, que++) { 5339 if (que->res != NULL) { 5340 pci_intr_disestablish(adapter->osdep.pc, 5341 adapter->osdep.ihs[i]); 5342 adapter->osdep.ihs[i] = NULL; 5343 } 5344 } 5345 5346 /* Clean the Legacy or Link interrupt last */ 5347 if (adapter->vector) /* we are doing MSIX */ 5348 rid = adapter->vector; 5349 else 5350 rid = 0; 5351 5352 if (adapter->osdep.ihs[rid] != NULL) { 5353 pci_intr_disestablish(adapter->osdep.pc, 5354 adapter->osdep.ihs[rid]); 5355 adapter->osdep.ihs[rid] = NULL; 5356 } 5357 5358 if (adapter->osdep.intrs != NULL) { 5359 pci_intr_release(adapter->osdep.pc, adapter->osdep.intrs, 5360 adapter->osdep.nintrs); 5361 adapter->osdep.intrs = NULL; 5362 } 5363 } /* ixgbe_free_pciintr_resources */ 5364 5365 /************************************************************************ 5366 * ixgbe_free_pci_resources 5367 ************************************************************************/ 5368 static void 5369 ixgbe_free_pci_resources(struct adapter *adapter) 5370 { 5371 5372 ixgbe_free_pciintr_resources(adapter); 5373 5374 if (adapter->osdep.mem_size != 0) { 5375 bus_space_unmap(adapter->osdep.mem_bus_space_tag, 5376 adapter->osdep.mem_bus_space_handle, 5377 adapter->osdep.mem_size); 5378 } 5379 5380 } /* ixgbe_free_pci_resources */ 5381 5382 /************************************************************************ 5383 * ixgbe_set_sysctl_value 5384 ************************************************************************/ 5385 static void 5386 ixgbe_set_sysctl_value(struct adapter *adapter, const char *name, 5387 const char *description, int *limit, int value) 5388 { 5389 device_t dev = adapter->dev; 5390 struct sysctllog **log; 5391 const struct sysctlnode *rnode, *cnode; 5392 5393 /* 5394 * It's not required to check recovery mode because this function never 5395 * touches hardware. 5396 */ 5397 5398 log = &adapter->sysctllog; 5399 if ((rnode = ixgbe_sysctl_instance(adapter)) == NULL) { 5400 aprint_error_dev(dev, "could not create sysctl root\n"); 5401 return; 5402 } 5403 if (sysctl_createv(log, 0, &rnode, &cnode, 5404 CTLFLAG_READWRITE, CTLTYPE_INT, 5405 name, SYSCTL_DESCR(description), 5406 NULL, 0, limit, 0, CTL_CREATE, CTL_EOL) != 0) 5407 aprint_error_dev(dev, "could not create sysctl\n"); 5408 *limit = value; 5409 } /* ixgbe_set_sysctl_value */ 5410 5411 /************************************************************************ 5412 * ixgbe_sysctl_flowcntl 5413 * 5414 * SYSCTL wrapper around setting Flow Control 5415 ************************************************************************/ 5416 static int 5417 ixgbe_sysctl_flowcntl(SYSCTLFN_ARGS) 5418 { 5419 struct sysctlnode node = *rnode; 5420 struct adapter *adapter = (struct adapter *)node.sysctl_data; 5421 int error, fc; 5422 5423 if (ixgbe_fw_recovery_mode_swflag(adapter)) 5424 return (EPERM); 5425 5426 fc = adapter->hw.fc.current_mode; 5427 node.sysctl_data = &fc; 5428 error = sysctl_lookup(SYSCTLFN_CALL(&node)); 5429 if (error != 0 || newp == NULL) 5430 return error; 5431 5432 /* Don't bother if it's not changed */ 5433 if (fc == adapter->hw.fc.current_mode) 5434 return (0); 5435 5436 return ixgbe_set_flowcntl(adapter, fc); 5437 } /* ixgbe_sysctl_flowcntl */ 5438 5439 /************************************************************************ 5440 * ixgbe_set_flowcntl - Set flow control 5441 * 5442 * Flow control values: 5443 * 0 - off 5444 * 1 - rx pause 5445 * 2 - tx pause 5446 * 3 - full 5447 ************************************************************************/ 5448 static int 5449 ixgbe_set_flowcntl(struct adapter *adapter, int fc) 5450 { 5451 switch (fc) { 5452 case ixgbe_fc_rx_pause: 5453 case ixgbe_fc_tx_pause: 5454 case ixgbe_fc_full: 5455 adapter->hw.fc.requested_mode = fc; 5456 if (adapter->num_queues > 1) 5457 ixgbe_disable_rx_drop(adapter); 5458 break; 5459 case ixgbe_fc_none: 5460 adapter->hw.fc.requested_mode = ixgbe_fc_none; 5461 if (adapter->num_queues > 1) 5462 ixgbe_enable_rx_drop(adapter); 5463 break; 5464 default: 5465 return (EINVAL); 5466 } 5467 5468 #if 0 /* XXX NetBSD */ 5469 /* Don't autoneg if forcing a value */ 5470 adapter->hw.fc.disable_fc_autoneg = TRUE; 5471 #endif 5472 ixgbe_fc_enable(&adapter->hw); 5473 5474 return (0); 5475 } /* ixgbe_set_flowcntl */ 5476 5477 /************************************************************************ 5478 * ixgbe_enable_rx_drop 5479 * 5480 * Enable the hardware to drop packets when the buffer is 5481 * full. This is useful with multiqueue, so that no single 5482 * queue being full stalls the entire RX engine. We only 5483 * enable this when Multiqueue is enabled AND Flow Control 5484 * is disabled. 5485 ************************************************************************/ 5486 static void 5487 ixgbe_enable_rx_drop(struct adapter *adapter) 5488 { 5489 struct ixgbe_hw *hw = &adapter->hw; 5490 struct rx_ring *rxr; 5491 u32 srrctl; 5492 5493 for (int i = 0; i < adapter->num_queues; i++) { 5494 rxr = &adapter->rx_rings[i]; 5495 srrctl = IXGBE_READ_REG(hw, IXGBE_SRRCTL(rxr->me)); 5496 srrctl |= IXGBE_SRRCTL_DROP_EN; 5497 IXGBE_WRITE_REG(hw, IXGBE_SRRCTL(rxr->me), srrctl); 5498 } 5499 5500 /* enable drop for each vf */ 5501 for (int i = 0; i < adapter->num_vfs; i++) { 5502 IXGBE_WRITE_REG(hw, IXGBE_QDE, 5503 (IXGBE_QDE_WRITE | (i << IXGBE_QDE_IDX_SHIFT) | 5504 IXGBE_QDE_ENABLE)); 5505 } 5506 } /* ixgbe_enable_rx_drop */ 5507 5508 /************************************************************************ 5509 * ixgbe_disable_rx_drop 5510 ************************************************************************/ 5511 static void 5512 ixgbe_disable_rx_drop(struct adapter *adapter) 5513 { 5514 struct ixgbe_hw *hw = &adapter->hw; 5515 struct rx_ring *rxr; 5516 u32 srrctl; 5517 5518 for (int i = 0; i < adapter->num_queues; i++) { 5519 rxr = &adapter->rx_rings[i]; 5520 srrctl = IXGBE_READ_REG(hw, IXGBE_SRRCTL(rxr->me)); 5521 srrctl &= ~IXGBE_SRRCTL_DROP_EN; 5522 IXGBE_WRITE_REG(hw, IXGBE_SRRCTL(rxr->me), srrctl); 5523 } 5524 5525 /* disable drop for each vf */ 5526 for (int i = 0; i < adapter->num_vfs; i++) { 5527 IXGBE_WRITE_REG(hw, IXGBE_QDE, 5528 (IXGBE_QDE_WRITE | (i << IXGBE_QDE_IDX_SHIFT))); 5529 } 5530 } /* ixgbe_disable_rx_drop */ 5531 5532 /************************************************************************ 5533 * ixgbe_sysctl_advertise 5534 * 5535 * SYSCTL wrapper around setting advertised speed 5536 ************************************************************************/ 5537 static int 5538 ixgbe_sysctl_advertise(SYSCTLFN_ARGS) 5539 { 5540 struct sysctlnode node = *rnode; 5541 struct adapter *adapter = (struct adapter *)node.sysctl_data; 5542 int error = 0, advertise; 5543 5544 if (ixgbe_fw_recovery_mode_swflag(adapter)) 5545 return (EPERM); 5546 5547 advertise = adapter->advertise; 5548 node.sysctl_data = &advertise; 5549 error = sysctl_lookup(SYSCTLFN_CALL(&node)); 5550 if (error != 0 || newp == NULL) 5551 return error; 5552 5553 return ixgbe_set_advertise(adapter, advertise); 5554 } /* ixgbe_sysctl_advertise */ 5555 5556 /************************************************************************ 5557 * ixgbe_set_advertise - Control advertised link speed 5558 * 5559 * Flags: 5560 * 0x00 - Default (all capable link speed) 5561 * 0x1 - advertise 100 Mb 5562 * 0x2 - advertise 1G 5563 * 0x4 - advertise 10G 5564 * 0x8 - advertise 10 Mb (yes, Mb) 5565 * 0x10 - advertise 2.5G 5566 * 0x20 - advertise 5G 5567 ************************************************************************/ 5568 static int 5569 ixgbe_set_advertise(struct adapter *adapter, int advertise) 5570 { 5571 device_t dev; 5572 struct ixgbe_hw *hw; 5573 ixgbe_link_speed speed = 0; 5574 ixgbe_link_speed link_caps = 0; 5575 s32 err = IXGBE_NOT_IMPLEMENTED; 5576 bool negotiate = FALSE; 5577 5578 /* Checks to validate new value */ 5579 if (adapter->advertise == advertise) /* no change */ 5580 return (0); 5581 5582 dev = adapter->dev; 5583 hw = &adapter->hw; 5584 5585 /* No speed changes for backplane media */ 5586 if (hw->phy.media_type == ixgbe_media_type_backplane) 5587 return (ENODEV); 5588 5589 if (!((hw->phy.media_type == ixgbe_media_type_copper) || 5590 (hw->phy.multispeed_fiber))) { 5591 device_printf(dev, 5592 "Advertised speed can only be set on copper or " 5593 "multispeed fiber media types.\n"); 5594 return (EINVAL); 5595 } 5596 5597 if (advertise < 0x0 || advertise > 0x3f) { 5598 device_printf(dev, "Invalid advertised speed; valid modes are 0x0 through 0x3f\n"); 5599 return (EINVAL); 5600 } 5601 5602 if (hw->mac.ops.get_link_capabilities) { 5603 err = hw->mac.ops.get_link_capabilities(hw, &link_caps, 5604 &negotiate); 5605 if (err != IXGBE_SUCCESS) { 5606 device_printf(dev, "Unable to determine supported advertise speeds\n"); 5607 return (ENODEV); 5608 } 5609 } 5610 5611 /* Set new value and report new advertised mode */ 5612 if (advertise & 0x1) { 5613 if (!(link_caps & IXGBE_LINK_SPEED_100_FULL)) { 5614 device_printf(dev, "Interface does not support 100Mb advertised speed\n"); 5615 return (EINVAL); 5616 } 5617 speed |= IXGBE_LINK_SPEED_100_FULL; 5618 } 5619 if (advertise & 0x2) { 5620 if (!(link_caps & IXGBE_LINK_SPEED_1GB_FULL)) { 5621 device_printf(dev, "Interface does not support 1Gb advertised speed\n"); 5622 return (EINVAL); 5623 } 5624 speed |= IXGBE_LINK_SPEED_1GB_FULL; 5625 } 5626 if (advertise & 0x4) { 5627 if (!(link_caps & IXGBE_LINK_SPEED_10GB_FULL)) { 5628 device_printf(dev, "Interface does not support 10Gb advertised speed\n"); 5629 return (EINVAL); 5630 } 5631 speed |= IXGBE_LINK_SPEED_10GB_FULL; 5632 } 5633 if (advertise & 0x8) { 5634 if (!(link_caps & IXGBE_LINK_SPEED_10_FULL)) { 5635 device_printf(dev, "Interface does not support 10Mb advertised speed\n"); 5636 return (EINVAL); 5637 } 5638 speed |= IXGBE_LINK_SPEED_10_FULL; 5639 } 5640 if (advertise & 0x10) { 5641 if (!(link_caps & IXGBE_LINK_SPEED_2_5GB_FULL)) { 5642 device_printf(dev, "Interface does not support 2.5Gb advertised speed\n"); 5643 return (EINVAL); 5644 } 5645 speed |= IXGBE_LINK_SPEED_2_5GB_FULL; 5646 } 5647 if (advertise & 0x20) { 5648 if (!(link_caps & IXGBE_LINK_SPEED_5GB_FULL)) { 5649 device_printf(dev, "Interface does not support 5Gb advertised speed\n"); 5650 return (EINVAL); 5651 } 5652 speed |= IXGBE_LINK_SPEED_5GB_FULL; 5653 } 5654 if (advertise == 0) 5655 speed = link_caps; /* All capable link speed */ 5656 5657 hw->mac.autotry_restart = TRUE; 5658 hw->mac.ops.setup_link(hw, speed, TRUE); 5659 adapter->advertise = advertise; 5660 5661 return (0); 5662 } /* ixgbe_set_advertise */ 5663 5664 /************************************************************************ 5665 * ixgbe_get_default_advertise - Get default advertised speed settings 5666 * 5667 * Formatted for sysctl usage. 5668 * Flags: 5669 * 0x1 - advertise 100 Mb 5670 * 0x2 - advertise 1G 5671 * 0x4 - advertise 10G 5672 * 0x8 - advertise 10 Mb (yes, Mb) 5673 * 0x10 - advertise 2.5G 5674 * 0x20 - advertise 5G 5675 ************************************************************************/ 5676 static int 5677 ixgbe_get_default_advertise(struct adapter *adapter) 5678 { 5679 struct ixgbe_hw *hw = &adapter->hw; 5680 int speed; 5681 ixgbe_link_speed link_caps = 0; 5682 s32 err; 5683 bool negotiate = FALSE; 5684 5685 /* 5686 * Advertised speed means nothing unless it's copper or 5687 * multi-speed fiber 5688 */ 5689 if (!(hw->phy.media_type == ixgbe_media_type_copper) && 5690 !(hw->phy.multispeed_fiber)) 5691 return (0); 5692 5693 err = hw->mac.ops.get_link_capabilities(hw, &link_caps, &negotiate); 5694 if (err != IXGBE_SUCCESS) 5695 return (0); 5696 5697 speed = 5698 ((link_caps & IXGBE_LINK_SPEED_10GB_FULL) ? 0x4 : 0) | 5699 ((link_caps & IXGBE_LINK_SPEED_5GB_FULL) ? 0x20 : 0) | 5700 ((link_caps & IXGBE_LINK_SPEED_2_5GB_FULL) ? 0x10 : 0) | 5701 ((link_caps & IXGBE_LINK_SPEED_1GB_FULL) ? 0x2 : 0) | 5702 ((link_caps & IXGBE_LINK_SPEED_100_FULL) ? 0x1 : 0) | 5703 ((link_caps & IXGBE_LINK_SPEED_10_FULL) ? 0x8 : 0); 5704 5705 return speed; 5706 } /* ixgbe_get_default_advertise */ 5707 5708 /************************************************************************ 5709 * ixgbe_sysctl_dmac - Manage DMA Coalescing 5710 * 5711 * Control values: 5712 * 0/1 - off / on (use default value of 1000) 5713 * 5714 * Legal timer values are: 5715 * 50,100,250,500,1000,2000,5000,10000 5716 * 5717 * Turning off interrupt moderation will also turn this off. 5718 ************************************************************************/ 5719 static int 5720 ixgbe_sysctl_dmac(SYSCTLFN_ARGS) 5721 { 5722 struct sysctlnode node = *rnode; 5723 struct adapter *adapter = (struct adapter *)node.sysctl_data; 5724 struct ifnet *ifp = adapter->ifp; 5725 int error; 5726 int newval; 5727 5728 if (ixgbe_fw_recovery_mode_swflag(adapter)) 5729 return (EPERM); 5730 5731 newval = adapter->dmac; 5732 node.sysctl_data = &newval; 5733 error = sysctl_lookup(SYSCTLFN_CALL(&node)); 5734 if ((error) || (newp == NULL)) 5735 return (error); 5736 5737 switch (newval) { 5738 case 0: 5739 /* Disabled */ 5740 adapter->dmac = 0; 5741 break; 5742 case 1: 5743 /* Enable and use default */ 5744 adapter->dmac = 1000; 5745 break; 5746 case 50: 5747 case 100: 5748 case 250: 5749 case 500: 5750 case 1000: 5751 case 2000: 5752 case 5000: 5753 case 10000: 5754 /* Legal values - allow */ 5755 adapter->dmac = newval; 5756 break; 5757 default: 5758 /* Do nothing, illegal value */ 5759 return (EINVAL); 5760 } 5761 5762 /* Re-initialize hardware if it's already running */ 5763 if (ifp->if_flags & IFF_RUNNING) 5764 if_init(ifp); 5765 5766 return (0); 5767 } 5768 5769 #ifdef IXGBE_DEBUG 5770 /************************************************************************ 5771 * ixgbe_sysctl_power_state 5772 * 5773 * Sysctl to test power states 5774 * Values: 5775 * 0 - set device to D0 5776 * 3 - set device to D3 5777 * (none) - get current device power state 5778 ************************************************************************/ 5779 static int 5780 ixgbe_sysctl_power_state(SYSCTLFN_ARGS) 5781 { 5782 #ifdef notyet 5783 struct sysctlnode node = *rnode; 5784 struct adapter *adapter = (struct adapter *)node.sysctl_data; 5785 device_t dev = adapter->dev; 5786 int curr_ps, new_ps, error = 0; 5787 5788 if (ixgbe_fw_recovery_mode_swflag(adapter)) 5789 return (EPERM); 5790 5791 curr_ps = new_ps = pci_get_powerstate(dev); 5792 5793 error = sysctl_lookup(SYSCTLFN_CALL(&node)); 5794 if ((error) || (req->newp == NULL)) 5795 return (error); 5796 5797 if (new_ps == curr_ps) 5798 return (0); 5799 5800 if (new_ps == 3 && curr_ps == 0) 5801 error = DEVICE_SUSPEND(dev); 5802 else if (new_ps == 0 && curr_ps == 3) 5803 error = DEVICE_RESUME(dev); 5804 else 5805 return (EINVAL); 5806 5807 device_printf(dev, "New state: %d\n", pci_get_powerstate(dev)); 5808 5809 return (error); 5810 #else 5811 return 0; 5812 #endif 5813 } /* ixgbe_sysctl_power_state */ 5814 #endif 5815 5816 /************************************************************************ 5817 * ixgbe_sysctl_wol_enable 5818 * 5819 * Sysctl to enable/disable the WoL capability, 5820 * if supported by the adapter. 5821 * 5822 * Values: 5823 * 0 - disabled 5824 * 1 - enabled 5825 ************************************************************************/ 5826 static int 5827 ixgbe_sysctl_wol_enable(SYSCTLFN_ARGS) 5828 { 5829 struct sysctlnode node = *rnode; 5830 struct adapter *adapter = (struct adapter *)node.sysctl_data; 5831 struct ixgbe_hw *hw = &adapter->hw; 5832 bool new_wol_enabled; 5833 int error = 0; 5834 5835 /* 5836 * It's not required to check recovery mode because this function never 5837 * touches hardware. 5838 */ 5839 new_wol_enabled = hw->wol_enabled; 5840 node.sysctl_data = &new_wol_enabled; 5841 error = sysctl_lookup(SYSCTLFN_CALL(&node)); 5842 if ((error) || (newp == NULL)) 5843 return (error); 5844 if (new_wol_enabled == hw->wol_enabled) 5845 return (0); 5846 5847 if (new_wol_enabled && !adapter->wol_support) 5848 return (ENODEV); 5849 else 5850 hw->wol_enabled = new_wol_enabled; 5851 5852 return (0); 5853 } /* ixgbe_sysctl_wol_enable */ 5854 5855 /************************************************************************ 5856 * ixgbe_sysctl_wufc - Wake Up Filter Control 5857 * 5858 * Sysctl to enable/disable the types of packets that the 5859 * adapter will wake up on upon receipt. 5860 * Flags: 5861 * 0x1 - Link Status Change 5862 * 0x2 - Magic Packet 5863 * 0x4 - Direct Exact 5864 * 0x8 - Directed Multicast 5865 * 0x10 - Broadcast 5866 * 0x20 - ARP/IPv4 Request Packet 5867 * 0x40 - Direct IPv4 Packet 5868 * 0x80 - Direct IPv6 Packet 5869 * 5870 * Settings not listed above will cause the sysctl to return an error. 5871 ************************************************************************/ 5872 static int 5873 ixgbe_sysctl_wufc(SYSCTLFN_ARGS) 5874 { 5875 struct sysctlnode node = *rnode; 5876 struct adapter *adapter = (struct adapter *)node.sysctl_data; 5877 int error = 0; 5878 u32 new_wufc; 5879 5880 /* 5881 * It's not required to check recovery mode because this function never 5882 * touches hardware. 5883 */ 5884 new_wufc = adapter->wufc; 5885 node.sysctl_data = &new_wufc; 5886 error = sysctl_lookup(SYSCTLFN_CALL(&node)); 5887 if ((error) || (newp == NULL)) 5888 return (error); 5889 if (new_wufc == adapter->wufc) 5890 return (0); 5891 5892 if (new_wufc & 0xffffff00) 5893 return (EINVAL); 5894 5895 new_wufc &= 0xff; 5896 new_wufc |= (0xffffff & adapter->wufc); 5897 adapter->wufc = new_wufc; 5898 5899 return (0); 5900 } /* ixgbe_sysctl_wufc */ 5901 5902 #ifdef IXGBE_DEBUG 5903 /************************************************************************ 5904 * ixgbe_sysctl_print_rss_config 5905 ************************************************************************/ 5906 static int 5907 ixgbe_sysctl_print_rss_config(SYSCTLFN_ARGS) 5908 { 5909 #ifdef notyet 5910 struct sysctlnode node = *rnode; 5911 struct adapter *adapter = (struct adapter *)node.sysctl_data; 5912 struct ixgbe_hw *hw = &adapter->hw; 5913 device_t dev = adapter->dev; 5914 struct sbuf *buf; 5915 int error = 0, reta_size; 5916 u32 reg; 5917 5918 if (ixgbe_fw_recovery_mode_swflag(adapter)) 5919 return (EPERM); 5920 5921 buf = sbuf_new_for_sysctl(NULL, NULL, 128, req); 5922 if (!buf) { 5923 device_printf(dev, "Could not allocate sbuf for output.\n"); 5924 return (ENOMEM); 5925 } 5926 5927 // TODO: use sbufs to make a string to print out 5928 /* Set multiplier for RETA setup and table size based on MAC */ 5929 switch (adapter->hw.mac.type) { 5930 case ixgbe_mac_X550: 5931 case ixgbe_mac_X550EM_x: 5932 case ixgbe_mac_X550EM_a: 5933 reta_size = 128; 5934 break; 5935 default: 5936 reta_size = 32; 5937 break; 5938 } 5939 5940 /* Print out the redirection table */ 5941 sbuf_cat(buf, "\n"); 5942 for (int i = 0; i < reta_size; i++) { 5943 if (i < 32) { 5944 reg = IXGBE_READ_REG(hw, IXGBE_RETA(i)); 5945 sbuf_printf(buf, "RETA(%2d): 0x%08x\n", i, reg); 5946 } else { 5947 reg = IXGBE_READ_REG(hw, IXGBE_ERETA(i - 32)); 5948 sbuf_printf(buf, "ERETA(%2d): 0x%08x\n", i - 32, reg); 5949 } 5950 } 5951 5952 // TODO: print more config 5953 5954 error = sbuf_finish(buf); 5955 if (error) 5956 device_printf(dev, "Error finishing sbuf: %d\n", error); 5957 5958 sbuf_delete(buf); 5959 #endif 5960 return (0); 5961 } /* ixgbe_sysctl_print_rss_config */ 5962 #endif /* IXGBE_DEBUG */ 5963 5964 /************************************************************************ 5965 * ixgbe_sysctl_phy_temp - Retrieve temperature of PHY 5966 * 5967 * For X552/X557-AT devices using an external PHY 5968 ************************************************************************/ 5969 static int 5970 ixgbe_sysctl_phy_temp(SYSCTLFN_ARGS) 5971 { 5972 struct sysctlnode node = *rnode; 5973 struct adapter *adapter = (struct adapter *)node.sysctl_data; 5974 struct ixgbe_hw *hw = &adapter->hw; 5975 int val; 5976 u16 reg; 5977 int error; 5978 5979 if (ixgbe_fw_recovery_mode_swflag(adapter)) 5980 return (EPERM); 5981 5982 if (hw->device_id != IXGBE_DEV_ID_X550EM_X_10G_T) { 5983 device_printf(adapter->dev, 5984 "Device has no supported external thermal sensor.\n"); 5985 return (ENODEV); 5986 } 5987 5988 if (hw->phy.ops.read_reg(hw, IXGBE_PHY_CURRENT_TEMP, 5989 IXGBE_MDIO_VENDOR_SPECIFIC_1_DEV_TYPE, ®)) { 5990 device_printf(adapter->dev, 5991 "Error reading from PHY's current temperature register\n"); 5992 return (EAGAIN); 5993 } 5994 5995 node.sysctl_data = &val; 5996 5997 /* Shift temp for output */ 5998 val = reg >> 8; 5999 6000 error = sysctl_lookup(SYSCTLFN_CALL(&node)); 6001 if ((error) || (newp == NULL)) 6002 return (error); 6003 6004 return (0); 6005 } /* ixgbe_sysctl_phy_temp */ 6006 6007 /************************************************************************ 6008 * ixgbe_sysctl_phy_overtemp_occurred 6009 * 6010 * Reports (directly from the PHY) whether the current PHY 6011 * temperature is over the overtemp threshold. 6012 ************************************************************************/ 6013 static int 6014 ixgbe_sysctl_phy_overtemp_occurred(SYSCTLFN_ARGS) 6015 { 6016 struct sysctlnode node = *rnode; 6017 struct adapter *adapter = (struct adapter *)node.sysctl_data; 6018 struct ixgbe_hw *hw = &adapter->hw; 6019 int val, error; 6020 u16 reg; 6021 6022 if (ixgbe_fw_recovery_mode_swflag(adapter)) 6023 return (EPERM); 6024 6025 if (hw->device_id != IXGBE_DEV_ID_X550EM_X_10G_T) { 6026 device_printf(adapter->dev, 6027 "Device has no supported external thermal sensor.\n"); 6028 return (ENODEV); 6029 } 6030 6031 if (hw->phy.ops.read_reg(hw, IXGBE_PHY_OVERTEMP_STATUS, 6032 IXGBE_MDIO_VENDOR_SPECIFIC_1_DEV_TYPE, ®)) { 6033 device_printf(adapter->dev, 6034 "Error reading from PHY's temperature status register\n"); 6035 return (EAGAIN); 6036 } 6037 6038 node.sysctl_data = &val; 6039 6040 /* Get occurrence bit */ 6041 val = !!(reg & 0x4000); 6042 6043 error = sysctl_lookup(SYSCTLFN_CALL(&node)); 6044 if ((error) || (newp == NULL)) 6045 return (error); 6046 6047 return (0); 6048 } /* ixgbe_sysctl_phy_overtemp_occurred */ 6049 6050 /************************************************************************ 6051 * ixgbe_sysctl_eee_state 6052 * 6053 * Sysctl to set EEE power saving feature 6054 * Values: 6055 * 0 - disable EEE 6056 * 1 - enable EEE 6057 * (none) - get current device EEE state 6058 ************************************************************************/ 6059 static int 6060 ixgbe_sysctl_eee_state(SYSCTLFN_ARGS) 6061 { 6062 struct sysctlnode node = *rnode; 6063 struct adapter *adapter = (struct adapter *)node.sysctl_data; 6064 struct ifnet *ifp = adapter->ifp; 6065 device_t dev = adapter->dev; 6066 int curr_eee, new_eee, error = 0; 6067 s32 retval; 6068 6069 if (ixgbe_fw_recovery_mode_swflag(adapter)) 6070 return (EPERM); 6071 6072 curr_eee = new_eee = !!(adapter->feat_en & IXGBE_FEATURE_EEE); 6073 node.sysctl_data = &new_eee; 6074 error = sysctl_lookup(SYSCTLFN_CALL(&node)); 6075 if ((error) || (newp == NULL)) 6076 return (error); 6077 6078 /* Nothing to do */ 6079 if (new_eee == curr_eee) 6080 return (0); 6081 6082 /* Not supported */ 6083 if (!(adapter->feat_cap & IXGBE_FEATURE_EEE)) 6084 return (EINVAL); 6085 6086 /* Bounds checking */ 6087 if ((new_eee < 0) || (new_eee > 1)) 6088 return (EINVAL); 6089 6090 retval = ixgbe_setup_eee(&adapter->hw, new_eee); 6091 if (retval) { 6092 device_printf(dev, "Error in EEE setup: 0x%08X\n", retval); 6093 return (EINVAL); 6094 } 6095 6096 /* Restart auto-neg */ 6097 if_init(ifp); 6098 6099 device_printf(dev, "New EEE state: %d\n", new_eee); 6100 6101 /* Cache new value */ 6102 if (new_eee) 6103 adapter->feat_en |= IXGBE_FEATURE_EEE; 6104 else 6105 adapter->feat_en &= ~IXGBE_FEATURE_EEE; 6106 6107 return (error); 6108 } /* ixgbe_sysctl_eee_state */ 6109 6110 #define PRINTQS(adapter, regname) \ 6111 do { \ 6112 struct ixgbe_hw *_hw = &(adapter)->hw; \ 6113 int _i; \ 6114 \ 6115 printf("%s: %s", device_xname((adapter)->dev), #regname); \ 6116 for (_i = 0; _i < (adapter)->num_queues; _i++) { \ 6117 printf((_i == 0) ? "\t" : " "); \ 6118 printf("%08x", IXGBE_READ_REG(_hw, \ 6119 IXGBE_##regname(_i))); \ 6120 } \ 6121 printf("\n"); \ 6122 } while (0) 6123 6124 /************************************************************************ 6125 * ixgbe_print_debug_info 6126 * 6127 * Called only when em_display_debug_stats is enabled. 6128 * Provides a way to take a look at important statistics 6129 * maintained by the driver and hardware. 6130 ************************************************************************/ 6131 static void 6132 ixgbe_print_debug_info(struct adapter *adapter) 6133 { 6134 device_t dev = adapter->dev; 6135 struct ixgbe_hw *hw = &adapter->hw; 6136 int table_size; 6137 int i; 6138 6139 switch (adapter->hw.mac.type) { 6140 case ixgbe_mac_X550: 6141 case ixgbe_mac_X550EM_x: 6142 case ixgbe_mac_X550EM_a: 6143 table_size = 128; 6144 break; 6145 default: 6146 table_size = 32; 6147 break; 6148 } 6149 6150 device_printf(dev, "[E]RETA:\n"); 6151 for (i = 0; i < table_size; i++) { 6152 if (i < 32) 6153 printf("%02x: %08x\n", i, IXGBE_READ_REG(hw, 6154 IXGBE_RETA(i))); 6155 else 6156 printf("%02x: %08x\n", i, IXGBE_READ_REG(hw, 6157 IXGBE_ERETA(i - 32))); 6158 } 6159 6160 device_printf(dev, "queue:"); 6161 for (i = 0; i < adapter->num_queues; i++) { 6162 printf((i == 0) ? "\t" : " "); 6163 printf("%8d", i); 6164 } 6165 printf("\n"); 6166 PRINTQS(adapter, RDBAL); 6167 PRINTQS(adapter, RDBAH); 6168 PRINTQS(adapter, RDLEN); 6169 PRINTQS(adapter, SRRCTL); 6170 PRINTQS(adapter, RDH); 6171 PRINTQS(adapter, RDT); 6172 PRINTQS(adapter, RXDCTL); 6173 6174 device_printf(dev, "RQSMR:"); 6175 for (i = 0; i < adapter->num_queues / 4; i++) { 6176 printf((i == 0) ? "\t" : " "); 6177 printf("%08x", IXGBE_READ_REG(hw, IXGBE_RQSMR(i))); 6178 } 6179 printf("\n"); 6180 6181 device_printf(dev, "disabled_count:"); 6182 for (i = 0; i < adapter->num_queues; i++) { 6183 printf((i == 0) ? "\t" : " "); 6184 printf("%8d", adapter->queues[i].disabled_count); 6185 } 6186 printf("\n"); 6187 6188 device_printf(dev, "EIMS:\t%08x\n", IXGBE_READ_REG(hw, IXGBE_EIMS)); 6189 if (hw->mac.type != ixgbe_mac_82598EB) { 6190 device_printf(dev, "EIMS_EX(0):\t%08x\n", 6191 IXGBE_READ_REG(hw, IXGBE_EIMS_EX(0))); 6192 device_printf(dev, "EIMS_EX(1):\t%08x\n", 6193 IXGBE_READ_REG(hw, IXGBE_EIMS_EX(1))); 6194 } 6195 device_printf(dev, "EIAM:\t%08x\n", IXGBE_READ_REG(hw, IXGBE_EIAM)); 6196 device_printf(dev, "EIAC:\t%08x\n", IXGBE_READ_REG(hw, IXGBE_EIAC)); 6197 } /* ixgbe_print_debug_info */ 6198 6199 /************************************************************************ 6200 * ixgbe_sysctl_debug 6201 ************************************************************************/ 6202 static int 6203 ixgbe_sysctl_debug(SYSCTLFN_ARGS) 6204 { 6205 struct sysctlnode node = *rnode; 6206 struct adapter *adapter = (struct adapter *)node.sysctl_data; 6207 int error, result = 0; 6208 6209 if (ixgbe_fw_recovery_mode_swflag(adapter)) 6210 return (EPERM); 6211 6212 node.sysctl_data = &result; 6213 error = sysctl_lookup(SYSCTLFN_CALL(&node)); 6214 6215 if (error || newp == NULL) 6216 return error; 6217 6218 if (result == 1) 6219 ixgbe_print_debug_info(adapter); 6220 6221 return 0; 6222 } /* ixgbe_sysctl_debug */ 6223 6224 /************************************************************************ 6225 * ixgbe_sysctl_rx_copy_len 6226 ************************************************************************/ 6227 static int 6228 ixgbe_sysctl_rx_copy_len(SYSCTLFN_ARGS) 6229 { 6230 struct sysctlnode node = *rnode; 6231 struct adapter *adapter = (struct adapter *)node.sysctl_data; 6232 int error; 6233 int result = adapter->rx_copy_len; 6234 6235 node.sysctl_data = &result; 6236 error = sysctl_lookup(SYSCTLFN_CALL(&node)); 6237 6238 if (error || newp == NULL) 6239 return error; 6240 6241 if ((result < 0) || (result > IXGBE_RX_COPY_LEN_MAX)) 6242 return EINVAL; 6243 6244 adapter->rx_copy_len = result; 6245 6246 return 0; 6247 } /* ixgbe_sysctl_rx_copy_len */ 6248 6249 /************************************************************************ 6250 * ixgbe_init_device_features 6251 ************************************************************************/ 6252 static void 6253 ixgbe_init_device_features(struct adapter *adapter) 6254 { 6255 adapter->feat_cap = IXGBE_FEATURE_NETMAP 6256 | IXGBE_FEATURE_RSS 6257 | IXGBE_FEATURE_MSI 6258 | IXGBE_FEATURE_MSIX 6259 | IXGBE_FEATURE_LEGACY_IRQ 6260 | IXGBE_FEATURE_LEGACY_TX; 6261 6262 /* Set capabilities first... */ 6263 switch (adapter->hw.mac.type) { 6264 case ixgbe_mac_82598EB: 6265 if (adapter->hw.device_id == IXGBE_DEV_ID_82598AT) 6266 adapter->feat_cap |= IXGBE_FEATURE_FAN_FAIL; 6267 break; 6268 case ixgbe_mac_X540: 6269 adapter->feat_cap |= IXGBE_FEATURE_SRIOV; 6270 adapter->feat_cap |= IXGBE_FEATURE_FDIR; 6271 if ((adapter->hw.device_id == IXGBE_DEV_ID_X540_BYPASS) && 6272 (adapter->hw.bus.func == 0)) 6273 adapter->feat_cap |= IXGBE_FEATURE_BYPASS; 6274 break; 6275 case ixgbe_mac_X550: 6276 /* 6277 * IXGBE_FEATURE_RECOVERY_MODE will be set after reading 6278 * NVM Image version. 6279 */ 6280 adapter->feat_cap |= IXGBE_FEATURE_TEMP_SENSOR; 6281 adapter->feat_cap |= IXGBE_FEATURE_SRIOV; 6282 adapter->feat_cap |= IXGBE_FEATURE_FDIR; 6283 break; 6284 case ixgbe_mac_X550EM_x: 6285 /* 6286 * IXGBE_FEATURE_RECOVERY_MODE will be set after reading 6287 * NVM Image version. 6288 */ 6289 adapter->feat_cap |= IXGBE_FEATURE_SRIOV; 6290 adapter->feat_cap |= IXGBE_FEATURE_FDIR; 6291 break; 6292 case ixgbe_mac_X550EM_a: 6293 /* 6294 * IXGBE_FEATURE_RECOVERY_MODE will be set after reading 6295 * NVM Image version. 6296 */ 6297 adapter->feat_cap |= IXGBE_FEATURE_SRIOV; 6298 adapter->feat_cap |= IXGBE_FEATURE_FDIR; 6299 adapter->feat_cap &= ~IXGBE_FEATURE_LEGACY_IRQ; 6300 if ((adapter->hw.device_id == IXGBE_DEV_ID_X550EM_A_1G_T) || 6301 (adapter->hw.device_id == IXGBE_DEV_ID_X550EM_A_1G_T_L)) { 6302 adapter->feat_cap |= IXGBE_FEATURE_TEMP_SENSOR; 6303 adapter->feat_cap |= IXGBE_FEATURE_EEE; 6304 } 6305 break; 6306 case ixgbe_mac_82599EB: 6307 adapter->feat_cap |= IXGBE_FEATURE_SRIOV; 6308 adapter->feat_cap |= IXGBE_FEATURE_FDIR; 6309 if ((adapter->hw.device_id == IXGBE_DEV_ID_82599_BYPASS) && 6310 (adapter->hw.bus.func == 0)) 6311 adapter->feat_cap |= IXGBE_FEATURE_BYPASS; 6312 if (adapter->hw.device_id == IXGBE_DEV_ID_82599_QSFP_SF_QP) 6313 adapter->feat_cap &= ~IXGBE_FEATURE_LEGACY_IRQ; 6314 break; 6315 default: 6316 break; 6317 } 6318 6319 /* Enabled by default... */ 6320 /* Fan failure detection */ 6321 if (adapter->feat_cap & IXGBE_FEATURE_FAN_FAIL) 6322 adapter->feat_en |= IXGBE_FEATURE_FAN_FAIL; 6323 /* Netmap */ 6324 if (adapter->feat_cap & IXGBE_FEATURE_NETMAP) 6325 adapter->feat_en |= IXGBE_FEATURE_NETMAP; 6326 /* EEE */ 6327 if (adapter->feat_cap & IXGBE_FEATURE_EEE) 6328 adapter->feat_en |= IXGBE_FEATURE_EEE; 6329 /* Thermal Sensor */ 6330 if (adapter->feat_cap & IXGBE_FEATURE_TEMP_SENSOR) 6331 adapter->feat_en |= IXGBE_FEATURE_TEMP_SENSOR; 6332 /* 6333 * Recovery mode: 6334 * NetBSD: IXGBE_FEATURE_RECOVERY_MODE will be controlled after reading 6335 * NVM Image version. 6336 */ 6337 6338 /* Enabled via global sysctl... */ 6339 /* Flow Director */ 6340 if (ixgbe_enable_fdir) { 6341 if (adapter->feat_cap & IXGBE_FEATURE_FDIR) 6342 adapter->feat_en |= IXGBE_FEATURE_FDIR; 6343 else 6344 device_printf(adapter->dev, "Device does not support Flow Director. Leaving disabled."); 6345 } 6346 /* Legacy (single queue) transmit */ 6347 if ((adapter->feat_cap & IXGBE_FEATURE_LEGACY_TX) && 6348 ixgbe_enable_legacy_tx) 6349 adapter->feat_en |= IXGBE_FEATURE_LEGACY_TX; 6350 /* 6351 * Message Signal Interrupts - Extended (MSI-X) 6352 * Normal MSI is only enabled if MSI-X calls fail. 6353 */ 6354 if (!ixgbe_enable_msix) 6355 adapter->feat_cap &= ~IXGBE_FEATURE_MSIX; 6356 /* Receive-Side Scaling (RSS) */ 6357 if ((adapter->feat_cap & IXGBE_FEATURE_RSS) && ixgbe_enable_rss) 6358 adapter->feat_en |= IXGBE_FEATURE_RSS; 6359 6360 /* Disable features with unmet dependencies... */ 6361 /* No MSI-X */ 6362 if (!(adapter->feat_cap & IXGBE_FEATURE_MSIX)) { 6363 adapter->feat_cap &= ~IXGBE_FEATURE_RSS; 6364 adapter->feat_cap &= ~IXGBE_FEATURE_SRIOV; 6365 adapter->feat_en &= ~IXGBE_FEATURE_RSS; 6366 adapter->feat_en &= ~IXGBE_FEATURE_SRIOV; 6367 } 6368 } /* ixgbe_init_device_features */ 6369 6370 /************************************************************************ 6371 * ixgbe_probe - Device identification routine 6372 * 6373 * Determines if the driver should be loaded on 6374 * adapter based on its PCI vendor/device ID. 6375 * 6376 * return BUS_PROBE_DEFAULT on success, positive on failure 6377 ************************************************************************/ 6378 static int 6379 ixgbe_probe(device_t dev, cfdata_t cf, void *aux) 6380 { 6381 const struct pci_attach_args *pa = aux; 6382 6383 return (ixgbe_lookup(pa) != NULL) ? 1 : 0; 6384 } 6385 6386 static const ixgbe_vendor_info_t * 6387 ixgbe_lookup(const struct pci_attach_args *pa) 6388 { 6389 const ixgbe_vendor_info_t *ent; 6390 pcireg_t subid; 6391 6392 INIT_DEBUGOUT("ixgbe_lookup: begin"); 6393 6394 if (PCI_VENDOR(pa->pa_id) != IXGBE_INTEL_VENDOR_ID) 6395 return NULL; 6396 6397 subid = pci_conf_read(pa->pa_pc, pa->pa_tag, PCI_SUBSYS_ID_REG); 6398 6399 for (ent = ixgbe_vendor_info_array; ent->vendor_id != 0; ent++) { 6400 if ((PCI_VENDOR(pa->pa_id) == ent->vendor_id) && 6401 (PCI_PRODUCT(pa->pa_id) == ent->device_id) && 6402 ((PCI_SUBSYS_VENDOR(subid) == ent->subvendor_id) || 6403 (ent->subvendor_id == 0)) && 6404 ((PCI_SUBSYS_ID(subid) == ent->subdevice_id) || 6405 (ent->subdevice_id == 0))) { 6406 return ent; 6407 } 6408 } 6409 return NULL; 6410 } 6411 6412 static int 6413 ixgbe_ifflags_cb(struct ethercom *ec) 6414 { 6415 struct ifnet *ifp = &ec->ec_if; 6416 struct adapter *adapter = ifp->if_softc; 6417 u_short change; 6418 int rv = 0; 6419 6420 IXGBE_CORE_LOCK(adapter); 6421 6422 change = ifp->if_flags ^ adapter->if_flags; 6423 if (change != 0) 6424 adapter->if_flags = ifp->if_flags; 6425 6426 if ((change & ~(IFF_CANTCHANGE | IFF_DEBUG)) != 0) { 6427 rv = ENETRESET; 6428 goto out; 6429 } else if ((change & IFF_PROMISC) != 0) 6430 ixgbe_set_rxfilter(adapter); 6431 6432 /* Check for ec_capenable. */ 6433 change = ec->ec_capenable ^ adapter->ec_capenable; 6434 adapter->ec_capenable = ec->ec_capenable; 6435 if ((change & ~(ETHERCAP_VLAN_MTU | ETHERCAP_VLAN_HWTAGGING 6436 | ETHERCAP_VLAN_HWFILTER)) != 0) { 6437 rv = ENETRESET; 6438 goto out; 6439 } 6440 6441 /* 6442 * Special handling is not required for ETHERCAP_VLAN_MTU. 6443 * MAXFRS(MHADD) does not include the 4bytes of the VLAN header. 6444 */ 6445 6446 /* Set up VLAN support and filter */ 6447 if ((change & (ETHERCAP_VLAN_HWTAGGING | ETHERCAP_VLAN_HWFILTER)) != 0) 6448 ixgbe_setup_vlan_hw_support(adapter); 6449 6450 out: 6451 IXGBE_CORE_UNLOCK(adapter); 6452 6453 return rv; 6454 } 6455 6456 /************************************************************************ 6457 * ixgbe_ioctl - Ioctl entry point 6458 * 6459 * Called when the user wants to configure the interface. 6460 * 6461 * return 0 on success, positive on failure 6462 ************************************************************************/ 6463 static int 6464 ixgbe_ioctl(struct ifnet *ifp, u_long command, void *data) 6465 { 6466 struct adapter *adapter = ifp->if_softc; 6467 struct ixgbe_hw *hw = &adapter->hw; 6468 struct ifcapreq *ifcr = data; 6469 struct ifreq *ifr = data; 6470 int error = 0; 6471 int l4csum_en; 6472 const int l4csum = IFCAP_CSUM_TCPv4_Rx | IFCAP_CSUM_UDPv4_Rx | 6473 IFCAP_CSUM_TCPv6_Rx | IFCAP_CSUM_UDPv6_Rx; 6474 6475 if (ixgbe_fw_recovery_mode_swflag(adapter)) 6476 return (EPERM); 6477 6478 switch (command) { 6479 case SIOCSIFFLAGS: 6480 IOCTL_DEBUGOUT("ioctl: SIOCSIFFLAGS (Set Interface Flags)"); 6481 break; 6482 case SIOCADDMULTI: 6483 case SIOCDELMULTI: 6484 IOCTL_DEBUGOUT("ioctl: SIOC(ADD|DEL)MULTI"); 6485 break; 6486 case SIOCSIFMEDIA: 6487 case SIOCGIFMEDIA: 6488 IOCTL_DEBUGOUT("ioctl: SIOCxIFMEDIA (Get/Set Interface Media)"); 6489 break; 6490 case SIOCSIFCAP: 6491 IOCTL_DEBUGOUT("ioctl: SIOCSIFCAP (Set Capabilities)"); 6492 break; 6493 case SIOCSIFMTU: 6494 IOCTL_DEBUGOUT("ioctl: SIOCSIFMTU (Set Interface MTU)"); 6495 break; 6496 #ifdef __NetBSD__ 6497 case SIOCINITIFADDR: 6498 IOCTL_DEBUGOUT("ioctl: SIOCINITIFADDR"); 6499 break; 6500 case SIOCGIFFLAGS: 6501 IOCTL_DEBUGOUT("ioctl: SIOCGIFFLAGS"); 6502 break; 6503 case SIOCGIFAFLAG_IN: 6504 IOCTL_DEBUGOUT("ioctl: SIOCGIFAFLAG_IN"); 6505 break; 6506 case SIOCGIFADDR: 6507 IOCTL_DEBUGOUT("ioctl: SIOCGIFADDR"); 6508 break; 6509 case SIOCGIFMTU: 6510 IOCTL_DEBUGOUT("ioctl: SIOCGIFMTU (Get Interface MTU)"); 6511 break; 6512 case SIOCGIFCAP: 6513 IOCTL_DEBUGOUT("ioctl: SIOCGIFCAP (Get IF cap)"); 6514 break; 6515 case SIOCGETHERCAP: 6516 IOCTL_DEBUGOUT("ioctl: SIOCGETHERCAP (Get ethercap)"); 6517 break; 6518 case SIOCGLIFADDR: 6519 IOCTL_DEBUGOUT("ioctl: SIOCGLIFADDR (Get Interface addr)"); 6520 break; 6521 case SIOCZIFDATA: 6522 IOCTL_DEBUGOUT("ioctl: SIOCZIFDATA (Zero counter)"); 6523 hw->mac.ops.clear_hw_cntrs(hw); 6524 ixgbe_clear_evcnt(adapter); 6525 break; 6526 case SIOCAIFADDR: 6527 IOCTL_DEBUGOUT("ioctl: SIOCAIFADDR (add/chg IF alias)"); 6528 break; 6529 #endif 6530 default: 6531 IOCTL_DEBUGOUT1("ioctl: UNKNOWN (0x%X)", (int)command); 6532 break; 6533 } 6534 6535 switch (command) { 6536 case SIOCGI2C: 6537 { 6538 struct ixgbe_i2c_req i2c; 6539 6540 IOCTL_DEBUGOUT("ioctl: SIOCGI2C (Get I2C Data)"); 6541 error = copyin(ifr->ifr_data, &i2c, sizeof(i2c)); 6542 if (error != 0) 6543 break; 6544 if (i2c.dev_addr != 0xA0 && i2c.dev_addr != 0xA2) { 6545 error = EINVAL; 6546 break; 6547 } 6548 if (i2c.len > sizeof(i2c.data)) { 6549 error = EINVAL; 6550 break; 6551 } 6552 6553 hw->phy.ops.read_i2c_byte(hw, i2c.offset, 6554 i2c.dev_addr, i2c.data); 6555 error = copyout(&i2c, ifr->ifr_data, sizeof(i2c)); 6556 break; 6557 } 6558 case SIOCSIFCAP: 6559 /* Layer-4 Rx checksum offload has to be turned on and 6560 * off as a unit. 6561 */ 6562 l4csum_en = ifcr->ifcr_capenable & l4csum; 6563 if (l4csum_en != l4csum && l4csum_en != 0) 6564 return EINVAL; 6565 /*FALLTHROUGH*/ 6566 case SIOCADDMULTI: 6567 case SIOCDELMULTI: 6568 case SIOCSIFFLAGS: 6569 case SIOCSIFMTU: 6570 default: 6571 if ((error = ether_ioctl(ifp, command, data)) != ENETRESET) 6572 return error; 6573 if ((ifp->if_flags & IFF_RUNNING) == 0) 6574 ; 6575 else if (command == SIOCSIFCAP || command == SIOCSIFMTU) { 6576 IXGBE_CORE_LOCK(adapter); 6577 if ((ifp->if_flags & IFF_RUNNING) != 0) 6578 ixgbe_init_locked(adapter); 6579 ixgbe_recalculate_max_frame(adapter); 6580 IXGBE_CORE_UNLOCK(adapter); 6581 } else if (command == SIOCADDMULTI || command == SIOCDELMULTI) { 6582 /* 6583 * Multicast list has changed; set the hardware filter 6584 * accordingly. 6585 */ 6586 IXGBE_CORE_LOCK(adapter); 6587 ixgbe_disable_intr(adapter); 6588 ixgbe_set_rxfilter(adapter); 6589 ixgbe_enable_intr(adapter); 6590 IXGBE_CORE_UNLOCK(adapter); 6591 } 6592 return 0; 6593 } 6594 6595 return error; 6596 } /* ixgbe_ioctl */ 6597 6598 /************************************************************************ 6599 * ixgbe_check_fan_failure 6600 ************************************************************************/ 6601 static int 6602 ixgbe_check_fan_failure(struct adapter *adapter, u32 reg, bool in_interrupt) 6603 { 6604 u32 mask; 6605 6606 mask = (in_interrupt) ? IXGBE_EICR_GPI_SDP1_BY_MAC(&adapter->hw) : 6607 IXGBE_ESDP_SDP1; 6608 6609 if (reg & mask) { 6610 device_printf(adapter->dev, 6611 "\nCRITICAL: FAN FAILURE!! REPLACE IMMEDIATELY!!\n"); 6612 return IXGBE_ERR_FAN_FAILURE; 6613 } 6614 6615 return IXGBE_SUCCESS; 6616 } /* ixgbe_check_fan_failure */ 6617 6618 /************************************************************************ 6619 * ixgbe_handle_que 6620 ************************************************************************/ 6621 static void 6622 ixgbe_handle_que(void *context) 6623 { 6624 struct ix_queue *que = context; 6625 struct adapter *adapter = que->adapter; 6626 struct tx_ring *txr = que->txr; 6627 struct ifnet *ifp = adapter->ifp; 6628 bool more = false; 6629 6630 IXGBE_EVC_ADD(&que->handleq, 1); 6631 6632 if (ifp->if_flags & IFF_RUNNING) { 6633 more = ixgbe_rxeof(que); 6634 IXGBE_TX_LOCK(txr); 6635 more |= ixgbe_txeof(txr); 6636 if (!(adapter->feat_en & IXGBE_FEATURE_LEGACY_TX)) 6637 if (!ixgbe_mq_ring_empty(ifp, txr->txr_interq)) 6638 ixgbe_mq_start_locked(ifp, txr); 6639 /* Only for queue 0 */ 6640 /* NetBSD still needs this for CBQ */ 6641 if ((&adapter->queues[0] == que) 6642 && (!ixgbe_legacy_ring_empty(ifp, NULL))) 6643 ixgbe_legacy_start_locked(ifp, txr); 6644 IXGBE_TX_UNLOCK(txr); 6645 } 6646 6647 if (more) { 6648 IXGBE_EVC_ADD(&que->req, 1); 6649 ixgbe_sched_handle_que(adapter, que); 6650 } else if (que->res != NULL) { 6651 /* MSIX: Re-enable this interrupt */ 6652 ixgbe_enable_queue(adapter, que->msix); 6653 } else { 6654 /* INTx or MSI */ 6655 ixgbe_enable_queue(adapter, 0); 6656 } 6657 6658 return; 6659 } /* ixgbe_handle_que */ 6660 6661 /************************************************************************ 6662 * ixgbe_handle_que_work 6663 ************************************************************************/ 6664 static void 6665 ixgbe_handle_que_work(struct work *wk, void *context) 6666 { 6667 struct ix_queue *que = container_of(wk, struct ix_queue, wq_cookie); 6668 6669 /* 6670 * "enqueued flag" is not required here. 6671 * See ixgbe_msix_que(). 6672 */ 6673 ixgbe_handle_que(que); 6674 } 6675 6676 /************************************************************************ 6677 * ixgbe_allocate_legacy - Setup the Legacy or MSI Interrupt handler 6678 ************************************************************************/ 6679 static int 6680 ixgbe_allocate_legacy(struct adapter *adapter, 6681 const struct pci_attach_args *pa) 6682 { 6683 device_t dev = adapter->dev; 6684 struct ix_queue *que = adapter->queues; 6685 struct tx_ring *txr = adapter->tx_rings; 6686 int counts[PCI_INTR_TYPE_SIZE]; 6687 pci_intr_type_t intr_type, max_type; 6688 char intrbuf[PCI_INTRSTR_LEN]; 6689 char wqname[MAXCOMLEN]; 6690 const char *intrstr = NULL; 6691 int defertx_error = 0, error; 6692 6693 /* We allocate a single interrupt resource */ 6694 max_type = PCI_INTR_TYPE_MSI; 6695 counts[PCI_INTR_TYPE_MSIX] = 0; 6696 counts[PCI_INTR_TYPE_MSI] = 6697 (adapter->feat_en & IXGBE_FEATURE_MSI) ? 1 : 0; 6698 /* Check not feat_en but feat_cap to fallback to INTx */ 6699 counts[PCI_INTR_TYPE_INTX] = 6700 (adapter->feat_cap & IXGBE_FEATURE_LEGACY_IRQ) ? 1 : 0; 6701 6702 alloc_retry: 6703 if (pci_intr_alloc(pa, &adapter->osdep.intrs, counts, max_type) != 0) { 6704 aprint_error_dev(dev, "couldn't alloc interrupt\n"); 6705 return ENXIO; 6706 } 6707 adapter->osdep.nintrs = 1; 6708 intrstr = pci_intr_string(adapter->osdep.pc, adapter->osdep.intrs[0], 6709 intrbuf, sizeof(intrbuf)); 6710 adapter->osdep.ihs[0] = pci_intr_establish_xname(adapter->osdep.pc, 6711 adapter->osdep.intrs[0], IPL_NET, ixgbe_legacy_irq, que, 6712 device_xname(dev)); 6713 intr_type = pci_intr_type(adapter->osdep.pc, adapter->osdep.intrs[0]); 6714 if (adapter->osdep.ihs[0] == NULL) { 6715 aprint_error_dev(dev,"unable to establish %s\n", 6716 (intr_type == PCI_INTR_TYPE_MSI) ? "MSI" : "INTx"); 6717 pci_intr_release(adapter->osdep.pc, adapter->osdep.intrs, 1); 6718 adapter->osdep.intrs = NULL; 6719 switch (intr_type) { 6720 case PCI_INTR_TYPE_MSI: 6721 /* The next try is for INTx: Disable MSI */ 6722 max_type = PCI_INTR_TYPE_INTX; 6723 counts[PCI_INTR_TYPE_INTX] = 1; 6724 adapter->feat_en &= ~IXGBE_FEATURE_MSI; 6725 if (adapter->feat_cap & IXGBE_FEATURE_LEGACY_IRQ) { 6726 adapter->feat_en |= IXGBE_FEATURE_LEGACY_IRQ; 6727 goto alloc_retry; 6728 } else 6729 break; 6730 case PCI_INTR_TYPE_INTX: 6731 default: 6732 /* See below */ 6733 break; 6734 } 6735 } 6736 if (intr_type == PCI_INTR_TYPE_INTX) { 6737 adapter->feat_en &= ~IXGBE_FEATURE_MSI; 6738 adapter->feat_en |= IXGBE_FEATURE_LEGACY_IRQ; 6739 } 6740 if (adapter->osdep.ihs[0] == NULL) { 6741 aprint_error_dev(dev, 6742 "couldn't establish interrupt%s%s\n", 6743 intrstr ? " at " : "", intrstr ? intrstr : ""); 6744 pci_intr_release(adapter->osdep.pc, adapter->osdep.intrs, 1); 6745 adapter->osdep.intrs = NULL; 6746 return ENXIO; 6747 } 6748 aprint_normal_dev(dev, "interrupting at %s\n", intrstr); 6749 /* 6750 * Try allocating a fast interrupt and the associated deferred 6751 * processing contexts. 6752 */ 6753 if (!(adapter->feat_en & IXGBE_FEATURE_LEGACY_TX)) { 6754 txr->txr_si = 6755 softint_establish(SOFTINT_NET | IXGBE_SOFTINT_FLAGS, 6756 ixgbe_deferred_mq_start, txr); 6757 6758 snprintf(wqname, sizeof(wqname), "%sdeferTx", 6759 device_xname(dev)); 6760 defertx_error = workqueue_create(&adapter->txr_wq, wqname, 6761 ixgbe_deferred_mq_start_work, adapter, IXGBE_WORKQUEUE_PRI, 6762 IPL_NET, IXGBE_WORKQUEUE_FLAGS); 6763 adapter->txr_wq_enqueued = percpu_alloc(sizeof(u_int)); 6764 } 6765 que->que_si = softint_establish(SOFTINT_NET | IXGBE_SOFTINT_FLAGS, 6766 ixgbe_handle_que, que); 6767 snprintf(wqname, sizeof(wqname), "%sTxRx", device_xname(dev)); 6768 error = workqueue_create(&adapter->que_wq, wqname, 6769 ixgbe_handle_que_work, adapter, IXGBE_WORKQUEUE_PRI, IPL_NET, 6770 IXGBE_WORKQUEUE_FLAGS); 6771 6772 if ((!(adapter->feat_en & IXGBE_FEATURE_LEGACY_TX) 6773 && ((txr->txr_si == NULL) || defertx_error != 0)) 6774 || (que->que_si == NULL) || error != 0) { 6775 aprint_error_dev(dev, 6776 "could not establish software interrupts\n"); 6777 6778 return ENXIO; 6779 } 6780 /* For simplicity in the handlers */ 6781 adapter->active_queues = IXGBE_EIMS_ENABLE_MASK; 6782 6783 return (0); 6784 } /* ixgbe_allocate_legacy */ 6785 6786 /************************************************************************ 6787 * ixgbe_allocate_msix - Setup MSI-X Interrupt resources and handlers 6788 ************************************************************************/ 6789 static int 6790 ixgbe_allocate_msix(struct adapter *adapter, const struct pci_attach_args *pa) 6791 { 6792 device_t dev = adapter->dev; 6793 struct ix_queue *que = adapter->queues; 6794 struct tx_ring *txr = adapter->tx_rings; 6795 pci_chipset_tag_t pc; 6796 char intrbuf[PCI_INTRSTR_LEN]; 6797 char intr_xname[32]; 6798 char wqname[MAXCOMLEN]; 6799 const char *intrstr = NULL; 6800 int error, vector = 0; 6801 int cpu_id = 0; 6802 kcpuset_t *affinity; 6803 #ifdef RSS 6804 unsigned int rss_buckets = 0; 6805 kcpuset_t cpu_mask; 6806 #endif 6807 6808 pc = adapter->osdep.pc; 6809 #ifdef RSS 6810 /* 6811 * If we're doing RSS, the number of queues needs to 6812 * match the number of RSS buckets that are configured. 6813 * 6814 * + If there's more queues than RSS buckets, we'll end 6815 * up with queues that get no traffic. 6816 * 6817 * + If there's more RSS buckets than queues, we'll end 6818 * up having multiple RSS buckets map to the same queue, 6819 * so there'll be some contention. 6820 */ 6821 rss_buckets = rss_getnumbuckets(); 6822 if ((adapter->feat_en & IXGBE_FEATURE_RSS) && 6823 (adapter->num_queues != rss_buckets)) { 6824 device_printf(dev, 6825 "%s: number of queues (%d) != number of RSS buckets (%d)" 6826 "; performance will be impacted.\n", 6827 __func__, adapter->num_queues, rss_buckets); 6828 } 6829 #endif 6830 6831 adapter->osdep.nintrs = adapter->num_queues + 1; 6832 if (pci_msix_alloc_exact(pa, &adapter->osdep.intrs, 6833 adapter->osdep.nintrs) != 0) { 6834 aprint_error_dev(dev, 6835 "failed to allocate MSI-X interrupt\n"); 6836 return (ENXIO); 6837 } 6838 6839 kcpuset_create(&affinity, false); 6840 for (int i = 0; i < adapter->num_queues; i++, vector++, que++, txr++) { 6841 snprintf(intr_xname, sizeof(intr_xname), "%s TXRX%d", 6842 device_xname(dev), i); 6843 intrstr = pci_intr_string(pc, adapter->osdep.intrs[i], intrbuf, 6844 sizeof(intrbuf)); 6845 #ifdef IXGBE_MPSAFE 6846 pci_intr_setattr(pc, &adapter->osdep.intrs[i], PCI_INTR_MPSAFE, 6847 true); 6848 #endif 6849 /* Set the handler function */ 6850 que->res = adapter->osdep.ihs[i] = pci_intr_establish_xname(pc, 6851 adapter->osdep.intrs[i], IPL_NET, ixgbe_msix_que, que, 6852 intr_xname); 6853 if (que->res == NULL) { 6854 aprint_error_dev(dev, 6855 "Failed to register QUE handler\n"); 6856 error = ENXIO; 6857 goto err_out; 6858 } 6859 que->msix = vector; 6860 adapter->active_queues |= 1ULL << que->msix; 6861 6862 if (adapter->feat_en & IXGBE_FEATURE_RSS) { 6863 #ifdef RSS 6864 /* 6865 * The queue ID is used as the RSS layer bucket ID. 6866 * We look up the queue ID -> RSS CPU ID and select 6867 * that. 6868 */ 6869 cpu_id = rss_getcpu(i % rss_getnumbuckets()); 6870 CPU_SETOF(cpu_id, &cpu_mask); 6871 #endif 6872 } else { 6873 /* 6874 * Bind the MSI-X vector, and thus the 6875 * rings to the corresponding CPU. 6876 * 6877 * This just happens to match the default RSS 6878 * round-robin bucket -> queue -> CPU allocation. 6879 */ 6880 if (adapter->num_queues > 1) 6881 cpu_id = i; 6882 } 6883 /* Round-robin affinity */ 6884 kcpuset_zero(affinity); 6885 kcpuset_set(affinity, cpu_id % ncpu); 6886 error = interrupt_distribute(adapter->osdep.ihs[i], affinity, 6887 NULL); 6888 aprint_normal_dev(dev, "for TX/RX, interrupting at %s", 6889 intrstr); 6890 if (error == 0) { 6891 #if 1 /* def IXGBE_DEBUG */ 6892 #ifdef RSS 6893 aprintf_normal(", bound RSS bucket %d to CPU %d", i, 6894 cpu_id % ncpu); 6895 #else 6896 aprint_normal(", bound queue %d to cpu %d", i, 6897 cpu_id % ncpu); 6898 #endif 6899 #endif /* IXGBE_DEBUG */ 6900 } 6901 aprint_normal("\n"); 6902 6903 if (!(adapter->feat_en & IXGBE_FEATURE_LEGACY_TX)) { 6904 txr->txr_si = softint_establish( 6905 SOFTINT_NET | IXGBE_SOFTINT_FLAGS, 6906 ixgbe_deferred_mq_start, txr); 6907 if (txr->txr_si == NULL) { 6908 aprint_error_dev(dev, 6909 "couldn't establish software interrupt\n"); 6910 error = ENXIO; 6911 goto err_out; 6912 } 6913 } 6914 que->que_si 6915 = softint_establish(SOFTINT_NET | IXGBE_SOFTINT_FLAGS, 6916 ixgbe_handle_que, que); 6917 if (que->que_si == NULL) { 6918 aprint_error_dev(dev, 6919 "couldn't establish software interrupt\n"); 6920 error = ENXIO; 6921 goto err_out; 6922 } 6923 } 6924 snprintf(wqname, sizeof(wqname), "%sdeferTx", device_xname(dev)); 6925 error = workqueue_create(&adapter->txr_wq, wqname, 6926 ixgbe_deferred_mq_start_work, adapter, IXGBE_WORKQUEUE_PRI, IPL_NET, 6927 IXGBE_WORKQUEUE_FLAGS); 6928 if (error) { 6929 aprint_error_dev(dev, 6930 "couldn't create workqueue for deferred Tx\n"); 6931 goto err_out; 6932 } 6933 adapter->txr_wq_enqueued = percpu_alloc(sizeof(u_int)); 6934 6935 snprintf(wqname, sizeof(wqname), "%sTxRx", device_xname(dev)); 6936 error = workqueue_create(&adapter->que_wq, wqname, 6937 ixgbe_handle_que_work, adapter, IXGBE_WORKQUEUE_PRI, IPL_NET, 6938 IXGBE_WORKQUEUE_FLAGS); 6939 if (error) { 6940 aprint_error_dev(dev, "couldn't create workqueue for Tx/Rx\n"); 6941 goto err_out; 6942 } 6943 6944 /* and Link */ 6945 cpu_id++; 6946 snprintf(intr_xname, sizeof(intr_xname), "%s link", device_xname(dev)); 6947 adapter->vector = vector; 6948 intrstr = pci_intr_string(pc, adapter->osdep.intrs[vector], intrbuf, 6949 sizeof(intrbuf)); 6950 #ifdef IXGBE_MPSAFE 6951 pci_intr_setattr(pc, &adapter->osdep.intrs[vector], PCI_INTR_MPSAFE, 6952 true); 6953 #endif 6954 /* Set the link handler function */ 6955 adapter->osdep.ihs[vector] = pci_intr_establish_xname(pc, 6956 adapter->osdep.intrs[vector], IPL_NET, ixgbe_msix_admin, adapter, 6957 intr_xname); 6958 if (adapter->osdep.ihs[vector] == NULL) { 6959 aprint_error_dev(dev, "Failed to register LINK handler\n"); 6960 error = ENXIO; 6961 goto err_out; 6962 } 6963 /* Round-robin affinity */ 6964 kcpuset_zero(affinity); 6965 kcpuset_set(affinity, cpu_id % ncpu); 6966 error = interrupt_distribute(adapter->osdep.ihs[vector], affinity, 6967 NULL); 6968 6969 aprint_normal_dev(dev, 6970 "for link, interrupting at %s", intrstr); 6971 if (error == 0) 6972 aprint_normal(", affinity to cpu %d\n", cpu_id % ncpu); 6973 else 6974 aprint_normal("\n"); 6975 6976 kcpuset_destroy(affinity); 6977 aprint_normal_dev(dev, 6978 "Using MSI-X interrupts with %d vectors\n", vector + 1); 6979 6980 return (0); 6981 6982 err_out: 6983 kcpuset_destroy(affinity); 6984 ixgbe_free_deferred_handlers(adapter); 6985 ixgbe_free_pciintr_resources(adapter); 6986 return (error); 6987 } /* ixgbe_allocate_msix */ 6988 6989 /************************************************************************ 6990 * ixgbe_configure_interrupts 6991 * 6992 * Setup MSI-X, MSI, or legacy interrupts (in that order). 6993 * This will also depend on user settings. 6994 ************************************************************************/ 6995 static int 6996 ixgbe_configure_interrupts(struct adapter *adapter) 6997 { 6998 device_t dev = adapter->dev; 6999 struct ixgbe_mac_info *mac = &adapter->hw.mac; 7000 int want, queues, msgs; 7001 7002 /* Default to 1 queue if MSI-X setup fails */ 7003 adapter->num_queues = 1; 7004 7005 /* Override by tuneable */ 7006 if (!(adapter->feat_cap & IXGBE_FEATURE_MSIX)) 7007 goto msi; 7008 7009 /* 7010 * NetBSD only: Use single vector MSI when number of CPU is 1 to save 7011 * interrupt slot. 7012 */ 7013 if (ncpu == 1) 7014 goto msi; 7015 7016 /* First try MSI-X */ 7017 msgs = pci_msix_count(adapter->osdep.pc, adapter->osdep.tag); 7018 msgs = MIN(msgs, IXG_MAX_NINTR); 7019 if (msgs < 2) 7020 goto msi; 7021 7022 adapter->msix_mem = (void *)1; /* XXX */ 7023 7024 /* Figure out a reasonable auto config value */ 7025 queues = (ncpu > (msgs - 1)) ? (msgs - 1) : ncpu; 7026 7027 #ifdef RSS 7028 /* If we're doing RSS, clamp at the number of RSS buckets */ 7029 if (adapter->feat_en & IXGBE_FEATURE_RSS) 7030 queues = uimin(queues, rss_getnumbuckets()); 7031 #endif 7032 if (ixgbe_num_queues > queues) { 7033 aprint_error_dev(adapter->dev, "ixgbe_num_queues (%d) is too large, using reduced amount (%d).\n", ixgbe_num_queues, queues); 7034 ixgbe_num_queues = queues; 7035 } 7036 7037 if (ixgbe_num_queues != 0) 7038 queues = ixgbe_num_queues; 7039 else 7040 queues = uimin(queues, 7041 uimin(mac->max_tx_queues, mac->max_rx_queues)); 7042 7043 /* reflect correct sysctl value */ 7044 ixgbe_num_queues = queues; 7045 7046 /* 7047 * Want one vector (RX/TX pair) per queue 7048 * plus an additional for Link. 7049 */ 7050 want = queues + 1; 7051 if (msgs >= want) 7052 msgs = want; 7053 else { 7054 aprint_error_dev(dev, "MSI-X Configuration Problem, " 7055 "%d vectors but %d queues wanted!\n", 7056 msgs, want); 7057 goto msi; 7058 } 7059 adapter->num_queues = queues; 7060 adapter->feat_en |= IXGBE_FEATURE_MSIX; 7061 return (0); 7062 7063 /* 7064 * MSI-X allocation failed or provided us with 7065 * less vectors than needed. Free MSI-X resources 7066 * and we'll try enabling MSI. 7067 */ 7068 msi: 7069 /* Without MSI-X, some features are no longer supported */ 7070 adapter->feat_cap &= ~IXGBE_FEATURE_RSS; 7071 adapter->feat_en &= ~IXGBE_FEATURE_RSS; 7072 adapter->feat_cap &= ~IXGBE_FEATURE_SRIOV; 7073 adapter->feat_en &= ~IXGBE_FEATURE_SRIOV; 7074 7075 msgs = pci_msi_count(adapter->osdep.pc, adapter->osdep.tag); 7076 adapter->msix_mem = NULL; /* XXX */ 7077 if (msgs > 1) 7078 msgs = 1; 7079 if (msgs != 0) { 7080 msgs = 1; 7081 adapter->feat_en |= IXGBE_FEATURE_MSI; 7082 return (0); 7083 } 7084 7085 if (!(adapter->feat_cap & IXGBE_FEATURE_LEGACY_IRQ)) { 7086 aprint_error_dev(dev, 7087 "Device does not support legacy interrupts.\n"); 7088 return 1; 7089 } 7090 7091 adapter->feat_en |= IXGBE_FEATURE_LEGACY_IRQ; 7092 7093 return (0); 7094 } /* ixgbe_configure_interrupts */ 7095 7096 7097 /************************************************************************ 7098 * ixgbe_handle_link - Tasklet for MSI-X Link interrupts 7099 * 7100 * Done outside of interrupt context since the driver might sleep 7101 ************************************************************************/ 7102 static void 7103 ixgbe_handle_link(void *context) 7104 { 7105 struct adapter *adapter = context; 7106 struct ixgbe_hw *hw = &adapter->hw; 7107 7108 KASSERT(mutex_owned(&adapter->core_mtx)); 7109 7110 IXGBE_EVC_ADD(&adapter->link_workev, 1); 7111 ixgbe_check_link(hw, &adapter->link_speed, &adapter->link_up, 0); 7112 ixgbe_update_link_status(adapter); 7113 7114 /* Re-enable link interrupts */ 7115 IXGBE_WRITE_REG(hw, IXGBE_EIMS, IXGBE_EIMS_LSC); 7116 } /* ixgbe_handle_link */ 7117 7118 #if 0 7119 /************************************************************************ 7120 * ixgbe_rearm_queues 7121 ************************************************************************/ 7122 static __inline void 7123 ixgbe_rearm_queues(struct adapter *adapter, u64 queues) 7124 { 7125 u32 mask; 7126 7127 switch (adapter->hw.mac.type) { 7128 case ixgbe_mac_82598EB: 7129 mask = (IXGBE_EIMS_RTX_QUEUE & queues); 7130 IXGBE_WRITE_REG(&adapter->hw, IXGBE_EICS, mask); 7131 break; 7132 case ixgbe_mac_82599EB: 7133 case ixgbe_mac_X540: 7134 case ixgbe_mac_X550: 7135 case ixgbe_mac_X550EM_x: 7136 case ixgbe_mac_X550EM_a: 7137 mask = (queues & 0xFFFFFFFF); 7138 IXGBE_WRITE_REG(&adapter->hw, IXGBE_EICS_EX(0), mask); 7139 mask = (queues >> 32); 7140 IXGBE_WRITE_REG(&adapter->hw, IXGBE_EICS_EX(1), mask); 7141 break; 7142 default: 7143 break; 7144 } 7145 } /* ixgbe_rearm_queues */ 7146 #endif 7147
Indexes created Tue Sep 30 11:09:46 GMT 2025