ixgbe.c revision 1.308
1 /* $NetBSD: ixgbe.c,v 1.308 2022/03/10 03:53:46 msaitoh Exp $ */ 2 3 /****************************************************************************** 4 5 Copyright (c) 2001-2017, Intel Corporation 6 All rights reserved. 7 8 Redistribution and use in source and binary forms, with or without 9 modification, are permitted provided that the following conditions are met: 10 11 1. Redistributions of source code must retain the above copyright notice, 12 this list of conditions and the following disclaimer. 13 14 2. Redistributions in binary form must reproduce the above copyright 15 notice, this list of conditions and the following disclaimer in the 16 documentation and/or other materials provided with the distribution. 17 18 3. Neither the name of the Intel Corporation nor the names of its 19 contributors may be used to endorse or promote products derived from 20 this software without specific prior written permission. 21 22 THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 23 AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 24 IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 25 ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE 26 LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 27 CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 28 SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 29 INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 30 CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 31 ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 32 POSSIBILITY OF SUCH DAMAGE. 33 34 ******************************************************************************/ 35 /*$FreeBSD: head/sys/dev/ixgbe/if_ix.c 331224 2018-03-19 20:55:05Z erj $*/ 36 37 /* 38 * Copyright (c) 2011 The NetBSD Foundation, Inc. 39 * All rights reserved. 40 * 41 * This code is derived from software contributed to The NetBSD Foundation 42 * by Coyote Point Systems, Inc. 43 * 44 * Redistribution and use in source and binary forms, with or without 45 * modification, are permitted provided that the following conditions 46 * are met: 47 * 1. Redistributions of source code must retain the above copyright 48 * notice, this list of conditions and the following disclaimer. 49 * 2. Redistributions in binary form must reproduce the above copyright 50 * notice, this list of conditions and the following disclaimer in the 51 * documentation and/or other materials provided with the distribution. 52 * 53 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS 54 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED 55 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 56 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS 57 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 58 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 59 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 60 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 61 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 62 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 63 * POSSIBILITY OF SUCH DAMAGE. 64 */ 65 66 #include <sys/cdefs.h> 67 __KERNEL_RCSID(0, "$NetBSD: ixgbe.c,v 1.308 2022/03/10 03:53:46 msaitoh Exp $"); 68 69 #ifdef _KERNEL_OPT 70 #include "opt_inet.h" 71 #include "opt_inet6.h" 72 #include "opt_net_mpsafe.h" 73 #endif 74 75 #include "ixgbe.h" 76 #include "ixgbe_phy.h" 77 #include "ixgbe_sriov.h" 78 79 #include <sys/cprng.h> 80 #include <dev/mii/mii.h> 81 #include <dev/mii/miivar.h> 82 83 /************************************************************************ 84 * Driver version 85 ************************************************************************/ 86 static const char ixgbe_driver_version[] = "4.0.1-k"; 87 /* XXX NetBSD: + 3.3.24 */ 88 89 /************************************************************************ 90 * PCI Device ID Table 91 * 92 * Used by probe to select devices to load on 93 * Last field stores an index into ixgbe_strings 94 * Last entry must be all 0s 95 * 96 * { Vendor ID, Device ID, SubVendor ID, SubDevice ID, String Index } 97 ************************************************************************/ 98 static const ixgbe_vendor_info_t ixgbe_vendor_info_array[] = 99 { 100 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598AF_DUAL_PORT, 0, 0, 0}, 101 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598AF_SINGLE_PORT, 0, 0, 0}, 102 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598EB_CX4, 0, 0, 0}, 103 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598AT, 0, 0, 0}, 104 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598AT2, 0, 0, 0}, 105 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598, 0, 0, 0}, 106 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598_BX, 0, 0, 0}, 107 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598_DA_DUAL_PORT, 0, 0, 0}, 108 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598_CX4_DUAL_PORT, 0, 0, 0}, 109 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598EB_XF_LR, 0, 0, 0}, 110 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598_SR_DUAL_PORT_EM, 0, 0, 0}, 111 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598EB_SFP_LOM, 0, 0, 0}, 112 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_KR, 0, 0, 0}, 113 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_KX4, 0, 0, 0}, 114 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_KX4_MEZZ, 0, 0, 0}, 115 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_SFP, 0, 0, 0}, 116 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_SFP_EM, 0, 0, 0}, 117 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_XAUI_LOM, 0, 0, 0}, 118 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_CX4, 0, 0, 0}, 119 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_T3_LOM, 0, 0, 0}, 120 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_COMBO_BACKPLANE, 0, 0, 0}, 121 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_BACKPLANE_FCOE, 0, 0, 0}, 122 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_SFP_SF2, 0, 0, 0}, 123 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_SFP_FCOE, 0, 0, 0}, 124 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599EN_SFP, 0, 0, 0}, 125 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_SFP_SF_QP, 0, 0, 0}, 126 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_QSFP_SF_QP, 0, 0, 0}, 127 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_X540T, 0, 0, 0}, 128 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_X540T1, 0, 0, 0}, 129 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_X550T, 0, 0, 0}, 130 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_X550T1, 0, 0, 0}, 131 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_X550EM_X_KR, 0, 0, 0}, 132 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_X550EM_X_KX4, 0, 0, 0}, 133 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_X550EM_X_10G_T, 0, 0, 0}, 134 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_X550EM_X_1G_T, 0, 0, 0}, 135 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_X550EM_X_SFP, 0, 0, 0}, 136 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_X550EM_X_XFI, 0, 0, 0}, 137 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_X550EM_A_KR, 0, 0, 0}, 138 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_X550EM_A_KR_L, 0, 0, 0}, 139 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_X550EM_A_QSFP, 0, 0, 0}, 140 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_X550EM_A_QSFP_N, 0, 0, 0}, 141 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_X550EM_A_SFP, 0, 0, 0}, 142 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_X550EM_A_SFP_N, 0, 0, 0}, 143 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_X550EM_A_SGMII, 0, 0, 0}, 144 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_X550EM_A_SGMII_L, 0, 0, 0}, 145 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_X550EM_A_10G_T, 0, 0, 0}, 146 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_X550EM_A_1G_T, 0, 0, 0}, 147 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_X550EM_A_1G_T_L, 0, 0, 0}, 148 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_X540_BYPASS, 0, 0, 0}, 149 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_BYPASS, 0, 0, 0}, 150 /* required last entry */ 151 {0, 0, 0, 0, 0} 152 }; 153 154 /************************************************************************ 155 * Table of branding strings 156 ************************************************************************/ 157 static const char *ixgbe_strings[] = { 158 "Intel(R) PRO/10GbE PCI-Express Network Driver" 159 }; 160 161 /************************************************************************ 162 * Function prototypes 163 ************************************************************************/ 164 static int ixgbe_probe(device_t, cfdata_t, void *); 165 static void ixgbe_quirks(struct adapter *); 166 static void ixgbe_attach(device_t, device_t, void *); 167 static int ixgbe_detach(device_t, int); 168 #if 0 169 static int ixgbe_shutdown(device_t); 170 #endif 171 static bool ixgbe_suspend(device_t, const pmf_qual_t *); 172 static bool ixgbe_resume(device_t, const pmf_qual_t *); 173 static int ixgbe_ifflags_cb(struct ethercom *); 174 static int ixgbe_ioctl(struct ifnet *, u_long, void *); 175 static int ixgbe_init(struct ifnet *); 176 static void ixgbe_init_locked(struct adapter *); 177 static void ixgbe_ifstop(struct ifnet *, int); 178 static void ixgbe_stop_locked(void *); 179 static void ixgbe_init_device_features(struct adapter *); 180 static int ixgbe_check_fan_failure(struct adapter *, u32, bool); 181 static void ixgbe_add_media_types(struct adapter *); 182 static void ixgbe_media_status(struct ifnet *, struct ifmediareq *); 183 static int ixgbe_media_change(struct ifnet *); 184 static int ixgbe_allocate_pci_resources(struct adapter *, 185 const struct pci_attach_args *); 186 static void ixgbe_free_deferred_handlers(struct adapter *); 187 static void ixgbe_get_slot_info(struct adapter *); 188 static int ixgbe_allocate_msix(struct adapter *, 189 const struct pci_attach_args *); 190 static int ixgbe_allocate_legacy(struct adapter *, 191 const struct pci_attach_args *); 192 static int ixgbe_configure_interrupts(struct adapter *); 193 static void ixgbe_free_pciintr_resources(struct adapter *); 194 static void ixgbe_free_pci_resources(struct adapter *); 195 static void ixgbe_local_timer(void *); 196 static void ixgbe_handle_timer(struct work *, void *); 197 static void ixgbe_recovery_mode_timer(void *); 198 static void ixgbe_handle_recovery_mode_timer(struct work *, void *); 199 static int ixgbe_setup_interface(device_t, struct adapter *); 200 static void ixgbe_config_gpie(struct adapter *); 201 static void ixgbe_config_dmac(struct adapter *); 202 static void ixgbe_config_delay_values(struct adapter *); 203 static void ixgbe_schedule_admin_tasklet(struct adapter *); 204 static void ixgbe_config_link(struct adapter *); 205 static void ixgbe_check_wol_support(struct adapter *); 206 static int ixgbe_setup_low_power_mode(struct adapter *); 207 #if 0 208 static void ixgbe_rearm_queues(struct adapter *, u64); 209 #endif 210 211 static void ixgbe_initialize_transmit_units(struct adapter *); 212 static void ixgbe_initialize_receive_units(struct adapter *); 213 static void ixgbe_enable_rx_drop(struct adapter *); 214 static void ixgbe_disable_rx_drop(struct adapter *); 215 static void ixgbe_initialize_rss_mapping(struct adapter *); 216 217 static void ixgbe_enable_intr(struct adapter *); 218 static void ixgbe_disable_intr(struct adapter *); 219 static void ixgbe_update_stats_counters(struct adapter *); 220 static void ixgbe_set_rxfilter(struct adapter *); 221 static void ixgbe_update_link_status(struct adapter *); 222 static void ixgbe_set_ivar(struct adapter *, u8, u8, s8); 223 static void ixgbe_configure_ivars(struct adapter *); 224 static u8 * ixgbe_mc_array_itr(struct ixgbe_hw *, u8 **, u32 *); 225 static void ixgbe_eitr_write(struct adapter *, uint32_t, uint32_t); 226 227 static void ixgbe_setup_vlan_hw_tagging(struct adapter *); 228 static void ixgbe_setup_vlan_hw_support(struct adapter *); 229 static int ixgbe_vlan_cb(struct ethercom *, uint16_t, bool); 230 static int ixgbe_register_vlan(struct adapter *, u16); 231 static int ixgbe_unregister_vlan(struct adapter *, u16); 232 233 static void ixgbe_add_device_sysctls(struct adapter *); 234 static void ixgbe_add_hw_stats(struct adapter *); 235 static void ixgbe_clear_evcnt(struct adapter *); 236 static int ixgbe_set_flowcntl(struct adapter *, int); 237 static int ixgbe_set_advertise(struct adapter *, int); 238 static int ixgbe_get_default_advertise(struct adapter *); 239 240 /* Sysctl handlers */ 241 static void ixgbe_set_sysctl_value(struct adapter *, const char *, 242 const char *, int *, int); 243 static int ixgbe_sysctl_flowcntl(SYSCTLFN_PROTO); 244 static int ixgbe_sysctl_advertise(SYSCTLFN_PROTO); 245 static int ixgbe_sysctl_interrupt_rate_handler(SYSCTLFN_PROTO); 246 static int ixgbe_sysctl_dmac(SYSCTLFN_PROTO); 247 static int ixgbe_sysctl_phy_temp(SYSCTLFN_PROTO); 248 static int ixgbe_sysctl_phy_overtemp_occurred(SYSCTLFN_PROTO); 249 #ifdef IXGBE_DEBUG 250 static int ixgbe_sysctl_power_state(SYSCTLFN_PROTO); 251 static int ixgbe_sysctl_print_rss_config(SYSCTLFN_PROTO); 252 #endif 253 static int ixgbe_sysctl_next_to_check_handler(SYSCTLFN_PROTO); 254 static int ixgbe_sysctl_next_to_refresh_handler(SYSCTLFN_PROTO); 255 static int ixgbe_sysctl_rdh_handler(SYSCTLFN_PROTO); 256 static int ixgbe_sysctl_rdt_handler(SYSCTLFN_PROTO); 257 static int ixgbe_sysctl_tdt_handler(SYSCTLFN_PROTO); 258 static int ixgbe_sysctl_tdh_handler(SYSCTLFN_PROTO); 259 static int ixgbe_sysctl_eee_state(SYSCTLFN_PROTO); 260 static int ixgbe_sysctl_debug(SYSCTLFN_PROTO); 261 static int ixgbe_sysctl_rx_copy_len(SYSCTLFN_PROTO); 262 static int ixgbe_sysctl_wol_enable(SYSCTLFN_PROTO); 263 static int ixgbe_sysctl_wufc(SYSCTLFN_PROTO); 264 265 /* Interrupt functions */ 266 static int ixgbe_msix_que(void *); 267 static int ixgbe_msix_admin(void *); 268 static void ixgbe_intr_admin_common(struct adapter *, u32, u32 *); 269 static int ixgbe_legacy_irq(void *); 270 271 /* Event handlers running on workqueue */ 272 static void ixgbe_handle_que(void *); 273 static void ixgbe_handle_link(void *); 274 static void ixgbe_handle_msf(void *); 275 static void ixgbe_handle_mod(void *, bool); 276 static void ixgbe_handle_phy(void *); 277 278 /* Deferred workqueue handlers */ 279 static void ixgbe_handle_admin(struct work *, void *); 280 static void ixgbe_handle_que_work(struct work *, void *); 281 282 static const ixgbe_vendor_info_t *ixgbe_lookup(const struct pci_attach_args *); 283 284 /************************************************************************ 285 * NetBSD Device Interface Entry Points 286 ************************************************************************/ 287 CFATTACH_DECL3_NEW(ixg, sizeof(struct adapter), 288 ixgbe_probe, ixgbe_attach, ixgbe_detach, NULL, NULL, NULL, 289 DVF_DETACH_SHUTDOWN); 290 291 #if 0 292 devclass_t ix_devclass; 293 DRIVER_MODULE(ix, pci, ix_driver, ix_devclass, 0, 0); 294 295 MODULE_DEPEND(ix, pci, 1, 1, 1); 296 MODULE_DEPEND(ix, ether, 1, 1, 1); 297 #ifdef DEV_NETMAP 298 MODULE_DEPEND(ix, netmap, 1, 1, 1); 299 #endif 300 #endif 301 302 /* 303 * TUNEABLE PARAMETERS: 304 */ 305 306 /* 307 * AIM: Adaptive Interrupt Moderation 308 * which means that the interrupt rate 309 * is varied over time based on the 310 * traffic for that interrupt vector 311 */ 312 static bool ixgbe_enable_aim = true; 313 #define SYSCTL_INT(_a1, _a2, _a3, _a4, _a5, _a6, _a7) 314 SYSCTL_INT(_hw_ix, OID_AUTO, enable_aim, CTLFLAG_RDTUN, &ixgbe_enable_aim, 0, 315 "Enable adaptive interrupt moderation"); 316 317 static int ixgbe_max_interrupt_rate = (4000000 / IXGBE_LOW_LATENCY); 318 SYSCTL_INT(_hw_ix, OID_AUTO, max_interrupt_rate, CTLFLAG_RDTUN, 319 &ixgbe_max_interrupt_rate, 0, "Maximum interrupts per second"); 320 321 /* How many packets rxeof tries to clean at a time */ 322 static int ixgbe_rx_process_limit = 256; 323 SYSCTL_INT(_hw_ix, OID_AUTO, rx_process_limit, CTLFLAG_RDTUN, 324 &ixgbe_rx_process_limit, 0, "Maximum number of received packets to process at a time, -1 means unlimited"); 325 326 /* How many packets txeof tries to clean at a time */ 327 static int ixgbe_tx_process_limit = 256; 328 SYSCTL_INT(_hw_ix, OID_AUTO, tx_process_limit, CTLFLAG_RDTUN, 329 &ixgbe_tx_process_limit, 0, 330 "Maximum number of sent packets to process at a time, -1 means unlimited"); 331 332 /* Flow control setting, default to full */ 333 static int ixgbe_flow_control = ixgbe_fc_full; 334 SYSCTL_INT(_hw_ix, OID_AUTO, flow_control, CTLFLAG_RDTUN, 335 &ixgbe_flow_control, 0, "Default flow control used for all adapters"); 336 337 /* Which packet processing uses workqueue or softint */ 338 static bool ixgbe_txrx_workqueue = false; 339 340 /* 341 * Smart speed setting, default to on 342 * this only works as a compile option 343 * right now as its during attach, set 344 * this to 'ixgbe_smart_speed_off' to 345 * disable. 346 */ 347 static int ixgbe_smart_speed = ixgbe_smart_speed_on; 348 349 /* 350 * MSI-X should be the default for best performance, 351 * but this allows it to be forced off for testing. 352 */ 353 static int ixgbe_enable_msix = 1; 354 SYSCTL_INT(_hw_ix, OID_AUTO, enable_msix, CTLFLAG_RDTUN, &ixgbe_enable_msix, 0, 355 "Enable MSI-X interrupts"); 356 357 /* 358 * Number of Queues, can be set to 0, 359 * it then autoconfigures based on the 360 * number of cpus with a max of 8. This 361 * can be overridden manually here. 362 */ 363 static int ixgbe_num_queues = 0; 364 SYSCTL_INT(_hw_ix, OID_AUTO, num_queues, CTLFLAG_RDTUN, &ixgbe_num_queues, 0, 365 "Number of queues to configure, 0 indicates autoconfigure"); 366 367 /* 368 * Number of TX descriptors per ring, 369 * setting higher than RX as this seems 370 * the better performing choice. 371 */ 372 static int ixgbe_txd = PERFORM_TXD; 373 SYSCTL_INT(_hw_ix, OID_AUTO, txd, CTLFLAG_RDTUN, &ixgbe_txd, 0, 374 "Number of transmit descriptors per queue"); 375 376 /* Number of RX descriptors per ring */ 377 static int ixgbe_rxd = PERFORM_RXD; 378 SYSCTL_INT(_hw_ix, OID_AUTO, rxd, CTLFLAG_RDTUN, &ixgbe_rxd, 0, 379 "Number of receive descriptors per queue"); 380 381 /* 382 * Defining this on will allow the use 383 * of unsupported SFP+ modules, note that 384 * doing so you are on your own :) 385 */ 386 static int allow_unsupported_sfp = false; 387 #define TUNABLE_INT(__x, __y) 388 TUNABLE_INT("hw.ix.unsupported_sfp", &allow_unsupported_sfp); 389 390 /* 391 * Not sure if Flow Director is fully baked, 392 * so we'll default to turning it off. 393 */ 394 static int ixgbe_enable_fdir = 0; 395 SYSCTL_INT(_hw_ix, OID_AUTO, enable_fdir, CTLFLAG_RDTUN, &ixgbe_enable_fdir, 0, 396 "Enable Flow Director"); 397 398 /* Legacy Transmit (single queue) */ 399 static int ixgbe_enable_legacy_tx = 0; 400 SYSCTL_INT(_hw_ix, OID_AUTO, enable_legacy_tx, CTLFLAG_RDTUN, 401 &ixgbe_enable_legacy_tx, 0, "Enable Legacy TX flow"); 402 403 /* Receive-Side Scaling */ 404 static int ixgbe_enable_rss = 1; 405 SYSCTL_INT(_hw_ix, OID_AUTO, enable_rss, CTLFLAG_RDTUN, &ixgbe_enable_rss, 0, 406 "Enable Receive-Side Scaling (RSS)"); 407 408 #if 0 409 static int (*ixgbe_start_locked)(struct ifnet *, struct tx_ring *); 410 static int (*ixgbe_ring_empty)(struct ifnet *, pcq_t *); 411 #endif 412 413 #ifdef NET_MPSAFE 414 #define IXGBE_MPSAFE 1 415 #define IXGBE_CALLOUT_FLAGS CALLOUT_MPSAFE 416 #define IXGBE_SOFTINT_FLAGS SOFTINT_MPSAFE 417 #define IXGBE_WORKQUEUE_FLAGS WQ_PERCPU | WQ_MPSAFE 418 #define IXGBE_TASKLET_WQ_FLAGS WQ_MPSAFE 419 #else 420 #define IXGBE_CALLOUT_FLAGS 0 421 #define IXGBE_SOFTINT_FLAGS 0 422 #define IXGBE_WORKQUEUE_FLAGS WQ_PERCPU 423 #define IXGBE_TASKLET_WQ_FLAGS 0 424 #endif 425 #define IXGBE_WORKQUEUE_PRI PRI_SOFTNET 426 427 /************************************************************************ 428 * ixgbe_initialize_rss_mapping 429 ************************************************************************/ 430 static void 431 ixgbe_initialize_rss_mapping(struct adapter *adapter) 432 { 433 struct ixgbe_hw *hw = &adapter->hw; 434 u32 reta = 0, mrqc, rss_key[10]; 435 int queue_id, table_size, index_mult; 436 int i, j; 437 u32 rss_hash_config; 438 439 /* force use default RSS key. */ 440 #ifdef __NetBSD__ 441 rss_getkey((uint8_t *) &rss_key); 442 #else 443 if (adapter->feat_en & IXGBE_FEATURE_RSS) { 444 /* Fetch the configured RSS key */ 445 rss_getkey((uint8_t *) &rss_key); 446 } else { 447 /* set up random bits */ 448 cprng_fast(&rss_key, sizeof(rss_key)); 449 } 450 #endif 451 452 /* Set multiplier for RETA setup and table size based on MAC */ 453 index_mult = 0x1; 454 table_size = 128; 455 switch (adapter->hw.mac.type) { 456 case ixgbe_mac_82598EB: 457 index_mult = 0x11; 458 break; 459 case ixgbe_mac_X550: 460 case ixgbe_mac_X550EM_x: 461 case ixgbe_mac_X550EM_a: 462 table_size = 512; 463 break; 464 default: 465 break; 466 } 467 468 /* Set up the redirection table */ 469 for (i = 0, j = 0; i < table_size; i++, j++) { 470 if (j == adapter->num_queues) 471 j = 0; 472 473 if (adapter->feat_en & IXGBE_FEATURE_RSS) { 474 /* 475 * Fetch the RSS bucket id for the given indirection 476 * entry. Cap it at the number of configured buckets 477 * (which is num_queues.) 478 */ 479 queue_id = rss_get_indirection_to_bucket(i); 480 queue_id = queue_id % adapter->num_queues; 481 } else 482 queue_id = (j * index_mult); 483 484 /* 485 * The low 8 bits are for hash value (n+0); 486 * The next 8 bits are for hash value (n+1), etc. 487 */ 488 reta = reta >> 8; 489 reta = reta | (((uint32_t) queue_id) << 24); 490 if ((i & 3) == 3) { 491 if (i < 128) 492 IXGBE_WRITE_REG(hw, IXGBE_RETA(i >> 2), reta); 493 else 494 IXGBE_WRITE_REG(hw, IXGBE_ERETA((i >> 2) - 32), 495 reta); 496 reta = 0; 497 } 498 } 499 500 /* Now fill our hash function seeds */ 501 for (i = 0; i < 10; i++) 502 IXGBE_WRITE_REG(hw, IXGBE_RSSRK(i), rss_key[i]); 503 504 /* Perform hash on these packet types */ 505 if (adapter->feat_en & IXGBE_FEATURE_RSS) 506 rss_hash_config = rss_gethashconfig(); 507 else { 508 /* 509 * Disable UDP - IP fragments aren't currently being handled 510 * and so we end up with a mix of 2-tuple and 4-tuple 511 * traffic. 512 */ 513 rss_hash_config = RSS_HASHTYPE_RSS_IPV4 514 | RSS_HASHTYPE_RSS_TCP_IPV4 515 | RSS_HASHTYPE_RSS_IPV6 516 | RSS_HASHTYPE_RSS_TCP_IPV6 517 | RSS_HASHTYPE_RSS_IPV6_EX 518 | RSS_HASHTYPE_RSS_TCP_IPV6_EX; 519 } 520 521 mrqc = IXGBE_MRQC_RSSEN; 522 if (rss_hash_config & RSS_HASHTYPE_RSS_IPV4) 523 mrqc |= IXGBE_MRQC_RSS_FIELD_IPV4; 524 if (rss_hash_config & RSS_HASHTYPE_RSS_TCP_IPV4) 525 mrqc |= IXGBE_MRQC_RSS_FIELD_IPV4_TCP; 526 if (rss_hash_config & RSS_HASHTYPE_RSS_IPV6) 527 mrqc |= IXGBE_MRQC_RSS_FIELD_IPV6; 528 if (rss_hash_config & RSS_HASHTYPE_RSS_TCP_IPV6) 529 mrqc |= IXGBE_MRQC_RSS_FIELD_IPV6_TCP; 530 if (rss_hash_config & RSS_HASHTYPE_RSS_IPV6_EX) 531 mrqc |= IXGBE_MRQC_RSS_FIELD_IPV6_EX; 532 if (rss_hash_config & RSS_HASHTYPE_RSS_TCP_IPV6_EX) 533 mrqc |= IXGBE_MRQC_RSS_FIELD_IPV6_EX_TCP; 534 if (rss_hash_config & RSS_HASHTYPE_RSS_UDP_IPV4) 535 mrqc |= IXGBE_MRQC_RSS_FIELD_IPV4_UDP; 536 if (rss_hash_config & RSS_HASHTYPE_RSS_UDP_IPV6) 537 mrqc |= IXGBE_MRQC_RSS_FIELD_IPV6_UDP; 538 if (rss_hash_config & RSS_HASHTYPE_RSS_UDP_IPV6_EX) 539 mrqc |= IXGBE_MRQC_RSS_FIELD_IPV6_EX_UDP; 540 mrqc |= ixgbe_get_mrqc(adapter->iov_mode); 541 IXGBE_WRITE_REG(hw, IXGBE_MRQC, mrqc); 542 } /* ixgbe_initialize_rss_mapping */ 543 544 /************************************************************************ 545 * ixgbe_initialize_receive_units - Setup receive registers and features. 546 ************************************************************************/ 547 #define BSIZEPKT_ROUNDUP ((1<<IXGBE_SRRCTL_BSIZEPKT_SHIFT)-1) 548 549 static void 550 ixgbe_initialize_receive_units(struct adapter *adapter) 551 { 552 struct rx_ring *rxr = adapter->rx_rings; 553 struct ixgbe_hw *hw = &adapter->hw; 554 struct ifnet *ifp = adapter->ifp; 555 int i, j; 556 u32 bufsz, fctrl, srrctl, rxcsum; 557 u32 hlreg; 558 559 /* 560 * Make sure receives are disabled while 561 * setting up the descriptor ring 562 */ 563 ixgbe_disable_rx(hw); 564 565 /* Enable broadcasts */ 566 fctrl = IXGBE_READ_REG(hw, IXGBE_FCTRL); 567 fctrl |= IXGBE_FCTRL_BAM; 568 if (adapter->hw.mac.type == ixgbe_mac_82598EB) { 569 fctrl |= IXGBE_FCTRL_DPF; 570 fctrl |= IXGBE_FCTRL_PMCF; 571 } 572 IXGBE_WRITE_REG(hw, IXGBE_FCTRL, fctrl); 573 574 /* Set for Jumbo Frames? */ 575 hlreg = IXGBE_READ_REG(hw, IXGBE_HLREG0); 576 if (ifp->if_mtu > ETHERMTU) 577 hlreg |= IXGBE_HLREG0_JUMBOEN; 578 else 579 hlreg &= ~IXGBE_HLREG0_JUMBOEN; 580 581 #ifdef DEV_NETMAP 582 /* CRC stripping is conditional in Netmap */ 583 if ((adapter->feat_en & IXGBE_FEATURE_NETMAP) && 584 (ifp->if_capenable & IFCAP_NETMAP) && 585 !ix_crcstrip) 586 hlreg &= ~IXGBE_HLREG0_RXCRCSTRP; 587 else 588 #endif /* DEV_NETMAP */ 589 hlreg |= IXGBE_HLREG0_RXCRCSTRP; 590 591 IXGBE_WRITE_REG(hw, IXGBE_HLREG0, hlreg); 592 593 bufsz = (adapter->rx_mbuf_sz + BSIZEPKT_ROUNDUP) >> 594 IXGBE_SRRCTL_BSIZEPKT_SHIFT; 595 596 for (i = 0; i < adapter->num_queues; i++, rxr++) { 597 u64 rdba = rxr->rxdma.dma_paddr; 598 u32 reg; 599 int regnum = i / 4; /* 1 register per 4 queues */ 600 int regshift = i % 4; /* 4 bits per 1 queue */ 601 j = rxr->me; 602 603 /* Setup the Base and Length of the Rx Descriptor Ring */ 604 IXGBE_WRITE_REG(hw, IXGBE_RDBAL(j), 605 (rdba & 0x00000000ffffffffULL)); 606 IXGBE_WRITE_REG(hw, IXGBE_RDBAH(j), (rdba >> 32)); 607 IXGBE_WRITE_REG(hw, IXGBE_RDLEN(j), 608 adapter->num_rx_desc * sizeof(union ixgbe_adv_rx_desc)); 609 610 /* Set up the SRRCTL register */ 611 srrctl = IXGBE_READ_REG(hw, IXGBE_SRRCTL(j)); 612 srrctl &= ~IXGBE_SRRCTL_BSIZEHDR_MASK; 613 srrctl &= ~IXGBE_SRRCTL_BSIZEPKT_MASK; 614 srrctl |= bufsz; 615 srrctl |= IXGBE_SRRCTL_DESCTYPE_ADV_ONEBUF; 616 617 /* Set RQSMR (Receive Queue Statistic Mapping) register */ 618 reg = IXGBE_READ_REG(hw, IXGBE_RQSMR(regnum)); 619 reg &= ~(0x000000ffUL << (regshift * 8)); 620 reg |= i << (regshift * 8); 621 IXGBE_WRITE_REG(hw, IXGBE_RQSMR(regnum), reg); 622 623 /* 624 * Set DROP_EN iff we have no flow control and >1 queue. 625 * Note that srrctl was cleared shortly before during reset, 626 * so we do not need to clear the bit, but do it just in case 627 * this code is moved elsewhere. 628 */ 629 if (adapter->num_queues > 1 && 630 adapter->hw.fc.requested_mode == ixgbe_fc_none) { 631 srrctl |= IXGBE_SRRCTL_DROP_EN; 632 } else { 633 srrctl &= ~IXGBE_SRRCTL_DROP_EN; 634 } 635 636 IXGBE_WRITE_REG(hw, IXGBE_SRRCTL(j), srrctl); 637 638 /* Setup the HW Rx Head and Tail Descriptor Pointers */ 639 IXGBE_WRITE_REG(hw, IXGBE_RDH(j), 0); 640 IXGBE_WRITE_REG(hw, IXGBE_RDT(j), 0); 641 642 /* Set the driver rx tail address */ 643 rxr->tail = IXGBE_RDT(rxr->me); 644 } 645 646 if (adapter->hw.mac.type != ixgbe_mac_82598EB) { 647 u32 psrtype = IXGBE_PSRTYPE_TCPHDR 648 | IXGBE_PSRTYPE_UDPHDR 649 | IXGBE_PSRTYPE_IPV4HDR 650 | IXGBE_PSRTYPE_IPV6HDR; 651 IXGBE_WRITE_REG(hw, IXGBE_PSRTYPE(0), psrtype); 652 } 653 654 rxcsum = IXGBE_READ_REG(hw, IXGBE_RXCSUM); 655 656 ixgbe_initialize_rss_mapping(adapter); 657 658 if (adapter->num_queues > 1) { 659 /* RSS and RX IPP Checksum are mutually exclusive */ 660 rxcsum |= IXGBE_RXCSUM_PCSD; 661 } 662 663 if (ifp->if_capenable & IFCAP_RXCSUM) 664 rxcsum |= IXGBE_RXCSUM_PCSD; 665 666 /* This is useful for calculating UDP/IP fragment checksums */ 667 if (!(rxcsum & IXGBE_RXCSUM_PCSD)) 668 rxcsum |= IXGBE_RXCSUM_IPPCSE; 669 670 IXGBE_WRITE_REG(hw, IXGBE_RXCSUM, rxcsum); 671 672 } /* ixgbe_initialize_receive_units */ 673 674 /************************************************************************ 675 * ixgbe_initialize_transmit_units - Enable transmit units. 676 ************************************************************************/ 677 static void 678 ixgbe_initialize_transmit_units(struct adapter *adapter) 679 { 680 struct tx_ring *txr = adapter->tx_rings; 681 struct ixgbe_hw *hw = &adapter->hw; 682 int i; 683 684 INIT_DEBUGOUT("ixgbe_initialize_transmit_units"); 685 686 /* Setup the Base and Length of the Tx Descriptor Ring */ 687 for (i = 0; i < adapter->num_queues; i++, txr++) { 688 u64 tdba = txr->txdma.dma_paddr; 689 u32 txctrl = 0; 690 u32 tqsmreg, reg; 691 int regnum = i / 4; /* 1 register per 4 queues */ 692 int regshift = i % 4; /* 4 bits per 1 queue */ 693 int j = txr->me; 694 695 IXGBE_WRITE_REG(hw, IXGBE_TDBAL(j), 696 (tdba & 0x00000000ffffffffULL)); 697 IXGBE_WRITE_REG(hw, IXGBE_TDBAH(j), (tdba >> 32)); 698 IXGBE_WRITE_REG(hw, IXGBE_TDLEN(j), 699 adapter->num_tx_desc * sizeof(union ixgbe_adv_tx_desc)); 700 701 /* 702 * Set TQSMR (Transmit Queue Statistic Mapping) register. 703 * Register location is different between 82598 and others. 704 */ 705 if (adapter->hw.mac.type == ixgbe_mac_82598EB) 706 tqsmreg = IXGBE_TQSMR(regnum); 707 else 708 tqsmreg = IXGBE_TQSM(regnum); 709 reg = IXGBE_READ_REG(hw, tqsmreg); 710 reg &= ~(0x000000ffUL << (regshift * 8)); 711 reg |= i << (regshift * 8); 712 IXGBE_WRITE_REG(hw, tqsmreg, reg); 713 714 /* Setup the HW Tx Head and Tail descriptor pointers */ 715 IXGBE_WRITE_REG(hw, IXGBE_TDH(j), 0); 716 IXGBE_WRITE_REG(hw, IXGBE_TDT(j), 0); 717 718 /* Cache the tail address */ 719 txr->tail = IXGBE_TDT(j); 720 721 txr->txr_no_space = false; 722 723 /* Disable Head Writeback */ 724 /* 725 * Note: for X550 series devices, these registers are actually 726 * prefixed with TPH_ instead of DCA_, but the addresses and 727 * fields remain the same. 728 */ 729 switch (hw->mac.type) { 730 case ixgbe_mac_82598EB: 731 txctrl = IXGBE_READ_REG(hw, IXGBE_DCA_TXCTRL(j)); 732 break; 733 default: 734 txctrl = IXGBE_READ_REG(hw, IXGBE_DCA_TXCTRL_82599(j)); 735 break; 736 } 737 txctrl &= ~IXGBE_DCA_TXCTRL_DESC_WRO_EN; 738 switch (hw->mac.type) { 739 case ixgbe_mac_82598EB: 740 IXGBE_WRITE_REG(hw, IXGBE_DCA_TXCTRL(j), txctrl); 741 break; 742 default: 743 IXGBE_WRITE_REG(hw, IXGBE_DCA_TXCTRL_82599(j), txctrl); 744 break; 745 } 746 747 } 748 749 if (hw->mac.type != ixgbe_mac_82598EB) { 750 u32 dmatxctl, rttdcs; 751 752 dmatxctl = IXGBE_READ_REG(hw, IXGBE_DMATXCTL); 753 dmatxctl |= IXGBE_DMATXCTL_TE; 754 IXGBE_WRITE_REG(hw, IXGBE_DMATXCTL, dmatxctl); 755 /* Disable arbiter to set MTQC */ 756 rttdcs = IXGBE_READ_REG(hw, IXGBE_RTTDCS); 757 rttdcs |= IXGBE_RTTDCS_ARBDIS; 758 IXGBE_WRITE_REG(hw, IXGBE_RTTDCS, rttdcs); 759 IXGBE_WRITE_REG(hw, IXGBE_MTQC, 760 ixgbe_get_mtqc(adapter->iov_mode)); 761 rttdcs &= ~IXGBE_RTTDCS_ARBDIS; 762 IXGBE_WRITE_REG(hw, IXGBE_RTTDCS, rttdcs); 763 } 764 765 return; 766 } /* ixgbe_initialize_transmit_units */ 767 768 static void 769 ixgbe_quirks(struct adapter *adapter) 770 { 771 device_t dev = adapter->dev; 772 struct ixgbe_hw *hw = &adapter->hw; 773 const char *vendor, *product; 774 775 if (hw->device_id == IXGBE_DEV_ID_X550EM_A_SFP_N) { 776 /* 777 * Quirk for inverted logic of SFP+'s MOD_ABS on GIGABYTE 778 * MA10-ST0. 779 */ 780 vendor = pmf_get_platform("system-vendor"); 781 product = pmf_get_platform("system-product"); 782 783 if ((vendor == NULL) || (product == NULL)) 784 return; 785 786 if ((strcmp(vendor, "GIGABYTE") == 0) && 787 (strcmp(product, "MA10-ST0") == 0)) { 788 aprint_verbose_dev(dev, 789 "Enable SFP+ MOD_ABS inverse quirk\n"); 790 adapter->hw.quirks |= IXGBE_QUIRK_MOD_ABS_INVERT; 791 } 792 } 793 } 794 795 /************************************************************************ 796 * ixgbe_attach - Device initialization routine 797 * 798 * Called when the driver is being loaded. 799 * Identifies the type of hardware, allocates all resources 800 * and initializes the hardware. 801 * 802 * return 0 on success, positive on failure 803 ************************************************************************/ 804 static void 805 ixgbe_attach(device_t parent, device_t dev, void *aux) 806 { 807 struct adapter *adapter; 808 struct ixgbe_hw *hw; 809 int error = -1; 810 u32 ctrl_ext; 811 u16 high, low, nvmreg; 812 pcireg_t id, subid; 813 const ixgbe_vendor_info_t *ent; 814 struct pci_attach_args *pa = aux; 815 bool unsupported_sfp = false; 816 const char *str; 817 char wqname[MAXCOMLEN]; 818 char buf[256]; 819 820 INIT_DEBUGOUT("ixgbe_attach: begin"); 821 822 /* Allocate, clear, and link in our adapter structure */ 823 adapter = device_private(dev); 824 adapter->hw.back = adapter; 825 adapter->dev = dev; 826 hw = &adapter->hw; 827 adapter->osdep.pc = pa->pa_pc; 828 adapter->osdep.tag = pa->pa_tag; 829 if (pci_dma64_available(pa)) 830 adapter->osdep.dmat = pa->pa_dmat64; 831 else 832 adapter->osdep.dmat = pa->pa_dmat; 833 adapter->osdep.attached = false; 834 adapter->osdep.detaching = false; 835 836 ent = ixgbe_lookup(pa); 837 838 KASSERT(ent != NULL); 839 840 aprint_normal(": %s, Version - %s\n", 841 ixgbe_strings[ent->index], ixgbe_driver_version); 842 843 /* Core Lock Init */ 844 IXGBE_CORE_LOCK_INIT(adapter, device_xname(dev)); 845 846 /* Set up the timer callout and workqueue */ 847 callout_init(&adapter->timer, IXGBE_CALLOUT_FLAGS); 848 snprintf(wqname, sizeof(wqname), "%s-timer", device_xname(dev)); 849 error = workqueue_create(&adapter->timer_wq, wqname, 850 ixgbe_handle_timer, adapter, IXGBE_WORKQUEUE_PRI, IPL_NET, 851 IXGBE_TASKLET_WQ_FLAGS); 852 if (error) { 853 aprint_error_dev(dev, 854 "could not create timer workqueue (%d)\n", error); 855 goto err_out; 856 } 857 858 /* Determine hardware revision */ 859 id = pci_conf_read(pa->pa_pc, pa->pa_tag, PCI_ID_REG); 860 subid = pci_conf_read(pa->pa_pc, pa->pa_tag, PCI_SUBSYS_ID_REG); 861 862 hw->vendor_id = PCI_VENDOR(id); 863 hw->device_id = PCI_PRODUCT(id); 864 hw->revision_id = 865 PCI_REVISION(pci_conf_read(pa->pa_pc, pa->pa_tag, PCI_CLASS_REG)); 866 hw->subsystem_vendor_id = PCI_SUBSYS_VENDOR(subid); 867 hw->subsystem_device_id = PCI_SUBSYS_ID(subid); 868 869 /* Set quirk flags */ 870 ixgbe_quirks(adapter); 871 872 /* 873 * Make sure BUSMASTER is set 874 */ 875 ixgbe_pci_enable_busmaster(pa->pa_pc, pa->pa_tag); 876 877 /* Do base PCI setup - map BAR0 */ 878 if (ixgbe_allocate_pci_resources(adapter, pa)) { 879 aprint_error_dev(dev, "Allocation of PCI resources failed\n"); 880 error = ENXIO; 881 goto err_out; 882 } 883 884 /* let hardware know driver is loaded */ 885 ctrl_ext = IXGBE_READ_REG(hw, IXGBE_CTRL_EXT); 886 ctrl_ext |= IXGBE_CTRL_EXT_DRV_LOAD; 887 IXGBE_WRITE_REG(hw, IXGBE_CTRL_EXT, ctrl_ext); 888 889 /* 890 * Initialize the shared code 891 */ 892 if (ixgbe_init_shared_code(hw) != 0) { 893 aprint_error_dev(dev, "Unable to initialize the shared code\n"); 894 error = ENXIO; 895 goto err_out; 896 } 897 898 switch (hw->mac.type) { 899 case ixgbe_mac_82598EB: 900 str = "82598EB"; 901 break; 902 case ixgbe_mac_82599EB: 903 str = "82599EB"; 904 break; 905 case ixgbe_mac_X540: 906 str = "X540"; 907 break; 908 case ixgbe_mac_X550: 909 str = "X550"; 910 break; 911 case ixgbe_mac_X550EM_x: 912 str = "X550EM X"; 913 break; 914 case ixgbe_mac_X550EM_a: 915 str = "X550EM A"; 916 break; 917 default: 918 str = "Unknown"; 919 break; 920 } 921 aprint_normal_dev(dev, "device %s\n", str); 922 923 hw->allow_unsupported_sfp = allow_unsupported_sfp; 924 925 /* Pick up the 82599 settings */ 926 if (hw->mac.type != ixgbe_mac_82598EB) 927 hw->phy.smart_speed = ixgbe_smart_speed; 928 929 /* Set the right number of segments */ 930 KASSERT(IXGBE_82599_SCATTER_MAX >= IXGBE_SCATTER_DEFAULT); 931 adapter->num_segs = IXGBE_SCATTER_DEFAULT; 932 933 /* Ensure SW/FW semaphore is free */ 934 ixgbe_init_swfw_semaphore(hw); 935 936 hw->mac.ops.set_lan_id(hw); 937 ixgbe_init_device_features(adapter); 938 939 if (ixgbe_configure_interrupts(adapter)) { 940 error = ENXIO; 941 goto err_out; 942 } 943 944 /* Allocate multicast array memory. */ 945 adapter->mta = malloc(sizeof(*adapter->mta) * 946 MAX_NUM_MULTICAST_ADDRESSES, M_DEVBUF, M_WAITOK); 947 948 /* Enable WoL (if supported) */ 949 ixgbe_check_wol_support(adapter); 950 951 /* Register for VLAN events */ 952 ether_set_vlan_cb(&adapter->osdep.ec, ixgbe_vlan_cb); 953 954 /* Verify adapter fan is still functional (if applicable) */ 955 if (adapter->feat_en & IXGBE_FEATURE_FAN_FAIL) { 956 u32 esdp = IXGBE_READ_REG(hw, IXGBE_ESDP); 957 ixgbe_check_fan_failure(adapter, esdp, FALSE); 958 } 959 960 /* Set an initial default flow control value */ 961 hw->fc.requested_mode = ixgbe_flow_control; 962 963 /* Sysctls for limiting the amount of work done in the taskqueues */ 964 ixgbe_set_sysctl_value(adapter, "rx_processing_limit", 965 "max number of rx packets to process", 966 &adapter->rx_process_limit, ixgbe_rx_process_limit); 967 968 ixgbe_set_sysctl_value(adapter, "tx_processing_limit", 969 "max number of tx packets to process", 970 &adapter->tx_process_limit, ixgbe_tx_process_limit); 971 972 /* Do descriptor calc and sanity checks */ 973 if (((ixgbe_txd * sizeof(union ixgbe_adv_tx_desc)) % DBA_ALIGN) != 0 || 974 ixgbe_txd < MIN_TXD || ixgbe_txd > MAX_TXD) { 975 aprint_error_dev(dev, "TXD config issue, using default!\n"); 976 adapter->num_tx_desc = DEFAULT_TXD; 977 } else 978 adapter->num_tx_desc = ixgbe_txd; 979 980 if (((ixgbe_rxd * sizeof(union ixgbe_adv_rx_desc)) % DBA_ALIGN) != 0 || 981 ixgbe_rxd < MIN_RXD || ixgbe_rxd > MAX_RXD) { 982 aprint_error_dev(dev, "RXD config issue, using default!\n"); 983 adapter->num_rx_desc = DEFAULT_RXD; 984 } else 985 adapter->num_rx_desc = ixgbe_rxd; 986 987 /* Set default high limit of copying mbuf in rxeof */ 988 adapter->rx_copy_len = IXGBE_RX_COPY_LEN_MAX; 989 990 /* Allocate our TX/RX Queues */ 991 if (ixgbe_allocate_queues(adapter)) { 992 error = ENOMEM; 993 goto err_out; 994 } 995 996 hw->phy.reset_if_overtemp = TRUE; 997 error = ixgbe_reset_hw(hw); 998 hw->phy.reset_if_overtemp = FALSE; 999 if (error == IXGBE_ERR_SFP_NOT_PRESENT) 1000 error = IXGBE_SUCCESS; 1001 else if (error == IXGBE_ERR_SFP_NOT_SUPPORTED) { 1002 aprint_error_dev(dev, "Unsupported SFP+ module detected!\n"); 1003 unsupported_sfp = true; 1004 error = IXGBE_SUCCESS; 1005 } else if (error) { 1006 aprint_error_dev(dev, 1007 "Hardware initialization failed(error = %d)\n", error); 1008 error = EIO; 1009 goto err_late; 1010 } 1011 1012 /* Make sure we have a good EEPROM before we read from it */ 1013 if (ixgbe_validate_eeprom_checksum(&adapter->hw, NULL) < 0) { 1014 aprint_error_dev(dev, "The EEPROM Checksum Is Not Valid\n"); 1015 error = EIO; 1016 goto err_late; 1017 } 1018 1019 aprint_normal("%s:", device_xname(dev)); 1020 /* NVM Image Version */ 1021 high = low = 0; 1022 switch (hw->mac.type) { 1023 case ixgbe_mac_82598EB: 1024 /* 1025 * Print version from the dev starter version (0x29). The 1026 * location is the same as newer device's IXGBE_NVM_MAP_VER. 1027 */ 1028 hw->eeprom.ops.read(hw, IXGBE_NVM_MAP_VER, &nvmreg); 1029 if (nvmreg == 0xffff) 1030 break; 1031 high = (nvmreg >> 12) & 0x0f; 1032 low = (nvmreg >> 4) & 0xff; 1033 id = nvmreg & 0x0f; 1034 /* 1035 * The following output might not be correct. Some 82598 cards 1036 * have 0x1070 or 0x2090. 82598 spec update notes about 2.9.0. 1037 */ 1038 aprint_normal(" NVM Image Version %u.%u.%u,", high, low, id); 1039 break; 1040 case ixgbe_mac_X540: 1041 case ixgbe_mac_X550EM_a: 1042 hw->eeprom.ops.read(hw, IXGBE_NVM_IMAGE_VER, &nvmreg); 1043 if (nvmreg == 0xffff) 1044 break; 1045 high = (nvmreg >> 12) & 0x0f; 1046 low = (nvmreg >> 4) & 0xff; 1047 id = nvmreg & 0x0f; 1048 aprint_normal(" NVM Image Version %u.", high); 1049 if (hw->mac.type == ixgbe_mac_X540) 1050 str = "%x"; 1051 else 1052 str = "%02x"; 1053 aprint_normal(str, low); 1054 aprint_normal(" ID 0x%x,", id); 1055 break; 1056 case ixgbe_mac_X550EM_x: 1057 case ixgbe_mac_X550: 1058 hw->eeprom.ops.read(hw, IXGBE_NVM_IMAGE_VER, &nvmreg); 1059 if (nvmreg == 0xffff) 1060 break; 1061 high = (nvmreg >> 12) & 0x0f; 1062 low = nvmreg & 0xff; 1063 aprint_normal(" NVM Image Version %u.%02x,", high, low); 1064 break; 1065 default: 1066 break; 1067 } 1068 hw->eeprom.nvm_image_ver_high = high; 1069 hw->eeprom.nvm_image_ver_low = low; 1070 1071 /* PHY firmware revision */ 1072 switch (hw->mac.type) { 1073 case ixgbe_mac_X540: 1074 case ixgbe_mac_X550: 1075 hw->eeprom.ops.read(hw, IXGBE_PHYFW_REV, &nvmreg); 1076 if (nvmreg == 0xffff) 1077 break; 1078 high = (nvmreg >> 12) & 0x0f; 1079 low = (nvmreg >> 4) & 0xff; 1080 id = nvmreg & 0x000f; 1081 aprint_normal(" PHY FW Revision %u.", high); 1082 if (hw->mac.type == ixgbe_mac_X540) 1083 str = "%x"; 1084 else 1085 str = "%02x"; 1086 aprint_normal(str, low); 1087 aprint_normal(" ID 0x%x,", id); 1088 break; 1089 default: 1090 break; 1091 } 1092 1093 /* NVM Map version & OEM NVM Image version */ 1094 switch (hw->mac.type) { 1095 case ixgbe_mac_X550: 1096 case ixgbe_mac_X550EM_x: 1097 case ixgbe_mac_X550EM_a: 1098 hw->eeprom.ops.read(hw, IXGBE_NVM_MAP_VER, &nvmreg); 1099 if (nvmreg != 0xffff) { 1100 high = (nvmreg >> 12) & 0x0f; 1101 low = nvmreg & 0x00ff; 1102 aprint_normal(" NVM Map version %u.%02x,", high, low); 1103 } 1104 hw->eeprom.ops.read(hw, IXGBE_OEM_NVM_IMAGE_VER, &nvmreg); 1105 if (nvmreg != 0xffff) { 1106 high = (nvmreg >> 12) & 0x0f; 1107 low = nvmreg & 0x00ff; 1108 aprint_verbose(" OEM NVM Image version %u.%02x,", high, 1109 low); 1110 } 1111 break; 1112 default: 1113 break; 1114 } 1115 1116 /* Print the ETrackID */ 1117 hw->eeprom.ops.read(hw, IXGBE_ETRACKID_H, &high); 1118 hw->eeprom.ops.read(hw, IXGBE_ETRACKID_L, &low); 1119 aprint_normal(" ETrackID %08x\n", ((uint32_t)high << 16) | low); 1120 1121 /* Printed Board Assembly number */ 1122 error = ixgbe_read_pba_string(hw, buf, IXGBE_PBANUM_LENGTH); 1123 aprint_normal_dev(dev, "PBA number %s\n", error ? "unknown" : buf); 1124 1125 if (adapter->feat_en & IXGBE_FEATURE_MSIX) { 1126 error = ixgbe_allocate_msix(adapter, pa); 1127 if (error) { 1128 /* Free allocated queue structures first */ 1129 ixgbe_free_queues(adapter); 1130 1131 /* Fallback to legacy interrupt */ 1132 adapter->feat_en &= ~IXGBE_FEATURE_MSIX; 1133 if (adapter->feat_cap & IXGBE_FEATURE_MSI) 1134 adapter->feat_en |= IXGBE_FEATURE_MSI; 1135 adapter->num_queues = 1; 1136 1137 /* Allocate our TX/RX Queues again */ 1138 if (ixgbe_allocate_queues(adapter)) { 1139 error = ENOMEM; 1140 goto err_out; 1141 } 1142 } 1143 } 1144 1145 /* Recovery mode */ 1146 switch (adapter->hw.mac.type) { 1147 case ixgbe_mac_X550: 1148 case ixgbe_mac_X550EM_x: 1149 case ixgbe_mac_X550EM_a: 1150 /* >= 2.00 */ 1151 if (hw->eeprom.nvm_image_ver_high >= 2) { 1152 adapter->feat_cap |= IXGBE_FEATURE_RECOVERY_MODE; 1153 adapter->feat_en |= IXGBE_FEATURE_RECOVERY_MODE; 1154 } 1155 break; 1156 default: 1157 break; 1158 } 1159 1160 if ((adapter->feat_en & IXGBE_FEATURE_MSIX) == 0) 1161 error = ixgbe_allocate_legacy(adapter, pa); 1162 if (error) 1163 goto err_late; 1164 1165 /* Tasklets for Link, SFP, Multispeed Fiber and Flow Director */ 1166 mutex_init(&(adapter)->admin_mtx, MUTEX_DEFAULT, IPL_NET); 1167 snprintf(wqname, sizeof(wqname), "%s-admin", device_xname(dev)); 1168 error = workqueue_create(&adapter->admin_wq, wqname, 1169 ixgbe_handle_admin, adapter, IXGBE_WORKQUEUE_PRI, IPL_NET, 1170 IXGBE_TASKLET_WQ_FLAGS); 1171 if (error) { 1172 aprint_error_dev(dev, 1173 "could not create admin workqueue (%d)\n", error); 1174 goto err_out; 1175 } 1176 1177 error = ixgbe_start_hw(hw); 1178 switch (error) { 1179 case IXGBE_ERR_EEPROM_VERSION: 1180 aprint_error_dev(dev, "This device is a pre-production adapter/" 1181 "LOM. Please be aware there may be issues associated " 1182 "with your hardware.\nIf you are experiencing problems " 1183 "please contact your Intel or hardware representative " 1184 "who provided you with this hardware.\n"); 1185 break; 1186 default: 1187 break; 1188 } 1189 1190 /* Setup OS specific network interface */ 1191 if (ixgbe_setup_interface(dev, adapter) != 0) 1192 goto err_late; 1193 1194 /* 1195 * Print PHY ID only for copper PHY. On device which has SFP(+) cage 1196 * and a module is inserted, phy.id is not MII PHY id but SFF 8024 ID. 1197 */ 1198 if (hw->phy.media_type == ixgbe_media_type_copper) { 1199 uint16_t id1, id2; 1200 int oui, model, rev; 1201 char descr[MII_MAX_DESCR_LEN]; 1202 1203 id1 = hw->phy.id >> 16; 1204 id2 = hw->phy.id & 0xffff; 1205 oui = MII_OUI(id1, id2); 1206 model = MII_MODEL(id2); 1207 rev = MII_REV(id2); 1208 mii_get_descr(descr, sizeof(descr), oui, model); 1209 if (descr[0]) 1210 aprint_normal_dev(dev, "PHY: %s, rev. %d\n", 1211 descr, rev); 1212 else 1213 aprint_normal_dev(dev, 1214 "PHY OUI 0x%06x, model 0x%04x, rev. %d\n", 1215 oui, model, rev); 1216 } 1217 1218 /* Enable EEE power saving */ 1219 if (adapter->feat_cap & IXGBE_FEATURE_EEE) 1220 hw->mac.ops.setup_eee(hw, 1221 adapter->feat_en & IXGBE_FEATURE_EEE); 1222 1223 /* Enable power to the phy. */ 1224 if (!unsupported_sfp) { 1225 /* Enable the optics for 82599 SFP+ fiber */ 1226 ixgbe_enable_tx_laser(hw); 1227 1228 /* 1229 * XXX Currently, ixgbe_set_phy_power() supports only copper 1230 * PHY, so it's not required to test with !unsupported_sfp. 1231 */ 1232 ixgbe_set_phy_power(hw, TRUE); 1233 } 1234 1235 /* Initialize statistics */ 1236 ixgbe_update_stats_counters(adapter); 1237 1238 /* Check PCIE slot type/speed/width */ 1239 ixgbe_get_slot_info(adapter); 1240 1241 /* 1242 * Do time init and sysctl init here, but 1243 * only on the first port of a bypass adapter. 1244 */ 1245 ixgbe_bypass_init(adapter); 1246 1247 /* Set an initial dmac value */ 1248 adapter->dmac = 0; 1249 /* Set initial advertised speeds (if applicable) */ 1250 adapter->advertise = ixgbe_get_default_advertise(adapter); 1251 1252 if (adapter->feat_cap & IXGBE_FEATURE_SRIOV) 1253 ixgbe_define_iov_schemas(dev, &error); 1254 1255 /* Add sysctls */ 1256 ixgbe_add_device_sysctls(adapter); 1257 ixgbe_add_hw_stats(adapter); 1258 1259 /* For Netmap */ 1260 adapter->init_locked = ixgbe_init_locked; 1261 adapter->stop_locked = ixgbe_stop_locked; 1262 1263 if (adapter->feat_en & IXGBE_FEATURE_NETMAP) 1264 ixgbe_netmap_attach(adapter); 1265 1266 snprintb(buf, sizeof(buf), IXGBE_FEATURE_FLAGS, adapter->feat_cap); 1267 aprint_verbose_dev(dev, "feature cap %s\n", buf); 1268 snprintb(buf, sizeof(buf), IXGBE_FEATURE_FLAGS, adapter->feat_en); 1269 aprint_verbose_dev(dev, "feature ena %s\n", buf); 1270 1271 if (pmf_device_register(dev, ixgbe_suspend, ixgbe_resume)) 1272 pmf_class_network_register(dev, adapter->ifp); 1273 else 1274 aprint_error_dev(dev, "couldn't establish power handler\n"); 1275 1276 /* Init recovery mode timer and state variable */ 1277 if (adapter->feat_en & IXGBE_FEATURE_RECOVERY_MODE) { 1278 adapter->recovery_mode = 0; 1279 1280 /* Set up the timer callout */ 1281 callout_init(&adapter->recovery_mode_timer, 1282 IXGBE_CALLOUT_FLAGS); 1283 snprintf(wqname, sizeof(wqname), "%s-recovery", 1284 device_xname(dev)); 1285 error = workqueue_create(&adapter->recovery_mode_timer_wq, 1286 wqname, ixgbe_handle_recovery_mode_timer, adapter, 1287 IXGBE_WORKQUEUE_PRI, IPL_NET, IXGBE_TASKLET_WQ_FLAGS); 1288 if (error) { 1289 aprint_error_dev(dev, "could not create " 1290 "recovery_mode_timer workqueue (%d)\n", error); 1291 goto err_out; 1292 } 1293 1294 /* Start the task */ 1295 callout_reset(&adapter->recovery_mode_timer, hz, 1296 ixgbe_recovery_mode_timer, adapter); 1297 } 1298 1299 INIT_DEBUGOUT("ixgbe_attach: end"); 1300 adapter->osdep.attached = true; 1301 1302 return; 1303 1304 err_late: 1305 ixgbe_free_queues(adapter); 1306 err_out: 1307 ctrl_ext = IXGBE_READ_REG(&adapter->hw, IXGBE_CTRL_EXT); 1308 ctrl_ext &= ~IXGBE_CTRL_EXT_DRV_LOAD; 1309 IXGBE_WRITE_REG(&adapter->hw, IXGBE_CTRL_EXT, ctrl_ext); 1310 ixgbe_free_deferred_handlers(adapter); 1311 ixgbe_free_pci_resources(adapter); 1312 if (adapter->mta != NULL) 1313 free(adapter->mta, M_DEVBUF); 1314 mutex_destroy(&(adapter)->admin_mtx); /* XXX appropriate order? */ 1315 IXGBE_CORE_LOCK_DESTROY(adapter); 1316 1317 return; 1318 } /* ixgbe_attach */ 1319 1320 /************************************************************************ 1321 * ixgbe_check_wol_support 1322 * 1323 * Checks whether the adapter's ports are capable of 1324 * Wake On LAN by reading the adapter's NVM. 1325 * 1326 * Sets each port's hw->wol_enabled value depending 1327 * on the value read here. 1328 ************************************************************************/ 1329 static void 1330 ixgbe_check_wol_support(struct adapter *adapter) 1331 { 1332 struct ixgbe_hw *hw = &adapter->hw; 1333 u16 dev_caps = 0; 1334 1335 /* Find out WoL support for port */ 1336 adapter->wol_support = hw->wol_enabled = 0; 1337 ixgbe_get_device_caps(hw, &dev_caps); 1338 if ((dev_caps & IXGBE_DEVICE_CAPS_WOL_PORT0_1) || 1339 ((dev_caps & IXGBE_DEVICE_CAPS_WOL_PORT0) && 1340 hw->bus.func == 0)) 1341 adapter->wol_support = hw->wol_enabled = 1; 1342 1343 /* Save initial wake up filter configuration */ 1344 adapter->wufc = IXGBE_READ_REG(hw, IXGBE_WUFC); 1345 1346 return; 1347 } /* ixgbe_check_wol_support */ 1348 1349 /************************************************************************ 1350 * ixgbe_setup_interface 1351 * 1352 * Setup networking device structure and register an interface. 1353 ************************************************************************/ 1354 static int 1355 ixgbe_setup_interface(device_t dev, struct adapter *adapter) 1356 { 1357 struct ethercom *ec = &adapter->osdep.ec; 1358 struct ifnet *ifp; 1359 1360 INIT_DEBUGOUT("ixgbe_setup_interface: begin"); 1361 1362 ifp = adapter->ifp = &ec->ec_if; 1363 strlcpy(ifp->if_xname, device_xname(dev), IFNAMSIZ); 1364 ifp->if_baudrate = IF_Gbps(10); 1365 ifp->if_init = ixgbe_init; 1366 ifp->if_stop = ixgbe_ifstop; 1367 ifp->if_softc = adapter; 1368 ifp->if_flags = IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST; 1369 #ifdef IXGBE_MPSAFE 1370 ifp->if_extflags = IFEF_MPSAFE; 1371 #endif 1372 ifp->if_ioctl = ixgbe_ioctl; 1373 #if __FreeBSD_version >= 1100045 1374 /* TSO parameters */ 1375 ifp->if_hw_tsomax = 65518; 1376 ifp->if_hw_tsomaxsegcount = IXGBE_82599_SCATTER; 1377 ifp->if_hw_tsomaxsegsize = 2048; 1378 #endif 1379 if (adapter->feat_en & IXGBE_FEATURE_LEGACY_TX) { 1380 #if 0 1381 ixgbe_start_locked = ixgbe_legacy_start_locked; 1382 #endif 1383 } else { 1384 ifp->if_transmit = ixgbe_mq_start; 1385 #if 0 1386 ixgbe_start_locked = ixgbe_mq_start_locked; 1387 #endif 1388 } 1389 ifp->if_start = ixgbe_legacy_start; 1390 IFQ_SET_MAXLEN(&ifp->if_snd, adapter->num_tx_desc - 2); 1391 IFQ_SET_READY(&ifp->if_snd); 1392 1393 if_initialize(ifp); 1394 adapter->ipq = if_percpuq_create(&adapter->osdep.ec.ec_if); 1395 ether_ifattach(ifp, adapter->hw.mac.addr); 1396 aprint_normal_dev(dev, "Ethernet address %s\n", 1397 ether_sprintf(adapter->hw.mac.addr)); 1398 /* 1399 * We use per TX queue softint, so if_deferred_start_init() isn't 1400 * used. 1401 */ 1402 ether_set_ifflags_cb(ec, ixgbe_ifflags_cb); 1403 1404 adapter->max_frame_size = ifp->if_mtu + ETHER_HDR_LEN + ETHER_CRC_LEN; 1405 1406 /* 1407 * Tell the upper layer(s) we support long frames. 1408 */ 1409 ifp->if_hdrlen = sizeof(struct ether_vlan_header); 1410 1411 /* Set capability flags */ 1412 ifp->if_capabilities |= IFCAP_RXCSUM 1413 | IFCAP_TXCSUM 1414 | IFCAP_TSOv4 1415 | IFCAP_TSOv6; 1416 ifp->if_capenable = 0; 1417 1418 ec->ec_capabilities |= ETHERCAP_VLAN_HWTAGGING 1419 | ETHERCAP_VLAN_HWCSUM 1420 | ETHERCAP_JUMBO_MTU 1421 | ETHERCAP_VLAN_MTU; 1422 1423 /* Enable the above capabilities by default */ 1424 ec->ec_capenable = ec->ec_capabilities; 1425 1426 /* 1427 * Don't turn this on by default, if vlans are 1428 * created on another pseudo device (eg. lagg) 1429 * then vlan events are not passed thru, breaking 1430 * operation, but with HW FILTER off it works. If 1431 * using vlans directly on the ixgbe driver you can 1432 * enable this and get full hardware tag filtering. 1433 */ 1434 ec->ec_capabilities |= ETHERCAP_VLAN_HWFILTER; 1435 1436 /* 1437 * Specify the media types supported by this adapter and register 1438 * callbacks to update media and link information 1439 */ 1440 ec->ec_ifmedia = &adapter->media; 1441 ifmedia_init_with_lock(&adapter->media, IFM_IMASK, ixgbe_media_change, 1442 ixgbe_media_status, &adapter->core_mtx); 1443 1444 adapter->phy_layer = ixgbe_get_supported_physical_layer(&adapter->hw); 1445 ixgbe_add_media_types(adapter); 1446 1447 /* Set autoselect media by default */ 1448 ifmedia_set(&adapter->media, IFM_ETHER | IFM_AUTO); 1449 1450 if_register(ifp); 1451 1452 return (0); 1453 } /* ixgbe_setup_interface */ 1454 1455 /************************************************************************ 1456 * ixgbe_add_media_types 1457 ************************************************************************/ 1458 static void 1459 ixgbe_add_media_types(struct adapter *adapter) 1460 { 1461 struct ixgbe_hw *hw = &adapter->hw; 1462 u64 layer; 1463 1464 layer = adapter->phy_layer; 1465 1466 #define ADD(mm, dd) \ 1467 ifmedia_add(&adapter->media, IFM_ETHER | (mm), (dd), NULL); 1468 1469 ADD(IFM_NONE, 0); 1470 1471 /* Media types with matching NetBSD media defines */ 1472 if (layer & IXGBE_PHYSICAL_LAYER_10GBASE_T) { 1473 ADD(IFM_10G_T | IFM_FDX, 0); 1474 } 1475 if (layer & IXGBE_PHYSICAL_LAYER_1000BASE_T) { 1476 ADD(IFM_1000_T | IFM_FDX, 0); 1477 } 1478 if (layer & IXGBE_PHYSICAL_LAYER_100BASE_TX) { 1479 ADD(IFM_100_TX | IFM_FDX, 0); 1480 } 1481 if (layer & IXGBE_PHYSICAL_LAYER_10BASE_T) { 1482 ADD(IFM_10_T | IFM_FDX, 0); 1483 } 1484 1485 if (layer & IXGBE_PHYSICAL_LAYER_SFP_PLUS_CU || 1486 layer & IXGBE_PHYSICAL_LAYER_SFP_ACTIVE_DA) { 1487 ADD(IFM_10G_TWINAX | IFM_FDX, 0); 1488 } 1489 1490 if (layer & IXGBE_PHYSICAL_LAYER_10GBASE_LR) { 1491 ADD(IFM_10G_LR | IFM_FDX, 0); 1492 if (hw->phy.multispeed_fiber) { 1493 ADD(IFM_1000_LX | IFM_FDX, 0); 1494 } 1495 } 1496 if (layer & IXGBE_PHYSICAL_LAYER_10GBASE_SR) { 1497 ADD(IFM_10G_SR | IFM_FDX, 0); 1498 if (hw->phy.multispeed_fiber) { 1499 ADD(IFM_1000_SX | IFM_FDX, 0); 1500 } 1501 } else if (layer & IXGBE_PHYSICAL_LAYER_1000BASE_SX) { 1502 ADD(IFM_1000_SX | IFM_FDX, 0); 1503 } 1504 if (layer & IXGBE_PHYSICAL_LAYER_10GBASE_CX4) { 1505 ADD(IFM_10G_CX4 | IFM_FDX, 0); 1506 } 1507 1508 if (layer & IXGBE_PHYSICAL_LAYER_10GBASE_KR) { 1509 ADD(IFM_10G_KR | IFM_FDX, 0); 1510 } 1511 if (layer & IXGBE_PHYSICAL_LAYER_10GBASE_KX4) { 1512 ADD(IFM_10G_KX4 | IFM_FDX, 0); 1513 } 1514 if (layer & IXGBE_PHYSICAL_LAYER_1000BASE_KX) { 1515 ADD(IFM_1000_KX | IFM_FDX, 0); 1516 } 1517 if (layer & IXGBE_PHYSICAL_LAYER_2500BASE_KX) { 1518 ADD(IFM_2500_KX | IFM_FDX, 0); 1519 } 1520 if (layer & IXGBE_PHYSICAL_LAYER_2500BASE_T) { 1521 ADD(IFM_2500_T | IFM_FDX, 0); 1522 } 1523 if (layer & IXGBE_PHYSICAL_LAYER_5GBASE_T) { 1524 ADD(IFM_5000_T | IFM_FDX, 0); 1525 } 1526 if (layer & IXGBE_PHYSICAL_LAYER_1000BASE_BX) 1527 ADD(IFM_1000_BX10 | IFM_FDX, 0); 1528 /* XXX no ifmedia_set? */ 1529 1530 ADD(IFM_AUTO, 0); 1531 1532 #undef ADD 1533 } /* ixgbe_add_media_types */ 1534 1535 /************************************************************************ 1536 * ixgbe_is_sfp 1537 ************************************************************************/ 1538 static inline bool 1539 ixgbe_is_sfp(struct ixgbe_hw *hw) 1540 { 1541 switch (hw->mac.type) { 1542 case ixgbe_mac_82598EB: 1543 if (hw->phy.type == ixgbe_phy_nl) 1544 return (TRUE); 1545 return (FALSE); 1546 case ixgbe_mac_82599EB: 1547 case ixgbe_mac_X550EM_x: 1548 case ixgbe_mac_X550EM_a: 1549 switch (hw->mac.ops.get_media_type(hw)) { 1550 case ixgbe_media_type_fiber: 1551 case ixgbe_media_type_fiber_qsfp: 1552 return (TRUE); 1553 default: 1554 return (FALSE); 1555 } 1556 default: 1557 return (FALSE); 1558 } 1559 } /* ixgbe_is_sfp */ 1560 1561 static void 1562 ixgbe_schedule_admin_tasklet(struct adapter *adapter) 1563 { 1564 1565 KASSERT(mutex_owned(&adapter->admin_mtx)); 1566 1567 if (__predict_true(adapter->osdep.detaching == false)) { 1568 if (adapter->admin_pending == 0) 1569 workqueue_enqueue(adapter->admin_wq, 1570 &adapter->admin_wc, NULL); 1571 adapter->admin_pending = 1; 1572 } 1573 } 1574 1575 /************************************************************************ 1576 * ixgbe_config_link 1577 ************************************************************************/ 1578 static void 1579 ixgbe_config_link(struct adapter *adapter) 1580 { 1581 struct ixgbe_hw *hw = &adapter->hw; 1582 u32 autoneg, err = 0; 1583 u32 task_requests = 0; 1584 bool sfp, negotiate = false; 1585 1586 sfp = ixgbe_is_sfp(hw); 1587 1588 if (sfp) { 1589 if (hw->phy.multispeed_fiber) { 1590 ixgbe_enable_tx_laser(hw); 1591 task_requests |= IXGBE_REQUEST_TASK_MSF_WOI; 1592 } 1593 task_requests |= IXGBE_REQUEST_TASK_MOD_WOI; 1594 1595 mutex_enter(&adapter->admin_mtx); 1596 adapter->task_requests |= task_requests; 1597 ixgbe_schedule_admin_tasklet(adapter); 1598 mutex_exit(&adapter->admin_mtx); 1599 } else { 1600 struct ifmedia *ifm = &adapter->media; 1601 1602 if (hw->mac.ops.check_link) 1603 err = ixgbe_check_link(hw, &adapter->link_speed, 1604 &adapter->link_up, FALSE); 1605 if (err) 1606 return; 1607 1608 /* 1609 * Check if it's the first call. If it's the first call, 1610 * get value for auto negotiation. 1611 */ 1612 autoneg = hw->phy.autoneg_advertised; 1613 if ((IFM_SUBTYPE(ifm->ifm_cur->ifm_media) != IFM_NONE) 1614 && ((!autoneg) && (hw->mac.ops.get_link_capabilities))) 1615 err = hw->mac.ops.get_link_capabilities(hw, &autoneg, 1616 &negotiate); 1617 if (err) 1618 return; 1619 if (hw->mac.ops.setup_link) 1620 err = hw->mac.ops.setup_link(hw, autoneg, 1621 adapter->link_up); 1622 } 1623 1624 } /* ixgbe_config_link */ 1625 1626 /************************************************************************ 1627 * ixgbe_update_stats_counters - Update board statistics counters. 1628 ************************************************************************/ 1629 static void 1630 ixgbe_update_stats_counters(struct adapter *adapter) 1631 { 1632 struct ifnet *ifp = adapter->ifp; 1633 struct ixgbe_hw *hw = &adapter->hw; 1634 struct ixgbe_hw_stats *stats = &adapter->stats.pf; 1635 u32 missed_rx = 0, bprc, lxontxc, lxofftxc; 1636 u64 total, total_missed_rx = 0; 1637 uint64_t crcerrs, illerrc, rlec, ruc, rfc, roc, rjc; 1638 unsigned int queue_counters; 1639 int i; 1640 1641 IXGBE_EVC_REGADD2(hw, stats, IXGBE_CRCERRS, crcerrs); 1642 IXGBE_EVC_REGADD2(hw, stats, IXGBE_ILLERRC, illerrc); 1643 1644 IXGBE_EVC_REGADD(hw, stats, IXGBE_ERRBC, errbc); 1645 IXGBE_EVC_REGADD(hw, stats, IXGBE_MSPDC, mspdc); 1646 if (hw->mac.type >= ixgbe_mac_X550) 1647 IXGBE_EVC_REGADD(hw, stats, IXGBE_MBSDC, mbsdc); 1648 1649 /* 16 registers exist */ 1650 queue_counters = uimin(__arraycount(stats->qprc), adapter->num_queues); 1651 for (i = 0; i < queue_counters; i++) { 1652 IXGBE_EVC_REGADD(hw, stats, IXGBE_QPRC(i), qprc[i]); 1653 IXGBE_EVC_REGADD(hw, stats, IXGBE_QPTC(i), qptc[i]); 1654 if (hw->mac.type >= ixgbe_mac_82599EB) 1655 IXGBE_EVC_REGADD(hw, stats, IXGBE_QPRDC(i), qprdc[i]); 1656 } 1657 1658 /* 8 registers exist */ 1659 for (i = 0; i < IXGBE_TC_COUNTER_NUM; i++) { 1660 uint32_t mp; 1661 1662 /* MPC */ 1663 mp = IXGBE_READ_REG(hw, IXGBE_MPC(i)); 1664 /* global total per queue */ 1665 IXGBE_EVC_ADD(&stats->mpc[i], mp); 1666 /* running comprehensive total for stats display */ 1667 total_missed_rx += mp; 1668 1669 if (hw->mac.type == ixgbe_mac_82598EB) 1670 IXGBE_EVC_REGADD(hw, stats, IXGBE_RNBC(i), rnbc[i]); 1671 1672 IXGBE_EVC_REGADD(hw, stats, IXGBE_PXONTXC(i), pxontxc[i]); 1673 IXGBE_EVC_REGADD(hw, stats, IXGBE_PXOFFTXC(i), pxofftxc[i]); 1674 if (hw->mac.type >= ixgbe_mac_82599EB) { 1675 IXGBE_EVC_REGADD(hw, stats, IXGBE_PXONRXCNT(i), pxonrxc[i]); 1676 IXGBE_EVC_REGADD(hw, stats, IXGBE_PXOFFRXCNT(i), pxoffrxc[i]); 1677 IXGBE_EVC_REGADD(hw, stats, IXGBE_PXON2OFFCNT(i), 1678 pxon2offc[i]); 1679 } else { 1680 IXGBE_EVC_REGADD(hw, stats, IXGBE_PXONRXC(i), pxonrxc[i]); 1681 IXGBE_EVC_REGADD(hw, stats, IXGBE_PXOFFRXC(i), pxoffrxc[i]); 1682 } 1683 } 1684 IXGBE_EVC_ADD(&stats->mpctotal, total_missed_rx); 1685 1686 /* Document says M[LR]FC are valid when link is up and 10Gbps */ 1687 if ((adapter->link_active == LINK_STATE_UP) 1688 && (adapter->link_speed == IXGBE_LINK_SPEED_10GB_FULL)) { 1689 IXGBE_EVC_REGADD(hw, stats, IXGBE_MLFC, mlfc); 1690 IXGBE_EVC_REGADD(hw, stats, IXGBE_MRFC, mrfc); 1691 } 1692 IXGBE_EVC_REGADD2(hw, stats, IXGBE_RLEC, rlec); 1693 1694 /* Hardware workaround, gprc counts missed packets */ 1695 IXGBE_EVC_ADD(&stats->gprc, 1696 IXGBE_READ_REG(hw, IXGBE_GPRC) - missed_rx); 1697 1698 IXGBE_EVC_REGADD2(hw, stats, IXGBE_LXONTXC, lxontxc); 1699 IXGBE_EVC_REGADD2(hw, stats, IXGBE_LXOFFTXC, lxofftxc); 1700 total = lxontxc + lxofftxc; 1701 1702 if (hw->mac.type != ixgbe_mac_82598EB) { 1703 IXGBE_EVC_ADD(&stats->gorc, IXGBE_READ_REG(hw, IXGBE_GORCL) + 1704 ((u64)IXGBE_READ_REG(hw, IXGBE_GORCH) << 32)); 1705 IXGBE_EVC_ADD(&stats->gotc, IXGBE_READ_REG(hw, IXGBE_GOTCL) + 1706 ((u64)IXGBE_READ_REG(hw, IXGBE_GOTCH) << 32) 1707 - total * ETHER_MIN_LEN); 1708 IXGBE_EVC_ADD(&stats->tor, IXGBE_READ_REG(hw, IXGBE_TORL) + 1709 ((u64)IXGBE_READ_REG(hw, IXGBE_TORH) << 32)); 1710 IXGBE_EVC_REGADD(hw, stats, IXGBE_LXONRXCNT, lxonrxc); 1711 IXGBE_EVC_REGADD(hw, stats, IXGBE_LXOFFRXCNT, lxoffrxc); 1712 } else { 1713 IXGBE_EVC_REGADD(hw, stats, IXGBE_LXONRXC, lxonrxc); 1714 IXGBE_EVC_REGADD(hw, stats, IXGBE_LXOFFRXC, lxoffrxc); 1715 /* 82598 only has a counter in the high register */ 1716 IXGBE_EVC_REGADD(hw, stats, IXGBE_GORCH, gorc); 1717 IXGBE_EVC_ADD(&stats->gotc, IXGBE_READ_REG(hw, IXGBE_GOTCH) 1718 - total * ETHER_MIN_LEN); 1719 IXGBE_EVC_REGADD(hw, stats, IXGBE_TORH, tor); 1720 } 1721 1722 /* 1723 * Workaround: mprc hardware is incorrectly counting 1724 * broadcasts, so for now we subtract those. 1725 */ 1726 IXGBE_EVC_REGADD2(hw, stats, IXGBE_BPRC, bprc); 1727 IXGBE_EVC_ADD(&stats->mprc, IXGBE_READ_REG(hw, IXGBE_MPRC) 1728 - ((hw->mac.type == ixgbe_mac_82598EB) ? bprc : 0)); 1729 1730 IXGBE_EVC_REGADD(hw, stats, IXGBE_PRC64, prc64); 1731 IXGBE_EVC_REGADD(hw, stats, IXGBE_PRC127, prc127); 1732 IXGBE_EVC_REGADD(hw, stats, IXGBE_PRC255, prc255); 1733 IXGBE_EVC_REGADD(hw, stats, IXGBE_PRC511, prc511); 1734 IXGBE_EVC_REGADD(hw, stats, IXGBE_PRC1023, prc1023); 1735 IXGBE_EVC_REGADD(hw, stats, IXGBE_PRC1522, prc1522); 1736 1737 IXGBE_EVC_ADD(&stats->gptc, IXGBE_READ_REG(hw, IXGBE_GPTC) - total); 1738 IXGBE_EVC_ADD(&stats->mptc, IXGBE_READ_REG(hw, IXGBE_MPTC) - total); 1739 IXGBE_EVC_ADD(&stats->ptc64, IXGBE_READ_REG(hw, IXGBE_PTC64) - total); 1740 1741 IXGBE_EVC_REGADD2(hw, stats, IXGBE_RUC, ruc); 1742 IXGBE_EVC_REGADD2(hw, stats, IXGBE_RFC, rfc); 1743 IXGBE_EVC_REGADD2(hw, stats, IXGBE_ROC, roc); 1744 IXGBE_EVC_REGADD2(hw, stats, IXGBE_RJC, rjc); 1745 1746 IXGBE_EVC_REGADD(hw, stats, IXGBE_MNGPRC, mngprc); 1747 IXGBE_EVC_REGADD(hw, stats, IXGBE_MNGPDC, mngpdc); 1748 IXGBE_EVC_REGADD(hw, stats, IXGBE_MNGPTC, mngptc); 1749 IXGBE_EVC_REGADD(hw, stats, IXGBE_TPR, tpr); 1750 IXGBE_EVC_REGADD(hw, stats, IXGBE_TPT, tpt); 1751 IXGBE_EVC_REGADD(hw, stats, IXGBE_PTC127, ptc127); 1752 IXGBE_EVC_REGADD(hw, stats, IXGBE_PTC255, ptc255); 1753 IXGBE_EVC_REGADD(hw, stats, IXGBE_PTC511, ptc511); 1754 IXGBE_EVC_REGADD(hw, stats, IXGBE_PTC1023, ptc1023); 1755 IXGBE_EVC_REGADD(hw, stats, IXGBE_PTC1522, ptc1522); 1756 IXGBE_EVC_REGADD(hw, stats, IXGBE_BPTC, bptc); 1757 IXGBE_EVC_REGADD(hw, stats, IXGBE_XEC, xec); 1758 IXGBE_EVC_REGADD(hw, stats, IXGBE_FCCRC, fccrc); 1759 IXGBE_EVC_REGADD(hw, stats, IXGBE_FCLAST, fclast); 1760 /* Only read FCOE on 82599 */ 1761 if (hw->mac.type != ixgbe_mac_82598EB) { 1762 IXGBE_EVC_REGADD(hw, stats, IXGBE_FCOERPDC, fcoerpdc); 1763 IXGBE_EVC_REGADD(hw, stats, IXGBE_FCOEPRC, fcoeprc); 1764 IXGBE_EVC_REGADD(hw, stats, IXGBE_FCOEPTC, fcoeptc); 1765 IXGBE_EVC_REGADD(hw, stats, IXGBE_FCOEDWRC, fcoedwrc); 1766 IXGBE_EVC_REGADD(hw, stats, IXGBE_FCOEDWTC, fcoedwtc); 1767 } 1768 1769 /* 1770 * Fill out the OS statistics structure. Only RX errors are required 1771 * here because all TX counters are incremented in the TX path and 1772 * normal RX counters are prepared in ether_input(). 1773 */ 1774 net_stat_ref_t nsr = IF_STAT_GETREF(ifp); 1775 if_statadd_ref(nsr, if_iqdrops, total_missed_rx); 1776 1777 /* 1778 * Aggregate following types of errors as RX errors: 1779 * - CRC error count, 1780 * - illegal byte error count, 1781 * - length error count, 1782 * - undersized packets count, 1783 * - fragmented packets count, 1784 * - oversized packets count, 1785 * - jabber count. 1786 */ 1787 if_statadd_ref(nsr, if_ierrors, 1788 crcerrs + illerrc + rlec + ruc + rfc + roc + rjc); 1789 1790 IF_STAT_PUTREF(ifp); 1791 } /* ixgbe_update_stats_counters */ 1792 1793 /************************************************************************ 1794 * ixgbe_add_hw_stats 1795 * 1796 * Add sysctl variables, one per statistic, to the system. 1797 ************************************************************************/ 1798 static void 1799 ixgbe_add_hw_stats(struct adapter *adapter) 1800 { 1801 device_t dev = adapter->dev; 1802 const struct sysctlnode *rnode, *cnode; 1803 struct sysctllog **log = &adapter->sysctllog; 1804 struct tx_ring *txr = adapter->tx_rings; 1805 struct rx_ring *rxr = adapter->rx_rings; 1806 struct ixgbe_hw *hw = &adapter->hw; 1807 struct ixgbe_hw_stats *stats = &adapter->stats.pf; 1808 const char *xname = device_xname(dev); 1809 int i; 1810 1811 /* Driver Statistics */ 1812 evcnt_attach_dynamic(&adapter->efbig_tx_dma_setup, EVCNT_TYPE_MISC, 1813 NULL, xname, "Driver tx dma soft fail EFBIG"); 1814 evcnt_attach_dynamic(&adapter->mbuf_defrag_failed, EVCNT_TYPE_MISC, 1815 NULL, xname, "m_defrag() failed"); 1816 evcnt_attach_dynamic(&adapter->efbig2_tx_dma_setup, EVCNT_TYPE_MISC, 1817 NULL, xname, "Driver tx dma hard fail EFBIG"); 1818 evcnt_attach_dynamic(&adapter->einval_tx_dma_setup, EVCNT_TYPE_MISC, 1819 NULL, xname, "Driver tx dma hard fail EINVAL"); 1820 evcnt_attach_dynamic(&adapter->other_tx_dma_setup, EVCNT_TYPE_MISC, 1821 NULL, xname, "Driver tx dma hard fail other"); 1822 evcnt_attach_dynamic(&adapter->eagain_tx_dma_setup, EVCNT_TYPE_MISC, 1823 NULL, xname, "Driver tx dma soft fail EAGAIN"); 1824 evcnt_attach_dynamic(&adapter->enomem_tx_dma_setup, EVCNT_TYPE_MISC, 1825 NULL, xname, "Driver tx dma soft fail ENOMEM"); 1826 evcnt_attach_dynamic(&adapter->watchdog_events, EVCNT_TYPE_MISC, 1827 NULL, xname, "Watchdog timeouts"); 1828 evcnt_attach_dynamic(&adapter->tso_err, EVCNT_TYPE_MISC, 1829 NULL, xname, "TSO errors"); 1830 evcnt_attach_dynamic(&adapter->admin_irqev, EVCNT_TYPE_INTR, 1831 NULL, xname, "Admin MSI-X IRQ Handled"); 1832 evcnt_attach_dynamic(&adapter->link_workev, EVCNT_TYPE_INTR, 1833 NULL, xname, "Link event"); 1834 evcnt_attach_dynamic(&adapter->mod_workev, EVCNT_TYPE_INTR, 1835 NULL, xname, "SFP+ module event"); 1836 evcnt_attach_dynamic(&adapter->msf_workev, EVCNT_TYPE_INTR, 1837 NULL, xname, "Multispeed event"); 1838 evcnt_attach_dynamic(&adapter->phy_workev, EVCNT_TYPE_INTR, 1839 NULL, xname, "External PHY event"); 1840 1841 /* Max number of traffic class is 8 */ 1842 KASSERT(IXGBE_DCB_MAX_TRAFFIC_CLASS == 8); 1843 for (i = 0; i < IXGBE_TC_COUNTER_NUM; i++) { 1844 snprintf(adapter->tcs[i].evnamebuf, 1845 sizeof(adapter->tcs[i].evnamebuf), "%s tc%d", 1846 xname, i); 1847 if (i < __arraycount(stats->mpc)) { 1848 evcnt_attach_dynamic(&stats->mpc[i], 1849 EVCNT_TYPE_MISC, NULL, adapter->tcs[i].evnamebuf, 1850 "RX Missed Packet Count"); 1851 if (hw->mac.type == ixgbe_mac_82598EB) 1852 evcnt_attach_dynamic(&stats->rnbc[i], 1853 EVCNT_TYPE_MISC, NULL, 1854 adapter->tcs[i].evnamebuf, 1855 "Receive No Buffers"); 1856 } 1857 if (i < __arraycount(stats->pxontxc)) { 1858 evcnt_attach_dynamic(&stats->pxontxc[i], 1859 EVCNT_TYPE_MISC, NULL, adapter->tcs[i].evnamebuf, 1860 "pxontxc"); 1861 evcnt_attach_dynamic(&stats->pxonrxc[i], 1862 EVCNT_TYPE_MISC, NULL, adapter->tcs[i].evnamebuf, 1863 "pxonrxc"); 1864 evcnt_attach_dynamic(&stats->pxofftxc[i], 1865 EVCNT_TYPE_MISC, NULL, adapter->tcs[i].evnamebuf, 1866 "pxofftxc"); 1867 evcnt_attach_dynamic(&stats->pxoffrxc[i], 1868 EVCNT_TYPE_MISC, NULL, adapter->tcs[i].evnamebuf, 1869 "pxoffrxc"); 1870 if (hw->mac.type >= ixgbe_mac_82599EB) 1871 evcnt_attach_dynamic(&stats->pxon2offc[i], 1872 EVCNT_TYPE_MISC, NULL, 1873 adapter->tcs[i].evnamebuf, 1874 "pxon2offc"); 1875 } 1876 } 1877 1878 for (i = 0; i < adapter->num_queues; i++, rxr++, txr++) { 1879 #ifdef LRO 1880 struct lro_ctrl *lro = &rxr->lro; 1881 #endif /* LRO */ 1882 1883 snprintf(adapter->queues[i].evnamebuf, 1884 sizeof(adapter->queues[i].evnamebuf), "%s q%d", 1885 xname, i); 1886 snprintf(adapter->queues[i].namebuf, 1887 sizeof(adapter->queues[i].namebuf), "q%d", i); 1888 1889 if ((rnode = ixgbe_sysctl_instance(adapter)) == NULL) { 1890 aprint_error_dev(dev, "could not create sysctl root\n"); 1891 break; 1892 } 1893 1894 if (sysctl_createv(log, 0, &rnode, &rnode, 1895 0, CTLTYPE_NODE, 1896 adapter->queues[i].namebuf, SYSCTL_DESCR("Queue Name"), 1897 NULL, 0, NULL, 0, CTL_CREATE, CTL_EOL) != 0) 1898 break; 1899 1900 if (sysctl_createv(log, 0, &rnode, &cnode, 1901 CTLFLAG_READWRITE, CTLTYPE_INT, 1902 "interrupt_rate", SYSCTL_DESCR("Interrupt Rate"), 1903 ixgbe_sysctl_interrupt_rate_handler, 0, 1904 (void *)&adapter->queues[i], 0, CTL_CREATE, CTL_EOL) != 0) 1905 break; 1906 1907 if (sysctl_createv(log, 0, &rnode, &cnode, 1908 CTLFLAG_READONLY, CTLTYPE_INT, 1909 "txd_head", SYSCTL_DESCR("Transmit Descriptor Head"), 1910 ixgbe_sysctl_tdh_handler, 0, (void *)txr, 1911 0, CTL_CREATE, CTL_EOL) != 0) 1912 break; 1913 1914 if (sysctl_createv(log, 0, &rnode, &cnode, 1915 CTLFLAG_READONLY, CTLTYPE_INT, 1916 "txd_tail", SYSCTL_DESCR("Transmit Descriptor Tail"), 1917 ixgbe_sysctl_tdt_handler, 0, (void *)txr, 1918 0, CTL_CREATE, CTL_EOL) != 0) 1919 break; 1920 1921 evcnt_attach_dynamic(&adapter->queues[i].irqs, EVCNT_TYPE_INTR, 1922 NULL, adapter->queues[i].evnamebuf, "IRQs on queue"); 1923 evcnt_attach_dynamic(&adapter->queues[i].handleq, 1924 EVCNT_TYPE_MISC, NULL, adapter->queues[i].evnamebuf, 1925 "Handled queue in softint"); 1926 evcnt_attach_dynamic(&adapter->queues[i].req, EVCNT_TYPE_MISC, 1927 NULL, adapter->queues[i].evnamebuf, "Requeued in softint"); 1928 evcnt_attach_dynamic(&txr->tso_tx, EVCNT_TYPE_MISC, 1929 NULL, adapter->queues[i].evnamebuf, "TSO"); 1930 evcnt_attach_dynamic(&txr->no_desc_avail, EVCNT_TYPE_MISC, 1931 NULL, adapter->queues[i].evnamebuf, 1932 "TX Queue No Descriptor Available"); 1933 evcnt_attach_dynamic(&txr->total_packets, EVCNT_TYPE_MISC, 1934 NULL, adapter->queues[i].evnamebuf, 1935 "Queue Packets Transmitted"); 1936 #ifndef IXGBE_LEGACY_TX 1937 evcnt_attach_dynamic(&txr->pcq_drops, EVCNT_TYPE_MISC, 1938 NULL, adapter->queues[i].evnamebuf, 1939 "Packets dropped in pcq"); 1940 #endif 1941 1942 if (sysctl_createv(log, 0, &rnode, &cnode, 1943 CTLFLAG_READONLY, CTLTYPE_INT, "rxd_nxck", 1944 SYSCTL_DESCR("Receive Descriptor next to check"), 1945 ixgbe_sysctl_next_to_check_handler, 0, (void *)rxr, 0, 1946 CTL_CREATE, CTL_EOL) != 0) 1947 break; 1948 1949 if (sysctl_createv(log, 0, &rnode, &cnode, 1950 CTLFLAG_READONLY, CTLTYPE_INT, "rxd_nxrf", 1951 SYSCTL_DESCR("Receive Descriptor next to refresh"), 1952 ixgbe_sysctl_next_to_refresh_handler, 0, (void *)rxr, 0, 1953 CTL_CREATE, CTL_EOL) != 0) 1954 break; 1955 1956 if (sysctl_createv(log, 0, &rnode, &cnode, 1957 CTLFLAG_READONLY, CTLTYPE_INT, "rxd_head", 1958 SYSCTL_DESCR("Receive Descriptor Head"), 1959 ixgbe_sysctl_rdh_handler, 0, (void *)rxr, 0, 1960 CTL_CREATE, CTL_EOL) != 0) 1961 break; 1962 1963 if (sysctl_createv(log, 0, &rnode, &cnode, 1964 CTLFLAG_READONLY, CTLTYPE_INT, "rxd_tail", 1965 SYSCTL_DESCR("Receive Descriptor Tail"), 1966 ixgbe_sysctl_rdt_handler, 0, (void *)rxr, 0, 1967 CTL_CREATE, CTL_EOL) != 0) 1968 break; 1969 1970 if (i < __arraycount(stats->qprc)) { 1971 evcnt_attach_dynamic(&stats->qprc[i], EVCNT_TYPE_MISC, 1972 NULL, adapter->queues[i].evnamebuf, "qprc"); 1973 evcnt_attach_dynamic(&stats->qptc[i], EVCNT_TYPE_MISC, 1974 NULL, adapter->queues[i].evnamebuf, "qptc"); 1975 evcnt_attach_dynamic(&stats->qbrc[i], EVCNT_TYPE_MISC, 1976 NULL, adapter->queues[i].evnamebuf, "qbrc"); 1977 evcnt_attach_dynamic(&stats->qbtc[i], EVCNT_TYPE_MISC, 1978 NULL, adapter->queues[i].evnamebuf, "qbtc"); 1979 if (hw->mac.type >= ixgbe_mac_82599EB) 1980 evcnt_attach_dynamic(&stats->qprdc[i], 1981 EVCNT_TYPE_MISC, NULL, 1982 adapter->queues[i].evnamebuf, "qprdc"); 1983 } 1984 1985 evcnt_attach_dynamic(&rxr->rx_packets, EVCNT_TYPE_MISC, 1986 NULL, adapter->queues[i].evnamebuf, 1987 "Queue Packets Received"); 1988 evcnt_attach_dynamic(&rxr->rx_bytes, EVCNT_TYPE_MISC, 1989 NULL, adapter->queues[i].evnamebuf, 1990 "Queue Bytes Received"); 1991 evcnt_attach_dynamic(&rxr->rx_copies, EVCNT_TYPE_MISC, 1992 NULL, adapter->queues[i].evnamebuf, "Copied RX Frames"); 1993 evcnt_attach_dynamic(&rxr->no_mbuf, EVCNT_TYPE_MISC, 1994 NULL, adapter->queues[i].evnamebuf, "Rx no mbuf"); 1995 evcnt_attach_dynamic(&rxr->rx_discarded, EVCNT_TYPE_MISC, 1996 NULL, adapter->queues[i].evnamebuf, "Rx discarded"); 1997 #ifdef LRO 1998 SYSCTL_ADD_INT(ctx, queue_list, OID_AUTO, "lro_queued", 1999 CTLFLAG_RD, &lro->lro_queued, 0, 2000 "LRO Queued"); 2001 SYSCTL_ADD_INT(ctx, queue_list, OID_AUTO, "lro_flushed", 2002 CTLFLAG_RD, &lro->lro_flushed, 0, 2003 "LRO Flushed"); 2004 #endif /* LRO */ 2005 } 2006 2007 /* MAC stats get their own sub node */ 2008 2009 snprintf(stats->namebuf, 2010 sizeof(stats->namebuf), "%s MAC Statistics", xname); 2011 2012 evcnt_attach_dynamic(&stats->ipcs, EVCNT_TYPE_MISC, NULL, 2013 stats->namebuf, "rx csum offload - IP"); 2014 evcnt_attach_dynamic(&stats->l4cs, EVCNT_TYPE_MISC, NULL, 2015 stats->namebuf, "rx csum offload - L4"); 2016 evcnt_attach_dynamic(&stats->ipcs_bad, EVCNT_TYPE_MISC, NULL, 2017 stats->namebuf, "rx csum offload - IP bad"); 2018 evcnt_attach_dynamic(&stats->l4cs_bad, EVCNT_TYPE_MISC, NULL, 2019 stats->namebuf, "rx csum offload - L4 bad"); 2020 evcnt_attach_dynamic(&stats->intzero, EVCNT_TYPE_MISC, NULL, 2021 stats->namebuf, "Interrupt conditions zero"); 2022 evcnt_attach_dynamic(&stats->legint, EVCNT_TYPE_MISC, NULL, 2023 stats->namebuf, "Legacy interrupts"); 2024 2025 evcnt_attach_dynamic(&stats->crcerrs, EVCNT_TYPE_MISC, NULL, 2026 stats->namebuf, "CRC Errors"); 2027 evcnt_attach_dynamic(&stats->illerrc, EVCNT_TYPE_MISC, NULL, 2028 stats->namebuf, "Illegal Byte Errors"); 2029 evcnt_attach_dynamic(&stats->errbc, EVCNT_TYPE_MISC, NULL, 2030 stats->namebuf, "Byte Errors"); 2031 evcnt_attach_dynamic(&stats->mspdc, EVCNT_TYPE_MISC, NULL, 2032 stats->namebuf, "MAC Short Packets Discarded"); 2033 if (hw->mac.type >= ixgbe_mac_X550) 2034 evcnt_attach_dynamic(&stats->mbsdc, EVCNT_TYPE_MISC, NULL, 2035 stats->namebuf, "Bad SFD"); 2036 evcnt_attach_dynamic(&stats->mpctotal, EVCNT_TYPE_MISC, NULL, 2037 stats->namebuf, "Total Packets Missed"); 2038 evcnt_attach_dynamic(&stats->mlfc, EVCNT_TYPE_MISC, NULL, 2039 stats->namebuf, "MAC Local Faults"); 2040 evcnt_attach_dynamic(&stats->mrfc, EVCNT_TYPE_MISC, NULL, 2041 stats->namebuf, "MAC Remote Faults"); 2042 evcnt_attach_dynamic(&stats->rlec, EVCNT_TYPE_MISC, NULL, 2043 stats->namebuf, "Receive Length Errors"); 2044 evcnt_attach_dynamic(&stats->lxontxc, EVCNT_TYPE_MISC, NULL, 2045 stats->namebuf, "Link XON Transmitted"); 2046 evcnt_attach_dynamic(&stats->lxonrxc, EVCNT_TYPE_MISC, NULL, 2047 stats->namebuf, "Link XON Received"); 2048 evcnt_attach_dynamic(&stats->lxofftxc, EVCNT_TYPE_MISC, NULL, 2049 stats->namebuf, "Link XOFF Transmitted"); 2050 evcnt_attach_dynamic(&stats->lxoffrxc, EVCNT_TYPE_MISC, NULL, 2051 stats->namebuf, "Link XOFF Received"); 2052 2053 /* Packet Reception Stats */ 2054 evcnt_attach_dynamic(&stats->tor, EVCNT_TYPE_MISC, NULL, 2055 stats->namebuf, "Total Octets Received"); 2056 evcnt_attach_dynamic(&stats->gorc, EVCNT_TYPE_MISC, NULL, 2057 stats->namebuf, "Good Octets Received"); 2058 evcnt_attach_dynamic(&stats->tpr, EVCNT_TYPE_MISC, NULL, 2059 stats->namebuf, "Total Packets Received"); 2060 evcnt_attach_dynamic(&stats->gprc, EVCNT_TYPE_MISC, NULL, 2061 stats->namebuf, "Good Packets Received"); 2062 evcnt_attach_dynamic(&stats->mprc, EVCNT_TYPE_MISC, NULL, 2063 stats->namebuf, "Multicast Packets Received"); 2064 evcnt_attach_dynamic(&stats->bprc, EVCNT_TYPE_MISC, NULL, 2065 stats->namebuf, "Broadcast Packets Received"); 2066 evcnt_attach_dynamic(&stats->prc64, EVCNT_TYPE_MISC, NULL, 2067 stats->namebuf, "64 byte frames received "); 2068 evcnt_attach_dynamic(&stats->prc127, EVCNT_TYPE_MISC, NULL, 2069 stats->namebuf, "65-127 byte frames received"); 2070 evcnt_attach_dynamic(&stats->prc255, EVCNT_TYPE_MISC, NULL, 2071 stats->namebuf, "128-255 byte frames received"); 2072 evcnt_attach_dynamic(&stats->prc511, EVCNT_TYPE_MISC, NULL, 2073 stats->namebuf, "256-511 byte frames received"); 2074 evcnt_attach_dynamic(&stats->prc1023, EVCNT_TYPE_MISC, NULL, 2075 stats->namebuf, "512-1023 byte frames received"); 2076 evcnt_attach_dynamic(&stats->prc1522, EVCNT_TYPE_MISC, NULL, 2077 stats->namebuf, "1023-1522 byte frames received"); 2078 evcnt_attach_dynamic(&stats->ruc, EVCNT_TYPE_MISC, NULL, 2079 stats->namebuf, "Receive Undersized"); 2080 evcnt_attach_dynamic(&stats->rfc, EVCNT_TYPE_MISC, NULL, 2081 stats->namebuf, "Fragmented Packets Received "); 2082 evcnt_attach_dynamic(&stats->roc, EVCNT_TYPE_MISC, NULL, 2083 stats->namebuf, "Oversized Packets Received"); 2084 evcnt_attach_dynamic(&stats->rjc, EVCNT_TYPE_MISC, NULL, 2085 stats->namebuf, "Received Jabber"); 2086 evcnt_attach_dynamic(&stats->mngprc, EVCNT_TYPE_MISC, NULL, 2087 stats->namebuf, "Management Packets Received"); 2088 evcnt_attach_dynamic(&stats->mngpdc, EVCNT_TYPE_MISC, NULL, 2089 stats->namebuf, "Management Packets Dropped"); 2090 evcnt_attach_dynamic(&stats->xec, EVCNT_TYPE_MISC, NULL, 2091 stats->namebuf, "Checksum Errors"); 2092 2093 /* Packet Transmission Stats */ 2094 evcnt_attach_dynamic(&stats->gotc, EVCNT_TYPE_MISC, NULL, 2095 stats->namebuf, "Good Octets Transmitted"); 2096 evcnt_attach_dynamic(&stats->tpt, EVCNT_TYPE_MISC, NULL, 2097 stats->namebuf, "Total Packets Transmitted"); 2098 evcnt_attach_dynamic(&stats->gptc, EVCNT_TYPE_MISC, NULL, 2099 stats->namebuf, "Good Packets Transmitted"); 2100 evcnt_attach_dynamic(&stats->bptc, EVCNT_TYPE_MISC, NULL, 2101 stats->namebuf, "Broadcast Packets Transmitted"); 2102 evcnt_attach_dynamic(&stats->mptc, EVCNT_TYPE_MISC, NULL, 2103 stats->namebuf, "Multicast Packets Transmitted"); 2104 evcnt_attach_dynamic(&stats->mngptc, EVCNT_TYPE_MISC, NULL, 2105 stats->namebuf, "Management Packets Transmitted"); 2106 evcnt_attach_dynamic(&stats->ptc64, EVCNT_TYPE_MISC, NULL, 2107 stats->namebuf, "64 byte frames transmitted "); 2108 evcnt_attach_dynamic(&stats->ptc127, EVCNT_TYPE_MISC, NULL, 2109 stats->namebuf, "65-127 byte frames transmitted"); 2110 evcnt_attach_dynamic(&stats->ptc255, EVCNT_TYPE_MISC, NULL, 2111 stats->namebuf, "128-255 byte frames transmitted"); 2112 evcnt_attach_dynamic(&stats->ptc511, EVCNT_TYPE_MISC, NULL, 2113 stats->namebuf, "256-511 byte frames transmitted"); 2114 evcnt_attach_dynamic(&stats->ptc1023, EVCNT_TYPE_MISC, NULL, 2115 stats->namebuf, "512-1023 byte frames transmitted"); 2116 evcnt_attach_dynamic(&stats->ptc1522, EVCNT_TYPE_MISC, NULL, 2117 stats->namebuf, "1024-1522 byte frames transmitted"); 2118 } /* ixgbe_add_hw_stats */ 2119 2120 static void 2121 ixgbe_clear_evcnt(struct adapter *adapter) 2122 { 2123 struct tx_ring *txr = adapter->tx_rings; 2124 struct rx_ring *rxr = adapter->rx_rings; 2125 struct ixgbe_hw *hw = &adapter->hw; 2126 struct ixgbe_hw_stats *stats = &adapter->stats.pf; 2127 int i; 2128 2129 IXGBE_EVC_STORE(&adapter->efbig_tx_dma_setup, 0); 2130 IXGBE_EVC_STORE(&adapter->mbuf_defrag_failed, 0); 2131 IXGBE_EVC_STORE(&adapter->efbig2_tx_dma_setup, 0); 2132 IXGBE_EVC_STORE(&adapter->einval_tx_dma_setup, 0); 2133 IXGBE_EVC_STORE(&adapter->other_tx_dma_setup, 0); 2134 IXGBE_EVC_STORE(&adapter->eagain_tx_dma_setup, 0); 2135 IXGBE_EVC_STORE(&adapter->enomem_tx_dma_setup, 0); 2136 IXGBE_EVC_STORE(&adapter->tso_err, 0); 2137 IXGBE_EVC_STORE(&adapter->watchdog_events, 0); 2138 IXGBE_EVC_STORE(&adapter->admin_irqev, 0); 2139 IXGBE_EVC_STORE(&adapter->link_workev, 0); 2140 IXGBE_EVC_STORE(&adapter->mod_workev, 0); 2141 IXGBE_EVC_STORE(&adapter->msf_workev, 0); 2142 IXGBE_EVC_STORE(&adapter->phy_workev, 0); 2143 2144 for (i = 0; i < IXGBE_TC_COUNTER_NUM; i++) { 2145 if (i < __arraycount(stats->mpc)) { 2146 IXGBE_EVC_STORE(&stats->mpc[i], 0); 2147 if (hw->mac.type == ixgbe_mac_82598EB) 2148 IXGBE_EVC_STORE(&stats->rnbc[i], 0); 2149 } 2150 if (i < __arraycount(stats->pxontxc)) { 2151 IXGBE_EVC_STORE(&stats->pxontxc[i], 0); 2152 IXGBE_EVC_STORE(&stats->pxonrxc[i], 0); 2153 IXGBE_EVC_STORE(&stats->pxofftxc[i], 0); 2154 IXGBE_EVC_STORE(&stats->pxoffrxc[i], 0); 2155 if (hw->mac.type >= ixgbe_mac_82599EB) 2156 IXGBE_EVC_STORE(&stats->pxon2offc[i], 0); 2157 } 2158 } 2159 2160 txr = adapter->tx_rings; 2161 for (i = 0; i < adapter->num_queues; i++, rxr++, txr++) { 2162 IXGBE_EVC_STORE(&adapter->queues[i].irqs, 0); 2163 IXGBE_EVC_STORE(&adapter->queues[i].handleq, 0); 2164 IXGBE_EVC_STORE(&adapter->queues[i].req, 0); 2165 IXGBE_EVC_STORE(&txr->no_desc_avail, 0); 2166 IXGBE_EVC_STORE(&txr->total_packets, 0); 2167 IXGBE_EVC_STORE(&txr->tso_tx, 0); 2168 #ifndef IXGBE_LEGACY_TX 2169 IXGBE_EVC_STORE(&txr->pcq_drops, 0); 2170 #endif 2171 txr->q_efbig_tx_dma_setup = 0; 2172 txr->q_mbuf_defrag_failed = 0; 2173 txr->q_efbig2_tx_dma_setup = 0; 2174 txr->q_einval_tx_dma_setup = 0; 2175 txr->q_other_tx_dma_setup = 0; 2176 txr->q_eagain_tx_dma_setup = 0; 2177 txr->q_enomem_tx_dma_setup = 0; 2178 txr->q_tso_err = 0; 2179 2180 if (i < __arraycount(stats->qprc)) { 2181 IXGBE_EVC_STORE(&stats->qprc[i], 0); 2182 IXGBE_EVC_STORE(&stats->qptc[i], 0); 2183 IXGBE_EVC_STORE(&stats->qbrc[i], 0); 2184 IXGBE_EVC_STORE(&stats->qbtc[i], 0); 2185 if (hw->mac.type >= ixgbe_mac_82599EB) 2186 IXGBE_EVC_STORE(&stats->qprdc[i], 0); 2187 } 2188 2189 IXGBE_EVC_STORE(&rxr->rx_packets, 0); 2190 IXGBE_EVC_STORE(&rxr->rx_bytes, 0); 2191 IXGBE_EVC_STORE(&rxr->rx_copies, 0); 2192 IXGBE_EVC_STORE(&rxr->no_mbuf, 0); 2193 IXGBE_EVC_STORE(&rxr->rx_discarded, 0); 2194 } 2195 IXGBE_EVC_STORE(&stats->ipcs, 0); 2196 IXGBE_EVC_STORE(&stats->l4cs, 0); 2197 IXGBE_EVC_STORE(&stats->ipcs_bad, 0); 2198 IXGBE_EVC_STORE(&stats->l4cs_bad, 0); 2199 IXGBE_EVC_STORE(&stats->intzero, 0); 2200 IXGBE_EVC_STORE(&stats->legint, 0); 2201 IXGBE_EVC_STORE(&stats->crcerrs, 0); 2202 IXGBE_EVC_STORE(&stats->illerrc, 0); 2203 IXGBE_EVC_STORE(&stats->errbc, 0); 2204 IXGBE_EVC_STORE(&stats->mspdc, 0); 2205 if (hw->mac.type >= ixgbe_mac_X550) 2206 IXGBE_EVC_STORE(&stats->mbsdc, 0); 2207 IXGBE_EVC_STORE(&stats->mpctotal, 0); 2208 IXGBE_EVC_STORE(&stats->mlfc, 0); 2209 IXGBE_EVC_STORE(&stats->mrfc, 0); 2210 IXGBE_EVC_STORE(&stats->rlec, 0); 2211 IXGBE_EVC_STORE(&stats->lxontxc, 0); 2212 IXGBE_EVC_STORE(&stats->lxonrxc, 0); 2213 IXGBE_EVC_STORE(&stats->lxofftxc, 0); 2214 IXGBE_EVC_STORE(&stats->lxoffrxc, 0); 2215 2216 /* Packet Reception Stats */ 2217 IXGBE_EVC_STORE(&stats->tor, 0); 2218 IXGBE_EVC_STORE(&stats->gorc, 0); 2219 IXGBE_EVC_STORE(&stats->tpr, 0); 2220 IXGBE_EVC_STORE(&stats->gprc, 0); 2221 IXGBE_EVC_STORE(&stats->mprc, 0); 2222 IXGBE_EVC_STORE(&stats->bprc, 0); 2223 IXGBE_EVC_STORE(&stats->prc64, 0); 2224 IXGBE_EVC_STORE(&stats->prc127, 0); 2225 IXGBE_EVC_STORE(&stats->prc255, 0); 2226 IXGBE_EVC_STORE(&stats->prc511, 0); 2227 IXGBE_EVC_STORE(&stats->prc1023, 0); 2228 IXGBE_EVC_STORE(&stats->prc1522, 0); 2229 IXGBE_EVC_STORE(&stats->ruc, 0); 2230 IXGBE_EVC_STORE(&stats->rfc, 0); 2231 IXGBE_EVC_STORE(&stats->roc, 0); 2232 IXGBE_EVC_STORE(&stats->rjc, 0); 2233 IXGBE_EVC_STORE(&stats->mngprc, 0); 2234 IXGBE_EVC_STORE(&stats->mngpdc, 0); 2235 IXGBE_EVC_STORE(&stats->xec, 0); 2236 2237 /* Packet Transmission Stats */ 2238 IXGBE_EVC_STORE(&stats->gotc, 0); 2239 IXGBE_EVC_STORE(&stats->tpt, 0); 2240 IXGBE_EVC_STORE(&stats->gptc, 0); 2241 IXGBE_EVC_STORE(&stats->bptc, 0); 2242 IXGBE_EVC_STORE(&stats->mptc, 0); 2243 IXGBE_EVC_STORE(&stats->mngptc, 0); 2244 IXGBE_EVC_STORE(&stats->ptc64, 0); 2245 IXGBE_EVC_STORE(&stats->ptc127, 0); 2246 IXGBE_EVC_STORE(&stats->ptc255, 0); 2247 IXGBE_EVC_STORE(&stats->ptc511, 0); 2248 IXGBE_EVC_STORE(&stats->ptc1023, 0); 2249 IXGBE_EVC_STORE(&stats->ptc1522, 0); 2250 } 2251 2252 /************************************************************************ 2253 * ixgbe_sysctl_tdh_handler - Transmit Descriptor Head handler function 2254 * 2255 * Retrieves the TDH value from the hardware 2256 ************************************************************************/ 2257 static int 2258 ixgbe_sysctl_tdh_handler(SYSCTLFN_ARGS) 2259 { 2260 struct sysctlnode node = *rnode; 2261 struct tx_ring *txr = (struct tx_ring *)node.sysctl_data; 2262 struct adapter *adapter; 2263 uint32_t val; 2264 2265 if (!txr) 2266 return (0); 2267 2268 adapter = txr->adapter; 2269 if (ixgbe_fw_recovery_mode_swflag(adapter)) 2270 return (EPERM); 2271 2272 val = IXGBE_READ_REG(&adapter->hw, IXGBE_TDH(txr->me)); 2273 node.sysctl_data = &val; 2274 return sysctl_lookup(SYSCTLFN_CALL(&node)); 2275 } /* ixgbe_sysctl_tdh_handler */ 2276 2277 /************************************************************************ 2278 * ixgbe_sysctl_tdt_handler - Transmit Descriptor Tail handler function 2279 * 2280 * Retrieves the TDT value from the hardware 2281 ************************************************************************/ 2282 static int 2283 ixgbe_sysctl_tdt_handler(SYSCTLFN_ARGS) 2284 { 2285 struct sysctlnode node = *rnode; 2286 struct tx_ring *txr = (struct tx_ring *)node.sysctl_data; 2287 struct adapter *adapter; 2288 uint32_t val; 2289 2290 if (!txr) 2291 return (0); 2292 2293 adapter = txr->adapter; 2294 if (ixgbe_fw_recovery_mode_swflag(adapter)) 2295 return (EPERM); 2296 2297 val = IXGBE_READ_REG(&adapter->hw, IXGBE_TDT(txr->me)); 2298 node.sysctl_data = &val; 2299 return sysctl_lookup(SYSCTLFN_CALL(&node)); 2300 } /* ixgbe_sysctl_tdt_handler */ 2301 2302 /************************************************************************ 2303 * ixgbe_sysctl_next_to_check_handler - Receive Descriptor next to check 2304 * handler function 2305 * 2306 * Retrieves the next_to_check value 2307 ************************************************************************/ 2308 static int 2309 ixgbe_sysctl_next_to_check_handler(SYSCTLFN_ARGS) 2310 { 2311 struct sysctlnode node = *rnode; 2312 struct rx_ring *rxr = (struct rx_ring *)node.sysctl_data; 2313 struct adapter *adapter; 2314 uint32_t val; 2315 2316 if (!rxr) 2317 return (0); 2318 2319 adapter = rxr->adapter; 2320 if (ixgbe_fw_recovery_mode_swflag(adapter)) 2321 return (EPERM); 2322 2323 val = rxr->next_to_check; 2324 node.sysctl_data = &val; 2325 return sysctl_lookup(SYSCTLFN_CALL(&node)); 2326 } /* ixgbe_sysctl_next_to_check_handler */ 2327 2328 /************************************************************************ 2329 * ixgbe_sysctl_next_to_refresh_handler - Receive Descriptor next to check 2330 * handler function 2331 * 2332 * Retrieves the next_to_refresh value 2333 ************************************************************************/ 2334 static int 2335 ixgbe_sysctl_next_to_refresh_handler(SYSCTLFN_ARGS) 2336 { 2337 struct sysctlnode node = *rnode; 2338 struct rx_ring *rxr = (struct rx_ring *)node.sysctl_data; 2339 struct adapter *adapter; 2340 uint32_t val; 2341 2342 if (!rxr) 2343 return (0); 2344 2345 adapter = rxr->adapter; 2346 if (ixgbe_fw_recovery_mode_swflag(adapter)) 2347 return (EPERM); 2348 2349 val = rxr->next_to_refresh; 2350 node.sysctl_data = &val; 2351 return sysctl_lookup(SYSCTLFN_CALL(&node)); 2352 } /* ixgbe_sysctl_next_to_refresh_handler */ 2353 2354 /************************************************************************ 2355 * ixgbe_sysctl_rdh_handler - Receive Descriptor Head handler function 2356 * 2357 * Retrieves the RDH value from the hardware 2358 ************************************************************************/ 2359 static int 2360 ixgbe_sysctl_rdh_handler(SYSCTLFN_ARGS) 2361 { 2362 struct sysctlnode node = *rnode; 2363 struct rx_ring *rxr = (struct rx_ring *)node.sysctl_data; 2364 struct adapter *adapter; 2365 uint32_t val; 2366 2367 if (!rxr) 2368 return (0); 2369 2370 adapter = rxr->adapter; 2371 if (ixgbe_fw_recovery_mode_swflag(adapter)) 2372 return (EPERM); 2373 2374 val = IXGBE_READ_REG(&adapter->hw, IXGBE_RDH(rxr->me)); 2375 node.sysctl_data = &val; 2376 return sysctl_lookup(SYSCTLFN_CALL(&node)); 2377 } /* ixgbe_sysctl_rdh_handler */ 2378 2379 /************************************************************************ 2380 * ixgbe_sysctl_rdt_handler - Receive Descriptor Tail handler function 2381 * 2382 * Retrieves the RDT value from the hardware 2383 ************************************************************************/ 2384 static int 2385 ixgbe_sysctl_rdt_handler(SYSCTLFN_ARGS) 2386 { 2387 struct sysctlnode node = *rnode; 2388 struct rx_ring *rxr = (struct rx_ring *)node.sysctl_data; 2389 struct adapter *adapter; 2390 uint32_t val; 2391 2392 if (!rxr) 2393 return (0); 2394 2395 adapter = rxr->adapter; 2396 if (ixgbe_fw_recovery_mode_swflag(adapter)) 2397 return (EPERM); 2398 2399 val = IXGBE_READ_REG(&adapter->hw, IXGBE_RDT(rxr->me)); 2400 node.sysctl_data = &val; 2401 return sysctl_lookup(SYSCTLFN_CALL(&node)); 2402 } /* ixgbe_sysctl_rdt_handler */ 2403 2404 static int 2405 ixgbe_vlan_cb(struct ethercom *ec, uint16_t vid, bool set) 2406 { 2407 struct ifnet *ifp = &ec->ec_if; 2408 struct adapter *adapter = ifp->if_softc; 2409 int rv; 2410 2411 if (set) 2412 rv = ixgbe_register_vlan(adapter, vid); 2413 else 2414 rv = ixgbe_unregister_vlan(adapter, vid); 2415 2416 if (rv != 0) 2417 return rv; 2418 2419 /* 2420 * Control VLAN HW tagging when ec_nvlan is changed from 1 to 0 2421 * or 0 to 1. 2422 */ 2423 if ((set && (ec->ec_nvlans == 1)) || (!set && (ec->ec_nvlans == 0))) 2424 ixgbe_setup_vlan_hw_tagging(adapter); 2425 2426 return rv; 2427 } 2428 2429 /************************************************************************ 2430 * ixgbe_register_vlan 2431 * 2432 * Run via vlan config EVENT, it enables us to use the 2433 * HW Filter table since we can get the vlan id. This 2434 * just creates the entry in the soft version of the 2435 * VFTA, init will repopulate the real table. 2436 ************************************************************************/ 2437 static int 2438 ixgbe_register_vlan(struct adapter *adapter, u16 vtag) 2439 { 2440 u16 index, bit; 2441 int error; 2442 2443 if ((vtag == 0) || (vtag > 4095)) /* Invalid */ 2444 return EINVAL; 2445 2446 IXGBE_CORE_LOCK(adapter); 2447 index = (vtag >> 5) & 0x7F; 2448 bit = vtag & 0x1F; 2449 adapter->shadow_vfta[index] |= ((u32)1 << bit); 2450 error = adapter->hw.mac.ops.set_vfta(&adapter->hw, vtag, 0, true, 2451 true); 2452 IXGBE_CORE_UNLOCK(adapter); 2453 if (error != 0) 2454 error = EACCES; 2455 2456 return error; 2457 } /* ixgbe_register_vlan */ 2458 2459 /************************************************************************ 2460 * ixgbe_unregister_vlan 2461 * 2462 * Run via vlan unconfig EVENT, remove our entry in the soft vfta. 2463 ************************************************************************/ 2464 static int 2465 ixgbe_unregister_vlan(struct adapter *adapter, u16 vtag) 2466 { 2467 u16 index, bit; 2468 int error; 2469 2470 if ((vtag == 0) || (vtag > 4095)) /* Invalid */ 2471 return EINVAL; 2472 2473 IXGBE_CORE_LOCK(adapter); 2474 index = (vtag >> 5) & 0x7F; 2475 bit = vtag & 0x1F; 2476 adapter->shadow_vfta[index] &= ~((u32)1 << bit); 2477 error = adapter->hw.mac.ops.set_vfta(&adapter->hw, vtag, 0, false, 2478 true); 2479 IXGBE_CORE_UNLOCK(adapter); 2480 if (error != 0) 2481 error = EACCES; 2482 2483 return error; 2484 } /* ixgbe_unregister_vlan */ 2485 2486 static void 2487 ixgbe_setup_vlan_hw_tagging(struct adapter *adapter) 2488 { 2489 struct ethercom *ec = &adapter->osdep.ec; 2490 struct ixgbe_hw *hw = &adapter->hw; 2491 struct rx_ring *rxr; 2492 u32 ctrl; 2493 int i; 2494 bool hwtagging; 2495 2496 /* Enable HW tagging only if any vlan is attached */ 2497 hwtagging = (ec->ec_capenable & ETHERCAP_VLAN_HWTAGGING) 2498 && VLAN_ATTACHED(ec); 2499 2500 /* Setup the queues for vlans */ 2501 for (i = 0; i < adapter->num_queues; i++) { 2502 rxr = &adapter->rx_rings[i]; 2503 /* 2504 * On 82599 and later, the VLAN enable is per/queue in RXDCTL. 2505 */ 2506 if (hw->mac.type != ixgbe_mac_82598EB) { 2507 ctrl = IXGBE_READ_REG(hw, IXGBE_RXDCTL(rxr->me)); 2508 if (hwtagging) 2509 ctrl |= IXGBE_RXDCTL_VME; 2510 else 2511 ctrl &= ~IXGBE_RXDCTL_VME; 2512 IXGBE_WRITE_REG(hw, IXGBE_RXDCTL(rxr->me), ctrl); 2513 } 2514 rxr->vtag_strip = hwtagging ? TRUE : FALSE; 2515 } 2516 2517 /* VLAN hw tagging for 82598 */ 2518 if (hw->mac.type == ixgbe_mac_82598EB) { 2519 ctrl = IXGBE_READ_REG(hw, IXGBE_VLNCTRL); 2520 if (hwtagging) 2521 ctrl |= IXGBE_VLNCTRL_VME; 2522 else 2523 ctrl &= ~IXGBE_VLNCTRL_VME; 2524 IXGBE_WRITE_REG(hw, IXGBE_VLNCTRL, ctrl); 2525 } 2526 } /* ixgbe_setup_vlan_hw_tagging */ 2527 2528 static void 2529 ixgbe_setup_vlan_hw_support(struct adapter *adapter) 2530 { 2531 struct ethercom *ec = &adapter->osdep.ec; 2532 struct ixgbe_hw *hw = &adapter->hw; 2533 int i; 2534 u32 ctrl; 2535 struct vlanid_list *vlanidp; 2536 2537 /* 2538 * This function is called from both if_init and ifflags_cb() 2539 * on NetBSD. 2540 */ 2541 2542 /* 2543 * Part 1: 2544 * Setup VLAN HW tagging 2545 */ 2546 ixgbe_setup_vlan_hw_tagging(adapter); 2547 2548 /* 2549 * Part 2: 2550 * Setup VLAN HW filter 2551 */ 2552 /* Cleanup shadow_vfta */ 2553 for (i = 0; i < IXGBE_VFTA_SIZE; i++) 2554 adapter->shadow_vfta[i] = 0; 2555 /* Generate shadow_vfta from ec_vids */ 2556 ETHER_LOCK(ec); 2557 SIMPLEQ_FOREACH(vlanidp, &ec->ec_vids, vid_list) { 2558 uint32_t idx; 2559 2560 idx = vlanidp->vid / 32; 2561 KASSERT(idx < IXGBE_VFTA_SIZE); 2562 adapter->shadow_vfta[idx] |= (u32)1 << (vlanidp->vid % 32); 2563 } 2564 ETHER_UNLOCK(ec); 2565 for (i = 0; i < IXGBE_VFTA_SIZE; i++) 2566 IXGBE_WRITE_REG(hw, IXGBE_VFTA(i), adapter->shadow_vfta[i]); 2567 2568 ctrl = IXGBE_READ_REG(hw, IXGBE_VLNCTRL); 2569 /* Enable the Filter Table if enabled */ 2570 if (ec->ec_capenable & ETHERCAP_VLAN_HWFILTER) 2571 ctrl |= IXGBE_VLNCTRL_VFE; 2572 else 2573 ctrl &= ~IXGBE_VLNCTRL_VFE; 2574 IXGBE_WRITE_REG(hw, IXGBE_VLNCTRL, ctrl); 2575 } /* ixgbe_setup_vlan_hw_support */ 2576 2577 /************************************************************************ 2578 * ixgbe_get_slot_info 2579 * 2580 * Get the width and transaction speed of 2581 * the slot this adapter is plugged into. 2582 ************************************************************************/ 2583 static void 2584 ixgbe_get_slot_info(struct adapter *adapter) 2585 { 2586 device_t dev = adapter->dev; 2587 struct ixgbe_hw *hw = &adapter->hw; 2588 u32 offset; 2589 u16 link; 2590 int bus_info_valid = TRUE; 2591 2592 /* Some devices are behind an internal bridge */ 2593 switch (hw->device_id) { 2594 case IXGBE_DEV_ID_82599_SFP_SF_QP: 2595 case IXGBE_DEV_ID_82599_QSFP_SF_QP: 2596 goto get_parent_info; 2597 default: 2598 break; 2599 } 2600 2601 ixgbe_get_bus_info(hw); 2602 2603 /* 2604 * Some devices don't use PCI-E, but there is no need 2605 * to display "Unknown" for bus speed and width. 2606 */ 2607 switch (hw->mac.type) { 2608 case ixgbe_mac_X550EM_x: 2609 case ixgbe_mac_X550EM_a: 2610 return; 2611 default: 2612 goto display; 2613 } 2614 2615 get_parent_info: 2616 /* 2617 * For the Quad port adapter we need to parse back 2618 * up the PCI tree to find the speed of the expansion 2619 * slot into which this adapter is plugged. A bit more work. 2620 */ 2621 dev = device_parent(device_parent(dev)); 2622 #if 0 2623 #ifdef IXGBE_DEBUG 2624 device_printf(dev, "parent pcib = %x,%x,%x\n", pci_get_bus(dev), 2625 pci_get_slot(dev), pci_get_function(dev)); 2626 #endif 2627 dev = device_parent(device_parent(dev)); 2628 #ifdef IXGBE_DEBUG 2629 device_printf(dev, "slot pcib = %x,%x,%x\n", pci_get_bus(dev), 2630 pci_get_slot(dev), pci_get_function(dev)); 2631 #endif 2632 #endif 2633 /* Now get the PCI Express Capabilities offset */ 2634 if (pci_get_capability(adapter->osdep.pc, adapter->osdep.tag, 2635 PCI_CAP_PCIEXPRESS, &offset, NULL)) { 2636 /* 2637 * Hmm...can't get PCI-Express capabilities. 2638 * Falling back to default method. 2639 */ 2640 bus_info_valid = FALSE; 2641 ixgbe_get_bus_info(hw); 2642 goto display; 2643 } 2644 /* ...and read the Link Status Register */ 2645 link = pci_conf_read(adapter->osdep.pc, adapter->osdep.tag, 2646 offset + PCIE_LCSR) >> 16; 2647 ixgbe_set_pci_config_data_generic(hw, link); 2648 2649 display: 2650 device_printf(dev, "PCI Express Bus: Speed %s Width %s\n", 2651 ((hw->bus.speed == ixgbe_bus_speed_8000) ? "8.0GT/s" : 2652 (hw->bus.speed == ixgbe_bus_speed_5000) ? "5.0GT/s" : 2653 (hw->bus.speed == ixgbe_bus_speed_2500) ? "2.5GT/s" : 2654 "Unknown"), 2655 ((hw->bus.width == ixgbe_bus_width_pcie_x8) ? "x8" : 2656 (hw->bus.width == ixgbe_bus_width_pcie_x4) ? "x4" : 2657 (hw->bus.width == ixgbe_bus_width_pcie_x1) ? "x1" : 2658 "Unknown")); 2659 2660 if (bus_info_valid) { 2661 if ((hw->device_id != IXGBE_DEV_ID_82599_SFP_SF_QP) && 2662 ((hw->bus.width <= ixgbe_bus_width_pcie_x4) && 2663 (hw->bus.speed == ixgbe_bus_speed_2500))) { 2664 device_printf(dev, "PCI-Express bandwidth available" 2665 " for this card\n is not sufficient for" 2666 " optimal performance.\n"); 2667 device_printf(dev, "For optimal performance a x8 " 2668 "PCIE, or x4 PCIE Gen2 slot is required.\n"); 2669 } 2670 if ((hw->device_id == IXGBE_DEV_ID_82599_SFP_SF_QP) && 2671 ((hw->bus.width <= ixgbe_bus_width_pcie_x8) && 2672 (hw->bus.speed < ixgbe_bus_speed_8000))) { 2673 device_printf(dev, "PCI-Express bandwidth available" 2674 " for this card\n is not sufficient for" 2675 " optimal performance.\n"); 2676 device_printf(dev, "For optimal performance a x8 " 2677 "PCIE Gen3 slot is required.\n"); 2678 } 2679 } else 2680 device_printf(dev, "Unable to determine slot speed/width. The speed/width reported are that of the internal switch.\n"); 2681 2682 return; 2683 } /* ixgbe_get_slot_info */ 2684 2685 /************************************************************************ 2686 * ixgbe_enable_queue - MSI-X Interrupt Handlers and Tasklets 2687 ************************************************************************/ 2688 static inline void 2689 ixgbe_enable_queue(struct adapter *adapter, u32 vector) 2690 { 2691 struct ixgbe_hw *hw = &adapter->hw; 2692 struct ix_queue *que = &adapter->queues[vector]; 2693 u64 queue = 1ULL << vector; 2694 u32 mask; 2695 2696 mutex_enter(&que->dc_mtx); 2697 if (que->disabled_count > 0 && --que->disabled_count > 0) 2698 goto out; 2699 2700 if (hw->mac.type == ixgbe_mac_82598EB) { 2701 mask = (IXGBE_EIMS_RTX_QUEUE & queue); 2702 IXGBE_WRITE_REG(hw, IXGBE_EIMS, mask); 2703 } else { 2704 mask = (queue & 0xFFFFFFFF); 2705 if (mask) 2706 IXGBE_WRITE_REG(hw, IXGBE_EIMS_EX(0), mask); 2707 mask = (queue >> 32); 2708 if (mask) 2709 IXGBE_WRITE_REG(hw, IXGBE_EIMS_EX(1), mask); 2710 } 2711 out: 2712 mutex_exit(&que->dc_mtx); 2713 } /* ixgbe_enable_queue */ 2714 2715 /************************************************************************ 2716 * ixgbe_disable_queue_internal 2717 ************************************************************************/ 2718 static inline void 2719 ixgbe_disable_queue_internal(struct adapter *adapter, u32 vector, bool nestok) 2720 { 2721 struct ixgbe_hw *hw = &adapter->hw; 2722 struct ix_queue *que = &adapter->queues[vector]; 2723 u64 queue = 1ULL << vector; 2724 u32 mask; 2725 2726 mutex_enter(&que->dc_mtx); 2727 2728 if (que->disabled_count > 0) { 2729 if (nestok) 2730 que->disabled_count++; 2731 goto out; 2732 } 2733 que->disabled_count++; 2734 2735 if (hw->mac.type == ixgbe_mac_82598EB) { 2736 mask = (IXGBE_EIMS_RTX_QUEUE & queue); 2737 IXGBE_WRITE_REG(hw, IXGBE_EIMC, mask); 2738 } else { 2739 mask = (queue & 0xFFFFFFFF); 2740 if (mask) 2741 IXGBE_WRITE_REG(hw, IXGBE_EIMC_EX(0), mask); 2742 mask = (queue >> 32); 2743 if (mask) 2744 IXGBE_WRITE_REG(hw, IXGBE_EIMC_EX(1), mask); 2745 } 2746 out: 2747 mutex_exit(&que->dc_mtx); 2748 } /* ixgbe_disable_queue_internal */ 2749 2750 /************************************************************************ 2751 * ixgbe_disable_queue 2752 ************************************************************************/ 2753 static inline void 2754 ixgbe_disable_queue(struct adapter *adapter, u32 vector) 2755 { 2756 2757 ixgbe_disable_queue_internal(adapter, vector, true); 2758 } /* ixgbe_disable_queue */ 2759 2760 /************************************************************************ 2761 * ixgbe_sched_handle_que - schedule deferred packet processing 2762 ************************************************************************/ 2763 static inline void 2764 ixgbe_sched_handle_que(struct adapter *adapter, struct ix_queue *que) 2765 { 2766 2767 if (que->txrx_use_workqueue) { 2768 /* 2769 * adapter->que_wq is bound to each CPU instead of 2770 * each NIC queue to reduce workqueue kthread. As we 2771 * should consider about interrupt affinity in this 2772 * function, the workqueue kthread must be WQ_PERCPU. 2773 * If create WQ_PERCPU workqueue kthread for each NIC 2774 * queue, that number of created workqueue kthread is 2775 * (number of used NIC queue) * (number of CPUs) = 2776 * (number of CPUs) ^ 2 most often. 2777 * 2778 * The same NIC queue's interrupts are avoided by 2779 * masking the queue's interrupt. And different 2780 * NIC queue's interrupts use different struct work 2781 * (que->wq_cookie). So, "enqueued flag" to avoid 2782 * twice workqueue_enqueue() is not required . 2783 */ 2784 workqueue_enqueue(adapter->que_wq, &que->wq_cookie, curcpu()); 2785 } else { 2786 softint_schedule(que->que_si); 2787 } 2788 } 2789 2790 /************************************************************************ 2791 * ixgbe_msix_que - MSI-X Queue Interrupt Service routine 2792 ************************************************************************/ 2793 static int 2794 ixgbe_msix_que(void *arg) 2795 { 2796 struct ix_queue *que = arg; 2797 struct adapter *adapter = que->adapter; 2798 struct ifnet *ifp = adapter->ifp; 2799 struct tx_ring *txr = que->txr; 2800 struct rx_ring *rxr = que->rxr; 2801 bool more; 2802 u32 newitr = 0; 2803 2804 /* Protect against spurious interrupts */ 2805 if ((ifp->if_flags & IFF_RUNNING) == 0) 2806 return 0; 2807 2808 ixgbe_disable_queue(adapter, que->msix); 2809 IXGBE_EVC_ADD(&que->irqs, 1); 2810 2811 /* 2812 * Don't change "que->txrx_use_workqueue" from this point to avoid 2813 * flip-flopping softint/workqueue mode in one deferred processing. 2814 */ 2815 que->txrx_use_workqueue = adapter->txrx_use_workqueue; 2816 2817 #ifdef __NetBSD__ 2818 /* Don't run ixgbe_rxeof in interrupt context */ 2819 more = true; 2820 #else 2821 more = ixgbe_rxeof(que); 2822 #endif 2823 2824 IXGBE_TX_LOCK(txr); 2825 ixgbe_txeof(txr); 2826 IXGBE_TX_UNLOCK(txr); 2827 2828 /* Do AIM now? */ 2829 2830 if (adapter->enable_aim == false) 2831 goto no_calc; 2832 /* 2833 * Do Adaptive Interrupt Moderation: 2834 * - Write out last calculated setting 2835 * - Calculate based on average size over 2836 * the last interval. 2837 */ 2838 if (que->eitr_setting) 2839 ixgbe_eitr_write(adapter, que->msix, que->eitr_setting); 2840 2841 que->eitr_setting = 0; 2842 2843 /* Idle, do nothing */ 2844 if ((txr->bytes == 0) && (rxr->bytes == 0)) 2845 goto no_calc; 2846 2847 if ((txr->bytes) && (txr->packets)) 2848 newitr = txr->bytes/txr->packets; 2849 if ((rxr->bytes) && (rxr->packets)) 2850 newitr = uimax(newitr, (rxr->bytes / rxr->packets)); 2851 newitr += 24; /* account for hardware frame, crc */ 2852 2853 /* set an upper boundary */ 2854 newitr = uimin(newitr, 3000); 2855 2856 /* Be nice to the mid range */ 2857 if ((newitr > 300) && (newitr < 1200)) 2858 newitr = (newitr / 3); 2859 else 2860 newitr = (newitr / 2); 2861 2862 /* 2863 * When RSC is used, ITR interval must be larger than RSC_DELAY. 2864 * Currently, we use 2us for RSC_DELAY. The minimum value is always 2865 * greater than 2us on 100M (and 10M?(not documented)), but it's not 2866 * on 1G and higher. 2867 */ 2868 if ((adapter->link_speed != IXGBE_LINK_SPEED_100_FULL) 2869 && (adapter->link_speed != IXGBE_LINK_SPEED_10_FULL)) { 2870 if (newitr < IXGBE_MIN_RSC_EITR_10G1G) 2871 newitr = IXGBE_MIN_RSC_EITR_10G1G; 2872 } 2873 2874 /* save for next interrupt */ 2875 que->eitr_setting = newitr; 2876 2877 /* Reset state */ 2878 txr->bytes = 0; 2879 txr->packets = 0; 2880 rxr->bytes = 0; 2881 rxr->packets = 0; 2882 2883 no_calc: 2884 if (more) 2885 ixgbe_sched_handle_que(adapter, que); 2886 else 2887 ixgbe_enable_queue(adapter, que->msix); 2888 2889 return 1; 2890 } /* ixgbe_msix_que */ 2891 2892 /************************************************************************ 2893 * ixgbe_media_status - Media Ioctl callback 2894 * 2895 * Called whenever the user queries the status of 2896 * the interface using ifconfig. 2897 ************************************************************************/ 2898 static void 2899 ixgbe_media_status(struct ifnet *ifp, struct ifmediareq *ifmr) 2900 { 2901 struct adapter *adapter = ifp->if_softc; 2902 struct ixgbe_hw *hw = &adapter->hw; 2903 int layer; 2904 2905 INIT_DEBUGOUT("ixgbe_media_status: begin"); 2906 ixgbe_update_link_status(adapter); 2907 2908 ifmr->ifm_status = IFM_AVALID; 2909 ifmr->ifm_active = IFM_ETHER; 2910 2911 if (adapter->link_active != LINK_STATE_UP) { 2912 ifmr->ifm_active |= IFM_NONE; 2913 return; 2914 } 2915 2916 ifmr->ifm_status |= IFM_ACTIVE; 2917 layer = adapter->phy_layer; 2918 2919 if (layer & IXGBE_PHYSICAL_LAYER_10GBASE_T || 2920 layer & IXGBE_PHYSICAL_LAYER_5GBASE_T || 2921 layer & IXGBE_PHYSICAL_LAYER_2500BASE_T || 2922 layer & IXGBE_PHYSICAL_LAYER_1000BASE_T || 2923 layer & IXGBE_PHYSICAL_LAYER_100BASE_TX || 2924 layer & IXGBE_PHYSICAL_LAYER_10BASE_T) 2925 switch (adapter->link_speed) { 2926 case IXGBE_LINK_SPEED_10GB_FULL: 2927 ifmr->ifm_active |= IFM_10G_T | IFM_FDX; 2928 break; 2929 case IXGBE_LINK_SPEED_5GB_FULL: 2930 ifmr->ifm_active |= IFM_5000_T | IFM_FDX; 2931 break; 2932 case IXGBE_LINK_SPEED_2_5GB_FULL: 2933 ifmr->ifm_active |= IFM_2500_T | IFM_FDX; 2934 break; 2935 case IXGBE_LINK_SPEED_1GB_FULL: 2936 ifmr->ifm_active |= IFM_1000_T | IFM_FDX; 2937 break; 2938 case IXGBE_LINK_SPEED_100_FULL: 2939 ifmr->ifm_active |= IFM_100_TX | IFM_FDX; 2940 break; 2941 case IXGBE_LINK_SPEED_10_FULL: 2942 ifmr->ifm_active |= IFM_10_T | IFM_FDX; 2943 break; 2944 } 2945 if (layer & IXGBE_PHYSICAL_LAYER_SFP_PLUS_CU || 2946 layer & IXGBE_PHYSICAL_LAYER_SFP_ACTIVE_DA) 2947 switch (adapter->link_speed) { 2948 case IXGBE_LINK_SPEED_10GB_FULL: 2949 ifmr->ifm_active |= IFM_10G_TWINAX | IFM_FDX; 2950 break; 2951 } 2952 if (layer & IXGBE_PHYSICAL_LAYER_10GBASE_LR) 2953 switch (adapter->link_speed) { 2954 case IXGBE_LINK_SPEED_10GB_FULL: 2955 ifmr->ifm_active |= IFM_10G_LR | IFM_FDX; 2956 break; 2957 case IXGBE_LINK_SPEED_1GB_FULL: 2958 ifmr->ifm_active |= IFM_1000_LX | IFM_FDX; 2959 break; 2960 } 2961 if (layer & IXGBE_PHYSICAL_LAYER_10GBASE_LRM) 2962 switch (adapter->link_speed) { 2963 case IXGBE_LINK_SPEED_10GB_FULL: 2964 ifmr->ifm_active |= IFM_10G_LRM | IFM_FDX; 2965 break; 2966 case IXGBE_LINK_SPEED_1GB_FULL: 2967 ifmr->ifm_active |= IFM_1000_LX | IFM_FDX; 2968 break; 2969 } 2970 if (layer & IXGBE_PHYSICAL_LAYER_10GBASE_SR || 2971 layer & IXGBE_PHYSICAL_LAYER_1000BASE_SX) 2972 switch (adapter->link_speed) { 2973 case IXGBE_LINK_SPEED_10GB_FULL: 2974 ifmr->ifm_active |= IFM_10G_SR | IFM_FDX; 2975 break; 2976 case IXGBE_LINK_SPEED_1GB_FULL: 2977 ifmr->ifm_active |= IFM_1000_SX | IFM_FDX; 2978 break; 2979 } 2980 if (layer & IXGBE_PHYSICAL_LAYER_10GBASE_CX4) 2981 switch (adapter->link_speed) { 2982 case IXGBE_LINK_SPEED_10GB_FULL: 2983 ifmr->ifm_active |= IFM_10G_CX4 | IFM_FDX; 2984 break; 2985 } 2986 /* 2987 * XXX: These need to use the proper media types once 2988 * they're added. 2989 */ 2990 if (layer & IXGBE_PHYSICAL_LAYER_10GBASE_KR) 2991 switch (adapter->link_speed) { 2992 case IXGBE_LINK_SPEED_10GB_FULL: 2993 ifmr->ifm_active |= IFM_10G_KR | IFM_FDX; 2994 break; 2995 case IXGBE_LINK_SPEED_2_5GB_FULL: 2996 ifmr->ifm_active |= IFM_2500_KX | IFM_FDX; 2997 break; 2998 case IXGBE_LINK_SPEED_1GB_FULL: 2999 ifmr->ifm_active |= IFM_1000_KX | IFM_FDX; 3000 break; 3001 } 3002 else if (layer & IXGBE_PHYSICAL_LAYER_10GBASE_KX4 || 3003 layer & IXGBE_PHYSICAL_LAYER_2500BASE_KX || 3004 layer & IXGBE_PHYSICAL_LAYER_1000BASE_KX) 3005 switch (adapter->link_speed) { 3006 case IXGBE_LINK_SPEED_10GB_FULL: 3007 ifmr->ifm_active |= IFM_10G_KX4 | IFM_FDX; 3008 break; 3009 case IXGBE_LINK_SPEED_2_5GB_FULL: 3010 ifmr->ifm_active |= IFM_2500_KX | IFM_FDX; 3011 break; 3012 case IXGBE_LINK_SPEED_1GB_FULL: 3013 ifmr->ifm_active |= IFM_1000_KX | IFM_FDX; 3014 break; 3015 } 3016 3017 /* If nothing is recognized... */ 3018 #if 0 3019 if (IFM_SUBTYPE(ifmr->ifm_active) == 0) 3020 ifmr->ifm_active |= IFM_UNKNOWN; 3021 #endif 3022 3023 ifp->if_baudrate = ifmedia_baudrate(ifmr->ifm_active); 3024 3025 /* Display current flow control setting used on link */ 3026 if (hw->fc.current_mode == ixgbe_fc_rx_pause || 3027 hw->fc.current_mode == ixgbe_fc_full) 3028 ifmr->ifm_active |= IFM_ETH_RXPAUSE; 3029 if (hw->fc.current_mode == ixgbe_fc_tx_pause || 3030 hw->fc.current_mode == ixgbe_fc_full) 3031 ifmr->ifm_active |= IFM_ETH_TXPAUSE; 3032 3033 return; 3034 } /* ixgbe_media_status */ 3035 3036 /************************************************************************ 3037 * ixgbe_media_change - Media Ioctl callback 3038 * 3039 * Called when the user changes speed/duplex using 3040 * media/mediopt option with ifconfig. 3041 ************************************************************************/ 3042 static int 3043 ixgbe_media_change(struct ifnet *ifp) 3044 { 3045 struct adapter *adapter = ifp->if_softc; 3046 struct ifmedia *ifm = &adapter->media; 3047 struct ixgbe_hw *hw = &adapter->hw; 3048 ixgbe_link_speed speed = 0; 3049 ixgbe_link_speed link_caps = 0; 3050 bool negotiate = false; 3051 s32 err = IXGBE_NOT_IMPLEMENTED; 3052 3053 INIT_DEBUGOUT("ixgbe_media_change: begin"); 3054 3055 if (IFM_TYPE(ifm->ifm_media) != IFM_ETHER) 3056 return (EINVAL); 3057 3058 if (hw->phy.media_type == ixgbe_media_type_backplane) 3059 return (EPERM); 3060 3061 /* 3062 * We don't actually need to check against the supported 3063 * media types of the adapter; ifmedia will take care of 3064 * that for us. 3065 */ 3066 switch (IFM_SUBTYPE(ifm->ifm_media)) { 3067 case IFM_AUTO: 3068 err = hw->mac.ops.get_link_capabilities(hw, &link_caps, 3069 &negotiate); 3070 if (err != IXGBE_SUCCESS) { 3071 device_printf(adapter->dev, "Unable to determine " 3072 "supported advertise speeds\n"); 3073 return (ENODEV); 3074 } 3075 speed |= link_caps; 3076 break; 3077 case IFM_10G_T: 3078 case IFM_10G_LRM: 3079 case IFM_10G_LR: 3080 case IFM_10G_TWINAX: 3081 case IFM_10G_SR: 3082 case IFM_10G_CX4: 3083 case IFM_10G_KR: 3084 case IFM_10G_KX4: 3085 speed |= IXGBE_LINK_SPEED_10GB_FULL; 3086 break; 3087 case IFM_5000_T: 3088 speed |= IXGBE_LINK_SPEED_5GB_FULL; 3089 break; 3090 case IFM_2500_T: 3091 case IFM_2500_KX: 3092 speed |= IXGBE_LINK_SPEED_2_5GB_FULL; 3093 break; 3094 case IFM_1000_T: 3095 case IFM_1000_LX: 3096 case IFM_1000_SX: 3097 case IFM_1000_KX: 3098 speed |= IXGBE_LINK_SPEED_1GB_FULL; 3099 break; 3100 case IFM_100_TX: 3101 speed |= IXGBE_LINK_SPEED_100_FULL; 3102 break; 3103 case IFM_10_T: 3104 speed |= IXGBE_LINK_SPEED_10_FULL; 3105 break; 3106 case IFM_NONE: 3107 break; 3108 default: 3109 goto invalid; 3110 } 3111 3112 hw->mac.autotry_restart = TRUE; 3113 hw->mac.ops.setup_link(hw, speed, TRUE); 3114 adapter->advertise = 0; 3115 if (IFM_SUBTYPE(ifm->ifm_media) != IFM_AUTO) { 3116 if ((speed & IXGBE_LINK_SPEED_10GB_FULL) != 0) 3117 adapter->advertise |= 1 << 2; 3118 if ((speed & IXGBE_LINK_SPEED_1GB_FULL) != 0) 3119 adapter->advertise |= 1 << 1; 3120 if ((speed & IXGBE_LINK_SPEED_100_FULL) != 0) 3121 adapter->advertise |= 1 << 0; 3122 if ((speed & IXGBE_LINK_SPEED_10_FULL) != 0) 3123 adapter->advertise |= 1 << 3; 3124 if ((speed & IXGBE_LINK_SPEED_2_5GB_FULL) != 0) 3125 adapter->advertise |= 1 << 4; 3126 if ((speed & IXGBE_LINK_SPEED_5GB_FULL) != 0) 3127 adapter->advertise |= 1 << 5; 3128 } 3129 3130 return (0); 3131 3132 invalid: 3133 device_printf(adapter->dev, "Invalid media type!\n"); 3134 3135 return (EINVAL); 3136 } /* ixgbe_media_change */ 3137 3138 /************************************************************************ 3139 * ixgbe_msix_admin - Link status change ISR (MSI/MSI-X) 3140 ************************************************************************/ 3141 static int 3142 ixgbe_msix_admin(void *arg) 3143 { 3144 struct adapter *adapter = arg; 3145 struct ixgbe_hw *hw = &adapter->hw; 3146 u32 eicr; 3147 u32 eims_orig; 3148 u32 eims_disable = 0; 3149 3150 IXGBE_EVC_ADD(&adapter->admin_irqev, 1); 3151 3152 eims_orig = IXGBE_READ_REG(hw, IXGBE_EIMS); 3153 /* Pause other interrupts */ 3154 IXGBE_WRITE_REG(hw, IXGBE_EIMC, IXGBE_MSIX_OTHER_CLEAR_MASK); 3155 3156 /* 3157 * First get the cause. 3158 * 3159 * The specifications of 82598, 82599, X540 and X550 say EICS register 3160 * is write only. However, Linux says it is a workaround for silicon 3161 * errata to read EICS instead of EICR to get interrupt cause. 3162 * At least, reading EICR clears lower 16bits of EIMS on 82598. 3163 */ 3164 eicr = IXGBE_READ_REG(hw, IXGBE_EICS); 3165 /* Be sure the queue bits are not cleared */ 3166 eicr &= ~IXGBE_EICR_RTX_QUEUE; 3167 /* Clear all OTHER interrupts with write */ 3168 IXGBE_WRITE_REG(hw, IXGBE_EICR, eicr); 3169 3170 ixgbe_intr_admin_common(adapter, eicr, &eims_disable); 3171 3172 /* Re-enable some OTHER interrupts */ 3173 IXGBE_WRITE_REG(hw, IXGBE_EIMS, eims_orig & ~eims_disable); 3174 3175 return 1; 3176 } /* ixgbe_msix_admin */ 3177 3178 static void 3179 ixgbe_intr_admin_common(struct adapter *adapter, u32 eicr, u32 *eims_disable) 3180 { 3181 struct ixgbe_hw *hw = &adapter->hw; 3182 u32 eicr_mask; 3183 u32 task_requests = 0; 3184 s32 retval; 3185 3186 /* Link status change */ 3187 if (eicr & IXGBE_EICR_LSC) { 3188 task_requests |= IXGBE_REQUEST_TASK_LSC; 3189 *eims_disable |= IXGBE_EIMS_LSC; 3190 } 3191 3192 if (ixgbe_is_sfp(hw)) { 3193 /* Pluggable optics-related interrupt */ 3194 if (hw->mac.type >= ixgbe_mac_X540) 3195 eicr_mask = IXGBE_EICR_GPI_SDP0_X540; 3196 else 3197 eicr_mask = IXGBE_EICR_GPI_SDP2_BY_MAC(hw); 3198 3199 /* 3200 * An interrupt might not arrive when a module is inserted. 3201 * When an link status change interrupt occurred and the driver 3202 * still regard SFP as unplugged, issue the module softint 3203 * and then issue LSC interrupt. 3204 */ 3205 if ((eicr & eicr_mask) 3206 || ((hw->phy.sfp_type == ixgbe_sfp_type_not_present) 3207 && (eicr & IXGBE_EICR_LSC))) { 3208 task_requests |= IXGBE_REQUEST_TASK_MOD; 3209 *eims_disable |= IXGBE_EIMS_LSC; 3210 } 3211 3212 if ((hw->mac.type == ixgbe_mac_82599EB) && 3213 (eicr & IXGBE_EICR_GPI_SDP1_BY_MAC(hw))) { 3214 task_requests |= IXGBE_REQUEST_TASK_MSF; 3215 *eims_disable |= IXGBE_EIMS_GPI_SDP1_BY_MAC(hw); 3216 } 3217 } 3218 3219 if (adapter->hw.mac.type != ixgbe_mac_82598EB) { 3220 if ((adapter->feat_en & IXGBE_FEATURE_FDIR) && 3221 (eicr & IXGBE_EICR_FLOW_DIR)) { 3222 if (!atomic_cas_uint(&adapter->fdir_reinit, 0, 1)) { 3223 task_requests |= IXGBE_REQUEST_TASK_FDIR; 3224 /* Disable the interrupt */ 3225 *eims_disable |= IXGBE_EIMS_FLOW_DIR; 3226 } 3227 } 3228 3229 if (eicr & IXGBE_EICR_ECC) { 3230 device_printf(adapter->dev, 3231 "CRITICAL: ECC ERROR!! Please Reboot!!\n"); 3232 /* Disable interrupt to prevent log spam */ 3233 *eims_disable |= IXGBE_EICR_ECC; 3234 } 3235 3236 /* Check for over temp condition */ 3237 if (adapter->feat_en & IXGBE_FEATURE_TEMP_SENSOR) { 3238 switch (adapter->hw.mac.type) { 3239 case ixgbe_mac_X550EM_a: 3240 if (!(eicr & IXGBE_EICR_GPI_SDP0_X550EM_a)) 3241 break; 3242 /* Disable interrupt to prevent log spam */ 3243 *eims_disable |= IXGBE_EICR_GPI_SDP0_X550EM_a; 3244 3245 retval = hw->phy.ops.check_overtemp(hw); 3246 if (retval != IXGBE_ERR_OVERTEMP) 3247 break; 3248 device_printf(adapter->dev, 3249 "CRITICAL: OVER TEMP!! " 3250 "PHY IS SHUT DOWN!!\n"); 3251 device_printf(adapter->dev, 3252 "System shutdown required!\n"); 3253 break; 3254 default: 3255 if (!(eicr & IXGBE_EICR_TS)) 3256 break; 3257 /* Disable interrupt to prevent log spam */ 3258 *eims_disable |= IXGBE_EIMS_TS; 3259 3260 retval = hw->phy.ops.check_overtemp(hw); 3261 if (retval != IXGBE_ERR_OVERTEMP) 3262 break; 3263 device_printf(adapter->dev, 3264 "CRITICAL: OVER TEMP!! " 3265 "PHY IS SHUT DOWN!!\n"); 3266 device_printf(adapter->dev, 3267 "System shutdown required!\n"); 3268 break; 3269 } 3270 } 3271 3272 /* Check for VF message */ 3273 if ((adapter->feat_en & IXGBE_FEATURE_SRIOV) && 3274 (eicr & IXGBE_EICR_MAILBOX)) { 3275 task_requests |= IXGBE_REQUEST_TASK_MBX; 3276 *eims_disable |= IXGBE_EIMS_MAILBOX; 3277 } 3278 } 3279 3280 /* Check for fan failure */ 3281 if (adapter->feat_en & IXGBE_FEATURE_FAN_FAIL) { 3282 retval = ixgbe_check_fan_failure(adapter, eicr, true); 3283 if (retval == IXGBE_ERR_FAN_FAILURE) { 3284 /* Disable interrupt to prevent log spam */ 3285 *eims_disable |= IXGBE_EIMS_GPI_SDP1_BY_MAC(hw); 3286 } 3287 } 3288 3289 /* External PHY interrupt */ 3290 if ((hw->phy.type == ixgbe_phy_x550em_ext_t) && 3291 (eicr & IXGBE_EICR_GPI_SDP0_X540)) { 3292 task_requests |= IXGBE_REQUEST_TASK_PHY; 3293 *eims_disable |= IXGBE_EICR_GPI_SDP0_X540; 3294 } 3295 3296 if (task_requests != 0) { 3297 mutex_enter(&adapter->admin_mtx); 3298 adapter->task_requests |= task_requests; 3299 ixgbe_schedule_admin_tasklet(adapter); 3300 mutex_exit(&adapter->admin_mtx); 3301 } 3302 3303 } 3304 3305 static void 3306 ixgbe_eitr_write(struct adapter *adapter, uint32_t index, uint32_t itr) 3307 { 3308 3309 if (adapter->hw.mac.type == ixgbe_mac_82598EB) 3310 itr |= itr << 16; 3311 else 3312 itr |= IXGBE_EITR_CNT_WDIS; 3313 3314 IXGBE_WRITE_REG(&adapter->hw, IXGBE_EITR(index), itr); 3315 } 3316 3317 3318 /************************************************************************ 3319 * ixgbe_sysctl_interrupt_rate_handler 3320 ************************************************************************/ 3321 static int 3322 ixgbe_sysctl_interrupt_rate_handler(SYSCTLFN_ARGS) 3323 { 3324 struct sysctlnode node = *rnode; 3325 struct ix_queue *que = (struct ix_queue *)node.sysctl_data; 3326 struct adapter *adapter; 3327 uint32_t reg, usec, rate; 3328 int error; 3329 3330 if (que == NULL) 3331 return 0; 3332 3333 adapter = que->adapter; 3334 if (ixgbe_fw_recovery_mode_swflag(adapter)) 3335 return (EPERM); 3336 3337 reg = IXGBE_READ_REG(&adapter->hw, IXGBE_EITR(que->msix)); 3338 usec = ((reg & 0x0FF8) >> 3); 3339 if (usec > 0) 3340 rate = 500000 / usec; 3341 else 3342 rate = 0; 3343 node.sysctl_data = &rate; 3344 error = sysctl_lookup(SYSCTLFN_CALL(&node)); 3345 if (error || newp == NULL) 3346 return error; 3347 reg &= ~0xfff; /* default, no limitation */ 3348 if (rate > 0 && rate < 500000) { 3349 if (rate < 1000) 3350 rate = 1000; 3351 reg |= ((4000000 / rate) & 0xff8); 3352 /* 3353 * When RSC is used, ITR interval must be larger than 3354 * RSC_DELAY. Currently, we use 2us for RSC_DELAY. 3355 * The minimum value is always greater than 2us on 100M 3356 * (and 10M?(not documented)), but it's not on 1G and higher. 3357 */ 3358 if ((adapter->link_speed != IXGBE_LINK_SPEED_100_FULL) 3359 && (adapter->link_speed != IXGBE_LINK_SPEED_10_FULL)) { 3360 if ((adapter->num_queues > 1) 3361 && (reg < IXGBE_MIN_RSC_EITR_10G1G)) 3362 return EINVAL; 3363 } 3364 ixgbe_max_interrupt_rate = rate; 3365 } else 3366 ixgbe_max_interrupt_rate = 0; 3367 ixgbe_eitr_write(adapter, que->msix, reg); 3368 3369 return (0); 3370 } /* ixgbe_sysctl_interrupt_rate_handler */ 3371 3372 const struct sysctlnode * 3373 ixgbe_sysctl_instance(struct adapter *adapter) 3374 { 3375 const char *dvname; 3376 struct sysctllog **log; 3377 int rc; 3378 const struct sysctlnode *rnode; 3379 3380 if (adapter->sysctltop != NULL) 3381 return adapter->sysctltop; 3382 3383 log = &adapter->sysctllog; 3384 dvname = device_xname(adapter->dev); 3385 3386 if ((rc = sysctl_createv(log, 0, NULL, &rnode, 3387 0, CTLTYPE_NODE, dvname, 3388 SYSCTL_DESCR("ixgbe information and settings"), 3389 NULL, 0, NULL, 0, CTL_HW, CTL_CREATE, CTL_EOL)) != 0) 3390 goto err; 3391 3392 return rnode; 3393 err: 3394 device_printf(adapter->dev, 3395 "%s: sysctl_createv failed, rc = %d\n", __func__, rc); 3396 return NULL; 3397 } 3398 3399 /************************************************************************ 3400 * ixgbe_add_device_sysctls 3401 ************************************************************************/ 3402 static void 3403 ixgbe_add_device_sysctls(struct adapter *adapter) 3404 { 3405 device_t dev = adapter->dev; 3406 struct ixgbe_hw *hw = &adapter->hw; 3407 struct sysctllog **log; 3408 const struct sysctlnode *rnode, *cnode; 3409 3410 log = &adapter->sysctllog; 3411 3412 if ((rnode = ixgbe_sysctl_instance(adapter)) == NULL) { 3413 aprint_error_dev(dev, "could not create sysctl root\n"); 3414 return; 3415 } 3416 3417 if (sysctl_createv(log, 0, &rnode, &cnode, 3418 CTLFLAG_READWRITE, CTLTYPE_INT, 3419 "debug", SYSCTL_DESCR("Debug Info"), 3420 ixgbe_sysctl_debug, 0, (void *)adapter, 0, CTL_CREATE, CTL_EOL) 3421 != 0) 3422 aprint_error_dev(dev, "could not create sysctl\n"); 3423 3424 if (sysctl_createv(log, 0, &rnode, &cnode, 3425 CTLFLAG_READWRITE, CTLTYPE_INT, 3426 "rx_copy_len", SYSCTL_DESCR("RX Copy Length"), 3427 ixgbe_sysctl_rx_copy_len, 0, 3428 (void *)adapter, 0, CTL_CREATE, CTL_EOL) != 0) 3429 aprint_error_dev(dev, "could not create sysctl\n"); 3430 3431 if (sysctl_createv(log, 0, &rnode, &cnode, 3432 CTLFLAG_READONLY, CTLTYPE_INT, 3433 "num_rx_desc", SYSCTL_DESCR("Number of rx descriptors"), 3434 NULL, 0, &adapter->num_rx_desc, 0, CTL_CREATE, CTL_EOL) != 0) 3435 aprint_error_dev(dev, "could not create sysctl\n"); 3436 3437 if (sysctl_createv(log, 0, &rnode, &cnode, 3438 CTLFLAG_READONLY, CTLTYPE_INT, 3439 "num_queues", SYSCTL_DESCR("Number of queues"), 3440 NULL, 0, &adapter->num_queues, 0, CTL_CREATE, CTL_EOL) != 0) 3441 aprint_error_dev(dev, "could not create sysctl\n"); 3442 3443 /* Sysctls for all devices */ 3444 if (sysctl_createv(log, 0, &rnode, &cnode, CTLFLAG_READWRITE, 3445 CTLTYPE_INT, "fc", SYSCTL_DESCR(IXGBE_SYSCTL_DESC_SET_FC), 3446 ixgbe_sysctl_flowcntl, 0, (void *)adapter, 0, CTL_CREATE, 3447 CTL_EOL) != 0) 3448 aprint_error_dev(dev, "could not create sysctl\n"); 3449 3450 adapter->enable_aim = ixgbe_enable_aim; 3451 if (sysctl_createv(log, 0, &rnode, &cnode, CTLFLAG_READWRITE, 3452 CTLTYPE_BOOL, "enable_aim", SYSCTL_DESCR("Interrupt Moderation"), 3453 NULL, 0, &adapter->enable_aim, 0, CTL_CREATE, CTL_EOL) != 0) 3454 aprint_error_dev(dev, "could not create sysctl\n"); 3455 3456 if (sysctl_createv(log, 0, &rnode, &cnode, 3457 CTLFLAG_READWRITE, CTLTYPE_INT, 3458 "advertise_speed", SYSCTL_DESCR(IXGBE_SYSCTL_DESC_ADV_SPEED), 3459 ixgbe_sysctl_advertise, 0, (void *)adapter, 0, CTL_CREATE, 3460 CTL_EOL) != 0) 3461 aprint_error_dev(dev, "could not create sysctl\n"); 3462 3463 /* 3464 * If each "que->txrx_use_workqueue" is changed in sysctl handler, 3465 * it causesflip-flopping softint/workqueue mode in one deferred 3466 * processing. Therefore, preempt_disable()/preempt_enable() are 3467 * required in ixgbe_sched_handle_que() to avoid 3468 * KASSERT(ixgbe_sched_handle_que()) in softint_schedule(). 3469 * I think changing "que->txrx_use_workqueue" in interrupt handler 3470 * is lighter than doing preempt_disable()/preempt_enable() in every 3471 * ixgbe_sched_handle_que(). 3472 */ 3473 adapter->txrx_use_workqueue = ixgbe_txrx_workqueue; 3474 if (sysctl_createv(log, 0, &rnode, &cnode, CTLFLAG_READWRITE, 3475 CTLTYPE_BOOL, "txrx_workqueue", 3476 SYSCTL_DESCR("Use workqueue for packet processing"), 3477 NULL, 0, &adapter->txrx_use_workqueue, 0, CTL_CREATE, 3478 CTL_EOL) != 0) 3479 aprint_error_dev(dev, "could not create sysctl\n"); 3480 3481 #ifdef IXGBE_DEBUG 3482 /* testing sysctls (for all devices) */ 3483 if (sysctl_createv(log, 0, &rnode, &cnode, CTLFLAG_READWRITE, 3484 CTLTYPE_INT, "power_state", SYSCTL_DESCR("PCI Power State"), 3485 ixgbe_sysctl_power_state, 0, (void *)adapter, 0, CTL_CREATE, 3486 CTL_EOL) != 0) 3487 aprint_error_dev(dev, "could not create sysctl\n"); 3488 3489 if (sysctl_createv(log, 0, &rnode, &cnode, CTLFLAG_READONLY, 3490 CTLTYPE_STRING, "print_rss_config", 3491 SYSCTL_DESCR("Prints RSS Configuration"), 3492 ixgbe_sysctl_print_rss_config, 0, (void *)adapter, 0, CTL_CREATE, 3493 CTL_EOL) != 0) 3494 aprint_error_dev(dev, "could not create sysctl\n"); 3495 #endif 3496 /* for X550 series devices */ 3497 if (hw->mac.type >= ixgbe_mac_X550) 3498 if (sysctl_createv(log, 0, &rnode, &cnode, CTLFLAG_READWRITE, 3499 CTLTYPE_INT, "dmac", SYSCTL_DESCR("DMA Coalesce"), 3500 ixgbe_sysctl_dmac, 0, (void *)adapter, 0, CTL_CREATE, 3501 CTL_EOL) != 0) 3502 aprint_error_dev(dev, "could not create sysctl\n"); 3503 3504 /* for WoL-capable devices */ 3505 if (adapter->wol_support) { 3506 if (sysctl_createv(log, 0, &rnode, &cnode, CTLFLAG_READWRITE, 3507 CTLTYPE_BOOL, "wol_enable", 3508 SYSCTL_DESCR("Enable/Disable Wake on LAN"), 3509 ixgbe_sysctl_wol_enable, 0, (void *)adapter, 0, CTL_CREATE, 3510 CTL_EOL) != 0) 3511 aprint_error_dev(dev, "could not create sysctl\n"); 3512 3513 if (sysctl_createv(log, 0, &rnode, &cnode, CTLFLAG_READWRITE, 3514 CTLTYPE_INT, "wufc", 3515 SYSCTL_DESCR("Enable/Disable Wake Up Filters"), 3516 ixgbe_sysctl_wufc, 0, (void *)adapter, 0, CTL_CREATE, 3517 CTL_EOL) != 0) 3518 aprint_error_dev(dev, "could not create sysctl\n"); 3519 } 3520 3521 /* for X552/X557-AT devices */ 3522 if (hw->device_id == IXGBE_DEV_ID_X550EM_X_10G_T) { 3523 const struct sysctlnode *phy_node; 3524 3525 if (sysctl_createv(log, 0, &rnode, &phy_node, 0, CTLTYPE_NODE, 3526 "phy", SYSCTL_DESCR("External PHY sysctls"), 3527 NULL, 0, NULL, 0, CTL_CREATE, CTL_EOL) != 0) { 3528 aprint_error_dev(dev, "could not create sysctl\n"); 3529 return; 3530 } 3531 3532 if (sysctl_createv(log, 0, &phy_node, &cnode, CTLFLAG_READONLY, 3533 CTLTYPE_INT, "temp", 3534 SYSCTL_DESCR("Current External PHY Temperature (Celsius)"), 3535 ixgbe_sysctl_phy_temp, 0, (void *)adapter, 0, CTL_CREATE, 3536 CTL_EOL) != 0) 3537 aprint_error_dev(dev, "could not create sysctl\n"); 3538 3539 if (sysctl_createv(log, 0, &phy_node, &cnode, CTLFLAG_READONLY, 3540 CTLTYPE_INT, "overtemp_occurred", 3541 SYSCTL_DESCR( 3542 "External PHY High Temperature Event Occurred"), 3543 ixgbe_sysctl_phy_overtemp_occurred, 0, (void *)adapter, 0, 3544 CTL_CREATE, CTL_EOL) != 0) 3545 aprint_error_dev(dev, "could not create sysctl\n"); 3546 } 3547 3548 if ((hw->mac.type == ixgbe_mac_X550EM_a) 3549 && (hw->phy.type == ixgbe_phy_fw)) 3550 if (sysctl_createv(log, 0, &rnode, &cnode, CTLFLAG_READWRITE, 3551 CTLTYPE_BOOL, "force_10_100_autonego", 3552 SYSCTL_DESCR("Force autonego on 10M and 100M"), 3553 NULL, 0, &hw->phy.force_10_100_autonego, 0, 3554 CTL_CREATE, CTL_EOL) != 0) 3555 aprint_error_dev(dev, "could not create sysctl\n"); 3556 3557 if (adapter->feat_cap & IXGBE_FEATURE_EEE) { 3558 if (sysctl_createv(log, 0, &rnode, &cnode, CTLFLAG_READWRITE, 3559 CTLTYPE_INT, "eee_state", 3560 SYSCTL_DESCR("EEE Power Save State"), 3561 ixgbe_sysctl_eee_state, 0, (void *)adapter, 0, CTL_CREATE, 3562 CTL_EOL) != 0) 3563 aprint_error_dev(dev, "could not create sysctl\n"); 3564 } 3565 } /* ixgbe_add_device_sysctls */ 3566 3567 /************************************************************************ 3568 * ixgbe_allocate_pci_resources 3569 ************************************************************************/ 3570 static int 3571 ixgbe_allocate_pci_resources(struct adapter *adapter, 3572 const struct pci_attach_args *pa) 3573 { 3574 pcireg_t memtype, csr; 3575 device_t dev = adapter->dev; 3576 bus_addr_t addr; 3577 int flags; 3578 3579 memtype = pci_mapreg_type(pa->pa_pc, pa->pa_tag, PCI_BAR(0)); 3580 switch (memtype) { 3581 case PCI_MAPREG_TYPE_MEM | PCI_MAPREG_MEM_TYPE_32BIT: 3582 case PCI_MAPREG_TYPE_MEM | PCI_MAPREG_MEM_TYPE_64BIT: 3583 adapter->osdep.mem_bus_space_tag = pa->pa_memt; 3584 if (pci_mapreg_info(pa->pa_pc, pa->pa_tag, PCI_BAR(0), 3585 memtype, &addr, &adapter->osdep.mem_size, &flags) != 0) 3586 goto map_err; 3587 if ((flags & BUS_SPACE_MAP_PREFETCHABLE) != 0) { 3588 aprint_normal_dev(dev, "clearing prefetchable bit\n"); 3589 flags &= ~BUS_SPACE_MAP_PREFETCHABLE; 3590 } 3591 if (bus_space_map(adapter->osdep.mem_bus_space_tag, addr, 3592 adapter->osdep.mem_size, flags, 3593 &adapter->osdep.mem_bus_space_handle) != 0) { 3594 map_err: 3595 adapter->osdep.mem_size = 0; 3596 aprint_error_dev(dev, "unable to map BAR0\n"); 3597 return ENXIO; 3598 } 3599 /* 3600 * Enable address decoding for memory range in case BIOS or 3601 * UEFI don't set it. 3602 */ 3603 csr = pci_conf_read(pa->pa_pc, pa->pa_tag, 3604 PCI_COMMAND_STATUS_REG); 3605 csr |= PCI_COMMAND_MEM_ENABLE; 3606 pci_conf_write(pa->pa_pc, pa->pa_tag, PCI_COMMAND_STATUS_REG, 3607 csr); 3608 break; 3609 default: 3610 aprint_error_dev(dev, "unexpected type on BAR0\n"); 3611 return ENXIO; 3612 } 3613 3614 return (0); 3615 } /* ixgbe_allocate_pci_resources */ 3616 3617 static void 3618 ixgbe_free_deferred_handlers(struct adapter *adapter) 3619 { 3620 struct ix_queue *que = adapter->queues; 3621 struct tx_ring *txr = adapter->tx_rings; 3622 int i; 3623 3624 for (i = 0; i < adapter->num_queues; i++, que++, txr++) { 3625 if (!(adapter->feat_en & IXGBE_FEATURE_LEGACY_TX)) { 3626 if (txr->txr_si != NULL) 3627 softint_disestablish(txr->txr_si); 3628 } 3629 if (que->que_si != NULL) 3630 softint_disestablish(que->que_si); 3631 } 3632 if (adapter->txr_wq != NULL) 3633 workqueue_destroy(adapter->txr_wq); 3634 if (adapter->txr_wq_enqueued != NULL) 3635 percpu_free(adapter->txr_wq_enqueued, sizeof(u_int)); 3636 if (adapter->que_wq != NULL) 3637 workqueue_destroy(adapter->que_wq); 3638 3639 if (adapter->admin_wq != NULL) { 3640 workqueue_destroy(adapter->admin_wq); 3641 adapter->admin_wq = NULL; 3642 } 3643 if (adapter->timer_wq != NULL) { 3644 workqueue_destroy(adapter->timer_wq); 3645 adapter->timer_wq = NULL; 3646 } 3647 if (adapter->recovery_mode_timer_wq != NULL) { 3648 /* 3649 * ixgbe_ifstop() doesn't call the workqueue_wait() for 3650 * the recovery_mode_timer workqueue, so call it here. 3651 */ 3652 workqueue_wait(adapter->recovery_mode_timer_wq, 3653 &adapter->recovery_mode_timer_wc); 3654 atomic_store_relaxed(&adapter->recovery_mode_timer_pending, 0); 3655 workqueue_destroy(adapter->recovery_mode_timer_wq); 3656 adapter->recovery_mode_timer_wq = NULL; 3657 } 3658 } /* ixgbe_free_deferred_handlers */ 3659 3660 /************************************************************************ 3661 * ixgbe_detach - Device removal routine 3662 * 3663 * Called when the driver is being removed. 3664 * Stops the adapter and deallocates all the resources 3665 * that were allocated for driver operation. 3666 * 3667 * return 0 on success, positive on failure 3668 ************************************************************************/ 3669 static int 3670 ixgbe_detach(device_t dev, int flags) 3671 { 3672 struct adapter *adapter = device_private(dev); 3673 struct rx_ring *rxr = adapter->rx_rings; 3674 struct tx_ring *txr = adapter->tx_rings; 3675 struct ixgbe_hw *hw = &adapter->hw; 3676 struct ixgbe_hw_stats *stats = &adapter->stats.pf; 3677 u32 ctrl_ext; 3678 int i; 3679 3680 INIT_DEBUGOUT("ixgbe_detach: begin"); 3681 if (adapter->osdep.attached == false) 3682 return 0; 3683 3684 if (ixgbe_pci_iov_detach(dev) != 0) { 3685 device_printf(dev, "SR-IOV in use; detach first.\n"); 3686 return (EBUSY); 3687 } 3688 3689 if (VLAN_ATTACHED(&adapter->osdep.ec) && 3690 (flags & (DETACH_SHUTDOWN | DETACH_FORCE)) == 0) { 3691 aprint_error_dev(dev, "VLANs in use, detach first\n"); 3692 return (EBUSY); 3693 } 3694 3695 ether_ifdetach(adapter->ifp); 3696 3697 adapter->osdep.detaching = true; 3698 /* 3699 * Stop the interface. ixgbe_setup_low_power_mode() calls 3700 * ixgbe_ifstop(), so it's not required to call ixgbe_ifstop() 3701 * directly. 3702 */ 3703 ixgbe_setup_low_power_mode(adapter); 3704 3705 callout_halt(&adapter->timer, NULL); 3706 if (adapter->feat_en & IXGBE_FEATURE_RECOVERY_MODE) 3707 callout_halt(&adapter->recovery_mode_timer, NULL); 3708 3709 workqueue_wait(adapter->admin_wq, &adapter->admin_wc); 3710 atomic_store_relaxed(&adapter->admin_pending, 0); 3711 workqueue_wait(adapter->timer_wq, &adapter->timer_wc); 3712 atomic_store_relaxed(&adapter->timer_pending, 0); 3713 3714 pmf_device_deregister(dev); 3715 3716 ixgbe_free_deferred_handlers(adapter); 3717 3718 /* let hardware know driver is unloading */ 3719 ctrl_ext = IXGBE_READ_REG(&adapter->hw, IXGBE_CTRL_EXT); 3720 ctrl_ext &= ~IXGBE_CTRL_EXT_DRV_LOAD; 3721 IXGBE_WRITE_REG(&adapter->hw, IXGBE_CTRL_EXT, ctrl_ext); 3722 3723 if (adapter->feat_en & IXGBE_FEATURE_NETMAP) 3724 netmap_detach(adapter->ifp); 3725 3726 ixgbe_free_pci_resources(adapter); 3727 #if 0 /* XXX the NetBSD port is probably missing something here */ 3728 bus_generic_detach(dev); 3729 #endif 3730 if_detach(adapter->ifp); 3731 ifmedia_fini(&adapter->media); 3732 if_percpuq_destroy(adapter->ipq); 3733 3734 sysctl_teardown(&adapter->sysctllog); 3735 evcnt_detach(&adapter->efbig_tx_dma_setup); 3736 evcnt_detach(&adapter->mbuf_defrag_failed); 3737 evcnt_detach(&adapter->efbig2_tx_dma_setup); 3738 evcnt_detach(&adapter->einval_tx_dma_setup); 3739 evcnt_detach(&adapter->other_tx_dma_setup); 3740 evcnt_detach(&adapter->eagain_tx_dma_setup); 3741 evcnt_detach(&adapter->enomem_tx_dma_setup); 3742 evcnt_detach(&adapter->watchdog_events); 3743 evcnt_detach(&adapter->tso_err); 3744 evcnt_detach(&adapter->admin_irqev); 3745 evcnt_detach(&adapter->link_workev); 3746 evcnt_detach(&adapter->mod_workev); 3747 evcnt_detach(&adapter->msf_workev); 3748 evcnt_detach(&adapter->phy_workev); 3749 3750 for (i = 0; i < IXGBE_TC_COUNTER_NUM; i++) { 3751 if (i < __arraycount(stats->mpc)) { 3752 evcnt_detach(&stats->mpc[i]); 3753 if (hw->mac.type == ixgbe_mac_82598EB) 3754 evcnt_detach(&stats->rnbc[i]); 3755 } 3756 if (i < __arraycount(stats->pxontxc)) { 3757 evcnt_detach(&stats->pxontxc[i]); 3758 evcnt_detach(&stats->pxonrxc[i]); 3759 evcnt_detach(&stats->pxofftxc[i]); 3760 evcnt_detach(&stats->pxoffrxc[i]); 3761 if (hw->mac.type >= ixgbe_mac_82599EB) 3762 evcnt_detach(&stats->pxon2offc[i]); 3763 } 3764 } 3765 3766 txr = adapter->tx_rings; 3767 for (i = 0; i < adapter->num_queues; i++, rxr++, txr++) { 3768 evcnt_detach(&adapter->queues[i].irqs); 3769 evcnt_detach(&adapter->queues[i].handleq); 3770 evcnt_detach(&adapter->queues[i].req); 3771 evcnt_detach(&txr->no_desc_avail); 3772 evcnt_detach(&txr->total_packets); 3773 evcnt_detach(&txr->tso_tx); 3774 #ifndef IXGBE_LEGACY_TX 3775 evcnt_detach(&txr->pcq_drops); 3776 #endif 3777 3778 if (i < __arraycount(stats->qprc)) { 3779 evcnt_detach(&stats->qprc[i]); 3780 evcnt_detach(&stats->qptc[i]); 3781 evcnt_detach(&stats->qbrc[i]); 3782 evcnt_detach(&stats->qbtc[i]); 3783 if (hw->mac.type >= ixgbe_mac_82599EB) 3784 evcnt_detach(&stats->qprdc[i]); 3785 } 3786 3787 evcnt_detach(&rxr->rx_packets); 3788 evcnt_detach(&rxr->rx_bytes); 3789 evcnt_detach(&rxr->rx_copies); 3790 evcnt_detach(&rxr->no_mbuf); 3791 evcnt_detach(&rxr->rx_discarded); 3792 } 3793 evcnt_detach(&stats->ipcs); 3794 evcnt_detach(&stats->l4cs); 3795 evcnt_detach(&stats->ipcs_bad); 3796 evcnt_detach(&stats->l4cs_bad); 3797 evcnt_detach(&stats->intzero); 3798 evcnt_detach(&stats->legint); 3799 evcnt_detach(&stats->crcerrs); 3800 evcnt_detach(&stats->illerrc); 3801 evcnt_detach(&stats->errbc); 3802 evcnt_detach(&stats->mspdc); 3803 if (hw->mac.type >= ixgbe_mac_X550) 3804 evcnt_detach(&stats->mbsdc); 3805 evcnt_detach(&stats->mpctotal); 3806 evcnt_detach(&stats->mlfc); 3807 evcnt_detach(&stats->mrfc); 3808 evcnt_detach(&stats->rlec); 3809 evcnt_detach(&stats->lxontxc); 3810 evcnt_detach(&stats->lxonrxc); 3811 evcnt_detach(&stats->lxofftxc); 3812 evcnt_detach(&stats->lxoffrxc); 3813 3814 /* Packet Reception Stats */ 3815 evcnt_detach(&stats->tor); 3816 evcnt_detach(&stats->gorc); 3817 evcnt_detach(&stats->tpr); 3818 evcnt_detach(&stats->gprc); 3819 evcnt_detach(&stats->mprc); 3820 evcnt_detach(&stats->bprc); 3821 evcnt_detach(&stats->prc64); 3822 evcnt_detach(&stats->prc127); 3823 evcnt_detach(&stats->prc255); 3824 evcnt_detach(&stats->prc511); 3825 evcnt_detach(&stats->prc1023); 3826 evcnt_detach(&stats->prc1522); 3827 evcnt_detach(&stats->ruc); 3828 evcnt_detach(&stats->rfc); 3829 evcnt_detach(&stats->roc); 3830 evcnt_detach(&stats->rjc); 3831 evcnt_detach(&stats->mngprc); 3832 evcnt_detach(&stats->mngpdc); 3833 evcnt_detach(&stats->xec); 3834 3835 /* Packet Transmission Stats */ 3836 evcnt_detach(&stats->gotc); 3837 evcnt_detach(&stats->tpt); 3838 evcnt_detach(&stats->gptc); 3839 evcnt_detach(&stats->bptc); 3840 evcnt_detach(&stats->mptc); 3841 evcnt_detach(&stats->mngptc); 3842 evcnt_detach(&stats->ptc64); 3843 evcnt_detach(&stats->ptc127); 3844 evcnt_detach(&stats->ptc255); 3845 evcnt_detach(&stats->ptc511); 3846 evcnt_detach(&stats->ptc1023); 3847 evcnt_detach(&stats->ptc1522); 3848 3849 ixgbe_free_queues(adapter); 3850 free(adapter->mta, M_DEVBUF); 3851 3852 mutex_destroy(&adapter->admin_mtx); /* XXX appropriate order? */ 3853 IXGBE_CORE_LOCK_DESTROY(adapter); 3854 3855 return (0); 3856 } /* ixgbe_detach */ 3857 3858 /************************************************************************ 3859 * ixgbe_setup_low_power_mode - LPLU/WoL preparation 3860 * 3861 * Prepare the adapter/port for LPLU and/or WoL 3862 ************************************************************************/ 3863 static int 3864 ixgbe_setup_low_power_mode(struct adapter *adapter) 3865 { 3866 struct ixgbe_hw *hw = &adapter->hw; 3867 device_t dev = adapter->dev; 3868 struct ifnet *ifp = adapter->ifp; 3869 s32 error = 0; 3870 3871 /* Limit power management flow to X550EM baseT */ 3872 if (hw->device_id == IXGBE_DEV_ID_X550EM_X_10G_T && 3873 hw->phy.ops.enter_lplu) { 3874 /* X550EM baseT adapters need a special LPLU flow */ 3875 hw->phy.reset_disable = true; 3876 ixgbe_ifstop(ifp, 1); 3877 error = hw->phy.ops.enter_lplu(hw); 3878 if (error) 3879 device_printf(dev, 3880 "Error entering LPLU: %d\n", error); 3881 hw->phy.reset_disable = false; 3882 } else { 3883 /* Just stop for other adapters */ 3884 ixgbe_ifstop(ifp, 1); 3885 } 3886 3887 IXGBE_CORE_LOCK(adapter); 3888 3889 if (!hw->wol_enabled) { 3890 ixgbe_set_phy_power(hw, FALSE); 3891 IXGBE_WRITE_REG(hw, IXGBE_WUFC, 0); 3892 IXGBE_WRITE_REG(hw, IXGBE_WUC, 0); 3893 } else { 3894 /* Turn off support for APM wakeup. (Using ACPI instead) */ 3895 IXGBE_WRITE_REG(hw, IXGBE_GRC_BY_MAC(hw), 3896 IXGBE_READ_REG(hw, IXGBE_GRC_BY_MAC(hw)) & ~(u32)2); 3897 3898 /* 3899 * Clear Wake Up Status register to prevent any previous wakeup 3900 * events from waking us up immediately after we suspend. 3901 */ 3902 IXGBE_WRITE_REG(hw, IXGBE_WUS, 0xffffffff); 3903 3904 /* 3905 * Program the Wakeup Filter Control register with user filter 3906 * settings 3907 */ 3908 IXGBE_WRITE_REG(hw, IXGBE_WUFC, adapter->wufc); 3909 3910 /* Enable wakeups and power management in Wakeup Control */ 3911 IXGBE_WRITE_REG(hw, IXGBE_WUC, 3912 IXGBE_WUC_WKEN | IXGBE_WUC_PME_EN); 3913 3914 } 3915 3916 IXGBE_CORE_UNLOCK(adapter); 3917 3918 return error; 3919 } /* ixgbe_setup_low_power_mode */ 3920 3921 /************************************************************************ 3922 * ixgbe_shutdown - Shutdown entry point 3923 ************************************************************************/ 3924 #if 0 /* XXX NetBSD ought to register something like this through pmf(9) */ 3925 static int 3926 ixgbe_shutdown(device_t dev) 3927 { 3928 struct adapter *adapter = device_private(dev); 3929 int error = 0; 3930 3931 INIT_DEBUGOUT("ixgbe_shutdown: begin"); 3932 3933 error = ixgbe_setup_low_power_mode(adapter); 3934 3935 return (error); 3936 } /* ixgbe_shutdown */ 3937 #endif 3938 3939 /************************************************************************ 3940 * ixgbe_suspend 3941 * 3942 * From D0 to D3 3943 ************************************************************************/ 3944 static bool 3945 ixgbe_suspend(device_t dev, const pmf_qual_t *qual) 3946 { 3947 struct adapter *adapter = device_private(dev); 3948 int error = 0; 3949 3950 INIT_DEBUGOUT("ixgbe_suspend: begin"); 3951 3952 error = ixgbe_setup_low_power_mode(adapter); 3953 3954 return (error); 3955 } /* ixgbe_suspend */ 3956 3957 /************************************************************************ 3958 * ixgbe_resume 3959 * 3960 * From D3 to D0 3961 ************************************************************************/ 3962 static bool 3963 ixgbe_resume(device_t dev, const pmf_qual_t *qual) 3964 { 3965 struct adapter *adapter = device_private(dev); 3966 struct ifnet *ifp = adapter->ifp; 3967 struct ixgbe_hw *hw = &adapter->hw; 3968 u32 wus; 3969 3970 INIT_DEBUGOUT("ixgbe_resume: begin"); 3971 3972 IXGBE_CORE_LOCK(adapter); 3973 3974 /* Read & clear WUS register */ 3975 wus = IXGBE_READ_REG(hw, IXGBE_WUS); 3976 if (wus) 3977 device_printf(dev, "Woken up by (WUS): %#010x\n", 3978 IXGBE_READ_REG(hw, IXGBE_WUS)); 3979 IXGBE_WRITE_REG(hw, IXGBE_WUS, 0xffffffff); 3980 /* And clear WUFC until next low-power transition */ 3981 IXGBE_WRITE_REG(hw, IXGBE_WUFC, 0); 3982 3983 /* 3984 * Required after D3->D0 transition; 3985 * will re-advertise all previous advertised speeds 3986 */ 3987 if (ifp->if_flags & IFF_UP) 3988 ixgbe_init_locked(adapter); 3989 3990 IXGBE_CORE_UNLOCK(adapter); 3991 3992 return true; 3993 } /* ixgbe_resume */ 3994 3995 /* 3996 * Set the various hardware offload abilities. 3997 * 3998 * This takes the ifnet's if_capenable flags (e.g. set by the user using 3999 * ifconfig) and indicates to the OS via the ifnet's if_hwassist field what 4000 * mbuf offload flags the driver will understand. 4001 */ 4002 static void 4003 ixgbe_set_if_hwassist(struct adapter *adapter) 4004 { 4005 /* XXX */ 4006 } 4007 4008 /************************************************************************ 4009 * ixgbe_init_locked - Init entry point 4010 * 4011 * Used in two ways: It is used by the stack as an init 4012 * entry point in network interface structure. It is also 4013 * used by the driver as a hw/sw initialization routine to 4014 * get to a consistent state. 4015 * 4016 * return 0 on success, positive on failure 4017 ************************************************************************/ 4018 static void 4019 ixgbe_init_locked(struct adapter *adapter) 4020 { 4021 struct ifnet *ifp = adapter->ifp; 4022 device_t dev = adapter->dev; 4023 struct ixgbe_hw *hw = &adapter->hw; 4024 struct ix_queue *que; 4025 struct tx_ring *txr; 4026 struct rx_ring *rxr; 4027 u32 txdctl, mhadd; 4028 u32 rxdctl, rxctrl; 4029 u32 ctrl_ext; 4030 bool unsupported_sfp = false; 4031 int i, j, error; 4032 4033 /* XXX check IFF_UP and IFF_RUNNING, power-saving state! */ 4034 4035 KASSERT(mutex_owned(&adapter->core_mtx)); 4036 INIT_DEBUGOUT("ixgbe_init_locked: begin"); 4037 4038 hw->need_unsupported_sfp_recovery = false; 4039 hw->adapter_stopped = FALSE; 4040 ixgbe_stop_adapter(hw); 4041 callout_stop(&adapter->timer); 4042 if (adapter->feat_en & IXGBE_FEATURE_RECOVERY_MODE) 4043 callout_stop(&adapter->recovery_mode_timer); 4044 for (i = 0, que = adapter->queues; i < adapter->num_queues; i++, que++) 4045 que->disabled_count = 0; 4046 4047 /* XXX I moved this here from the SIOCSIFMTU case in ixgbe_ioctl(). */ 4048 adapter->max_frame_size = 4049 ifp->if_mtu + ETHER_HDR_LEN + ETHER_CRC_LEN; 4050 4051 /* Queue indices may change with IOV mode */ 4052 ixgbe_align_all_queue_indices(adapter); 4053 4054 /* reprogram the RAR[0] in case user changed it. */ 4055 ixgbe_set_rar(hw, 0, hw->mac.addr, adapter->pool, IXGBE_RAH_AV); 4056 4057 /* Get the latest mac address, User can use a LAA */ 4058 memcpy(hw->mac.addr, CLLADDR(ifp->if_sadl), 4059 IXGBE_ETH_LENGTH_OF_ADDRESS); 4060 ixgbe_set_rar(hw, 0, hw->mac.addr, adapter->pool, 1); 4061 hw->addr_ctrl.rar_used_count = 1; 4062 4063 /* Set hardware offload abilities from ifnet flags */ 4064 ixgbe_set_if_hwassist(adapter); 4065 4066 /* Prepare transmit descriptors and buffers */ 4067 if (ixgbe_setup_transmit_structures(adapter)) { 4068 device_printf(dev, "Could not setup transmit structures\n"); 4069 ixgbe_stop_locked(adapter); 4070 return; 4071 } 4072 4073 ixgbe_init_hw(hw); 4074 4075 ixgbe_initialize_iov(adapter); 4076 4077 ixgbe_initialize_transmit_units(adapter); 4078 4079 /* Setup Multicast table */ 4080 ixgbe_set_rxfilter(adapter); 4081 4082 /* Use fixed buffer size, even for jumbo frames */ 4083 adapter->rx_mbuf_sz = MCLBYTES; 4084 4085 /* Prepare receive descriptors and buffers */ 4086 error = ixgbe_setup_receive_structures(adapter); 4087 if (error) { 4088 device_printf(dev, 4089 "Could not setup receive structures (err = %d)\n", error); 4090 ixgbe_stop_locked(adapter); 4091 return; 4092 } 4093 4094 /* Configure RX settings */ 4095 ixgbe_initialize_receive_units(adapter); 4096 4097 /* Initialize variable holding task enqueue requests interrupts */ 4098 adapter->task_requests = 0; 4099 4100 /* Enable SDP & MSI-X interrupts based on adapter */ 4101 ixgbe_config_gpie(adapter); 4102 4103 /* Set MTU size */ 4104 if (ifp->if_mtu > ETHERMTU) { 4105 /* aka IXGBE_MAXFRS on 82599 and newer */ 4106 mhadd = IXGBE_READ_REG(hw, IXGBE_MHADD); 4107 mhadd &= ~IXGBE_MHADD_MFS_MASK; 4108 mhadd |= adapter->max_frame_size << IXGBE_MHADD_MFS_SHIFT; 4109 IXGBE_WRITE_REG(hw, IXGBE_MHADD, mhadd); 4110 } 4111 4112 /* Now enable all the queues */ 4113 for (i = 0; i < adapter->num_queues; i++) { 4114 txr = &adapter->tx_rings[i]; 4115 txdctl = IXGBE_READ_REG(hw, IXGBE_TXDCTL(txr->me)); 4116 txdctl |= IXGBE_TXDCTL_ENABLE; 4117 /* Set WTHRESH to 8, burst writeback */ 4118 txdctl |= IXGBE_TX_WTHRESH << IXGBE_TXDCTL_WTHRESH_SHIFT; 4119 /* 4120 * When the internal queue falls below PTHRESH (32), 4121 * start prefetching as long as there are at least 4122 * HTHRESH (1) buffers ready. The values are taken 4123 * from the Intel linux driver 3.8.21. 4124 * Prefetching enables tx line rate even with 1 queue. 4125 */ 4126 txdctl |= (32 << 0) | (1 << 8); 4127 IXGBE_WRITE_REG(hw, IXGBE_TXDCTL(txr->me), txdctl); 4128 } 4129 4130 for (i = 0; i < adapter->num_queues; i++) { 4131 rxr = &adapter->rx_rings[i]; 4132 rxdctl = IXGBE_READ_REG(hw, IXGBE_RXDCTL(rxr->me)); 4133 if (hw->mac.type == ixgbe_mac_82598EB) { 4134 /* 4135 * PTHRESH = 21 4136 * HTHRESH = 4 4137 * WTHRESH = 8 4138 */ 4139 rxdctl &= ~0x3FFFFF; 4140 rxdctl |= 0x080420; 4141 } 4142 rxdctl |= IXGBE_RXDCTL_ENABLE; 4143 IXGBE_WRITE_REG(hw, IXGBE_RXDCTL(rxr->me), rxdctl); 4144 for (j = 0; j < 10; j++) { 4145 if (IXGBE_READ_REG(hw, IXGBE_RXDCTL(rxr->me)) & 4146 IXGBE_RXDCTL_ENABLE) 4147 break; 4148 else 4149 msec_delay(1); 4150 } 4151 IXGBE_WRITE_BARRIER(hw); 4152 4153 /* 4154 * In netmap mode, we must preserve the buffers made 4155 * available to userspace before the if_init() 4156 * (this is true by default on the TX side, because 4157 * init makes all buffers available to userspace). 4158 * 4159 * netmap_reset() and the device specific routines 4160 * (e.g. ixgbe_setup_receive_rings()) map these 4161 * buffers at the end of the NIC ring, so here we 4162 * must set the RDT (tail) register to make sure 4163 * they are not overwritten. 4164 * 4165 * In this driver the NIC ring starts at RDH = 0, 4166 * RDT points to the last slot available for reception (?), 4167 * so RDT = num_rx_desc - 1 means the whole ring is available. 4168 */ 4169 #ifdef DEV_NETMAP 4170 if ((adapter->feat_en & IXGBE_FEATURE_NETMAP) && 4171 (ifp->if_capenable & IFCAP_NETMAP)) { 4172 struct netmap_adapter *na = NA(adapter->ifp); 4173 struct netmap_kring *kring = na->rx_rings[i]; 4174 int t = na->num_rx_desc - 1 - nm_kr_rxspace(kring); 4175 4176 IXGBE_WRITE_REG(hw, IXGBE_RDT(rxr->me), t); 4177 } else 4178 #endif /* DEV_NETMAP */ 4179 IXGBE_WRITE_REG(hw, IXGBE_RDT(rxr->me), 4180 adapter->num_rx_desc - 1); 4181 } 4182 4183 /* Enable Receive engine */ 4184 rxctrl = IXGBE_READ_REG(hw, IXGBE_RXCTRL); 4185 if (hw->mac.type == ixgbe_mac_82598EB) 4186 rxctrl |= IXGBE_RXCTRL_DMBYPS; 4187 rxctrl |= IXGBE_RXCTRL_RXEN; 4188 ixgbe_enable_rx_dma(hw, rxctrl); 4189 4190 callout_reset(&adapter->timer, hz, ixgbe_local_timer, adapter); 4191 atomic_store_relaxed(&adapter->timer_pending, 0); 4192 if (adapter->feat_en & IXGBE_FEATURE_RECOVERY_MODE) 4193 callout_reset(&adapter->recovery_mode_timer, hz, 4194 ixgbe_recovery_mode_timer, adapter); 4195 4196 /* Set up MSI/MSI-X routing */ 4197 if (adapter->feat_en & IXGBE_FEATURE_MSIX) { 4198 ixgbe_configure_ivars(adapter); 4199 /* Set up auto-mask */ 4200 if (hw->mac.type == ixgbe_mac_82598EB) 4201 IXGBE_WRITE_REG(hw, IXGBE_EIAM, IXGBE_EICS_RTX_QUEUE); 4202 else { 4203 IXGBE_WRITE_REG(hw, IXGBE_EIAM_EX(0), 0xFFFFFFFF); 4204 IXGBE_WRITE_REG(hw, IXGBE_EIAM_EX(1), 0xFFFFFFFF); 4205 } 4206 } else { /* Simple settings for Legacy/MSI */ 4207 ixgbe_set_ivar(adapter, 0, 0, 0); 4208 ixgbe_set_ivar(adapter, 0, 0, 1); 4209 IXGBE_WRITE_REG(hw, IXGBE_EIAM, IXGBE_EICS_RTX_QUEUE); 4210 } 4211 4212 ixgbe_init_fdir(adapter); 4213 4214 /* 4215 * Check on any SFP devices that 4216 * need to be kick-started 4217 */ 4218 if (hw->phy.type == ixgbe_phy_none) { 4219 error = hw->phy.ops.identify(hw); 4220 if (error == IXGBE_ERR_SFP_NOT_SUPPORTED) 4221 unsupported_sfp = true; 4222 } else if (hw->phy.type == ixgbe_phy_sfp_unsupported) 4223 unsupported_sfp = true; 4224 4225 if (unsupported_sfp) 4226 device_printf(dev, 4227 "Unsupported SFP+ module type was detected.\n"); 4228 4229 /* Set moderation on the Link interrupt */ 4230 ixgbe_eitr_write(adapter, adapter->vector, IXGBE_LINK_ITR); 4231 4232 /* Enable EEE power saving */ 4233 if (adapter->feat_cap & IXGBE_FEATURE_EEE) 4234 hw->mac.ops.setup_eee(hw, 4235 adapter->feat_en & IXGBE_FEATURE_EEE); 4236 4237 /* Enable power to the phy. */ 4238 if (!unsupported_sfp) { 4239 ixgbe_set_phy_power(hw, TRUE); 4240 4241 /* Config/Enable Link */ 4242 ixgbe_config_link(adapter); 4243 } 4244 4245 /* Hardware Packet Buffer & Flow Control setup */ 4246 ixgbe_config_delay_values(adapter); 4247 4248 /* Initialize the FC settings */ 4249 ixgbe_start_hw(hw); 4250 4251 /* Set up VLAN support and filter */ 4252 ixgbe_setup_vlan_hw_support(adapter); 4253 4254 /* Setup DMA Coalescing */ 4255 ixgbe_config_dmac(adapter); 4256 4257 /* OK to schedule workqueues. */ 4258 adapter->schedule_wqs_ok = true; 4259 4260 /* And now turn on interrupts */ 4261 ixgbe_enable_intr(adapter); 4262 4263 /* Enable the use of the MBX by the VF's */ 4264 if (adapter->feat_en & IXGBE_FEATURE_SRIOV) { 4265 ctrl_ext = IXGBE_READ_REG(hw, IXGBE_CTRL_EXT); 4266 ctrl_ext |= IXGBE_CTRL_EXT_PFRSTD; 4267 IXGBE_WRITE_REG(hw, IXGBE_CTRL_EXT, ctrl_ext); 4268 } 4269 4270 /* Update saved flags. See ixgbe_ifflags_cb() */ 4271 adapter->if_flags = ifp->if_flags; 4272 adapter->ec_capenable = adapter->osdep.ec.ec_capenable; 4273 4274 /* Now inform the stack we're ready */ 4275 ifp->if_flags |= IFF_RUNNING; 4276 4277 return; 4278 } /* ixgbe_init_locked */ 4279 4280 /************************************************************************ 4281 * ixgbe_init 4282 ************************************************************************/ 4283 static int 4284 ixgbe_init(struct ifnet *ifp) 4285 { 4286 struct adapter *adapter = ifp->if_softc; 4287 4288 IXGBE_CORE_LOCK(adapter); 4289 ixgbe_init_locked(adapter); 4290 IXGBE_CORE_UNLOCK(adapter); 4291 4292 return 0; /* XXX ixgbe_init_locked cannot fail? really? */ 4293 } /* ixgbe_init */ 4294 4295 /************************************************************************ 4296 * ixgbe_set_ivar 4297 * 4298 * Setup the correct IVAR register for a particular MSI-X interrupt 4299 * (yes this is all very magic and confusing :) 4300 * - entry is the register array entry 4301 * - vector is the MSI-X vector for this queue 4302 * - type is RX/TX/MISC 4303 ************************************************************************/ 4304 static void 4305 ixgbe_set_ivar(struct adapter *adapter, u8 entry, u8 vector, s8 type) 4306 { 4307 struct ixgbe_hw *hw = &adapter->hw; 4308 u32 ivar, index; 4309 4310 vector |= IXGBE_IVAR_ALLOC_VAL; 4311 4312 switch (hw->mac.type) { 4313 case ixgbe_mac_82598EB: 4314 if (type == -1) 4315 entry = IXGBE_IVAR_OTHER_CAUSES_INDEX; 4316 else 4317 entry += (type * 64); 4318 index = (entry >> 2) & 0x1F; 4319 ivar = IXGBE_READ_REG(hw, IXGBE_IVAR(index)); 4320 ivar &= ~(0xffUL << (8 * (entry & 0x3))); 4321 ivar |= ((u32)vector << (8 * (entry & 0x3))); 4322 IXGBE_WRITE_REG(&adapter->hw, IXGBE_IVAR(index), ivar); 4323 break; 4324 case ixgbe_mac_82599EB: 4325 case ixgbe_mac_X540: 4326 case ixgbe_mac_X550: 4327 case ixgbe_mac_X550EM_x: 4328 case ixgbe_mac_X550EM_a: 4329 if (type == -1) { /* MISC IVAR */ 4330 index = (entry & 1) * 8; 4331 ivar = IXGBE_READ_REG(hw, IXGBE_IVAR_MISC); 4332 ivar &= ~(0xffUL << index); 4333 ivar |= ((u32)vector << index); 4334 IXGBE_WRITE_REG(hw, IXGBE_IVAR_MISC, ivar); 4335 } else { /* RX/TX IVARS */ 4336 index = (16 * (entry & 1)) + (8 * type); 4337 ivar = IXGBE_READ_REG(hw, IXGBE_IVAR(entry >> 1)); 4338 ivar &= ~(0xffUL << index); 4339 ivar |= ((u32)vector << index); 4340 IXGBE_WRITE_REG(hw, IXGBE_IVAR(entry >> 1), ivar); 4341 } 4342 break; 4343 default: 4344 break; 4345 } 4346 } /* ixgbe_set_ivar */ 4347 4348 /************************************************************************ 4349 * ixgbe_configure_ivars 4350 ************************************************************************/ 4351 static void 4352 ixgbe_configure_ivars(struct adapter *adapter) 4353 { 4354 struct ix_queue *que = adapter->queues; 4355 u32 newitr; 4356 4357 if (ixgbe_max_interrupt_rate > 0) 4358 newitr = (4000000 / ixgbe_max_interrupt_rate) & 0x0FF8; 4359 else { 4360 /* 4361 * Disable DMA coalescing if interrupt moderation is 4362 * disabled. 4363 */ 4364 adapter->dmac = 0; 4365 newitr = 0; 4366 } 4367 4368 for (int i = 0; i < adapter->num_queues; i++, que++) { 4369 struct rx_ring *rxr = &adapter->rx_rings[i]; 4370 struct tx_ring *txr = &adapter->tx_rings[i]; 4371 /* First the RX queue entry */ 4372 ixgbe_set_ivar(adapter, rxr->me, que->msix, 0); 4373 /* ... and the TX */ 4374 ixgbe_set_ivar(adapter, txr->me, que->msix, 1); 4375 /* Set an Initial EITR value */ 4376 ixgbe_eitr_write(adapter, que->msix, newitr); 4377 /* 4378 * To eliminate influence of the previous state. 4379 * At this point, Tx/Rx interrupt handler 4380 * (ixgbe_msix_que()) cannot be called, so both 4381 * IXGBE_TX_LOCK and IXGBE_RX_LOCK are not required. 4382 */ 4383 que->eitr_setting = 0; 4384 } 4385 4386 /* For the Link interrupt */ 4387 ixgbe_set_ivar(adapter, 1, adapter->vector, -1); 4388 } /* ixgbe_configure_ivars */ 4389 4390 /************************************************************************ 4391 * ixgbe_config_gpie 4392 ************************************************************************/ 4393 static void 4394 ixgbe_config_gpie(struct adapter *adapter) 4395 { 4396 struct ixgbe_hw *hw = &adapter->hw; 4397 u32 gpie; 4398 4399 gpie = IXGBE_READ_REG(hw, IXGBE_GPIE); 4400 4401 if (adapter->feat_en & IXGBE_FEATURE_MSIX) { 4402 /* Enable Enhanced MSI-X mode */ 4403 gpie |= IXGBE_GPIE_MSIX_MODE 4404 | IXGBE_GPIE_EIAME 4405 | IXGBE_GPIE_PBA_SUPPORT 4406 | IXGBE_GPIE_OCD; 4407 } 4408 4409 /* Fan Failure Interrupt */ 4410 if (adapter->feat_en & IXGBE_FEATURE_FAN_FAIL) 4411 gpie |= IXGBE_SDP1_GPIEN; 4412 4413 /* Thermal Sensor Interrupt */ 4414 if (adapter->feat_en & IXGBE_FEATURE_TEMP_SENSOR) 4415 gpie |= IXGBE_SDP0_GPIEN_X540; 4416 4417 /* Link detection */ 4418 switch (hw->mac.type) { 4419 case ixgbe_mac_82599EB: 4420 gpie |= IXGBE_SDP1_GPIEN | IXGBE_SDP2_GPIEN; 4421 break; 4422 case ixgbe_mac_X550EM_x: 4423 case ixgbe_mac_X550EM_a: 4424 gpie |= IXGBE_SDP0_GPIEN_X540; 4425 break; 4426 default: 4427 break; 4428 } 4429 4430 IXGBE_WRITE_REG(hw, IXGBE_GPIE, gpie); 4431 4432 } /* ixgbe_config_gpie */ 4433 4434 /************************************************************************ 4435 * ixgbe_config_delay_values 4436 * 4437 * Requires adapter->max_frame_size to be set. 4438 ************************************************************************/ 4439 static void 4440 ixgbe_config_delay_values(struct adapter *adapter) 4441 { 4442 struct ixgbe_hw *hw = &adapter->hw; 4443 u32 rxpb, frame, size, tmp; 4444 4445 frame = adapter->max_frame_size; 4446 4447 /* Calculate High Water */ 4448 switch (hw->mac.type) { 4449 case ixgbe_mac_X540: 4450 case ixgbe_mac_X550: 4451 case ixgbe_mac_X550EM_x: 4452 case ixgbe_mac_X550EM_a: 4453 tmp = IXGBE_DV_X540(frame, frame); 4454 break; 4455 default: 4456 tmp = IXGBE_DV(frame, frame); 4457 break; 4458 } 4459 size = IXGBE_BT2KB(tmp); 4460 rxpb = IXGBE_READ_REG(hw, IXGBE_RXPBSIZE(0)) >> 10; 4461 hw->fc.high_water[0] = rxpb - size; 4462 4463 /* Now calculate Low Water */ 4464 switch (hw->mac.type) { 4465 case ixgbe_mac_X540: 4466 case ixgbe_mac_X550: 4467 case ixgbe_mac_X550EM_x: 4468 case ixgbe_mac_X550EM_a: 4469 tmp = IXGBE_LOW_DV_X540(frame); 4470 break; 4471 default: 4472 tmp = IXGBE_LOW_DV(frame); 4473 break; 4474 } 4475 hw->fc.low_water[0] = IXGBE_BT2KB(tmp); 4476 4477 hw->fc.pause_time = IXGBE_FC_PAUSE; 4478 hw->fc.send_xon = TRUE; 4479 } /* ixgbe_config_delay_values */ 4480 4481 /************************************************************************ 4482 * ixgbe_set_rxfilter - Multicast Update 4483 * 4484 * Called whenever multicast address list is updated. 4485 ************************************************************************/ 4486 static void 4487 ixgbe_set_rxfilter(struct adapter *adapter) 4488 { 4489 struct ixgbe_mc_addr *mta; 4490 struct ifnet *ifp = adapter->ifp; 4491 u8 *update_ptr; 4492 int mcnt = 0; 4493 u32 fctrl; 4494 struct ethercom *ec = &adapter->osdep.ec; 4495 struct ether_multi *enm; 4496 struct ether_multistep step; 4497 4498 KASSERT(mutex_owned(&adapter->core_mtx)); 4499 IOCTL_DEBUGOUT("ixgbe_set_rxfilter: begin"); 4500 4501 mta = adapter->mta; 4502 bzero(mta, sizeof(*mta) * MAX_NUM_MULTICAST_ADDRESSES); 4503 4504 ETHER_LOCK(ec); 4505 ec->ec_flags &= ~ETHER_F_ALLMULTI; 4506 ETHER_FIRST_MULTI(step, ec, enm); 4507 while (enm != NULL) { 4508 if ((mcnt == MAX_NUM_MULTICAST_ADDRESSES) || 4509 (memcmp(enm->enm_addrlo, enm->enm_addrhi, 4510 ETHER_ADDR_LEN) != 0)) { 4511 ec->ec_flags |= ETHER_F_ALLMULTI; 4512 break; 4513 } 4514 bcopy(enm->enm_addrlo, 4515 mta[mcnt].addr, IXGBE_ETH_LENGTH_OF_ADDRESS); 4516 mta[mcnt].vmdq = adapter->pool; 4517 mcnt++; 4518 ETHER_NEXT_MULTI(step, enm); 4519 } 4520 4521 fctrl = IXGBE_READ_REG(&adapter->hw, IXGBE_FCTRL); 4522 if (ifp->if_flags & IFF_PROMISC) 4523 fctrl |= (IXGBE_FCTRL_UPE | IXGBE_FCTRL_MPE); 4524 else if (ec->ec_flags & ETHER_F_ALLMULTI) { 4525 fctrl |= IXGBE_FCTRL_MPE; 4526 fctrl &= ~IXGBE_FCTRL_UPE; 4527 } else 4528 fctrl &= ~(IXGBE_FCTRL_UPE | IXGBE_FCTRL_MPE); 4529 4530 IXGBE_WRITE_REG(&adapter->hw, IXGBE_FCTRL, fctrl); 4531 4532 /* Update multicast filter entries only when it's not ALLMULTI */ 4533 if ((ec->ec_flags & ETHER_F_ALLMULTI) == 0) { 4534 ETHER_UNLOCK(ec); 4535 update_ptr = (u8 *)mta; 4536 ixgbe_update_mc_addr_list(&adapter->hw, update_ptr, mcnt, 4537 ixgbe_mc_array_itr, TRUE); 4538 } else 4539 ETHER_UNLOCK(ec); 4540 } /* ixgbe_set_rxfilter */ 4541 4542 /************************************************************************ 4543 * ixgbe_mc_array_itr 4544 * 4545 * An iterator function needed by the multicast shared code. 4546 * It feeds the shared code routine the addresses in the 4547 * array of ixgbe_set_rxfilter() one by one. 4548 ************************************************************************/ 4549 static u8 * 4550 ixgbe_mc_array_itr(struct ixgbe_hw *hw, u8 **update_ptr, u32 *vmdq) 4551 { 4552 struct ixgbe_mc_addr *mta; 4553 4554 mta = (struct ixgbe_mc_addr *)*update_ptr; 4555 *vmdq = mta->vmdq; 4556 4557 *update_ptr = (u8*)(mta + 1); 4558 4559 return (mta->addr); 4560 } /* ixgbe_mc_array_itr */ 4561 4562 /************************************************************************ 4563 * ixgbe_local_timer - Timer routine 4564 * 4565 * Checks for link status, updates statistics, 4566 * and runs the watchdog check. 4567 ************************************************************************/ 4568 static void 4569 ixgbe_local_timer(void *arg) 4570 { 4571 struct adapter *adapter = arg; 4572 4573 if (adapter->schedule_wqs_ok) { 4574 if (atomic_cas_uint(&adapter->timer_pending, 0, 1) == 0) 4575 workqueue_enqueue(adapter->timer_wq, 4576 &adapter->timer_wc, NULL); 4577 } 4578 } 4579 4580 static void 4581 ixgbe_handle_timer(struct work *wk, void *context) 4582 { 4583 struct adapter *adapter = context; 4584 struct ixgbe_hw *hw = &adapter->hw; 4585 device_t dev = adapter->dev; 4586 struct ix_queue *que = adapter->queues; 4587 u64 queues = 0; 4588 u64 v0, v1, v2, v3, v4, v5, v6, v7; 4589 int hung = 0; 4590 int i; 4591 4592 IXGBE_CORE_LOCK(adapter); 4593 4594 /* Check for pluggable optics */ 4595 if (ixgbe_is_sfp(hw)) { 4596 bool sched_mod_task = false; 4597 4598 if (hw->mac.type == ixgbe_mac_82598EB) { 4599 /* 4600 * On 82598EB, SFP+'s MOD_ABS pin is not connected to 4601 * any GPIO(SDP). So just schedule TASK_MOD. 4602 */ 4603 sched_mod_task = true; 4604 } else { 4605 bool was_full, is_full; 4606 4607 was_full = 4608 hw->phy.sfp_type != ixgbe_sfp_type_not_present; 4609 is_full = ixgbe_sfp_cage_full(hw); 4610 4611 /* Do probe if cage state changed */ 4612 if (was_full ^ is_full) 4613 sched_mod_task = true; 4614 } 4615 if (sched_mod_task) { 4616 mutex_enter(&adapter->admin_mtx); 4617 adapter->task_requests |= IXGBE_REQUEST_TASK_MOD_WOI; 4618 ixgbe_schedule_admin_tasklet(adapter); 4619 mutex_exit(&adapter->admin_mtx); 4620 } 4621 } 4622 4623 ixgbe_update_link_status(adapter); 4624 ixgbe_update_stats_counters(adapter); 4625 4626 /* Update some event counters */ 4627 v0 = v1 = v2 = v3 = v4 = v5 = v6 = v7 = 0; 4628 que = adapter->queues; 4629 for (i = 0; i < adapter->num_queues; i++, que++) { 4630 struct tx_ring *txr = que->txr; 4631 4632 v0 += txr->q_efbig_tx_dma_setup; 4633 v1 += txr->q_mbuf_defrag_failed; 4634 v2 += txr->q_efbig2_tx_dma_setup; 4635 v3 += txr->q_einval_tx_dma_setup; 4636 v4 += txr->q_other_tx_dma_setup; 4637 v5 += txr->q_eagain_tx_dma_setup; 4638 v6 += txr->q_enomem_tx_dma_setup; 4639 v7 += txr->q_tso_err; 4640 } 4641 IXGBE_EVC_STORE(&adapter->efbig_tx_dma_setup, v0); 4642 IXGBE_EVC_STORE(&adapter->mbuf_defrag_failed, v1); 4643 IXGBE_EVC_STORE(&adapter->efbig2_tx_dma_setup, v2); 4644 IXGBE_EVC_STORE(&adapter->einval_tx_dma_setup, v3); 4645 IXGBE_EVC_STORE(&adapter->other_tx_dma_setup, v4); 4646 IXGBE_EVC_STORE(&adapter->eagain_tx_dma_setup, v5); 4647 IXGBE_EVC_STORE(&adapter->enomem_tx_dma_setup, v6); 4648 IXGBE_EVC_STORE(&adapter->tso_err, v7); 4649 4650 /* 4651 * Check the TX queues status 4652 * - mark hung queues so we don't schedule on them 4653 * - watchdog only if all queues show hung 4654 */ 4655 que = adapter->queues; 4656 for (i = 0; i < adapter->num_queues; i++, que++) { 4657 /* Keep track of queues with work for soft irq */ 4658 if (que->txr->busy) 4659 queues |= 1ULL << que->me; 4660 /* 4661 * Each time txeof runs without cleaning, but there 4662 * are uncleaned descriptors it increments busy. If 4663 * we get to the MAX we declare it hung. 4664 */ 4665 if (que->busy == IXGBE_QUEUE_HUNG) { 4666 ++hung; 4667 /* Mark the queue as inactive */ 4668 adapter->active_queues &= ~(1ULL << que->me); 4669 continue; 4670 } else { 4671 /* Check if we've come back from hung */ 4672 if ((adapter->active_queues & (1ULL << que->me)) == 0) 4673 adapter->active_queues |= 1ULL << que->me; 4674 } 4675 if (que->busy >= IXGBE_MAX_TX_BUSY) { 4676 device_printf(dev, 4677 "Warning queue %d appears to be hung!\n", i); 4678 que->txr->busy = IXGBE_QUEUE_HUNG; 4679 ++hung; 4680 } 4681 } 4682 4683 /* Only truly watchdog if all queues show hung */ 4684 if (hung == adapter->num_queues) 4685 goto watchdog; 4686 #if 0 /* XXX Avoid unexpectedly disabling interrupt forever (PR#53294) */ 4687 else if (queues != 0) { /* Force an IRQ on queues with work */ 4688 que = adapter->queues; 4689 for (i = 0; i < adapter->num_queues; i++, que++) { 4690 mutex_enter(&que->dc_mtx); 4691 if (que->disabled_count == 0) 4692 ixgbe_rearm_queues(adapter, 4693 queues & ((u64)1 << i)); 4694 mutex_exit(&que->dc_mtx); 4695 } 4696 } 4697 #endif 4698 4699 atomic_store_relaxed(&adapter->timer_pending, 0); 4700 IXGBE_CORE_UNLOCK(adapter); 4701 callout_reset(&adapter->timer, hz, ixgbe_local_timer, adapter); 4702 return; 4703 4704 watchdog: 4705 device_printf(adapter->dev, "Watchdog timeout -- resetting\n"); 4706 adapter->ifp->if_flags &= ~IFF_RUNNING; 4707 IXGBE_EVC_ADD(&adapter->watchdog_events, 1); 4708 ixgbe_init_locked(adapter); 4709 IXGBE_CORE_UNLOCK(adapter); 4710 } /* ixgbe_handle_timer */ 4711 4712 /************************************************************************ 4713 * ixgbe_recovery_mode_timer - Recovery mode timer routine 4714 ************************************************************************/ 4715 static void 4716 ixgbe_recovery_mode_timer(void *arg) 4717 { 4718 struct adapter *adapter = arg; 4719 4720 if (__predict_true(adapter->osdep.detaching == false)) { 4721 if (atomic_cas_uint(&adapter->recovery_mode_timer_pending, 4722 0, 1) == 0) { 4723 workqueue_enqueue(adapter->recovery_mode_timer_wq, 4724 &adapter->recovery_mode_timer_wc, NULL); 4725 } 4726 } 4727 } 4728 4729 static void 4730 ixgbe_handle_recovery_mode_timer(struct work *wk, void *context) 4731 { 4732 struct adapter *adapter = context; 4733 struct ixgbe_hw *hw = &adapter->hw; 4734 4735 IXGBE_CORE_LOCK(adapter); 4736 if (ixgbe_fw_recovery_mode(hw)) { 4737 if (atomic_cas_uint(&adapter->recovery_mode, 0, 1)) { 4738 /* Firmware error detected, entering recovery mode */ 4739 device_printf(adapter->dev, "Firmware recovery mode detected. Limiting functionality. Refer to the Intel(R) Ethernet Adapters and Devices User Guide for details on firmware recovery mode.\n"); 4740 4741 if (hw->adapter_stopped == FALSE) 4742 ixgbe_stop_locked(adapter); 4743 } 4744 } else 4745 atomic_cas_uint(&adapter->recovery_mode, 1, 0); 4746 4747 atomic_store_relaxed(&adapter->recovery_mode_timer_pending, 0); 4748 callout_reset(&adapter->recovery_mode_timer, hz, 4749 ixgbe_recovery_mode_timer, adapter); 4750 IXGBE_CORE_UNLOCK(adapter); 4751 } /* ixgbe_handle_recovery_mode_timer */ 4752 4753 /************************************************************************ 4754 * ixgbe_handle_mod - Tasklet for SFP module interrupts 4755 * bool int_en: true if it's called when the interrupt is enabled. 4756 ************************************************************************/ 4757 static void 4758 ixgbe_handle_mod(void *context, bool int_en) 4759 { 4760 struct adapter *adapter = context; 4761 struct ixgbe_hw *hw = &adapter->hw; 4762 device_t dev = adapter->dev; 4763 enum ixgbe_sfp_type last_sfp_type; 4764 u32 err; 4765 bool last_unsupported_sfp_recovery; 4766 4767 KASSERT(mutex_owned(&adapter->core_mtx)); 4768 4769 last_sfp_type = hw->phy.sfp_type; 4770 last_unsupported_sfp_recovery = hw->need_unsupported_sfp_recovery; 4771 IXGBE_EVC_ADD(&adapter->mod_workev, 1); 4772 if (adapter->hw.need_crosstalk_fix) { 4773 if ((hw->mac.type != ixgbe_mac_82598EB) && 4774 !ixgbe_sfp_cage_full(hw)) 4775 goto out; 4776 } 4777 4778 err = hw->phy.ops.identify_sfp(hw); 4779 if (err == IXGBE_ERR_SFP_NOT_SUPPORTED) { 4780 if (last_unsupported_sfp_recovery == false) 4781 device_printf(dev, 4782 "Unsupported SFP+ module type was detected.\n"); 4783 goto out; 4784 } 4785 4786 if (hw->need_unsupported_sfp_recovery) { 4787 device_printf(dev, "Recovering from unsupported SFP\n"); 4788 /* 4789 * We could recover the status by calling setup_sfp(), 4790 * setup_link() and some others. It's complex and might not 4791 * work correctly on some unknown cases. To avoid such type of 4792 * problem, call ixgbe_init_locked(). It's simple and safe 4793 * approach. 4794 */ 4795 ixgbe_init_locked(adapter); 4796 } else if ((hw->phy.sfp_type != ixgbe_sfp_type_not_present) && 4797 (hw->phy.sfp_type != last_sfp_type)) { 4798 /* A module is inserted and changed. */ 4799 4800 if (hw->mac.type == ixgbe_mac_82598EB) 4801 err = hw->phy.ops.reset(hw); 4802 else { 4803 err = hw->mac.ops.setup_sfp(hw); 4804 hw->phy.sfp_setup_needed = FALSE; 4805 } 4806 if (err == IXGBE_ERR_SFP_NOT_SUPPORTED) { 4807 device_printf(dev, 4808 "Setup failure - unsupported SFP+ module type.\n"); 4809 goto out; 4810 } 4811 } 4812 4813 out: 4814 /* get_supported_phy_layer will call hw->phy.ops.identify_sfp() */ 4815 adapter->phy_layer = ixgbe_get_supported_physical_layer(hw); 4816 4817 /* Adjust media types shown in ifconfig */ 4818 IXGBE_CORE_UNLOCK(adapter); 4819 ifmedia_removeall(&adapter->media); 4820 ixgbe_add_media_types(adapter); 4821 ifmedia_set(&adapter->media, IFM_ETHER | IFM_AUTO); 4822 IXGBE_CORE_LOCK(adapter); 4823 4824 /* 4825 * Don't schedule MSF event if the chip is 82598. 82598 doesn't support 4826 * MSF. At least, calling ixgbe_handle_msf on 82598 DA makes the link 4827 * flap because the function calls setup_link(). 4828 */ 4829 if (hw->mac.type != ixgbe_mac_82598EB) { 4830 mutex_enter(&adapter->admin_mtx); 4831 if (int_en) 4832 adapter->task_requests |= IXGBE_REQUEST_TASK_MSF; 4833 else 4834 adapter->task_requests |= IXGBE_REQUEST_TASK_MSF_WOI; 4835 mutex_exit(&adapter->admin_mtx); 4836 } 4837 4838 /* 4839 * Don't call ixgbe_schedule_admin_tasklet() because we are on 4840 * the workqueue now. 4841 */ 4842 } /* ixgbe_handle_mod */ 4843 4844 4845 /************************************************************************ 4846 * ixgbe_handle_msf - Tasklet for MSF (multispeed fiber) interrupts 4847 ************************************************************************/ 4848 static void 4849 ixgbe_handle_msf(void *context) 4850 { 4851 struct adapter *adapter = context; 4852 struct ixgbe_hw *hw = &adapter->hw; 4853 u32 autoneg; 4854 bool negotiate; 4855 4856 KASSERT(mutex_owned(&adapter->core_mtx)); 4857 4858 IXGBE_EVC_ADD(&adapter->msf_workev, 1); 4859 4860 autoneg = hw->phy.autoneg_advertised; 4861 if ((!autoneg) && (hw->mac.ops.get_link_capabilities)) 4862 hw->mac.ops.get_link_capabilities(hw, &autoneg, &negotiate); 4863 if (hw->mac.ops.setup_link) 4864 hw->mac.ops.setup_link(hw, autoneg, TRUE); 4865 } /* ixgbe_handle_msf */ 4866 4867 /************************************************************************ 4868 * ixgbe_handle_phy - Tasklet for external PHY interrupts 4869 ************************************************************************/ 4870 static void 4871 ixgbe_handle_phy(void *context) 4872 { 4873 struct adapter *adapter = context; 4874 struct ixgbe_hw *hw = &adapter->hw; 4875 int error; 4876 4877 KASSERT(mutex_owned(&adapter->core_mtx)); 4878 4879 IXGBE_EVC_ADD(&adapter->phy_workev, 1); 4880 error = hw->phy.ops.handle_lasi(hw); 4881 if (error == IXGBE_ERR_OVERTEMP) 4882 device_printf(adapter->dev, 4883 "CRITICAL: EXTERNAL PHY OVER TEMP!! " 4884 " PHY will downshift to lower power state!\n"); 4885 else if (error) 4886 device_printf(adapter->dev, 4887 "Error handling LASI interrupt: %d\n", error); 4888 } /* ixgbe_handle_phy */ 4889 4890 static void 4891 ixgbe_handle_admin(struct work *wk, void *context) 4892 { 4893 struct adapter *adapter = context; 4894 struct ifnet *ifp = adapter->ifp; 4895 struct ixgbe_hw *hw = &adapter->hw; 4896 u32 task_requests; 4897 u32 eims_enable = 0; 4898 4899 mutex_enter(&adapter->admin_mtx); 4900 adapter->admin_pending = 0; 4901 task_requests = adapter->task_requests; 4902 adapter->task_requests = 0; 4903 mutex_exit(&adapter->admin_mtx); 4904 4905 /* 4906 * Hold the IFNET_LOCK across this entire call. This will 4907 * prevent additional changes to adapter->phy_layer 4908 * and serialize calls to this tasklet. We cannot hold the 4909 * CORE_LOCK while calling into the ifmedia functions as 4910 * they call ifmedia_lock() and the lock is CORE_LOCK. 4911 */ 4912 IFNET_LOCK(ifp); 4913 IXGBE_CORE_LOCK(adapter); 4914 if ((task_requests & IXGBE_REQUEST_TASK_LSC) != 0) { 4915 ixgbe_handle_link(adapter); 4916 eims_enable |= IXGBE_EIMS_LSC; 4917 } 4918 if ((task_requests & IXGBE_REQUEST_TASK_MOD_WOI) != 0) { 4919 ixgbe_handle_mod(adapter, false); 4920 } 4921 if ((task_requests & IXGBE_REQUEST_TASK_MOD) != 0) { 4922 ixgbe_handle_mod(adapter, true); 4923 if (hw->mac.type >= ixgbe_mac_X540) 4924 eims_enable |= IXGBE_EICR_GPI_SDP0_X540; 4925 else 4926 eims_enable |= IXGBE_EICR_GPI_SDP2_BY_MAC(hw); 4927 } 4928 if ((task_requests 4929 & (IXGBE_REQUEST_TASK_MSF_WOI | IXGBE_REQUEST_TASK_MSF)) != 0) { 4930 ixgbe_handle_msf(adapter); 4931 if (((task_requests & IXGBE_REQUEST_TASK_MSF) != 0) && 4932 (hw->mac.type == ixgbe_mac_82599EB)) 4933 eims_enable |= IXGBE_EIMS_GPI_SDP1_BY_MAC(hw); 4934 } 4935 if ((task_requests & IXGBE_REQUEST_TASK_PHY) != 0) { 4936 ixgbe_handle_phy(adapter); 4937 eims_enable |= IXGBE_EICR_GPI_SDP0_X540; 4938 } 4939 if ((task_requests & IXGBE_REQUEST_TASK_FDIR) != 0) { 4940 ixgbe_reinit_fdir(adapter); 4941 eims_enable |= IXGBE_EIMS_FLOW_DIR; 4942 } 4943 #if 0 /* notyet */ 4944 if ((task_requests & IXGBE_REQUEST_TASK_MBX) != 0) { 4945 ixgbe_handle_mbx(adapter); 4946 eims_enable |= IXGBE_EIMS_MAILBOX; 4947 } 4948 #endif 4949 IXGBE_WRITE_REG(hw, IXGBE_EIMS, eims_enable); 4950 4951 IXGBE_CORE_UNLOCK(adapter); 4952 IFNET_UNLOCK(ifp); 4953 } /* ixgbe_handle_admin */ 4954 4955 static void 4956 ixgbe_ifstop(struct ifnet *ifp, int disable) 4957 { 4958 struct adapter *adapter = ifp->if_softc; 4959 4960 IXGBE_CORE_LOCK(adapter); 4961 ixgbe_stop_locked(adapter); 4962 IXGBE_CORE_UNLOCK(adapter); 4963 4964 workqueue_wait(adapter->timer_wq, &adapter->timer_wc); 4965 atomic_store_relaxed(&adapter->timer_pending, 0); 4966 } 4967 4968 /************************************************************************ 4969 * ixgbe_stop_locked - Stop the hardware 4970 * 4971 * Disables all traffic on the adapter by issuing a 4972 * global reset on the MAC and deallocates TX/RX buffers. 4973 ************************************************************************/ 4974 static void 4975 ixgbe_stop_locked(void *arg) 4976 { 4977 struct ifnet *ifp; 4978 struct adapter *adapter = arg; 4979 struct ixgbe_hw *hw = &adapter->hw; 4980 4981 ifp = adapter->ifp; 4982 4983 KASSERT(mutex_owned(&adapter->core_mtx)); 4984 4985 INIT_DEBUGOUT("ixgbe_stop_locked: begin\n"); 4986 ixgbe_disable_intr(adapter); 4987 callout_stop(&adapter->timer); 4988 4989 /* Don't schedule workqueues. */ 4990 adapter->schedule_wqs_ok = false; 4991 4992 /* Let the stack know...*/ 4993 ifp->if_flags &= ~IFF_RUNNING; 4994 4995 ixgbe_reset_hw(hw); 4996 hw->adapter_stopped = FALSE; 4997 ixgbe_stop_adapter(hw); 4998 if (hw->mac.type == ixgbe_mac_82599EB) 4999 ixgbe_stop_mac_link_on_d3_82599(hw); 5000 /* Turn off the laser - noop with no optics */ 5001 ixgbe_disable_tx_laser(hw); 5002 5003 /* Update the stack */ 5004 adapter->link_up = FALSE; 5005 ixgbe_update_link_status(adapter); 5006 5007 /* reprogram the RAR[0] in case user changed it. */ 5008 ixgbe_set_rar(&adapter->hw, 0, adapter->hw.mac.addr, 0, IXGBE_RAH_AV); 5009 5010 return; 5011 } /* ixgbe_stop_locked */ 5012 5013 /************************************************************************ 5014 * ixgbe_update_link_status - Update OS on link state 5015 * 5016 * Note: Only updates the OS on the cached link state. 5017 * The real check of the hardware only happens with 5018 * a link interrupt. 5019 ************************************************************************/ 5020 static void 5021 ixgbe_update_link_status(struct adapter *adapter) 5022 { 5023 struct ifnet *ifp = adapter->ifp; 5024 device_t dev = adapter->dev; 5025 struct ixgbe_hw *hw = &adapter->hw; 5026 5027 KASSERT(mutex_owned(&adapter->core_mtx)); 5028 5029 if (adapter->link_up) { 5030 if (adapter->link_active != LINK_STATE_UP) { 5031 /* 5032 * To eliminate influence of the previous state 5033 * in the same way as ixgbe_init_locked(). 5034 */ 5035 struct ix_queue *que = adapter->queues; 5036 for (int i = 0; i < adapter->num_queues; i++, que++) 5037 que->eitr_setting = 0; 5038 5039 if (adapter->link_speed == IXGBE_LINK_SPEED_10GB_FULL){ 5040 /* 5041 * Discard count for both MAC Local Fault and 5042 * Remote Fault because those registers are 5043 * valid only when the link speed is up and 5044 * 10Gbps. 5045 */ 5046 IXGBE_READ_REG(hw, IXGBE_MLFC); 5047 IXGBE_READ_REG(hw, IXGBE_MRFC); 5048 } 5049 5050 if (bootverbose) { 5051 const char *bpsmsg; 5052 5053 switch (adapter->link_speed) { 5054 case IXGBE_LINK_SPEED_10GB_FULL: 5055 bpsmsg = "10 Gbps"; 5056 break; 5057 case IXGBE_LINK_SPEED_5GB_FULL: 5058 bpsmsg = "5 Gbps"; 5059 break; 5060 case IXGBE_LINK_SPEED_2_5GB_FULL: 5061 bpsmsg = "2.5 Gbps"; 5062 break; 5063 case IXGBE_LINK_SPEED_1GB_FULL: 5064 bpsmsg = "1 Gbps"; 5065 break; 5066 case IXGBE_LINK_SPEED_100_FULL: 5067 bpsmsg = "100 Mbps"; 5068 break; 5069 case IXGBE_LINK_SPEED_10_FULL: 5070 bpsmsg = "10 Mbps"; 5071 break; 5072 default: 5073 bpsmsg = "unknown speed"; 5074 break; 5075 } 5076 device_printf(dev, "Link is up %s %s \n", 5077 bpsmsg, "Full Duplex"); 5078 } 5079 adapter->link_active = LINK_STATE_UP; 5080 /* Update any Flow Control changes */ 5081 ixgbe_fc_enable(&adapter->hw); 5082 /* Update DMA coalescing config */ 5083 ixgbe_config_dmac(adapter); 5084 if_link_state_change(ifp, LINK_STATE_UP); 5085 5086 if (adapter->feat_en & IXGBE_FEATURE_SRIOV) 5087 ixgbe_ping_all_vfs(adapter); 5088 } 5089 } else { 5090 /* 5091 * Do it when link active changes to DOWN. i.e. 5092 * a) LINK_STATE_UNKNOWN -> LINK_STATE_DOWN 5093 * b) LINK_STATE_UP -> LINK_STATE_DOWN 5094 */ 5095 if (adapter->link_active != LINK_STATE_DOWN) { 5096 if (bootverbose) 5097 device_printf(dev, "Link is Down\n"); 5098 if_link_state_change(ifp, LINK_STATE_DOWN); 5099 adapter->link_active = LINK_STATE_DOWN; 5100 if (adapter->feat_en & IXGBE_FEATURE_SRIOV) 5101 ixgbe_ping_all_vfs(adapter); 5102 ixgbe_drain_all(adapter); 5103 } 5104 } 5105 } /* ixgbe_update_link_status */ 5106 5107 /************************************************************************ 5108 * ixgbe_config_dmac - Configure DMA Coalescing 5109 ************************************************************************/ 5110 static void 5111 ixgbe_config_dmac(struct adapter *adapter) 5112 { 5113 struct ixgbe_hw *hw = &adapter->hw; 5114 struct ixgbe_dmac_config *dcfg = &hw->mac.dmac_config; 5115 5116 if (hw->mac.type < ixgbe_mac_X550 || !hw->mac.ops.dmac_config) 5117 return; 5118 5119 if (dcfg->watchdog_timer ^ adapter->dmac || 5120 dcfg->link_speed ^ adapter->link_speed) { 5121 dcfg->watchdog_timer = adapter->dmac; 5122 dcfg->fcoe_en = false; 5123 dcfg->link_speed = adapter->link_speed; 5124 dcfg->num_tcs = 1; 5125 5126 INIT_DEBUGOUT2("dmac settings: watchdog %d, link speed %d\n", 5127 dcfg->watchdog_timer, dcfg->link_speed); 5128 5129 hw->mac.ops.dmac_config(hw); 5130 } 5131 } /* ixgbe_config_dmac */ 5132 5133 /************************************************************************ 5134 * ixgbe_enable_intr 5135 ************************************************************************/ 5136 static void 5137 ixgbe_enable_intr(struct adapter *adapter) 5138 { 5139 struct ixgbe_hw *hw = &adapter->hw; 5140 struct ix_queue *que = adapter->queues; 5141 u32 mask, fwsm; 5142 5143 mask = (IXGBE_EIMS_ENABLE_MASK & ~IXGBE_EIMS_RTX_QUEUE); 5144 5145 switch (adapter->hw.mac.type) { 5146 case ixgbe_mac_82599EB: 5147 mask |= IXGBE_EIMS_ECC; 5148 /* Temperature sensor on some adapters */ 5149 mask |= IXGBE_EIMS_GPI_SDP0; 5150 /* SFP+ (RX_LOS_N & MOD_ABS_N) */ 5151 mask |= IXGBE_EIMS_GPI_SDP1; 5152 mask |= IXGBE_EIMS_GPI_SDP2; 5153 break; 5154 case ixgbe_mac_X540: 5155 /* Detect if Thermal Sensor is enabled */ 5156 fwsm = IXGBE_READ_REG(hw, IXGBE_FWSM); 5157 if (fwsm & IXGBE_FWSM_TS_ENABLED) 5158 mask |= IXGBE_EIMS_TS; 5159 mask |= IXGBE_EIMS_ECC; 5160 break; 5161 case ixgbe_mac_X550: 5162 /* MAC thermal sensor is automatically enabled */ 5163 mask |= IXGBE_EIMS_TS; 5164 mask |= IXGBE_EIMS_ECC; 5165 break; 5166 case ixgbe_mac_X550EM_x: 5167 case ixgbe_mac_X550EM_a: 5168 /* Some devices use SDP0 for important information */ 5169 if (hw->device_id == IXGBE_DEV_ID_X550EM_X_SFP || 5170 hw->device_id == IXGBE_DEV_ID_X550EM_A_SFP || 5171 hw->device_id == IXGBE_DEV_ID_X550EM_A_SFP_N || 5172 hw->device_id == IXGBE_DEV_ID_X550EM_X_10G_T) 5173 mask |= IXGBE_EIMS_GPI_SDP0_BY_MAC(hw); 5174 if (hw->phy.type == ixgbe_phy_x550em_ext_t) 5175 mask |= IXGBE_EICR_GPI_SDP0_X540; 5176 mask |= IXGBE_EIMS_ECC; 5177 break; 5178 default: 5179 break; 5180 } 5181 5182 /* Enable Fan Failure detection */ 5183 if (adapter->feat_en & IXGBE_FEATURE_FAN_FAIL) 5184 mask |= IXGBE_EIMS_GPI_SDP1; 5185 /* Enable SR-IOV */ 5186 if (adapter->feat_en & IXGBE_FEATURE_SRIOV) 5187 mask |= IXGBE_EIMS_MAILBOX; 5188 /* Enable Flow Director */ 5189 if (adapter->feat_en & IXGBE_FEATURE_FDIR) 5190 mask |= IXGBE_EIMS_FLOW_DIR; 5191 5192 IXGBE_WRITE_REG(hw, IXGBE_EIMS, mask); 5193 5194 /* With MSI-X we use auto clear */ 5195 if (adapter->msix_mem) { 5196 /* 5197 * It's not required to set TCP_TIMER because we don't use 5198 * it. 5199 */ 5200 IXGBE_WRITE_REG(hw, IXGBE_EIAC, IXGBE_EIMS_RTX_QUEUE); 5201 } 5202 5203 /* 5204 * Now enable all queues, this is done separately to 5205 * allow for handling the extended (beyond 32) MSI-X 5206 * vectors that can be used by 82599 5207 */ 5208 for (int i = 0; i < adapter->num_queues; i++, que++) 5209 ixgbe_enable_queue(adapter, que->msix); 5210 5211 IXGBE_WRITE_FLUSH(hw); 5212 5213 } /* ixgbe_enable_intr */ 5214 5215 /************************************************************************ 5216 * ixgbe_disable_intr_internal 5217 ************************************************************************/ 5218 static void 5219 ixgbe_disable_intr_internal(struct adapter *adapter, bool nestok) 5220 { 5221 struct ix_queue *que = adapter->queues; 5222 5223 /* disable interrupts other than queues */ 5224 IXGBE_WRITE_REG(&adapter->hw, IXGBE_EIMC, ~IXGBE_EIMC_RTX_QUEUE); 5225 5226 if (adapter->msix_mem) 5227 IXGBE_WRITE_REG(&adapter->hw, IXGBE_EIAC, 0); 5228 5229 for (int i = 0; i < adapter->num_queues; i++, que++) 5230 ixgbe_disable_queue_internal(adapter, que->msix, nestok); 5231 5232 IXGBE_WRITE_FLUSH(&adapter->hw); 5233 5234 } /* ixgbe_do_disable_intr_internal */ 5235 5236 /************************************************************************ 5237 * ixgbe_disable_intr 5238 ************************************************************************/ 5239 static void 5240 ixgbe_disable_intr(struct adapter *adapter) 5241 { 5242 5243 ixgbe_disable_intr_internal(adapter, true); 5244 } /* ixgbe_disable_intr */ 5245 5246 /************************************************************************ 5247 * ixgbe_ensure_disabled_intr 5248 ************************************************************************/ 5249 void 5250 ixgbe_ensure_disabled_intr(struct adapter *adapter) 5251 { 5252 5253 ixgbe_disable_intr_internal(adapter, false); 5254 } /* ixgbe_ensure_disabled_intr */ 5255 5256 /************************************************************************ 5257 * ixgbe_legacy_irq - Legacy Interrupt Service routine 5258 ************************************************************************/ 5259 static int 5260 ixgbe_legacy_irq(void *arg) 5261 { 5262 struct ix_queue *que = arg; 5263 struct adapter *adapter = que->adapter; 5264 struct ixgbe_hw *hw = &adapter->hw; 5265 struct ifnet *ifp = adapter->ifp; 5266 struct tx_ring *txr = adapter->tx_rings; 5267 u32 eicr; 5268 u32 eims_orig; 5269 u32 eims_enable = 0; 5270 u32 eims_disable = 0; 5271 5272 eims_orig = IXGBE_READ_REG(hw, IXGBE_EIMS); 5273 /* 5274 * Silicon errata #26 on 82598. Disable all interrupts before reading 5275 * EICR. 5276 */ 5277 IXGBE_WRITE_REG(hw, IXGBE_EIMC, IXGBE_IRQ_CLEAR_MASK); 5278 5279 /* Read and clear EICR */ 5280 eicr = IXGBE_READ_REG(hw, IXGBE_EICR); 5281 5282 if (eicr == 0) { 5283 IXGBE_EVC_ADD(&adapter->stats.pf.intzero, 1); 5284 IXGBE_WRITE_REG(hw, IXGBE_EIMS, eims_orig); 5285 return 0; 5286 } 5287 IXGBE_EVC_ADD(&adapter->stats.pf.legint, 1); 5288 5289 /* Queue (0) intr */ 5290 if (((ifp->if_flags & IFF_RUNNING) != 0) && 5291 (eicr & IXGBE_EIMC_RTX_QUEUE) != 0) { 5292 IXGBE_EVC_ADD(&que->irqs, 1); 5293 5294 /* 5295 * The same as ixgbe_msix_que() about 5296 * "que->txrx_use_workqueue". 5297 */ 5298 que->txrx_use_workqueue = adapter->txrx_use_workqueue; 5299 5300 IXGBE_TX_LOCK(txr); 5301 ixgbe_txeof(txr); 5302 #ifdef notyet 5303 if (!ixgbe_ring_empty(ifp, txr->br)) 5304 ixgbe_start_locked(ifp, txr); 5305 #endif 5306 IXGBE_TX_UNLOCK(txr); 5307 5308 IXGBE_EVC_ADD(&que->req, 1); 5309 ixgbe_sched_handle_que(adapter, que); 5310 /* Disable queue 0 interrupt */ 5311 eims_disable |= 1UL << 0; 5312 5313 } else 5314 eims_enable |= IXGBE_EIMC_RTX_QUEUE; 5315 5316 ixgbe_intr_admin_common(adapter, eicr, &eims_disable); 5317 5318 /* Re-enable some interrupts */ 5319 IXGBE_WRITE_REG(hw, IXGBE_EIMS, 5320 (eims_orig & ~eims_disable) | eims_enable); 5321 5322 return 1; 5323 } /* ixgbe_legacy_irq */ 5324 5325 /************************************************************************ 5326 * ixgbe_free_pciintr_resources 5327 ************************************************************************/ 5328 static void 5329 ixgbe_free_pciintr_resources(struct adapter *adapter) 5330 { 5331 struct ix_queue *que = adapter->queues; 5332 int rid; 5333 5334 /* 5335 * Release all msix queue resources: 5336 */ 5337 for (int i = 0; i < adapter->num_queues; i++, que++) { 5338 if (que->res != NULL) { 5339 pci_intr_disestablish(adapter->osdep.pc, 5340 adapter->osdep.ihs[i]); 5341 adapter->osdep.ihs[i] = NULL; 5342 } 5343 } 5344 5345 /* Clean the Legacy or Link interrupt last */ 5346 if (adapter->vector) /* we are doing MSIX */ 5347 rid = adapter->vector; 5348 else 5349 rid = 0; 5350 5351 if (adapter->osdep.ihs[rid] != NULL) { 5352 pci_intr_disestablish(adapter->osdep.pc, 5353 adapter->osdep.ihs[rid]); 5354 adapter->osdep.ihs[rid] = NULL; 5355 } 5356 5357 if (adapter->osdep.intrs != NULL) { 5358 pci_intr_release(adapter->osdep.pc, adapter->osdep.intrs, 5359 adapter->osdep.nintrs); 5360 adapter->osdep.intrs = NULL; 5361 } 5362 } /* ixgbe_free_pciintr_resources */ 5363 5364 /************************************************************************ 5365 * ixgbe_free_pci_resources 5366 ************************************************************************/ 5367 static void 5368 ixgbe_free_pci_resources(struct adapter *adapter) 5369 { 5370 5371 ixgbe_free_pciintr_resources(adapter); 5372 5373 if (adapter->osdep.mem_size != 0) { 5374 bus_space_unmap(adapter->osdep.mem_bus_space_tag, 5375 adapter->osdep.mem_bus_space_handle, 5376 adapter->osdep.mem_size); 5377 } 5378 5379 } /* ixgbe_free_pci_resources */ 5380 5381 /************************************************************************ 5382 * ixgbe_set_sysctl_value 5383 ************************************************************************/ 5384 static void 5385 ixgbe_set_sysctl_value(struct adapter *adapter, const char *name, 5386 const char *description, int *limit, int value) 5387 { 5388 device_t dev = adapter->dev; 5389 struct sysctllog **log; 5390 const struct sysctlnode *rnode, *cnode; 5391 5392 /* 5393 * It's not required to check recovery mode because this function never 5394 * touches hardware. 5395 */ 5396 5397 log = &adapter->sysctllog; 5398 if ((rnode = ixgbe_sysctl_instance(adapter)) == NULL) { 5399 aprint_error_dev(dev, "could not create sysctl root\n"); 5400 return; 5401 } 5402 if (sysctl_createv(log, 0, &rnode, &cnode, 5403 CTLFLAG_READWRITE, CTLTYPE_INT, 5404 name, SYSCTL_DESCR(description), 5405 NULL, 0, limit, 0, CTL_CREATE, CTL_EOL) != 0) 5406 aprint_error_dev(dev, "could not create sysctl\n"); 5407 *limit = value; 5408 } /* ixgbe_set_sysctl_value */ 5409 5410 /************************************************************************ 5411 * ixgbe_sysctl_flowcntl 5412 * 5413 * SYSCTL wrapper around setting Flow Control 5414 ************************************************************************/ 5415 static int 5416 ixgbe_sysctl_flowcntl(SYSCTLFN_ARGS) 5417 { 5418 struct sysctlnode node = *rnode; 5419 struct adapter *adapter = (struct adapter *)node.sysctl_data; 5420 int error, fc; 5421 5422 if (ixgbe_fw_recovery_mode_swflag(adapter)) 5423 return (EPERM); 5424 5425 fc = adapter->hw.fc.current_mode; 5426 node.sysctl_data = &fc; 5427 error = sysctl_lookup(SYSCTLFN_CALL(&node)); 5428 if (error != 0 || newp == NULL) 5429 return error; 5430 5431 /* Don't bother if it's not changed */ 5432 if (fc == adapter->hw.fc.current_mode) 5433 return (0); 5434 5435 return ixgbe_set_flowcntl(adapter, fc); 5436 } /* ixgbe_sysctl_flowcntl */ 5437 5438 /************************************************************************ 5439 * ixgbe_set_flowcntl - Set flow control 5440 * 5441 * Flow control values: 5442 * 0 - off 5443 * 1 - rx pause 5444 * 2 - tx pause 5445 * 3 - full 5446 ************************************************************************/ 5447 static int 5448 ixgbe_set_flowcntl(struct adapter *adapter, int fc) 5449 { 5450 switch (fc) { 5451 case ixgbe_fc_rx_pause: 5452 case ixgbe_fc_tx_pause: 5453 case ixgbe_fc_full: 5454 adapter->hw.fc.requested_mode = fc; 5455 if (adapter->num_queues > 1) 5456 ixgbe_disable_rx_drop(adapter); 5457 break; 5458 case ixgbe_fc_none: 5459 adapter->hw.fc.requested_mode = ixgbe_fc_none; 5460 if (adapter->num_queues > 1) 5461 ixgbe_enable_rx_drop(adapter); 5462 break; 5463 default: 5464 return (EINVAL); 5465 } 5466 5467 #if 0 /* XXX NetBSD */ 5468 /* Don't autoneg if forcing a value */ 5469 adapter->hw.fc.disable_fc_autoneg = TRUE; 5470 #endif 5471 ixgbe_fc_enable(&adapter->hw); 5472 5473 return (0); 5474 } /* ixgbe_set_flowcntl */ 5475 5476 /************************************************************************ 5477 * ixgbe_enable_rx_drop 5478 * 5479 * Enable the hardware to drop packets when the buffer is 5480 * full. This is useful with multiqueue, so that no single 5481 * queue being full stalls the entire RX engine. We only 5482 * enable this when Multiqueue is enabled AND Flow Control 5483 * is disabled. 5484 ************************************************************************/ 5485 static void 5486 ixgbe_enable_rx_drop(struct adapter *adapter) 5487 { 5488 struct ixgbe_hw *hw = &adapter->hw; 5489 struct rx_ring *rxr; 5490 u32 srrctl; 5491 5492 for (int i = 0; i < adapter->num_queues; i++) { 5493 rxr = &adapter->rx_rings[i]; 5494 srrctl = IXGBE_READ_REG(hw, IXGBE_SRRCTL(rxr->me)); 5495 srrctl |= IXGBE_SRRCTL_DROP_EN; 5496 IXGBE_WRITE_REG(hw, IXGBE_SRRCTL(rxr->me), srrctl); 5497 } 5498 5499 /* enable drop for each vf */ 5500 for (int i = 0; i < adapter->num_vfs; i++) { 5501 IXGBE_WRITE_REG(hw, IXGBE_QDE, 5502 (IXGBE_QDE_WRITE | (i << IXGBE_QDE_IDX_SHIFT) | 5503 IXGBE_QDE_ENABLE)); 5504 } 5505 } /* ixgbe_enable_rx_drop */ 5506 5507 /************************************************************************ 5508 * ixgbe_disable_rx_drop 5509 ************************************************************************/ 5510 static void 5511 ixgbe_disable_rx_drop(struct adapter *adapter) 5512 { 5513 struct ixgbe_hw *hw = &adapter->hw; 5514 struct rx_ring *rxr; 5515 u32 srrctl; 5516 5517 for (int i = 0; i < adapter->num_queues; i++) { 5518 rxr = &adapter->rx_rings[i]; 5519 srrctl = IXGBE_READ_REG(hw, IXGBE_SRRCTL(rxr->me)); 5520 srrctl &= ~IXGBE_SRRCTL_DROP_EN; 5521 IXGBE_WRITE_REG(hw, IXGBE_SRRCTL(rxr->me), srrctl); 5522 } 5523 5524 /* disable drop for each vf */ 5525 for (int i = 0; i < adapter->num_vfs; i++) { 5526 IXGBE_WRITE_REG(hw, IXGBE_QDE, 5527 (IXGBE_QDE_WRITE | (i << IXGBE_QDE_IDX_SHIFT))); 5528 } 5529 } /* ixgbe_disable_rx_drop */ 5530 5531 /************************************************************************ 5532 * ixgbe_sysctl_advertise 5533 * 5534 * SYSCTL wrapper around setting advertised speed 5535 ************************************************************************/ 5536 static int 5537 ixgbe_sysctl_advertise(SYSCTLFN_ARGS) 5538 { 5539 struct sysctlnode node = *rnode; 5540 struct adapter *adapter = (struct adapter *)node.sysctl_data; 5541 int error = 0, advertise; 5542 5543 if (ixgbe_fw_recovery_mode_swflag(adapter)) 5544 return (EPERM); 5545 5546 advertise = adapter->advertise; 5547 node.sysctl_data = &advertise; 5548 error = sysctl_lookup(SYSCTLFN_CALL(&node)); 5549 if (error != 0 || newp == NULL) 5550 return error; 5551 5552 return ixgbe_set_advertise(adapter, advertise); 5553 } /* ixgbe_sysctl_advertise */ 5554 5555 /************************************************************************ 5556 * ixgbe_set_advertise - Control advertised link speed 5557 * 5558 * Flags: 5559 * 0x00 - Default (all capable link speed) 5560 * 0x1 - advertise 100 Mb 5561 * 0x2 - advertise 1G 5562 * 0x4 - advertise 10G 5563 * 0x8 - advertise 10 Mb (yes, Mb) 5564 * 0x10 - advertise 2.5G 5565 * 0x20 - advertise 5G 5566 ************************************************************************/ 5567 static int 5568 ixgbe_set_advertise(struct adapter *adapter, int advertise) 5569 { 5570 device_t dev; 5571 struct ixgbe_hw *hw; 5572 ixgbe_link_speed speed = 0; 5573 ixgbe_link_speed link_caps = 0; 5574 s32 err = IXGBE_NOT_IMPLEMENTED; 5575 bool negotiate = FALSE; 5576 5577 /* Checks to validate new value */ 5578 if (adapter->advertise == advertise) /* no change */ 5579 return (0); 5580 5581 dev = adapter->dev; 5582 hw = &adapter->hw; 5583 5584 /* No speed changes for backplane media */ 5585 if (hw->phy.media_type == ixgbe_media_type_backplane) 5586 return (ENODEV); 5587 5588 if (!((hw->phy.media_type == ixgbe_media_type_copper) || 5589 (hw->phy.multispeed_fiber))) { 5590 device_printf(dev, 5591 "Advertised speed can only be set on copper or " 5592 "multispeed fiber media types.\n"); 5593 return (EINVAL); 5594 } 5595 5596 if (advertise < 0x0 || advertise > 0x3f) { 5597 device_printf(dev, "Invalid advertised speed; valid modes are 0x0 through 0x3f\n"); 5598 return (EINVAL); 5599 } 5600 5601 if (hw->mac.ops.get_link_capabilities) { 5602 err = hw->mac.ops.get_link_capabilities(hw, &link_caps, 5603 &negotiate); 5604 if (err != IXGBE_SUCCESS) { 5605 device_printf(dev, "Unable to determine supported advertise speeds\n"); 5606 return (ENODEV); 5607 } 5608 } 5609 5610 /* Set new value and report new advertised mode */ 5611 if (advertise & 0x1) { 5612 if (!(link_caps & IXGBE_LINK_SPEED_100_FULL)) { 5613 device_printf(dev, "Interface does not support 100Mb advertised speed\n"); 5614 return (EINVAL); 5615 } 5616 speed |= IXGBE_LINK_SPEED_100_FULL; 5617 } 5618 if (advertise & 0x2) { 5619 if (!(link_caps & IXGBE_LINK_SPEED_1GB_FULL)) { 5620 device_printf(dev, "Interface does not support 1Gb advertised speed\n"); 5621 return (EINVAL); 5622 } 5623 speed |= IXGBE_LINK_SPEED_1GB_FULL; 5624 } 5625 if (advertise & 0x4) { 5626 if (!(link_caps & IXGBE_LINK_SPEED_10GB_FULL)) { 5627 device_printf(dev, "Interface does not support 10Gb advertised speed\n"); 5628 return (EINVAL); 5629 } 5630 speed |= IXGBE_LINK_SPEED_10GB_FULL; 5631 } 5632 if (advertise & 0x8) { 5633 if (!(link_caps & IXGBE_LINK_SPEED_10_FULL)) { 5634 device_printf(dev, "Interface does not support 10Mb advertised speed\n"); 5635 return (EINVAL); 5636 } 5637 speed |= IXGBE_LINK_SPEED_10_FULL; 5638 } 5639 if (advertise & 0x10) { 5640 if (!(link_caps & IXGBE_LINK_SPEED_2_5GB_FULL)) { 5641 device_printf(dev, "Interface does not support 2.5Gb advertised speed\n"); 5642 return (EINVAL); 5643 } 5644 speed |= IXGBE_LINK_SPEED_2_5GB_FULL; 5645 } 5646 if (advertise & 0x20) { 5647 if (!(link_caps & IXGBE_LINK_SPEED_5GB_FULL)) { 5648 device_printf(dev, "Interface does not support 5Gb advertised speed\n"); 5649 return (EINVAL); 5650 } 5651 speed |= IXGBE_LINK_SPEED_5GB_FULL; 5652 } 5653 if (advertise == 0) 5654 speed = link_caps; /* All capable link speed */ 5655 5656 hw->mac.autotry_restart = TRUE; 5657 hw->mac.ops.setup_link(hw, speed, TRUE); 5658 adapter->advertise = advertise; 5659 5660 return (0); 5661 } /* ixgbe_set_advertise */ 5662 5663 /************************************************************************ 5664 * ixgbe_get_default_advertise - Get default advertised speed settings 5665 * 5666 * Formatted for sysctl usage. 5667 * Flags: 5668 * 0x1 - advertise 100 Mb 5669 * 0x2 - advertise 1G 5670 * 0x4 - advertise 10G 5671 * 0x8 - advertise 10 Mb (yes, Mb) 5672 * 0x10 - advertise 2.5G 5673 * 0x20 - advertise 5G 5674 ************************************************************************/ 5675 static int 5676 ixgbe_get_default_advertise(struct adapter *adapter) 5677 { 5678 struct ixgbe_hw *hw = &adapter->hw; 5679 int speed; 5680 ixgbe_link_speed link_caps = 0; 5681 s32 err; 5682 bool negotiate = FALSE; 5683 5684 /* 5685 * Advertised speed means nothing unless it's copper or 5686 * multi-speed fiber 5687 */ 5688 if (!(hw->phy.media_type == ixgbe_media_type_copper) && 5689 !(hw->phy.multispeed_fiber)) 5690 return (0); 5691 5692 err = hw->mac.ops.get_link_capabilities(hw, &link_caps, &negotiate); 5693 if (err != IXGBE_SUCCESS) 5694 return (0); 5695 5696 speed = 5697 ((link_caps & IXGBE_LINK_SPEED_10GB_FULL) ? 0x4 : 0) | 5698 ((link_caps & IXGBE_LINK_SPEED_5GB_FULL) ? 0x20 : 0) | 5699 ((link_caps & IXGBE_LINK_SPEED_2_5GB_FULL) ? 0x10 : 0) | 5700 ((link_caps & IXGBE_LINK_SPEED_1GB_FULL) ? 0x2 : 0) | 5701 ((link_caps & IXGBE_LINK_SPEED_100_FULL) ? 0x1 : 0) | 5702 ((link_caps & IXGBE_LINK_SPEED_10_FULL) ? 0x8 : 0); 5703 5704 return speed; 5705 } /* ixgbe_get_default_advertise */ 5706 5707 /************************************************************************ 5708 * ixgbe_sysctl_dmac - Manage DMA Coalescing 5709 * 5710 * Control values: 5711 * 0/1 - off / on (use default value of 1000) 5712 * 5713 * Legal timer values are: 5714 * 50,100,250,500,1000,2000,5000,10000 5715 * 5716 * Turning off interrupt moderation will also turn this off. 5717 ************************************************************************/ 5718 static int 5719 ixgbe_sysctl_dmac(SYSCTLFN_ARGS) 5720 { 5721 struct sysctlnode node = *rnode; 5722 struct adapter *adapter = (struct adapter *)node.sysctl_data; 5723 struct ifnet *ifp = adapter->ifp; 5724 int error; 5725 int newval; 5726 5727 if (ixgbe_fw_recovery_mode_swflag(adapter)) 5728 return (EPERM); 5729 5730 newval = adapter->dmac; 5731 node.sysctl_data = &newval; 5732 error = sysctl_lookup(SYSCTLFN_CALL(&node)); 5733 if ((error) || (newp == NULL)) 5734 return (error); 5735 5736 switch (newval) { 5737 case 0: 5738 /* Disabled */ 5739 adapter->dmac = 0; 5740 break; 5741 case 1: 5742 /* Enable and use default */ 5743 adapter->dmac = 1000; 5744 break; 5745 case 50: 5746 case 100: 5747 case 250: 5748 case 500: 5749 case 1000: 5750 case 2000: 5751 case 5000: 5752 case 10000: 5753 /* Legal values - allow */ 5754 adapter->dmac = newval; 5755 break; 5756 default: 5757 /* Do nothing, illegal value */ 5758 return (EINVAL); 5759 } 5760 5761 /* Re-initialize hardware if it's already running */ 5762 if (ifp->if_flags & IFF_RUNNING) 5763 if_init(ifp); 5764 5765 return (0); 5766 } 5767 5768 #ifdef IXGBE_DEBUG 5769 /************************************************************************ 5770 * ixgbe_sysctl_power_state 5771 * 5772 * Sysctl to test power states 5773 * Values: 5774 * 0 - set device to D0 5775 * 3 - set device to D3 5776 * (none) - get current device power state 5777 ************************************************************************/ 5778 static int 5779 ixgbe_sysctl_power_state(SYSCTLFN_ARGS) 5780 { 5781 #ifdef notyet 5782 struct sysctlnode node = *rnode; 5783 struct adapter *adapter = (struct adapter *)node.sysctl_data; 5784 device_t dev = adapter->dev; 5785 int curr_ps, new_ps, error = 0; 5786 5787 if (ixgbe_fw_recovery_mode_swflag(adapter)) 5788 return (EPERM); 5789 5790 curr_ps = new_ps = pci_get_powerstate(dev); 5791 5792 error = sysctl_lookup(SYSCTLFN_CALL(&node)); 5793 if ((error) || (req->newp == NULL)) 5794 return (error); 5795 5796 if (new_ps == curr_ps) 5797 return (0); 5798 5799 if (new_ps == 3 && curr_ps == 0) 5800 error = DEVICE_SUSPEND(dev); 5801 else if (new_ps == 0 && curr_ps == 3) 5802 error = DEVICE_RESUME(dev); 5803 else 5804 return (EINVAL); 5805 5806 device_printf(dev, "New state: %d\n", pci_get_powerstate(dev)); 5807 5808 return (error); 5809 #else 5810 return 0; 5811 #endif 5812 } /* ixgbe_sysctl_power_state */ 5813 #endif 5814 5815 /************************************************************************ 5816 * ixgbe_sysctl_wol_enable 5817 * 5818 * Sysctl to enable/disable the WoL capability, 5819 * if supported by the adapter. 5820 * 5821 * Values: 5822 * 0 - disabled 5823 * 1 - enabled 5824 ************************************************************************/ 5825 static int 5826 ixgbe_sysctl_wol_enable(SYSCTLFN_ARGS) 5827 { 5828 struct sysctlnode node = *rnode; 5829 struct adapter *adapter = (struct adapter *)node.sysctl_data; 5830 struct ixgbe_hw *hw = &adapter->hw; 5831 bool new_wol_enabled; 5832 int error = 0; 5833 5834 /* 5835 * It's not required to check recovery mode because this function never 5836 * touches hardware. 5837 */ 5838 new_wol_enabled = hw->wol_enabled; 5839 node.sysctl_data = &new_wol_enabled; 5840 error = sysctl_lookup(SYSCTLFN_CALL(&node)); 5841 if ((error) || (newp == NULL)) 5842 return (error); 5843 if (new_wol_enabled == hw->wol_enabled) 5844 return (0); 5845 5846 if (new_wol_enabled && !adapter->wol_support) 5847 return (ENODEV); 5848 else 5849 hw->wol_enabled = new_wol_enabled; 5850 5851 return (0); 5852 } /* ixgbe_sysctl_wol_enable */ 5853 5854 /************************************************************************ 5855 * ixgbe_sysctl_wufc - Wake Up Filter Control 5856 * 5857 * Sysctl to enable/disable the types of packets that the 5858 * adapter will wake up on upon receipt. 5859 * Flags: 5860 * 0x1 - Link Status Change 5861 * 0x2 - Magic Packet 5862 * 0x4 - Direct Exact 5863 * 0x8 - Directed Multicast 5864 * 0x10 - Broadcast 5865 * 0x20 - ARP/IPv4 Request Packet 5866 * 0x40 - Direct IPv4 Packet 5867 * 0x80 - Direct IPv6 Packet 5868 * 5869 * Settings not listed above will cause the sysctl to return an error. 5870 ************************************************************************/ 5871 static int 5872 ixgbe_sysctl_wufc(SYSCTLFN_ARGS) 5873 { 5874 struct sysctlnode node = *rnode; 5875 struct adapter *adapter = (struct adapter *)node.sysctl_data; 5876 int error = 0; 5877 u32 new_wufc; 5878 5879 /* 5880 * It's not required to check recovery mode because this function never 5881 * touches hardware. 5882 */ 5883 new_wufc = adapter->wufc; 5884 node.sysctl_data = &new_wufc; 5885 error = sysctl_lookup(SYSCTLFN_CALL(&node)); 5886 if ((error) || (newp == NULL)) 5887 return (error); 5888 if (new_wufc == adapter->wufc) 5889 return (0); 5890 5891 if (new_wufc & 0xffffff00) 5892 return (EINVAL); 5893 5894 new_wufc &= 0xff; 5895 new_wufc |= (0xffffff & adapter->wufc); 5896 adapter->wufc = new_wufc; 5897 5898 return (0); 5899 } /* ixgbe_sysctl_wufc */ 5900 5901 #ifdef IXGBE_DEBUG 5902 /************************************************************************ 5903 * ixgbe_sysctl_print_rss_config 5904 ************************************************************************/ 5905 static int 5906 ixgbe_sysctl_print_rss_config(SYSCTLFN_ARGS) 5907 { 5908 #ifdef notyet 5909 struct sysctlnode node = *rnode; 5910 struct adapter *adapter = (struct adapter *)node.sysctl_data; 5911 struct ixgbe_hw *hw = &adapter->hw; 5912 device_t dev = adapter->dev; 5913 struct sbuf *buf; 5914 int error = 0, reta_size; 5915 u32 reg; 5916 5917 if (ixgbe_fw_recovery_mode_swflag(adapter)) 5918 return (EPERM); 5919 5920 buf = sbuf_new_for_sysctl(NULL, NULL, 128, req); 5921 if (!buf) { 5922 device_printf(dev, "Could not allocate sbuf for output.\n"); 5923 return (ENOMEM); 5924 } 5925 5926 // TODO: use sbufs to make a string to print out 5927 /* Set multiplier for RETA setup and table size based on MAC */ 5928 switch (adapter->hw.mac.type) { 5929 case ixgbe_mac_X550: 5930 case ixgbe_mac_X550EM_x: 5931 case ixgbe_mac_X550EM_a: 5932 reta_size = 128; 5933 break; 5934 default: 5935 reta_size = 32; 5936 break; 5937 } 5938 5939 /* Print out the redirection table */ 5940 sbuf_cat(buf, "\n"); 5941 for (int i = 0; i < reta_size; i++) { 5942 if (i < 32) { 5943 reg = IXGBE_READ_REG(hw, IXGBE_RETA(i)); 5944 sbuf_printf(buf, "RETA(%2d): 0x%08x\n", i, reg); 5945 } else { 5946 reg = IXGBE_READ_REG(hw, IXGBE_ERETA(i - 32)); 5947 sbuf_printf(buf, "ERETA(%2d): 0x%08x\n", i - 32, reg); 5948 } 5949 } 5950 5951 // TODO: print more config 5952 5953 error = sbuf_finish(buf); 5954 if (error) 5955 device_printf(dev, "Error finishing sbuf: %d\n", error); 5956 5957 sbuf_delete(buf); 5958 #endif 5959 return (0); 5960 } /* ixgbe_sysctl_print_rss_config */ 5961 #endif /* IXGBE_DEBUG */ 5962 5963 /************************************************************************ 5964 * ixgbe_sysctl_phy_temp - Retrieve temperature of PHY 5965 * 5966 * For X552/X557-AT devices using an external PHY 5967 ************************************************************************/ 5968 static int 5969 ixgbe_sysctl_phy_temp(SYSCTLFN_ARGS) 5970 { 5971 struct sysctlnode node = *rnode; 5972 struct adapter *adapter = (struct adapter *)node.sysctl_data; 5973 struct ixgbe_hw *hw = &adapter->hw; 5974 int val; 5975 u16 reg; 5976 int error; 5977 5978 if (ixgbe_fw_recovery_mode_swflag(adapter)) 5979 return (EPERM); 5980 5981 if (hw->device_id != IXGBE_DEV_ID_X550EM_X_10G_T) { 5982 device_printf(adapter->dev, 5983 "Device has no supported external thermal sensor.\n"); 5984 return (ENODEV); 5985 } 5986 5987 if (hw->phy.ops.read_reg(hw, IXGBE_PHY_CURRENT_TEMP, 5988 IXGBE_MDIO_VENDOR_SPECIFIC_1_DEV_TYPE, ®)) { 5989 device_printf(adapter->dev, 5990 "Error reading from PHY's current temperature register\n"); 5991 return (EAGAIN); 5992 } 5993 5994 node.sysctl_data = &val; 5995 5996 /* Shift temp for output */ 5997 val = reg >> 8; 5998 5999 error = sysctl_lookup(SYSCTLFN_CALL(&node)); 6000 if ((error) || (newp == NULL)) 6001 return (error); 6002 6003 return (0); 6004 } /* ixgbe_sysctl_phy_temp */ 6005 6006 /************************************************************************ 6007 * ixgbe_sysctl_phy_overtemp_occurred 6008 * 6009 * Reports (directly from the PHY) whether the current PHY 6010 * temperature is over the overtemp threshold. 6011 ************************************************************************/ 6012 static int 6013 ixgbe_sysctl_phy_overtemp_occurred(SYSCTLFN_ARGS) 6014 { 6015 struct sysctlnode node = *rnode; 6016 struct adapter *adapter = (struct adapter *)node.sysctl_data; 6017 struct ixgbe_hw *hw = &adapter->hw; 6018 int val, error; 6019 u16 reg; 6020 6021 if (ixgbe_fw_recovery_mode_swflag(adapter)) 6022 return (EPERM); 6023 6024 if (hw->device_id != IXGBE_DEV_ID_X550EM_X_10G_T) { 6025 device_printf(adapter->dev, 6026 "Device has no supported external thermal sensor.\n"); 6027 return (ENODEV); 6028 } 6029 6030 if (hw->phy.ops.read_reg(hw, IXGBE_PHY_OVERTEMP_STATUS, 6031 IXGBE_MDIO_VENDOR_SPECIFIC_1_DEV_TYPE, ®)) { 6032 device_printf(adapter->dev, 6033 "Error reading from PHY's temperature status register\n"); 6034 return (EAGAIN); 6035 } 6036 6037 node.sysctl_data = &val; 6038 6039 /* Get occurrence bit */ 6040 val = !!(reg & 0x4000); 6041 6042 error = sysctl_lookup(SYSCTLFN_CALL(&node)); 6043 if ((error) || (newp == NULL)) 6044 return (error); 6045 6046 return (0); 6047 } /* ixgbe_sysctl_phy_overtemp_occurred */ 6048 6049 /************************************************************************ 6050 * ixgbe_sysctl_eee_state 6051 * 6052 * Sysctl to set EEE power saving feature 6053 * Values: 6054 * 0 - disable EEE 6055 * 1 - enable EEE 6056 * (none) - get current device EEE state 6057 ************************************************************************/ 6058 static int 6059 ixgbe_sysctl_eee_state(SYSCTLFN_ARGS) 6060 { 6061 struct sysctlnode node = *rnode; 6062 struct adapter *adapter = (struct adapter *)node.sysctl_data; 6063 struct ifnet *ifp = adapter->ifp; 6064 device_t dev = adapter->dev; 6065 int curr_eee, new_eee, error = 0; 6066 s32 retval; 6067 6068 if (ixgbe_fw_recovery_mode_swflag(adapter)) 6069 return (EPERM); 6070 6071 curr_eee = new_eee = !!(adapter->feat_en & IXGBE_FEATURE_EEE); 6072 node.sysctl_data = &new_eee; 6073 error = sysctl_lookup(SYSCTLFN_CALL(&node)); 6074 if ((error) || (newp == NULL)) 6075 return (error); 6076 6077 /* Nothing to do */ 6078 if (new_eee == curr_eee) 6079 return (0); 6080 6081 /* Not supported */ 6082 if (!(adapter->feat_cap & IXGBE_FEATURE_EEE)) 6083 return (EINVAL); 6084 6085 /* Bounds checking */ 6086 if ((new_eee < 0) || (new_eee > 1)) 6087 return (EINVAL); 6088 6089 retval = ixgbe_setup_eee(&adapter->hw, new_eee); 6090 if (retval) { 6091 device_printf(dev, "Error in EEE setup: 0x%08X\n", retval); 6092 return (EINVAL); 6093 } 6094 6095 /* Restart auto-neg */ 6096 if_init(ifp); 6097 6098 device_printf(dev, "New EEE state: %d\n", new_eee); 6099 6100 /* Cache new value */ 6101 if (new_eee) 6102 adapter->feat_en |= IXGBE_FEATURE_EEE; 6103 else 6104 adapter->feat_en &= ~IXGBE_FEATURE_EEE; 6105 6106 return (error); 6107 } /* ixgbe_sysctl_eee_state */ 6108 6109 #define PRINTQS(adapter, regname) \ 6110 do { \ 6111 struct ixgbe_hw *_hw = &(adapter)->hw; \ 6112 int _i; \ 6113 \ 6114 printf("%s: %s", device_xname((adapter)->dev), #regname); \ 6115 for (_i = 0; _i < (adapter)->num_queues; _i++) { \ 6116 printf((_i == 0) ? "\t" : " "); \ 6117 printf("%08x", IXGBE_READ_REG(_hw, \ 6118 IXGBE_##regname(_i))); \ 6119 } \ 6120 printf("\n"); \ 6121 } while (0) 6122 6123 /************************************************************************ 6124 * ixgbe_print_debug_info 6125 * 6126 * Called only when em_display_debug_stats is enabled. 6127 * Provides a way to take a look at important statistics 6128 * maintained by the driver and hardware. 6129 ************************************************************************/ 6130 static void 6131 ixgbe_print_debug_info(struct adapter *adapter) 6132 { 6133 device_t dev = adapter->dev; 6134 struct ixgbe_hw *hw = &adapter->hw; 6135 int table_size; 6136 int i; 6137 6138 switch (adapter->hw.mac.type) { 6139 case ixgbe_mac_X550: 6140 case ixgbe_mac_X550EM_x: 6141 case ixgbe_mac_X550EM_a: 6142 table_size = 128; 6143 break; 6144 default: 6145 table_size = 32; 6146 break; 6147 } 6148 6149 device_printf(dev, "[E]RETA:\n"); 6150 for (i = 0; i < table_size; i++) { 6151 if (i < 32) 6152 printf("%02x: %08x\n", i, IXGBE_READ_REG(hw, 6153 IXGBE_RETA(i))); 6154 else 6155 printf("%02x: %08x\n", i, IXGBE_READ_REG(hw, 6156 IXGBE_ERETA(i - 32))); 6157 } 6158 6159 device_printf(dev, "queue:"); 6160 for (i = 0; i < adapter->num_queues; i++) { 6161 printf((i == 0) ? "\t" : " "); 6162 printf("%8d", i); 6163 } 6164 printf("\n"); 6165 PRINTQS(adapter, RDBAL); 6166 PRINTQS(adapter, RDBAH); 6167 PRINTQS(adapter, RDLEN); 6168 PRINTQS(adapter, SRRCTL); 6169 PRINTQS(adapter, RDH); 6170 PRINTQS(adapter, RDT); 6171 PRINTQS(adapter, RXDCTL); 6172 6173 device_printf(dev, "RQSMR:"); 6174 for (i = 0; i < adapter->num_queues / 4; i++) { 6175 printf((i == 0) ? "\t" : " "); 6176 printf("%08x", IXGBE_READ_REG(hw, IXGBE_RQSMR(i))); 6177 } 6178 printf("\n"); 6179 6180 device_printf(dev, "disabled_count:"); 6181 for (i = 0; i < adapter->num_queues; i++) { 6182 printf((i == 0) ? "\t" : " "); 6183 printf("%8d", adapter->queues[i].disabled_count); 6184 } 6185 printf("\n"); 6186 6187 device_printf(dev, "EIMS:\t%08x\n", IXGBE_READ_REG(hw, IXGBE_EIMS)); 6188 if (hw->mac.type != ixgbe_mac_82598EB) { 6189 device_printf(dev, "EIMS_EX(0):\t%08x\n", 6190 IXGBE_READ_REG(hw, IXGBE_EIMS_EX(0))); 6191 device_printf(dev, "EIMS_EX(1):\t%08x\n", 6192 IXGBE_READ_REG(hw, IXGBE_EIMS_EX(1))); 6193 } 6194 device_printf(dev, "EIAM:\t%08x\n", IXGBE_READ_REG(hw, IXGBE_EIAM)); 6195 device_printf(dev, "EIAC:\t%08x\n", IXGBE_READ_REG(hw, IXGBE_EIAC)); 6196 } /* ixgbe_print_debug_info */ 6197 6198 /************************************************************************ 6199 * ixgbe_sysctl_debug 6200 ************************************************************************/ 6201 static int 6202 ixgbe_sysctl_debug(SYSCTLFN_ARGS) 6203 { 6204 struct sysctlnode node = *rnode; 6205 struct adapter *adapter = (struct adapter *)node.sysctl_data; 6206 int error, result = 0; 6207 6208 if (ixgbe_fw_recovery_mode_swflag(adapter)) 6209 return (EPERM); 6210 6211 node.sysctl_data = &result; 6212 error = sysctl_lookup(SYSCTLFN_CALL(&node)); 6213 6214 if (error || newp == NULL) 6215 return error; 6216 6217 if (result == 1) 6218 ixgbe_print_debug_info(adapter); 6219 6220 return 0; 6221 } /* ixgbe_sysctl_debug */ 6222 6223 /************************************************************************ 6224 * ixgbe_sysctl_rx_copy_len 6225 ************************************************************************/ 6226 static int 6227 ixgbe_sysctl_rx_copy_len(SYSCTLFN_ARGS) 6228 { 6229 struct sysctlnode node = *rnode; 6230 struct adapter *adapter = (struct adapter *)node.sysctl_data; 6231 int error; 6232 int result = adapter->rx_copy_len; 6233 6234 node.sysctl_data = &result; 6235 error = sysctl_lookup(SYSCTLFN_CALL(&node)); 6236 6237 if (error || newp == NULL) 6238 return error; 6239 6240 if ((result < 0) || (result > IXGBE_RX_COPY_LEN_MAX)) 6241 return EINVAL; 6242 6243 adapter->rx_copy_len = result; 6244 6245 return 0; 6246 } /* ixgbe_sysctl_rx_copy_len */ 6247 6248 /************************************************************************ 6249 * ixgbe_init_device_features 6250 ************************************************************************/ 6251 static void 6252 ixgbe_init_device_features(struct adapter *adapter) 6253 { 6254 adapter->feat_cap = IXGBE_FEATURE_NETMAP 6255 | IXGBE_FEATURE_RSS 6256 | IXGBE_FEATURE_MSI 6257 | IXGBE_FEATURE_MSIX 6258 | IXGBE_FEATURE_LEGACY_IRQ 6259 | IXGBE_FEATURE_LEGACY_TX; 6260 6261 /* Set capabilities first... */ 6262 switch (adapter->hw.mac.type) { 6263 case ixgbe_mac_82598EB: 6264 if (adapter->hw.device_id == IXGBE_DEV_ID_82598AT) 6265 adapter->feat_cap |= IXGBE_FEATURE_FAN_FAIL; 6266 break; 6267 case ixgbe_mac_X540: 6268 adapter->feat_cap |= IXGBE_FEATURE_SRIOV; 6269 adapter->feat_cap |= IXGBE_FEATURE_FDIR; 6270 if ((adapter->hw.device_id == IXGBE_DEV_ID_X540_BYPASS) && 6271 (adapter->hw.bus.func == 0)) 6272 adapter->feat_cap |= IXGBE_FEATURE_BYPASS; 6273 break; 6274 case ixgbe_mac_X550: 6275 /* 6276 * IXGBE_FEATURE_RECOVERY_MODE will be set after reading 6277 * NVM Image version. 6278 */ 6279 adapter->feat_cap |= IXGBE_FEATURE_TEMP_SENSOR; 6280 adapter->feat_cap |= IXGBE_FEATURE_SRIOV; 6281 adapter->feat_cap |= IXGBE_FEATURE_FDIR; 6282 break; 6283 case ixgbe_mac_X550EM_x: 6284 /* 6285 * IXGBE_FEATURE_RECOVERY_MODE will be set after reading 6286 * NVM Image version. 6287 */ 6288 adapter->feat_cap |= IXGBE_FEATURE_SRIOV; 6289 adapter->feat_cap |= IXGBE_FEATURE_FDIR; 6290 break; 6291 case ixgbe_mac_X550EM_a: 6292 /* 6293 * IXGBE_FEATURE_RECOVERY_MODE will be set after reading 6294 * NVM Image version. 6295 */ 6296 adapter->feat_cap |= IXGBE_FEATURE_SRIOV; 6297 adapter->feat_cap |= IXGBE_FEATURE_FDIR; 6298 adapter->feat_cap &= ~IXGBE_FEATURE_LEGACY_IRQ; 6299 if ((adapter->hw.device_id == IXGBE_DEV_ID_X550EM_A_1G_T) || 6300 (adapter->hw.device_id == IXGBE_DEV_ID_X550EM_A_1G_T_L)) { 6301 adapter->feat_cap |= IXGBE_FEATURE_TEMP_SENSOR; 6302 adapter->feat_cap |= IXGBE_FEATURE_EEE; 6303 } 6304 break; 6305 case ixgbe_mac_82599EB: 6306 adapter->feat_cap |= IXGBE_FEATURE_SRIOV; 6307 adapter->feat_cap |= IXGBE_FEATURE_FDIR; 6308 if ((adapter->hw.device_id == IXGBE_DEV_ID_82599_BYPASS) && 6309 (adapter->hw.bus.func == 0)) 6310 adapter->feat_cap |= IXGBE_FEATURE_BYPASS; 6311 if (adapter->hw.device_id == IXGBE_DEV_ID_82599_QSFP_SF_QP) 6312 adapter->feat_cap &= ~IXGBE_FEATURE_LEGACY_IRQ; 6313 break; 6314 default: 6315 break; 6316 } 6317 6318 /* Enabled by default... */ 6319 /* Fan failure detection */ 6320 if (adapter->feat_cap & IXGBE_FEATURE_FAN_FAIL) 6321 adapter->feat_en |= IXGBE_FEATURE_FAN_FAIL; 6322 /* Netmap */ 6323 if (adapter->feat_cap & IXGBE_FEATURE_NETMAP) 6324 adapter->feat_en |= IXGBE_FEATURE_NETMAP; 6325 /* EEE */ 6326 if (adapter->feat_cap & IXGBE_FEATURE_EEE) 6327 adapter->feat_en |= IXGBE_FEATURE_EEE; 6328 /* Thermal Sensor */ 6329 if (adapter->feat_cap & IXGBE_FEATURE_TEMP_SENSOR) 6330 adapter->feat_en |= IXGBE_FEATURE_TEMP_SENSOR; 6331 /* 6332 * Recovery mode: 6333 * NetBSD: IXGBE_FEATURE_RECOVERY_MODE will be controlled after reading 6334 * NVM Image version. 6335 */ 6336 6337 /* Enabled via global sysctl... */ 6338 /* Flow Director */ 6339 if (ixgbe_enable_fdir) { 6340 if (adapter->feat_cap & IXGBE_FEATURE_FDIR) 6341 adapter->feat_en |= IXGBE_FEATURE_FDIR; 6342 else 6343 device_printf(adapter->dev, "Device does not support Flow Director. Leaving disabled."); 6344 } 6345 /* Legacy (single queue) transmit */ 6346 if ((adapter->feat_cap & IXGBE_FEATURE_LEGACY_TX) && 6347 ixgbe_enable_legacy_tx) 6348 adapter->feat_en |= IXGBE_FEATURE_LEGACY_TX; 6349 /* 6350 * Message Signal Interrupts - Extended (MSI-X) 6351 * Normal MSI is only enabled if MSI-X calls fail. 6352 */ 6353 if (!ixgbe_enable_msix) 6354 adapter->feat_cap &= ~IXGBE_FEATURE_MSIX; 6355 /* Receive-Side Scaling (RSS) */ 6356 if ((adapter->feat_cap & IXGBE_FEATURE_RSS) && ixgbe_enable_rss) 6357 adapter->feat_en |= IXGBE_FEATURE_RSS; 6358 6359 /* Disable features with unmet dependencies... */ 6360 /* No MSI-X */ 6361 if (!(adapter->feat_cap & IXGBE_FEATURE_MSIX)) { 6362 adapter->feat_cap &= ~IXGBE_FEATURE_RSS; 6363 adapter->feat_cap &= ~IXGBE_FEATURE_SRIOV; 6364 adapter->feat_en &= ~IXGBE_FEATURE_RSS; 6365 adapter->feat_en &= ~IXGBE_FEATURE_SRIOV; 6366 } 6367 } /* ixgbe_init_device_features */ 6368 6369 /************************************************************************ 6370 * ixgbe_probe - Device identification routine 6371 * 6372 * Determines if the driver should be loaded on 6373 * adapter based on its PCI vendor/device ID. 6374 * 6375 * return BUS_PROBE_DEFAULT on success, positive on failure 6376 ************************************************************************/ 6377 static int 6378 ixgbe_probe(device_t dev, cfdata_t cf, void *aux) 6379 { 6380 const struct pci_attach_args *pa = aux; 6381 6382 return (ixgbe_lookup(pa) != NULL) ? 1 : 0; 6383 } 6384 6385 static const ixgbe_vendor_info_t * 6386 ixgbe_lookup(const struct pci_attach_args *pa) 6387 { 6388 const ixgbe_vendor_info_t *ent; 6389 pcireg_t subid; 6390 6391 INIT_DEBUGOUT("ixgbe_lookup: begin"); 6392 6393 if (PCI_VENDOR(pa->pa_id) != IXGBE_INTEL_VENDOR_ID) 6394 return NULL; 6395 6396 subid = pci_conf_read(pa->pa_pc, pa->pa_tag, PCI_SUBSYS_ID_REG); 6397 6398 for (ent = ixgbe_vendor_info_array; ent->vendor_id != 0; ent++) { 6399 if ((PCI_VENDOR(pa->pa_id) == ent->vendor_id) && 6400 (PCI_PRODUCT(pa->pa_id) == ent->device_id) && 6401 ((PCI_SUBSYS_VENDOR(subid) == ent->subvendor_id) || 6402 (ent->subvendor_id == 0)) && 6403 ((PCI_SUBSYS_ID(subid) == ent->subdevice_id) || 6404 (ent->subdevice_id == 0))) { 6405 return ent; 6406 } 6407 } 6408 return NULL; 6409 } 6410 6411 static int 6412 ixgbe_ifflags_cb(struct ethercom *ec) 6413 { 6414 struct ifnet *ifp = &ec->ec_if; 6415 struct adapter *adapter = ifp->if_softc; 6416 u_short change; 6417 int rv = 0; 6418 6419 IXGBE_CORE_LOCK(adapter); 6420 6421 change = ifp->if_flags ^ adapter->if_flags; 6422 if (change != 0) 6423 adapter->if_flags = ifp->if_flags; 6424 6425 if ((change & ~(IFF_CANTCHANGE | IFF_DEBUG)) != 0) { 6426 rv = ENETRESET; 6427 goto out; 6428 } else if ((change & IFF_PROMISC) != 0) 6429 ixgbe_set_rxfilter(adapter); 6430 6431 /* Check for ec_capenable. */ 6432 change = ec->ec_capenable ^ adapter->ec_capenable; 6433 adapter->ec_capenable = ec->ec_capenable; 6434 if ((change & ~(ETHERCAP_VLAN_MTU | ETHERCAP_VLAN_HWTAGGING 6435 | ETHERCAP_VLAN_HWFILTER)) != 0) { 6436 rv = ENETRESET; 6437 goto out; 6438 } 6439 6440 /* 6441 * Special handling is not required for ETHERCAP_VLAN_MTU. 6442 * MAXFRS(MHADD) does not include the 4bytes of the VLAN header. 6443 */ 6444 6445 /* Set up VLAN support and filter */ 6446 if ((change & (ETHERCAP_VLAN_HWTAGGING | ETHERCAP_VLAN_HWFILTER)) != 0) 6447 ixgbe_setup_vlan_hw_support(adapter); 6448 6449 out: 6450 IXGBE_CORE_UNLOCK(adapter); 6451 6452 return rv; 6453 } 6454 6455 /************************************************************************ 6456 * ixgbe_ioctl - Ioctl entry point 6457 * 6458 * Called when the user wants to configure the interface. 6459 * 6460 * return 0 on success, positive on failure 6461 ************************************************************************/ 6462 static int 6463 ixgbe_ioctl(struct ifnet *ifp, u_long command, void *data) 6464 { 6465 struct adapter *adapter = ifp->if_softc; 6466 struct ixgbe_hw *hw = &adapter->hw; 6467 struct ifcapreq *ifcr = data; 6468 struct ifreq *ifr = data; 6469 int error = 0; 6470 int l4csum_en; 6471 const int l4csum = IFCAP_CSUM_TCPv4_Rx | IFCAP_CSUM_UDPv4_Rx | 6472 IFCAP_CSUM_TCPv6_Rx | IFCAP_CSUM_UDPv6_Rx; 6473 6474 if (ixgbe_fw_recovery_mode_swflag(adapter)) 6475 return (EPERM); 6476 6477 switch (command) { 6478 case SIOCSIFFLAGS: 6479 IOCTL_DEBUGOUT("ioctl: SIOCSIFFLAGS (Set Interface Flags)"); 6480 break; 6481 case SIOCADDMULTI: 6482 case SIOCDELMULTI: 6483 IOCTL_DEBUGOUT("ioctl: SIOC(ADD|DEL)MULTI"); 6484 break; 6485 case SIOCSIFMEDIA: 6486 case SIOCGIFMEDIA: 6487 IOCTL_DEBUGOUT("ioctl: SIOCxIFMEDIA (Get/Set Interface Media)"); 6488 break; 6489 case SIOCSIFCAP: 6490 IOCTL_DEBUGOUT("ioctl: SIOCSIFCAP (Set Capabilities)"); 6491 break; 6492 case SIOCSIFMTU: 6493 IOCTL_DEBUGOUT("ioctl: SIOCSIFMTU (Set Interface MTU)"); 6494 break; 6495 #ifdef __NetBSD__ 6496 case SIOCINITIFADDR: 6497 IOCTL_DEBUGOUT("ioctl: SIOCINITIFADDR"); 6498 break; 6499 case SIOCGIFFLAGS: 6500 IOCTL_DEBUGOUT("ioctl: SIOCGIFFLAGS"); 6501 break; 6502 case SIOCGIFAFLAG_IN: 6503 IOCTL_DEBUGOUT("ioctl: SIOCGIFAFLAG_IN"); 6504 break; 6505 case SIOCGIFADDR: 6506 IOCTL_DEBUGOUT("ioctl: SIOCGIFADDR"); 6507 break; 6508 case SIOCGIFMTU: 6509 IOCTL_DEBUGOUT("ioctl: SIOCGIFMTU (Get Interface MTU)"); 6510 break; 6511 case SIOCGIFCAP: 6512 IOCTL_DEBUGOUT("ioctl: SIOCGIFCAP (Get IF cap)"); 6513 break; 6514 case SIOCGETHERCAP: 6515 IOCTL_DEBUGOUT("ioctl: SIOCGETHERCAP (Get ethercap)"); 6516 break; 6517 case SIOCGLIFADDR: 6518 IOCTL_DEBUGOUT("ioctl: SIOCGLIFADDR (Get Interface addr)"); 6519 break; 6520 case SIOCZIFDATA: 6521 IOCTL_DEBUGOUT("ioctl: SIOCZIFDATA (Zero counter)"); 6522 hw->mac.ops.clear_hw_cntrs(hw); 6523 ixgbe_clear_evcnt(adapter); 6524 break; 6525 case SIOCAIFADDR: 6526 IOCTL_DEBUGOUT("ioctl: SIOCAIFADDR (add/chg IF alias)"); 6527 break; 6528 #endif 6529 default: 6530 IOCTL_DEBUGOUT1("ioctl: UNKNOWN (0x%X)", (int)command); 6531 break; 6532 } 6533 6534 switch (command) { 6535 case SIOCGI2C: 6536 { 6537 struct ixgbe_i2c_req i2c; 6538 6539 IOCTL_DEBUGOUT("ioctl: SIOCGI2C (Get I2C Data)"); 6540 error = copyin(ifr->ifr_data, &i2c, sizeof(i2c)); 6541 if (error != 0) 6542 break; 6543 if (i2c.dev_addr != 0xA0 && i2c.dev_addr != 0xA2) { 6544 error = EINVAL; 6545 break; 6546 } 6547 if (i2c.len > sizeof(i2c.data)) { 6548 error = EINVAL; 6549 break; 6550 } 6551 6552 hw->phy.ops.read_i2c_byte(hw, i2c.offset, 6553 i2c.dev_addr, i2c.data); 6554 error = copyout(&i2c, ifr->ifr_data, sizeof(i2c)); 6555 break; 6556 } 6557 case SIOCSIFCAP: 6558 /* Layer-4 Rx checksum offload has to be turned on and 6559 * off as a unit. 6560 */ 6561 l4csum_en = ifcr->ifcr_capenable & l4csum; 6562 if (l4csum_en != l4csum && l4csum_en != 0) 6563 return EINVAL; 6564 /*FALLTHROUGH*/ 6565 case SIOCADDMULTI: 6566 case SIOCDELMULTI: 6567 case SIOCSIFFLAGS: 6568 case SIOCSIFMTU: 6569 default: 6570 if ((error = ether_ioctl(ifp, command, data)) != ENETRESET) 6571 return error; 6572 if ((ifp->if_flags & IFF_RUNNING) == 0) 6573 ; 6574 else if (command == SIOCSIFCAP || command == SIOCSIFMTU) { 6575 IXGBE_CORE_LOCK(adapter); 6576 if ((ifp->if_flags & IFF_RUNNING) != 0) 6577 ixgbe_init_locked(adapter); 6578 ixgbe_recalculate_max_frame(adapter); 6579 IXGBE_CORE_UNLOCK(adapter); 6580 } else if (command == SIOCADDMULTI || command == SIOCDELMULTI) { 6581 /* 6582 * Multicast list has changed; set the hardware filter 6583 * accordingly. 6584 */ 6585 IXGBE_CORE_LOCK(adapter); 6586 ixgbe_disable_intr(adapter); 6587 ixgbe_set_rxfilter(adapter); 6588 ixgbe_enable_intr(adapter); 6589 IXGBE_CORE_UNLOCK(adapter); 6590 } 6591 return 0; 6592 } 6593 6594 return error; 6595 } /* ixgbe_ioctl */ 6596 6597 /************************************************************************ 6598 * ixgbe_check_fan_failure 6599 ************************************************************************/ 6600 static int 6601 ixgbe_check_fan_failure(struct adapter *adapter, u32 reg, bool in_interrupt) 6602 { 6603 u32 mask; 6604 6605 mask = (in_interrupt) ? IXGBE_EICR_GPI_SDP1_BY_MAC(&adapter->hw) : 6606 IXGBE_ESDP_SDP1; 6607 6608 if (reg & mask) { 6609 device_printf(adapter->dev, 6610 "\nCRITICAL: FAN FAILURE!! REPLACE IMMEDIATELY!!\n"); 6611 return IXGBE_ERR_FAN_FAILURE; 6612 } 6613 6614 return IXGBE_SUCCESS; 6615 } /* ixgbe_check_fan_failure */ 6616 6617 /************************************************************************ 6618 * ixgbe_handle_que 6619 ************************************************************************/ 6620 static void 6621 ixgbe_handle_que(void *context) 6622 { 6623 struct ix_queue *que = context; 6624 struct adapter *adapter = que->adapter; 6625 struct tx_ring *txr = que->txr; 6626 struct ifnet *ifp = adapter->ifp; 6627 bool more = false; 6628 6629 IXGBE_EVC_ADD(&que->handleq, 1); 6630 6631 if (ifp->if_flags & IFF_RUNNING) { 6632 more = ixgbe_rxeof(que); 6633 IXGBE_TX_LOCK(txr); 6634 more |= ixgbe_txeof(txr); 6635 if (!(adapter->feat_en & IXGBE_FEATURE_LEGACY_TX)) 6636 if (!ixgbe_mq_ring_empty(ifp, txr->txr_interq)) 6637 ixgbe_mq_start_locked(ifp, txr); 6638 /* Only for queue 0 */ 6639 /* NetBSD still needs this for CBQ */ 6640 if ((&adapter->queues[0] == que) 6641 && (!ixgbe_legacy_ring_empty(ifp, NULL))) 6642 ixgbe_legacy_start_locked(ifp, txr); 6643 IXGBE_TX_UNLOCK(txr); 6644 } 6645 6646 if (more) { 6647 IXGBE_EVC_ADD(&que->req, 1); 6648 ixgbe_sched_handle_que(adapter, que); 6649 } else if (que->res != NULL) { 6650 /* MSIX: Re-enable this interrupt */ 6651 ixgbe_enable_queue(adapter, que->msix); 6652 } else { 6653 /* INTx or MSI */ 6654 ixgbe_enable_queue(adapter, 0); 6655 } 6656 6657 return; 6658 } /* ixgbe_handle_que */ 6659 6660 /************************************************************************ 6661 * ixgbe_handle_que_work 6662 ************************************************************************/ 6663 static void 6664 ixgbe_handle_que_work(struct work *wk, void *context) 6665 { 6666 struct ix_queue *que = container_of(wk, struct ix_queue, wq_cookie); 6667 6668 /* 6669 * "enqueued flag" is not required here. 6670 * See ixgbe_msix_que(). 6671 */ 6672 ixgbe_handle_que(que); 6673 } 6674 6675 /************************************************************************ 6676 * ixgbe_allocate_legacy - Setup the Legacy or MSI Interrupt handler 6677 ************************************************************************/ 6678 static int 6679 ixgbe_allocate_legacy(struct adapter *adapter, 6680 const struct pci_attach_args *pa) 6681 { 6682 device_t dev = adapter->dev; 6683 struct ix_queue *que = adapter->queues; 6684 struct tx_ring *txr = adapter->tx_rings; 6685 int counts[PCI_INTR_TYPE_SIZE]; 6686 pci_intr_type_t intr_type, max_type; 6687 char intrbuf[PCI_INTRSTR_LEN]; 6688 char wqname[MAXCOMLEN]; 6689 const char *intrstr = NULL; 6690 int defertx_error = 0, error; 6691 6692 /* We allocate a single interrupt resource */ 6693 max_type = PCI_INTR_TYPE_MSI; 6694 counts[PCI_INTR_TYPE_MSIX] = 0; 6695 counts[PCI_INTR_TYPE_MSI] = 6696 (adapter->feat_en & IXGBE_FEATURE_MSI) ? 1 : 0; 6697 /* Check not feat_en but feat_cap to fallback to INTx */ 6698 counts[PCI_INTR_TYPE_INTX] = 6699 (adapter->feat_cap & IXGBE_FEATURE_LEGACY_IRQ) ? 1 : 0; 6700 6701 alloc_retry: 6702 if (pci_intr_alloc(pa, &adapter->osdep.intrs, counts, max_type) != 0) { 6703 aprint_error_dev(dev, "couldn't alloc interrupt\n"); 6704 return ENXIO; 6705 } 6706 adapter->osdep.nintrs = 1; 6707 intrstr = pci_intr_string(adapter->osdep.pc, adapter->osdep.intrs[0], 6708 intrbuf, sizeof(intrbuf)); 6709 adapter->osdep.ihs[0] = pci_intr_establish_xname(adapter->osdep.pc, 6710 adapter->osdep.intrs[0], IPL_NET, ixgbe_legacy_irq, que, 6711 device_xname(dev)); 6712 intr_type = pci_intr_type(adapter->osdep.pc, adapter->osdep.intrs[0]); 6713 if (adapter->osdep.ihs[0] == NULL) { 6714 aprint_error_dev(dev,"unable to establish %s\n", 6715 (intr_type == PCI_INTR_TYPE_MSI) ? "MSI" : "INTx"); 6716 pci_intr_release(adapter->osdep.pc, adapter->osdep.intrs, 1); 6717 adapter->osdep.intrs = NULL; 6718 switch (intr_type) { 6719 case PCI_INTR_TYPE_MSI: 6720 /* The next try is for INTx: Disable MSI */ 6721 max_type = PCI_INTR_TYPE_INTX; 6722 counts[PCI_INTR_TYPE_INTX] = 1; 6723 adapter->feat_en &= ~IXGBE_FEATURE_MSI; 6724 if (adapter->feat_cap & IXGBE_FEATURE_LEGACY_IRQ) { 6725 adapter->feat_en |= IXGBE_FEATURE_LEGACY_IRQ; 6726 goto alloc_retry; 6727 } else 6728 break; 6729 case PCI_INTR_TYPE_INTX: 6730 default: 6731 /* See below */ 6732 break; 6733 } 6734 } 6735 if (intr_type == PCI_INTR_TYPE_INTX) { 6736 adapter->feat_en &= ~IXGBE_FEATURE_MSI; 6737 adapter->feat_en |= IXGBE_FEATURE_LEGACY_IRQ; 6738 } 6739 if (adapter->osdep.ihs[0] == NULL) { 6740 aprint_error_dev(dev, 6741 "couldn't establish interrupt%s%s\n", 6742 intrstr ? " at " : "", intrstr ? intrstr : ""); 6743 pci_intr_release(adapter->osdep.pc, adapter->osdep.intrs, 1); 6744 adapter->osdep.intrs = NULL; 6745 return ENXIO; 6746 } 6747 aprint_normal_dev(dev, "interrupting at %s\n", intrstr); 6748 /* 6749 * Try allocating a fast interrupt and the associated deferred 6750 * processing contexts. 6751 */ 6752 if (!(adapter->feat_en & IXGBE_FEATURE_LEGACY_TX)) { 6753 txr->txr_si = 6754 softint_establish(SOFTINT_NET | IXGBE_SOFTINT_FLAGS, 6755 ixgbe_deferred_mq_start, txr); 6756 6757 snprintf(wqname, sizeof(wqname), "%sdeferTx", 6758 device_xname(dev)); 6759 defertx_error = workqueue_create(&adapter->txr_wq, wqname, 6760 ixgbe_deferred_mq_start_work, adapter, IXGBE_WORKQUEUE_PRI, 6761 IPL_NET, IXGBE_WORKQUEUE_FLAGS); 6762 adapter->txr_wq_enqueued = percpu_alloc(sizeof(u_int)); 6763 } 6764 que->que_si = softint_establish(SOFTINT_NET | IXGBE_SOFTINT_FLAGS, 6765 ixgbe_handle_que, que); 6766 snprintf(wqname, sizeof(wqname), "%sTxRx", device_xname(dev)); 6767 error = workqueue_create(&adapter->que_wq, wqname, 6768 ixgbe_handle_que_work, adapter, IXGBE_WORKQUEUE_PRI, IPL_NET, 6769 IXGBE_WORKQUEUE_FLAGS); 6770 6771 if ((!(adapter->feat_en & IXGBE_FEATURE_LEGACY_TX) 6772 && ((txr->txr_si == NULL) || defertx_error != 0)) 6773 || (que->que_si == NULL) || error != 0) { 6774 aprint_error_dev(dev, 6775 "could not establish software interrupts\n"); 6776 6777 return ENXIO; 6778 } 6779 /* For simplicity in the handlers */ 6780 adapter->active_queues = IXGBE_EIMS_ENABLE_MASK; 6781 6782 return (0); 6783 } /* ixgbe_allocate_legacy */ 6784 6785 /************************************************************************ 6786 * ixgbe_allocate_msix - Setup MSI-X Interrupt resources and handlers 6787 ************************************************************************/ 6788 static int 6789 ixgbe_allocate_msix(struct adapter *adapter, const struct pci_attach_args *pa) 6790 { 6791 device_t dev = adapter->dev; 6792 struct ix_queue *que = adapter->queues; 6793 struct tx_ring *txr = adapter->tx_rings; 6794 pci_chipset_tag_t pc; 6795 char intrbuf[PCI_INTRSTR_LEN]; 6796 char intr_xname[32]; 6797 char wqname[MAXCOMLEN]; 6798 const char *intrstr = NULL; 6799 int error, vector = 0; 6800 int cpu_id = 0; 6801 kcpuset_t *affinity; 6802 #ifdef RSS 6803 unsigned int rss_buckets = 0; 6804 kcpuset_t cpu_mask; 6805 #endif 6806 6807 pc = adapter->osdep.pc; 6808 #ifdef RSS 6809 /* 6810 * If we're doing RSS, the number of queues needs to 6811 * match the number of RSS buckets that are configured. 6812 * 6813 * + If there's more queues than RSS buckets, we'll end 6814 * up with queues that get no traffic. 6815 * 6816 * + If there's more RSS buckets than queues, we'll end 6817 * up having multiple RSS buckets map to the same queue, 6818 * so there'll be some contention. 6819 */ 6820 rss_buckets = rss_getnumbuckets(); 6821 if ((adapter->feat_en & IXGBE_FEATURE_RSS) && 6822 (adapter->num_queues != rss_buckets)) { 6823 device_printf(dev, 6824 "%s: number of queues (%d) != number of RSS buckets (%d)" 6825 "; performance will be impacted.\n", 6826 __func__, adapter->num_queues, rss_buckets); 6827 } 6828 #endif 6829 6830 adapter->osdep.nintrs = adapter->num_queues + 1; 6831 if (pci_msix_alloc_exact(pa, &adapter->osdep.intrs, 6832 adapter->osdep.nintrs) != 0) { 6833 aprint_error_dev(dev, 6834 "failed to allocate MSI-X interrupt\n"); 6835 return (ENXIO); 6836 } 6837 6838 kcpuset_create(&affinity, false); 6839 for (int i = 0; i < adapter->num_queues; i++, vector++, que++, txr++) { 6840 snprintf(intr_xname, sizeof(intr_xname), "%s TXRX%d", 6841 device_xname(dev), i); 6842 intrstr = pci_intr_string(pc, adapter->osdep.intrs[i], intrbuf, 6843 sizeof(intrbuf)); 6844 #ifdef IXGBE_MPSAFE 6845 pci_intr_setattr(pc, &adapter->osdep.intrs[i], PCI_INTR_MPSAFE, 6846 true); 6847 #endif 6848 /* Set the handler function */ 6849 que->res = adapter->osdep.ihs[i] = pci_intr_establish_xname(pc, 6850 adapter->osdep.intrs[i], IPL_NET, ixgbe_msix_que, que, 6851 intr_xname); 6852 if (que->res == NULL) { 6853 aprint_error_dev(dev, 6854 "Failed to register QUE handler\n"); 6855 error = ENXIO; 6856 goto err_out; 6857 } 6858 que->msix = vector; 6859 adapter->active_queues |= 1ULL << que->msix; 6860 6861 if (adapter->feat_en & IXGBE_FEATURE_RSS) { 6862 #ifdef RSS 6863 /* 6864 * The queue ID is used as the RSS layer bucket ID. 6865 * We look up the queue ID -> RSS CPU ID and select 6866 * that. 6867 */ 6868 cpu_id = rss_getcpu(i % rss_getnumbuckets()); 6869 CPU_SETOF(cpu_id, &cpu_mask); 6870 #endif 6871 } else { 6872 /* 6873 * Bind the MSI-X vector, and thus the 6874 * rings to the corresponding CPU. 6875 * 6876 * This just happens to match the default RSS 6877 * round-robin bucket -> queue -> CPU allocation. 6878 */ 6879 if (adapter->num_queues > 1) 6880 cpu_id = i; 6881 } 6882 /* Round-robin affinity */ 6883 kcpuset_zero(affinity); 6884 kcpuset_set(affinity, cpu_id % ncpu); 6885 error = interrupt_distribute(adapter->osdep.ihs[i], affinity, 6886 NULL); 6887 aprint_normal_dev(dev, "for TX/RX, interrupting at %s", 6888 intrstr); 6889 if (error == 0) { 6890 #if 1 /* def IXGBE_DEBUG */ 6891 #ifdef RSS 6892 aprintf_normal(", bound RSS bucket %d to CPU %d", i, 6893 cpu_id % ncpu); 6894 #else 6895 aprint_normal(", bound queue %d to cpu %d", i, 6896 cpu_id % ncpu); 6897 #endif 6898 #endif /* IXGBE_DEBUG */ 6899 } 6900 aprint_normal("\n"); 6901 6902 if (!(adapter->feat_en & IXGBE_FEATURE_LEGACY_TX)) { 6903 txr->txr_si = softint_establish( 6904 SOFTINT_NET | IXGBE_SOFTINT_FLAGS, 6905 ixgbe_deferred_mq_start, txr); 6906 if (txr->txr_si == NULL) { 6907 aprint_error_dev(dev, 6908 "couldn't establish software interrupt\n"); 6909 error = ENXIO; 6910 goto err_out; 6911 } 6912 } 6913 que->que_si 6914 = softint_establish(SOFTINT_NET | IXGBE_SOFTINT_FLAGS, 6915 ixgbe_handle_que, que); 6916 if (que->que_si == NULL) { 6917 aprint_error_dev(dev, 6918 "couldn't establish software interrupt\n"); 6919 error = ENXIO; 6920 goto err_out; 6921 } 6922 } 6923 snprintf(wqname, sizeof(wqname), "%sdeferTx", device_xname(dev)); 6924 error = workqueue_create(&adapter->txr_wq, wqname, 6925 ixgbe_deferred_mq_start_work, adapter, IXGBE_WORKQUEUE_PRI, IPL_NET, 6926 IXGBE_WORKQUEUE_FLAGS); 6927 if (error) { 6928 aprint_error_dev(dev, 6929 "couldn't create workqueue for deferred Tx\n"); 6930 goto err_out; 6931 } 6932 adapter->txr_wq_enqueued = percpu_alloc(sizeof(u_int)); 6933 6934 snprintf(wqname, sizeof(wqname), "%sTxRx", device_xname(dev)); 6935 error = workqueue_create(&adapter->que_wq, wqname, 6936 ixgbe_handle_que_work, adapter, IXGBE_WORKQUEUE_PRI, IPL_NET, 6937 IXGBE_WORKQUEUE_FLAGS); 6938 if (error) { 6939 aprint_error_dev(dev, "couldn't create workqueue for Tx/Rx\n"); 6940 goto err_out; 6941 } 6942 6943 /* and Link */ 6944 cpu_id++; 6945 snprintf(intr_xname, sizeof(intr_xname), "%s link", device_xname(dev)); 6946 adapter->vector = vector; 6947 intrstr = pci_intr_string(pc, adapter->osdep.intrs[vector], intrbuf, 6948 sizeof(intrbuf)); 6949 #ifdef IXGBE_MPSAFE 6950 pci_intr_setattr(pc, &adapter->osdep.intrs[vector], PCI_INTR_MPSAFE, 6951 true); 6952 #endif 6953 /* Set the link handler function */ 6954 adapter->osdep.ihs[vector] = pci_intr_establish_xname(pc, 6955 adapter->osdep.intrs[vector], IPL_NET, ixgbe_msix_admin, adapter, 6956 intr_xname); 6957 if (adapter->osdep.ihs[vector] == NULL) { 6958 aprint_error_dev(dev, "Failed to register LINK handler\n"); 6959 error = ENXIO; 6960 goto err_out; 6961 } 6962 /* Round-robin affinity */ 6963 kcpuset_zero(affinity); 6964 kcpuset_set(affinity, cpu_id % ncpu); 6965 error = interrupt_distribute(adapter->osdep.ihs[vector], affinity, 6966 NULL); 6967 6968 aprint_normal_dev(dev, 6969 "for link, interrupting at %s", intrstr); 6970 if (error == 0) 6971 aprint_normal(", affinity to cpu %d\n", cpu_id % ncpu); 6972 else 6973 aprint_normal("\n"); 6974 6975 kcpuset_destroy(affinity); 6976 aprint_normal_dev(dev, 6977 "Using MSI-X interrupts with %d vectors\n", vector + 1); 6978 6979 return (0); 6980 6981 err_out: 6982 kcpuset_destroy(affinity); 6983 ixgbe_free_deferred_handlers(adapter); 6984 ixgbe_free_pciintr_resources(adapter); 6985 return (error); 6986 } /* ixgbe_allocate_msix */ 6987 6988 /************************************************************************ 6989 * ixgbe_configure_interrupts 6990 * 6991 * Setup MSI-X, MSI, or legacy interrupts (in that order). 6992 * This will also depend on user settings. 6993 ************************************************************************/ 6994 static int 6995 ixgbe_configure_interrupts(struct adapter *adapter) 6996 { 6997 device_t dev = adapter->dev; 6998 struct ixgbe_mac_info *mac = &adapter->hw.mac; 6999 int want, queues, msgs; 7000 7001 /* Default to 1 queue if MSI-X setup fails */ 7002 adapter->num_queues = 1; 7003 7004 /* Override by tuneable */ 7005 if (!(adapter->feat_cap & IXGBE_FEATURE_MSIX)) 7006 goto msi; 7007 7008 /* 7009 * NetBSD only: Use single vector MSI when number of CPU is 1 to save 7010 * interrupt slot. 7011 */ 7012 if (ncpu == 1) 7013 goto msi; 7014 7015 /* First try MSI-X */ 7016 msgs = pci_msix_count(adapter->osdep.pc, adapter->osdep.tag); 7017 msgs = MIN(msgs, IXG_MAX_NINTR); 7018 if (msgs < 2) 7019 goto msi; 7020 7021 adapter->msix_mem = (void *)1; /* XXX */ 7022 7023 /* Figure out a reasonable auto config value */ 7024 queues = (ncpu > (msgs - 1)) ? (msgs - 1) : ncpu; 7025 7026 #ifdef RSS 7027 /* If we're doing RSS, clamp at the number of RSS buckets */ 7028 if (adapter->feat_en & IXGBE_FEATURE_RSS) 7029 queues = uimin(queues, rss_getnumbuckets()); 7030 #endif 7031 if (ixgbe_num_queues > queues) { 7032 aprint_error_dev(adapter->dev, "ixgbe_num_queues (%d) is too large, using reduced amount (%d).\n", ixgbe_num_queues, queues); 7033 ixgbe_num_queues = queues; 7034 } 7035 7036 if (ixgbe_num_queues != 0) 7037 queues = ixgbe_num_queues; 7038 else 7039 queues = uimin(queues, 7040 uimin(mac->max_tx_queues, mac->max_rx_queues)); 7041 7042 /* reflect correct sysctl value */ 7043 ixgbe_num_queues = queues; 7044 7045 /* 7046 * Want one vector (RX/TX pair) per queue 7047 * plus an additional for Link. 7048 */ 7049 want = queues + 1; 7050 if (msgs >= want) 7051 msgs = want; 7052 else { 7053 aprint_error_dev(dev, "MSI-X Configuration Problem, " 7054 "%d vectors but %d queues wanted!\n", 7055 msgs, want); 7056 goto msi; 7057 } 7058 adapter->num_queues = queues; 7059 adapter->feat_en |= IXGBE_FEATURE_MSIX; 7060 return (0); 7061 7062 /* 7063 * MSI-X allocation failed or provided us with 7064 * less vectors than needed. Free MSI-X resources 7065 * and we'll try enabling MSI. 7066 */ 7067 msi: 7068 /* Without MSI-X, some features are no longer supported */ 7069 adapter->feat_cap &= ~IXGBE_FEATURE_RSS; 7070 adapter->feat_en &= ~IXGBE_FEATURE_RSS; 7071 adapter->feat_cap &= ~IXGBE_FEATURE_SRIOV; 7072 adapter->feat_en &= ~IXGBE_FEATURE_SRIOV; 7073 7074 msgs = pci_msi_count(adapter->osdep.pc, adapter->osdep.tag); 7075 adapter->msix_mem = NULL; /* XXX */ 7076 if (msgs > 1) 7077 msgs = 1; 7078 if (msgs != 0) { 7079 msgs = 1; 7080 adapter->feat_en |= IXGBE_FEATURE_MSI; 7081 return (0); 7082 } 7083 7084 if (!(adapter->feat_cap & IXGBE_FEATURE_LEGACY_IRQ)) { 7085 aprint_error_dev(dev, 7086 "Device does not support legacy interrupts.\n"); 7087 return 1; 7088 } 7089 7090 adapter->feat_en |= IXGBE_FEATURE_LEGACY_IRQ; 7091 7092 return (0); 7093 } /* ixgbe_configure_interrupts */ 7094 7095 7096 /************************************************************************ 7097 * ixgbe_handle_link - Tasklet for MSI-X Link interrupts 7098 * 7099 * Done outside of interrupt context since the driver might sleep 7100 ************************************************************************/ 7101 static void 7102 ixgbe_handle_link(void *context) 7103 { 7104 struct adapter *adapter = context; 7105 struct ixgbe_hw *hw = &adapter->hw; 7106 7107 KASSERT(mutex_owned(&adapter->core_mtx)); 7108 7109 IXGBE_EVC_ADD(&adapter->link_workev, 1); 7110 ixgbe_check_link(hw, &adapter->link_speed, &adapter->link_up, 0); 7111 ixgbe_update_link_status(adapter); 7112 7113 /* Re-enable link interrupts */ 7114 IXGBE_WRITE_REG(hw, IXGBE_EIMS, IXGBE_EIMS_LSC); 7115 } /* ixgbe_handle_link */ 7116 7117 #if 0 7118 /************************************************************************ 7119 * ixgbe_rearm_queues 7120 ************************************************************************/ 7121 static __inline void 7122 ixgbe_rearm_queues(struct adapter *adapter, u64 queues) 7123 { 7124 u32 mask; 7125 7126 switch (adapter->hw.mac.type) { 7127 case ixgbe_mac_82598EB: 7128 mask = (IXGBE_EIMS_RTX_QUEUE & queues); 7129 IXGBE_WRITE_REG(&adapter->hw, IXGBE_EICS, mask); 7130 break; 7131 case ixgbe_mac_82599EB: 7132 case ixgbe_mac_X540: 7133 case ixgbe_mac_X550: 7134 case ixgbe_mac_X550EM_x: 7135 case ixgbe_mac_X550EM_a: 7136 mask = (queues & 0xFFFFFFFF); 7137 IXGBE_WRITE_REG(&adapter->hw, IXGBE_EICS_EX(0), mask); 7138 mask = (queues >> 32); 7139 IXGBE_WRITE_REG(&adapter->hw, IXGBE_EICS_EX(1), mask); 7140 break; 7141 default: 7142 break; 7143 } 7144 } /* ixgbe_rearm_queues */ 7145 #endif 7146
Indexes created Tue Sep 30 11:09:46 GMT 2025