ixgbe.c revision 1.316
1 /* $NetBSD: ixgbe.c,v 1.316 2022/06/01 02:07:24 msaitoh Exp $ */ 2 3 /****************************************************************************** 4 5 Copyright (c) 2001-2017, Intel Corporation 6 All rights reserved. 7 8 Redistribution and use in source and binary forms, with or without 9 modification, are permitted provided that the following conditions are met: 10 11 1. Redistributions of source code must retain the above copyright notice, 12 this list of conditions and the following disclaimer. 13 14 2. Redistributions in binary form must reproduce the above copyright 15 notice, this list of conditions and the following disclaimer in the 16 documentation and/or other materials provided with the distribution. 17 18 3. Neither the name of the Intel Corporation nor the names of its 19 contributors may be used to endorse or promote products derived from 20 this software without specific prior written permission. 21 22 THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 23 AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 24 IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 25 ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE 26 LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 27 CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 28 SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 29 INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 30 CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 31 ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 32 POSSIBILITY OF SUCH DAMAGE. 33 34 ******************************************************************************/ 35 /*$FreeBSD: head/sys/dev/ixgbe/if_ix.c 331224 2018-03-19 20:55:05Z erj $*/ 36 37 /* 38 * Copyright (c) 2011 The NetBSD Foundation, Inc. 39 * All rights reserved. 40 * 41 * This code is derived from software contributed to The NetBSD Foundation 42 * by Coyote Point Systems, Inc. 43 * 44 * Redistribution and use in source and binary forms, with or without 45 * modification, are permitted provided that the following conditions 46 * are met: 47 * 1. Redistributions of source code must retain the above copyright 48 * notice, this list of conditions and the following disclaimer. 49 * 2. Redistributions in binary form must reproduce the above copyright 50 * notice, this list of conditions and the following disclaimer in the 51 * documentation and/or other materials provided with the distribution. 52 * 53 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS 54 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED 55 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 56 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS 57 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 58 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 59 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 60 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 61 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 62 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 63 * POSSIBILITY OF SUCH DAMAGE. 64 */ 65 66 #include <sys/cdefs.h> 67 __KERNEL_RCSID(0, "$NetBSD: ixgbe.c,v 1.316 2022/06/01 02:07:24 msaitoh Exp $"); 68 69 #ifdef _KERNEL_OPT 70 #include "opt_inet.h" 71 #include "opt_inet6.h" 72 #include "opt_net_mpsafe.h" 73 #endif 74 75 #include "ixgbe.h" 76 #include "ixgbe_phy.h" 77 #include "ixgbe_sriov.h" 78 79 #include <sys/cprng.h> 80 #include <dev/mii/mii.h> 81 #include <dev/mii/miivar.h> 82 83 /************************************************************************ 84 * Driver version 85 ************************************************************************/ 86 static const char ixgbe_driver_version[] = "4.0.1-k"; 87 /* XXX NetBSD: + 3.3.24 */ 88 89 /************************************************************************ 90 * PCI Device ID Table 91 * 92 * Used by probe to select devices to load on 93 * Last field stores an index into ixgbe_strings 94 * Last entry must be all 0s 95 * 96 * { Vendor ID, Device ID, SubVendor ID, SubDevice ID, String Index } 97 ************************************************************************/ 98 static const ixgbe_vendor_info_t ixgbe_vendor_info_array[] = 99 { 100 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598AF_DUAL_PORT, 0, 0, 0}, 101 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598AF_SINGLE_PORT, 0, 0, 0}, 102 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598EB_CX4, 0, 0, 0}, 103 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598AT, 0, 0, 0}, 104 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598AT2, 0, 0, 0}, 105 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598, 0, 0, 0}, 106 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598_BX, 0, 0, 0}, 107 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598_DA_DUAL_PORT, 0, 0, 0}, 108 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598_CX4_DUAL_PORT, 0, 0, 0}, 109 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598EB_XF_LR, 0, 0, 0}, 110 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598_SR_DUAL_PORT_EM, 0, 0, 0}, 111 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598EB_SFP_LOM, 0, 0, 0}, 112 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_KR, 0, 0, 0}, 113 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_KX4, 0, 0, 0}, 114 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_KX4_MEZZ, 0, 0, 0}, 115 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_SFP, 0, 0, 0}, 116 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_SFP_EM, 0, 0, 0}, 117 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_XAUI_LOM, 0, 0, 0}, 118 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_CX4, 0, 0, 0}, 119 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_T3_LOM, 0, 0, 0}, 120 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_COMBO_BACKPLANE, 0, 0, 0}, 121 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_BACKPLANE_FCOE, 0, 0, 0}, 122 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_SFP_SF2, 0, 0, 0}, 123 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_SFP_FCOE, 0, 0, 0}, 124 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599EN_SFP, 0, 0, 0}, 125 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_SFP_SF_QP, 0, 0, 0}, 126 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_QSFP_SF_QP, 0, 0, 0}, 127 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_X540T, 0, 0, 0}, 128 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_X540T1, 0, 0, 0}, 129 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_X550T, 0, 0, 0}, 130 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_X550T1, 0, 0, 0}, 131 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_X550EM_X_KR, 0, 0, 0}, 132 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_X550EM_X_KX4, 0, 0, 0}, 133 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_X550EM_X_10G_T, 0, 0, 0}, 134 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_X550EM_X_1G_T, 0, 0, 0}, 135 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_X550EM_X_SFP, 0, 0, 0}, 136 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_X550EM_X_XFI, 0, 0, 0}, 137 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_X550EM_A_KR, 0, 0, 0}, 138 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_X550EM_A_KR_L, 0, 0, 0}, 139 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_X550EM_A_QSFP, 0, 0, 0}, 140 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_X550EM_A_QSFP_N, 0, 0, 0}, 141 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_X550EM_A_SFP, 0, 0, 0}, 142 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_X550EM_A_SFP_N, 0, 0, 0}, 143 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_X550EM_A_SGMII, 0, 0, 0}, 144 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_X550EM_A_SGMII_L, 0, 0, 0}, 145 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_X550EM_A_10G_T, 0, 0, 0}, 146 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_X550EM_A_1G_T, 0, 0, 0}, 147 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_X550EM_A_1G_T_L, 0, 0, 0}, 148 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_X540_BYPASS, 0, 0, 0}, 149 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_BYPASS, 0, 0, 0}, 150 /* required last entry */ 151 {0, 0, 0, 0, 0} 152 }; 153 154 /************************************************************************ 155 * Table of branding strings 156 ************************************************************************/ 157 static const char *ixgbe_strings[] = { 158 "Intel(R) PRO/10GbE PCI-Express Network Driver" 159 }; 160 161 /************************************************************************ 162 * Function prototypes 163 ************************************************************************/ 164 static int ixgbe_probe(device_t, cfdata_t, void *); 165 static void ixgbe_quirks(struct adapter *); 166 static void ixgbe_attach(device_t, device_t, void *); 167 static int ixgbe_detach(device_t, int); 168 #if 0 169 static int ixgbe_shutdown(device_t); 170 #endif 171 static bool ixgbe_suspend(device_t, const pmf_qual_t *); 172 static bool ixgbe_resume(device_t, const pmf_qual_t *); 173 static int ixgbe_ifflags_cb(struct ethercom *); 174 static int ixgbe_ioctl(struct ifnet *, u_long, void *); 175 static int ixgbe_init(struct ifnet *); 176 static void ixgbe_init_locked(struct adapter *); 177 static void ixgbe_ifstop(struct ifnet *, int); 178 static void ixgbe_stop_locked(void *); 179 static void ixgbe_init_device_features(struct adapter *); 180 static int ixgbe_check_fan_failure(struct adapter *, u32, bool); 181 static void ixgbe_add_media_types(struct adapter *); 182 static void ixgbe_media_status(struct ifnet *, struct ifmediareq *); 183 static int ixgbe_media_change(struct ifnet *); 184 static int ixgbe_allocate_pci_resources(struct adapter *, 185 const struct pci_attach_args *); 186 static void ixgbe_free_deferred_handlers(struct adapter *); 187 static void ixgbe_get_slot_info(struct adapter *); 188 static int ixgbe_allocate_msix(struct adapter *, 189 const struct pci_attach_args *); 190 static int ixgbe_allocate_legacy(struct adapter *, 191 const struct pci_attach_args *); 192 static int ixgbe_configure_interrupts(struct adapter *); 193 static void ixgbe_free_pciintr_resources(struct adapter *); 194 static void ixgbe_free_pci_resources(struct adapter *); 195 static void ixgbe_local_timer(void *); 196 static void ixgbe_handle_timer(struct work *, void *); 197 static void ixgbe_recovery_mode_timer(void *); 198 static void ixgbe_handle_recovery_mode_timer(struct work *, void *); 199 static int ixgbe_setup_interface(device_t, struct adapter *); 200 static void ixgbe_config_gpie(struct adapter *); 201 static void ixgbe_config_dmac(struct adapter *); 202 static void ixgbe_config_delay_values(struct adapter *); 203 static void ixgbe_schedule_admin_tasklet(struct adapter *); 204 static void ixgbe_config_link(struct adapter *); 205 static void ixgbe_check_wol_support(struct adapter *); 206 static int ixgbe_setup_low_power_mode(struct adapter *); 207 #if 0 208 static void ixgbe_rearm_queues(struct adapter *, u64); 209 #endif 210 211 static void ixgbe_initialize_transmit_units(struct adapter *); 212 static void ixgbe_initialize_receive_units(struct adapter *); 213 static void ixgbe_enable_rx_drop(struct adapter *); 214 static void ixgbe_disable_rx_drop(struct adapter *); 215 static void ixgbe_initialize_rss_mapping(struct adapter *); 216 217 static void ixgbe_enable_intr(struct adapter *); 218 static void ixgbe_disable_intr(struct adapter *); 219 static void ixgbe_update_stats_counters(struct adapter *); 220 static void ixgbe_set_rxfilter(struct adapter *); 221 static void ixgbe_update_link_status(struct adapter *); 222 static void ixgbe_set_ivar(struct adapter *, u8, u8, s8); 223 static void ixgbe_configure_ivars(struct adapter *); 224 static u8 * ixgbe_mc_array_itr(struct ixgbe_hw *, u8 **, u32 *); 225 static void ixgbe_eitr_write(struct adapter *, uint32_t, uint32_t); 226 227 static void ixgbe_setup_vlan_hw_tagging(struct adapter *); 228 static void ixgbe_setup_vlan_hw_support(struct adapter *); 229 static int ixgbe_vlan_cb(struct ethercom *, uint16_t, bool); 230 static int ixgbe_register_vlan(struct adapter *, u16); 231 static int ixgbe_unregister_vlan(struct adapter *, u16); 232 233 static void ixgbe_add_device_sysctls(struct adapter *); 234 static void ixgbe_add_hw_stats(struct adapter *); 235 static void ixgbe_clear_evcnt(struct adapter *); 236 static int ixgbe_set_flowcntl(struct adapter *, int); 237 static int ixgbe_set_advertise(struct adapter *, int); 238 static int ixgbe_get_default_advertise(struct adapter *); 239 240 /* Sysctl handlers */ 241 static int ixgbe_sysctl_flowcntl(SYSCTLFN_PROTO); 242 static int ixgbe_sysctl_advertise(SYSCTLFN_PROTO); 243 static int ixgbe_sysctl_interrupt_rate_handler(SYSCTLFN_PROTO); 244 static int ixgbe_sysctl_dmac(SYSCTLFN_PROTO); 245 static int ixgbe_sysctl_phy_temp(SYSCTLFN_PROTO); 246 static int ixgbe_sysctl_phy_overtemp_occurred(SYSCTLFN_PROTO); 247 #ifdef IXGBE_DEBUG 248 static int ixgbe_sysctl_power_state(SYSCTLFN_PROTO); 249 static int ixgbe_sysctl_print_rss_config(SYSCTLFN_PROTO); 250 #endif 251 static int ixgbe_sysctl_next_to_check_handler(SYSCTLFN_PROTO); 252 static int ixgbe_sysctl_next_to_refresh_handler(SYSCTLFN_PROTO); 253 static int ixgbe_sysctl_rdh_handler(SYSCTLFN_PROTO); 254 static int ixgbe_sysctl_rdt_handler(SYSCTLFN_PROTO); 255 static int ixgbe_sysctl_tdt_handler(SYSCTLFN_PROTO); 256 static int ixgbe_sysctl_tdh_handler(SYSCTLFN_PROTO); 257 static int ixgbe_sysctl_eee_state(SYSCTLFN_PROTO); 258 static int ixgbe_sysctl_debug(SYSCTLFN_PROTO); 259 static int ixgbe_sysctl_rx_copy_len(SYSCTLFN_PROTO); 260 static int ixgbe_sysctl_tx_process_limit(SYSCTLFN_PROTO); 261 static int ixgbe_sysctl_rx_process_limit(SYSCTLFN_PROTO); 262 static int ixgbe_sysctl_wol_enable(SYSCTLFN_PROTO); 263 static int ixgbe_sysctl_wufc(SYSCTLFN_PROTO); 264 265 /* Interrupt functions */ 266 static int ixgbe_msix_que(void *); 267 static int ixgbe_msix_admin(void *); 268 static void ixgbe_intr_admin_common(struct adapter *, u32, u32 *); 269 static int ixgbe_legacy_irq(void *); 270 271 /* Event handlers running on workqueue */ 272 static void ixgbe_handle_que(void *); 273 static void ixgbe_handle_link(void *); 274 static void ixgbe_handle_msf(void *); 275 static void ixgbe_handle_mod(void *, bool); 276 static void ixgbe_handle_phy(void *); 277 278 /* Deferred workqueue handlers */ 279 static void ixgbe_handle_admin(struct work *, void *); 280 static void ixgbe_handle_que_work(struct work *, void *); 281 282 static const ixgbe_vendor_info_t *ixgbe_lookup(const struct pci_attach_args *); 283 284 /************************************************************************ 285 * NetBSD Device Interface Entry Points 286 ************************************************************************/ 287 CFATTACH_DECL3_NEW(ixg, sizeof(struct adapter), 288 ixgbe_probe, ixgbe_attach, ixgbe_detach, NULL, NULL, NULL, 289 DVF_DETACH_SHUTDOWN); 290 291 #if 0 292 devclass_t ix_devclass; 293 DRIVER_MODULE(ix, pci, ix_driver, ix_devclass, 0, 0); 294 295 MODULE_DEPEND(ix, pci, 1, 1, 1); 296 MODULE_DEPEND(ix, ether, 1, 1, 1); 297 #ifdef DEV_NETMAP 298 MODULE_DEPEND(ix, netmap, 1, 1, 1); 299 #endif 300 #endif 301 302 /* 303 * TUNEABLE PARAMETERS: 304 */ 305 306 /* 307 * AIM: Adaptive Interrupt Moderation 308 * which means that the interrupt rate 309 * is varied over time based on the 310 * traffic for that interrupt vector 311 */ 312 static bool ixgbe_enable_aim = true; 313 #define SYSCTL_INT(_a1, _a2, _a3, _a4, _a5, _a6, _a7) 314 SYSCTL_INT(_hw_ix, OID_AUTO, enable_aim, CTLFLAG_RDTUN, &ixgbe_enable_aim, 0, 315 "Enable adaptive interrupt moderation"); 316 317 static int ixgbe_max_interrupt_rate = (4000000 / IXGBE_LOW_LATENCY); 318 SYSCTL_INT(_hw_ix, OID_AUTO, max_interrupt_rate, CTLFLAG_RDTUN, 319 &ixgbe_max_interrupt_rate, 0, "Maximum interrupts per second"); 320 321 /* How many packets rxeof tries to clean at a time */ 322 static int ixgbe_rx_process_limit = 256; 323 SYSCTL_INT(_hw_ix, OID_AUTO, rx_process_limit, CTLFLAG_RDTUN, 324 &ixgbe_rx_process_limit, 0, "Maximum number of received packets to process at a time, -1 means unlimited"); 325 326 /* How many packets txeof tries to clean at a time */ 327 static int ixgbe_tx_process_limit = 256; 328 SYSCTL_INT(_hw_ix, OID_AUTO, tx_process_limit, CTLFLAG_RDTUN, 329 &ixgbe_tx_process_limit, 0, 330 "Maximum number of sent packets to process at a time, -1 means unlimited"); 331 332 /* Flow control setting, default to full */ 333 static int ixgbe_flow_control = ixgbe_fc_full; 334 SYSCTL_INT(_hw_ix, OID_AUTO, flow_control, CTLFLAG_RDTUN, 335 &ixgbe_flow_control, 0, "Default flow control used for all adapters"); 336 337 /* Which packet processing uses workqueue or softint */ 338 static bool ixgbe_txrx_workqueue = false; 339 340 /* 341 * Smart speed setting, default to on 342 * this only works as a compile option 343 * right now as its during attach, set 344 * this to 'ixgbe_smart_speed_off' to 345 * disable. 346 */ 347 static int ixgbe_smart_speed = ixgbe_smart_speed_on; 348 349 /* 350 * MSI-X should be the default for best performance, 351 * but this allows it to be forced off for testing. 352 */ 353 static int ixgbe_enable_msix = 1; 354 SYSCTL_INT(_hw_ix, OID_AUTO, enable_msix, CTLFLAG_RDTUN, &ixgbe_enable_msix, 0, 355 "Enable MSI-X interrupts"); 356 357 /* 358 * Number of Queues, can be set to 0, 359 * it then autoconfigures based on the 360 * number of cpus with a max of 8. This 361 * can be overridden manually here. 362 */ 363 static int ixgbe_num_queues = 0; 364 SYSCTL_INT(_hw_ix, OID_AUTO, num_queues, CTLFLAG_RDTUN, &ixgbe_num_queues, 0, 365 "Number of queues to configure, 0 indicates autoconfigure"); 366 367 /* 368 * Number of TX descriptors per ring, 369 * setting higher than RX as this seems 370 * the better performing choice. 371 */ 372 static int ixgbe_txd = PERFORM_TXD; 373 SYSCTL_INT(_hw_ix, OID_AUTO, txd, CTLFLAG_RDTUN, &ixgbe_txd, 0, 374 "Number of transmit descriptors per queue"); 375 376 /* Number of RX descriptors per ring */ 377 static int ixgbe_rxd = PERFORM_RXD; 378 SYSCTL_INT(_hw_ix, OID_AUTO, rxd, CTLFLAG_RDTUN, &ixgbe_rxd, 0, 379 "Number of receive descriptors per queue"); 380 381 /* 382 * Defining this on will allow the use 383 * of unsupported SFP+ modules, note that 384 * doing so you are on your own :) 385 */ 386 static int allow_unsupported_sfp = false; 387 #define TUNABLE_INT(__x, __y) 388 TUNABLE_INT("hw.ix.unsupported_sfp", &allow_unsupported_sfp); 389 390 /* 391 * Not sure if Flow Director is fully baked, 392 * so we'll default to turning it off. 393 */ 394 static int ixgbe_enable_fdir = 0; 395 SYSCTL_INT(_hw_ix, OID_AUTO, enable_fdir, CTLFLAG_RDTUN, &ixgbe_enable_fdir, 0, 396 "Enable Flow Director"); 397 398 /* Legacy Transmit (single queue) */ 399 static int ixgbe_enable_legacy_tx = 0; 400 SYSCTL_INT(_hw_ix, OID_AUTO, enable_legacy_tx, CTLFLAG_RDTUN, 401 &ixgbe_enable_legacy_tx, 0, "Enable Legacy TX flow"); 402 403 /* Receive-Side Scaling */ 404 static int ixgbe_enable_rss = 1; 405 SYSCTL_INT(_hw_ix, OID_AUTO, enable_rss, CTLFLAG_RDTUN, &ixgbe_enable_rss, 0, 406 "Enable Receive-Side Scaling (RSS)"); 407 408 #if 0 409 static int (*ixgbe_start_locked)(struct ifnet *, struct tx_ring *); 410 static int (*ixgbe_ring_empty)(struct ifnet *, pcq_t *); 411 #endif 412 413 #ifdef NET_MPSAFE 414 #define IXGBE_MPSAFE 1 415 #define IXGBE_CALLOUT_FLAGS CALLOUT_MPSAFE 416 #define IXGBE_SOFTINT_FLAGS SOFTINT_MPSAFE 417 #define IXGBE_WORKQUEUE_FLAGS WQ_PERCPU | WQ_MPSAFE 418 #define IXGBE_TASKLET_WQ_FLAGS WQ_MPSAFE 419 #else 420 #define IXGBE_CALLOUT_FLAGS 0 421 #define IXGBE_SOFTINT_FLAGS 0 422 #define IXGBE_WORKQUEUE_FLAGS WQ_PERCPU 423 #define IXGBE_TASKLET_WQ_FLAGS 0 424 #endif 425 #define IXGBE_WORKQUEUE_PRI PRI_SOFTNET 426 427 /* Interval between reports of errors */ 428 static const struct timeval ixgbe_errlog_intrvl = { 60, 0 }; /* 60s */ 429 430 /************************************************************************ 431 * ixgbe_initialize_rss_mapping 432 ************************************************************************/ 433 static void 434 ixgbe_initialize_rss_mapping(struct adapter *adapter) 435 { 436 struct ixgbe_hw *hw = &adapter->hw; 437 u32 reta = 0, mrqc, rss_key[10]; 438 int queue_id, table_size, index_mult; 439 int i, j; 440 u32 rss_hash_config; 441 442 /* force use default RSS key. */ 443 #ifdef __NetBSD__ 444 rss_getkey((uint8_t *) &rss_key); 445 #else 446 if (adapter->feat_en & IXGBE_FEATURE_RSS) { 447 /* Fetch the configured RSS key */ 448 rss_getkey((uint8_t *) &rss_key); 449 } else { 450 /* set up random bits */ 451 cprng_fast(&rss_key, sizeof(rss_key)); 452 } 453 #endif 454 455 /* Set multiplier for RETA setup and table size based on MAC */ 456 index_mult = 0x1; 457 table_size = 128; 458 switch (adapter->hw.mac.type) { 459 case ixgbe_mac_82598EB: 460 index_mult = 0x11; 461 break; 462 case ixgbe_mac_X550: 463 case ixgbe_mac_X550EM_x: 464 case ixgbe_mac_X550EM_a: 465 table_size = 512; 466 break; 467 default: 468 break; 469 } 470 471 /* Set up the redirection table */ 472 for (i = 0, j = 0; i < table_size; i++, j++) { 473 if (j == adapter->num_queues) 474 j = 0; 475 476 if (adapter->feat_en & IXGBE_FEATURE_RSS) { 477 /* 478 * Fetch the RSS bucket id for the given indirection 479 * entry. Cap it at the number of configured buckets 480 * (which is num_queues.) 481 */ 482 queue_id = rss_get_indirection_to_bucket(i); 483 queue_id = queue_id % adapter->num_queues; 484 } else 485 queue_id = (j * index_mult); 486 487 /* 488 * The low 8 bits are for hash value (n+0); 489 * The next 8 bits are for hash value (n+1), etc. 490 */ 491 reta = reta >> 8; 492 reta = reta | (((uint32_t) queue_id) << 24); 493 if ((i & 3) == 3) { 494 if (i < 128) 495 IXGBE_WRITE_REG(hw, IXGBE_RETA(i >> 2), reta); 496 else 497 IXGBE_WRITE_REG(hw, IXGBE_ERETA((i >> 2) - 32), 498 reta); 499 reta = 0; 500 } 501 } 502 503 /* Now fill our hash function seeds */ 504 for (i = 0; i < 10; i++) 505 IXGBE_WRITE_REG(hw, IXGBE_RSSRK(i), rss_key[i]); 506 507 /* Perform hash on these packet types */ 508 if (adapter->feat_en & IXGBE_FEATURE_RSS) 509 rss_hash_config = rss_gethashconfig(); 510 else { 511 /* 512 * Disable UDP - IP fragments aren't currently being handled 513 * and so we end up with a mix of 2-tuple and 4-tuple 514 * traffic. 515 */ 516 rss_hash_config = RSS_HASHTYPE_RSS_IPV4 517 | RSS_HASHTYPE_RSS_TCP_IPV4 518 | RSS_HASHTYPE_RSS_IPV6 519 | RSS_HASHTYPE_RSS_TCP_IPV6 520 | RSS_HASHTYPE_RSS_IPV6_EX 521 | RSS_HASHTYPE_RSS_TCP_IPV6_EX; 522 } 523 524 mrqc = IXGBE_MRQC_RSSEN; 525 if (rss_hash_config & RSS_HASHTYPE_RSS_IPV4) 526 mrqc |= IXGBE_MRQC_RSS_FIELD_IPV4; 527 if (rss_hash_config & RSS_HASHTYPE_RSS_TCP_IPV4) 528 mrqc |= IXGBE_MRQC_RSS_FIELD_IPV4_TCP; 529 if (rss_hash_config & RSS_HASHTYPE_RSS_IPV6) 530 mrqc |= IXGBE_MRQC_RSS_FIELD_IPV6; 531 if (rss_hash_config & RSS_HASHTYPE_RSS_TCP_IPV6) 532 mrqc |= IXGBE_MRQC_RSS_FIELD_IPV6_TCP; 533 if (rss_hash_config & RSS_HASHTYPE_RSS_IPV6_EX) 534 mrqc |= IXGBE_MRQC_RSS_FIELD_IPV6_EX; 535 if (rss_hash_config & RSS_HASHTYPE_RSS_TCP_IPV6_EX) 536 mrqc |= IXGBE_MRQC_RSS_FIELD_IPV6_EX_TCP; 537 if (rss_hash_config & RSS_HASHTYPE_RSS_UDP_IPV4) 538 mrqc |= IXGBE_MRQC_RSS_FIELD_IPV4_UDP; 539 if (rss_hash_config & RSS_HASHTYPE_RSS_UDP_IPV6) 540 mrqc |= IXGBE_MRQC_RSS_FIELD_IPV6_UDP; 541 if (rss_hash_config & RSS_HASHTYPE_RSS_UDP_IPV6_EX) 542 mrqc |= IXGBE_MRQC_RSS_FIELD_IPV6_EX_UDP; 543 mrqc |= ixgbe_get_mrqc(adapter->iov_mode); 544 IXGBE_WRITE_REG(hw, IXGBE_MRQC, mrqc); 545 } /* ixgbe_initialize_rss_mapping */ 546 547 /************************************************************************ 548 * ixgbe_initialize_receive_units - Setup receive registers and features. 549 ************************************************************************/ 550 #define BSIZEPKT_ROUNDUP ((1<<IXGBE_SRRCTL_BSIZEPKT_SHIFT)-1) 551 552 static void 553 ixgbe_initialize_receive_units(struct adapter *adapter) 554 { 555 struct rx_ring *rxr = adapter->rx_rings; 556 struct ixgbe_hw *hw = &adapter->hw; 557 struct ifnet *ifp = adapter->ifp; 558 int i, j; 559 u32 bufsz, fctrl, srrctl, rxcsum; 560 u32 hlreg; 561 562 /* 563 * Make sure receives are disabled while 564 * setting up the descriptor ring 565 */ 566 ixgbe_disable_rx(hw); 567 568 /* Enable broadcasts */ 569 fctrl = IXGBE_READ_REG(hw, IXGBE_FCTRL); 570 fctrl |= IXGBE_FCTRL_BAM; 571 if (adapter->hw.mac.type == ixgbe_mac_82598EB) { 572 fctrl |= IXGBE_FCTRL_DPF; 573 fctrl |= IXGBE_FCTRL_PMCF; 574 } 575 IXGBE_WRITE_REG(hw, IXGBE_FCTRL, fctrl); 576 577 /* Set for Jumbo Frames? */ 578 hlreg = IXGBE_READ_REG(hw, IXGBE_HLREG0); 579 if (ifp->if_mtu > ETHERMTU) 580 hlreg |= IXGBE_HLREG0_JUMBOEN; 581 else 582 hlreg &= ~IXGBE_HLREG0_JUMBOEN; 583 584 #ifdef DEV_NETMAP 585 /* CRC stripping is conditional in Netmap */ 586 if ((adapter->feat_en & IXGBE_FEATURE_NETMAP) && 587 (ifp->if_capenable & IFCAP_NETMAP) && 588 !ix_crcstrip) 589 hlreg &= ~IXGBE_HLREG0_RXCRCSTRP; 590 else 591 #endif /* DEV_NETMAP */ 592 hlreg |= IXGBE_HLREG0_RXCRCSTRP; 593 594 IXGBE_WRITE_REG(hw, IXGBE_HLREG0, hlreg); 595 596 bufsz = (adapter->rx_mbuf_sz + BSIZEPKT_ROUNDUP) >> 597 IXGBE_SRRCTL_BSIZEPKT_SHIFT; 598 599 for (i = 0; i < adapter->num_queues; i++, rxr++) { 600 u64 rdba = rxr->rxdma.dma_paddr; 601 u32 reg; 602 int regnum = i / 4; /* 1 register per 4 queues */ 603 int regshift = i % 4; /* 4 bits per 1 queue */ 604 j = rxr->me; 605 606 /* Setup the Base and Length of the Rx Descriptor Ring */ 607 IXGBE_WRITE_REG(hw, IXGBE_RDBAL(j), 608 (rdba & 0x00000000ffffffffULL)); 609 IXGBE_WRITE_REG(hw, IXGBE_RDBAH(j), (rdba >> 32)); 610 IXGBE_WRITE_REG(hw, IXGBE_RDLEN(j), 611 adapter->num_rx_desc * sizeof(union ixgbe_adv_rx_desc)); 612 613 /* Set up the SRRCTL register */ 614 srrctl = IXGBE_READ_REG(hw, IXGBE_SRRCTL(j)); 615 srrctl &= ~IXGBE_SRRCTL_BSIZEHDR_MASK; 616 srrctl &= ~IXGBE_SRRCTL_BSIZEPKT_MASK; 617 srrctl |= bufsz; 618 srrctl |= IXGBE_SRRCTL_DESCTYPE_ADV_ONEBUF; 619 620 /* Set RQSMR (Receive Queue Statistic Mapping) register */ 621 reg = IXGBE_READ_REG(hw, IXGBE_RQSMR(regnum)); 622 reg &= ~(0x000000ffUL << (regshift * 8)); 623 reg |= i << (regshift * 8); 624 IXGBE_WRITE_REG(hw, IXGBE_RQSMR(regnum), reg); 625 626 /* 627 * Set DROP_EN iff we have no flow control and >1 queue. 628 * Note that srrctl was cleared shortly before during reset, 629 * so we do not need to clear the bit, but do it just in case 630 * this code is moved elsewhere. 631 */ 632 if (adapter->num_queues > 1 && 633 adapter->hw.fc.requested_mode == ixgbe_fc_none) { 634 srrctl |= IXGBE_SRRCTL_DROP_EN; 635 } else { 636 srrctl &= ~IXGBE_SRRCTL_DROP_EN; 637 } 638 639 IXGBE_WRITE_REG(hw, IXGBE_SRRCTL(j), srrctl); 640 641 /* Setup the HW Rx Head and Tail Descriptor Pointers */ 642 IXGBE_WRITE_REG(hw, IXGBE_RDH(j), 0); 643 IXGBE_WRITE_REG(hw, IXGBE_RDT(j), 0); 644 645 /* Set the driver rx tail address */ 646 rxr->tail = IXGBE_RDT(rxr->me); 647 } 648 649 if (adapter->hw.mac.type != ixgbe_mac_82598EB) { 650 u32 psrtype = IXGBE_PSRTYPE_TCPHDR 651 | IXGBE_PSRTYPE_UDPHDR 652 | IXGBE_PSRTYPE_IPV4HDR 653 | IXGBE_PSRTYPE_IPV6HDR; 654 IXGBE_WRITE_REG(hw, IXGBE_PSRTYPE(0), psrtype); 655 } 656 657 rxcsum = IXGBE_READ_REG(hw, IXGBE_RXCSUM); 658 659 ixgbe_initialize_rss_mapping(adapter); 660 661 if (adapter->num_queues > 1) { 662 /* RSS and RX IPP Checksum are mutually exclusive */ 663 rxcsum |= IXGBE_RXCSUM_PCSD; 664 } 665 666 if (ifp->if_capenable & IFCAP_RXCSUM) 667 rxcsum |= IXGBE_RXCSUM_PCSD; 668 669 /* This is useful for calculating UDP/IP fragment checksums */ 670 if (!(rxcsum & IXGBE_RXCSUM_PCSD)) 671 rxcsum |= IXGBE_RXCSUM_IPPCSE; 672 673 IXGBE_WRITE_REG(hw, IXGBE_RXCSUM, rxcsum); 674 675 } /* ixgbe_initialize_receive_units */ 676 677 /************************************************************************ 678 * ixgbe_initialize_transmit_units - Enable transmit units. 679 ************************************************************************/ 680 static void 681 ixgbe_initialize_transmit_units(struct adapter *adapter) 682 { 683 struct tx_ring *txr = adapter->tx_rings; 684 struct ixgbe_hw *hw = &adapter->hw; 685 int i; 686 687 INIT_DEBUGOUT("ixgbe_initialize_transmit_units"); 688 689 /* Setup the Base and Length of the Tx Descriptor Ring */ 690 for (i = 0; i < adapter->num_queues; i++, txr++) { 691 u64 tdba = txr->txdma.dma_paddr; 692 u32 txctrl = 0; 693 u32 tqsmreg, reg; 694 int regnum = i / 4; /* 1 register per 4 queues */ 695 int regshift = i % 4; /* 4 bits per 1 queue */ 696 int j = txr->me; 697 698 IXGBE_WRITE_REG(hw, IXGBE_TDBAL(j), 699 (tdba & 0x00000000ffffffffULL)); 700 IXGBE_WRITE_REG(hw, IXGBE_TDBAH(j), (tdba >> 32)); 701 IXGBE_WRITE_REG(hw, IXGBE_TDLEN(j), 702 adapter->num_tx_desc * sizeof(union ixgbe_adv_tx_desc)); 703 704 /* 705 * Set TQSMR (Transmit Queue Statistic Mapping) register. 706 * Register location is different between 82598 and others. 707 */ 708 if (adapter->hw.mac.type == ixgbe_mac_82598EB) 709 tqsmreg = IXGBE_TQSMR(regnum); 710 else 711 tqsmreg = IXGBE_TQSM(regnum); 712 reg = IXGBE_READ_REG(hw, tqsmreg); 713 reg &= ~(0x000000ffUL << (regshift * 8)); 714 reg |= i << (regshift * 8); 715 IXGBE_WRITE_REG(hw, tqsmreg, reg); 716 717 /* Setup the HW Tx Head and Tail descriptor pointers */ 718 IXGBE_WRITE_REG(hw, IXGBE_TDH(j), 0); 719 IXGBE_WRITE_REG(hw, IXGBE_TDT(j), 0); 720 721 /* Cache the tail address */ 722 txr->tail = IXGBE_TDT(j); 723 724 txr->txr_no_space = false; 725 726 /* Disable Head Writeback */ 727 /* 728 * Note: for X550 series devices, these registers are actually 729 * prefixed with TPH_ instead of DCA_, but the addresses and 730 * fields remain the same. 731 */ 732 switch (hw->mac.type) { 733 case ixgbe_mac_82598EB: 734 txctrl = IXGBE_READ_REG(hw, IXGBE_DCA_TXCTRL(j)); 735 break; 736 default: 737 txctrl = IXGBE_READ_REG(hw, IXGBE_DCA_TXCTRL_82599(j)); 738 break; 739 } 740 txctrl &= ~IXGBE_DCA_TXCTRL_DESC_WRO_EN; 741 switch (hw->mac.type) { 742 case ixgbe_mac_82598EB: 743 IXGBE_WRITE_REG(hw, IXGBE_DCA_TXCTRL(j), txctrl); 744 break; 745 default: 746 IXGBE_WRITE_REG(hw, IXGBE_DCA_TXCTRL_82599(j), txctrl); 747 break; 748 } 749 750 } 751 752 if (hw->mac.type != ixgbe_mac_82598EB) { 753 u32 dmatxctl, rttdcs; 754 755 dmatxctl = IXGBE_READ_REG(hw, IXGBE_DMATXCTL); 756 dmatxctl |= IXGBE_DMATXCTL_TE; 757 IXGBE_WRITE_REG(hw, IXGBE_DMATXCTL, dmatxctl); 758 /* Disable arbiter to set MTQC */ 759 rttdcs = IXGBE_READ_REG(hw, IXGBE_RTTDCS); 760 rttdcs |= IXGBE_RTTDCS_ARBDIS; 761 IXGBE_WRITE_REG(hw, IXGBE_RTTDCS, rttdcs); 762 IXGBE_WRITE_REG(hw, IXGBE_MTQC, 763 ixgbe_get_mtqc(adapter->iov_mode)); 764 rttdcs &= ~IXGBE_RTTDCS_ARBDIS; 765 IXGBE_WRITE_REG(hw, IXGBE_RTTDCS, rttdcs); 766 } 767 768 return; 769 } /* ixgbe_initialize_transmit_units */ 770 771 static void 772 ixgbe_quirks(struct adapter *adapter) 773 { 774 device_t dev = adapter->dev; 775 struct ixgbe_hw *hw = &adapter->hw; 776 const char *vendor, *product; 777 778 if (hw->device_id == IXGBE_DEV_ID_X550EM_A_SFP_N) { 779 /* 780 * Quirk for inverted logic of SFP+'s MOD_ABS on GIGABYTE 781 * MA10-ST0. 782 */ 783 vendor = pmf_get_platform("system-vendor"); 784 product = pmf_get_platform("system-product"); 785 786 if ((vendor == NULL) || (product == NULL)) 787 return; 788 789 if ((strcmp(vendor, "GIGABYTE") == 0) && 790 (strcmp(product, "MA10-ST0") == 0)) { 791 aprint_verbose_dev(dev, 792 "Enable SFP+ MOD_ABS inverse quirk\n"); 793 adapter->hw.quirks |= IXGBE_QUIRK_MOD_ABS_INVERT; 794 } 795 } 796 } 797 798 /************************************************************************ 799 * ixgbe_attach - Device initialization routine 800 * 801 * Called when the driver is being loaded. 802 * Identifies the type of hardware, allocates all resources 803 * and initializes the hardware. 804 * 805 * return 0 on success, positive on failure 806 ************************************************************************/ 807 static void 808 ixgbe_attach(device_t parent, device_t dev, void *aux) 809 { 810 struct adapter *adapter; 811 struct ixgbe_hw *hw; 812 int error = -1; 813 u32 ctrl_ext; 814 u16 high, low, nvmreg; 815 pcireg_t id, subid; 816 const ixgbe_vendor_info_t *ent; 817 struct pci_attach_args *pa = aux; 818 bool unsupported_sfp = false; 819 const char *str; 820 char wqname[MAXCOMLEN]; 821 char buf[256]; 822 823 INIT_DEBUGOUT("ixgbe_attach: begin"); 824 825 /* Allocate, clear, and link in our adapter structure */ 826 adapter = device_private(dev); 827 adapter->hw.back = adapter; 828 adapter->dev = dev; 829 hw = &adapter->hw; 830 adapter->osdep.pc = pa->pa_pc; 831 adapter->osdep.tag = pa->pa_tag; 832 if (pci_dma64_available(pa)) 833 adapter->osdep.dmat = pa->pa_dmat64; 834 else 835 adapter->osdep.dmat = pa->pa_dmat; 836 adapter->osdep.attached = false; 837 adapter->osdep.detaching = false; 838 839 ent = ixgbe_lookup(pa); 840 841 KASSERT(ent != NULL); 842 843 aprint_normal(": %s, Version - %s\n", 844 ixgbe_strings[ent->index], ixgbe_driver_version); 845 846 /* Core Lock Init */ 847 IXGBE_CORE_LOCK_INIT(adapter, device_xname(dev)); 848 849 /* Set up the timer callout and workqueue */ 850 callout_init(&adapter->timer, IXGBE_CALLOUT_FLAGS); 851 snprintf(wqname, sizeof(wqname), "%s-timer", device_xname(dev)); 852 error = workqueue_create(&adapter->timer_wq, wqname, 853 ixgbe_handle_timer, adapter, IXGBE_WORKQUEUE_PRI, IPL_NET, 854 IXGBE_TASKLET_WQ_FLAGS); 855 if (error) { 856 aprint_error_dev(dev, 857 "could not create timer workqueue (%d)\n", error); 858 goto err_out; 859 } 860 861 /* Determine hardware revision */ 862 id = pci_conf_read(pa->pa_pc, pa->pa_tag, PCI_ID_REG); 863 subid = pci_conf_read(pa->pa_pc, pa->pa_tag, PCI_SUBSYS_ID_REG); 864 865 hw->vendor_id = PCI_VENDOR(id); 866 hw->device_id = PCI_PRODUCT(id); 867 hw->revision_id = 868 PCI_REVISION(pci_conf_read(pa->pa_pc, pa->pa_tag, PCI_CLASS_REG)); 869 hw->subsystem_vendor_id = PCI_SUBSYS_VENDOR(subid); 870 hw->subsystem_device_id = PCI_SUBSYS_ID(subid); 871 872 /* Set quirk flags */ 873 ixgbe_quirks(adapter); 874 875 /* 876 * Make sure BUSMASTER is set 877 */ 878 ixgbe_pci_enable_busmaster(pa->pa_pc, pa->pa_tag); 879 880 /* Do base PCI setup - map BAR0 */ 881 if (ixgbe_allocate_pci_resources(adapter, pa)) { 882 aprint_error_dev(dev, "Allocation of PCI resources failed\n"); 883 error = ENXIO; 884 goto err_out; 885 } 886 887 /* let hardware know driver is loaded */ 888 ctrl_ext = IXGBE_READ_REG(hw, IXGBE_CTRL_EXT); 889 ctrl_ext |= IXGBE_CTRL_EXT_DRV_LOAD; 890 IXGBE_WRITE_REG(hw, IXGBE_CTRL_EXT, ctrl_ext); 891 892 /* 893 * Initialize the shared code 894 */ 895 if (ixgbe_init_shared_code(hw) != 0) { 896 aprint_error_dev(dev, "Unable to initialize the shared code\n"); 897 error = ENXIO; 898 goto err_out; 899 } 900 901 switch (hw->mac.type) { 902 case ixgbe_mac_82598EB: 903 str = "82598EB"; 904 break; 905 case ixgbe_mac_82599EB: 906 str = "82599EB"; 907 break; 908 case ixgbe_mac_X540: 909 str = "X540"; 910 break; 911 case ixgbe_mac_X550: 912 str = "X550"; 913 break; 914 case ixgbe_mac_X550EM_x: 915 str = "X550EM X"; 916 break; 917 case ixgbe_mac_X550EM_a: 918 str = "X550EM A"; 919 break; 920 default: 921 str = "Unknown"; 922 break; 923 } 924 aprint_normal_dev(dev, "device %s\n", str); 925 926 hw->allow_unsupported_sfp = allow_unsupported_sfp; 927 928 /* Pick up the 82599 settings */ 929 if (hw->mac.type != ixgbe_mac_82598EB) 930 hw->phy.smart_speed = ixgbe_smart_speed; 931 932 /* Set the right number of segments */ 933 KASSERT(IXGBE_82599_SCATTER_MAX >= IXGBE_SCATTER_DEFAULT); 934 adapter->num_segs = IXGBE_SCATTER_DEFAULT; 935 936 /* Ensure SW/FW semaphore is free */ 937 ixgbe_init_swfw_semaphore(hw); 938 939 hw->mac.ops.set_lan_id(hw); 940 ixgbe_init_device_features(adapter); 941 942 if (ixgbe_configure_interrupts(adapter)) { 943 error = ENXIO; 944 goto err_out; 945 } 946 947 /* Allocate multicast array memory. */ 948 adapter->mta = malloc(sizeof(*adapter->mta) * 949 MAX_NUM_MULTICAST_ADDRESSES, M_DEVBUF, M_WAITOK); 950 951 /* Enable WoL (if supported) */ 952 ixgbe_check_wol_support(adapter); 953 954 /* Register for VLAN events */ 955 ether_set_vlan_cb(&adapter->osdep.ec, ixgbe_vlan_cb); 956 957 /* Verify adapter fan is still functional (if applicable) */ 958 if (adapter->feat_en & IXGBE_FEATURE_FAN_FAIL) { 959 u32 esdp = IXGBE_READ_REG(hw, IXGBE_ESDP); 960 ixgbe_check_fan_failure(adapter, esdp, FALSE); 961 } 962 963 /* Set an initial default flow control value */ 964 hw->fc.requested_mode = ixgbe_flow_control; 965 966 /* Do descriptor calc and sanity checks */ 967 if (((ixgbe_txd * sizeof(union ixgbe_adv_tx_desc)) % DBA_ALIGN) != 0 || 968 ixgbe_txd < MIN_TXD || ixgbe_txd > MAX_TXD) { 969 aprint_error_dev(dev, "TXD config issue, using default!\n"); 970 adapter->num_tx_desc = DEFAULT_TXD; 971 } else 972 adapter->num_tx_desc = ixgbe_txd; 973 974 if (((ixgbe_rxd * sizeof(union ixgbe_adv_rx_desc)) % DBA_ALIGN) != 0 || 975 ixgbe_rxd < MIN_RXD || ixgbe_rxd > MAX_RXD) { 976 aprint_error_dev(dev, "RXD config issue, using default!\n"); 977 adapter->num_rx_desc = DEFAULT_RXD; 978 } else 979 adapter->num_rx_desc = ixgbe_rxd; 980 981 /* Sysctls for limiting the amount of work done in the taskqueues */ 982 adapter->rx_process_limit 983 = (ixgbe_rx_process_limit <= adapter->num_rx_desc) 984 ? ixgbe_rx_process_limit : adapter->num_rx_desc; 985 adapter->tx_process_limit 986 = (ixgbe_tx_process_limit <= adapter->num_tx_desc) 987 ? ixgbe_tx_process_limit : adapter->num_tx_desc; 988 989 /* Set default high limit of copying mbuf in rxeof */ 990 adapter->rx_copy_len = IXGBE_RX_COPY_LEN_MAX; 991 992 /* Allocate our TX/RX Queues */ 993 if (ixgbe_allocate_queues(adapter)) { 994 error = ENOMEM; 995 goto err_out; 996 } 997 998 hw->phy.reset_if_overtemp = TRUE; 999 error = ixgbe_reset_hw(hw); 1000 hw->phy.reset_if_overtemp = FALSE; 1001 if (error == IXGBE_ERR_SFP_NOT_PRESENT) 1002 error = IXGBE_SUCCESS; 1003 else if (error == IXGBE_ERR_SFP_NOT_SUPPORTED) { 1004 aprint_error_dev(dev, "Unsupported SFP+ module detected!\n"); 1005 unsupported_sfp = true; 1006 error = IXGBE_SUCCESS; 1007 } else if (error) { 1008 aprint_error_dev(dev, 1009 "Hardware initialization failed(error = %d)\n", error); 1010 error = EIO; 1011 goto err_late; 1012 } 1013 1014 /* Make sure we have a good EEPROM before we read from it */ 1015 if (ixgbe_validate_eeprom_checksum(&adapter->hw, NULL) < 0) { 1016 aprint_error_dev(dev, "The EEPROM Checksum Is Not Valid\n"); 1017 error = EIO; 1018 goto err_late; 1019 } 1020 1021 aprint_normal("%s:", device_xname(dev)); 1022 /* NVM Image Version */ 1023 high = low = 0; 1024 switch (hw->mac.type) { 1025 case ixgbe_mac_82598EB: 1026 /* 1027 * Print version from the dev starter version (0x29). The 1028 * location is the same as newer device's IXGBE_NVM_MAP_VER. 1029 */ 1030 hw->eeprom.ops.read(hw, IXGBE_NVM_MAP_VER, &nvmreg); 1031 if (nvmreg == 0xffff) 1032 break; 1033 high = (nvmreg >> 12) & 0x0f; 1034 low = (nvmreg >> 4) & 0xff; 1035 id = nvmreg & 0x0f; 1036 /* 1037 * The following output might not be correct. Some 82598 cards 1038 * have 0x1070 or 0x2090. 82598 spec update notes about 2.9.0. 1039 */ 1040 aprint_normal(" NVM Image Version %u.%u.%u,", high, low, id); 1041 break; 1042 case ixgbe_mac_X540: 1043 case ixgbe_mac_X550EM_a: 1044 hw->eeprom.ops.read(hw, IXGBE_NVM_IMAGE_VER, &nvmreg); 1045 if (nvmreg == 0xffff) 1046 break; 1047 high = (nvmreg >> 12) & 0x0f; 1048 low = (nvmreg >> 4) & 0xff; 1049 id = nvmreg & 0x0f; 1050 aprint_normal(" NVM Image Version %u.", high); 1051 if (hw->mac.type == ixgbe_mac_X540) 1052 str = "%x"; 1053 else 1054 str = "%02x"; 1055 aprint_normal(str, low); 1056 aprint_normal(" ID 0x%x,", id); 1057 break; 1058 case ixgbe_mac_X550EM_x: 1059 case ixgbe_mac_X550: 1060 hw->eeprom.ops.read(hw, IXGBE_NVM_IMAGE_VER, &nvmreg); 1061 if (nvmreg == 0xffff) 1062 break; 1063 high = (nvmreg >> 12) & 0x0f; 1064 low = nvmreg & 0xff; 1065 aprint_normal(" NVM Image Version %u.%02x,", high, low); 1066 break; 1067 default: 1068 break; 1069 } 1070 hw->eeprom.nvm_image_ver_high = high; 1071 hw->eeprom.nvm_image_ver_low = low; 1072 1073 /* PHY firmware revision */ 1074 switch (hw->mac.type) { 1075 case ixgbe_mac_X540: 1076 case ixgbe_mac_X550: 1077 hw->eeprom.ops.read(hw, IXGBE_PHYFW_REV, &nvmreg); 1078 if (nvmreg == 0xffff) 1079 break; 1080 high = (nvmreg >> 12) & 0x0f; 1081 low = (nvmreg >> 4) & 0xff; 1082 id = nvmreg & 0x000f; 1083 aprint_normal(" PHY FW Revision %u.", high); 1084 if (hw->mac.type == ixgbe_mac_X540) 1085 str = "%x"; 1086 else 1087 str = "%02x"; 1088 aprint_normal(str, low); 1089 aprint_normal(" ID 0x%x,", id); 1090 break; 1091 default: 1092 break; 1093 } 1094 1095 /* NVM Map version & OEM NVM Image version */ 1096 switch (hw->mac.type) { 1097 case ixgbe_mac_X550: 1098 case ixgbe_mac_X550EM_x: 1099 case ixgbe_mac_X550EM_a: 1100 hw->eeprom.ops.read(hw, IXGBE_NVM_MAP_VER, &nvmreg); 1101 if (nvmreg != 0xffff) { 1102 high = (nvmreg >> 12) & 0x0f; 1103 low = nvmreg & 0x00ff; 1104 aprint_normal(" NVM Map version %u.%02x,", high, low); 1105 } 1106 hw->eeprom.ops.read(hw, IXGBE_OEM_NVM_IMAGE_VER, &nvmreg); 1107 if (nvmreg != 0xffff) { 1108 high = (nvmreg >> 12) & 0x0f; 1109 low = nvmreg & 0x00ff; 1110 aprint_verbose(" OEM NVM Image version %u.%02x,", high, 1111 low); 1112 } 1113 break; 1114 default: 1115 break; 1116 } 1117 1118 /* Print the ETrackID */ 1119 hw->eeprom.ops.read(hw, IXGBE_ETRACKID_H, &high); 1120 hw->eeprom.ops.read(hw, IXGBE_ETRACKID_L, &low); 1121 aprint_normal(" ETrackID %08x\n", ((uint32_t)high << 16) | low); 1122 1123 /* Printed Board Assembly number */ 1124 error = ixgbe_read_pba_string(hw, buf, IXGBE_PBANUM_LENGTH); 1125 aprint_normal_dev(dev, "PBA number %s\n", error ? "unknown" : buf); 1126 1127 if (adapter->feat_en & IXGBE_FEATURE_MSIX) { 1128 error = ixgbe_allocate_msix(adapter, pa); 1129 if (error) { 1130 /* Free allocated queue structures first */ 1131 ixgbe_free_queues(adapter); 1132 1133 /* Fallback to legacy interrupt */ 1134 if (adapter->feat_cap & IXGBE_FEATURE_MSI) 1135 adapter->feat_en |= IXGBE_FEATURE_MSI; 1136 adapter->num_queues = 1; 1137 1138 /* Allocate our TX/RX Queues again */ 1139 if (ixgbe_allocate_queues(adapter)) { 1140 error = ENOMEM; 1141 goto err_out; 1142 } 1143 } 1144 } 1145 1146 /* Recovery mode */ 1147 switch (adapter->hw.mac.type) { 1148 case ixgbe_mac_X550: 1149 case ixgbe_mac_X550EM_x: 1150 case ixgbe_mac_X550EM_a: 1151 /* >= 2.00 */ 1152 if (hw->eeprom.nvm_image_ver_high >= 2) { 1153 adapter->feat_cap |= IXGBE_FEATURE_RECOVERY_MODE; 1154 adapter->feat_en |= IXGBE_FEATURE_RECOVERY_MODE; 1155 } 1156 break; 1157 default: 1158 break; 1159 } 1160 1161 if ((adapter->feat_en & IXGBE_FEATURE_MSIX) == 0) 1162 error = ixgbe_allocate_legacy(adapter, pa); 1163 if (error) 1164 goto err_late; 1165 1166 /* Tasklets for Link, SFP, Multispeed Fiber and Flow Director */ 1167 mutex_init(&(adapter)->admin_mtx, MUTEX_DEFAULT, IPL_NET); 1168 snprintf(wqname, sizeof(wqname), "%s-admin", device_xname(dev)); 1169 error = workqueue_create(&adapter->admin_wq, wqname, 1170 ixgbe_handle_admin, adapter, IXGBE_WORKQUEUE_PRI, IPL_NET, 1171 IXGBE_TASKLET_WQ_FLAGS); 1172 if (error) { 1173 aprint_error_dev(dev, 1174 "could not create admin workqueue (%d)\n", error); 1175 goto err_out; 1176 } 1177 1178 error = ixgbe_start_hw(hw); 1179 switch (error) { 1180 case IXGBE_ERR_EEPROM_VERSION: 1181 aprint_error_dev(dev, "This device is a pre-production adapter/" 1182 "LOM. Please be aware there may be issues associated " 1183 "with your hardware.\nIf you are experiencing problems " 1184 "please contact your Intel or hardware representative " 1185 "who provided you with this hardware.\n"); 1186 break; 1187 default: 1188 break; 1189 } 1190 1191 /* Setup OS specific network interface */ 1192 if (ixgbe_setup_interface(dev, adapter) != 0) 1193 goto err_late; 1194 1195 /* 1196 * Print PHY ID only for copper PHY. On device which has SFP(+) cage 1197 * and a module is inserted, phy.id is not MII PHY id but SFF 8024 ID. 1198 */ 1199 if (hw->phy.media_type == ixgbe_media_type_copper) { 1200 uint16_t id1, id2; 1201 int oui, model, rev; 1202 char descr[MII_MAX_DESCR_LEN]; 1203 1204 id1 = hw->phy.id >> 16; 1205 id2 = hw->phy.id & 0xffff; 1206 oui = MII_OUI(id1, id2); 1207 model = MII_MODEL(id2); 1208 rev = MII_REV(id2); 1209 mii_get_descr(descr, sizeof(descr), oui, model); 1210 if (descr[0]) 1211 aprint_normal_dev(dev, "PHY: %s, rev. %d\n", 1212 descr, rev); 1213 else 1214 aprint_normal_dev(dev, 1215 "PHY OUI 0x%06x, model 0x%04x, rev. %d\n", 1216 oui, model, rev); 1217 } 1218 1219 /* Enable EEE power saving */ 1220 if (adapter->feat_cap & IXGBE_FEATURE_EEE) 1221 hw->mac.ops.setup_eee(hw, 1222 adapter->feat_en & IXGBE_FEATURE_EEE); 1223 1224 /* Enable power to the phy. */ 1225 if (!unsupported_sfp) { 1226 /* Enable the optics for 82599 SFP+ fiber */ 1227 ixgbe_enable_tx_laser(hw); 1228 1229 /* 1230 * XXX Currently, ixgbe_set_phy_power() supports only copper 1231 * PHY, so it's not required to test with !unsupported_sfp. 1232 */ 1233 ixgbe_set_phy_power(hw, TRUE); 1234 } 1235 1236 /* Initialize statistics */ 1237 ixgbe_update_stats_counters(adapter); 1238 1239 /* Check PCIE slot type/speed/width */ 1240 ixgbe_get_slot_info(adapter); 1241 1242 /* 1243 * Do time init and sysctl init here, but 1244 * only on the first port of a bypass adapter. 1245 */ 1246 ixgbe_bypass_init(adapter); 1247 1248 /* Set an initial dmac value */ 1249 adapter->dmac = 0; 1250 /* Set initial advertised speeds (if applicable) */ 1251 adapter->advertise = ixgbe_get_default_advertise(adapter); 1252 1253 if (adapter->feat_cap & IXGBE_FEATURE_SRIOV) 1254 ixgbe_define_iov_schemas(dev, &error); 1255 1256 /* Add sysctls */ 1257 ixgbe_add_device_sysctls(adapter); 1258 ixgbe_add_hw_stats(adapter); 1259 1260 /* For Netmap */ 1261 adapter->init_locked = ixgbe_init_locked; 1262 adapter->stop_locked = ixgbe_stop_locked; 1263 1264 if (adapter->feat_en & IXGBE_FEATURE_NETMAP) 1265 ixgbe_netmap_attach(adapter); 1266 1267 snprintb(buf, sizeof(buf), IXGBE_FEATURE_FLAGS, adapter->feat_cap); 1268 aprint_verbose_dev(dev, "feature cap %s\n", buf); 1269 snprintb(buf, sizeof(buf), IXGBE_FEATURE_FLAGS, adapter->feat_en); 1270 aprint_verbose_dev(dev, "feature ena %s\n", buf); 1271 1272 if (pmf_device_register(dev, ixgbe_suspend, ixgbe_resume)) 1273 pmf_class_network_register(dev, adapter->ifp); 1274 else 1275 aprint_error_dev(dev, "couldn't establish power handler\n"); 1276 1277 /* Init recovery mode timer and state variable */ 1278 if (adapter->feat_en & IXGBE_FEATURE_RECOVERY_MODE) { 1279 adapter->recovery_mode = 0; 1280 1281 /* Set up the timer callout */ 1282 callout_init(&adapter->recovery_mode_timer, 1283 IXGBE_CALLOUT_FLAGS); 1284 snprintf(wqname, sizeof(wqname), "%s-recovery", 1285 device_xname(dev)); 1286 error = workqueue_create(&adapter->recovery_mode_timer_wq, 1287 wqname, ixgbe_handle_recovery_mode_timer, adapter, 1288 IXGBE_WORKQUEUE_PRI, IPL_NET, IXGBE_TASKLET_WQ_FLAGS); 1289 if (error) { 1290 aprint_error_dev(dev, "could not create " 1291 "recovery_mode_timer workqueue (%d)\n", error); 1292 goto err_out; 1293 } 1294 1295 /* Start the task */ 1296 callout_reset(&adapter->recovery_mode_timer, hz, 1297 ixgbe_recovery_mode_timer, adapter); 1298 } 1299 1300 INIT_DEBUGOUT("ixgbe_attach: end"); 1301 adapter->osdep.attached = true; 1302 1303 return; 1304 1305 err_late: 1306 ixgbe_free_queues(adapter); 1307 err_out: 1308 ctrl_ext = IXGBE_READ_REG(&adapter->hw, IXGBE_CTRL_EXT); 1309 ctrl_ext &= ~IXGBE_CTRL_EXT_DRV_LOAD; 1310 IXGBE_WRITE_REG(&adapter->hw, IXGBE_CTRL_EXT, ctrl_ext); 1311 ixgbe_free_deferred_handlers(adapter); 1312 ixgbe_free_pci_resources(adapter); 1313 if (adapter->mta != NULL) 1314 free(adapter->mta, M_DEVBUF); 1315 mutex_destroy(&(adapter)->admin_mtx); /* XXX appropriate order? */ 1316 IXGBE_CORE_LOCK_DESTROY(adapter); 1317 1318 return; 1319 } /* ixgbe_attach */ 1320 1321 /************************************************************************ 1322 * ixgbe_check_wol_support 1323 * 1324 * Checks whether the adapter's ports are capable of 1325 * Wake On LAN by reading the adapter's NVM. 1326 * 1327 * Sets each port's hw->wol_enabled value depending 1328 * on the value read here. 1329 ************************************************************************/ 1330 static void 1331 ixgbe_check_wol_support(struct adapter *adapter) 1332 { 1333 struct ixgbe_hw *hw = &adapter->hw; 1334 u16 dev_caps = 0; 1335 1336 /* Find out WoL support for port */ 1337 adapter->wol_support = hw->wol_enabled = 0; 1338 ixgbe_get_device_caps(hw, &dev_caps); 1339 if ((dev_caps & IXGBE_DEVICE_CAPS_WOL_PORT0_1) || 1340 ((dev_caps & IXGBE_DEVICE_CAPS_WOL_PORT0) && 1341 hw->bus.func == 0)) 1342 adapter->wol_support = hw->wol_enabled = 1; 1343 1344 /* Save initial wake up filter configuration */ 1345 adapter->wufc = IXGBE_READ_REG(hw, IXGBE_WUFC); 1346 1347 return; 1348 } /* ixgbe_check_wol_support */ 1349 1350 /************************************************************************ 1351 * ixgbe_setup_interface 1352 * 1353 * Setup networking device structure and register an interface. 1354 ************************************************************************/ 1355 static int 1356 ixgbe_setup_interface(device_t dev, struct adapter *adapter) 1357 { 1358 struct ethercom *ec = &adapter->osdep.ec; 1359 struct ifnet *ifp; 1360 1361 INIT_DEBUGOUT("ixgbe_setup_interface: begin"); 1362 1363 ifp = adapter->ifp = &ec->ec_if; 1364 strlcpy(ifp->if_xname, device_xname(dev), IFNAMSIZ); 1365 ifp->if_baudrate = IF_Gbps(10); 1366 ifp->if_init = ixgbe_init; 1367 ifp->if_stop = ixgbe_ifstop; 1368 ifp->if_softc = adapter; 1369 ifp->if_flags = IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST; 1370 #ifdef IXGBE_MPSAFE 1371 ifp->if_extflags = IFEF_MPSAFE; 1372 #endif 1373 ifp->if_ioctl = ixgbe_ioctl; 1374 #if __FreeBSD_version >= 1100045 1375 /* TSO parameters */ 1376 ifp->if_hw_tsomax = 65518; 1377 ifp->if_hw_tsomaxsegcount = IXGBE_82599_SCATTER; 1378 ifp->if_hw_tsomaxsegsize = 2048; 1379 #endif 1380 if (adapter->feat_en & IXGBE_FEATURE_LEGACY_TX) { 1381 #if 0 1382 ixgbe_start_locked = ixgbe_legacy_start_locked; 1383 #endif 1384 } else { 1385 ifp->if_transmit = ixgbe_mq_start; 1386 #if 0 1387 ixgbe_start_locked = ixgbe_mq_start_locked; 1388 #endif 1389 } 1390 ifp->if_start = ixgbe_legacy_start; 1391 IFQ_SET_MAXLEN(&ifp->if_snd, adapter->num_tx_desc - 2); 1392 IFQ_SET_READY(&ifp->if_snd); 1393 1394 if_initialize(ifp); 1395 adapter->ipq = if_percpuq_create(&adapter->osdep.ec.ec_if); 1396 ether_ifattach(ifp, adapter->hw.mac.addr); 1397 aprint_normal_dev(dev, "Ethernet address %s\n", 1398 ether_sprintf(adapter->hw.mac.addr)); 1399 /* 1400 * We use per TX queue softint, so if_deferred_start_init() isn't 1401 * used. 1402 */ 1403 ether_set_ifflags_cb(ec, ixgbe_ifflags_cb); 1404 1405 adapter->max_frame_size = ifp->if_mtu + ETHER_HDR_LEN + ETHER_CRC_LEN; 1406 1407 /* 1408 * Tell the upper layer(s) we support long frames. 1409 */ 1410 ifp->if_hdrlen = sizeof(struct ether_vlan_header); 1411 1412 /* Set capability flags */ 1413 ifp->if_capabilities |= IFCAP_RXCSUM 1414 | IFCAP_TXCSUM 1415 | IFCAP_TSOv4 1416 | IFCAP_TSOv6; 1417 ifp->if_capenable = 0; 1418 1419 ec->ec_capabilities |= ETHERCAP_VLAN_HWTAGGING 1420 | ETHERCAP_VLAN_HWCSUM 1421 | ETHERCAP_JUMBO_MTU 1422 | ETHERCAP_VLAN_MTU; 1423 1424 /* Enable the above capabilities by default */ 1425 ec->ec_capenable = ec->ec_capabilities; 1426 1427 /* 1428 * Don't turn this on by default, if vlans are 1429 * created on another pseudo device (eg. lagg) 1430 * then vlan events are not passed thru, breaking 1431 * operation, but with HW FILTER off it works. If 1432 * using vlans directly on the ixgbe driver you can 1433 * enable this and get full hardware tag filtering. 1434 */ 1435 ec->ec_capabilities |= ETHERCAP_VLAN_HWFILTER; 1436 1437 /* 1438 * Specify the media types supported by this adapter and register 1439 * callbacks to update media and link information 1440 */ 1441 ec->ec_ifmedia = &adapter->media; 1442 ifmedia_init_with_lock(&adapter->media, IFM_IMASK, ixgbe_media_change, 1443 ixgbe_media_status, &adapter->core_mtx); 1444 1445 adapter->phy_layer = ixgbe_get_supported_physical_layer(&adapter->hw); 1446 ixgbe_add_media_types(adapter); 1447 1448 /* Set autoselect media by default */ 1449 ifmedia_set(&adapter->media, IFM_ETHER | IFM_AUTO); 1450 1451 if_register(ifp); 1452 1453 return (0); 1454 } /* ixgbe_setup_interface */ 1455 1456 /************************************************************************ 1457 * ixgbe_add_media_types 1458 ************************************************************************/ 1459 static void 1460 ixgbe_add_media_types(struct adapter *adapter) 1461 { 1462 struct ixgbe_hw *hw = &adapter->hw; 1463 u64 layer; 1464 1465 layer = adapter->phy_layer; 1466 1467 #define ADD(mm, dd) \ 1468 ifmedia_add(&adapter->media, IFM_ETHER | (mm), (dd), NULL); 1469 1470 ADD(IFM_NONE, 0); 1471 1472 /* Media types with matching NetBSD media defines */ 1473 if (layer & IXGBE_PHYSICAL_LAYER_10GBASE_T) { 1474 ADD(IFM_10G_T | IFM_FDX, 0); 1475 } 1476 if (layer & IXGBE_PHYSICAL_LAYER_1000BASE_T) { 1477 ADD(IFM_1000_T | IFM_FDX, 0); 1478 } 1479 if (layer & IXGBE_PHYSICAL_LAYER_100BASE_TX) { 1480 ADD(IFM_100_TX | IFM_FDX, 0); 1481 } 1482 if (layer & IXGBE_PHYSICAL_LAYER_10BASE_T) { 1483 ADD(IFM_10_T | IFM_FDX, 0); 1484 } 1485 1486 if (layer & IXGBE_PHYSICAL_LAYER_SFP_PLUS_CU || 1487 layer & IXGBE_PHYSICAL_LAYER_SFP_ACTIVE_DA) { 1488 ADD(IFM_10G_TWINAX | IFM_FDX, 0); 1489 } 1490 1491 if (layer & IXGBE_PHYSICAL_LAYER_10GBASE_LR) { 1492 ADD(IFM_10G_LR | IFM_FDX, 0); 1493 if (hw->phy.multispeed_fiber) { 1494 ADD(IFM_1000_LX | IFM_FDX, 0); 1495 } 1496 } 1497 if (layer & IXGBE_PHYSICAL_LAYER_10GBASE_SR) { 1498 ADD(IFM_10G_SR | IFM_FDX, 0); 1499 if (hw->phy.multispeed_fiber) { 1500 ADD(IFM_1000_SX | IFM_FDX, 0); 1501 } 1502 } else if (layer & IXGBE_PHYSICAL_LAYER_1000BASE_SX) { 1503 ADD(IFM_1000_SX | IFM_FDX, 0); 1504 } 1505 if (layer & IXGBE_PHYSICAL_LAYER_10GBASE_CX4) { 1506 ADD(IFM_10G_CX4 | IFM_FDX, 0); 1507 } 1508 1509 if (layer & IXGBE_PHYSICAL_LAYER_10GBASE_KR) { 1510 ADD(IFM_10G_KR | IFM_FDX, 0); 1511 } 1512 if (layer & IXGBE_PHYSICAL_LAYER_10GBASE_KX4) { 1513 ADD(IFM_10G_KX4 | IFM_FDX, 0); 1514 } 1515 if (layer & IXGBE_PHYSICAL_LAYER_1000BASE_KX) { 1516 ADD(IFM_1000_KX | IFM_FDX, 0); 1517 } 1518 if (layer & IXGBE_PHYSICAL_LAYER_2500BASE_KX) { 1519 ADD(IFM_2500_KX | IFM_FDX, 0); 1520 } 1521 if (layer & IXGBE_PHYSICAL_LAYER_2500BASE_T) { 1522 ADD(IFM_2500_T | IFM_FDX, 0); 1523 } 1524 if (layer & IXGBE_PHYSICAL_LAYER_5GBASE_T) { 1525 ADD(IFM_5000_T | IFM_FDX, 0); 1526 } 1527 if (layer & IXGBE_PHYSICAL_LAYER_1000BASE_BX) 1528 ADD(IFM_1000_BX10 | IFM_FDX, 0); 1529 /* XXX no ifmedia_set? */ 1530 1531 ADD(IFM_AUTO, 0); 1532 1533 #undef ADD 1534 } /* ixgbe_add_media_types */ 1535 1536 /************************************************************************ 1537 * ixgbe_is_sfp 1538 ************************************************************************/ 1539 static inline bool 1540 ixgbe_is_sfp(struct ixgbe_hw *hw) 1541 { 1542 switch (hw->mac.type) { 1543 case ixgbe_mac_82598EB: 1544 if (hw->phy.type == ixgbe_phy_nl) 1545 return (TRUE); 1546 return (FALSE); 1547 case ixgbe_mac_82599EB: 1548 case ixgbe_mac_X550EM_x: 1549 case ixgbe_mac_X550EM_a: 1550 switch (hw->mac.ops.get_media_type(hw)) { 1551 case ixgbe_media_type_fiber: 1552 case ixgbe_media_type_fiber_qsfp: 1553 return (TRUE); 1554 default: 1555 return (FALSE); 1556 } 1557 default: 1558 return (FALSE); 1559 } 1560 } /* ixgbe_is_sfp */ 1561 1562 static void 1563 ixgbe_schedule_admin_tasklet(struct adapter *adapter) 1564 { 1565 1566 KASSERT(mutex_owned(&adapter->admin_mtx)); 1567 1568 if (__predict_true(adapter->osdep.detaching == false)) { 1569 if (adapter->admin_pending == 0) 1570 workqueue_enqueue(adapter->admin_wq, 1571 &adapter->admin_wc, NULL); 1572 adapter->admin_pending = 1; 1573 } 1574 } 1575 1576 /************************************************************************ 1577 * ixgbe_config_link 1578 ************************************************************************/ 1579 static void 1580 ixgbe_config_link(struct adapter *adapter) 1581 { 1582 struct ixgbe_hw *hw = &adapter->hw; 1583 u32 autoneg, err = 0; 1584 u32 task_requests = 0; 1585 bool sfp, negotiate = false; 1586 1587 sfp = ixgbe_is_sfp(hw); 1588 1589 if (sfp) { 1590 if (hw->phy.multispeed_fiber) { 1591 ixgbe_enable_tx_laser(hw); 1592 task_requests |= IXGBE_REQUEST_TASK_MSF_WOI; 1593 } 1594 task_requests |= IXGBE_REQUEST_TASK_MOD_WOI; 1595 1596 mutex_enter(&adapter->admin_mtx); 1597 adapter->task_requests |= task_requests; 1598 ixgbe_schedule_admin_tasklet(adapter); 1599 mutex_exit(&adapter->admin_mtx); 1600 } else { 1601 struct ifmedia *ifm = &adapter->media; 1602 1603 if (hw->mac.ops.check_link) 1604 err = ixgbe_check_link(hw, &adapter->link_speed, 1605 &adapter->link_up, FALSE); 1606 if (err) 1607 return; 1608 1609 /* 1610 * Check if it's the first call. If it's the first call, 1611 * get value for auto negotiation. 1612 */ 1613 autoneg = hw->phy.autoneg_advertised; 1614 if ((IFM_SUBTYPE(ifm->ifm_cur->ifm_media) != IFM_NONE) 1615 && ((!autoneg) && (hw->mac.ops.get_link_capabilities))) 1616 err = hw->mac.ops.get_link_capabilities(hw, &autoneg, 1617 &negotiate); 1618 if (err) 1619 return; 1620 if (hw->mac.ops.setup_link) 1621 err = hw->mac.ops.setup_link(hw, autoneg, 1622 adapter->link_up); 1623 } 1624 1625 } /* ixgbe_config_link */ 1626 1627 /************************************************************************ 1628 * ixgbe_update_stats_counters - Update board statistics counters. 1629 ************************************************************************/ 1630 static void 1631 ixgbe_update_stats_counters(struct adapter *adapter) 1632 { 1633 struct ifnet *ifp = adapter->ifp; 1634 struct ixgbe_hw *hw = &adapter->hw; 1635 struct ixgbe_hw_stats *stats = &adapter->stats.pf; 1636 u32 missed_rx = 0, bprc, lxontxc, lxofftxc; 1637 u64 total, total_missed_rx = 0; 1638 uint64_t crcerrs, illerrc, rlec, ruc, rfc, roc, rjc; 1639 unsigned int queue_counters; 1640 int i; 1641 1642 IXGBE_EVC_REGADD2(hw, stats, IXGBE_CRCERRS, crcerrs); 1643 IXGBE_EVC_REGADD2(hw, stats, IXGBE_ILLERRC, illerrc); 1644 1645 IXGBE_EVC_REGADD(hw, stats, IXGBE_ERRBC, errbc); 1646 IXGBE_EVC_REGADD(hw, stats, IXGBE_MSPDC, mspdc); 1647 if (hw->mac.type >= ixgbe_mac_X550) 1648 IXGBE_EVC_REGADD(hw, stats, IXGBE_MBSDC, mbsdc); 1649 1650 /* 16 registers exist */ 1651 queue_counters = uimin(__arraycount(stats->qprc), adapter->num_queues); 1652 for (i = 0; i < queue_counters; i++) { 1653 IXGBE_EVC_REGADD(hw, stats, IXGBE_QPRC(i), qprc[i]); 1654 IXGBE_EVC_REGADD(hw, stats, IXGBE_QPTC(i), qptc[i]); 1655 if (hw->mac.type >= ixgbe_mac_82599EB) 1656 IXGBE_EVC_REGADD(hw, stats, IXGBE_QPRDC(i), qprdc[i]); 1657 } 1658 1659 /* 8 registers exist */ 1660 for (i = 0; i < IXGBE_TC_COUNTER_NUM; i++) { 1661 uint32_t mp; 1662 1663 /* MPC */ 1664 mp = IXGBE_READ_REG(hw, IXGBE_MPC(i)); 1665 /* global total per queue */ 1666 IXGBE_EVC_ADD(&stats->mpc[i], mp); 1667 /* running comprehensive total for stats display */ 1668 total_missed_rx += mp; 1669 1670 if (hw->mac.type == ixgbe_mac_82598EB) 1671 IXGBE_EVC_REGADD(hw, stats, IXGBE_RNBC(i), rnbc[i]); 1672 1673 IXGBE_EVC_REGADD(hw, stats, IXGBE_PXONTXC(i), pxontxc[i]); 1674 IXGBE_EVC_REGADD(hw, stats, IXGBE_PXOFFTXC(i), pxofftxc[i]); 1675 if (hw->mac.type >= ixgbe_mac_82599EB) { 1676 IXGBE_EVC_REGADD(hw, stats, IXGBE_PXONRXCNT(i), pxonrxc[i]); 1677 IXGBE_EVC_REGADD(hw, stats, IXGBE_PXOFFRXCNT(i), pxoffrxc[i]); 1678 IXGBE_EVC_REGADD(hw, stats, IXGBE_PXON2OFFCNT(i), 1679 pxon2offc[i]); 1680 } else { 1681 IXGBE_EVC_REGADD(hw, stats, IXGBE_PXONRXC(i), pxonrxc[i]); 1682 IXGBE_EVC_REGADD(hw, stats, IXGBE_PXOFFRXC(i), pxoffrxc[i]); 1683 } 1684 } 1685 IXGBE_EVC_ADD(&stats->mpctotal, total_missed_rx); 1686 1687 /* Document says M[LR]FC are valid when link is up and 10Gbps */ 1688 if ((adapter->link_active == LINK_STATE_UP) 1689 && (adapter->link_speed == IXGBE_LINK_SPEED_10GB_FULL)) { 1690 IXGBE_EVC_REGADD(hw, stats, IXGBE_MLFC, mlfc); 1691 IXGBE_EVC_REGADD(hw, stats, IXGBE_MRFC, mrfc); 1692 } 1693 IXGBE_EVC_REGADD2(hw, stats, IXGBE_RLEC, rlec); 1694 1695 /* Hardware workaround, gprc counts missed packets */ 1696 IXGBE_EVC_ADD(&stats->gprc, 1697 IXGBE_READ_REG(hw, IXGBE_GPRC) - missed_rx); 1698 1699 IXGBE_EVC_REGADD2(hw, stats, IXGBE_LXONTXC, lxontxc); 1700 IXGBE_EVC_REGADD2(hw, stats, IXGBE_LXOFFTXC, lxofftxc); 1701 total = lxontxc + lxofftxc; 1702 1703 if (hw->mac.type != ixgbe_mac_82598EB) { 1704 IXGBE_EVC_ADD(&stats->gorc, IXGBE_READ_REG(hw, IXGBE_GORCL) + 1705 ((u64)IXGBE_READ_REG(hw, IXGBE_GORCH) << 32)); 1706 IXGBE_EVC_ADD(&stats->gotc, IXGBE_READ_REG(hw, IXGBE_GOTCL) + 1707 ((u64)IXGBE_READ_REG(hw, IXGBE_GOTCH) << 32) 1708 - total * ETHER_MIN_LEN); 1709 IXGBE_EVC_ADD(&stats->tor, IXGBE_READ_REG(hw, IXGBE_TORL) + 1710 ((u64)IXGBE_READ_REG(hw, IXGBE_TORH) << 32)); 1711 IXGBE_EVC_REGADD(hw, stats, IXGBE_LXONRXCNT, lxonrxc); 1712 IXGBE_EVC_REGADD(hw, stats, IXGBE_LXOFFRXCNT, lxoffrxc); 1713 } else { 1714 IXGBE_EVC_REGADD(hw, stats, IXGBE_LXONRXC, lxonrxc); 1715 IXGBE_EVC_REGADD(hw, stats, IXGBE_LXOFFRXC, lxoffrxc); 1716 /* 82598 only has a counter in the high register */ 1717 IXGBE_EVC_REGADD(hw, stats, IXGBE_GORCH, gorc); 1718 IXGBE_EVC_ADD(&stats->gotc, IXGBE_READ_REG(hw, IXGBE_GOTCH) 1719 - total * ETHER_MIN_LEN); 1720 IXGBE_EVC_REGADD(hw, stats, IXGBE_TORH, tor); 1721 } 1722 1723 /* 1724 * Workaround: mprc hardware is incorrectly counting 1725 * broadcasts, so for now we subtract those. 1726 */ 1727 IXGBE_EVC_REGADD2(hw, stats, IXGBE_BPRC, bprc); 1728 IXGBE_EVC_ADD(&stats->mprc, IXGBE_READ_REG(hw, IXGBE_MPRC) 1729 - ((hw->mac.type == ixgbe_mac_82598EB) ? bprc : 0)); 1730 1731 IXGBE_EVC_REGADD(hw, stats, IXGBE_PRC64, prc64); 1732 IXGBE_EVC_REGADD(hw, stats, IXGBE_PRC127, prc127); 1733 IXGBE_EVC_REGADD(hw, stats, IXGBE_PRC255, prc255); 1734 IXGBE_EVC_REGADD(hw, stats, IXGBE_PRC511, prc511); 1735 IXGBE_EVC_REGADD(hw, stats, IXGBE_PRC1023, prc1023); 1736 IXGBE_EVC_REGADD(hw, stats, IXGBE_PRC1522, prc1522); 1737 1738 IXGBE_EVC_ADD(&stats->gptc, IXGBE_READ_REG(hw, IXGBE_GPTC) - total); 1739 IXGBE_EVC_ADD(&stats->mptc, IXGBE_READ_REG(hw, IXGBE_MPTC) - total); 1740 IXGBE_EVC_ADD(&stats->ptc64, IXGBE_READ_REG(hw, IXGBE_PTC64) - total); 1741 1742 IXGBE_EVC_REGADD2(hw, stats, IXGBE_RUC, ruc); 1743 IXGBE_EVC_REGADD2(hw, stats, IXGBE_RFC, rfc); 1744 IXGBE_EVC_REGADD2(hw, stats, IXGBE_ROC, roc); 1745 IXGBE_EVC_REGADD2(hw, stats, IXGBE_RJC, rjc); 1746 1747 IXGBE_EVC_REGADD(hw, stats, IXGBE_MNGPRC, mngprc); 1748 IXGBE_EVC_REGADD(hw, stats, IXGBE_MNGPDC, mngpdc); 1749 IXGBE_EVC_REGADD(hw, stats, IXGBE_MNGPTC, mngptc); 1750 IXGBE_EVC_REGADD(hw, stats, IXGBE_TPR, tpr); 1751 IXGBE_EVC_REGADD(hw, stats, IXGBE_TPT, tpt); 1752 IXGBE_EVC_REGADD(hw, stats, IXGBE_PTC127, ptc127); 1753 IXGBE_EVC_REGADD(hw, stats, IXGBE_PTC255, ptc255); 1754 IXGBE_EVC_REGADD(hw, stats, IXGBE_PTC511, ptc511); 1755 IXGBE_EVC_REGADD(hw, stats, IXGBE_PTC1023, ptc1023); 1756 IXGBE_EVC_REGADD(hw, stats, IXGBE_PTC1522, ptc1522); 1757 IXGBE_EVC_REGADD(hw, stats, IXGBE_BPTC, bptc); 1758 IXGBE_EVC_REGADD(hw, stats, IXGBE_XEC, xec); 1759 IXGBE_EVC_REGADD(hw, stats, IXGBE_FCCRC, fccrc); 1760 IXGBE_EVC_REGADD(hw, stats, IXGBE_FCLAST, fclast); 1761 /* Only read FCOE on 82599 */ 1762 if (hw->mac.type != ixgbe_mac_82598EB) { 1763 IXGBE_EVC_REGADD(hw, stats, IXGBE_FCOERPDC, fcoerpdc); 1764 IXGBE_EVC_REGADD(hw, stats, IXGBE_FCOEPRC, fcoeprc); 1765 IXGBE_EVC_REGADD(hw, stats, IXGBE_FCOEPTC, fcoeptc); 1766 IXGBE_EVC_REGADD(hw, stats, IXGBE_FCOEDWRC, fcoedwrc); 1767 IXGBE_EVC_REGADD(hw, stats, IXGBE_FCOEDWTC, fcoedwtc); 1768 } 1769 1770 /* 1771 * Fill out the OS statistics structure. Only RX errors are required 1772 * here because all TX counters are incremented in the TX path and 1773 * normal RX counters are prepared in ether_input(). 1774 */ 1775 net_stat_ref_t nsr = IF_STAT_GETREF(ifp); 1776 if_statadd_ref(nsr, if_iqdrops, total_missed_rx); 1777 1778 /* 1779 * Aggregate following types of errors as RX errors: 1780 * - CRC error count, 1781 * - illegal byte error count, 1782 * - length error count, 1783 * - undersized packets count, 1784 * - fragmented packets count, 1785 * - oversized packets count, 1786 * - jabber count. 1787 */ 1788 if_statadd_ref(nsr, if_ierrors, 1789 crcerrs + illerrc + rlec + ruc + rfc + roc + rjc); 1790 1791 IF_STAT_PUTREF(ifp); 1792 } /* ixgbe_update_stats_counters */ 1793 1794 /************************************************************************ 1795 * ixgbe_add_hw_stats 1796 * 1797 * Add sysctl variables, one per statistic, to the system. 1798 ************************************************************************/ 1799 static void 1800 ixgbe_add_hw_stats(struct adapter *adapter) 1801 { 1802 device_t dev = adapter->dev; 1803 const struct sysctlnode *rnode, *cnode; 1804 struct sysctllog **log = &adapter->sysctllog; 1805 struct tx_ring *txr = adapter->tx_rings; 1806 struct rx_ring *rxr = adapter->rx_rings; 1807 struct ixgbe_hw *hw = &adapter->hw; 1808 struct ixgbe_hw_stats *stats = &adapter->stats.pf; 1809 const char *xname = device_xname(dev); 1810 int i; 1811 1812 /* Driver Statistics */ 1813 evcnt_attach_dynamic(&adapter->efbig_tx_dma_setup, EVCNT_TYPE_MISC, 1814 NULL, xname, "Driver tx dma soft fail EFBIG"); 1815 evcnt_attach_dynamic(&adapter->mbuf_defrag_failed, EVCNT_TYPE_MISC, 1816 NULL, xname, "m_defrag() failed"); 1817 evcnt_attach_dynamic(&adapter->efbig2_tx_dma_setup, EVCNT_TYPE_MISC, 1818 NULL, xname, "Driver tx dma hard fail EFBIG"); 1819 evcnt_attach_dynamic(&adapter->einval_tx_dma_setup, EVCNT_TYPE_MISC, 1820 NULL, xname, "Driver tx dma hard fail EINVAL"); 1821 evcnt_attach_dynamic(&adapter->other_tx_dma_setup, EVCNT_TYPE_MISC, 1822 NULL, xname, "Driver tx dma hard fail other"); 1823 evcnt_attach_dynamic(&adapter->eagain_tx_dma_setup, EVCNT_TYPE_MISC, 1824 NULL, xname, "Driver tx dma soft fail EAGAIN"); 1825 evcnt_attach_dynamic(&adapter->enomem_tx_dma_setup, EVCNT_TYPE_MISC, 1826 NULL, xname, "Driver tx dma soft fail ENOMEM"); 1827 evcnt_attach_dynamic(&adapter->watchdog_events, EVCNT_TYPE_MISC, 1828 NULL, xname, "Watchdog timeouts"); 1829 evcnt_attach_dynamic(&adapter->tso_err, EVCNT_TYPE_MISC, 1830 NULL, xname, "TSO errors"); 1831 evcnt_attach_dynamic(&adapter->admin_irqev, EVCNT_TYPE_INTR, 1832 NULL, xname, "Admin MSI-X IRQ Handled"); 1833 evcnt_attach_dynamic(&adapter->link_workev, EVCNT_TYPE_INTR, 1834 NULL, xname, "Link event"); 1835 evcnt_attach_dynamic(&adapter->mod_workev, EVCNT_TYPE_INTR, 1836 NULL, xname, "SFP+ module event"); 1837 evcnt_attach_dynamic(&adapter->msf_workev, EVCNT_TYPE_INTR, 1838 NULL, xname, "Multispeed event"); 1839 evcnt_attach_dynamic(&adapter->phy_workev, EVCNT_TYPE_INTR, 1840 NULL, xname, "External PHY event"); 1841 1842 /* Max number of traffic class is 8 */ 1843 KASSERT(IXGBE_DCB_MAX_TRAFFIC_CLASS == 8); 1844 for (i = 0; i < IXGBE_TC_COUNTER_NUM; i++) { 1845 snprintf(adapter->tcs[i].evnamebuf, 1846 sizeof(adapter->tcs[i].evnamebuf), "%s tc%d", 1847 xname, i); 1848 if (i < __arraycount(stats->mpc)) { 1849 evcnt_attach_dynamic(&stats->mpc[i], 1850 EVCNT_TYPE_MISC, NULL, adapter->tcs[i].evnamebuf, 1851 "RX Missed Packet Count"); 1852 if (hw->mac.type == ixgbe_mac_82598EB) 1853 evcnt_attach_dynamic(&stats->rnbc[i], 1854 EVCNT_TYPE_MISC, NULL, 1855 adapter->tcs[i].evnamebuf, 1856 "Receive No Buffers"); 1857 } 1858 if (i < __arraycount(stats->pxontxc)) { 1859 evcnt_attach_dynamic(&stats->pxontxc[i], 1860 EVCNT_TYPE_MISC, NULL, adapter->tcs[i].evnamebuf, 1861 "pxontxc"); 1862 evcnt_attach_dynamic(&stats->pxonrxc[i], 1863 EVCNT_TYPE_MISC, NULL, adapter->tcs[i].evnamebuf, 1864 "pxonrxc"); 1865 evcnt_attach_dynamic(&stats->pxofftxc[i], 1866 EVCNT_TYPE_MISC, NULL, adapter->tcs[i].evnamebuf, 1867 "pxofftxc"); 1868 evcnt_attach_dynamic(&stats->pxoffrxc[i], 1869 EVCNT_TYPE_MISC, NULL, adapter->tcs[i].evnamebuf, 1870 "pxoffrxc"); 1871 if (hw->mac.type >= ixgbe_mac_82599EB) 1872 evcnt_attach_dynamic(&stats->pxon2offc[i], 1873 EVCNT_TYPE_MISC, NULL, 1874 adapter->tcs[i].evnamebuf, 1875 "pxon2offc"); 1876 } 1877 } 1878 1879 for (i = 0; i < adapter->num_queues; i++, rxr++, txr++) { 1880 #ifdef LRO 1881 struct lro_ctrl *lro = &rxr->lro; 1882 #endif /* LRO */ 1883 1884 snprintf(adapter->queues[i].evnamebuf, 1885 sizeof(adapter->queues[i].evnamebuf), "%s q%d", 1886 xname, i); 1887 snprintf(adapter->queues[i].namebuf, 1888 sizeof(adapter->queues[i].namebuf), "q%d", i); 1889 1890 if ((rnode = ixgbe_sysctl_instance(adapter)) == NULL) { 1891 aprint_error_dev(dev, "could not create sysctl root\n"); 1892 break; 1893 } 1894 1895 if (sysctl_createv(log, 0, &rnode, &rnode, 1896 0, CTLTYPE_NODE, 1897 adapter->queues[i].namebuf, SYSCTL_DESCR("Queue Name"), 1898 NULL, 0, NULL, 0, CTL_CREATE, CTL_EOL) != 0) 1899 break; 1900 1901 if (sysctl_createv(log, 0, &rnode, &cnode, 1902 CTLFLAG_READWRITE, CTLTYPE_INT, 1903 "interrupt_rate", SYSCTL_DESCR("Interrupt Rate"), 1904 ixgbe_sysctl_interrupt_rate_handler, 0, 1905 (void *)&adapter->queues[i], 0, CTL_CREATE, CTL_EOL) != 0) 1906 break; 1907 1908 if (sysctl_createv(log, 0, &rnode, &cnode, 1909 CTLFLAG_READONLY, CTLTYPE_INT, 1910 "txd_head", SYSCTL_DESCR("Transmit Descriptor Head"), 1911 ixgbe_sysctl_tdh_handler, 0, (void *)txr, 1912 0, CTL_CREATE, CTL_EOL) != 0) 1913 break; 1914 1915 if (sysctl_createv(log, 0, &rnode, &cnode, 1916 CTLFLAG_READONLY, CTLTYPE_INT, 1917 "txd_tail", SYSCTL_DESCR("Transmit Descriptor Tail"), 1918 ixgbe_sysctl_tdt_handler, 0, (void *)txr, 1919 0, CTL_CREATE, CTL_EOL) != 0) 1920 break; 1921 1922 evcnt_attach_dynamic(&adapter->queues[i].irqs, EVCNT_TYPE_INTR, 1923 NULL, adapter->queues[i].evnamebuf, "IRQs on queue"); 1924 evcnt_attach_dynamic(&adapter->queues[i].handleq, 1925 EVCNT_TYPE_MISC, NULL, adapter->queues[i].evnamebuf, 1926 "Handled queue in softint"); 1927 evcnt_attach_dynamic(&adapter->queues[i].req, EVCNT_TYPE_MISC, 1928 NULL, adapter->queues[i].evnamebuf, "Requeued in softint"); 1929 evcnt_attach_dynamic(&txr->tso_tx, EVCNT_TYPE_MISC, 1930 NULL, adapter->queues[i].evnamebuf, "TSO"); 1931 evcnt_attach_dynamic(&txr->no_desc_avail, EVCNT_TYPE_MISC, 1932 NULL, adapter->queues[i].evnamebuf, 1933 "TX Queue No Descriptor Available"); 1934 evcnt_attach_dynamic(&txr->total_packets, EVCNT_TYPE_MISC, 1935 NULL, adapter->queues[i].evnamebuf, 1936 "Queue Packets Transmitted"); 1937 #ifndef IXGBE_LEGACY_TX 1938 evcnt_attach_dynamic(&txr->pcq_drops, EVCNT_TYPE_MISC, 1939 NULL, adapter->queues[i].evnamebuf, 1940 "Packets dropped in pcq"); 1941 #endif 1942 1943 if (sysctl_createv(log, 0, &rnode, &cnode, 1944 CTLFLAG_READONLY, CTLTYPE_INT, "rxd_nxck", 1945 SYSCTL_DESCR("Receive Descriptor next to check"), 1946 ixgbe_sysctl_next_to_check_handler, 0, (void *)rxr, 0, 1947 CTL_CREATE, CTL_EOL) != 0) 1948 break; 1949 1950 if (sysctl_createv(log, 0, &rnode, &cnode, 1951 CTLFLAG_READONLY, CTLTYPE_INT, "rxd_nxrf", 1952 SYSCTL_DESCR("Receive Descriptor next to refresh"), 1953 ixgbe_sysctl_next_to_refresh_handler, 0, (void *)rxr, 0, 1954 CTL_CREATE, CTL_EOL) != 0) 1955 break; 1956 1957 if (sysctl_createv(log, 0, &rnode, &cnode, 1958 CTLFLAG_READONLY, CTLTYPE_INT, "rxd_head", 1959 SYSCTL_DESCR("Receive Descriptor Head"), 1960 ixgbe_sysctl_rdh_handler, 0, (void *)rxr, 0, 1961 CTL_CREATE, CTL_EOL) != 0) 1962 break; 1963 1964 if (sysctl_createv(log, 0, &rnode, &cnode, 1965 CTLFLAG_READONLY, CTLTYPE_INT, "rxd_tail", 1966 SYSCTL_DESCR("Receive Descriptor Tail"), 1967 ixgbe_sysctl_rdt_handler, 0, (void *)rxr, 0, 1968 CTL_CREATE, CTL_EOL) != 0) 1969 break; 1970 1971 if (i < __arraycount(stats->qprc)) { 1972 evcnt_attach_dynamic(&stats->qprc[i], EVCNT_TYPE_MISC, 1973 NULL, adapter->queues[i].evnamebuf, "qprc"); 1974 evcnt_attach_dynamic(&stats->qptc[i], EVCNT_TYPE_MISC, 1975 NULL, adapter->queues[i].evnamebuf, "qptc"); 1976 evcnt_attach_dynamic(&stats->qbrc[i], EVCNT_TYPE_MISC, 1977 NULL, adapter->queues[i].evnamebuf, "qbrc"); 1978 evcnt_attach_dynamic(&stats->qbtc[i], EVCNT_TYPE_MISC, 1979 NULL, adapter->queues[i].evnamebuf, "qbtc"); 1980 if (hw->mac.type >= ixgbe_mac_82599EB) 1981 evcnt_attach_dynamic(&stats->qprdc[i], 1982 EVCNT_TYPE_MISC, NULL, 1983 adapter->queues[i].evnamebuf, "qprdc"); 1984 } 1985 1986 evcnt_attach_dynamic(&rxr->rx_packets, EVCNT_TYPE_MISC, 1987 NULL, adapter->queues[i].evnamebuf, 1988 "Queue Packets Received"); 1989 evcnt_attach_dynamic(&rxr->rx_bytes, EVCNT_TYPE_MISC, 1990 NULL, adapter->queues[i].evnamebuf, 1991 "Queue Bytes Received"); 1992 evcnt_attach_dynamic(&rxr->rx_copies, EVCNT_TYPE_MISC, 1993 NULL, adapter->queues[i].evnamebuf, "Copied RX Frames"); 1994 evcnt_attach_dynamic(&rxr->no_mbuf, EVCNT_TYPE_MISC, 1995 NULL, adapter->queues[i].evnamebuf, "Rx no mbuf"); 1996 evcnt_attach_dynamic(&rxr->rx_discarded, EVCNT_TYPE_MISC, 1997 NULL, adapter->queues[i].evnamebuf, "Rx discarded"); 1998 #ifdef LRO 1999 SYSCTL_ADD_INT(ctx, queue_list, OID_AUTO, "lro_queued", 2000 CTLFLAG_RD, &lro->lro_queued, 0, 2001 "LRO Queued"); 2002 SYSCTL_ADD_INT(ctx, queue_list, OID_AUTO, "lro_flushed", 2003 CTLFLAG_RD, &lro->lro_flushed, 0, 2004 "LRO Flushed"); 2005 #endif /* LRO */ 2006 } 2007 2008 /* MAC stats get their own sub node */ 2009 2010 snprintf(stats->namebuf, 2011 sizeof(stats->namebuf), "%s MAC Statistics", xname); 2012 2013 evcnt_attach_dynamic(&stats->ipcs, EVCNT_TYPE_MISC, NULL, 2014 stats->namebuf, "rx csum offload - IP"); 2015 evcnt_attach_dynamic(&stats->l4cs, EVCNT_TYPE_MISC, NULL, 2016 stats->namebuf, "rx csum offload - L4"); 2017 evcnt_attach_dynamic(&stats->ipcs_bad, EVCNT_TYPE_MISC, NULL, 2018 stats->namebuf, "rx csum offload - IP bad"); 2019 evcnt_attach_dynamic(&stats->l4cs_bad, EVCNT_TYPE_MISC, NULL, 2020 stats->namebuf, "rx csum offload - L4 bad"); 2021 evcnt_attach_dynamic(&stats->intzero, EVCNT_TYPE_MISC, NULL, 2022 stats->namebuf, "Interrupt conditions zero"); 2023 evcnt_attach_dynamic(&stats->legint, EVCNT_TYPE_MISC, NULL, 2024 stats->namebuf, "Legacy interrupts"); 2025 2026 evcnt_attach_dynamic(&stats->crcerrs, EVCNT_TYPE_MISC, NULL, 2027 stats->namebuf, "CRC Errors"); 2028 evcnt_attach_dynamic(&stats->illerrc, EVCNT_TYPE_MISC, NULL, 2029 stats->namebuf, "Illegal Byte Errors"); 2030 evcnt_attach_dynamic(&stats->errbc, EVCNT_TYPE_MISC, NULL, 2031 stats->namebuf, "Byte Errors"); 2032 evcnt_attach_dynamic(&stats->mspdc, EVCNT_TYPE_MISC, NULL, 2033 stats->namebuf, "MAC Short Packets Discarded"); 2034 if (hw->mac.type >= ixgbe_mac_X550) 2035 evcnt_attach_dynamic(&stats->mbsdc, EVCNT_TYPE_MISC, NULL, 2036 stats->namebuf, "Bad SFD"); 2037 evcnt_attach_dynamic(&stats->mpctotal, EVCNT_TYPE_MISC, NULL, 2038 stats->namebuf, "Total Packets Missed"); 2039 evcnt_attach_dynamic(&stats->mlfc, EVCNT_TYPE_MISC, NULL, 2040 stats->namebuf, "MAC Local Faults"); 2041 evcnt_attach_dynamic(&stats->mrfc, EVCNT_TYPE_MISC, NULL, 2042 stats->namebuf, "MAC Remote Faults"); 2043 evcnt_attach_dynamic(&stats->rlec, EVCNT_TYPE_MISC, NULL, 2044 stats->namebuf, "Receive Length Errors"); 2045 evcnt_attach_dynamic(&stats->lxontxc, EVCNT_TYPE_MISC, NULL, 2046 stats->namebuf, "Link XON Transmitted"); 2047 evcnt_attach_dynamic(&stats->lxonrxc, EVCNT_TYPE_MISC, NULL, 2048 stats->namebuf, "Link XON Received"); 2049 evcnt_attach_dynamic(&stats->lxofftxc, EVCNT_TYPE_MISC, NULL, 2050 stats->namebuf, "Link XOFF Transmitted"); 2051 evcnt_attach_dynamic(&stats->lxoffrxc, EVCNT_TYPE_MISC, NULL, 2052 stats->namebuf, "Link XOFF Received"); 2053 2054 /* Packet Reception Stats */ 2055 evcnt_attach_dynamic(&stats->tor, EVCNT_TYPE_MISC, NULL, 2056 stats->namebuf, "Total Octets Received"); 2057 evcnt_attach_dynamic(&stats->gorc, EVCNT_TYPE_MISC, NULL, 2058 stats->namebuf, "Good Octets Received"); 2059 evcnt_attach_dynamic(&stats->tpr, EVCNT_TYPE_MISC, NULL, 2060 stats->namebuf, "Total Packets Received"); 2061 evcnt_attach_dynamic(&stats->gprc, EVCNT_TYPE_MISC, NULL, 2062 stats->namebuf, "Good Packets Received"); 2063 evcnt_attach_dynamic(&stats->mprc, EVCNT_TYPE_MISC, NULL, 2064 stats->namebuf, "Multicast Packets Received"); 2065 evcnt_attach_dynamic(&stats->bprc, EVCNT_TYPE_MISC, NULL, 2066 stats->namebuf, "Broadcast Packets Received"); 2067 evcnt_attach_dynamic(&stats->prc64, EVCNT_TYPE_MISC, NULL, 2068 stats->namebuf, "64 byte frames received "); 2069 evcnt_attach_dynamic(&stats->prc127, EVCNT_TYPE_MISC, NULL, 2070 stats->namebuf, "65-127 byte frames received"); 2071 evcnt_attach_dynamic(&stats->prc255, EVCNT_TYPE_MISC, NULL, 2072 stats->namebuf, "128-255 byte frames received"); 2073 evcnt_attach_dynamic(&stats->prc511, EVCNT_TYPE_MISC, NULL, 2074 stats->namebuf, "256-511 byte frames received"); 2075 evcnt_attach_dynamic(&stats->prc1023, EVCNT_TYPE_MISC, NULL, 2076 stats->namebuf, "512-1023 byte frames received"); 2077 evcnt_attach_dynamic(&stats->prc1522, EVCNT_TYPE_MISC, NULL, 2078 stats->namebuf, "1023-1522 byte frames received"); 2079 evcnt_attach_dynamic(&stats->ruc, EVCNT_TYPE_MISC, NULL, 2080 stats->namebuf, "Receive Undersized"); 2081 evcnt_attach_dynamic(&stats->rfc, EVCNT_TYPE_MISC, NULL, 2082 stats->namebuf, "Fragmented Packets Received "); 2083 evcnt_attach_dynamic(&stats->roc, EVCNT_TYPE_MISC, NULL, 2084 stats->namebuf, "Oversized Packets Received"); 2085 evcnt_attach_dynamic(&stats->rjc, EVCNT_TYPE_MISC, NULL, 2086 stats->namebuf, "Received Jabber"); 2087 evcnt_attach_dynamic(&stats->mngprc, EVCNT_TYPE_MISC, NULL, 2088 stats->namebuf, "Management Packets Received"); 2089 evcnt_attach_dynamic(&stats->mngpdc, EVCNT_TYPE_MISC, NULL, 2090 stats->namebuf, "Management Packets Dropped"); 2091 evcnt_attach_dynamic(&stats->xec, EVCNT_TYPE_MISC, NULL, 2092 stats->namebuf, "Checksum Errors"); 2093 2094 /* Packet Transmission Stats */ 2095 evcnt_attach_dynamic(&stats->gotc, EVCNT_TYPE_MISC, NULL, 2096 stats->namebuf, "Good Octets Transmitted"); 2097 evcnt_attach_dynamic(&stats->tpt, EVCNT_TYPE_MISC, NULL, 2098 stats->namebuf, "Total Packets Transmitted"); 2099 evcnt_attach_dynamic(&stats->gptc, EVCNT_TYPE_MISC, NULL, 2100 stats->namebuf, "Good Packets Transmitted"); 2101 evcnt_attach_dynamic(&stats->bptc, EVCNT_TYPE_MISC, NULL, 2102 stats->namebuf, "Broadcast Packets Transmitted"); 2103 evcnt_attach_dynamic(&stats->mptc, EVCNT_TYPE_MISC, NULL, 2104 stats->namebuf, "Multicast Packets Transmitted"); 2105 evcnt_attach_dynamic(&stats->mngptc, EVCNT_TYPE_MISC, NULL, 2106 stats->namebuf, "Management Packets Transmitted"); 2107 evcnt_attach_dynamic(&stats->ptc64, EVCNT_TYPE_MISC, NULL, 2108 stats->namebuf, "64 byte frames transmitted "); 2109 evcnt_attach_dynamic(&stats->ptc127, EVCNT_TYPE_MISC, NULL, 2110 stats->namebuf, "65-127 byte frames transmitted"); 2111 evcnt_attach_dynamic(&stats->ptc255, EVCNT_TYPE_MISC, NULL, 2112 stats->namebuf, "128-255 byte frames transmitted"); 2113 evcnt_attach_dynamic(&stats->ptc511, EVCNT_TYPE_MISC, NULL, 2114 stats->namebuf, "256-511 byte frames transmitted"); 2115 evcnt_attach_dynamic(&stats->ptc1023, EVCNT_TYPE_MISC, NULL, 2116 stats->namebuf, "512-1023 byte frames transmitted"); 2117 evcnt_attach_dynamic(&stats->ptc1522, EVCNT_TYPE_MISC, NULL, 2118 stats->namebuf, "1024-1522 byte frames transmitted"); 2119 } /* ixgbe_add_hw_stats */ 2120 2121 static void 2122 ixgbe_clear_evcnt(struct adapter *adapter) 2123 { 2124 struct tx_ring *txr = adapter->tx_rings; 2125 struct rx_ring *rxr = adapter->rx_rings; 2126 struct ixgbe_hw *hw = &adapter->hw; 2127 struct ixgbe_hw_stats *stats = &adapter->stats.pf; 2128 int i; 2129 2130 IXGBE_EVC_STORE(&adapter->efbig_tx_dma_setup, 0); 2131 IXGBE_EVC_STORE(&adapter->mbuf_defrag_failed, 0); 2132 IXGBE_EVC_STORE(&adapter->efbig2_tx_dma_setup, 0); 2133 IXGBE_EVC_STORE(&adapter->einval_tx_dma_setup, 0); 2134 IXGBE_EVC_STORE(&adapter->other_tx_dma_setup, 0); 2135 IXGBE_EVC_STORE(&adapter->eagain_tx_dma_setup, 0); 2136 IXGBE_EVC_STORE(&adapter->enomem_tx_dma_setup, 0); 2137 IXGBE_EVC_STORE(&adapter->tso_err, 0); 2138 IXGBE_EVC_STORE(&adapter->watchdog_events, 0); 2139 IXGBE_EVC_STORE(&adapter->admin_irqev, 0); 2140 IXGBE_EVC_STORE(&adapter->link_workev, 0); 2141 IXGBE_EVC_STORE(&adapter->mod_workev, 0); 2142 IXGBE_EVC_STORE(&adapter->msf_workev, 0); 2143 IXGBE_EVC_STORE(&adapter->phy_workev, 0); 2144 2145 for (i = 0; i < IXGBE_TC_COUNTER_NUM; i++) { 2146 if (i < __arraycount(stats->mpc)) { 2147 IXGBE_EVC_STORE(&stats->mpc[i], 0); 2148 if (hw->mac.type == ixgbe_mac_82598EB) 2149 IXGBE_EVC_STORE(&stats->rnbc[i], 0); 2150 } 2151 if (i < __arraycount(stats->pxontxc)) { 2152 IXGBE_EVC_STORE(&stats->pxontxc[i], 0); 2153 IXGBE_EVC_STORE(&stats->pxonrxc[i], 0); 2154 IXGBE_EVC_STORE(&stats->pxofftxc[i], 0); 2155 IXGBE_EVC_STORE(&stats->pxoffrxc[i], 0); 2156 if (hw->mac.type >= ixgbe_mac_82599EB) 2157 IXGBE_EVC_STORE(&stats->pxon2offc[i], 0); 2158 } 2159 } 2160 2161 txr = adapter->tx_rings; 2162 for (i = 0; i < adapter->num_queues; i++, rxr++, txr++) { 2163 IXGBE_EVC_STORE(&adapter->queues[i].irqs, 0); 2164 IXGBE_EVC_STORE(&adapter->queues[i].handleq, 0); 2165 IXGBE_EVC_STORE(&adapter->queues[i].req, 0); 2166 IXGBE_EVC_STORE(&txr->no_desc_avail, 0); 2167 IXGBE_EVC_STORE(&txr->total_packets, 0); 2168 IXGBE_EVC_STORE(&txr->tso_tx, 0); 2169 #ifndef IXGBE_LEGACY_TX 2170 IXGBE_EVC_STORE(&txr->pcq_drops, 0); 2171 #endif 2172 txr->q_efbig_tx_dma_setup = 0; 2173 txr->q_mbuf_defrag_failed = 0; 2174 txr->q_efbig2_tx_dma_setup = 0; 2175 txr->q_einval_tx_dma_setup = 0; 2176 txr->q_other_tx_dma_setup = 0; 2177 txr->q_eagain_tx_dma_setup = 0; 2178 txr->q_enomem_tx_dma_setup = 0; 2179 txr->q_tso_err = 0; 2180 2181 if (i < __arraycount(stats->qprc)) { 2182 IXGBE_EVC_STORE(&stats->qprc[i], 0); 2183 IXGBE_EVC_STORE(&stats->qptc[i], 0); 2184 IXGBE_EVC_STORE(&stats->qbrc[i], 0); 2185 IXGBE_EVC_STORE(&stats->qbtc[i], 0); 2186 if (hw->mac.type >= ixgbe_mac_82599EB) 2187 IXGBE_EVC_STORE(&stats->qprdc[i], 0); 2188 } 2189 2190 IXGBE_EVC_STORE(&rxr->rx_packets, 0); 2191 IXGBE_EVC_STORE(&rxr->rx_bytes, 0); 2192 IXGBE_EVC_STORE(&rxr->rx_copies, 0); 2193 IXGBE_EVC_STORE(&rxr->no_mbuf, 0); 2194 IXGBE_EVC_STORE(&rxr->rx_discarded, 0); 2195 } 2196 IXGBE_EVC_STORE(&stats->ipcs, 0); 2197 IXGBE_EVC_STORE(&stats->l4cs, 0); 2198 IXGBE_EVC_STORE(&stats->ipcs_bad, 0); 2199 IXGBE_EVC_STORE(&stats->l4cs_bad, 0); 2200 IXGBE_EVC_STORE(&stats->intzero, 0); 2201 IXGBE_EVC_STORE(&stats->legint, 0); 2202 IXGBE_EVC_STORE(&stats->crcerrs, 0); 2203 IXGBE_EVC_STORE(&stats->illerrc, 0); 2204 IXGBE_EVC_STORE(&stats->errbc, 0); 2205 IXGBE_EVC_STORE(&stats->mspdc, 0); 2206 if (hw->mac.type >= ixgbe_mac_X550) 2207 IXGBE_EVC_STORE(&stats->mbsdc, 0); 2208 IXGBE_EVC_STORE(&stats->mpctotal, 0); 2209 IXGBE_EVC_STORE(&stats->mlfc, 0); 2210 IXGBE_EVC_STORE(&stats->mrfc, 0); 2211 IXGBE_EVC_STORE(&stats->rlec, 0); 2212 IXGBE_EVC_STORE(&stats->lxontxc, 0); 2213 IXGBE_EVC_STORE(&stats->lxonrxc, 0); 2214 IXGBE_EVC_STORE(&stats->lxofftxc, 0); 2215 IXGBE_EVC_STORE(&stats->lxoffrxc, 0); 2216 2217 /* Packet Reception Stats */ 2218 IXGBE_EVC_STORE(&stats->tor, 0); 2219 IXGBE_EVC_STORE(&stats->gorc, 0); 2220 IXGBE_EVC_STORE(&stats->tpr, 0); 2221 IXGBE_EVC_STORE(&stats->gprc, 0); 2222 IXGBE_EVC_STORE(&stats->mprc, 0); 2223 IXGBE_EVC_STORE(&stats->bprc, 0); 2224 IXGBE_EVC_STORE(&stats->prc64, 0); 2225 IXGBE_EVC_STORE(&stats->prc127, 0); 2226 IXGBE_EVC_STORE(&stats->prc255, 0); 2227 IXGBE_EVC_STORE(&stats->prc511, 0); 2228 IXGBE_EVC_STORE(&stats->prc1023, 0); 2229 IXGBE_EVC_STORE(&stats->prc1522, 0); 2230 IXGBE_EVC_STORE(&stats->ruc, 0); 2231 IXGBE_EVC_STORE(&stats->rfc, 0); 2232 IXGBE_EVC_STORE(&stats->roc, 0); 2233 IXGBE_EVC_STORE(&stats->rjc, 0); 2234 IXGBE_EVC_STORE(&stats->mngprc, 0); 2235 IXGBE_EVC_STORE(&stats->mngpdc, 0); 2236 IXGBE_EVC_STORE(&stats->xec, 0); 2237 2238 /* Packet Transmission Stats */ 2239 IXGBE_EVC_STORE(&stats->gotc, 0); 2240 IXGBE_EVC_STORE(&stats->tpt, 0); 2241 IXGBE_EVC_STORE(&stats->gptc, 0); 2242 IXGBE_EVC_STORE(&stats->bptc, 0); 2243 IXGBE_EVC_STORE(&stats->mptc, 0); 2244 IXGBE_EVC_STORE(&stats->mngptc, 0); 2245 IXGBE_EVC_STORE(&stats->ptc64, 0); 2246 IXGBE_EVC_STORE(&stats->ptc127, 0); 2247 IXGBE_EVC_STORE(&stats->ptc255, 0); 2248 IXGBE_EVC_STORE(&stats->ptc511, 0); 2249 IXGBE_EVC_STORE(&stats->ptc1023, 0); 2250 IXGBE_EVC_STORE(&stats->ptc1522, 0); 2251 } 2252 2253 /************************************************************************ 2254 * ixgbe_sysctl_tdh_handler - Transmit Descriptor Head handler function 2255 * 2256 * Retrieves the TDH value from the hardware 2257 ************************************************************************/ 2258 static int 2259 ixgbe_sysctl_tdh_handler(SYSCTLFN_ARGS) 2260 { 2261 struct sysctlnode node = *rnode; 2262 struct tx_ring *txr = (struct tx_ring *)node.sysctl_data; 2263 struct adapter *adapter; 2264 uint32_t val; 2265 2266 if (!txr) 2267 return (0); 2268 2269 adapter = txr->adapter; 2270 if (ixgbe_fw_recovery_mode_swflag(adapter)) 2271 return (EPERM); 2272 2273 val = IXGBE_READ_REG(&adapter->hw, IXGBE_TDH(txr->me)); 2274 node.sysctl_data = &val; 2275 return sysctl_lookup(SYSCTLFN_CALL(&node)); 2276 } /* ixgbe_sysctl_tdh_handler */ 2277 2278 /************************************************************************ 2279 * ixgbe_sysctl_tdt_handler - Transmit Descriptor Tail handler function 2280 * 2281 * Retrieves the TDT value from the hardware 2282 ************************************************************************/ 2283 static int 2284 ixgbe_sysctl_tdt_handler(SYSCTLFN_ARGS) 2285 { 2286 struct sysctlnode node = *rnode; 2287 struct tx_ring *txr = (struct tx_ring *)node.sysctl_data; 2288 struct adapter *adapter; 2289 uint32_t val; 2290 2291 if (!txr) 2292 return (0); 2293 2294 adapter = txr->adapter; 2295 if (ixgbe_fw_recovery_mode_swflag(adapter)) 2296 return (EPERM); 2297 2298 val = IXGBE_READ_REG(&adapter->hw, IXGBE_TDT(txr->me)); 2299 node.sysctl_data = &val; 2300 return sysctl_lookup(SYSCTLFN_CALL(&node)); 2301 } /* ixgbe_sysctl_tdt_handler */ 2302 2303 /************************************************************************ 2304 * ixgbe_sysctl_next_to_check_handler - Receive Descriptor next to check 2305 * handler function 2306 * 2307 * Retrieves the next_to_check value 2308 ************************************************************************/ 2309 static int 2310 ixgbe_sysctl_next_to_check_handler(SYSCTLFN_ARGS) 2311 { 2312 struct sysctlnode node = *rnode; 2313 struct rx_ring *rxr = (struct rx_ring *)node.sysctl_data; 2314 struct adapter *adapter; 2315 uint32_t val; 2316 2317 if (!rxr) 2318 return (0); 2319 2320 adapter = rxr->adapter; 2321 if (ixgbe_fw_recovery_mode_swflag(adapter)) 2322 return (EPERM); 2323 2324 val = rxr->next_to_check; 2325 node.sysctl_data = &val; 2326 return sysctl_lookup(SYSCTLFN_CALL(&node)); 2327 } /* ixgbe_sysctl_next_to_check_handler */ 2328 2329 /************************************************************************ 2330 * ixgbe_sysctl_next_to_refresh_handler - Receive Descriptor next to check 2331 * handler function 2332 * 2333 * Retrieves the next_to_refresh value 2334 ************************************************************************/ 2335 static int 2336 ixgbe_sysctl_next_to_refresh_handler(SYSCTLFN_ARGS) 2337 { 2338 struct sysctlnode node = *rnode; 2339 struct rx_ring *rxr = (struct rx_ring *)node.sysctl_data; 2340 struct adapter *adapter; 2341 uint32_t val; 2342 2343 if (!rxr) 2344 return (0); 2345 2346 adapter = rxr->adapter; 2347 if (ixgbe_fw_recovery_mode_swflag(adapter)) 2348 return (EPERM); 2349 2350 val = rxr->next_to_refresh; 2351 node.sysctl_data = &val; 2352 return sysctl_lookup(SYSCTLFN_CALL(&node)); 2353 } /* ixgbe_sysctl_next_to_refresh_handler */ 2354 2355 /************************************************************************ 2356 * ixgbe_sysctl_rdh_handler - Receive Descriptor Head handler function 2357 * 2358 * Retrieves the RDH value from the hardware 2359 ************************************************************************/ 2360 static int 2361 ixgbe_sysctl_rdh_handler(SYSCTLFN_ARGS) 2362 { 2363 struct sysctlnode node = *rnode; 2364 struct rx_ring *rxr = (struct rx_ring *)node.sysctl_data; 2365 struct adapter *adapter; 2366 uint32_t val; 2367 2368 if (!rxr) 2369 return (0); 2370 2371 adapter = rxr->adapter; 2372 if (ixgbe_fw_recovery_mode_swflag(adapter)) 2373 return (EPERM); 2374 2375 val = IXGBE_READ_REG(&adapter->hw, IXGBE_RDH(rxr->me)); 2376 node.sysctl_data = &val; 2377 return sysctl_lookup(SYSCTLFN_CALL(&node)); 2378 } /* ixgbe_sysctl_rdh_handler */ 2379 2380 /************************************************************************ 2381 * ixgbe_sysctl_rdt_handler - Receive Descriptor Tail handler function 2382 * 2383 * Retrieves the RDT value from the hardware 2384 ************************************************************************/ 2385 static int 2386 ixgbe_sysctl_rdt_handler(SYSCTLFN_ARGS) 2387 { 2388 struct sysctlnode node = *rnode; 2389 struct rx_ring *rxr = (struct rx_ring *)node.sysctl_data; 2390 struct adapter *adapter; 2391 uint32_t val; 2392 2393 if (!rxr) 2394 return (0); 2395 2396 adapter = rxr->adapter; 2397 if (ixgbe_fw_recovery_mode_swflag(adapter)) 2398 return (EPERM); 2399 2400 val = IXGBE_READ_REG(&adapter->hw, IXGBE_RDT(rxr->me)); 2401 node.sysctl_data = &val; 2402 return sysctl_lookup(SYSCTLFN_CALL(&node)); 2403 } /* ixgbe_sysctl_rdt_handler */ 2404 2405 static int 2406 ixgbe_vlan_cb(struct ethercom *ec, uint16_t vid, bool set) 2407 { 2408 struct ifnet *ifp = &ec->ec_if; 2409 struct adapter *adapter = ifp->if_softc; 2410 int rv; 2411 2412 if (set) 2413 rv = ixgbe_register_vlan(adapter, vid); 2414 else 2415 rv = ixgbe_unregister_vlan(adapter, vid); 2416 2417 if (rv != 0) 2418 return rv; 2419 2420 /* 2421 * Control VLAN HW tagging when ec_nvlan is changed from 1 to 0 2422 * or 0 to 1. 2423 */ 2424 if ((set && (ec->ec_nvlans == 1)) || (!set && (ec->ec_nvlans == 0))) 2425 ixgbe_setup_vlan_hw_tagging(adapter); 2426 2427 return rv; 2428 } 2429 2430 /************************************************************************ 2431 * ixgbe_register_vlan 2432 * 2433 * Run via vlan config EVENT, it enables us to use the 2434 * HW Filter table since we can get the vlan id. This 2435 * just creates the entry in the soft version of the 2436 * VFTA, init will repopulate the real table. 2437 ************************************************************************/ 2438 static int 2439 ixgbe_register_vlan(struct adapter *adapter, u16 vtag) 2440 { 2441 u16 index, bit; 2442 int error; 2443 2444 if ((vtag == 0) || (vtag > 4095)) /* Invalid */ 2445 return EINVAL; 2446 2447 IXGBE_CORE_LOCK(adapter); 2448 index = (vtag >> 5) & 0x7F; 2449 bit = vtag & 0x1F; 2450 adapter->shadow_vfta[index] |= ((u32)1 << bit); 2451 error = adapter->hw.mac.ops.set_vfta(&adapter->hw, vtag, 0, true, 2452 true); 2453 IXGBE_CORE_UNLOCK(adapter); 2454 if (error != 0) 2455 error = EACCES; 2456 2457 return error; 2458 } /* ixgbe_register_vlan */ 2459 2460 /************************************************************************ 2461 * ixgbe_unregister_vlan 2462 * 2463 * Run via vlan unconfig EVENT, remove our entry in the soft vfta. 2464 ************************************************************************/ 2465 static int 2466 ixgbe_unregister_vlan(struct adapter *adapter, u16 vtag) 2467 { 2468 u16 index, bit; 2469 int error; 2470 2471 if ((vtag == 0) || (vtag > 4095)) /* Invalid */ 2472 return EINVAL; 2473 2474 IXGBE_CORE_LOCK(adapter); 2475 index = (vtag >> 5) & 0x7F; 2476 bit = vtag & 0x1F; 2477 adapter->shadow_vfta[index] &= ~((u32)1 << bit); 2478 error = adapter->hw.mac.ops.set_vfta(&adapter->hw, vtag, 0, false, 2479 true); 2480 IXGBE_CORE_UNLOCK(adapter); 2481 if (error != 0) 2482 error = EACCES; 2483 2484 return error; 2485 } /* ixgbe_unregister_vlan */ 2486 2487 static void 2488 ixgbe_setup_vlan_hw_tagging(struct adapter *adapter) 2489 { 2490 struct ethercom *ec = &adapter->osdep.ec; 2491 struct ixgbe_hw *hw = &adapter->hw; 2492 struct rx_ring *rxr; 2493 u32 ctrl; 2494 int i; 2495 bool hwtagging; 2496 2497 /* Enable HW tagging only if any vlan is attached */ 2498 hwtagging = (ec->ec_capenable & ETHERCAP_VLAN_HWTAGGING) 2499 && VLAN_ATTACHED(ec); 2500 2501 /* Setup the queues for vlans */ 2502 for (i = 0; i < adapter->num_queues; i++) { 2503 rxr = &adapter->rx_rings[i]; 2504 /* 2505 * On 82599 and later, the VLAN enable is per/queue in RXDCTL. 2506 */ 2507 if (hw->mac.type != ixgbe_mac_82598EB) { 2508 ctrl = IXGBE_READ_REG(hw, IXGBE_RXDCTL(rxr->me)); 2509 if (hwtagging) 2510 ctrl |= IXGBE_RXDCTL_VME; 2511 else 2512 ctrl &= ~IXGBE_RXDCTL_VME; 2513 IXGBE_WRITE_REG(hw, IXGBE_RXDCTL(rxr->me), ctrl); 2514 } 2515 rxr->vtag_strip = hwtagging ? TRUE : FALSE; 2516 } 2517 2518 /* VLAN hw tagging for 82598 */ 2519 if (hw->mac.type == ixgbe_mac_82598EB) { 2520 ctrl = IXGBE_READ_REG(hw, IXGBE_VLNCTRL); 2521 if (hwtagging) 2522 ctrl |= IXGBE_VLNCTRL_VME; 2523 else 2524 ctrl &= ~IXGBE_VLNCTRL_VME; 2525 IXGBE_WRITE_REG(hw, IXGBE_VLNCTRL, ctrl); 2526 } 2527 } /* ixgbe_setup_vlan_hw_tagging */ 2528 2529 static void 2530 ixgbe_setup_vlan_hw_support(struct adapter *adapter) 2531 { 2532 struct ethercom *ec = &adapter->osdep.ec; 2533 struct ixgbe_hw *hw = &adapter->hw; 2534 int i; 2535 u32 ctrl; 2536 struct vlanid_list *vlanidp; 2537 2538 /* 2539 * This function is called from both if_init and ifflags_cb() 2540 * on NetBSD. 2541 */ 2542 2543 /* 2544 * Part 1: 2545 * Setup VLAN HW tagging 2546 */ 2547 ixgbe_setup_vlan_hw_tagging(adapter); 2548 2549 /* 2550 * Part 2: 2551 * Setup VLAN HW filter 2552 */ 2553 /* Cleanup shadow_vfta */ 2554 for (i = 0; i < IXGBE_VFTA_SIZE; i++) 2555 adapter->shadow_vfta[i] = 0; 2556 /* Generate shadow_vfta from ec_vids */ 2557 ETHER_LOCK(ec); 2558 SIMPLEQ_FOREACH(vlanidp, &ec->ec_vids, vid_list) { 2559 uint32_t idx; 2560 2561 idx = vlanidp->vid / 32; 2562 KASSERT(idx < IXGBE_VFTA_SIZE); 2563 adapter->shadow_vfta[idx] |= (u32)1 << (vlanidp->vid % 32); 2564 } 2565 ETHER_UNLOCK(ec); 2566 for (i = 0; i < IXGBE_VFTA_SIZE; i++) 2567 IXGBE_WRITE_REG(hw, IXGBE_VFTA(i), adapter->shadow_vfta[i]); 2568 2569 ctrl = IXGBE_READ_REG(hw, IXGBE_VLNCTRL); 2570 /* Enable the Filter Table if enabled */ 2571 if (ec->ec_capenable & ETHERCAP_VLAN_HWFILTER) 2572 ctrl |= IXGBE_VLNCTRL_VFE; 2573 else 2574 ctrl &= ~IXGBE_VLNCTRL_VFE; 2575 IXGBE_WRITE_REG(hw, IXGBE_VLNCTRL, ctrl); 2576 } /* ixgbe_setup_vlan_hw_support */ 2577 2578 /************************************************************************ 2579 * ixgbe_get_slot_info 2580 * 2581 * Get the width and transaction speed of 2582 * the slot this adapter is plugged into. 2583 ************************************************************************/ 2584 static void 2585 ixgbe_get_slot_info(struct adapter *adapter) 2586 { 2587 device_t dev = adapter->dev; 2588 struct ixgbe_hw *hw = &adapter->hw; 2589 u32 offset; 2590 u16 link; 2591 int bus_info_valid = TRUE; 2592 2593 /* Some devices are behind an internal bridge */ 2594 switch (hw->device_id) { 2595 case IXGBE_DEV_ID_82599_SFP_SF_QP: 2596 case IXGBE_DEV_ID_82599_QSFP_SF_QP: 2597 goto get_parent_info; 2598 default: 2599 break; 2600 } 2601 2602 ixgbe_get_bus_info(hw); 2603 2604 /* 2605 * Some devices don't use PCI-E, but there is no need 2606 * to display "Unknown" for bus speed and width. 2607 */ 2608 switch (hw->mac.type) { 2609 case ixgbe_mac_X550EM_x: 2610 case ixgbe_mac_X550EM_a: 2611 return; 2612 default: 2613 goto display; 2614 } 2615 2616 get_parent_info: 2617 /* 2618 * For the Quad port adapter we need to parse back 2619 * up the PCI tree to find the speed of the expansion 2620 * slot into which this adapter is plugged. A bit more work. 2621 */ 2622 dev = device_parent(device_parent(dev)); 2623 #if 0 2624 #ifdef IXGBE_DEBUG 2625 device_printf(dev, "parent pcib = %x,%x,%x\n", pci_get_bus(dev), 2626 pci_get_slot(dev), pci_get_function(dev)); 2627 #endif 2628 dev = device_parent(device_parent(dev)); 2629 #ifdef IXGBE_DEBUG 2630 device_printf(dev, "slot pcib = %x,%x,%x\n", pci_get_bus(dev), 2631 pci_get_slot(dev), pci_get_function(dev)); 2632 #endif 2633 #endif 2634 /* Now get the PCI Express Capabilities offset */ 2635 if (pci_get_capability(adapter->osdep.pc, adapter->osdep.tag, 2636 PCI_CAP_PCIEXPRESS, &offset, NULL)) { 2637 /* 2638 * Hmm...can't get PCI-Express capabilities. 2639 * Falling back to default method. 2640 */ 2641 bus_info_valid = FALSE; 2642 ixgbe_get_bus_info(hw); 2643 goto display; 2644 } 2645 /* ...and read the Link Status Register */ 2646 link = pci_conf_read(adapter->osdep.pc, adapter->osdep.tag, 2647 offset + PCIE_LCSR) >> 16; 2648 ixgbe_set_pci_config_data_generic(hw, link); 2649 2650 display: 2651 device_printf(dev, "PCI Express Bus: Speed %s Width %s\n", 2652 ((hw->bus.speed == ixgbe_bus_speed_8000) ? "8.0GT/s" : 2653 (hw->bus.speed == ixgbe_bus_speed_5000) ? "5.0GT/s" : 2654 (hw->bus.speed == ixgbe_bus_speed_2500) ? "2.5GT/s" : 2655 "Unknown"), 2656 ((hw->bus.width == ixgbe_bus_width_pcie_x8) ? "x8" : 2657 (hw->bus.width == ixgbe_bus_width_pcie_x4) ? "x4" : 2658 (hw->bus.width == ixgbe_bus_width_pcie_x1) ? "x1" : 2659 "Unknown")); 2660 2661 if (bus_info_valid) { 2662 if ((hw->device_id != IXGBE_DEV_ID_82599_SFP_SF_QP) && 2663 ((hw->bus.width <= ixgbe_bus_width_pcie_x4) && 2664 (hw->bus.speed == ixgbe_bus_speed_2500))) { 2665 device_printf(dev, "PCI-Express bandwidth available" 2666 " for this card\n is not sufficient for" 2667 " optimal performance.\n"); 2668 device_printf(dev, "For optimal performance a x8 " 2669 "PCIE, or x4 PCIE Gen2 slot is required.\n"); 2670 } 2671 if ((hw->device_id == IXGBE_DEV_ID_82599_SFP_SF_QP) && 2672 ((hw->bus.width <= ixgbe_bus_width_pcie_x8) && 2673 (hw->bus.speed < ixgbe_bus_speed_8000))) { 2674 device_printf(dev, "PCI-Express bandwidth available" 2675 " for this card\n is not sufficient for" 2676 " optimal performance.\n"); 2677 device_printf(dev, "For optimal performance a x8 " 2678 "PCIE Gen3 slot is required.\n"); 2679 } 2680 } else 2681 device_printf(dev, "Unable to determine slot speed/width. The speed/width reported are that of the internal switch.\n"); 2682 2683 return; 2684 } /* ixgbe_get_slot_info */ 2685 2686 /************************************************************************ 2687 * ixgbe_enable_queue - MSI-X Interrupt Handlers and Tasklets 2688 ************************************************************************/ 2689 static inline void 2690 ixgbe_enable_queue(struct adapter *adapter, u32 vector) 2691 { 2692 struct ixgbe_hw *hw = &adapter->hw; 2693 struct ix_queue *que = &adapter->queues[vector]; 2694 u64 queue = 1ULL << vector; 2695 u32 mask; 2696 2697 mutex_enter(&que->dc_mtx); 2698 if (que->disabled_count > 0 && --que->disabled_count > 0) 2699 goto out; 2700 2701 if (hw->mac.type == ixgbe_mac_82598EB) { 2702 mask = (IXGBE_EIMS_RTX_QUEUE & queue); 2703 IXGBE_WRITE_REG(hw, IXGBE_EIMS, mask); 2704 } else { 2705 mask = (queue & 0xFFFFFFFF); 2706 if (mask) 2707 IXGBE_WRITE_REG(hw, IXGBE_EIMS_EX(0), mask); 2708 mask = (queue >> 32); 2709 if (mask) 2710 IXGBE_WRITE_REG(hw, IXGBE_EIMS_EX(1), mask); 2711 } 2712 out: 2713 mutex_exit(&que->dc_mtx); 2714 } /* ixgbe_enable_queue */ 2715 2716 /************************************************************************ 2717 * ixgbe_disable_queue_internal 2718 ************************************************************************/ 2719 static inline void 2720 ixgbe_disable_queue_internal(struct adapter *adapter, u32 vector, bool nestok) 2721 { 2722 struct ixgbe_hw *hw = &adapter->hw; 2723 struct ix_queue *que = &adapter->queues[vector]; 2724 u64 queue = 1ULL << vector; 2725 u32 mask; 2726 2727 mutex_enter(&que->dc_mtx); 2728 2729 if (que->disabled_count > 0) { 2730 if (nestok) 2731 que->disabled_count++; 2732 goto out; 2733 } 2734 que->disabled_count++; 2735 2736 if (hw->mac.type == ixgbe_mac_82598EB) { 2737 mask = (IXGBE_EIMS_RTX_QUEUE & queue); 2738 IXGBE_WRITE_REG(hw, IXGBE_EIMC, mask); 2739 } else { 2740 mask = (queue & 0xFFFFFFFF); 2741 if (mask) 2742 IXGBE_WRITE_REG(hw, IXGBE_EIMC_EX(0), mask); 2743 mask = (queue >> 32); 2744 if (mask) 2745 IXGBE_WRITE_REG(hw, IXGBE_EIMC_EX(1), mask); 2746 } 2747 out: 2748 mutex_exit(&que->dc_mtx); 2749 } /* ixgbe_disable_queue_internal */ 2750 2751 /************************************************************************ 2752 * ixgbe_disable_queue 2753 ************************************************************************/ 2754 static inline void 2755 ixgbe_disable_queue(struct adapter *adapter, u32 vector) 2756 { 2757 2758 ixgbe_disable_queue_internal(adapter, vector, true); 2759 } /* ixgbe_disable_queue */ 2760 2761 /************************************************************************ 2762 * ixgbe_sched_handle_que - schedule deferred packet processing 2763 ************************************************************************/ 2764 static inline void 2765 ixgbe_sched_handle_que(struct adapter *adapter, struct ix_queue *que) 2766 { 2767 2768 if (que->txrx_use_workqueue) { 2769 /* 2770 * adapter->que_wq is bound to each CPU instead of 2771 * each NIC queue to reduce workqueue kthread. As we 2772 * should consider about interrupt affinity in this 2773 * function, the workqueue kthread must be WQ_PERCPU. 2774 * If create WQ_PERCPU workqueue kthread for each NIC 2775 * queue, that number of created workqueue kthread is 2776 * (number of used NIC queue) * (number of CPUs) = 2777 * (number of CPUs) ^ 2 most often. 2778 * 2779 * The same NIC queue's interrupts are avoided by 2780 * masking the queue's interrupt. And different 2781 * NIC queue's interrupts use different struct work 2782 * (que->wq_cookie). So, "enqueued flag" to avoid 2783 * twice workqueue_enqueue() is not required . 2784 */ 2785 workqueue_enqueue(adapter->que_wq, &que->wq_cookie, curcpu()); 2786 } else { 2787 softint_schedule(que->que_si); 2788 } 2789 } 2790 2791 /************************************************************************ 2792 * ixgbe_msix_que - MSI-X Queue Interrupt Service routine 2793 ************************************************************************/ 2794 static int 2795 ixgbe_msix_que(void *arg) 2796 { 2797 struct ix_queue *que = arg; 2798 struct adapter *adapter = que->adapter; 2799 struct ifnet *ifp = adapter->ifp; 2800 struct tx_ring *txr = que->txr; 2801 struct rx_ring *rxr = que->rxr; 2802 u32 newitr = 0; 2803 2804 /* Protect against spurious interrupts */ 2805 if ((ifp->if_flags & IFF_RUNNING) == 0) 2806 return 0; 2807 2808 ixgbe_disable_queue(adapter, que->msix); 2809 IXGBE_EVC_ADD(&que->irqs, 1); 2810 2811 /* 2812 * Don't change "que->txrx_use_workqueue" from this point to avoid 2813 * flip-flopping softint/workqueue mode in one deferred processing. 2814 */ 2815 que->txrx_use_workqueue = adapter->txrx_use_workqueue; 2816 2817 IXGBE_TX_LOCK(txr); 2818 ixgbe_txeof(txr); 2819 IXGBE_TX_UNLOCK(txr); 2820 2821 /* Do AIM now? */ 2822 2823 if (adapter->enable_aim == false) 2824 goto no_calc; 2825 /* 2826 * Do Adaptive Interrupt Moderation: 2827 * - Write out last calculated setting 2828 * - Calculate based on average size over 2829 * the last interval. 2830 */ 2831 if (que->eitr_setting) 2832 ixgbe_eitr_write(adapter, que->msix, que->eitr_setting); 2833 2834 que->eitr_setting = 0; 2835 2836 /* Idle, do nothing */ 2837 if ((txr->bytes == 0) && (rxr->bytes == 0)) 2838 goto no_calc; 2839 2840 if ((txr->bytes) && (txr->packets)) 2841 newitr = txr->bytes/txr->packets; 2842 if ((rxr->bytes) && (rxr->packets)) 2843 newitr = uimax(newitr, (rxr->bytes / rxr->packets)); 2844 newitr += 24; /* account for hardware frame, crc */ 2845 2846 /* set an upper boundary */ 2847 newitr = uimin(newitr, 3000); 2848 2849 /* Be nice to the mid range */ 2850 if ((newitr > 300) && (newitr < 1200)) 2851 newitr = (newitr / 3); 2852 else 2853 newitr = (newitr / 2); 2854 2855 /* 2856 * When RSC is used, ITR interval must be larger than RSC_DELAY. 2857 * Currently, we use 2us for RSC_DELAY. The minimum value is always 2858 * greater than 2us on 100M (and 10M?(not documented)), but it's not 2859 * on 1G and higher. 2860 */ 2861 if ((adapter->link_speed != IXGBE_LINK_SPEED_100_FULL) 2862 && (adapter->link_speed != IXGBE_LINK_SPEED_10_FULL)) { 2863 if (newitr < IXGBE_MIN_RSC_EITR_10G1G) 2864 newitr = IXGBE_MIN_RSC_EITR_10G1G; 2865 } 2866 2867 /* save for next interrupt */ 2868 que->eitr_setting = newitr; 2869 2870 /* Reset state */ 2871 txr->bytes = 0; 2872 txr->packets = 0; 2873 rxr->bytes = 0; 2874 rxr->packets = 0; 2875 2876 no_calc: 2877 ixgbe_sched_handle_que(adapter, que); 2878 2879 return 1; 2880 } /* ixgbe_msix_que */ 2881 2882 /************************************************************************ 2883 * ixgbe_media_status - Media Ioctl callback 2884 * 2885 * Called whenever the user queries the status of 2886 * the interface using ifconfig. 2887 ************************************************************************/ 2888 static void 2889 ixgbe_media_status(struct ifnet *ifp, struct ifmediareq *ifmr) 2890 { 2891 struct adapter *adapter = ifp->if_softc; 2892 struct ixgbe_hw *hw = &adapter->hw; 2893 int layer; 2894 2895 INIT_DEBUGOUT("ixgbe_media_status: begin"); 2896 ixgbe_update_link_status(adapter); 2897 2898 ifmr->ifm_status = IFM_AVALID; 2899 ifmr->ifm_active = IFM_ETHER; 2900 2901 if (adapter->link_active != LINK_STATE_UP) { 2902 ifmr->ifm_active |= IFM_NONE; 2903 return; 2904 } 2905 2906 ifmr->ifm_status |= IFM_ACTIVE; 2907 layer = adapter->phy_layer; 2908 2909 if (layer & IXGBE_PHYSICAL_LAYER_10GBASE_T || 2910 layer & IXGBE_PHYSICAL_LAYER_5GBASE_T || 2911 layer & IXGBE_PHYSICAL_LAYER_2500BASE_T || 2912 layer & IXGBE_PHYSICAL_LAYER_1000BASE_T || 2913 layer & IXGBE_PHYSICAL_LAYER_100BASE_TX || 2914 layer & IXGBE_PHYSICAL_LAYER_10BASE_T) 2915 switch (adapter->link_speed) { 2916 case IXGBE_LINK_SPEED_10GB_FULL: 2917 ifmr->ifm_active |= IFM_10G_T | IFM_FDX; 2918 break; 2919 case IXGBE_LINK_SPEED_5GB_FULL: 2920 ifmr->ifm_active |= IFM_5000_T | IFM_FDX; 2921 break; 2922 case IXGBE_LINK_SPEED_2_5GB_FULL: 2923 ifmr->ifm_active |= IFM_2500_T | IFM_FDX; 2924 break; 2925 case IXGBE_LINK_SPEED_1GB_FULL: 2926 ifmr->ifm_active |= IFM_1000_T | IFM_FDX; 2927 break; 2928 case IXGBE_LINK_SPEED_100_FULL: 2929 ifmr->ifm_active |= IFM_100_TX | IFM_FDX; 2930 break; 2931 case IXGBE_LINK_SPEED_10_FULL: 2932 ifmr->ifm_active |= IFM_10_T | IFM_FDX; 2933 break; 2934 } 2935 if (layer & IXGBE_PHYSICAL_LAYER_SFP_PLUS_CU || 2936 layer & IXGBE_PHYSICAL_LAYER_SFP_ACTIVE_DA) 2937 switch (adapter->link_speed) { 2938 case IXGBE_LINK_SPEED_10GB_FULL: 2939 ifmr->ifm_active |= IFM_10G_TWINAX | IFM_FDX; 2940 break; 2941 } 2942 if (layer & IXGBE_PHYSICAL_LAYER_10GBASE_LR) 2943 switch (adapter->link_speed) { 2944 case IXGBE_LINK_SPEED_10GB_FULL: 2945 ifmr->ifm_active |= IFM_10G_LR | IFM_FDX; 2946 break; 2947 case IXGBE_LINK_SPEED_1GB_FULL: 2948 ifmr->ifm_active |= IFM_1000_LX | IFM_FDX; 2949 break; 2950 } 2951 if (layer & IXGBE_PHYSICAL_LAYER_10GBASE_LRM) 2952 switch (adapter->link_speed) { 2953 case IXGBE_LINK_SPEED_10GB_FULL: 2954 ifmr->ifm_active |= IFM_10G_LRM | IFM_FDX; 2955 break; 2956 case IXGBE_LINK_SPEED_1GB_FULL: 2957 ifmr->ifm_active |= IFM_1000_LX | IFM_FDX; 2958 break; 2959 } 2960 if (layer & IXGBE_PHYSICAL_LAYER_10GBASE_SR || 2961 layer & IXGBE_PHYSICAL_LAYER_1000BASE_SX) 2962 switch (adapter->link_speed) { 2963 case IXGBE_LINK_SPEED_10GB_FULL: 2964 ifmr->ifm_active |= IFM_10G_SR | IFM_FDX; 2965 break; 2966 case IXGBE_LINK_SPEED_1GB_FULL: 2967 ifmr->ifm_active |= IFM_1000_SX | IFM_FDX; 2968 break; 2969 } 2970 if (layer & IXGBE_PHYSICAL_LAYER_10GBASE_CX4) 2971 switch (adapter->link_speed) { 2972 case IXGBE_LINK_SPEED_10GB_FULL: 2973 ifmr->ifm_active |= IFM_10G_CX4 | IFM_FDX; 2974 break; 2975 } 2976 /* 2977 * XXX: These need to use the proper media types once 2978 * they're added. 2979 */ 2980 if (layer & IXGBE_PHYSICAL_LAYER_10GBASE_KR) 2981 switch (adapter->link_speed) { 2982 case IXGBE_LINK_SPEED_10GB_FULL: 2983 ifmr->ifm_active |= IFM_10G_KR | IFM_FDX; 2984 break; 2985 case IXGBE_LINK_SPEED_2_5GB_FULL: 2986 ifmr->ifm_active |= IFM_2500_KX | IFM_FDX; 2987 break; 2988 case IXGBE_LINK_SPEED_1GB_FULL: 2989 ifmr->ifm_active |= IFM_1000_KX | IFM_FDX; 2990 break; 2991 } 2992 else if (layer & IXGBE_PHYSICAL_LAYER_10GBASE_KX4 || 2993 layer & IXGBE_PHYSICAL_LAYER_2500BASE_KX || 2994 layer & IXGBE_PHYSICAL_LAYER_1000BASE_KX) 2995 switch (adapter->link_speed) { 2996 case IXGBE_LINK_SPEED_10GB_FULL: 2997 ifmr->ifm_active |= IFM_10G_KX4 | IFM_FDX; 2998 break; 2999 case IXGBE_LINK_SPEED_2_5GB_FULL: 3000 ifmr->ifm_active |= IFM_2500_KX | IFM_FDX; 3001 break; 3002 case IXGBE_LINK_SPEED_1GB_FULL: 3003 ifmr->ifm_active |= IFM_1000_KX | IFM_FDX; 3004 break; 3005 } 3006 3007 /* If nothing is recognized... */ 3008 #if 0 3009 if (IFM_SUBTYPE(ifmr->ifm_active) == 0) 3010 ifmr->ifm_active |= IFM_UNKNOWN; 3011 #endif 3012 3013 ifp->if_baudrate = ifmedia_baudrate(ifmr->ifm_active); 3014 3015 /* Display current flow control setting used on link */ 3016 if (hw->fc.current_mode == ixgbe_fc_rx_pause || 3017 hw->fc.current_mode == ixgbe_fc_full) 3018 ifmr->ifm_active |= IFM_ETH_RXPAUSE; 3019 if (hw->fc.current_mode == ixgbe_fc_tx_pause || 3020 hw->fc.current_mode == ixgbe_fc_full) 3021 ifmr->ifm_active |= IFM_ETH_TXPAUSE; 3022 3023 return; 3024 } /* ixgbe_media_status */ 3025 3026 /************************************************************************ 3027 * ixgbe_media_change - Media Ioctl callback 3028 * 3029 * Called when the user changes speed/duplex using 3030 * media/mediopt option with ifconfig. 3031 ************************************************************************/ 3032 static int 3033 ixgbe_media_change(struct ifnet *ifp) 3034 { 3035 struct adapter *adapter = ifp->if_softc; 3036 struct ifmedia *ifm = &adapter->media; 3037 struct ixgbe_hw *hw = &adapter->hw; 3038 ixgbe_link_speed speed = 0; 3039 ixgbe_link_speed link_caps = 0; 3040 bool negotiate = false; 3041 s32 err = IXGBE_NOT_IMPLEMENTED; 3042 3043 INIT_DEBUGOUT("ixgbe_media_change: begin"); 3044 3045 if (IFM_TYPE(ifm->ifm_media) != IFM_ETHER) 3046 return (EINVAL); 3047 3048 if (hw->phy.media_type == ixgbe_media_type_backplane) 3049 return (EPERM); 3050 3051 /* 3052 * We don't actually need to check against the supported 3053 * media types of the adapter; ifmedia will take care of 3054 * that for us. 3055 */ 3056 switch (IFM_SUBTYPE(ifm->ifm_media)) { 3057 case IFM_AUTO: 3058 err = hw->mac.ops.get_link_capabilities(hw, &link_caps, 3059 &negotiate); 3060 if (err != IXGBE_SUCCESS) { 3061 device_printf(adapter->dev, "Unable to determine " 3062 "supported advertise speeds\n"); 3063 return (ENODEV); 3064 } 3065 speed |= link_caps; 3066 break; 3067 case IFM_10G_T: 3068 case IFM_10G_LRM: 3069 case IFM_10G_LR: 3070 case IFM_10G_TWINAX: 3071 case IFM_10G_SR: 3072 case IFM_10G_CX4: 3073 case IFM_10G_KR: 3074 case IFM_10G_KX4: 3075 speed |= IXGBE_LINK_SPEED_10GB_FULL; 3076 break; 3077 case IFM_5000_T: 3078 speed |= IXGBE_LINK_SPEED_5GB_FULL; 3079 break; 3080 case IFM_2500_T: 3081 case IFM_2500_KX: 3082 speed |= IXGBE_LINK_SPEED_2_5GB_FULL; 3083 break; 3084 case IFM_1000_T: 3085 case IFM_1000_LX: 3086 case IFM_1000_SX: 3087 case IFM_1000_KX: 3088 speed |= IXGBE_LINK_SPEED_1GB_FULL; 3089 break; 3090 case IFM_100_TX: 3091 speed |= IXGBE_LINK_SPEED_100_FULL; 3092 break; 3093 case IFM_10_T: 3094 speed |= IXGBE_LINK_SPEED_10_FULL; 3095 break; 3096 case IFM_NONE: 3097 break; 3098 default: 3099 goto invalid; 3100 } 3101 3102 hw->mac.autotry_restart = TRUE; 3103 hw->mac.ops.setup_link(hw, speed, TRUE); 3104 adapter->advertise = 0; 3105 if (IFM_SUBTYPE(ifm->ifm_media) != IFM_AUTO) { 3106 if ((speed & IXGBE_LINK_SPEED_10GB_FULL) != 0) 3107 adapter->advertise |= 1 << 2; 3108 if ((speed & IXGBE_LINK_SPEED_1GB_FULL) != 0) 3109 adapter->advertise |= 1 << 1; 3110 if ((speed & IXGBE_LINK_SPEED_100_FULL) != 0) 3111 adapter->advertise |= 1 << 0; 3112 if ((speed & IXGBE_LINK_SPEED_10_FULL) != 0) 3113 adapter->advertise |= 1 << 3; 3114 if ((speed & IXGBE_LINK_SPEED_2_5GB_FULL) != 0) 3115 adapter->advertise |= 1 << 4; 3116 if ((speed & IXGBE_LINK_SPEED_5GB_FULL) != 0) 3117 adapter->advertise |= 1 << 5; 3118 } 3119 3120 return (0); 3121 3122 invalid: 3123 device_printf(adapter->dev, "Invalid media type!\n"); 3124 3125 return (EINVAL); 3126 } /* ixgbe_media_change */ 3127 3128 /************************************************************************ 3129 * ixgbe_msix_admin - Link status change ISR (MSI/MSI-X) 3130 ************************************************************************/ 3131 static int 3132 ixgbe_msix_admin(void *arg) 3133 { 3134 struct adapter *adapter = arg; 3135 struct ixgbe_hw *hw = &adapter->hw; 3136 u32 eicr; 3137 u32 eims_orig; 3138 u32 eims_disable = 0; 3139 3140 IXGBE_EVC_ADD(&adapter->admin_irqev, 1); 3141 3142 eims_orig = IXGBE_READ_REG(hw, IXGBE_EIMS); 3143 /* Pause other interrupts */ 3144 IXGBE_WRITE_REG(hw, IXGBE_EIMC, IXGBE_MSIX_OTHER_CLEAR_MASK); 3145 3146 /* 3147 * First get the cause. 3148 * 3149 * The specifications of 82598, 82599, X540 and X550 say EICS register 3150 * is write only. However, Linux says it is a workaround for silicon 3151 * errata to read EICS instead of EICR to get interrupt cause. 3152 * At least, reading EICR clears lower 16bits of EIMS on 82598. 3153 */ 3154 eicr = IXGBE_READ_REG(hw, IXGBE_EICS); 3155 /* Be sure the queue bits are not cleared */ 3156 eicr &= ~IXGBE_EICR_RTX_QUEUE; 3157 /* Clear all OTHER interrupts with write */ 3158 IXGBE_WRITE_REG(hw, IXGBE_EICR, eicr); 3159 3160 ixgbe_intr_admin_common(adapter, eicr, &eims_disable); 3161 3162 /* Re-enable some OTHER interrupts */ 3163 IXGBE_WRITE_REG(hw, IXGBE_EIMS, eims_orig & ~eims_disable); 3164 3165 return 1; 3166 } /* ixgbe_msix_admin */ 3167 3168 static void 3169 ixgbe_intr_admin_common(struct adapter *adapter, u32 eicr, u32 *eims_disable) 3170 { 3171 struct ixgbe_hw *hw = &adapter->hw; 3172 u32 task_requests = 0; 3173 s32 retval; 3174 3175 /* Link status change */ 3176 if (eicr & IXGBE_EICR_LSC) { 3177 task_requests |= IXGBE_REQUEST_TASK_LSC; 3178 *eims_disable |= IXGBE_EIMS_LSC; 3179 } 3180 3181 if (ixgbe_is_sfp(hw)) { 3182 u32 eicr_mask; 3183 3184 /* Pluggable optics-related interrupt */ 3185 if (hw->mac.type >= ixgbe_mac_X540) 3186 eicr_mask = IXGBE_EICR_GPI_SDP0_X540; 3187 else 3188 eicr_mask = IXGBE_EICR_GPI_SDP2_BY_MAC(hw); 3189 3190 /* 3191 * An interrupt might not arrive when a module is inserted. 3192 * When an link status change interrupt occurred and the driver 3193 * still regard SFP as unplugged, issue the module softint 3194 * and then issue LSC interrupt. 3195 */ 3196 if ((eicr & eicr_mask) 3197 || ((hw->phy.sfp_type == ixgbe_sfp_type_not_present) 3198 && (eicr & IXGBE_EICR_LSC))) { 3199 task_requests |= IXGBE_REQUEST_TASK_MOD; 3200 *eims_disable |= IXGBE_EIMS_LSC; 3201 } 3202 3203 if ((hw->mac.type == ixgbe_mac_82599EB) && 3204 (eicr & IXGBE_EICR_GPI_SDP1_BY_MAC(hw))) { 3205 task_requests |= IXGBE_REQUEST_TASK_MSF; 3206 *eims_disable |= IXGBE_EIMS_GPI_SDP1_BY_MAC(hw); 3207 } 3208 } 3209 3210 if (adapter->hw.mac.type != ixgbe_mac_82598EB) { 3211 #ifdef IXGBE_FDIR 3212 if ((adapter->feat_en & IXGBE_FEATURE_FDIR) && 3213 (eicr & IXGBE_EICR_FLOW_DIR)) { 3214 if (!atomic_cas_uint(&adapter->fdir_reinit, 0, 1)) { 3215 task_requests |= IXGBE_REQUEST_TASK_FDIR; 3216 /* Disable the interrupt */ 3217 *eims_disable |= IXGBE_EIMS_FLOW_DIR; 3218 } 3219 } 3220 #endif 3221 3222 if (eicr & IXGBE_EICR_ECC) { 3223 if (ratecheck(&adapter->lasterr_time, 3224 &ixgbe_errlog_intrvl)) 3225 device_printf(adapter->dev, 3226 "CRITICAL: ECC ERROR!! Please Reboot!!\n"); 3227 } 3228 3229 /* Check for over temp condition */ 3230 if (adapter->feat_en & IXGBE_FEATURE_TEMP_SENSOR) { 3231 switch (adapter->hw.mac.type) { 3232 case ixgbe_mac_X550EM_a: 3233 if (!(eicr & IXGBE_EICR_GPI_SDP0_X550EM_a)) 3234 break; 3235 retval = hw->phy.ops.check_overtemp(hw); 3236 if (retval != IXGBE_ERR_OVERTEMP) 3237 break; 3238 if (ratecheck(&adapter->lasterr_time, 3239 &ixgbe_errlog_intrvl)) { 3240 device_printf(adapter->dev, 3241 "CRITICAL: OVER TEMP!! " 3242 "PHY IS SHUT DOWN!!\n"); 3243 device_printf(adapter->dev, 3244 "System shutdown required!\n"); 3245 } 3246 break; 3247 default: 3248 if (!(eicr & IXGBE_EICR_TS)) 3249 break; 3250 retval = hw->phy.ops.check_overtemp(hw); 3251 if (retval != IXGBE_ERR_OVERTEMP) 3252 break; 3253 if (ratecheck(&adapter->lasterr_time, 3254 &ixgbe_errlog_intrvl)) { 3255 device_printf(adapter->dev, 3256 "CRITICAL: OVER TEMP!! " 3257 "PHY IS SHUT DOWN!!\n"); 3258 device_printf(adapter->dev, 3259 "System shutdown required!\n"); 3260 } 3261 break; 3262 } 3263 } 3264 3265 /* Check for VF message */ 3266 if ((adapter->feat_en & IXGBE_FEATURE_SRIOV) && 3267 (eicr & IXGBE_EICR_MAILBOX)) { 3268 task_requests |= IXGBE_REQUEST_TASK_MBX; 3269 *eims_disable |= IXGBE_EIMS_MAILBOX; 3270 } 3271 } 3272 3273 /* Check for fan failure */ 3274 if (adapter->feat_en & IXGBE_FEATURE_FAN_FAIL) 3275 ixgbe_check_fan_failure(adapter, eicr, true); 3276 3277 /* External PHY interrupt */ 3278 if ((hw->phy.type == ixgbe_phy_x550em_ext_t) && 3279 (eicr & IXGBE_EICR_GPI_SDP0_X540)) { 3280 task_requests |= IXGBE_REQUEST_TASK_PHY; 3281 *eims_disable |= IXGBE_EICR_GPI_SDP0_X540; 3282 } 3283 3284 if (task_requests != 0) { 3285 mutex_enter(&adapter->admin_mtx); 3286 adapter->task_requests |= task_requests; 3287 ixgbe_schedule_admin_tasklet(adapter); 3288 mutex_exit(&adapter->admin_mtx); 3289 } 3290 3291 } 3292 3293 static void 3294 ixgbe_eitr_write(struct adapter *adapter, uint32_t index, uint32_t itr) 3295 { 3296 3297 if (adapter->hw.mac.type == ixgbe_mac_82598EB) 3298 itr |= itr << 16; 3299 else 3300 itr |= IXGBE_EITR_CNT_WDIS; 3301 3302 IXGBE_WRITE_REG(&adapter->hw, IXGBE_EITR(index), itr); 3303 } 3304 3305 3306 /************************************************************************ 3307 * ixgbe_sysctl_interrupt_rate_handler 3308 ************************************************************************/ 3309 static int 3310 ixgbe_sysctl_interrupt_rate_handler(SYSCTLFN_ARGS) 3311 { 3312 struct sysctlnode node = *rnode; 3313 struct ix_queue *que = (struct ix_queue *)node.sysctl_data; 3314 struct adapter *adapter; 3315 uint32_t reg, usec, rate; 3316 int error; 3317 3318 if (que == NULL) 3319 return 0; 3320 3321 adapter = que->adapter; 3322 if (ixgbe_fw_recovery_mode_swflag(adapter)) 3323 return (EPERM); 3324 3325 reg = IXGBE_READ_REG(&adapter->hw, IXGBE_EITR(que->msix)); 3326 usec = ((reg & 0x0FF8) >> 3); 3327 if (usec > 0) 3328 rate = 500000 / usec; 3329 else 3330 rate = 0; 3331 node.sysctl_data = &rate; 3332 error = sysctl_lookup(SYSCTLFN_CALL(&node)); 3333 if (error || newp == NULL) 3334 return error; 3335 reg &= ~0xfff; /* default, no limitation */ 3336 if (rate > 0 && rate < 500000) { 3337 if (rate < 1000) 3338 rate = 1000; 3339 reg |= ((4000000 / rate) & 0xff8); 3340 /* 3341 * When RSC is used, ITR interval must be larger than 3342 * RSC_DELAY. Currently, we use 2us for RSC_DELAY. 3343 * The minimum value is always greater than 2us on 100M 3344 * (and 10M?(not documented)), but it's not on 1G and higher. 3345 */ 3346 if ((adapter->link_speed != IXGBE_LINK_SPEED_100_FULL) 3347 && (adapter->link_speed != IXGBE_LINK_SPEED_10_FULL)) { 3348 if ((adapter->num_queues > 1) 3349 && (reg < IXGBE_MIN_RSC_EITR_10G1G)) 3350 return EINVAL; 3351 } 3352 ixgbe_max_interrupt_rate = rate; 3353 } else 3354 ixgbe_max_interrupt_rate = 0; 3355 ixgbe_eitr_write(adapter, que->msix, reg); 3356 3357 return (0); 3358 } /* ixgbe_sysctl_interrupt_rate_handler */ 3359 3360 const struct sysctlnode * 3361 ixgbe_sysctl_instance(struct adapter *adapter) 3362 { 3363 const char *dvname; 3364 struct sysctllog **log; 3365 int rc; 3366 const struct sysctlnode *rnode; 3367 3368 if (adapter->sysctltop != NULL) 3369 return adapter->sysctltop; 3370 3371 log = &adapter->sysctllog; 3372 dvname = device_xname(adapter->dev); 3373 3374 if ((rc = sysctl_createv(log, 0, NULL, &rnode, 3375 0, CTLTYPE_NODE, dvname, 3376 SYSCTL_DESCR("ixgbe information and settings"), 3377 NULL, 0, NULL, 0, CTL_HW, CTL_CREATE, CTL_EOL)) != 0) 3378 goto err; 3379 3380 return rnode; 3381 err: 3382 device_printf(adapter->dev, 3383 "%s: sysctl_createv failed, rc = %d\n", __func__, rc); 3384 return NULL; 3385 } 3386 3387 /************************************************************************ 3388 * ixgbe_add_device_sysctls 3389 ************************************************************************/ 3390 static void 3391 ixgbe_add_device_sysctls(struct adapter *adapter) 3392 { 3393 device_t dev = adapter->dev; 3394 struct ixgbe_hw *hw = &adapter->hw; 3395 struct sysctllog **log; 3396 const struct sysctlnode *rnode, *cnode; 3397 3398 log = &adapter->sysctllog; 3399 3400 if ((rnode = ixgbe_sysctl_instance(adapter)) == NULL) { 3401 aprint_error_dev(dev, "could not create sysctl root\n"); 3402 return; 3403 } 3404 3405 if (sysctl_createv(log, 0, &rnode, &cnode, 3406 CTLFLAG_READWRITE, CTLTYPE_INT, 3407 "debug", SYSCTL_DESCR("Debug Info"), 3408 ixgbe_sysctl_debug, 0, (void *)adapter, 0, CTL_CREATE, CTL_EOL) 3409 != 0) 3410 aprint_error_dev(dev, "could not create sysctl\n"); 3411 3412 if (sysctl_createv(log, 0, &rnode, &cnode, 3413 CTLFLAG_READWRITE, CTLTYPE_INT, 3414 "rx_copy_len", SYSCTL_DESCR("RX Copy Length"), 3415 ixgbe_sysctl_rx_copy_len, 0, 3416 (void *)adapter, 0, CTL_CREATE, CTL_EOL) != 0) 3417 aprint_error_dev(dev, "could not create sysctl\n"); 3418 3419 if (sysctl_createv(log, 0, &rnode, &cnode, 3420 CTLFLAG_READONLY, CTLTYPE_INT, 3421 "num_tx_desc", SYSCTL_DESCR("Number of TX descriptors"), 3422 NULL, 0, &adapter->num_tx_desc, 0, CTL_CREATE, CTL_EOL) != 0) 3423 aprint_error_dev(dev, "could not create sysctl\n"); 3424 3425 if (sysctl_createv(log, 0, &rnode, &cnode, 3426 CTLFLAG_READONLY, CTLTYPE_INT, 3427 "num_rx_desc", SYSCTL_DESCR("Number of RX descriptors"), 3428 NULL, 0, &adapter->num_rx_desc, 0, CTL_CREATE, CTL_EOL) != 0) 3429 aprint_error_dev(dev, "could not create sysctl\n"); 3430 3431 if (sysctl_createv(log, 0, &rnode, &cnode, 3432 CTLFLAG_READWRITE, CTLTYPE_INT, "rx_process_limit", 3433 SYSCTL_DESCR("max number of RX packets to process"), 3434 ixgbe_sysctl_rx_process_limit, 0, (void *)adapter, 0, CTL_CREATE, 3435 CTL_EOL) != 0) 3436 aprint_error_dev(dev, "could not create sysctl\n"); 3437 3438 if (sysctl_createv(log, 0, &rnode, &cnode, 3439 CTLFLAG_READWRITE, CTLTYPE_INT, "tx_process_limit", 3440 SYSCTL_DESCR("max number of TX packets to process"), 3441 ixgbe_sysctl_tx_process_limit, 0, (void *)adapter, 0, CTL_CREATE, 3442 CTL_EOL) != 0) 3443 aprint_error_dev(dev, "could not create sysctl\n"); 3444 3445 if (sysctl_createv(log, 0, &rnode, &cnode, 3446 CTLFLAG_READONLY, CTLTYPE_INT, 3447 "num_queues", SYSCTL_DESCR("Number of queues"), 3448 NULL, 0, &adapter->num_queues, 0, CTL_CREATE, CTL_EOL) != 0) 3449 aprint_error_dev(dev, "could not create sysctl\n"); 3450 3451 /* Sysctls for all devices */ 3452 if (sysctl_createv(log, 0, &rnode, &cnode, CTLFLAG_READWRITE, 3453 CTLTYPE_INT, "fc", SYSCTL_DESCR(IXGBE_SYSCTL_DESC_SET_FC), 3454 ixgbe_sysctl_flowcntl, 0, (void *)adapter, 0, CTL_CREATE, 3455 CTL_EOL) != 0) 3456 aprint_error_dev(dev, "could not create sysctl\n"); 3457 3458 adapter->enable_aim = ixgbe_enable_aim; 3459 if (sysctl_createv(log, 0, &rnode, &cnode, CTLFLAG_READWRITE, 3460 CTLTYPE_BOOL, "enable_aim", SYSCTL_DESCR("Interrupt Moderation"), 3461 NULL, 0, &adapter->enable_aim, 0, CTL_CREATE, CTL_EOL) != 0) 3462 aprint_error_dev(dev, "could not create sysctl\n"); 3463 3464 if (sysctl_createv(log, 0, &rnode, &cnode, 3465 CTLFLAG_READWRITE, CTLTYPE_INT, 3466 "advertise_speed", SYSCTL_DESCR(IXGBE_SYSCTL_DESC_ADV_SPEED), 3467 ixgbe_sysctl_advertise, 0, (void *)adapter, 0, CTL_CREATE, 3468 CTL_EOL) != 0) 3469 aprint_error_dev(dev, "could not create sysctl\n"); 3470 3471 /* 3472 * If each "que->txrx_use_workqueue" is changed in sysctl handler, 3473 * it causesflip-flopping softint/workqueue mode in one deferred 3474 * processing. Therefore, preempt_disable()/preempt_enable() are 3475 * required in ixgbe_sched_handle_que() to avoid 3476 * KASSERT(ixgbe_sched_handle_que()) in softint_schedule(). 3477 * I think changing "que->txrx_use_workqueue" in interrupt handler 3478 * is lighter than doing preempt_disable()/preempt_enable() in every 3479 * ixgbe_sched_handle_que(). 3480 */ 3481 adapter->txrx_use_workqueue = ixgbe_txrx_workqueue; 3482 if (sysctl_createv(log, 0, &rnode, &cnode, CTLFLAG_READWRITE, 3483 CTLTYPE_BOOL, "txrx_workqueue", 3484 SYSCTL_DESCR("Use workqueue for packet processing"), 3485 NULL, 0, &adapter->txrx_use_workqueue, 0, CTL_CREATE, 3486 CTL_EOL) != 0) 3487 aprint_error_dev(dev, "could not create sysctl\n"); 3488 3489 #ifdef IXGBE_DEBUG 3490 /* testing sysctls (for all devices) */ 3491 if (sysctl_createv(log, 0, &rnode, &cnode, CTLFLAG_READWRITE, 3492 CTLTYPE_INT, "power_state", SYSCTL_DESCR("PCI Power State"), 3493 ixgbe_sysctl_power_state, 0, (void *)adapter, 0, CTL_CREATE, 3494 CTL_EOL) != 0) 3495 aprint_error_dev(dev, "could not create sysctl\n"); 3496 3497 if (sysctl_createv(log, 0, &rnode, &cnode, CTLFLAG_READONLY, 3498 CTLTYPE_STRING, "print_rss_config", 3499 SYSCTL_DESCR("Prints RSS Configuration"), 3500 ixgbe_sysctl_print_rss_config, 0, (void *)adapter, 0, CTL_CREATE, 3501 CTL_EOL) != 0) 3502 aprint_error_dev(dev, "could not create sysctl\n"); 3503 #endif 3504 /* for X550 series devices */ 3505 if (hw->mac.type >= ixgbe_mac_X550) 3506 if (sysctl_createv(log, 0, &rnode, &cnode, CTLFLAG_READWRITE, 3507 CTLTYPE_INT, "dmac", SYSCTL_DESCR("DMA Coalesce"), 3508 ixgbe_sysctl_dmac, 0, (void *)adapter, 0, CTL_CREATE, 3509 CTL_EOL) != 0) 3510 aprint_error_dev(dev, "could not create sysctl\n"); 3511 3512 /* for WoL-capable devices */ 3513 if (adapter->wol_support) { 3514 if (sysctl_createv(log, 0, &rnode, &cnode, CTLFLAG_READWRITE, 3515 CTLTYPE_BOOL, "wol_enable", 3516 SYSCTL_DESCR("Enable/Disable Wake on LAN"), 3517 ixgbe_sysctl_wol_enable, 0, (void *)adapter, 0, CTL_CREATE, 3518 CTL_EOL) != 0) 3519 aprint_error_dev(dev, "could not create sysctl\n"); 3520 3521 if (sysctl_createv(log, 0, &rnode, &cnode, CTLFLAG_READWRITE, 3522 CTLTYPE_INT, "wufc", 3523 SYSCTL_DESCR("Enable/Disable Wake Up Filters"), 3524 ixgbe_sysctl_wufc, 0, (void *)adapter, 0, CTL_CREATE, 3525 CTL_EOL) != 0) 3526 aprint_error_dev(dev, "could not create sysctl\n"); 3527 } 3528 3529 /* for X552/X557-AT devices */ 3530 if (hw->device_id == IXGBE_DEV_ID_X550EM_X_10G_T) { 3531 const struct sysctlnode *phy_node; 3532 3533 if (sysctl_createv(log, 0, &rnode, &phy_node, 0, CTLTYPE_NODE, 3534 "phy", SYSCTL_DESCR("External PHY sysctls"), 3535 NULL, 0, NULL, 0, CTL_CREATE, CTL_EOL) != 0) { 3536 aprint_error_dev(dev, "could not create sysctl\n"); 3537 return; 3538 } 3539 3540 if (sysctl_createv(log, 0, &phy_node, &cnode, CTLFLAG_READONLY, 3541 CTLTYPE_INT, "temp", 3542 SYSCTL_DESCR("Current External PHY Temperature (Celsius)"), 3543 ixgbe_sysctl_phy_temp, 0, (void *)adapter, 0, CTL_CREATE, 3544 CTL_EOL) != 0) 3545 aprint_error_dev(dev, "could not create sysctl\n"); 3546 3547 if (sysctl_createv(log, 0, &phy_node, &cnode, CTLFLAG_READONLY, 3548 CTLTYPE_INT, "overtemp_occurred", 3549 SYSCTL_DESCR( 3550 "External PHY High Temperature Event Occurred"), 3551 ixgbe_sysctl_phy_overtemp_occurred, 0, (void *)adapter, 0, 3552 CTL_CREATE, CTL_EOL) != 0) 3553 aprint_error_dev(dev, "could not create sysctl\n"); 3554 } 3555 3556 if ((hw->mac.type == ixgbe_mac_X550EM_a) 3557 && (hw->phy.type == ixgbe_phy_fw)) 3558 if (sysctl_createv(log, 0, &rnode, &cnode, CTLFLAG_READWRITE, 3559 CTLTYPE_BOOL, "force_10_100_autonego", 3560 SYSCTL_DESCR("Force autonego on 10M and 100M"), 3561 NULL, 0, &hw->phy.force_10_100_autonego, 0, 3562 CTL_CREATE, CTL_EOL) != 0) 3563 aprint_error_dev(dev, "could not create sysctl\n"); 3564 3565 if (adapter->feat_cap & IXGBE_FEATURE_EEE) { 3566 if (sysctl_createv(log, 0, &rnode, &cnode, CTLFLAG_READWRITE, 3567 CTLTYPE_INT, "eee_state", 3568 SYSCTL_DESCR("EEE Power Save State"), 3569 ixgbe_sysctl_eee_state, 0, (void *)adapter, 0, CTL_CREATE, 3570 CTL_EOL) != 0) 3571 aprint_error_dev(dev, "could not create sysctl\n"); 3572 } 3573 } /* ixgbe_add_device_sysctls */ 3574 3575 /************************************************************************ 3576 * ixgbe_allocate_pci_resources 3577 ************************************************************************/ 3578 static int 3579 ixgbe_allocate_pci_resources(struct adapter *adapter, 3580 const struct pci_attach_args *pa) 3581 { 3582 pcireg_t memtype, csr; 3583 device_t dev = adapter->dev; 3584 bus_addr_t addr; 3585 int flags; 3586 3587 memtype = pci_mapreg_type(pa->pa_pc, pa->pa_tag, PCI_BAR(0)); 3588 switch (memtype) { 3589 case PCI_MAPREG_TYPE_MEM | PCI_MAPREG_MEM_TYPE_32BIT: 3590 case PCI_MAPREG_TYPE_MEM | PCI_MAPREG_MEM_TYPE_64BIT: 3591 adapter->osdep.mem_bus_space_tag = pa->pa_memt; 3592 if (pci_mapreg_info(pa->pa_pc, pa->pa_tag, PCI_BAR(0), 3593 memtype, &addr, &adapter->osdep.mem_size, &flags) != 0) 3594 goto map_err; 3595 if ((flags & BUS_SPACE_MAP_PREFETCHABLE) != 0) { 3596 aprint_normal_dev(dev, "clearing prefetchable bit\n"); 3597 flags &= ~BUS_SPACE_MAP_PREFETCHABLE; 3598 } 3599 if (bus_space_map(adapter->osdep.mem_bus_space_tag, addr, 3600 adapter->osdep.mem_size, flags, 3601 &adapter->osdep.mem_bus_space_handle) != 0) { 3602 map_err: 3603 adapter->osdep.mem_size = 0; 3604 aprint_error_dev(dev, "unable to map BAR0\n"); 3605 return ENXIO; 3606 } 3607 /* 3608 * Enable address decoding for memory range in case BIOS or 3609 * UEFI don't set it. 3610 */ 3611 csr = pci_conf_read(pa->pa_pc, pa->pa_tag, 3612 PCI_COMMAND_STATUS_REG); 3613 csr |= PCI_COMMAND_MEM_ENABLE; 3614 pci_conf_write(pa->pa_pc, pa->pa_tag, PCI_COMMAND_STATUS_REG, 3615 csr); 3616 break; 3617 default: 3618 aprint_error_dev(dev, "unexpected type on BAR0\n"); 3619 return ENXIO; 3620 } 3621 3622 return (0); 3623 } /* ixgbe_allocate_pci_resources */ 3624 3625 static void 3626 ixgbe_free_deferred_handlers(struct adapter *adapter) 3627 { 3628 struct ix_queue *que = adapter->queues; 3629 struct tx_ring *txr = adapter->tx_rings; 3630 int i; 3631 3632 for (i = 0; i < adapter->num_queues; i++, que++, txr++) { 3633 if (!(adapter->feat_en & IXGBE_FEATURE_LEGACY_TX)) { 3634 if (txr->txr_si != NULL) 3635 softint_disestablish(txr->txr_si); 3636 } 3637 if (que->que_si != NULL) 3638 softint_disestablish(que->que_si); 3639 } 3640 if (adapter->txr_wq != NULL) 3641 workqueue_destroy(adapter->txr_wq); 3642 if (adapter->txr_wq_enqueued != NULL) 3643 percpu_free(adapter->txr_wq_enqueued, sizeof(u_int)); 3644 if (adapter->que_wq != NULL) 3645 workqueue_destroy(adapter->que_wq); 3646 3647 if (adapter->admin_wq != NULL) { 3648 workqueue_destroy(adapter->admin_wq); 3649 adapter->admin_wq = NULL; 3650 } 3651 if (adapter->timer_wq != NULL) { 3652 workqueue_destroy(adapter->timer_wq); 3653 adapter->timer_wq = NULL; 3654 } 3655 if (adapter->recovery_mode_timer_wq != NULL) { 3656 /* 3657 * ixgbe_ifstop() doesn't call the workqueue_wait() for 3658 * the recovery_mode_timer workqueue, so call it here. 3659 */ 3660 workqueue_wait(adapter->recovery_mode_timer_wq, 3661 &adapter->recovery_mode_timer_wc); 3662 atomic_store_relaxed(&adapter->recovery_mode_timer_pending, 0); 3663 workqueue_destroy(adapter->recovery_mode_timer_wq); 3664 adapter->recovery_mode_timer_wq = NULL; 3665 } 3666 } /* ixgbe_free_deferred_handlers */ 3667 3668 /************************************************************************ 3669 * ixgbe_detach - Device removal routine 3670 * 3671 * Called when the driver is being removed. 3672 * Stops the adapter and deallocates all the resources 3673 * that were allocated for driver operation. 3674 * 3675 * return 0 on success, positive on failure 3676 ************************************************************************/ 3677 static int 3678 ixgbe_detach(device_t dev, int flags) 3679 { 3680 struct adapter *adapter = device_private(dev); 3681 struct rx_ring *rxr = adapter->rx_rings; 3682 struct tx_ring *txr = adapter->tx_rings; 3683 struct ixgbe_hw *hw = &adapter->hw; 3684 struct ixgbe_hw_stats *stats = &adapter->stats.pf; 3685 u32 ctrl_ext; 3686 int i; 3687 3688 INIT_DEBUGOUT("ixgbe_detach: begin"); 3689 if (adapter->osdep.attached == false) 3690 return 0; 3691 3692 if (ixgbe_pci_iov_detach(dev) != 0) { 3693 device_printf(dev, "SR-IOV in use; detach first.\n"); 3694 return (EBUSY); 3695 } 3696 3697 if (VLAN_ATTACHED(&adapter->osdep.ec) && 3698 (flags & (DETACH_SHUTDOWN | DETACH_FORCE)) == 0) { 3699 aprint_error_dev(dev, "VLANs in use, detach first\n"); 3700 return (EBUSY); 3701 } 3702 3703 ether_ifdetach(adapter->ifp); 3704 3705 adapter->osdep.detaching = true; 3706 /* 3707 * Stop the interface. ixgbe_setup_low_power_mode() calls 3708 * ixgbe_ifstop(), so it's not required to call ixgbe_ifstop() 3709 * directly. 3710 */ 3711 ixgbe_setup_low_power_mode(adapter); 3712 3713 callout_halt(&adapter->timer, NULL); 3714 if (adapter->feat_en & IXGBE_FEATURE_RECOVERY_MODE) 3715 callout_halt(&adapter->recovery_mode_timer, NULL); 3716 3717 workqueue_wait(adapter->admin_wq, &adapter->admin_wc); 3718 atomic_store_relaxed(&adapter->admin_pending, 0); 3719 workqueue_wait(adapter->timer_wq, &adapter->timer_wc); 3720 atomic_store_relaxed(&adapter->timer_pending, 0); 3721 3722 pmf_device_deregister(dev); 3723 3724 ixgbe_free_deferred_handlers(adapter); 3725 3726 /* let hardware know driver is unloading */ 3727 ctrl_ext = IXGBE_READ_REG(&adapter->hw, IXGBE_CTRL_EXT); 3728 ctrl_ext &= ~IXGBE_CTRL_EXT_DRV_LOAD; 3729 IXGBE_WRITE_REG(&adapter->hw, IXGBE_CTRL_EXT, ctrl_ext); 3730 3731 if (adapter->feat_en & IXGBE_FEATURE_NETMAP) 3732 netmap_detach(adapter->ifp); 3733 3734 ixgbe_free_pci_resources(adapter); 3735 #if 0 /* XXX the NetBSD port is probably missing something here */ 3736 bus_generic_detach(dev); 3737 #endif 3738 if_detach(adapter->ifp); 3739 ifmedia_fini(&adapter->media); 3740 if_percpuq_destroy(adapter->ipq); 3741 3742 sysctl_teardown(&adapter->sysctllog); 3743 evcnt_detach(&adapter->efbig_tx_dma_setup); 3744 evcnt_detach(&adapter->mbuf_defrag_failed); 3745 evcnt_detach(&adapter->efbig2_tx_dma_setup); 3746 evcnt_detach(&adapter->einval_tx_dma_setup); 3747 evcnt_detach(&adapter->other_tx_dma_setup); 3748 evcnt_detach(&adapter->eagain_tx_dma_setup); 3749 evcnt_detach(&adapter->enomem_tx_dma_setup); 3750 evcnt_detach(&adapter->watchdog_events); 3751 evcnt_detach(&adapter->tso_err); 3752 evcnt_detach(&adapter->admin_irqev); 3753 evcnt_detach(&adapter->link_workev); 3754 evcnt_detach(&adapter->mod_workev); 3755 evcnt_detach(&adapter->msf_workev); 3756 evcnt_detach(&adapter->phy_workev); 3757 3758 for (i = 0; i < IXGBE_TC_COUNTER_NUM; i++) { 3759 if (i < __arraycount(stats->mpc)) { 3760 evcnt_detach(&stats->mpc[i]); 3761 if (hw->mac.type == ixgbe_mac_82598EB) 3762 evcnt_detach(&stats->rnbc[i]); 3763 } 3764 if (i < __arraycount(stats->pxontxc)) { 3765 evcnt_detach(&stats->pxontxc[i]); 3766 evcnt_detach(&stats->pxonrxc[i]); 3767 evcnt_detach(&stats->pxofftxc[i]); 3768 evcnt_detach(&stats->pxoffrxc[i]); 3769 if (hw->mac.type >= ixgbe_mac_82599EB) 3770 evcnt_detach(&stats->pxon2offc[i]); 3771 } 3772 } 3773 3774 txr = adapter->tx_rings; 3775 for (i = 0; i < adapter->num_queues; i++, rxr++, txr++) { 3776 evcnt_detach(&adapter->queues[i].irqs); 3777 evcnt_detach(&adapter->queues[i].handleq); 3778 evcnt_detach(&adapter->queues[i].req); 3779 evcnt_detach(&txr->no_desc_avail); 3780 evcnt_detach(&txr->total_packets); 3781 evcnt_detach(&txr->tso_tx); 3782 #ifndef IXGBE_LEGACY_TX 3783 evcnt_detach(&txr->pcq_drops); 3784 #endif 3785 3786 if (i < __arraycount(stats->qprc)) { 3787 evcnt_detach(&stats->qprc[i]); 3788 evcnt_detach(&stats->qptc[i]); 3789 evcnt_detach(&stats->qbrc[i]); 3790 evcnt_detach(&stats->qbtc[i]); 3791 if (hw->mac.type >= ixgbe_mac_82599EB) 3792 evcnt_detach(&stats->qprdc[i]); 3793 } 3794 3795 evcnt_detach(&rxr->rx_packets); 3796 evcnt_detach(&rxr->rx_bytes); 3797 evcnt_detach(&rxr->rx_copies); 3798 evcnt_detach(&rxr->no_mbuf); 3799 evcnt_detach(&rxr->rx_discarded); 3800 } 3801 evcnt_detach(&stats->ipcs); 3802 evcnt_detach(&stats->l4cs); 3803 evcnt_detach(&stats->ipcs_bad); 3804 evcnt_detach(&stats->l4cs_bad); 3805 evcnt_detach(&stats->intzero); 3806 evcnt_detach(&stats->legint); 3807 evcnt_detach(&stats->crcerrs); 3808 evcnt_detach(&stats->illerrc); 3809 evcnt_detach(&stats->errbc); 3810 evcnt_detach(&stats->mspdc); 3811 if (hw->mac.type >= ixgbe_mac_X550) 3812 evcnt_detach(&stats->mbsdc); 3813 evcnt_detach(&stats->mpctotal); 3814 evcnt_detach(&stats->mlfc); 3815 evcnt_detach(&stats->mrfc); 3816 evcnt_detach(&stats->rlec); 3817 evcnt_detach(&stats->lxontxc); 3818 evcnt_detach(&stats->lxonrxc); 3819 evcnt_detach(&stats->lxofftxc); 3820 evcnt_detach(&stats->lxoffrxc); 3821 3822 /* Packet Reception Stats */ 3823 evcnt_detach(&stats->tor); 3824 evcnt_detach(&stats->gorc); 3825 evcnt_detach(&stats->tpr); 3826 evcnt_detach(&stats->gprc); 3827 evcnt_detach(&stats->mprc); 3828 evcnt_detach(&stats->bprc); 3829 evcnt_detach(&stats->prc64); 3830 evcnt_detach(&stats->prc127); 3831 evcnt_detach(&stats->prc255); 3832 evcnt_detach(&stats->prc511); 3833 evcnt_detach(&stats->prc1023); 3834 evcnt_detach(&stats->prc1522); 3835 evcnt_detach(&stats->ruc); 3836 evcnt_detach(&stats->rfc); 3837 evcnt_detach(&stats->roc); 3838 evcnt_detach(&stats->rjc); 3839 evcnt_detach(&stats->mngprc); 3840 evcnt_detach(&stats->mngpdc); 3841 evcnt_detach(&stats->xec); 3842 3843 /* Packet Transmission Stats */ 3844 evcnt_detach(&stats->gotc); 3845 evcnt_detach(&stats->tpt); 3846 evcnt_detach(&stats->gptc); 3847 evcnt_detach(&stats->bptc); 3848 evcnt_detach(&stats->mptc); 3849 evcnt_detach(&stats->mngptc); 3850 evcnt_detach(&stats->ptc64); 3851 evcnt_detach(&stats->ptc127); 3852 evcnt_detach(&stats->ptc255); 3853 evcnt_detach(&stats->ptc511); 3854 evcnt_detach(&stats->ptc1023); 3855 evcnt_detach(&stats->ptc1522); 3856 3857 ixgbe_free_queues(adapter); 3858 free(adapter->mta, M_DEVBUF); 3859 3860 mutex_destroy(&adapter->admin_mtx); /* XXX appropriate order? */ 3861 IXGBE_CORE_LOCK_DESTROY(adapter); 3862 3863 return (0); 3864 } /* ixgbe_detach */ 3865 3866 /************************************************************************ 3867 * ixgbe_setup_low_power_mode - LPLU/WoL preparation 3868 * 3869 * Prepare the adapter/port for LPLU and/or WoL 3870 ************************************************************************/ 3871 static int 3872 ixgbe_setup_low_power_mode(struct adapter *adapter) 3873 { 3874 struct ixgbe_hw *hw = &adapter->hw; 3875 device_t dev = adapter->dev; 3876 struct ifnet *ifp = adapter->ifp; 3877 s32 error = 0; 3878 3879 /* Limit power management flow to X550EM baseT */ 3880 if (hw->device_id == IXGBE_DEV_ID_X550EM_X_10G_T && 3881 hw->phy.ops.enter_lplu) { 3882 /* X550EM baseT adapters need a special LPLU flow */ 3883 hw->phy.reset_disable = true; 3884 ixgbe_ifstop(ifp, 1); 3885 error = hw->phy.ops.enter_lplu(hw); 3886 if (error) 3887 device_printf(dev, 3888 "Error entering LPLU: %d\n", error); 3889 hw->phy.reset_disable = false; 3890 } else { 3891 /* Just stop for other adapters */ 3892 ixgbe_ifstop(ifp, 1); 3893 } 3894 3895 IXGBE_CORE_LOCK(adapter); 3896 3897 if (!hw->wol_enabled) { 3898 ixgbe_set_phy_power(hw, FALSE); 3899 IXGBE_WRITE_REG(hw, IXGBE_WUFC, 0); 3900 IXGBE_WRITE_REG(hw, IXGBE_WUC, 0); 3901 } else { 3902 /* Turn off support for APM wakeup. (Using ACPI instead) */ 3903 IXGBE_WRITE_REG(hw, IXGBE_GRC_BY_MAC(hw), 3904 IXGBE_READ_REG(hw, IXGBE_GRC_BY_MAC(hw)) & ~(u32)2); 3905 3906 /* 3907 * Clear Wake Up Status register to prevent any previous wakeup 3908 * events from waking us up immediately after we suspend. 3909 */ 3910 IXGBE_WRITE_REG(hw, IXGBE_WUS, 0xffffffff); 3911 3912 /* 3913 * Program the Wakeup Filter Control register with user filter 3914 * settings 3915 */ 3916 IXGBE_WRITE_REG(hw, IXGBE_WUFC, adapter->wufc); 3917 3918 /* Enable wakeups and power management in Wakeup Control */ 3919 IXGBE_WRITE_REG(hw, IXGBE_WUC, 3920 IXGBE_WUC_WKEN | IXGBE_WUC_PME_EN); 3921 3922 } 3923 3924 IXGBE_CORE_UNLOCK(adapter); 3925 3926 return error; 3927 } /* ixgbe_setup_low_power_mode */ 3928 3929 /************************************************************************ 3930 * ixgbe_shutdown - Shutdown entry point 3931 ************************************************************************/ 3932 #if 0 /* XXX NetBSD ought to register something like this through pmf(9) */ 3933 static int 3934 ixgbe_shutdown(device_t dev) 3935 { 3936 struct adapter *adapter = device_private(dev); 3937 int error = 0; 3938 3939 INIT_DEBUGOUT("ixgbe_shutdown: begin"); 3940 3941 error = ixgbe_setup_low_power_mode(adapter); 3942 3943 return (error); 3944 } /* ixgbe_shutdown */ 3945 #endif 3946 3947 /************************************************************************ 3948 * ixgbe_suspend 3949 * 3950 * From D0 to D3 3951 ************************************************************************/ 3952 static bool 3953 ixgbe_suspend(device_t dev, const pmf_qual_t *qual) 3954 { 3955 struct adapter *adapter = device_private(dev); 3956 int error = 0; 3957 3958 INIT_DEBUGOUT("ixgbe_suspend: begin"); 3959 3960 error = ixgbe_setup_low_power_mode(adapter); 3961 3962 return (error); 3963 } /* ixgbe_suspend */ 3964 3965 /************************************************************************ 3966 * ixgbe_resume 3967 * 3968 * From D3 to D0 3969 ************************************************************************/ 3970 static bool 3971 ixgbe_resume(device_t dev, const pmf_qual_t *qual) 3972 { 3973 struct adapter *adapter = device_private(dev); 3974 struct ifnet *ifp = adapter->ifp; 3975 struct ixgbe_hw *hw = &adapter->hw; 3976 u32 wus; 3977 3978 INIT_DEBUGOUT("ixgbe_resume: begin"); 3979 3980 IXGBE_CORE_LOCK(adapter); 3981 3982 /* Read & clear WUS register */ 3983 wus = IXGBE_READ_REG(hw, IXGBE_WUS); 3984 if (wus) 3985 device_printf(dev, "Woken up by (WUS): %#010x\n", 3986 IXGBE_READ_REG(hw, IXGBE_WUS)); 3987 IXGBE_WRITE_REG(hw, IXGBE_WUS, 0xffffffff); 3988 /* And clear WUFC until next low-power transition */ 3989 IXGBE_WRITE_REG(hw, IXGBE_WUFC, 0); 3990 3991 /* 3992 * Required after D3->D0 transition; 3993 * will re-advertise all previous advertised speeds 3994 */ 3995 if (ifp->if_flags & IFF_UP) 3996 ixgbe_init_locked(adapter); 3997 3998 IXGBE_CORE_UNLOCK(adapter); 3999 4000 return true; 4001 } /* ixgbe_resume */ 4002 4003 /* 4004 * Set the various hardware offload abilities. 4005 * 4006 * This takes the ifnet's if_capenable flags (e.g. set by the user using 4007 * ifconfig) and indicates to the OS via the ifnet's if_hwassist field what 4008 * mbuf offload flags the driver will understand. 4009 */ 4010 static void 4011 ixgbe_set_if_hwassist(struct adapter *adapter) 4012 { 4013 /* XXX */ 4014 } 4015 4016 /************************************************************************ 4017 * ixgbe_init_locked - Init entry point 4018 * 4019 * Used in two ways: It is used by the stack as an init 4020 * entry point in network interface structure. It is also 4021 * used by the driver as a hw/sw initialization routine to 4022 * get to a consistent state. 4023 * 4024 * return 0 on success, positive on failure 4025 ************************************************************************/ 4026 static void 4027 ixgbe_init_locked(struct adapter *adapter) 4028 { 4029 struct ifnet *ifp = adapter->ifp; 4030 device_t dev = adapter->dev; 4031 struct ixgbe_hw *hw = &adapter->hw; 4032 struct ix_queue *que; 4033 struct tx_ring *txr; 4034 struct rx_ring *rxr; 4035 u32 txdctl, mhadd; 4036 u32 rxdctl, rxctrl; 4037 u32 ctrl_ext; 4038 bool unsupported_sfp = false; 4039 int i, j, error; 4040 4041 /* XXX check IFF_UP and IFF_RUNNING, power-saving state! */ 4042 4043 KASSERT(mutex_owned(&adapter->core_mtx)); 4044 INIT_DEBUGOUT("ixgbe_init_locked: begin"); 4045 4046 hw->need_unsupported_sfp_recovery = false; 4047 hw->adapter_stopped = FALSE; 4048 ixgbe_stop_adapter(hw); 4049 callout_stop(&adapter->timer); 4050 if (adapter->feat_en & IXGBE_FEATURE_RECOVERY_MODE) 4051 callout_stop(&adapter->recovery_mode_timer); 4052 for (i = 0, que = adapter->queues; i < adapter->num_queues; i++, que++) 4053 que->disabled_count = 0; 4054 4055 /* XXX I moved this here from the SIOCSIFMTU case in ixgbe_ioctl(). */ 4056 adapter->max_frame_size = 4057 ifp->if_mtu + ETHER_HDR_LEN + ETHER_CRC_LEN; 4058 4059 /* Queue indices may change with IOV mode */ 4060 ixgbe_align_all_queue_indices(adapter); 4061 4062 /* reprogram the RAR[0] in case user changed it. */ 4063 ixgbe_set_rar(hw, 0, hw->mac.addr, adapter->pool, IXGBE_RAH_AV); 4064 4065 /* Get the latest mac address, User can use a LAA */ 4066 memcpy(hw->mac.addr, CLLADDR(ifp->if_sadl), 4067 IXGBE_ETH_LENGTH_OF_ADDRESS); 4068 ixgbe_set_rar(hw, 0, hw->mac.addr, adapter->pool, 1); 4069 hw->addr_ctrl.rar_used_count = 1; 4070 4071 /* Set hardware offload abilities from ifnet flags */ 4072 ixgbe_set_if_hwassist(adapter); 4073 4074 /* Prepare transmit descriptors and buffers */ 4075 if (ixgbe_setup_transmit_structures(adapter)) { 4076 device_printf(dev, "Could not setup transmit structures\n"); 4077 ixgbe_stop_locked(adapter); 4078 return; 4079 } 4080 4081 ixgbe_init_hw(hw); 4082 4083 ixgbe_initialize_iov(adapter); 4084 4085 ixgbe_initialize_transmit_units(adapter); 4086 4087 /* Setup Multicast table */ 4088 ixgbe_set_rxfilter(adapter); 4089 4090 /* Use fixed buffer size, even for jumbo frames */ 4091 adapter->rx_mbuf_sz = MCLBYTES; 4092 4093 /* Prepare receive descriptors and buffers */ 4094 error = ixgbe_setup_receive_structures(adapter); 4095 if (error) { 4096 device_printf(dev, 4097 "Could not setup receive structures (err = %d)\n", error); 4098 ixgbe_stop_locked(adapter); 4099 return; 4100 } 4101 4102 /* Configure RX settings */ 4103 ixgbe_initialize_receive_units(adapter); 4104 4105 /* Initialize variable holding task enqueue requests interrupts */ 4106 adapter->task_requests = 0; 4107 4108 /* Enable SDP & MSI-X interrupts based on adapter */ 4109 ixgbe_config_gpie(adapter); 4110 4111 /* Set MTU size */ 4112 if (ifp->if_mtu > ETHERMTU) { 4113 /* aka IXGBE_MAXFRS on 82599 and newer */ 4114 mhadd = IXGBE_READ_REG(hw, IXGBE_MHADD); 4115 mhadd &= ~IXGBE_MHADD_MFS_MASK; 4116 mhadd |= adapter->max_frame_size << IXGBE_MHADD_MFS_SHIFT; 4117 IXGBE_WRITE_REG(hw, IXGBE_MHADD, mhadd); 4118 } 4119 4120 /* Now enable all the queues */ 4121 for (i = 0; i < adapter->num_queues; i++) { 4122 txr = &adapter->tx_rings[i]; 4123 txdctl = IXGBE_READ_REG(hw, IXGBE_TXDCTL(txr->me)); 4124 txdctl |= IXGBE_TXDCTL_ENABLE; 4125 /* Set WTHRESH to 8, burst writeback */ 4126 txdctl |= IXGBE_TX_WTHRESH << IXGBE_TXDCTL_WTHRESH_SHIFT; 4127 /* 4128 * When the internal queue falls below PTHRESH (32), 4129 * start prefetching as long as there are at least 4130 * HTHRESH (1) buffers ready. The values are taken 4131 * from the Intel linux driver 3.8.21. 4132 * Prefetching enables tx line rate even with 1 queue. 4133 */ 4134 txdctl |= (32 << 0) | (1 << 8); 4135 IXGBE_WRITE_REG(hw, IXGBE_TXDCTL(txr->me), txdctl); 4136 } 4137 4138 for (i = 0; i < adapter->num_queues; i++) { 4139 rxr = &adapter->rx_rings[i]; 4140 rxdctl = IXGBE_READ_REG(hw, IXGBE_RXDCTL(rxr->me)); 4141 if (hw->mac.type == ixgbe_mac_82598EB) { 4142 /* 4143 * PTHRESH = 21 4144 * HTHRESH = 4 4145 * WTHRESH = 8 4146 */ 4147 rxdctl &= ~0x3FFFFF; 4148 rxdctl |= 0x080420; 4149 } 4150 rxdctl |= IXGBE_RXDCTL_ENABLE; 4151 IXGBE_WRITE_REG(hw, IXGBE_RXDCTL(rxr->me), rxdctl); 4152 for (j = 0; j < 10; j++) { 4153 if (IXGBE_READ_REG(hw, IXGBE_RXDCTL(rxr->me)) & 4154 IXGBE_RXDCTL_ENABLE) 4155 break; 4156 else 4157 msec_delay(1); 4158 } 4159 IXGBE_WRITE_BARRIER(hw); 4160 4161 /* 4162 * In netmap mode, we must preserve the buffers made 4163 * available to userspace before the if_init() 4164 * (this is true by default on the TX side, because 4165 * init makes all buffers available to userspace). 4166 * 4167 * netmap_reset() and the device specific routines 4168 * (e.g. ixgbe_setup_receive_rings()) map these 4169 * buffers at the end of the NIC ring, so here we 4170 * must set the RDT (tail) register to make sure 4171 * they are not overwritten. 4172 * 4173 * In this driver the NIC ring starts at RDH = 0, 4174 * RDT points to the last slot available for reception (?), 4175 * so RDT = num_rx_desc - 1 means the whole ring is available. 4176 */ 4177 #ifdef DEV_NETMAP 4178 if ((adapter->feat_en & IXGBE_FEATURE_NETMAP) && 4179 (ifp->if_capenable & IFCAP_NETMAP)) { 4180 struct netmap_adapter *na = NA(adapter->ifp); 4181 struct netmap_kring *kring = na->rx_rings[i]; 4182 int t = na->num_rx_desc - 1 - nm_kr_rxspace(kring); 4183 4184 IXGBE_WRITE_REG(hw, IXGBE_RDT(rxr->me), t); 4185 } else 4186 #endif /* DEV_NETMAP */ 4187 IXGBE_WRITE_REG(hw, IXGBE_RDT(rxr->me), 4188 adapter->num_rx_desc - 1); 4189 } 4190 4191 /* Enable Receive engine */ 4192 rxctrl = IXGBE_READ_REG(hw, IXGBE_RXCTRL); 4193 if (hw->mac.type == ixgbe_mac_82598EB) 4194 rxctrl |= IXGBE_RXCTRL_DMBYPS; 4195 rxctrl |= IXGBE_RXCTRL_RXEN; 4196 ixgbe_enable_rx_dma(hw, rxctrl); 4197 4198 callout_reset(&adapter->timer, hz, ixgbe_local_timer, adapter); 4199 atomic_store_relaxed(&adapter->timer_pending, 0); 4200 if (adapter->feat_en & IXGBE_FEATURE_RECOVERY_MODE) 4201 callout_reset(&adapter->recovery_mode_timer, hz, 4202 ixgbe_recovery_mode_timer, adapter); 4203 4204 /* Set up MSI/MSI-X routing */ 4205 if (adapter->feat_en & IXGBE_FEATURE_MSIX) { 4206 ixgbe_configure_ivars(adapter); 4207 /* Set up auto-mask */ 4208 if (hw->mac.type == ixgbe_mac_82598EB) 4209 IXGBE_WRITE_REG(hw, IXGBE_EIAM, IXGBE_EICS_RTX_QUEUE); 4210 else { 4211 IXGBE_WRITE_REG(hw, IXGBE_EIAM_EX(0), 0xFFFFFFFF); 4212 IXGBE_WRITE_REG(hw, IXGBE_EIAM_EX(1), 0xFFFFFFFF); 4213 } 4214 } else { /* Simple settings for Legacy/MSI */ 4215 ixgbe_set_ivar(adapter, 0, 0, 0); 4216 ixgbe_set_ivar(adapter, 0, 0, 1); 4217 IXGBE_WRITE_REG(hw, IXGBE_EIAM, IXGBE_EICS_RTX_QUEUE); 4218 } 4219 4220 ixgbe_init_fdir(adapter); 4221 4222 /* 4223 * Check on any SFP devices that 4224 * need to be kick-started 4225 */ 4226 if (hw->phy.type == ixgbe_phy_none) { 4227 error = hw->phy.ops.identify(hw); 4228 if (error == IXGBE_ERR_SFP_NOT_SUPPORTED) 4229 unsupported_sfp = true; 4230 } else if (hw->phy.type == ixgbe_phy_sfp_unsupported) 4231 unsupported_sfp = true; 4232 4233 if (unsupported_sfp) 4234 device_printf(dev, 4235 "Unsupported SFP+ module type was detected.\n"); 4236 4237 /* Set moderation on the Link interrupt */ 4238 ixgbe_eitr_write(adapter, adapter->vector, IXGBE_LINK_ITR); 4239 4240 /* Enable EEE power saving */ 4241 if (adapter->feat_cap & IXGBE_FEATURE_EEE) 4242 hw->mac.ops.setup_eee(hw, 4243 adapter->feat_en & IXGBE_FEATURE_EEE); 4244 4245 /* Enable power to the phy. */ 4246 if (!unsupported_sfp) { 4247 ixgbe_set_phy_power(hw, TRUE); 4248 4249 /* Config/Enable Link */ 4250 ixgbe_config_link(adapter); 4251 } 4252 4253 /* Hardware Packet Buffer & Flow Control setup */ 4254 ixgbe_config_delay_values(adapter); 4255 4256 /* Initialize the FC settings */ 4257 ixgbe_start_hw(hw); 4258 4259 /* Set up VLAN support and filter */ 4260 ixgbe_setup_vlan_hw_support(adapter); 4261 4262 /* Setup DMA Coalescing */ 4263 ixgbe_config_dmac(adapter); 4264 4265 /* OK to schedule workqueues. */ 4266 adapter->schedule_wqs_ok = true; 4267 4268 /* And now turn on interrupts */ 4269 ixgbe_enable_intr(adapter); 4270 4271 /* Enable the use of the MBX by the VF's */ 4272 if (adapter->feat_en & IXGBE_FEATURE_SRIOV) { 4273 ctrl_ext = IXGBE_READ_REG(hw, IXGBE_CTRL_EXT); 4274 ctrl_ext |= IXGBE_CTRL_EXT_PFRSTD; 4275 IXGBE_WRITE_REG(hw, IXGBE_CTRL_EXT, ctrl_ext); 4276 } 4277 4278 /* Update saved flags. See ixgbe_ifflags_cb() */ 4279 adapter->if_flags = ifp->if_flags; 4280 adapter->ec_capenable = adapter->osdep.ec.ec_capenable; 4281 4282 /* Now inform the stack we're ready */ 4283 ifp->if_flags |= IFF_RUNNING; 4284 4285 return; 4286 } /* ixgbe_init_locked */ 4287 4288 /************************************************************************ 4289 * ixgbe_init 4290 ************************************************************************/ 4291 static int 4292 ixgbe_init(struct ifnet *ifp) 4293 { 4294 struct adapter *adapter = ifp->if_softc; 4295 4296 IXGBE_CORE_LOCK(adapter); 4297 ixgbe_init_locked(adapter); 4298 IXGBE_CORE_UNLOCK(adapter); 4299 4300 return 0; /* XXX ixgbe_init_locked cannot fail? really? */ 4301 } /* ixgbe_init */ 4302 4303 /************************************************************************ 4304 * ixgbe_set_ivar 4305 * 4306 * Setup the correct IVAR register for a particular MSI-X interrupt 4307 * (yes this is all very magic and confusing :) 4308 * - entry is the register array entry 4309 * - vector is the MSI-X vector for this queue 4310 * - type is RX/TX/MISC 4311 ************************************************************************/ 4312 static void 4313 ixgbe_set_ivar(struct adapter *adapter, u8 entry, u8 vector, s8 type) 4314 { 4315 struct ixgbe_hw *hw = &adapter->hw; 4316 u32 ivar, index; 4317 4318 vector |= IXGBE_IVAR_ALLOC_VAL; 4319 4320 switch (hw->mac.type) { 4321 case ixgbe_mac_82598EB: 4322 if (type == -1) 4323 entry = IXGBE_IVAR_OTHER_CAUSES_INDEX; 4324 else 4325 entry += (type * 64); 4326 index = (entry >> 2) & 0x1F; 4327 ivar = IXGBE_READ_REG(hw, IXGBE_IVAR(index)); 4328 ivar &= ~(0xffUL << (8 * (entry & 0x3))); 4329 ivar |= ((u32)vector << (8 * (entry & 0x3))); 4330 IXGBE_WRITE_REG(&adapter->hw, IXGBE_IVAR(index), ivar); 4331 break; 4332 case ixgbe_mac_82599EB: 4333 case ixgbe_mac_X540: 4334 case ixgbe_mac_X550: 4335 case ixgbe_mac_X550EM_x: 4336 case ixgbe_mac_X550EM_a: 4337 if (type == -1) { /* MISC IVAR */ 4338 index = (entry & 1) * 8; 4339 ivar = IXGBE_READ_REG(hw, IXGBE_IVAR_MISC); 4340 ivar &= ~(0xffUL << index); 4341 ivar |= ((u32)vector << index); 4342 IXGBE_WRITE_REG(hw, IXGBE_IVAR_MISC, ivar); 4343 } else { /* RX/TX IVARS */ 4344 index = (16 * (entry & 1)) + (8 * type); 4345 ivar = IXGBE_READ_REG(hw, IXGBE_IVAR(entry >> 1)); 4346 ivar &= ~(0xffUL << index); 4347 ivar |= ((u32)vector << index); 4348 IXGBE_WRITE_REG(hw, IXGBE_IVAR(entry >> 1), ivar); 4349 } 4350 break; 4351 default: 4352 break; 4353 } 4354 } /* ixgbe_set_ivar */ 4355 4356 /************************************************************************ 4357 * ixgbe_configure_ivars 4358 ************************************************************************/ 4359 static void 4360 ixgbe_configure_ivars(struct adapter *adapter) 4361 { 4362 struct ix_queue *que = adapter->queues; 4363 u32 newitr; 4364 4365 if (ixgbe_max_interrupt_rate > 0) 4366 newitr = (4000000 / ixgbe_max_interrupt_rate) & 0x0FF8; 4367 else { 4368 /* 4369 * Disable DMA coalescing if interrupt moderation is 4370 * disabled. 4371 */ 4372 adapter->dmac = 0; 4373 newitr = 0; 4374 } 4375 4376 for (int i = 0; i < adapter->num_queues; i++, que++) { 4377 struct rx_ring *rxr = &adapter->rx_rings[i]; 4378 struct tx_ring *txr = &adapter->tx_rings[i]; 4379 /* First the RX queue entry */ 4380 ixgbe_set_ivar(adapter, rxr->me, que->msix, 0); 4381 /* ... and the TX */ 4382 ixgbe_set_ivar(adapter, txr->me, que->msix, 1); 4383 /* Set an Initial EITR value */ 4384 ixgbe_eitr_write(adapter, que->msix, newitr); 4385 /* 4386 * To eliminate influence of the previous state. 4387 * At this point, Tx/Rx interrupt handler 4388 * (ixgbe_msix_que()) cannot be called, so both 4389 * IXGBE_TX_LOCK and IXGBE_RX_LOCK are not required. 4390 */ 4391 que->eitr_setting = 0; 4392 } 4393 4394 /* For the Link interrupt */ 4395 ixgbe_set_ivar(adapter, 1, adapter->vector, -1); 4396 } /* ixgbe_configure_ivars */ 4397 4398 /************************************************************************ 4399 * ixgbe_config_gpie 4400 ************************************************************************/ 4401 static void 4402 ixgbe_config_gpie(struct adapter *adapter) 4403 { 4404 struct ixgbe_hw *hw = &adapter->hw; 4405 u32 gpie; 4406 4407 gpie = IXGBE_READ_REG(hw, IXGBE_GPIE); 4408 4409 if (adapter->feat_en & IXGBE_FEATURE_MSIX) { 4410 /* Enable Enhanced MSI-X mode */ 4411 gpie |= IXGBE_GPIE_MSIX_MODE 4412 | IXGBE_GPIE_EIAME 4413 | IXGBE_GPIE_PBA_SUPPORT 4414 | IXGBE_GPIE_OCD; 4415 } 4416 4417 /* Fan Failure Interrupt */ 4418 if (adapter->feat_en & IXGBE_FEATURE_FAN_FAIL) 4419 gpie |= IXGBE_SDP1_GPIEN; 4420 4421 /* Thermal Sensor Interrupt */ 4422 if (adapter->feat_en & IXGBE_FEATURE_TEMP_SENSOR) 4423 gpie |= IXGBE_SDP0_GPIEN_X540; 4424 4425 /* Link detection */ 4426 switch (hw->mac.type) { 4427 case ixgbe_mac_82599EB: 4428 gpie |= IXGBE_SDP1_GPIEN | IXGBE_SDP2_GPIEN; 4429 break; 4430 case ixgbe_mac_X550EM_x: 4431 case ixgbe_mac_X550EM_a: 4432 gpie |= IXGBE_SDP0_GPIEN_X540; 4433 break; 4434 default: 4435 break; 4436 } 4437 4438 IXGBE_WRITE_REG(hw, IXGBE_GPIE, gpie); 4439 4440 } /* ixgbe_config_gpie */ 4441 4442 /************************************************************************ 4443 * ixgbe_config_delay_values 4444 * 4445 * Requires adapter->max_frame_size to be set. 4446 ************************************************************************/ 4447 static void 4448 ixgbe_config_delay_values(struct adapter *adapter) 4449 { 4450 struct ixgbe_hw *hw = &adapter->hw; 4451 u32 rxpb, frame, size, tmp; 4452 4453 frame = adapter->max_frame_size; 4454 4455 /* Calculate High Water */ 4456 switch (hw->mac.type) { 4457 case ixgbe_mac_X540: 4458 case ixgbe_mac_X550: 4459 case ixgbe_mac_X550EM_x: 4460 case ixgbe_mac_X550EM_a: 4461 tmp = IXGBE_DV_X540(frame, frame); 4462 break; 4463 default: 4464 tmp = IXGBE_DV(frame, frame); 4465 break; 4466 } 4467 size = IXGBE_BT2KB(tmp); 4468 rxpb = IXGBE_READ_REG(hw, IXGBE_RXPBSIZE(0)) >> 10; 4469 hw->fc.high_water[0] = rxpb - size; 4470 4471 /* Now calculate Low Water */ 4472 switch (hw->mac.type) { 4473 case ixgbe_mac_X540: 4474 case ixgbe_mac_X550: 4475 case ixgbe_mac_X550EM_x: 4476 case ixgbe_mac_X550EM_a: 4477 tmp = IXGBE_LOW_DV_X540(frame); 4478 break; 4479 default: 4480 tmp = IXGBE_LOW_DV(frame); 4481 break; 4482 } 4483 hw->fc.low_water[0] = IXGBE_BT2KB(tmp); 4484 4485 hw->fc.pause_time = IXGBE_FC_PAUSE; 4486 hw->fc.send_xon = TRUE; 4487 } /* ixgbe_config_delay_values */ 4488 4489 /************************************************************************ 4490 * ixgbe_set_rxfilter - Multicast Update 4491 * 4492 * Called whenever multicast address list is updated. 4493 ************************************************************************/ 4494 static void 4495 ixgbe_set_rxfilter(struct adapter *adapter) 4496 { 4497 struct ixgbe_mc_addr *mta; 4498 struct ifnet *ifp = adapter->ifp; 4499 u8 *update_ptr; 4500 int mcnt = 0; 4501 u32 fctrl; 4502 struct ethercom *ec = &adapter->osdep.ec; 4503 struct ether_multi *enm; 4504 struct ether_multistep step; 4505 4506 KASSERT(mutex_owned(&adapter->core_mtx)); 4507 IOCTL_DEBUGOUT("ixgbe_set_rxfilter: begin"); 4508 4509 mta = adapter->mta; 4510 bzero(mta, sizeof(*mta) * MAX_NUM_MULTICAST_ADDRESSES); 4511 4512 ETHER_LOCK(ec); 4513 ec->ec_flags &= ~ETHER_F_ALLMULTI; 4514 ETHER_FIRST_MULTI(step, ec, enm); 4515 while (enm != NULL) { 4516 if ((mcnt == MAX_NUM_MULTICAST_ADDRESSES) || 4517 (memcmp(enm->enm_addrlo, enm->enm_addrhi, 4518 ETHER_ADDR_LEN) != 0)) { 4519 ec->ec_flags |= ETHER_F_ALLMULTI; 4520 break; 4521 } 4522 bcopy(enm->enm_addrlo, 4523 mta[mcnt].addr, IXGBE_ETH_LENGTH_OF_ADDRESS); 4524 mta[mcnt].vmdq = adapter->pool; 4525 mcnt++; 4526 ETHER_NEXT_MULTI(step, enm); 4527 } 4528 4529 fctrl = IXGBE_READ_REG(&adapter->hw, IXGBE_FCTRL); 4530 if (ifp->if_flags & IFF_PROMISC) 4531 fctrl |= (IXGBE_FCTRL_UPE | IXGBE_FCTRL_MPE); 4532 else if (ec->ec_flags & ETHER_F_ALLMULTI) { 4533 fctrl |= IXGBE_FCTRL_MPE; 4534 fctrl &= ~IXGBE_FCTRL_UPE; 4535 } else 4536 fctrl &= ~(IXGBE_FCTRL_UPE | IXGBE_FCTRL_MPE); 4537 4538 IXGBE_WRITE_REG(&adapter->hw, IXGBE_FCTRL, fctrl); 4539 4540 /* Update multicast filter entries only when it's not ALLMULTI */ 4541 if ((ec->ec_flags & ETHER_F_ALLMULTI) == 0) { 4542 ETHER_UNLOCK(ec); 4543 update_ptr = (u8 *)mta; 4544 ixgbe_update_mc_addr_list(&adapter->hw, update_ptr, mcnt, 4545 ixgbe_mc_array_itr, TRUE); 4546 } else 4547 ETHER_UNLOCK(ec); 4548 } /* ixgbe_set_rxfilter */ 4549 4550 /************************************************************************ 4551 * ixgbe_mc_array_itr 4552 * 4553 * An iterator function needed by the multicast shared code. 4554 * It feeds the shared code routine the addresses in the 4555 * array of ixgbe_set_rxfilter() one by one. 4556 ************************************************************************/ 4557 static u8 * 4558 ixgbe_mc_array_itr(struct ixgbe_hw *hw, u8 **update_ptr, u32 *vmdq) 4559 { 4560 struct ixgbe_mc_addr *mta; 4561 4562 mta = (struct ixgbe_mc_addr *)*update_ptr; 4563 *vmdq = mta->vmdq; 4564 4565 *update_ptr = (u8*)(mta + 1); 4566 4567 return (mta->addr); 4568 } /* ixgbe_mc_array_itr */ 4569 4570 /************************************************************************ 4571 * ixgbe_local_timer - Timer routine 4572 * 4573 * Checks for link status, updates statistics, 4574 * and runs the watchdog check. 4575 ************************************************************************/ 4576 static void 4577 ixgbe_local_timer(void *arg) 4578 { 4579 struct adapter *adapter = arg; 4580 4581 if (adapter->schedule_wqs_ok) { 4582 if (atomic_cas_uint(&adapter->timer_pending, 0, 1) == 0) 4583 workqueue_enqueue(adapter->timer_wq, 4584 &adapter->timer_wc, NULL); 4585 } 4586 } 4587 4588 static void 4589 ixgbe_handle_timer(struct work *wk, void *context) 4590 { 4591 struct adapter *adapter = context; 4592 struct ixgbe_hw *hw = &adapter->hw; 4593 device_t dev = adapter->dev; 4594 struct ix_queue *que = adapter->queues; 4595 u64 queues = 0; 4596 u64 v0, v1, v2, v3, v4, v5, v6, v7; 4597 int hung = 0; 4598 int i; 4599 4600 IXGBE_CORE_LOCK(adapter); 4601 4602 /* Check for pluggable optics */ 4603 if (ixgbe_is_sfp(hw)) { 4604 bool sched_mod_task = false; 4605 4606 if (hw->mac.type == ixgbe_mac_82598EB) { 4607 /* 4608 * On 82598EB, SFP+'s MOD_ABS pin is not connected to 4609 * any GPIO(SDP). So just schedule TASK_MOD. 4610 */ 4611 sched_mod_task = true; 4612 } else { 4613 bool was_full, is_full; 4614 4615 was_full = 4616 hw->phy.sfp_type != ixgbe_sfp_type_not_present; 4617 is_full = ixgbe_sfp_cage_full(hw); 4618 4619 /* Do probe if cage state changed */ 4620 if (was_full ^ is_full) 4621 sched_mod_task = true; 4622 } 4623 if (sched_mod_task) { 4624 mutex_enter(&adapter->admin_mtx); 4625 adapter->task_requests |= IXGBE_REQUEST_TASK_MOD_WOI; 4626 ixgbe_schedule_admin_tasklet(adapter); 4627 mutex_exit(&adapter->admin_mtx); 4628 } 4629 } 4630 4631 ixgbe_update_link_status(adapter); 4632 ixgbe_update_stats_counters(adapter); 4633 4634 /* Update some event counters */ 4635 v0 = v1 = v2 = v3 = v4 = v5 = v6 = v7 = 0; 4636 que = adapter->queues; 4637 for (i = 0; i < adapter->num_queues; i++, que++) { 4638 struct tx_ring *txr = que->txr; 4639 4640 v0 += txr->q_efbig_tx_dma_setup; 4641 v1 += txr->q_mbuf_defrag_failed; 4642 v2 += txr->q_efbig2_tx_dma_setup; 4643 v3 += txr->q_einval_tx_dma_setup; 4644 v4 += txr->q_other_tx_dma_setup; 4645 v5 += txr->q_eagain_tx_dma_setup; 4646 v6 += txr->q_enomem_tx_dma_setup; 4647 v7 += txr->q_tso_err; 4648 } 4649 IXGBE_EVC_STORE(&adapter->efbig_tx_dma_setup, v0); 4650 IXGBE_EVC_STORE(&adapter->mbuf_defrag_failed, v1); 4651 IXGBE_EVC_STORE(&adapter->efbig2_tx_dma_setup, v2); 4652 IXGBE_EVC_STORE(&adapter->einval_tx_dma_setup, v3); 4653 IXGBE_EVC_STORE(&adapter->other_tx_dma_setup, v4); 4654 IXGBE_EVC_STORE(&adapter->eagain_tx_dma_setup, v5); 4655 IXGBE_EVC_STORE(&adapter->enomem_tx_dma_setup, v6); 4656 IXGBE_EVC_STORE(&adapter->tso_err, v7); 4657 4658 /* 4659 * Check the TX queues status 4660 * - mark hung queues so we don't schedule on them 4661 * - watchdog only if all queues show hung 4662 */ 4663 que = adapter->queues; 4664 for (i = 0; i < adapter->num_queues; i++, que++) { 4665 /* Keep track of queues with work for soft irq */ 4666 if (que->txr->busy) 4667 queues |= 1ULL << que->me; 4668 /* 4669 * Each time txeof runs without cleaning, but there 4670 * are uncleaned descriptors it increments busy. If 4671 * we get to the MAX we declare it hung. 4672 */ 4673 if (que->busy == IXGBE_QUEUE_HUNG) { 4674 ++hung; 4675 /* Mark the queue as inactive */ 4676 adapter->active_queues &= ~(1ULL << que->me); 4677 continue; 4678 } else { 4679 /* Check if we've come back from hung */ 4680 if ((adapter->active_queues & (1ULL << que->me)) == 0) 4681 adapter->active_queues |= 1ULL << que->me; 4682 } 4683 if (que->busy >= IXGBE_MAX_TX_BUSY) { 4684 device_printf(dev, 4685 "Warning queue %d appears to be hung!\n", i); 4686 que->txr->busy = IXGBE_QUEUE_HUNG; 4687 ++hung; 4688 } 4689 } 4690 4691 /* Only truly watchdog if all queues show hung */ 4692 if (hung == adapter->num_queues) 4693 goto watchdog; 4694 #if 0 /* XXX Avoid unexpectedly disabling interrupt forever (PR#53294) */ 4695 else if (queues != 0) { /* Force an IRQ on queues with work */ 4696 que = adapter->queues; 4697 for (i = 0; i < adapter->num_queues; i++, que++) { 4698 mutex_enter(&que->dc_mtx); 4699 if (que->disabled_count == 0) 4700 ixgbe_rearm_queues(adapter, 4701 queues & ((u64)1 << i)); 4702 mutex_exit(&que->dc_mtx); 4703 } 4704 } 4705 #endif 4706 4707 atomic_store_relaxed(&adapter->timer_pending, 0); 4708 IXGBE_CORE_UNLOCK(adapter); 4709 callout_reset(&adapter->timer, hz, ixgbe_local_timer, adapter); 4710 return; 4711 4712 watchdog: 4713 device_printf(adapter->dev, "Watchdog timeout -- resetting\n"); 4714 adapter->ifp->if_flags &= ~IFF_RUNNING; 4715 IXGBE_EVC_ADD(&adapter->watchdog_events, 1); 4716 ixgbe_init_locked(adapter); 4717 IXGBE_CORE_UNLOCK(adapter); 4718 } /* ixgbe_handle_timer */ 4719 4720 /************************************************************************ 4721 * ixgbe_recovery_mode_timer - Recovery mode timer routine 4722 ************************************************************************/ 4723 static void 4724 ixgbe_recovery_mode_timer(void *arg) 4725 { 4726 struct adapter *adapter = arg; 4727 4728 if (__predict_true(adapter->osdep.detaching == false)) { 4729 if (atomic_cas_uint(&adapter->recovery_mode_timer_pending, 4730 0, 1) == 0) { 4731 workqueue_enqueue(adapter->recovery_mode_timer_wq, 4732 &adapter->recovery_mode_timer_wc, NULL); 4733 } 4734 } 4735 } 4736 4737 static void 4738 ixgbe_handle_recovery_mode_timer(struct work *wk, void *context) 4739 { 4740 struct adapter *adapter = context; 4741 struct ixgbe_hw *hw = &adapter->hw; 4742 4743 IXGBE_CORE_LOCK(adapter); 4744 if (ixgbe_fw_recovery_mode(hw)) { 4745 if (atomic_cas_uint(&adapter->recovery_mode, 0, 1)) { 4746 /* Firmware error detected, entering recovery mode */ 4747 device_printf(adapter->dev, "Firmware recovery mode detected. Limiting functionality. Refer to the Intel(R) Ethernet Adapters and Devices User Guide for details on firmware recovery mode.\n"); 4748 4749 if (hw->adapter_stopped == FALSE) 4750 ixgbe_stop_locked(adapter); 4751 } 4752 } else 4753 atomic_cas_uint(&adapter->recovery_mode, 1, 0); 4754 4755 atomic_store_relaxed(&adapter->recovery_mode_timer_pending, 0); 4756 callout_reset(&adapter->recovery_mode_timer, hz, 4757 ixgbe_recovery_mode_timer, adapter); 4758 IXGBE_CORE_UNLOCK(adapter); 4759 } /* ixgbe_handle_recovery_mode_timer */ 4760 4761 /************************************************************************ 4762 * ixgbe_handle_mod - Tasklet for SFP module interrupts 4763 * bool int_en: true if it's called when the interrupt is enabled. 4764 ************************************************************************/ 4765 static void 4766 ixgbe_handle_mod(void *context, bool int_en) 4767 { 4768 struct adapter *adapter = context; 4769 struct ixgbe_hw *hw = &adapter->hw; 4770 device_t dev = adapter->dev; 4771 enum ixgbe_sfp_type last_sfp_type; 4772 u32 err; 4773 bool last_unsupported_sfp_recovery; 4774 4775 KASSERT(mutex_owned(&adapter->core_mtx)); 4776 4777 last_sfp_type = hw->phy.sfp_type; 4778 last_unsupported_sfp_recovery = hw->need_unsupported_sfp_recovery; 4779 IXGBE_EVC_ADD(&adapter->mod_workev, 1); 4780 if (adapter->hw.need_crosstalk_fix) { 4781 if ((hw->mac.type != ixgbe_mac_82598EB) && 4782 !ixgbe_sfp_cage_full(hw)) 4783 goto out; 4784 } 4785 4786 err = hw->phy.ops.identify_sfp(hw); 4787 if (err == IXGBE_ERR_SFP_NOT_SUPPORTED) { 4788 if (last_unsupported_sfp_recovery == false) 4789 device_printf(dev, 4790 "Unsupported SFP+ module type was detected.\n"); 4791 goto out; 4792 } 4793 4794 if (hw->need_unsupported_sfp_recovery) { 4795 device_printf(dev, "Recovering from unsupported SFP\n"); 4796 /* 4797 * We could recover the status by calling setup_sfp(), 4798 * setup_link() and some others. It's complex and might not 4799 * work correctly on some unknown cases. To avoid such type of 4800 * problem, call ixgbe_init_locked(). It's simple and safe 4801 * approach. 4802 */ 4803 ixgbe_init_locked(adapter); 4804 } else if ((hw->phy.sfp_type != ixgbe_sfp_type_not_present) && 4805 (hw->phy.sfp_type != last_sfp_type)) { 4806 /* A module is inserted and changed. */ 4807 4808 if (hw->mac.type == ixgbe_mac_82598EB) 4809 err = hw->phy.ops.reset(hw); 4810 else { 4811 err = hw->mac.ops.setup_sfp(hw); 4812 hw->phy.sfp_setup_needed = FALSE; 4813 } 4814 if (err == IXGBE_ERR_SFP_NOT_SUPPORTED) { 4815 device_printf(dev, 4816 "Setup failure - unsupported SFP+ module type.\n"); 4817 goto out; 4818 } 4819 } 4820 4821 out: 4822 /* get_supported_phy_layer will call hw->phy.ops.identify_sfp() */ 4823 adapter->phy_layer = ixgbe_get_supported_physical_layer(hw); 4824 4825 /* Adjust media types shown in ifconfig */ 4826 IXGBE_CORE_UNLOCK(adapter); 4827 ifmedia_removeall(&adapter->media); 4828 ixgbe_add_media_types(adapter); 4829 ifmedia_set(&adapter->media, IFM_ETHER | IFM_AUTO); 4830 IXGBE_CORE_LOCK(adapter); 4831 4832 /* 4833 * Don't schedule MSF event if the chip is 82598. 82598 doesn't support 4834 * MSF. At least, calling ixgbe_handle_msf on 82598 DA makes the link 4835 * flap because the function calls setup_link(). 4836 */ 4837 if (hw->mac.type != ixgbe_mac_82598EB) { 4838 mutex_enter(&adapter->admin_mtx); 4839 if (int_en) 4840 adapter->task_requests |= IXGBE_REQUEST_TASK_MSF; 4841 else 4842 adapter->task_requests |= IXGBE_REQUEST_TASK_MSF_WOI; 4843 mutex_exit(&adapter->admin_mtx); 4844 } 4845 4846 /* 4847 * Don't call ixgbe_schedule_admin_tasklet() because we are on 4848 * the workqueue now. 4849 */ 4850 } /* ixgbe_handle_mod */ 4851 4852 4853 /************************************************************************ 4854 * ixgbe_handle_msf - Tasklet for MSF (multispeed fiber) interrupts 4855 ************************************************************************/ 4856 static void 4857 ixgbe_handle_msf(void *context) 4858 { 4859 struct adapter *adapter = context; 4860 struct ixgbe_hw *hw = &adapter->hw; 4861 u32 autoneg; 4862 bool negotiate; 4863 4864 KASSERT(mutex_owned(&adapter->core_mtx)); 4865 4866 IXGBE_EVC_ADD(&adapter->msf_workev, 1); 4867 4868 autoneg = hw->phy.autoneg_advertised; 4869 if ((!autoneg) && (hw->mac.ops.get_link_capabilities)) 4870 hw->mac.ops.get_link_capabilities(hw, &autoneg, &negotiate); 4871 if (hw->mac.ops.setup_link) 4872 hw->mac.ops.setup_link(hw, autoneg, TRUE); 4873 } /* ixgbe_handle_msf */ 4874 4875 /************************************************************************ 4876 * ixgbe_handle_phy - Tasklet for external PHY interrupts 4877 ************************************************************************/ 4878 static void 4879 ixgbe_handle_phy(void *context) 4880 { 4881 struct adapter *adapter = context; 4882 struct ixgbe_hw *hw = &adapter->hw; 4883 int error; 4884 4885 KASSERT(mutex_owned(&adapter->core_mtx)); 4886 4887 IXGBE_EVC_ADD(&adapter->phy_workev, 1); 4888 error = hw->phy.ops.handle_lasi(hw); 4889 if (error == IXGBE_ERR_OVERTEMP) 4890 device_printf(adapter->dev, 4891 "CRITICAL: EXTERNAL PHY OVER TEMP!! " 4892 " PHY will downshift to lower power state!\n"); 4893 else if (error) 4894 device_printf(adapter->dev, 4895 "Error handling LASI interrupt: %d\n", error); 4896 } /* ixgbe_handle_phy */ 4897 4898 static void 4899 ixgbe_handle_admin(struct work *wk, void *context) 4900 { 4901 struct adapter *adapter = context; 4902 struct ifnet *ifp = adapter->ifp; 4903 struct ixgbe_hw *hw = &adapter->hw; 4904 u32 task_requests; 4905 u32 eims_enable = 0; 4906 4907 mutex_enter(&adapter->admin_mtx); 4908 adapter->admin_pending = 0; 4909 task_requests = adapter->task_requests; 4910 adapter->task_requests = 0; 4911 mutex_exit(&adapter->admin_mtx); 4912 4913 /* 4914 * Hold the IFNET_LOCK across this entire call. This will 4915 * prevent additional changes to adapter->phy_layer 4916 * and serialize calls to this tasklet. We cannot hold the 4917 * CORE_LOCK while calling into the ifmedia functions as 4918 * they call ifmedia_lock() and the lock is CORE_LOCK. 4919 */ 4920 IFNET_LOCK(ifp); 4921 IXGBE_CORE_LOCK(adapter); 4922 if ((task_requests & IXGBE_REQUEST_TASK_LSC) != 0) { 4923 ixgbe_handle_link(adapter); 4924 eims_enable |= IXGBE_EIMS_LSC; 4925 } 4926 if ((task_requests & IXGBE_REQUEST_TASK_MOD_WOI) != 0) { 4927 ixgbe_handle_mod(adapter, false); 4928 } 4929 if ((task_requests & IXGBE_REQUEST_TASK_MOD) != 0) { 4930 ixgbe_handle_mod(adapter, true); 4931 if (hw->mac.type >= ixgbe_mac_X540) 4932 eims_enable |= IXGBE_EICR_GPI_SDP0_X540; 4933 else 4934 eims_enable |= IXGBE_EICR_GPI_SDP2_BY_MAC(hw); 4935 } 4936 if ((task_requests 4937 & (IXGBE_REQUEST_TASK_MSF_WOI | IXGBE_REQUEST_TASK_MSF)) != 0) { 4938 ixgbe_handle_msf(adapter); 4939 if (((task_requests & IXGBE_REQUEST_TASK_MSF) != 0) && 4940 (hw->mac.type == ixgbe_mac_82599EB)) 4941 eims_enable |= IXGBE_EIMS_GPI_SDP1_BY_MAC(hw); 4942 } 4943 if ((task_requests & IXGBE_REQUEST_TASK_PHY) != 0) { 4944 ixgbe_handle_phy(adapter); 4945 eims_enable |= IXGBE_EICR_GPI_SDP0_X540; 4946 } 4947 if ((task_requests & IXGBE_REQUEST_TASK_FDIR) != 0) { 4948 ixgbe_reinit_fdir(adapter); 4949 eims_enable |= IXGBE_EIMS_FLOW_DIR; 4950 } 4951 #if 0 /* notyet */ 4952 if ((task_requests & IXGBE_REQUEST_TASK_MBX) != 0) { 4953 ixgbe_handle_mbx(adapter); 4954 eims_enable |= IXGBE_EIMS_MAILBOX; 4955 } 4956 #endif 4957 IXGBE_WRITE_REG(hw, IXGBE_EIMS, eims_enable); 4958 4959 IXGBE_CORE_UNLOCK(adapter); 4960 IFNET_UNLOCK(ifp); 4961 } /* ixgbe_handle_admin */ 4962 4963 static void 4964 ixgbe_ifstop(struct ifnet *ifp, int disable) 4965 { 4966 struct adapter *adapter = ifp->if_softc; 4967 4968 IXGBE_CORE_LOCK(adapter); 4969 ixgbe_stop_locked(adapter); 4970 IXGBE_CORE_UNLOCK(adapter); 4971 4972 workqueue_wait(adapter->timer_wq, &adapter->timer_wc); 4973 atomic_store_relaxed(&adapter->timer_pending, 0); 4974 } 4975 4976 /************************************************************************ 4977 * ixgbe_stop_locked - Stop the hardware 4978 * 4979 * Disables all traffic on the adapter by issuing a 4980 * global reset on the MAC and deallocates TX/RX buffers. 4981 ************************************************************************/ 4982 static void 4983 ixgbe_stop_locked(void *arg) 4984 { 4985 struct ifnet *ifp; 4986 struct adapter *adapter = arg; 4987 struct ixgbe_hw *hw = &adapter->hw; 4988 4989 ifp = adapter->ifp; 4990 4991 KASSERT(mutex_owned(&adapter->core_mtx)); 4992 4993 INIT_DEBUGOUT("ixgbe_stop_locked: begin\n"); 4994 ixgbe_disable_intr(adapter); 4995 callout_stop(&adapter->timer); 4996 4997 /* Don't schedule workqueues. */ 4998 adapter->schedule_wqs_ok = false; 4999 5000 /* Let the stack know...*/ 5001 ifp->if_flags &= ~IFF_RUNNING; 5002 5003 ixgbe_reset_hw(hw); 5004 hw->adapter_stopped = FALSE; 5005 ixgbe_stop_adapter(hw); 5006 if (hw->mac.type == ixgbe_mac_82599EB) 5007 ixgbe_stop_mac_link_on_d3_82599(hw); 5008 /* Turn off the laser - noop with no optics */ 5009 ixgbe_disable_tx_laser(hw); 5010 5011 /* Update the stack */ 5012 adapter->link_up = FALSE; 5013 ixgbe_update_link_status(adapter); 5014 5015 /* reprogram the RAR[0] in case user changed it. */ 5016 ixgbe_set_rar(&adapter->hw, 0, adapter->hw.mac.addr, 0, IXGBE_RAH_AV); 5017 5018 return; 5019 } /* ixgbe_stop_locked */ 5020 5021 /************************************************************************ 5022 * ixgbe_update_link_status - Update OS on link state 5023 * 5024 * Note: Only updates the OS on the cached link state. 5025 * The real check of the hardware only happens with 5026 * a link interrupt. 5027 ************************************************************************/ 5028 static void 5029 ixgbe_update_link_status(struct adapter *adapter) 5030 { 5031 struct ifnet *ifp = adapter->ifp; 5032 device_t dev = adapter->dev; 5033 struct ixgbe_hw *hw = &adapter->hw; 5034 5035 KASSERT(mutex_owned(&adapter->core_mtx)); 5036 5037 if (adapter->link_up) { 5038 if (adapter->link_active != LINK_STATE_UP) { 5039 /* 5040 * To eliminate influence of the previous state 5041 * in the same way as ixgbe_init_locked(). 5042 */ 5043 struct ix_queue *que = adapter->queues; 5044 for (int i = 0; i < adapter->num_queues; i++, que++) 5045 que->eitr_setting = 0; 5046 5047 if (adapter->link_speed == IXGBE_LINK_SPEED_10GB_FULL){ 5048 /* 5049 * Discard count for both MAC Local Fault and 5050 * Remote Fault because those registers are 5051 * valid only when the link speed is up and 5052 * 10Gbps. 5053 */ 5054 IXGBE_READ_REG(hw, IXGBE_MLFC); 5055 IXGBE_READ_REG(hw, IXGBE_MRFC); 5056 } 5057 5058 if (bootverbose) { 5059 const char *bpsmsg; 5060 5061 switch (adapter->link_speed) { 5062 case IXGBE_LINK_SPEED_10GB_FULL: 5063 bpsmsg = "10 Gbps"; 5064 break; 5065 case IXGBE_LINK_SPEED_5GB_FULL: 5066 bpsmsg = "5 Gbps"; 5067 break; 5068 case IXGBE_LINK_SPEED_2_5GB_FULL: 5069 bpsmsg = "2.5 Gbps"; 5070 break; 5071 case IXGBE_LINK_SPEED_1GB_FULL: 5072 bpsmsg = "1 Gbps"; 5073 break; 5074 case IXGBE_LINK_SPEED_100_FULL: 5075 bpsmsg = "100 Mbps"; 5076 break; 5077 case IXGBE_LINK_SPEED_10_FULL: 5078 bpsmsg = "10 Mbps"; 5079 break; 5080 default: 5081 bpsmsg = "unknown speed"; 5082 break; 5083 } 5084 device_printf(dev, "Link is up %s %s \n", 5085 bpsmsg, "Full Duplex"); 5086 } 5087 adapter->link_active = LINK_STATE_UP; 5088 /* Update any Flow Control changes */ 5089 ixgbe_fc_enable(&adapter->hw); 5090 /* Update DMA coalescing config */ 5091 ixgbe_config_dmac(adapter); 5092 if_link_state_change(ifp, LINK_STATE_UP); 5093 5094 if (adapter->feat_en & IXGBE_FEATURE_SRIOV) 5095 ixgbe_ping_all_vfs(adapter); 5096 } 5097 } else { 5098 /* 5099 * Do it when link active changes to DOWN. i.e. 5100 * a) LINK_STATE_UNKNOWN -> LINK_STATE_DOWN 5101 * b) LINK_STATE_UP -> LINK_STATE_DOWN 5102 */ 5103 if (adapter->link_active != LINK_STATE_DOWN) { 5104 if (bootverbose) 5105 device_printf(dev, "Link is Down\n"); 5106 if_link_state_change(ifp, LINK_STATE_DOWN); 5107 adapter->link_active = LINK_STATE_DOWN; 5108 if (adapter->feat_en & IXGBE_FEATURE_SRIOV) 5109 ixgbe_ping_all_vfs(adapter); 5110 ixgbe_drain_all(adapter); 5111 } 5112 } 5113 } /* ixgbe_update_link_status */ 5114 5115 /************************************************************************ 5116 * ixgbe_config_dmac - Configure DMA Coalescing 5117 ************************************************************************/ 5118 static void 5119 ixgbe_config_dmac(struct adapter *adapter) 5120 { 5121 struct ixgbe_hw *hw = &adapter->hw; 5122 struct ixgbe_dmac_config *dcfg = &hw->mac.dmac_config; 5123 5124 if (hw->mac.type < ixgbe_mac_X550 || !hw->mac.ops.dmac_config) 5125 return; 5126 5127 if (dcfg->watchdog_timer ^ adapter->dmac || 5128 dcfg->link_speed ^ adapter->link_speed) { 5129 dcfg->watchdog_timer = adapter->dmac; 5130 dcfg->fcoe_en = false; 5131 dcfg->link_speed = adapter->link_speed; 5132 dcfg->num_tcs = 1; 5133 5134 INIT_DEBUGOUT2("dmac settings: watchdog %d, link speed %d\n", 5135 dcfg->watchdog_timer, dcfg->link_speed); 5136 5137 hw->mac.ops.dmac_config(hw); 5138 } 5139 } /* ixgbe_config_dmac */ 5140 5141 /************************************************************************ 5142 * ixgbe_enable_intr 5143 ************************************************************************/ 5144 static void 5145 ixgbe_enable_intr(struct adapter *adapter) 5146 { 5147 struct ixgbe_hw *hw = &adapter->hw; 5148 struct ix_queue *que = adapter->queues; 5149 u32 mask, fwsm; 5150 5151 mask = (IXGBE_EIMS_ENABLE_MASK & ~IXGBE_EIMS_RTX_QUEUE); 5152 5153 switch (adapter->hw.mac.type) { 5154 case ixgbe_mac_82599EB: 5155 mask |= IXGBE_EIMS_ECC; 5156 /* Temperature sensor on some adapters */ 5157 mask |= IXGBE_EIMS_GPI_SDP0; 5158 /* SFP+ (RX_LOS_N & MOD_ABS_N) */ 5159 mask |= IXGBE_EIMS_GPI_SDP1; 5160 mask |= IXGBE_EIMS_GPI_SDP2; 5161 break; 5162 case ixgbe_mac_X540: 5163 /* Detect if Thermal Sensor is enabled */ 5164 fwsm = IXGBE_READ_REG(hw, IXGBE_FWSM); 5165 if (fwsm & IXGBE_FWSM_TS_ENABLED) 5166 mask |= IXGBE_EIMS_TS; 5167 mask |= IXGBE_EIMS_ECC; 5168 break; 5169 case ixgbe_mac_X550: 5170 /* MAC thermal sensor is automatically enabled */ 5171 mask |= IXGBE_EIMS_TS; 5172 mask |= IXGBE_EIMS_ECC; 5173 break; 5174 case ixgbe_mac_X550EM_x: 5175 case ixgbe_mac_X550EM_a: 5176 /* Some devices use SDP0 for important information */ 5177 if (hw->device_id == IXGBE_DEV_ID_X550EM_X_SFP || 5178 hw->device_id == IXGBE_DEV_ID_X550EM_A_SFP || 5179 hw->device_id == IXGBE_DEV_ID_X550EM_A_SFP_N || 5180 hw->device_id == IXGBE_DEV_ID_X550EM_X_10G_T) 5181 mask |= IXGBE_EIMS_GPI_SDP0_BY_MAC(hw); 5182 if (hw->phy.type == ixgbe_phy_x550em_ext_t) 5183 mask |= IXGBE_EICR_GPI_SDP0_X540; 5184 mask |= IXGBE_EIMS_ECC; 5185 break; 5186 default: 5187 break; 5188 } 5189 5190 /* Enable Fan Failure detection */ 5191 if (adapter->feat_en & IXGBE_FEATURE_FAN_FAIL) 5192 mask |= IXGBE_EIMS_GPI_SDP1; 5193 /* Enable SR-IOV */ 5194 if (adapter->feat_en & IXGBE_FEATURE_SRIOV) 5195 mask |= IXGBE_EIMS_MAILBOX; 5196 /* Enable Flow Director */ 5197 if (adapter->feat_en & IXGBE_FEATURE_FDIR) 5198 mask |= IXGBE_EIMS_FLOW_DIR; 5199 5200 IXGBE_WRITE_REG(hw, IXGBE_EIMS, mask); 5201 5202 /* With MSI-X we use auto clear */ 5203 if ((adapter->feat_en & IXGBE_FEATURE_MSIX) != 0) { 5204 /* 5205 * We use auto clear for RTX_QUEUE only. Don't use other 5206 * interrupts (e.g. link interrupt). BTW, we don't use 5207 * TCP_TIMER interrupt itself. 5208 */ 5209 IXGBE_WRITE_REG(hw, IXGBE_EIAC, IXGBE_EIMS_RTX_QUEUE); 5210 } 5211 5212 /* 5213 * Now enable all queues, this is done separately to 5214 * allow for handling the extended (beyond 32) MSI-X 5215 * vectors that can be used by 82599 5216 */ 5217 for (int i = 0; i < adapter->num_queues; i++, que++) 5218 ixgbe_enable_queue(adapter, que->msix); 5219 5220 IXGBE_WRITE_FLUSH(hw); 5221 5222 } /* ixgbe_enable_intr */ 5223 5224 /************************************************************************ 5225 * ixgbe_disable_intr_internal 5226 ************************************************************************/ 5227 static void 5228 ixgbe_disable_intr_internal(struct adapter *adapter, bool nestok) 5229 { 5230 struct ix_queue *que = adapter->queues; 5231 5232 /* disable interrupts other than queues */ 5233 IXGBE_WRITE_REG(&adapter->hw, IXGBE_EIMC, ~IXGBE_EIMC_RTX_QUEUE); 5234 5235 if ((adapter->feat_en & IXGBE_FEATURE_MSIX) != 0) 5236 IXGBE_WRITE_REG(&adapter->hw, IXGBE_EIAC, 0); 5237 5238 for (int i = 0; i < adapter->num_queues; i++, que++) 5239 ixgbe_disable_queue_internal(adapter, que->msix, nestok); 5240 5241 IXGBE_WRITE_FLUSH(&adapter->hw); 5242 5243 } /* ixgbe_do_disable_intr_internal */ 5244 5245 /************************************************************************ 5246 * ixgbe_disable_intr 5247 ************************************************************************/ 5248 static void 5249 ixgbe_disable_intr(struct adapter *adapter) 5250 { 5251 5252 ixgbe_disable_intr_internal(adapter, true); 5253 } /* ixgbe_disable_intr */ 5254 5255 /************************************************************************ 5256 * ixgbe_ensure_disabled_intr 5257 ************************************************************************/ 5258 void 5259 ixgbe_ensure_disabled_intr(struct adapter *adapter) 5260 { 5261 5262 ixgbe_disable_intr_internal(adapter, false); 5263 } /* ixgbe_ensure_disabled_intr */ 5264 5265 /************************************************************************ 5266 * ixgbe_legacy_irq - Legacy Interrupt Service routine 5267 ************************************************************************/ 5268 static int 5269 ixgbe_legacy_irq(void *arg) 5270 { 5271 struct ix_queue *que = arg; 5272 struct adapter *adapter = que->adapter; 5273 struct ixgbe_hw *hw = &adapter->hw; 5274 struct ifnet *ifp = adapter->ifp; 5275 struct tx_ring *txr = adapter->tx_rings; 5276 u32 eicr; 5277 u32 eims_orig; 5278 u32 eims_enable = 0; 5279 u32 eims_disable = 0; 5280 5281 eims_orig = IXGBE_READ_REG(hw, IXGBE_EIMS); 5282 /* 5283 * Silicon errata #26 on 82598. Disable all interrupts before reading 5284 * EICR. 5285 */ 5286 IXGBE_WRITE_REG(hw, IXGBE_EIMC, IXGBE_IRQ_CLEAR_MASK); 5287 5288 /* Read and clear EICR */ 5289 eicr = IXGBE_READ_REG(hw, IXGBE_EICR); 5290 5291 if (eicr == 0) { 5292 IXGBE_EVC_ADD(&adapter->stats.pf.intzero, 1); 5293 IXGBE_WRITE_REG(hw, IXGBE_EIMS, eims_orig); 5294 return 0; 5295 } 5296 IXGBE_EVC_ADD(&adapter->stats.pf.legint, 1); 5297 5298 /* Queue (0) intr */ 5299 if (((ifp->if_flags & IFF_RUNNING) != 0) && 5300 (eicr & IXGBE_EIMC_RTX_QUEUE) != 0) { 5301 IXGBE_EVC_ADD(&que->irqs, 1); 5302 5303 /* 5304 * The same as ixgbe_msix_que() about 5305 * "que->txrx_use_workqueue". 5306 */ 5307 que->txrx_use_workqueue = adapter->txrx_use_workqueue; 5308 5309 IXGBE_TX_LOCK(txr); 5310 ixgbe_txeof(txr); 5311 #ifdef notyet 5312 if (!ixgbe_ring_empty(ifp, txr->br)) 5313 ixgbe_start_locked(ifp, txr); 5314 #endif 5315 IXGBE_TX_UNLOCK(txr); 5316 5317 IXGBE_EVC_ADD(&que->req, 1); 5318 ixgbe_sched_handle_que(adapter, que); 5319 /* Disable queue 0 interrupt */ 5320 eims_disable |= 1UL << 0; 5321 5322 } else 5323 eims_enable |= IXGBE_EIMC_RTX_QUEUE; 5324 5325 ixgbe_intr_admin_common(adapter, eicr, &eims_disable); 5326 5327 /* Re-enable some interrupts */ 5328 IXGBE_WRITE_REG(hw, IXGBE_EIMS, 5329 (eims_orig & ~eims_disable) | eims_enable); 5330 5331 return 1; 5332 } /* ixgbe_legacy_irq */ 5333 5334 /************************************************************************ 5335 * ixgbe_free_pciintr_resources 5336 ************************************************************************/ 5337 static void 5338 ixgbe_free_pciintr_resources(struct adapter *adapter) 5339 { 5340 struct ix_queue *que = adapter->queues; 5341 int rid; 5342 5343 /* 5344 * Release all msix queue resources: 5345 */ 5346 for (int i = 0; i < adapter->num_queues; i++, que++) { 5347 if (que->res != NULL) { 5348 pci_intr_disestablish(adapter->osdep.pc, 5349 adapter->osdep.ihs[i]); 5350 adapter->osdep.ihs[i] = NULL; 5351 } 5352 } 5353 5354 /* Clean the Legacy or Link interrupt last */ 5355 if (adapter->vector) /* we are doing MSIX */ 5356 rid = adapter->vector; 5357 else 5358 rid = 0; 5359 5360 if (adapter->osdep.ihs[rid] != NULL) { 5361 pci_intr_disestablish(adapter->osdep.pc, 5362 adapter->osdep.ihs[rid]); 5363 adapter->osdep.ihs[rid] = NULL; 5364 } 5365 5366 if (adapter->osdep.intrs != NULL) { 5367 pci_intr_release(adapter->osdep.pc, adapter->osdep.intrs, 5368 adapter->osdep.nintrs); 5369 adapter->osdep.intrs = NULL; 5370 } 5371 } /* ixgbe_free_pciintr_resources */ 5372 5373 /************************************************************************ 5374 * ixgbe_free_pci_resources 5375 ************************************************************************/ 5376 static void 5377 ixgbe_free_pci_resources(struct adapter *adapter) 5378 { 5379 5380 ixgbe_free_pciintr_resources(adapter); 5381 5382 if (adapter->osdep.mem_size != 0) { 5383 bus_space_unmap(adapter->osdep.mem_bus_space_tag, 5384 adapter->osdep.mem_bus_space_handle, 5385 adapter->osdep.mem_size); 5386 } 5387 5388 } /* ixgbe_free_pci_resources */ 5389 5390 /************************************************************************ 5391 * ixgbe_sysctl_flowcntl 5392 * 5393 * SYSCTL wrapper around setting Flow Control 5394 ************************************************************************/ 5395 static int 5396 ixgbe_sysctl_flowcntl(SYSCTLFN_ARGS) 5397 { 5398 struct sysctlnode node = *rnode; 5399 struct adapter *adapter = (struct adapter *)node.sysctl_data; 5400 int error, fc; 5401 5402 if (ixgbe_fw_recovery_mode_swflag(adapter)) 5403 return (EPERM); 5404 5405 fc = adapter->hw.fc.current_mode; 5406 node.sysctl_data = &fc; 5407 error = sysctl_lookup(SYSCTLFN_CALL(&node)); 5408 if (error != 0 || newp == NULL) 5409 return error; 5410 5411 /* Don't bother if it's not changed */ 5412 if (fc == adapter->hw.fc.current_mode) 5413 return (0); 5414 5415 return ixgbe_set_flowcntl(adapter, fc); 5416 } /* ixgbe_sysctl_flowcntl */ 5417 5418 /************************************************************************ 5419 * ixgbe_set_flowcntl - Set flow control 5420 * 5421 * Flow control values: 5422 * 0 - off 5423 * 1 - rx pause 5424 * 2 - tx pause 5425 * 3 - full 5426 ************************************************************************/ 5427 static int 5428 ixgbe_set_flowcntl(struct adapter *adapter, int fc) 5429 { 5430 switch (fc) { 5431 case ixgbe_fc_rx_pause: 5432 case ixgbe_fc_tx_pause: 5433 case ixgbe_fc_full: 5434 adapter->hw.fc.requested_mode = fc; 5435 if (adapter->num_queues > 1) 5436 ixgbe_disable_rx_drop(adapter); 5437 break; 5438 case ixgbe_fc_none: 5439 adapter->hw.fc.requested_mode = ixgbe_fc_none; 5440 if (adapter->num_queues > 1) 5441 ixgbe_enable_rx_drop(adapter); 5442 break; 5443 default: 5444 return (EINVAL); 5445 } 5446 5447 #if 0 /* XXX NetBSD */ 5448 /* Don't autoneg if forcing a value */ 5449 adapter->hw.fc.disable_fc_autoneg = TRUE; 5450 #endif 5451 ixgbe_fc_enable(&adapter->hw); 5452 5453 return (0); 5454 } /* ixgbe_set_flowcntl */ 5455 5456 /************************************************************************ 5457 * ixgbe_enable_rx_drop 5458 * 5459 * Enable the hardware to drop packets when the buffer is 5460 * full. This is useful with multiqueue, so that no single 5461 * queue being full stalls the entire RX engine. We only 5462 * enable this when Multiqueue is enabled AND Flow Control 5463 * is disabled. 5464 ************************************************************************/ 5465 static void 5466 ixgbe_enable_rx_drop(struct adapter *adapter) 5467 { 5468 struct ixgbe_hw *hw = &adapter->hw; 5469 struct rx_ring *rxr; 5470 u32 srrctl; 5471 5472 for (int i = 0; i < adapter->num_queues; i++) { 5473 rxr = &adapter->rx_rings[i]; 5474 srrctl = IXGBE_READ_REG(hw, IXGBE_SRRCTL(rxr->me)); 5475 srrctl |= IXGBE_SRRCTL_DROP_EN; 5476 IXGBE_WRITE_REG(hw, IXGBE_SRRCTL(rxr->me), srrctl); 5477 } 5478 5479 /* enable drop for each vf */ 5480 for (int i = 0; i < adapter->num_vfs; i++) { 5481 IXGBE_WRITE_REG(hw, IXGBE_QDE, 5482 (IXGBE_QDE_WRITE | (i << IXGBE_QDE_IDX_SHIFT) | 5483 IXGBE_QDE_ENABLE)); 5484 } 5485 } /* ixgbe_enable_rx_drop */ 5486 5487 /************************************************************************ 5488 * ixgbe_disable_rx_drop 5489 ************************************************************************/ 5490 static void 5491 ixgbe_disable_rx_drop(struct adapter *adapter) 5492 { 5493 struct ixgbe_hw *hw = &adapter->hw; 5494 struct rx_ring *rxr; 5495 u32 srrctl; 5496 5497 for (int i = 0; i < adapter->num_queues; i++) { 5498 rxr = &adapter->rx_rings[i]; 5499 srrctl = IXGBE_READ_REG(hw, IXGBE_SRRCTL(rxr->me)); 5500 srrctl &= ~IXGBE_SRRCTL_DROP_EN; 5501 IXGBE_WRITE_REG(hw, IXGBE_SRRCTL(rxr->me), srrctl); 5502 } 5503 5504 /* disable drop for each vf */ 5505 for (int i = 0; i < adapter->num_vfs; i++) { 5506 IXGBE_WRITE_REG(hw, IXGBE_QDE, 5507 (IXGBE_QDE_WRITE | (i << IXGBE_QDE_IDX_SHIFT))); 5508 } 5509 } /* ixgbe_disable_rx_drop */ 5510 5511 /************************************************************************ 5512 * ixgbe_sysctl_advertise 5513 * 5514 * SYSCTL wrapper around setting advertised speed 5515 ************************************************************************/ 5516 static int 5517 ixgbe_sysctl_advertise(SYSCTLFN_ARGS) 5518 { 5519 struct sysctlnode node = *rnode; 5520 struct adapter *adapter = (struct adapter *)node.sysctl_data; 5521 int error = 0, advertise; 5522 5523 if (ixgbe_fw_recovery_mode_swflag(adapter)) 5524 return (EPERM); 5525 5526 advertise = adapter->advertise; 5527 node.sysctl_data = &advertise; 5528 error = sysctl_lookup(SYSCTLFN_CALL(&node)); 5529 if (error != 0 || newp == NULL) 5530 return error; 5531 5532 return ixgbe_set_advertise(adapter, advertise); 5533 } /* ixgbe_sysctl_advertise */ 5534 5535 /************************************************************************ 5536 * ixgbe_set_advertise - Control advertised link speed 5537 * 5538 * Flags: 5539 * 0x00 - Default (all capable link speed) 5540 * 0x1 - advertise 100 Mb 5541 * 0x2 - advertise 1G 5542 * 0x4 - advertise 10G 5543 * 0x8 - advertise 10 Mb (yes, Mb) 5544 * 0x10 - advertise 2.5G 5545 * 0x20 - advertise 5G 5546 ************************************************************************/ 5547 static int 5548 ixgbe_set_advertise(struct adapter *adapter, int advertise) 5549 { 5550 device_t dev; 5551 struct ixgbe_hw *hw; 5552 ixgbe_link_speed speed = 0; 5553 ixgbe_link_speed link_caps = 0; 5554 s32 err = IXGBE_NOT_IMPLEMENTED; 5555 bool negotiate = FALSE; 5556 5557 /* Checks to validate new value */ 5558 if (adapter->advertise == advertise) /* no change */ 5559 return (0); 5560 5561 dev = adapter->dev; 5562 hw = &adapter->hw; 5563 5564 /* No speed changes for backplane media */ 5565 if (hw->phy.media_type == ixgbe_media_type_backplane) 5566 return (ENODEV); 5567 5568 if (!((hw->phy.media_type == ixgbe_media_type_copper) || 5569 (hw->phy.multispeed_fiber))) { 5570 device_printf(dev, 5571 "Advertised speed can only be set on copper or " 5572 "multispeed fiber media types.\n"); 5573 return (EINVAL); 5574 } 5575 5576 if (advertise < 0x0 || advertise > 0x3f) { 5577 device_printf(dev, "Invalid advertised speed; valid modes are 0x0 through 0x3f\n"); 5578 return (EINVAL); 5579 } 5580 5581 if (hw->mac.ops.get_link_capabilities) { 5582 err = hw->mac.ops.get_link_capabilities(hw, &link_caps, 5583 &negotiate); 5584 if (err != IXGBE_SUCCESS) { 5585 device_printf(dev, "Unable to determine supported advertise speeds\n"); 5586 return (ENODEV); 5587 } 5588 } 5589 5590 /* Set new value and report new advertised mode */ 5591 if (advertise & 0x1) { 5592 if (!(link_caps & IXGBE_LINK_SPEED_100_FULL)) { 5593 device_printf(dev, "Interface does not support 100Mb advertised speed\n"); 5594 return (EINVAL); 5595 } 5596 speed |= IXGBE_LINK_SPEED_100_FULL; 5597 } 5598 if (advertise & 0x2) { 5599 if (!(link_caps & IXGBE_LINK_SPEED_1GB_FULL)) { 5600 device_printf(dev, "Interface does not support 1Gb advertised speed\n"); 5601 return (EINVAL); 5602 } 5603 speed |= IXGBE_LINK_SPEED_1GB_FULL; 5604 } 5605 if (advertise & 0x4) { 5606 if (!(link_caps & IXGBE_LINK_SPEED_10GB_FULL)) { 5607 device_printf(dev, "Interface does not support 10Gb advertised speed\n"); 5608 return (EINVAL); 5609 } 5610 speed |= IXGBE_LINK_SPEED_10GB_FULL; 5611 } 5612 if (advertise & 0x8) { 5613 if (!(link_caps & IXGBE_LINK_SPEED_10_FULL)) { 5614 device_printf(dev, "Interface does not support 10Mb advertised speed\n"); 5615 return (EINVAL); 5616 } 5617 speed |= IXGBE_LINK_SPEED_10_FULL; 5618 } 5619 if (advertise & 0x10) { 5620 if (!(link_caps & IXGBE_LINK_SPEED_2_5GB_FULL)) { 5621 device_printf(dev, "Interface does not support 2.5Gb advertised speed\n"); 5622 return (EINVAL); 5623 } 5624 speed |= IXGBE_LINK_SPEED_2_5GB_FULL; 5625 } 5626 if (advertise & 0x20) { 5627 if (!(link_caps & IXGBE_LINK_SPEED_5GB_FULL)) { 5628 device_printf(dev, "Interface does not support 5Gb advertised speed\n"); 5629 return (EINVAL); 5630 } 5631 speed |= IXGBE_LINK_SPEED_5GB_FULL; 5632 } 5633 if (advertise == 0) 5634 speed = link_caps; /* All capable link speed */ 5635 5636 hw->mac.autotry_restart = TRUE; 5637 hw->mac.ops.setup_link(hw, speed, TRUE); 5638 adapter->advertise = advertise; 5639 5640 return (0); 5641 } /* ixgbe_set_advertise */ 5642 5643 /************************************************************************ 5644 * ixgbe_get_default_advertise - Get default advertised speed settings 5645 * 5646 * Formatted for sysctl usage. 5647 * Flags: 5648 * 0x1 - advertise 100 Mb 5649 * 0x2 - advertise 1G 5650 * 0x4 - advertise 10G 5651 * 0x8 - advertise 10 Mb (yes, Mb) 5652 * 0x10 - advertise 2.5G 5653 * 0x20 - advertise 5G 5654 ************************************************************************/ 5655 static int 5656 ixgbe_get_default_advertise(struct adapter *adapter) 5657 { 5658 struct ixgbe_hw *hw = &adapter->hw; 5659 int speed; 5660 ixgbe_link_speed link_caps = 0; 5661 s32 err; 5662 bool negotiate = FALSE; 5663 5664 /* 5665 * Advertised speed means nothing unless it's copper or 5666 * multi-speed fiber 5667 */ 5668 if (!(hw->phy.media_type == ixgbe_media_type_copper) && 5669 !(hw->phy.multispeed_fiber)) 5670 return (0); 5671 5672 err = hw->mac.ops.get_link_capabilities(hw, &link_caps, &negotiate); 5673 if (err != IXGBE_SUCCESS) 5674 return (0); 5675 5676 speed = 5677 ((link_caps & IXGBE_LINK_SPEED_10GB_FULL) ? 0x4 : 0) | 5678 ((link_caps & IXGBE_LINK_SPEED_5GB_FULL) ? 0x20 : 0) | 5679 ((link_caps & IXGBE_LINK_SPEED_2_5GB_FULL) ? 0x10 : 0) | 5680 ((link_caps & IXGBE_LINK_SPEED_1GB_FULL) ? 0x2 : 0) | 5681 ((link_caps & IXGBE_LINK_SPEED_100_FULL) ? 0x1 : 0) | 5682 ((link_caps & IXGBE_LINK_SPEED_10_FULL) ? 0x8 : 0); 5683 5684 return speed; 5685 } /* ixgbe_get_default_advertise */ 5686 5687 /************************************************************************ 5688 * ixgbe_sysctl_dmac - Manage DMA Coalescing 5689 * 5690 * Control values: 5691 * 0/1 - off / on (use default value of 1000) 5692 * 5693 * Legal timer values are: 5694 * 50,100,250,500,1000,2000,5000,10000 5695 * 5696 * Turning off interrupt moderation will also turn this off. 5697 ************************************************************************/ 5698 static int 5699 ixgbe_sysctl_dmac(SYSCTLFN_ARGS) 5700 { 5701 struct sysctlnode node = *rnode; 5702 struct adapter *adapter = (struct adapter *)node.sysctl_data; 5703 struct ifnet *ifp = adapter->ifp; 5704 int error; 5705 int newval; 5706 5707 if (ixgbe_fw_recovery_mode_swflag(adapter)) 5708 return (EPERM); 5709 5710 newval = adapter->dmac; 5711 node.sysctl_data = &newval; 5712 error = sysctl_lookup(SYSCTLFN_CALL(&node)); 5713 if ((error) || (newp == NULL)) 5714 return (error); 5715 5716 switch (newval) { 5717 case 0: 5718 /* Disabled */ 5719 adapter->dmac = 0; 5720 break; 5721 case 1: 5722 /* Enable and use default */ 5723 adapter->dmac = 1000; 5724 break; 5725 case 50: 5726 case 100: 5727 case 250: 5728 case 500: 5729 case 1000: 5730 case 2000: 5731 case 5000: 5732 case 10000: 5733 /* Legal values - allow */ 5734 adapter->dmac = newval; 5735 break; 5736 default: 5737 /* Do nothing, illegal value */ 5738 return (EINVAL); 5739 } 5740 5741 /* Re-initialize hardware if it's already running */ 5742 if (ifp->if_flags & IFF_RUNNING) 5743 if_init(ifp); 5744 5745 return (0); 5746 } 5747 5748 #ifdef IXGBE_DEBUG 5749 /************************************************************************ 5750 * ixgbe_sysctl_power_state 5751 * 5752 * Sysctl to test power states 5753 * Values: 5754 * 0 - set device to D0 5755 * 3 - set device to D3 5756 * (none) - get current device power state 5757 ************************************************************************/ 5758 static int 5759 ixgbe_sysctl_power_state(SYSCTLFN_ARGS) 5760 { 5761 #ifdef notyet 5762 struct sysctlnode node = *rnode; 5763 struct adapter *adapter = (struct adapter *)node.sysctl_data; 5764 device_t dev = adapter->dev; 5765 int curr_ps, new_ps, error = 0; 5766 5767 if (ixgbe_fw_recovery_mode_swflag(adapter)) 5768 return (EPERM); 5769 5770 curr_ps = new_ps = pci_get_powerstate(dev); 5771 5772 error = sysctl_lookup(SYSCTLFN_CALL(&node)); 5773 if ((error) || (req->newp == NULL)) 5774 return (error); 5775 5776 if (new_ps == curr_ps) 5777 return (0); 5778 5779 if (new_ps == 3 && curr_ps == 0) 5780 error = DEVICE_SUSPEND(dev); 5781 else if (new_ps == 0 && curr_ps == 3) 5782 error = DEVICE_RESUME(dev); 5783 else 5784 return (EINVAL); 5785 5786 device_printf(dev, "New state: %d\n", pci_get_powerstate(dev)); 5787 5788 return (error); 5789 #else 5790 return 0; 5791 #endif 5792 } /* ixgbe_sysctl_power_state */ 5793 #endif 5794 5795 /************************************************************************ 5796 * ixgbe_sysctl_wol_enable 5797 * 5798 * Sysctl to enable/disable the WoL capability, 5799 * if supported by the adapter. 5800 * 5801 * Values: 5802 * 0 - disabled 5803 * 1 - enabled 5804 ************************************************************************/ 5805 static int 5806 ixgbe_sysctl_wol_enable(SYSCTLFN_ARGS) 5807 { 5808 struct sysctlnode node = *rnode; 5809 struct adapter *adapter = (struct adapter *)node.sysctl_data; 5810 struct ixgbe_hw *hw = &adapter->hw; 5811 bool new_wol_enabled; 5812 int error = 0; 5813 5814 /* 5815 * It's not required to check recovery mode because this function never 5816 * touches hardware. 5817 */ 5818 new_wol_enabled = hw->wol_enabled; 5819 node.sysctl_data = &new_wol_enabled; 5820 error = sysctl_lookup(SYSCTLFN_CALL(&node)); 5821 if ((error) || (newp == NULL)) 5822 return (error); 5823 if (new_wol_enabled == hw->wol_enabled) 5824 return (0); 5825 5826 if (new_wol_enabled && !adapter->wol_support) 5827 return (ENODEV); 5828 else 5829 hw->wol_enabled = new_wol_enabled; 5830 5831 return (0); 5832 } /* ixgbe_sysctl_wol_enable */ 5833 5834 /************************************************************************ 5835 * ixgbe_sysctl_wufc - Wake Up Filter Control 5836 * 5837 * Sysctl to enable/disable the types of packets that the 5838 * adapter will wake up on upon receipt. 5839 * Flags: 5840 * 0x1 - Link Status Change 5841 * 0x2 - Magic Packet 5842 * 0x4 - Direct Exact 5843 * 0x8 - Directed Multicast 5844 * 0x10 - Broadcast 5845 * 0x20 - ARP/IPv4 Request Packet 5846 * 0x40 - Direct IPv4 Packet 5847 * 0x80 - Direct IPv6 Packet 5848 * 5849 * Settings not listed above will cause the sysctl to return an error. 5850 ************************************************************************/ 5851 static int 5852 ixgbe_sysctl_wufc(SYSCTLFN_ARGS) 5853 { 5854 struct sysctlnode node = *rnode; 5855 struct adapter *adapter = (struct adapter *)node.sysctl_data; 5856 int error = 0; 5857 u32 new_wufc; 5858 5859 /* 5860 * It's not required to check recovery mode because this function never 5861 * touches hardware. 5862 */ 5863 new_wufc = adapter->wufc; 5864 node.sysctl_data = &new_wufc; 5865 error = sysctl_lookup(SYSCTLFN_CALL(&node)); 5866 if ((error) || (newp == NULL)) 5867 return (error); 5868 if (new_wufc == adapter->wufc) 5869 return (0); 5870 5871 if (new_wufc & 0xffffff00) 5872 return (EINVAL); 5873 5874 new_wufc &= 0xff; 5875 new_wufc |= (0xffffff & adapter->wufc); 5876 adapter->wufc = new_wufc; 5877 5878 return (0); 5879 } /* ixgbe_sysctl_wufc */ 5880 5881 #ifdef IXGBE_DEBUG 5882 /************************************************************************ 5883 * ixgbe_sysctl_print_rss_config 5884 ************************************************************************/ 5885 static int 5886 ixgbe_sysctl_print_rss_config(SYSCTLFN_ARGS) 5887 { 5888 #ifdef notyet 5889 struct sysctlnode node = *rnode; 5890 struct adapter *adapter = (struct adapter *)node.sysctl_data; 5891 struct ixgbe_hw *hw = &adapter->hw; 5892 device_t dev = adapter->dev; 5893 struct sbuf *buf; 5894 int error = 0, reta_size; 5895 u32 reg; 5896 5897 if (ixgbe_fw_recovery_mode_swflag(adapter)) 5898 return (EPERM); 5899 5900 buf = sbuf_new_for_sysctl(NULL, NULL, 128, req); 5901 if (!buf) { 5902 device_printf(dev, "Could not allocate sbuf for output.\n"); 5903 return (ENOMEM); 5904 } 5905 5906 // TODO: use sbufs to make a string to print out 5907 /* Set multiplier for RETA setup and table size based on MAC */ 5908 switch (adapter->hw.mac.type) { 5909 case ixgbe_mac_X550: 5910 case ixgbe_mac_X550EM_x: 5911 case ixgbe_mac_X550EM_a: 5912 reta_size = 128; 5913 break; 5914 default: 5915 reta_size = 32; 5916 break; 5917 } 5918 5919 /* Print out the redirection table */ 5920 sbuf_cat(buf, "\n"); 5921 for (int i = 0; i < reta_size; i++) { 5922 if (i < 32) { 5923 reg = IXGBE_READ_REG(hw, IXGBE_RETA(i)); 5924 sbuf_printf(buf, "RETA(%2d): 0x%08x\n", i, reg); 5925 } else { 5926 reg = IXGBE_READ_REG(hw, IXGBE_ERETA(i - 32)); 5927 sbuf_printf(buf, "ERETA(%2d): 0x%08x\n", i - 32, reg); 5928 } 5929 } 5930 5931 // TODO: print more config 5932 5933 error = sbuf_finish(buf); 5934 if (error) 5935 device_printf(dev, "Error finishing sbuf: %d\n", error); 5936 5937 sbuf_delete(buf); 5938 #endif 5939 return (0); 5940 } /* ixgbe_sysctl_print_rss_config */ 5941 #endif /* IXGBE_DEBUG */ 5942 5943 /************************************************************************ 5944 * ixgbe_sysctl_phy_temp - Retrieve temperature of PHY 5945 * 5946 * For X552/X557-AT devices using an external PHY 5947 ************************************************************************/ 5948 static int 5949 ixgbe_sysctl_phy_temp(SYSCTLFN_ARGS) 5950 { 5951 struct sysctlnode node = *rnode; 5952 struct adapter *adapter = (struct adapter *)node.sysctl_data; 5953 struct ixgbe_hw *hw = &adapter->hw; 5954 int val; 5955 u16 reg; 5956 int error; 5957 5958 if (ixgbe_fw_recovery_mode_swflag(adapter)) 5959 return (EPERM); 5960 5961 if (hw->device_id != IXGBE_DEV_ID_X550EM_X_10G_T) { 5962 device_printf(adapter->dev, 5963 "Device has no supported external thermal sensor.\n"); 5964 return (ENODEV); 5965 } 5966 5967 if (hw->phy.ops.read_reg(hw, IXGBE_PHY_CURRENT_TEMP, 5968 IXGBE_MDIO_VENDOR_SPECIFIC_1_DEV_TYPE, ®)) { 5969 device_printf(adapter->dev, 5970 "Error reading from PHY's current temperature register\n"); 5971 return (EAGAIN); 5972 } 5973 5974 node.sysctl_data = &val; 5975 5976 /* Shift temp for output */ 5977 val = reg >> 8; 5978 5979 error = sysctl_lookup(SYSCTLFN_CALL(&node)); 5980 if ((error) || (newp == NULL)) 5981 return (error); 5982 5983 return (0); 5984 } /* ixgbe_sysctl_phy_temp */ 5985 5986 /************************************************************************ 5987 * ixgbe_sysctl_phy_overtemp_occurred 5988 * 5989 * Reports (directly from the PHY) whether the current PHY 5990 * temperature is over the overtemp threshold. 5991 ************************************************************************/ 5992 static int 5993 ixgbe_sysctl_phy_overtemp_occurred(SYSCTLFN_ARGS) 5994 { 5995 struct sysctlnode node = *rnode; 5996 struct adapter *adapter = (struct adapter *)node.sysctl_data; 5997 struct ixgbe_hw *hw = &adapter->hw; 5998 int val, error; 5999 u16 reg; 6000 6001 if (ixgbe_fw_recovery_mode_swflag(adapter)) 6002 return (EPERM); 6003 6004 if (hw->device_id != IXGBE_DEV_ID_X550EM_X_10G_T) { 6005 device_printf(adapter->dev, 6006 "Device has no supported external thermal sensor.\n"); 6007 return (ENODEV); 6008 } 6009 6010 if (hw->phy.ops.read_reg(hw, IXGBE_PHY_OVERTEMP_STATUS, 6011 IXGBE_MDIO_VENDOR_SPECIFIC_1_DEV_TYPE, ®)) { 6012 device_printf(adapter->dev, 6013 "Error reading from PHY's temperature status register\n"); 6014 return (EAGAIN); 6015 } 6016 6017 node.sysctl_data = &val; 6018 6019 /* Get occurrence bit */ 6020 val = !!(reg & 0x4000); 6021 6022 error = sysctl_lookup(SYSCTLFN_CALL(&node)); 6023 if ((error) || (newp == NULL)) 6024 return (error); 6025 6026 return (0); 6027 } /* ixgbe_sysctl_phy_overtemp_occurred */ 6028 6029 /************************************************************************ 6030 * ixgbe_sysctl_eee_state 6031 * 6032 * Sysctl to set EEE power saving feature 6033 * Values: 6034 * 0 - disable EEE 6035 * 1 - enable EEE 6036 * (none) - get current device EEE state 6037 ************************************************************************/ 6038 static int 6039 ixgbe_sysctl_eee_state(SYSCTLFN_ARGS) 6040 { 6041 struct sysctlnode node = *rnode; 6042 struct adapter *adapter = (struct adapter *)node.sysctl_data; 6043 struct ifnet *ifp = adapter->ifp; 6044 device_t dev = adapter->dev; 6045 int curr_eee, new_eee, error = 0; 6046 s32 retval; 6047 6048 if (ixgbe_fw_recovery_mode_swflag(adapter)) 6049 return (EPERM); 6050 6051 curr_eee = new_eee = !!(adapter->feat_en & IXGBE_FEATURE_EEE); 6052 node.sysctl_data = &new_eee; 6053 error = sysctl_lookup(SYSCTLFN_CALL(&node)); 6054 if ((error) || (newp == NULL)) 6055 return (error); 6056 6057 /* Nothing to do */ 6058 if (new_eee == curr_eee) 6059 return (0); 6060 6061 /* Not supported */ 6062 if (!(adapter->feat_cap & IXGBE_FEATURE_EEE)) 6063 return (EINVAL); 6064 6065 /* Bounds checking */ 6066 if ((new_eee < 0) || (new_eee > 1)) 6067 return (EINVAL); 6068 6069 retval = ixgbe_setup_eee(&adapter->hw, new_eee); 6070 if (retval) { 6071 device_printf(dev, "Error in EEE setup: 0x%08X\n", retval); 6072 return (EINVAL); 6073 } 6074 6075 /* Restart auto-neg */ 6076 if_init(ifp); 6077 6078 device_printf(dev, "New EEE state: %d\n", new_eee); 6079 6080 /* Cache new value */ 6081 if (new_eee) 6082 adapter->feat_en |= IXGBE_FEATURE_EEE; 6083 else 6084 adapter->feat_en &= ~IXGBE_FEATURE_EEE; 6085 6086 return (error); 6087 } /* ixgbe_sysctl_eee_state */ 6088 6089 #define PRINTQS(adapter, regname) \ 6090 do { \ 6091 struct ixgbe_hw *_hw = &(adapter)->hw; \ 6092 int _i; \ 6093 \ 6094 printf("%s: %s", device_xname((adapter)->dev), #regname); \ 6095 for (_i = 0; _i < (adapter)->num_queues; _i++) { \ 6096 printf((_i == 0) ? "\t" : " "); \ 6097 printf("%08x", IXGBE_READ_REG(_hw, \ 6098 IXGBE_##regname(_i))); \ 6099 } \ 6100 printf("\n"); \ 6101 } while (0) 6102 6103 /************************************************************************ 6104 * ixgbe_print_debug_info 6105 * 6106 * Called only when em_display_debug_stats is enabled. 6107 * Provides a way to take a look at important statistics 6108 * maintained by the driver and hardware. 6109 ************************************************************************/ 6110 static void 6111 ixgbe_print_debug_info(struct adapter *adapter) 6112 { 6113 device_t dev = adapter->dev; 6114 struct ixgbe_hw *hw = &adapter->hw; 6115 int table_size; 6116 int i; 6117 6118 switch (adapter->hw.mac.type) { 6119 case ixgbe_mac_X550: 6120 case ixgbe_mac_X550EM_x: 6121 case ixgbe_mac_X550EM_a: 6122 table_size = 128; 6123 break; 6124 default: 6125 table_size = 32; 6126 break; 6127 } 6128 6129 device_printf(dev, "[E]RETA:\n"); 6130 for (i = 0; i < table_size; i++) { 6131 if (i < 32) 6132 printf("%02x: %08x\n", i, IXGBE_READ_REG(hw, 6133 IXGBE_RETA(i))); 6134 else 6135 printf("%02x: %08x\n", i, IXGBE_READ_REG(hw, 6136 IXGBE_ERETA(i - 32))); 6137 } 6138 6139 device_printf(dev, "queue:"); 6140 for (i = 0; i < adapter->num_queues; i++) { 6141 printf((i == 0) ? "\t" : " "); 6142 printf("%8d", i); 6143 } 6144 printf("\n"); 6145 PRINTQS(adapter, RDBAL); 6146 PRINTQS(adapter, RDBAH); 6147 PRINTQS(adapter, RDLEN); 6148 PRINTQS(adapter, SRRCTL); 6149 PRINTQS(adapter, RDH); 6150 PRINTQS(adapter, RDT); 6151 PRINTQS(adapter, RXDCTL); 6152 6153 device_printf(dev, "RQSMR:"); 6154 for (i = 0; i < adapter->num_queues / 4; i++) { 6155 printf((i == 0) ? "\t" : " "); 6156 printf("%08x", IXGBE_READ_REG(hw, IXGBE_RQSMR(i))); 6157 } 6158 printf("\n"); 6159 6160 device_printf(dev, "disabled_count:"); 6161 for (i = 0; i < adapter->num_queues; i++) { 6162 printf((i == 0) ? "\t" : " "); 6163 printf("%8d", adapter->queues[i].disabled_count); 6164 } 6165 printf("\n"); 6166 6167 device_printf(dev, "EIMS:\t%08x\n", IXGBE_READ_REG(hw, IXGBE_EIMS)); 6168 if (hw->mac.type != ixgbe_mac_82598EB) { 6169 device_printf(dev, "EIMS_EX(0):\t%08x\n", 6170 IXGBE_READ_REG(hw, IXGBE_EIMS_EX(0))); 6171 device_printf(dev, "EIMS_EX(1):\t%08x\n", 6172 IXGBE_READ_REG(hw, IXGBE_EIMS_EX(1))); 6173 } 6174 device_printf(dev, "EIAM:\t%08x\n", IXGBE_READ_REG(hw, IXGBE_EIAM)); 6175 device_printf(dev, "EIAC:\t%08x\n", IXGBE_READ_REG(hw, IXGBE_EIAC)); 6176 } /* ixgbe_print_debug_info */ 6177 6178 /************************************************************************ 6179 * ixgbe_sysctl_debug 6180 ************************************************************************/ 6181 static int 6182 ixgbe_sysctl_debug(SYSCTLFN_ARGS) 6183 { 6184 struct sysctlnode node = *rnode; 6185 struct adapter *adapter = (struct adapter *)node.sysctl_data; 6186 int error, result = 0; 6187 6188 if (ixgbe_fw_recovery_mode_swflag(adapter)) 6189 return (EPERM); 6190 6191 node.sysctl_data = &result; 6192 error = sysctl_lookup(SYSCTLFN_CALL(&node)); 6193 6194 if (error || newp == NULL) 6195 return error; 6196 6197 if (result == 1) 6198 ixgbe_print_debug_info(adapter); 6199 6200 return 0; 6201 } /* ixgbe_sysctl_debug */ 6202 6203 /************************************************************************ 6204 * ixgbe_sysctl_rx_copy_len 6205 ************************************************************************/ 6206 static int 6207 ixgbe_sysctl_rx_copy_len(SYSCTLFN_ARGS) 6208 { 6209 struct sysctlnode node = *rnode; 6210 struct adapter *adapter = (struct adapter *)node.sysctl_data; 6211 int error; 6212 int result = adapter->rx_copy_len; 6213 6214 node.sysctl_data = &result; 6215 error = sysctl_lookup(SYSCTLFN_CALL(&node)); 6216 6217 if (error || newp == NULL) 6218 return error; 6219 6220 if ((result < 0) || (result > IXGBE_RX_COPY_LEN_MAX)) 6221 return EINVAL; 6222 6223 adapter->rx_copy_len = result; 6224 6225 return 0; 6226 } /* ixgbe_sysctl_rx_copy_len */ 6227 6228 /************************************************************************ 6229 * ixgbe_sysctl_tx_process_limit 6230 ************************************************************************/ 6231 static int 6232 ixgbe_sysctl_tx_process_limit(SYSCTLFN_ARGS) 6233 { 6234 struct sysctlnode node = *rnode; 6235 struct adapter *adapter = (struct adapter *)node.sysctl_data; 6236 int error; 6237 int result = adapter->tx_process_limit; 6238 6239 node.sysctl_data = &result; 6240 error = sysctl_lookup(SYSCTLFN_CALL(&node)); 6241 6242 if (error || newp == NULL) 6243 return error; 6244 6245 if ((result <= 0) || (result > adapter->num_tx_desc)) 6246 return EINVAL; 6247 6248 adapter->tx_process_limit = result; 6249 6250 return 0; 6251 } /* ixgbe_sysctl_tx_process_limit */ 6252 6253 /************************************************************************ 6254 * ixgbe_sysctl_rx_process_limit 6255 ************************************************************************/ 6256 static int 6257 ixgbe_sysctl_rx_process_limit(SYSCTLFN_ARGS) 6258 { 6259 struct sysctlnode node = *rnode; 6260 struct adapter *adapter = (struct adapter *)node.sysctl_data; 6261 int error; 6262 int result = adapter->rx_process_limit; 6263 6264 node.sysctl_data = &result; 6265 error = sysctl_lookup(SYSCTLFN_CALL(&node)); 6266 6267 if (error || newp == NULL) 6268 return error; 6269 6270 if ((result <= 0) || (result > adapter->num_rx_desc)) 6271 return EINVAL; 6272 6273 adapter->rx_process_limit = result; 6274 6275 return 0; 6276 } /* ixgbe_sysctl_rx_process_limit */ 6277 6278 /************************************************************************ 6279 * ixgbe_init_device_features 6280 ************************************************************************/ 6281 static void 6282 ixgbe_init_device_features(struct adapter *adapter) 6283 { 6284 adapter->feat_cap = IXGBE_FEATURE_NETMAP 6285 | IXGBE_FEATURE_RSS 6286 | IXGBE_FEATURE_MSI 6287 | IXGBE_FEATURE_MSIX 6288 | IXGBE_FEATURE_LEGACY_IRQ 6289 | IXGBE_FEATURE_LEGACY_TX; 6290 6291 /* Set capabilities first... */ 6292 switch (adapter->hw.mac.type) { 6293 case ixgbe_mac_82598EB: 6294 if (adapter->hw.device_id == IXGBE_DEV_ID_82598AT) 6295 adapter->feat_cap |= IXGBE_FEATURE_FAN_FAIL; 6296 break; 6297 case ixgbe_mac_X540: 6298 adapter->feat_cap |= IXGBE_FEATURE_SRIOV; 6299 adapter->feat_cap |= IXGBE_FEATURE_FDIR; 6300 if ((adapter->hw.device_id == IXGBE_DEV_ID_X540_BYPASS) && 6301 (adapter->hw.bus.func == 0)) 6302 adapter->feat_cap |= IXGBE_FEATURE_BYPASS; 6303 break; 6304 case ixgbe_mac_X550: 6305 /* 6306 * IXGBE_FEATURE_RECOVERY_MODE will be set after reading 6307 * NVM Image version. 6308 */ 6309 adapter->feat_cap |= IXGBE_FEATURE_TEMP_SENSOR; 6310 adapter->feat_cap |= IXGBE_FEATURE_SRIOV; 6311 adapter->feat_cap |= IXGBE_FEATURE_FDIR; 6312 break; 6313 case ixgbe_mac_X550EM_x: 6314 /* 6315 * IXGBE_FEATURE_RECOVERY_MODE will be set after reading 6316 * NVM Image version. 6317 */ 6318 adapter->feat_cap |= IXGBE_FEATURE_SRIOV; 6319 adapter->feat_cap |= IXGBE_FEATURE_FDIR; 6320 break; 6321 case ixgbe_mac_X550EM_a: 6322 /* 6323 * IXGBE_FEATURE_RECOVERY_MODE will be set after reading 6324 * NVM Image version. 6325 */ 6326 adapter->feat_cap |= IXGBE_FEATURE_SRIOV; 6327 adapter->feat_cap |= IXGBE_FEATURE_FDIR; 6328 adapter->feat_cap &= ~IXGBE_FEATURE_LEGACY_IRQ; 6329 if ((adapter->hw.device_id == IXGBE_DEV_ID_X550EM_A_1G_T) || 6330 (adapter->hw.device_id == IXGBE_DEV_ID_X550EM_A_1G_T_L)) { 6331 adapter->feat_cap |= IXGBE_FEATURE_TEMP_SENSOR; 6332 adapter->feat_cap |= IXGBE_FEATURE_EEE; 6333 } 6334 break; 6335 case ixgbe_mac_82599EB: 6336 adapter->feat_cap |= IXGBE_FEATURE_SRIOV; 6337 adapter->feat_cap |= IXGBE_FEATURE_FDIR; 6338 if ((adapter->hw.device_id == IXGBE_DEV_ID_82599_BYPASS) && 6339 (adapter->hw.bus.func == 0)) 6340 adapter->feat_cap |= IXGBE_FEATURE_BYPASS; 6341 if (adapter->hw.device_id == IXGBE_DEV_ID_82599_QSFP_SF_QP) 6342 adapter->feat_cap &= ~IXGBE_FEATURE_LEGACY_IRQ; 6343 break; 6344 default: 6345 break; 6346 } 6347 6348 /* Enabled by default... */ 6349 /* Fan failure detection */ 6350 if (adapter->feat_cap & IXGBE_FEATURE_FAN_FAIL) 6351 adapter->feat_en |= IXGBE_FEATURE_FAN_FAIL; 6352 /* Netmap */ 6353 if (adapter->feat_cap & IXGBE_FEATURE_NETMAP) 6354 adapter->feat_en |= IXGBE_FEATURE_NETMAP; 6355 /* EEE */ 6356 if (adapter->feat_cap & IXGBE_FEATURE_EEE) 6357 adapter->feat_en |= IXGBE_FEATURE_EEE; 6358 /* Thermal Sensor */ 6359 if (adapter->feat_cap & IXGBE_FEATURE_TEMP_SENSOR) 6360 adapter->feat_en |= IXGBE_FEATURE_TEMP_SENSOR; 6361 /* 6362 * Recovery mode: 6363 * NetBSD: IXGBE_FEATURE_RECOVERY_MODE will be controlled after reading 6364 * NVM Image version. 6365 */ 6366 6367 /* Enabled via global sysctl... */ 6368 /* Flow Director */ 6369 if (ixgbe_enable_fdir) { 6370 if (adapter->feat_cap & IXGBE_FEATURE_FDIR) 6371 adapter->feat_en |= IXGBE_FEATURE_FDIR; 6372 else 6373 device_printf(adapter->dev, "Device does not support Flow Director. Leaving disabled."); 6374 } 6375 /* Legacy (single queue) transmit */ 6376 if ((adapter->feat_cap & IXGBE_FEATURE_LEGACY_TX) && 6377 ixgbe_enable_legacy_tx) 6378 adapter->feat_en |= IXGBE_FEATURE_LEGACY_TX; 6379 /* 6380 * Message Signal Interrupts - Extended (MSI-X) 6381 * Normal MSI is only enabled if MSI-X calls fail. 6382 */ 6383 if (!ixgbe_enable_msix) 6384 adapter->feat_cap &= ~IXGBE_FEATURE_MSIX; 6385 /* Receive-Side Scaling (RSS) */ 6386 if ((adapter->feat_cap & IXGBE_FEATURE_RSS) && ixgbe_enable_rss) 6387 adapter->feat_en |= IXGBE_FEATURE_RSS; 6388 6389 /* Disable features with unmet dependencies... */ 6390 /* No MSI-X */ 6391 if (!(adapter->feat_cap & IXGBE_FEATURE_MSIX)) { 6392 adapter->feat_cap &= ~IXGBE_FEATURE_RSS; 6393 adapter->feat_cap &= ~IXGBE_FEATURE_SRIOV; 6394 adapter->feat_en &= ~IXGBE_FEATURE_RSS; 6395 adapter->feat_en &= ~IXGBE_FEATURE_SRIOV; 6396 } 6397 } /* ixgbe_init_device_features */ 6398 6399 /************************************************************************ 6400 * ixgbe_probe - Device identification routine 6401 * 6402 * Determines if the driver should be loaded on 6403 * adapter based on its PCI vendor/device ID. 6404 * 6405 * return BUS_PROBE_DEFAULT on success, positive on failure 6406 ************************************************************************/ 6407 static int 6408 ixgbe_probe(device_t dev, cfdata_t cf, void *aux) 6409 { 6410 const struct pci_attach_args *pa = aux; 6411 6412 return (ixgbe_lookup(pa) != NULL) ? 1 : 0; 6413 } 6414 6415 static const ixgbe_vendor_info_t * 6416 ixgbe_lookup(const struct pci_attach_args *pa) 6417 { 6418 const ixgbe_vendor_info_t *ent; 6419 pcireg_t subid; 6420 6421 INIT_DEBUGOUT("ixgbe_lookup: begin"); 6422 6423 if (PCI_VENDOR(pa->pa_id) != IXGBE_INTEL_VENDOR_ID) 6424 return NULL; 6425 6426 subid = pci_conf_read(pa->pa_pc, pa->pa_tag, PCI_SUBSYS_ID_REG); 6427 6428 for (ent = ixgbe_vendor_info_array; ent->vendor_id != 0; ent++) { 6429 if ((PCI_VENDOR(pa->pa_id) == ent->vendor_id) && 6430 (PCI_PRODUCT(pa->pa_id) == ent->device_id) && 6431 ((PCI_SUBSYS_VENDOR(subid) == ent->subvendor_id) || 6432 (ent->subvendor_id == 0)) && 6433 ((PCI_SUBSYS_ID(subid) == ent->subdevice_id) || 6434 (ent->subdevice_id == 0))) { 6435 return ent; 6436 } 6437 } 6438 return NULL; 6439 } 6440 6441 static int 6442 ixgbe_ifflags_cb(struct ethercom *ec) 6443 { 6444 struct ifnet *ifp = &ec->ec_if; 6445 struct adapter *adapter = ifp->if_softc; 6446 u_short change; 6447 int rv = 0; 6448 6449 IXGBE_CORE_LOCK(adapter); 6450 6451 change = ifp->if_flags ^ adapter->if_flags; 6452 if (change != 0) 6453 adapter->if_flags = ifp->if_flags; 6454 6455 if ((change & ~(IFF_CANTCHANGE | IFF_DEBUG)) != 0) { 6456 rv = ENETRESET; 6457 goto out; 6458 } else if ((change & IFF_PROMISC) != 0) 6459 ixgbe_set_rxfilter(adapter); 6460 6461 /* Check for ec_capenable. */ 6462 change = ec->ec_capenable ^ adapter->ec_capenable; 6463 adapter->ec_capenable = ec->ec_capenable; 6464 if ((change & ~(ETHERCAP_VLAN_MTU | ETHERCAP_VLAN_HWTAGGING 6465 | ETHERCAP_VLAN_HWFILTER)) != 0) { 6466 rv = ENETRESET; 6467 goto out; 6468 } 6469 6470 /* 6471 * Special handling is not required for ETHERCAP_VLAN_MTU. 6472 * MAXFRS(MHADD) does not include the 4bytes of the VLAN header. 6473 */ 6474 6475 /* Set up VLAN support and filter */ 6476 if ((change & (ETHERCAP_VLAN_HWTAGGING | ETHERCAP_VLAN_HWFILTER)) != 0) 6477 ixgbe_setup_vlan_hw_support(adapter); 6478 6479 out: 6480 IXGBE_CORE_UNLOCK(adapter); 6481 6482 return rv; 6483 } 6484 6485 /************************************************************************ 6486 * ixgbe_ioctl - Ioctl entry point 6487 * 6488 * Called when the user wants to configure the interface. 6489 * 6490 * return 0 on success, positive on failure 6491 ************************************************************************/ 6492 static int 6493 ixgbe_ioctl(struct ifnet *ifp, u_long command, void *data) 6494 { 6495 struct adapter *adapter = ifp->if_softc; 6496 struct ixgbe_hw *hw = &adapter->hw; 6497 struct ifcapreq *ifcr = data; 6498 struct ifreq *ifr = data; 6499 int error = 0; 6500 int l4csum_en; 6501 const int l4csum = IFCAP_CSUM_TCPv4_Rx | IFCAP_CSUM_UDPv4_Rx | 6502 IFCAP_CSUM_TCPv6_Rx | IFCAP_CSUM_UDPv6_Rx; 6503 6504 if (ixgbe_fw_recovery_mode_swflag(adapter)) 6505 return (EPERM); 6506 6507 switch (command) { 6508 case SIOCSIFFLAGS: 6509 IOCTL_DEBUGOUT("ioctl: SIOCSIFFLAGS (Set Interface Flags)"); 6510 break; 6511 case SIOCADDMULTI: 6512 case SIOCDELMULTI: 6513 IOCTL_DEBUGOUT("ioctl: SIOC(ADD|DEL)MULTI"); 6514 break; 6515 case SIOCSIFMEDIA: 6516 case SIOCGIFMEDIA: 6517 IOCTL_DEBUGOUT("ioctl: SIOCxIFMEDIA (Get/Set Interface Media)"); 6518 break; 6519 case SIOCSIFCAP: 6520 IOCTL_DEBUGOUT("ioctl: SIOCSIFCAP (Set Capabilities)"); 6521 break; 6522 case SIOCSIFMTU: 6523 IOCTL_DEBUGOUT("ioctl: SIOCSIFMTU (Set Interface MTU)"); 6524 break; 6525 #ifdef __NetBSD__ 6526 case SIOCINITIFADDR: 6527 IOCTL_DEBUGOUT("ioctl: SIOCINITIFADDR"); 6528 break; 6529 case SIOCGIFFLAGS: 6530 IOCTL_DEBUGOUT("ioctl: SIOCGIFFLAGS"); 6531 break; 6532 case SIOCGIFAFLAG_IN: 6533 IOCTL_DEBUGOUT("ioctl: SIOCGIFAFLAG_IN"); 6534 break; 6535 case SIOCGIFADDR: 6536 IOCTL_DEBUGOUT("ioctl: SIOCGIFADDR"); 6537 break; 6538 case SIOCGIFMTU: 6539 IOCTL_DEBUGOUT("ioctl: SIOCGIFMTU (Get Interface MTU)"); 6540 break; 6541 case SIOCGIFCAP: 6542 IOCTL_DEBUGOUT("ioctl: SIOCGIFCAP (Get IF cap)"); 6543 break; 6544 case SIOCGETHERCAP: 6545 IOCTL_DEBUGOUT("ioctl: SIOCGETHERCAP (Get ethercap)"); 6546 break; 6547 case SIOCGLIFADDR: 6548 IOCTL_DEBUGOUT("ioctl: SIOCGLIFADDR (Get Interface addr)"); 6549 break; 6550 case SIOCZIFDATA: 6551 IOCTL_DEBUGOUT("ioctl: SIOCZIFDATA (Zero counter)"); 6552 hw->mac.ops.clear_hw_cntrs(hw); 6553 ixgbe_clear_evcnt(adapter); 6554 break; 6555 case SIOCAIFADDR: 6556 IOCTL_DEBUGOUT("ioctl: SIOCAIFADDR (add/chg IF alias)"); 6557 break; 6558 #endif 6559 default: 6560 IOCTL_DEBUGOUT1("ioctl: UNKNOWN (0x%X)", (int)command); 6561 break; 6562 } 6563 6564 switch (command) { 6565 case SIOCGI2C: 6566 { 6567 struct ixgbe_i2c_req i2c; 6568 6569 IOCTL_DEBUGOUT("ioctl: SIOCGI2C (Get I2C Data)"); 6570 error = copyin(ifr->ifr_data, &i2c, sizeof(i2c)); 6571 if (error != 0) 6572 break; 6573 if (i2c.dev_addr != 0xA0 && i2c.dev_addr != 0xA2) { 6574 error = EINVAL; 6575 break; 6576 } 6577 if (i2c.len > sizeof(i2c.data)) { 6578 error = EINVAL; 6579 break; 6580 } 6581 6582 hw->phy.ops.read_i2c_byte(hw, i2c.offset, 6583 i2c.dev_addr, i2c.data); 6584 error = copyout(&i2c, ifr->ifr_data, sizeof(i2c)); 6585 break; 6586 } 6587 case SIOCSIFCAP: 6588 /* Layer-4 Rx checksum offload has to be turned on and 6589 * off as a unit. 6590 */ 6591 l4csum_en = ifcr->ifcr_capenable & l4csum; 6592 if (l4csum_en != l4csum && l4csum_en != 0) 6593 return EINVAL; 6594 /*FALLTHROUGH*/ 6595 case SIOCADDMULTI: 6596 case SIOCDELMULTI: 6597 case SIOCSIFFLAGS: 6598 case SIOCSIFMTU: 6599 default: 6600 if ((error = ether_ioctl(ifp, command, data)) != ENETRESET) 6601 return error; 6602 if ((ifp->if_flags & IFF_RUNNING) == 0) 6603 ; 6604 else if (command == SIOCSIFCAP || command == SIOCSIFMTU) { 6605 IXGBE_CORE_LOCK(adapter); 6606 if ((ifp->if_flags & IFF_RUNNING) != 0) 6607 ixgbe_init_locked(adapter); 6608 ixgbe_recalculate_max_frame(adapter); 6609 IXGBE_CORE_UNLOCK(adapter); 6610 } else if (command == SIOCADDMULTI || command == SIOCDELMULTI) { 6611 /* 6612 * Multicast list has changed; set the hardware filter 6613 * accordingly. 6614 */ 6615 IXGBE_CORE_LOCK(adapter); 6616 ixgbe_disable_intr(adapter); 6617 ixgbe_set_rxfilter(adapter); 6618 ixgbe_enable_intr(adapter); 6619 IXGBE_CORE_UNLOCK(adapter); 6620 } 6621 return 0; 6622 } 6623 6624 return error; 6625 } /* ixgbe_ioctl */ 6626 6627 /************************************************************************ 6628 * ixgbe_check_fan_failure 6629 ************************************************************************/ 6630 static int 6631 ixgbe_check_fan_failure(struct adapter *adapter, u32 reg, bool in_interrupt) 6632 { 6633 u32 mask; 6634 6635 mask = (in_interrupt) ? IXGBE_EICR_GPI_SDP1_BY_MAC(&adapter->hw) : 6636 IXGBE_ESDP_SDP1; 6637 6638 if ((reg & mask) == 0) 6639 return IXGBE_SUCCESS; 6640 6641 /* 6642 * Use ratecheck() just in case interrupt occur frequently. 6643 * When EXPX9501AT's fan stopped, interrupt occurred only once, 6644 * an red LED on the board turned on and link never up until 6645 * power off. 6646 */ 6647 if (ratecheck(&adapter->lasterr_time, &ixgbe_errlog_intrvl)) 6648 device_printf(adapter->dev, 6649 "\nCRITICAL: FAN FAILURE!! REPLACE IMMEDIATELY!!\n"); 6650 6651 return IXGBE_ERR_FAN_FAILURE; 6652 } /* ixgbe_check_fan_failure */ 6653 6654 /************************************************************************ 6655 * ixgbe_handle_que 6656 ************************************************************************/ 6657 static void 6658 ixgbe_handle_que(void *context) 6659 { 6660 struct ix_queue *que = context; 6661 struct adapter *adapter = que->adapter; 6662 struct tx_ring *txr = que->txr; 6663 struct ifnet *ifp = adapter->ifp; 6664 bool more = false; 6665 6666 IXGBE_EVC_ADD(&que->handleq, 1); 6667 6668 if (ifp->if_flags & IFF_RUNNING) { 6669 more = ixgbe_rxeof(que); 6670 IXGBE_TX_LOCK(txr); 6671 more |= ixgbe_txeof(txr); 6672 if (!(adapter->feat_en & IXGBE_FEATURE_LEGACY_TX)) 6673 if (!ixgbe_mq_ring_empty(ifp, txr->txr_interq)) 6674 ixgbe_mq_start_locked(ifp, txr); 6675 /* Only for queue 0 */ 6676 /* NetBSD still needs this for CBQ */ 6677 if ((&adapter->queues[0] == que) 6678 && (!ixgbe_legacy_ring_empty(ifp, NULL))) 6679 ixgbe_legacy_start_locked(ifp, txr); 6680 IXGBE_TX_UNLOCK(txr); 6681 } 6682 6683 if (more) { 6684 IXGBE_EVC_ADD(&que->req, 1); 6685 ixgbe_sched_handle_que(adapter, que); 6686 } else if (que->res != NULL) { 6687 /* MSIX: Re-enable this interrupt */ 6688 ixgbe_enable_queue(adapter, que->msix); 6689 } else { 6690 /* INTx or MSI */ 6691 ixgbe_enable_queue(adapter, 0); 6692 } 6693 6694 return; 6695 } /* ixgbe_handle_que */ 6696 6697 /************************************************************************ 6698 * ixgbe_handle_que_work 6699 ************************************************************************/ 6700 static void 6701 ixgbe_handle_que_work(struct work *wk, void *context) 6702 { 6703 struct ix_queue *que = container_of(wk, struct ix_queue, wq_cookie); 6704 6705 /* 6706 * "enqueued flag" is not required here. 6707 * See ixgbe_msix_que(). 6708 */ 6709 ixgbe_handle_que(que); 6710 } 6711 6712 /************************************************************************ 6713 * ixgbe_allocate_legacy - Setup the Legacy or MSI Interrupt handler 6714 ************************************************************************/ 6715 static int 6716 ixgbe_allocate_legacy(struct adapter *adapter, 6717 const struct pci_attach_args *pa) 6718 { 6719 device_t dev = adapter->dev; 6720 struct ix_queue *que = adapter->queues; 6721 struct tx_ring *txr = adapter->tx_rings; 6722 int counts[PCI_INTR_TYPE_SIZE]; 6723 pci_intr_type_t intr_type, max_type; 6724 char intrbuf[PCI_INTRSTR_LEN]; 6725 char wqname[MAXCOMLEN]; 6726 const char *intrstr = NULL; 6727 int defertx_error = 0, error; 6728 6729 /* We allocate a single interrupt resource */ 6730 max_type = PCI_INTR_TYPE_MSI; 6731 counts[PCI_INTR_TYPE_MSIX] = 0; 6732 counts[PCI_INTR_TYPE_MSI] = 6733 (adapter->feat_en & IXGBE_FEATURE_MSI) ? 1 : 0; 6734 /* Check not feat_en but feat_cap to fallback to INTx */ 6735 counts[PCI_INTR_TYPE_INTX] = 6736 (adapter->feat_cap & IXGBE_FEATURE_LEGACY_IRQ) ? 1 : 0; 6737 6738 alloc_retry: 6739 if (pci_intr_alloc(pa, &adapter->osdep.intrs, counts, max_type) != 0) { 6740 aprint_error_dev(dev, "couldn't alloc interrupt\n"); 6741 return ENXIO; 6742 } 6743 adapter->osdep.nintrs = 1; 6744 intrstr = pci_intr_string(adapter->osdep.pc, adapter->osdep.intrs[0], 6745 intrbuf, sizeof(intrbuf)); 6746 adapter->osdep.ihs[0] = pci_intr_establish_xname(adapter->osdep.pc, 6747 adapter->osdep.intrs[0], IPL_NET, ixgbe_legacy_irq, que, 6748 device_xname(dev)); 6749 intr_type = pci_intr_type(adapter->osdep.pc, adapter->osdep.intrs[0]); 6750 if (adapter->osdep.ihs[0] == NULL) { 6751 aprint_error_dev(dev,"unable to establish %s\n", 6752 (intr_type == PCI_INTR_TYPE_MSI) ? "MSI" : "INTx"); 6753 pci_intr_release(adapter->osdep.pc, adapter->osdep.intrs, 1); 6754 adapter->osdep.intrs = NULL; 6755 switch (intr_type) { 6756 case PCI_INTR_TYPE_MSI: 6757 /* The next try is for INTx: Disable MSI */ 6758 max_type = PCI_INTR_TYPE_INTX; 6759 counts[PCI_INTR_TYPE_INTX] = 1; 6760 adapter->feat_en &= ~IXGBE_FEATURE_MSI; 6761 if (adapter->feat_cap & IXGBE_FEATURE_LEGACY_IRQ) { 6762 adapter->feat_en |= IXGBE_FEATURE_LEGACY_IRQ; 6763 goto alloc_retry; 6764 } else 6765 break; 6766 case PCI_INTR_TYPE_INTX: 6767 default: 6768 /* See below */ 6769 break; 6770 } 6771 } 6772 if (intr_type == PCI_INTR_TYPE_INTX) { 6773 adapter->feat_en &= ~IXGBE_FEATURE_MSI; 6774 adapter->feat_en |= IXGBE_FEATURE_LEGACY_IRQ; 6775 } 6776 if (adapter->osdep.ihs[0] == NULL) { 6777 aprint_error_dev(dev, 6778 "couldn't establish interrupt%s%s\n", 6779 intrstr ? " at " : "", intrstr ? intrstr : ""); 6780 pci_intr_release(adapter->osdep.pc, adapter->osdep.intrs, 1); 6781 adapter->osdep.intrs = NULL; 6782 return ENXIO; 6783 } 6784 aprint_normal_dev(dev, "interrupting at %s\n", intrstr); 6785 /* 6786 * Try allocating a fast interrupt and the associated deferred 6787 * processing contexts. 6788 */ 6789 if (!(adapter->feat_en & IXGBE_FEATURE_LEGACY_TX)) { 6790 txr->txr_si = 6791 softint_establish(SOFTINT_NET | IXGBE_SOFTINT_FLAGS, 6792 ixgbe_deferred_mq_start, txr); 6793 6794 snprintf(wqname, sizeof(wqname), "%sdeferTx", 6795 device_xname(dev)); 6796 defertx_error = workqueue_create(&adapter->txr_wq, wqname, 6797 ixgbe_deferred_mq_start_work, adapter, IXGBE_WORKQUEUE_PRI, 6798 IPL_NET, IXGBE_WORKQUEUE_FLAGS); 6799 adapter->txr_wq_enqueued = percpu_alloc(sizeof(u_int)); 6800 } 6801 que->que_si = softint_establish(SOFTINT_NET | IXGBE_SOFTINT_FLAGS, 6802 ixgbe_handle_que, que); 6803 snprintf(wqname, sizeof(wqname), "%sTxRx", device_xname(dev)); 6804 error = workqueue_create(&adapter->que_wq, wqname, 6805 ixgbe_handle_que_work, adapter, IXGBE_WORKQUEUE_PRI, IPL_NET, 6806 IXGBE_WORKQUEUE_FLAGS); 6807 6808 if ((!(adapter->feat_en & IXGBE_FEATURE_LEGACY_TX) 6809 && ((txr->txr_si == NULL) || defertx_error != 0)) 6810 || (que->que_si == NULL) || error != 0) { 6811 aprint_error_dev(dev, 6812 "could not establish software interrupts\n"); 6813 6814 return ENXIO; 6815 } 6816 /* For simplicity in the handlers */ 6817 adapter->active_queues = IXGBE_EIMS_ENABLE_MASK; 6818 6819 return (0); 6820 } /* ixgbe_allocate_legacy */ 6821 6822 /************************************************************************ 6823 * ixgbe_allocate_msix - Setup MSI-X Interrupt resources and handlers 6824 ************************************************************************/ 6825 static int 6826 ixgbe_allocate_msix(struct adapter *adapter, const struct pci_attach_args *pa) 6827 { 6828 device_t dev = adapter->dev; 6829 struct ix_queue *que = adapter->queues; 6830 struct tx_ring *txr = adapter->tx_rings; 6831 pci_chipset_tag_t pc; 6832 char intrbuf[PCI_INTRSTR_LEN]; 6833 char intr_xname[32]; 6834 char wqname[MAXCOMLEN]; 6835 const char *intrstr = NULL; 6836 int error, vector = 0; 6837 int cpu_id = 0; 6838 kcpuset_t *affinity; 6839 #ifdef RSS 6840 unsigned int rss_buckets = 0; 6841 kcpuset_t cpu_mask; 6842 #endif 6843 6844 pc = adapter->osdep.pc; 6845 #ifdef RSS 6846 /* 6847 * If we're doing RSS, the number of queues needs to 6848 * match the number of RSS buckets that are configured. 6849 * 6850 * + If there's more queues than RSS buckets, we'll end 6851 * up with queues that get no traffic. 6852 * 6853 * + If there's more RSS buckets than queues, we'll end 6854 * up having multiple RSS buckets map to the same queue, 6855 * so there'll be some contention. 6856 */ 6857 rss_buckets = rss_getnumbuckets(); 6858 if ((adapter->feat_en & IXGBE_FEATURE_RSS) && 6859 (adapter->num_queues != rss_buckets)) { 6860 device_printf(dev, 6861 "%s: number of queues (%d) != number of RSS buckets (%d)" 6862 "; performance will be impacted.\n", 6863 __func__, adapter->num_queues, rss_buckets); 6864 } 6865 #endif 6866 6867 adapter->osdep.nintrs = adapter->num_queues + 1; 6868 if (pci_msix_alloc_exact(pa, &adapter->osdep.intrs, 6869 adapter->osdep.nintrs) != 0) { 6870 aprint_error_dev(dev, 6871 "failed to allocate MSI-X interrupt\n"); 6872 adapter->feat_en &= ~IXGBE_FEATURE_MSIX; 6873 return (ENXIO); 6874 } 6875 6876 kcpuset_create(&affinity, false); 6877 for (int i = 0; i < adapter->num_queues; i++, vector++, que++, txr++) { 6878 snprintf(intr_xname, sizeof(intr_xname), "%s TXRX%d", 6879 device_xname(dev), i); 6880 intrstr = pci_intr_string(pc, adapter->osdep.intrs[i], intrbuf, 6881 sizeof(intrbuf)); 6882 #ifdef IXGBE_MPSAFE 6883 pci_intr_setattr(pc, &adapter->osdep.intrs[i], PCI_INTR_MPSAFE, 6884 true); 6885 #endif 6886 /* Set the handler function */ 6887 que->res = adapter->osdep.ihs[i] = pci_intr_establish_xname(pc, 6888 adapter->osdep.intrs[i], IPL_NET, ixgbe_msix_que, que, 6889 intr_xname); 6890 if (que->res == NULL) { 6891 aprint_error_dev(dev, 6892 "Failed to register QUE handler\n"); 6893 error = ENXIO; 6894 goto err_out; 6895 } 6896 que->msix = vector; 6897 adapter->active_queues |= 1ULL << que->msix; 6898 6899 if (adapter->feat_en & IXGBE_FEATURE_RSS) { 6900 #ifdef RSS 6901 /* 6902 * The queue ID is used as the RSS layer bucket ID. 6903 * We look up the queue ID -> RSS CPU ID and select 6904 * that. 6905 */ 6906 cpu_id = rss_getcpu(i % rss_getnumbuckets()); 6907 CPU_SETOF(cpu_id, &cpu_mask); 6908 #endif 6909 } else { 6910 /* 6911 * Bind the MSI-X vector, and thus the 6912 * rings to the corresponding CPU. 6913 * 6914 * This just happens to match the default RSS 6915 * round-robin bucket -> queue -> CPU allocation. 6916 */ 6917 if (adapter->num_queues > 1) 6918 cpu_id = i; 6919 } 6920 /* Round-robin affinity */ 6921 kcpuset_zero(affinity); 6922 kcpuset_set(affinity, cpu_id % ncpu); 6923 error = interrupt_distribute(adapter->osdep.ihs[i], affinity, 6924 NULL); 6925 aprint_normal_dev(dev, "for TX/RX, interrupting at %s", 6926 intrstr); 6927 if (error == 0) { 6928 #if 1 /* def IXGBE_DEBUG */ 6929 #ifdef RSS 6930 aprintf_normal(", bound RSS bucket %d to CPU %d", i, 6931 cpu_id % ncpu); 6932 #else 6933 aprint_normal(", bound queue %d to cpu %d", i, 6934 cpu_id % ncpu); 6935 #endif 6936 #endif /* IXGBE_DEBUG */ 6937 } 6938 aprint_normal("\n"); 6939 6940 if (!(adapter->feat_en & IXGBE_FEATURE_LEGACY_TX)) { 6941 txr->txr_si = softint_establish( 6942 SOFTINT_NET | IXGBE_SOFTINT_FLAGS, 6943 ixgbe_deferred_mq_start, txr); 6944 if (txr->txr_si == NULL) { 6945 aprint_error_dev(dev, 6946 "couldn't establish software interrupt\n"); 6947 error = ENXIO; 6948 goto err_out; 6949 } 6950 } 6951 que->que_si 6952 = softint_establish(SOFTINT_NET | IXGBE_SOFTINT_FLAGS, 6953 ixgbe_handle_que, que); 6954 if (que->que_si == NULL) { 6955 aprint_error_dev(dev, 6956 "couldn't establish software interrupt\n"); 6957 error = ENXIO; 6958 goto err_out; 6959 } 6960 } 6961 snprintf(wqname, sizeof(wqname), "%sdeferTx", device_xname(dev)); 6962 error = workqueue_create(&adapter->txr_wq, wqname, 6963 ixgbe_deferred_mq_start_work, adapter, IXGBE_WORKQUEUE_PRI, IPL_NET, 6964 IXGBE_WORKQUEUE_FLAGS); 6965 if (error) { 6966 aprint_error_dev(dev, 6967 "couldn't create workqueue for deferred Tx\n"); 6968 goto err_out; 6969 } 6970 adapter->txr_wq_enqueued = percpu_alloc(sizeof(u_int)); 6971 6972 snprintf(wqname, sizeof(wqname), "%sTxRx", device_xname(dev)); 6973 error = workqueue_create(&adapter->que_wq, wqname, 6974 ixgbe_handle_que_work, adapter, IXGBE_WORKQUEUE_PRI, IPL_NET, 6975 IXGBE_WORKQUEUE_FLAGS); 6976 if (error) { 6977 aprint_error_dev(dev, "couldn't create workqueue for Tx/Rx\n"); 6978 goto err_out; 6979 } 6980 6981 /* and Link */ 6982 cpu_id++; 6983 snprintf(intr_xname, sizeof(intr_xname), "%s link", device_xname(dev)); 6984 adapter->vector = vector; 6985 intrstr = pci_intr_string(pc, adapter->osdep.intrs[vector], intrbuf, 6986 sizeof(intrbuf)); 6987 #ifdef IXGBE_MPSAFE 6988 pci_intr_setattr(pc, &adapter->osdep.intrs[vector], PCI_INTR_MPSAFE, 6989 true); 6990 #endif 6991 /* Set the link handler function */ 6992 adapter->osdep.ihs[vector] = pci_intr_establish_xname(pc, 6993 adapter->osdep.intrs[vector], IPL_NET, ixgbe_msix_admin, adapter, 6994 intr_xname); 6995 if (adapter->osdep.ihs[vector] == NULL) { 6996 aprint_error_dev(dev, "Failed to register LINK handler\n"); 6997 error = ENXIO; 6998 goto err_out; 6999 } 7000 /* Round-robin affinity */ 7001 kcpuset_zero(affinity); 7002 kcpuset_set(affinity, cpu_id % ncpu); 7003 error = interrupt_distribute(adapter->osdep.ihs[vector], affinity, 7004 NULL); 7005 7006 aprint_normal_dev(dev, 7007 "for link, interrupting at %s", intrstr); 7008 if (error == 0) 7009 aprint_normal(", affinity to cpu %d\n", cpu_id % ncpu); 7010 else 7011 aprint_normal("\n"); 7012 7013 kcpuset_destroy(affinity); 7014 aprint_normal_dev(dev, 7015 "Using MSI-X interrupts with %d vectors\n", vector + 1); 7016 7017 return (0); 7018 7019 err_out: 7020 kcpuset_destroy(affinity); 7021 ixgbe_free_deferred_handlers(adapter); 7022 ixgbe_free_pciintr_resources(adapter); 7023 return (error); 7024 } /* ixgbe_allocate_msix */ 7025 7026 /************************************************************************ 7027 * ixgbe_configure_interrupts 7028 * 7029 * Setup MSI-X, MSI, or legacy interrupts (in that order). 7030 * This will also depend on user settings. 7031 ************************************************************************/ 7032 static int 7033 ixgbe_configure_interrupts(struct adapter *adapter) 7034 { 7035 device_t dev = adapter->dev; 7036 struct ixgbe_mac_info *mac = &adapter->hw.mac; 7037 int want, queues, msgs; 7038 7039 /* Default to 1 queue if MSI-X setup fails */ 7040 adapter->num_queues = 1; 7041 7042 /* Override by tuneable */ 7043 if (!(adapter->feat_cap & IXGBE_FEATURE_MSIX)) 7044 goto msi; 7045 7046 /* 7047 * NetBSD only: Use single vector MSI when number of CPU is 1 to save 7048 * interrupt slot. 7049 */ 7050 if (ncpu == 1) 7051 goto msi; 7052 7053 /* First try MSI-X */ 7054 msgs = pci_msix_count(adapter->osdep.pc, adapter->osdep.tag); 7055 msgs = MIN(msgs, IXG_MAX_NINTR); 7056 if (msgs < 2) 7057 goto msi; 7058 7059 adapter->feat_en |= IXGBE_FEATURE_MSIX; 7060 7061 /* Figure out a reasonable auto config value */ 7062 queues = (ncpu > (msgs - 1)) ? (msgs - 1) : ncpu; 7063 7064 #ifdef RSS 7065 /* If we're doing RSS, clamp at the number of RSS buckets */ 7066 if (adapter->feat_en & IXGBE_FEATURE_RSS) 7067 queues = uimin(queues, rss_getnumbuckets()); 7068 #endif 7069 if (ixgbe_num_queues > queues) { 7070 aprint_error_dev(adapter->dev, "ixgbe_num_queues (%d) is too large, using reduced amount (%d).\n", ixgbe_num_queues, queues); 7071 ixgbe_num_queues = queues; 7072 } 7073 7074 if (ixgbe_num_queues != 0) 7075 queues = ixgbe_num_queues; 7076 else 7077 queues = uimin(queues, 7078 uimin(mac->max_tx_queues, mac->max_rx_queues)); 7079 7080 /* reflect correct sysctl value */ 7081 ixgbe_num_queues = queues; 7082 7083 /* 7084 * Want one vector (RX/TX pair) per queue 7085 * plus an additional for Link. 7086 */ 7087 want = queues + 1; 7088 if (msgs >= want) 7089 msgs = want; 7090 else { 7091 aprint_error_dev(dev, "MSI-X Configuration Problem, " 7092 "%d vectors but %d queues wanted!\n", 7093 msgs, want); 7094 goto msi; 7095 } 7096 adapter->num_queues = queues; 7097 adapter->feat_en |= IXGBE_FEATURE_MSIX; 7098 return (0); 7099 7100 /* 7101 * MSI-X allocation failed or provided us with 7102 * less vectors than needed. Free MSI-X resources 7103 * and we'll try enabling MSI. 7104 */ 7105 msi: 7106 /* Without MSI-X, some features are no longer supported */ 7107 adapter->feat_cap &= ~IXGBE_FEATURE_RSS; 7108 adapter->feat_en &= ~IXGBE_FEATURE_RSS; 7109 adapter->feat_cap &= ~IXGBE_FEATURE_SRIOV; 7110 adapter->feat_en &= ~IXGBE_FEATURE_SRIOV; 7111 7112 msgs = pci_msi_count(adapter->osdep.pc, adapter->osdep.tag); 7113 adapter->feat_en &= ~IXGBE_FEATURE_MSIX; 7114 if (msgs > 1) 7115 msgs = 1; 7116 if (msgs != 0) { 7117 msgs = 1; 7118 adapter->feat_en |= IXGBE_FEATURE_MSI; 7119 return (0); 7120 } 7121 7122 if (!(adapter->feat_cap & IXGBE_FEATURE_LEGACY_IRQ)) { 7123 aprint_error_dev(dev, 7124 "Device does not support legacy interrupts.\n"); 7125 return 1; 7126 } 7127 7128 adapter->feat_en |= IXGBE_FEATURE_LEGACY_IRQ; 7129 7130 return (0); 7131 } /* ixgbe_configure_interrupts */ 7132 7133 7134 /************************************************************************ 7135 * ixgbe_handle_link - Tasklet for MSI-X Link interrupts 7136 * 7137 * Done outside of interrupt context since the driver might sleep 7138 ************************************************************************/ 7139 static void 7140 ixgbe_handle_link(void *context) 7141 { 7142 struct adapter *adapter = context; 7143 struct ixgbe_hw *hw = &adapter->hw; 7144 7145 KASSERT(mutex_owned(&adapter->core_mtx)); 7146 7147 IXGBE_EVC_ADD(&adapter->link_workev, 1); 7148 ixgbe_check_link(hw, &adapter->link_speed, &adapter->link_up, 0); 7149 ixgbe_update_link_status(adapter); 7150 7151 /* Re-enable link interrupts */ 7152 IXGBE_WRITE_REG(hw, IXGBE_EIMS, IXGBE_EIMS_LSC); 7153 } /* ixgbe_handle_link */ 7154 7155 #if 0 7156 /************************************************************************ 7157 * ixgbe_rearm_queues 7158 ************************************************************************/ 7159 static __inline void 7160 ixgbe_rearm_queues(struct adapter *adapter, u64 queues) 7161 { 7162 u32 mask; 7163 7164 switch (adapter->hw.mac.type) { 7165 case ixgbe_mac_82598EB: 7166 mask = (IXGBE_EIMS_RTX_QUEUE & queues); 7167 IXGBE_WRITE_REG(&adapter->hw, IXGBE_EICS, mask); 7168 break; 7169 case ixgbe_mac_82599EB: 7170 case ixgbe_mac_X540: 7171 case ixgbe_mac_X550: 7172 case ixgbe_mac_X550EM_x: 7173 case ixgbe_mac_X550EM_a: 7174 mask = (queues & 0xFFFFFFFF); 7175 IXGBE_WRITE_REG(&adapter->hw, IXGBE_EICS_EX(0), mask); 7176 mask = (queues >> 32); 7177 IXGBE_WRITE_REG(&adapter->hw, IXGBE_EICS_EX(1), mask); 7178 break; 7179 default: 7180 break; 7181 } 7182 } /* ixgbe_rearm_queues */ 7183 #endif 7184
Indexes created Tue Sep 30 11:09:46 GMT 2025