ixgbe.c revision 1.314
1 /* $NetBSD: ixgbe.c,v 1.314 2022/04/25 07:56:47 msaitoh Exp $ */ 2 3 /****************************************************************************** 4 5 Copyright (c) 2001-2017, Intel Corporation 6 All rights reserved. 7 8 Redistribution and use in source and binary forms, with or without 9 modification, are permitted provided that the following conditions are met: 10 11 1. Redistributions of source code must retain the above copyright notice, 12 this list of conditions and the following disclaimer. 13 14 2. Redistributions in binary form must reproduce the above copyright 15 notice, this list of conditions and the following disclaimer in the 16 documentation and/or other materials provided with the distribution. 17 18 3. Neither the name of the Intel Corporation nor the names of its 19 contributors may be used to endorse or promote products derived from 20 this software without specific prior written permission. 21 22 THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 23 AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 24 IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 25 ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE 26 LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 27 CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 28 SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 29 INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 30 CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 31 ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 32 POSSIBILITY OF SUCH DAMAGE. 33 34 ******************************************************************************/ 35 /*$FreeBSD: head/sys/dev/ixgbe/if_ix.c 331224 2018-03-19 20:55:05Z erj $*/ 36 37 /* 38 * Copyright (c) 2011 The NetBSD Foundation, Inc. 39 * All rights reserved. 40 * 41 * This code is derived from software contributed to The NetBSD Foundation 42 * by Coyote Point Systems, Inc. 43 * 44 * Redistribution and use in source and binary forms, with or without 45 * modification, are permitted provided that the following conditions 46 * are met: 47 * 1. Redistributions of source code must retain the above copyright 48 * notice, this list of conditions and the following disclaimer. 49 * 2. Redistributions in binary form must reproduce the above copyright 50 * notice, this list of conditions and the following disclaimer in the 51 * documentation and/or other materials provided with the distribution. 52 * 53 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS 54 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED 55 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 56 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS 57 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 58 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 59 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 60 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 61 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 62 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 63 * POSSIBILITY OF SUCH DAMAGE. 64 */ 65 66 #include <sys/cdefs.h> 67 __KERNEL_RCSID(0, "$NetBSD: ixgbe.c,v 1.314 2022/04/25 07:56:47 msaitoh Exp $"); 68 69 #ifdef _KERNEL_OPT 70 #include "opt_inet.h" 71 #include "opt_inet6.h" 72 #include "opt_net_mpsafe.h" 73 #endif 74 75 #include "ixgbe.h" 76 #include "ixgbe_phy.h" 77 #include "ixgbe_sriov.h" 78 79 #include <sys/cprng.h> 80 #include <dev/mii/mii.h> 81 #include <dev/mii/miivar.h> 82 83 /************************************************************************ 84 * Driver version 85 ************************************************************************/ 86 static const char ixgbe_driver_version[] = "4.0.1-k"; 87 /* XXX NetBSD: + 3.3.24 */ 88 89 /************************************************************************ 90 * PCI Device ID Table 91 * 92 * Used by probe to select devices to load on 93 * Last field stores an index into ixgbe_strings 94 * Last entry must be all 0s 95 * 96 * { Vendor ID, Device ID, SubVendor ID, SubDevice ID, String Index } 97 ************************************************************************/ 98 static const ixgbe_vendor_info_t ixgbe_vendor_info_array[] = 99 { 100 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598AF_DUAL_PORT, 0, 0, 0}, 101 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598AF_SINGLE_PORT, 0, 0, 0}, 102 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598EB_CX4, 0, 0, 0}, 103 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598AT, 0, 0, 0}, 104 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598AT2, 0, 0, 0}, 105 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598, 0, 0, 0}, 106 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598_BX, 0, 0, 0}, 107 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598_DA_DUAL_PORT, 0, 0, 0}, 108 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598_CX4_DUAL_PORT, 0, 0, 0}, 109 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598EB_XF_LR, 0, 0, 0}, 110 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598_SR_DUAL_PORT_EM, 0, 0, 0}, 111 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598EB_SFP_LOM, 0, 0, 0}, 112 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_KR, 0, 0, 0}, 113 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_KX4, 0, 0, 0}, 114 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_KX4_MEZZ, 0, 0, 0}, 115 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_SFP, 0, 0, 0}, 116 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_SFP_EM, 0, 0, 0}, 117 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_XAUI_LOM, 0, 0, 0}, 118 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_CX4, 0, 0, 0}, 119 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_T3_LOM, 0, 0, 0}, 120 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_COMBO_BACKPLANE, 0, 0, 0}, 121 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_BACKPLANE_FCOE, 0, 0, 0}, 122 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_SFP_SF2, 0, 0, 0}, 123 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_SFP_FCOE, 0, 0, 0}, 124 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599EN_SFP, 0, 0, 0}, 125 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_SFP_SF_QP, 0, 0, 0}, 126 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_QSFP_SF_QP, 0, 0, 0}, 127 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_X540T, 0, 0, 0}, 128 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_X540T1, 0, 0, 0}, 129 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_X550T, 0, 0, 0}, 130 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_X550T1, 0, 0, 0}, 131 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_X550EM_X_KR, 0, 0, 0}, 132 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_X550EM_X_KX4, 0, 0, 0}, 133 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_X550EM_X_10G_T, 0, 0, 0}, 134 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_X550EM_X_1G_T, 0, 0, 0}, 135 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_X550EM_X_SFP, 0, 0, 0}, 136 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_X550EM_X_XFI, 0, 0, 0}, 137 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_X550EM_A_KR, 0, 0, 0}, 138 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_X550EM_A_KR_L, 0, 0, 0}, 139 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_X550EM_A_QSFP, 0, 0, 0}, 140 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_X550EM_A_QSFP_N, 0, 0, 0}, 141 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_X550EM_A_SFP, 0, 0, 0}, 142 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_X550EM_A_SFP_N, 0, 0, 0}, 143 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_X550EM_A_SGMII, 0, 0, 0}, 144 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_X550EM_A_SGMII_L, 0, 0, 0}, 145 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_X550EM_A_10G_T, 0, 0, 0}, 146 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_X550EM_A_1G_T, 0, 0, 0}, 147 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_X550EM_A_1G_T_L, 0, 0, 0}, 148 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_X540_BYPASS, 0, 0, 0}, 149 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_BYPASS, 0, 0, 0}, 150 /* required last entry */ 151 {0, 0, 0, 0, 0} 152 }; 153 154 /************************************************************************ 155 * Table of branding strings 156 ************************************************************************/ 157 static const char *ixgbe_strings[] = { 158 "Intel(R) PRO/10GbE PCI-Express Network Driver" 159 }; 160 161 /************************************************************************ 162 * Function prototypes 163 ************************************************************************/ 164 static int ixgbe_probe(device_t, cfdata_t, void *); 165 static void ixgbe_quirks(struct adapter *); 166 static void ixgbe_attach(device_t, device_t, void *); 167 static int ixgbe_detach(device_t, int); 168 #if 0 169 static int ixgbe_shutdown(device_t); 170 #endif 171 static bool ixgbe_suspend(device_t, const pmf_qual_t *); 172 static bool ixgbe_resume(device_t, const pmf_qual_t *); 173 static int ixgbe_ifflags_cb(struct ethercom *); 174 static int ixgbe_ioctl(struct ifnet *, u_long, void *); 175 static int ixgbe_init(struct ifnet *); 176 static void ixgbe_init_locked(struct adapter *); 177 static void ixgbe_ifstop(struct ifnet *, int); 178 static void ixgbe_stop_locked(void *); 179 static void ixgbe_init_device_features(struct adapter *); 180 static int ixgbe_check_fan_failure(struct adapter *, u32, bool); 181 static void ixgbe_add_media_types(struct adapter *); 182 static void ixgbe_media_status(struct ifnet *, struct ifmediareq *); 183 static int ixgbe_media_change(struct ifnet *); 184 static int ixgbe_allocate_pci_resources(struct adapter *, 185 const struct pci_attach_args *); 186 static void ixgbe_free_deferred_handlers(struct adapter *); 187 static void ixgbe_get_slot_info(struct adapter *); 188 static int ixgbe_allocate_msix(struct adapter *, 189 const struct pci_attach_args *); 190 static int ixgbe_allocate_legacy(struct adapter *, 191 const struct pci_attach_args *); 192 static int ixgbe_configure_interrupts(struct adapter *); 193 static void ixgbe_free_pciintr_resources(struct adapter *); 194 static void ixgbe_free_pci_resources(struct adapter *); 195 static void ixgbe_local_timer(void *); 196 static void ixgbe_handle_timer(struct work *, void *); 197 static void ixgbe_recovery_mode_timer(void *); 198 static void ixgbe_handle_recovery_mode_timer(struct work *, void *); 199 static int ixgbe_setup_interface(device_t, struct adapter *); 200 static void ixgbe_config_gpie(struct adapter *); 201 static void ixgbe_config_dmac(struct adapter *); 202 static void ixgbe_config_delay_values(struct adapter *); 203 static void ixgbe_schedule_admin_tasklet(struct adapter *); 204 static void ixgbe_config_link(struct adapter *); 205 static void ixgbe_check_wol_support(struct adapter *); 206 static int ixgbe_setup_low_power_mode(struct adapter *); 207 #if 0 208 static void ixgbe_rearm_queues(struct adapter *, u64); 209 #endif 210 211 static void ixgbe_initialize_transmit_units(struct adapter *); 212 static void ixgbe_initialize_receive_units(struct adapter *); 213 static void ixgbe_enable_rx_drop(struct adapter *); 214 static void ixgbe_disable_rx_drop(struct adapter *); 215 static void ixgbe_initialize_rss_mapping(struct adapter *); 216 217 static void ixgbe_enable_intr(struct adapter *); 218 static void ixgbe_disable_intr(struct adapter *); 219 static void ixgbe_update_stats_counters(struct adapter *); 220 static void ixgbe_set_rxfilter(struct adapter *); 221 static void ixgbe_update_link_status(struct adapter *); 222 static void ixgbe_set_ivar(struct adapter *, u8, u8, s8); 223 static void ixgbe_configure_ivars(struct adapter *); 224 static u8 * ixgbe_mc_array_itr(struct ixgbe_hw *, u8 **, u32 *); 225 static void ixgbe_eitr_write(struct adapter *, uint32_t, uint32_t); 226 227 static void ixgbe_setup_vlan_hw_tagging(struct adapter *); 228 static void ixgbe_setup_vlan_hw_support(struct adapter *); 229 static int ixgbe_vlan_cb(struct ethercom *, uint16_t, bool); 230 static int ixgbe_register_vlan(struct adapter *, u16); 231 static int ixgbe_unregister_vlan(struct adapter *, u16); 232 233 static void ixgbe_add_device_sysctls(struct adapter *); 234 static void ixgbe_add_hw_stats(struct adapter *); 235 static void ixgbe_clear_evcnt(struct adapter *); 236 static int ixgbe_set_flowcntl(struct adapter *, int); 237 static int ixgbe_set_advertise(struct adapter *, int); 238 static int ixgbe_get_default_advertise(struct adapter *); 239 240 /* Sysctl handlers */ 241 static int ixgbe_sysctl_flowcntl(SYSCTLFN_PROTO); 242 static int ixgbe_sysctl_advertise(SYSCTLFN_PROTO); 243 static int ixgbe_sysctl_interrupt_rate_handler(SYSCTLFN_PROTO); 244 static int ixgbe_sysctl_dmac(SYSCTLFN_PROTO); 245 static int ixgbe_sysctl_phy_temp(SYSCTLFN_PROTO); 246 static int ixgbe_sysctl_phy_overtemp_occurred(SYSCTLFN_PROTO); 247 #ifdef IXGBE_DEBUG 248 static int ixgbe_sysctl_power_state(SYSCTLFN_PROTO); 249 static int ixgbe_sysctl_print_rss_config(SYSCTLFN_PROTO); 250 #endif 251 static int ixgbe_sysctl_next_to_check_handler(SYSCTLFN_PROTO); 252 static int ixgbe_sysctl_next_to_refresh_handler(SYSCTLFN_PROTO); 253 static int ixgbe_sysctl_rdh_handler(SYSCTLFN_PROTO); 254 static int ixgbe_sysctl_rdt_handler(SYSCTLFN_PROTO); 255 static int ixgbe_sysctl_tdt_handler(SYSCTLFN_PROTO); 256 static int ixgbe_sysctl_tdh_handler(SYSCTLFN_PROTO); 257 static int ixgbe_sysctl_eee_state(SYSCTLFN_PROTO); 258 static int ixgbe_sysctl_debug(SYSCTLFN_PROTO); 259 static int ixgbe_sysctl_rx_copy_len(SYSCTLFN_PROTO); 260 static int ixgbe_sysctl_tx_process_limit(SYSCTLFN_PROTO); 261 static int ixgbe_sysctl_rx_process_limit(SYSCTLFN_PROTO); 262 static int ixgbe_sysctl_wol_enable(SYSCTLFN_PROTO); 263 static int ixgbe_sysctl_wufc(SYSCTLFN_PROTO); 264 265 /* Interrupt functions */ 266 static int ixgbe_msix_que(void *); 267 static int ixgbe_msix_admin(void *); 268 static void ixgbe_intr_admin_common(struct adapter *, u32, u32 *); 269 static int ixgbe_legacy_irq(void *); 270 271 /* Event handlers running on workqueue */ 272 static void ixgbe_handle_que(void *); 273 static void ixgbe_handle_link(void *); 274 static void ixgbe_handle_msf(void *); 275 static void ixgbe_handle_mod(void *, bool); 276 static void ixgbe_handle_phy(void *); 277 278 /* Deferred workqueue handlers */ 279 static void ixgbe_handle_admin(struct work *, void *); 280 static void ixgbe_handle_que_work(struct work *, void *); 281 282 static const ixgbe_vendor_info_t *ixgbe_lookup(const struct pci_attach_args *); 283 284 /************************************************************************ 285 * NetBSD Device Interface Entry Points 286 ************************************************************************/ 287 CFATTACH_DECL3_NEW(ixg, sizeof(struct adapter), 288 ixgbe_probe, ixgbe_attach, ixgbe_detach, NULL, NULL, NULL, 289 DVF_DETACH_SHUTDOWN); 290 291 #if 0 292 devclass_t ix_devclass; 293 DRIVER_MODULE(ix, pci, ix_driver, ix_devclass, 0, 0); 294 295 MODULE_DEPEND(ix, pci, 1, 1, 1); 296 MODULE_DEPEND(ix, ether, 1, 1, 1); 297 #ifdef DEV_NETMAP 298 MODULE_DEPEND(ix, netmap, 1, 1, 1); 299 #endif 300 #endif 301 302 /* 303 * TUNEABLE PARAMETERS: 304 */ 305 306 /* 307 * AIM: Adaptive Interrupt Moderation 308 * which means that the interrupt rate 309 * is varied over time based on the 310 * traffic for that interrupt vector 311 */ 312 static bool ixgbe_enable_aim = true; 313 #define SYSCTL_INT(_a1, _a2, _a3, _a4, _a5, _a6, _a7) 314 SYSCTL_INT(_hw_ix, OID_AUTO, enable_aim, CTLFLAG_RDTUN, &ixgbe_enable_aim, 0, 315 "Enable adaptive interrupt moderation"); 316 317 static int ixgbe_max_interrupt_rate = (4000000 / IXGBE_LOW_LATENCY); 318 SYSCTL_INT(_hw_ix, OID_AUTO, max_interrupt_rate, CTLFLAG_RDTUN, 319 &ixgbe_max_interrupt_rate, 0, "Maximum interrupts per second"); 320 321 /* How many packets rxeof tries to clean at a time */ 322 static int ixgbe_rx_process_limit = 256; 323 SYSCTL_INT(_hw_ix, OID_AUTO, rx_process_limit, CTLFLAG_RDTUN, 324 &ixgbe_rx_process_limit, 0, "Maximum number of received packets to process at a time, -1 means unlimited"); 325 326 /* How many packets txeof tries to clean at a time */ 327 static int ixgbe_tx_process_limit = 256; 328 SYSCTL_INT(_hw_ix, OID_AUTO, tx_process_limit, CTLFLAG_RDTUN, 329 &ixgbe_tx_process_limit, 0, 330 "Maximum number of sent packets to process at a time, -1 means unlimited"); 331 332 /* Flow control setting, default to full */ 333 static int ixgbe_flow_control = ixgbe_fc_full; 334 SYSCTL_INT(_hw_ix, OID_AUTO, flow_control, CTLFLAG_RDTUN, 335 &ixgbe_flow_control, 0, "Default flow control used for all adapters"); 336 337 /* Which packet processing uses workqueue or softint */ 338 static bool ixgbe_txrx_workqueue = false; 339 340 /* 341 * Smart speed setting, default to on 342 * this only works as a compile option 343 * right now as its during attach, set 344 * this to 'ixgbe_smart_speed_off' to 345 * disable. 346 */ 347 static int ixgbe_smart_speed = ixgbe_smart_speed_on; 348 349 /* 350 * MSI-X should be the default for best performance, 351 * but this allows it to be forced off for testing. 352 */ 353 static int ixgbe_enable_msix = 1; 354 SYSCTL_INT(_hw_ix, OID_AUTO, enable_msix, CTLFLAG_RDTUN, &ixgbe_enable_msix, 0, 355 "Enable MSI-X interrupts"); 356 357 /* 358 * Number of Queues, can be set to 0, 359 * it then autoconfigures based on the 360 * number of cpus with a max of 8. This 361 * can be overridden manually here. 362 */ 363 static int ixgbe_num_queues = 0; 364 SYSCTL_INT(_hw_ix, OID_AUTO, num_queues, CTLFLAG_RDTUN, &ixgbe_num_queues, 0, 365 "Number of queues to configure, 0 indicates autoconfigure"); 366 367 /* 368 * Number of TX descriptors per ring, 369 * setting higher than RX as this seems 370 * the better performing choice. 371 */ 372 static int ixgbe_txd = PERFORM_TXD; 373 SYSCTL_INT(_hw_ix, OID_AUTO, txd, CTLFLAG_RDTUN, &ixgbe_txd, 0, 374 "Number of transmit descriptors per queue"); 375 376 /* Number of RX descriptors per ring */ 377 static int ixgbe_rxd = PERFORM_RXD; 378 SYSCTL_INT(_hw_ix, OID_AUTO, rxd, CTLFLAG_RDTUN, &ixgbe_rxd, 0, 379 "Number of receive descriptors per queue"); 380 381 /* 382 * Defining this on will allow the use 383 * of unsupported SFP+ modules, note that 384 * doing so you are on your own :) 385 */ 386 static int allow_unsupported_sfp = false; 387 #define TUNABLE_INT(__x, __y) 388 TUNABLE_INT("hw.ix.unsupported_sfp", &allow_unsupported_sfp); 389 390 /* 391 * Not sure if Flow Director is fully baked, 392 * so we'll default to turning it off. 393 */ 394 static int ixgbe_enable_fdir = 0; 395 SYSCTL_INT(_hw_ix, OID_AUTO, enable_fdir, CTLFLAG_RDTUN, &ixgbe_enable_fdir, 0, 396 "Enable Flow Director"); 397 398 /* Legacy Transmit (single queue) */ 399 static int ixgbe_enable_legacy_tx = 0; 400 SYSCTL_INT(_hw_ix, OID_AUTO, enable_legacy_tx, CTLFLAG_RDTUN, 401 &ixgbe_enable_legacy_tx, 0, "Enable Legacy TX flow"); 402 403 /* Receive-Side Scaling */ 404 static int ixgbe_enable_rss = 1; 405 SYSCTL_INT(_hw_ix, OID_AUTO, enable_rss, CTLFLAG_RDTUN, &ixgbe_enable_rss, 0, 406 "Enable Receive-Side Scaling (RSS)"); 407 408 #if 0 409 static int (*ixgbe_start_locked)(struct ifnet *, struct tx_ring *); 410 static int (*ixgbe_ring_empty)(struct ifnet *, pcq_t *); 411 #endif 412 413 #ifdef NET_MPSAFE 414 #define IXGBE_MPSAFE 1 415 #define IXGBE_CALLOUT_FLAGS CALLOUT_MPSAFE 416 #define IXGBE_SOFTINT_FLAGS SOFTINT_MPSAFE 417 #define IXGBE_WORKQUEUE_FLAGS WQ_PERCPU | WQ_MPSAFE 418 #define IXGBE_TASKLET_WQ_FLAGS WQ_MPSAFE 419 #else 420 #define IXGBE_CALLOUT_FLAGS 0 421 #define IXGBE_SOFTINT_FLAGS 0 422 #define IXGBE_WORKQUEUE_FLAGS WQ_PERCPU 423 #define IXGBE_TASKLET_WQ_FLAGS 0 424 #endif 425 #define IXGBE_WORKQUEUE_PRI PRI_SOFTNET 426 427 /* Interval between reports of errors */ 428 static const struct timeval ixgbe_errlog_intrvl = { 60, 0 }; /* 60s */ 429 430 /************************************************************************ 431 * ixgbe_initialize_rss_mapping 432 ************************************************************************/ 433 static void 434 ixgbe_initialize_rss_mapping(struct adapter *adapter) 435 { 436 struct ixgbe_hw *hw = &adapter->hw; 437 u32 reta = 0, mrqc, rss_key[10]; 438 int queue_id, table_size, index_mult; 439 int i, j; 440 u32 rss_hash_config; 441 442 /* force use default RSS key. */ 443 #ifdef __NetBSD__ 444 rss_getkey((uint8_t *) &rss_key); 445 #else 446 if (adapter->feat_en & IXGBE_FEATURE_RSS) { 447 /* Fetch the configured RSS key */ 448 rss_getkey((uint8_t *) &rss_key); 449 } else { 450 /* set up random bits */ 451 cprng_fast(&rss_key, sizeof(rss_key)); 452 } 453 #endif 454 455 /* Set multiplier for RETA setup and table size based on MAC */ 456 index_mult = 0x1; 457 table_size = 128; 458 switch (adapter->hw.mac.type) { 459 case ixgbe_mac_82598EB: 460 index_mult = 0x11; 461 break; 462 case ixgbe_mac_X550: 463 case ixgbe_mac_X550EM_x: 464 case ixgbe_mac_X550EM_a: 465 table_size = 512; 466 break; 467 default: 468 break; 469 } 470 471 /* Set up the redirection table */ 472 for (i = 0, j = 0; i < table_size; i++, j++) { 473 if (j == adapter->num_queues) 474 j = 0; 475 476 if (adapter->feat_en & IXGBE_FEATURE_RSS) { 477 /* 478 * Fetch the RSS bucket id for the given indirection 479 * entry. Cap it at the number of configured buckets 480 * (which is num_queues.) 481 */ 482 queue_id = rss_get_indirection_to_bucket(i); 483 queue_id = queue_id % adapter->num_queues; 484 } else 485 queue_id = (j * index_mult); 486 487 /* 488 * The low 8 bits are for hash value (n+0); 489 * The next 8 bits are for hash value (n+1), etc. 490 */ 491 reta = reta >> 8; 492 reta = reta | (((uint32_t) queue_id) << 24); 493 if ((i & 3) == 3) { 494 if (i < 128) 495 IXGBE_WRITE_REG(hw, IXGBE_RETA(i >> 2), reta); 496 else 497 IXGBE_WRITE_REG(hw, IXGBE_ERETA((i >> 2) - 32), 498 reta); 499 reta = 0; 500 } 501 } 502 503 /* Now fill our hash function seeds */ 504 for (i = 0; i < 10; i++) 505 IXGBE_WRITE_REG(hw, IXGBE_RSSRK(i), rss_key[i]); 506 507 /* Perform hash on these packet types */ 508 if (adapter->feat_en & IXGBE_FEATURE_RSS) 509 rss_hash_config = rss_gethashconfig(); 510 else { 511 /* 512 * Disable UDP - IP fragments aren't currently being handled 513 * and so we end up with a mix of 2-tuple and 4-tuple 514 * traffic. 515 */ 516 rss_hash_config = RSS_HASHTYPE_RSS_IPV4 517 | RSS_HASHTYPE_RSS_TCP_IPV4 518 | RSS_HASHTYPE_RSS_IPV6 519 | RSS_HASHTYPE_RSS_TCP_IPV6 520 | RSS_HASHTYPE_RSS_IPV6_EX 521 | RSS_HASHTYPE_RSS_TCP_IPV6_EX; 522 } 523 524 mrqc = IXGBE_MRQC_RSSEN; 525 if (rss_hash_config & RSS_HASHTYPE_RSS_IPV4) 526 mrqc |= IXGBE_MRQC_RSS_FIELD_IPV4; 527 if (rss_hash_config & RSS_HASHTYPE_RSS_TCP_IPV4) 528 mrqc |= IXGBE_MRQC_RSS_FIELD_IPV4_TCP; 529 if (rss_hash_config & RSS_HASHTYPE_RSS_IPV6) 530 mrqc |= IXGBE_MRQC_RSS_FIELD_IPV6; 531 if (rss_hash_config & RSS_HASHTYPE_RSS_TCP_IPV6) 532 mrqc |= IXGBE_MRQC_RSS_FIELD_IPV6_TCP; 533 if (rss_hash_config & RSS_HASHTYPE_RSS_IPV6_EX) 534 mrqc |= IXGBE_MRQC_RSS_FIELD_IPV6_EX; 535 if (rss_hash_config & RSS_HASHTYPE_RSS_TCP_IPV6_EX) 536 mrqc |= IXGBE_MRQC_RSS_FIELD_IPV6_EX_TCP; 537 if (rss_hash_config & RSS_HASHTYPE_RSS_UDP_IPV4) 538 mrqc |= IXGBE_MRQC_RSS_FIELD_IPV4_UDP; 539 if (rss_hash_config & RSS_HASHTYPE_RSS_UDP_IPV6) 540 mrqc |= IXGBE_MRQC_RSS_FIELD_IPV6_UDP; 541 if (rss_hash_config & RSS_HASHTYPE_RSS_UDP_IPV6_EX) 542 mrqc |= IXGBE_MRQC_RSS_FIELD_IPV6_EX_UDP; 543 mrqc |= ixgbe_get_mrqc(adapter->iov_mode); 544 IXGBE_WRITE_REG(hw, IXGBE_MRQC, mrqc); 545 } /* ixgbe_initialize_rss_mapping */ 546 547 /************************************************************************ 548 * ixgbe_initialize_receive_units - Setup receive registers and features. 549 ************************************************************************/ 550 #define BSIZEPKT_ROUNDUP ((1<<IXGBE_SRRCTL_BSIZEPKT_SHIFT)-1) 551 552 static void 553 ixgbe_initialize_receive_units(struct adapter *adapter) 554 { 555 struct rx_ring *rxr = adapter->rx_rings; 556 struct ixgbe_hw *hw = &adapter->hw; 557 struct ifnet *ifp = adapter->ifp; 558 int i, j; 559 u32 bufsz, fctrl, srrctl, rxcsum; 560 u32 hlreg; 561 562 /* 563 * Make sure receives are disabled while 564 * setting up the descriptor ring 565 */ 566 ixgbe_disable_rx(hw); 567 568 /* Enable broadcasts */ 569 fctrl = IXGBE_READ_REG(hw, IXGBE_FCTRL); 570 fctrl |= IXGBE_FCTRL_BAM; 571 if (adapter->hw.mac.type == ixgbe_mac_82598EB) { 572 fctrl |= IXGBE_FCTRL_DPF; 573 fctrl |= IXGBE_FCTRL_PMCF; 574 } 575 IXGBE_WRITE_REG(hw, IXGBE_FCTRL, fctrl); 576 577 /* Set for Jumbo Frames? */ 578 hlreg = IXGBE_READ_REG(hw, IXGBE_HLREG0); 579 if (ifp->if_mtu > ETHERMTU) 580 hlreg |= IXGBE_HLREG0_JUMBOEN; 581 else 582 hlreg &= ~IXGBE_HLREG0_JUMBOEN; 583 584 #ifdef DEV_NETMAP 585 /* CRC stripping is conditional in Netmap */ 586 if ((adapter->feat_en & IXGBE_FEATURE_NETMAP) && 587 (ifp->if_capenable & IFCAP_NETMAP) && 588 !ix_crcstrip) 589 hlreg &= ~IXGBE_HLREG0_RXCRCSTRP; 590 else 591 #endif /* DEV_NETMAP */ 592 hlreg |= IXGBE_HLREG0_RXCRCSTRP; 593 594 IXGBE_WRITE_REG(hw, IXGBE_HLREG0, hlreg); 595 596 bufsz = (adapter->rx_mbuf_sz + BSIZEPKT_ROUNDUP) >> 597 IXGBE_SRRCTL_BSIZEPKT_SHIFT; 598 599 for (i = 0; i < adapter->num_queues; i++, rxr++) { 600 u64 rdba = rxr->rxdma.dma_paddr; 601 u32 reg; 602 int regnum = i / 4; /* 1 register per 4 queues */ 603 int regshift = i % 4; /* 4 bits per 1 queue */ 604 j = rxr->me; 605 606 /* Setup the Base and Length of the Rx Descriptor Ring */ 607 IXGBE_WRITE_REG(hw, IXGBE_RDBAL(j), 608 (rdba & 0x00000000ffffffffULL)); 609 IXGBE_WRITE_REG(hw, IXGBE_RDBAH(j), (rdba >> 32)); 610 IXGBE_WRITE_REG(hw, IXGBE_RDLEN(j), 611 adapter->num_rx_desc * sizeof(union ixgbe_adv_rx_desc)); 612 613 /* Set up the SRRCTL register */ 614 srrctl = IXGBE_READ_REG(hw, IXGBE_SRRCTL(j)); 615 srrctl &= ~IXGBE_SRRCTL_BSIZEHDR_MASK; 616 srrctl &= ~IXGBE_SRRCTL_BSIZEPKT_MASK; 617 srrctl |= bufsz; 618 srrctl |= IXGBE_SRRCTL_DESCTYPE_ADV_ONEBUF; 619 620 /* Set RQSMR (Receive Queue Statistic Mapping) register */ 621 reg = IXGBE_READ_REG(hw, IXGBE_RQSMR(regnum)); 622 reg &= ~(0x000000ffUL << (regshift * 8)); 623 reg |= i << (regshift * 8); 624 IXGBE_WRITE_REG(hw, IXGBE_RQSMR(regnum), reg); 625 626 /* 627 * Set DROP_EN iff we have no flow control and >1 queue. 628 * Note that srrctl was cleared shortly before during reset, 629 * so we do not need to clear the bit, but do it just in case 630 * this code is moved elsewhere. 631 */ 632 if (adapter->num_queues > 1 && 633 adapter->hw.fc.requested_mode == ixgbe_fc_none) { 634 srrctl |= IXGBE_SRRCTL_DROP_EN; 635 } else { 636 srrctl &= ~IXGBE_SRRCTL_DROP_EN; 637 } 638 639 IXGBE_WRITE_REG(hw, IXGBE_SRRCTL(j), srrctl); 640 641 /* Setup the HW Rx Head and Tail Descriptor Pointers */ 642 IXGBE_WRITE_REG(hw, IXGBE_RDH(j), 0); 643 IXGBE_WRITE_REG(hw, IXGBE_RDT(j), 0); 644 645 /* Set the driver rx tail address */ 646 rxr->tail = IXGBE_RDT(rxr->me); 647 } 648 649 if (adapter->hw.mac.type != ixgbe_mac_82598EB) { 650 u32 psrtype = IXGBE_PSRTYPE_TCPHDR 651 | IXGBE_PSRTYPE_UDPHDR 652 | IXGBE_PSRTYPE_IPV4HDR 653 | IXGBE_PSRTYPE_IPV6HDR; 654 IXGBE_WRITE_REG(hw, IXGBE_PSRTYPE(0), psrtype); 655 } 656 657 rxcsum = IXGBE_READ_REG(hw, IXGBE_RXCSUM); 658 659 ixgbe_initialize_rss_mapping(adapter); 660 661 if (adapter->num_queues > 1) { 662 /* RSS and RX IPP Checksum are mutually exclusive */ 663 rxcsum |= IXGBE_RXCSUM_PCSD; 664 } 665 666 if (ifp->if_capenable & IFCAP_RXCSUM) 667 rxcsum |= IXGBE_RXCSUM_PCSD; 668 669 /* This is useful for calculating UDP/IP fragment checksums */ 670 if (!(rxcsum & IXGBE_RXCSUM_PCSD)) 671 rxcsum |= IXGBE_RXCSUM_IPPCSE; 672 673 IXGBE_WRITE_REG(hw, IXGBE_RXCSUM, rxcsum); 674 675 } /* ixgbe_initialize_receive_units */ 676 677 /************************************************************************ 678 * ixgbe_initialize_transmit_units - Enable transmit units. 679 ************************************************************************/ 680 static void 681 ixgbe_initialize_transmit_units(struct adapter *adapter) 682 { 683 struct tx_ring *txr = adapter->tx_rings; 684 struct ixgbe_hw *hw = &adapter->hw; 685 int i; 686 687 INIT_DEBUGOUT("ixgbe_initialize_transmit_units"); 688 689 /* Setup the Base and Length of the Tx Descriptor Ring */ 690 for (i = 0; i < adapter->num_queues; i++, txr++) { 691 u64 tdba = txr->txdma.dma_paddr; 692 u32 txctrl = 0; 693 u32 tqsmreg, reg; 694 int regnum = i / 4; /* 1 register per 4 queues */ 695 int regshift = i % 4; /* 4 bits per 1 queue */ 696 int j = txr->me; 697 698 IXGBE_WRITE_REG(hw, IXGBE_TDBAL(j), 699 (tdba & 0x00000000ffffffffULL)); 700 IXGBE_WRITE_REG(hw, IXGBE_TDBAH(j), (tdba >> 32)); 701 IXGBE_WRITE_REG(hw, IXGBE_TDLEN(j), 702 adapter->num_tx_desc * sizeof(union ixgbe_adv_tx_desc)); 703 704 /* 705 * Set TQSMR (Transmit Queue Statistic Mapping) register. 706 * Register location is different between 82598 and others. 707 */ 708 if (adapter->hw.mac.type == ixgbe_mac_82598EB) 709 tqsmreg = IXGBE_TQSMR(regnum); 710 else 711 tqsmreg = IXGBE_TQSM(regnum); 712 reg = IXGBE_READ_REG(hw, tqsmreg); 713 reg &= ~(0x000000ffUL << (regshift * 8)); 714 reg |= i << (regshift * 8); 715 IXGBE_WRITE_REG(hw, tqsmreg, reg); 716 717 /* Setup the HW Tx Head and Tail descriptor pointers */ 718 IXGBE_WRITE_REG(hw, IXGBE_TDH(j), 0); 719 IXGBE_WRITE_REG(hw, IXGBE_TDT(j), 0); 720 721 /* Cache the tail address */ 722 txr->tail = IXGBE_TDT(j); 723 724 txr->txr_no_space = false; 725 726 /* Disable Head Writeback */ 727 /* 728 * Note: for X550 series devices, these registers are actually 729 * prefixed with TPH_ instead of DCA_, but the addresses and 730 * fields remain the same. 731 */ 732 switch (hw->mac.type) { 733 case ixgbe_mac_82598EB: 734 txctrl = IXGBE_READ_REG(hw, IXGBE_DCA_TXCTRL(j)); 735 break; 736 default: 737 txctrl = IXGBE_READ_REG(hw, IXGBE_DCA_TXCTRL_82599(j)); 738 break; 739 } 740 txctrl &= ~IXGBE_DCA_TXCTRL_DESC_WRO_EN; 741 switch (hw->mac.type) { 742 case ixgbe_mac_82598EB: 743 IXGBE_WRITE_REG(hw, IXGBE_DCA_TXCTRL(j), txctrl); 744 break; 745 default: 746 IXGBE_WRITE_REG(hw, IXGBE_DCA_TXCTRL_82599(j), txctrl); 747 break; 748 } 749 750 } 751 752 if (hw->mac.type != ixgbe_mac_82598EB) { 753 u32 dmatxctl, rttdcs; 754 755 dmatxctl = IXGBE_READ_REG(hw, IXGBE_DMATXCTL); 756 dmatxctl |= IXGBE_DMATXCTL_TE; 757 IXGBE_WRITE_REG(hw, IXGBE_DMATXCTL, dmatxctl); 758 /* Disable arbiter to set MTQC */ 759 rttdcs = IXGBE_READ_REG(hw, IXGBE_RTTDCS); 760 rttdcs |= IXGBE_RTTDCS_ARBDIS; 761 IXGBE_WRITE_REG(hw, IXGBE_RTTDCS, rttdcs); 762 IXGBE_WRITE_REG(hw, IXGBE_MTQC, 763 ixgbe_get_mtqc(adapter->iov_mode)); 764 rttdcs &= ~IXGBE_RTTDCS_ARBDIS; 765 IXGBE_WRITE_REG(hw, IXGBE_RTTDCS, rttdcs); 766 } 767 768 return; 769 } /* ixgbe_initialize_transmit_units */ 770 771 static void 772 ixgbe_quirks(struct adapter *adapter) 773 { 774 device_t dev = adapter->dev; 775 struct ixgbe_hw *hw = &adapter->hw; 776 const char *vendor, *product; 777 778 if (hw->device_id == IXGBE_DEV_ID_X550EM_A_SFP_N) { 779 /* 780 * Quirk for inverted logic of SFP+'s MOD_ABS on GIGABYTE 781 * MA10-ST0. 782 */ 783 vendor = pmf_get_platform("system-vendor"); 784 product = pmf_get_platform("system-product"); 785 786 if ((vendor == NULL) || (product == NULL)) 787 return; 788 789 if ((strcmp(vendor, "GIGABYTE") == 0) && 790 (strcmp(product, "MA10-ST0") == 0)) { 791 aprint_verbose_dev(dev, 792 "Enable SFP+ MOD_ABS inverse quirk\n"); 793 adapter->hw.quirks |= IXGBE_QUIRK_MOD_ABS_INVERT; 794 } 795 } 796 } 797 798 /************************************************************************ 799 * ixgbe_attach - Device initialization routine 800 * 801 * Called when the driver is being loaded. 802 * Identifies the type of hardware, allocates all resources 803 * and initializes the hardware. 804 * 805 * return 0 on success, positive on failure 806 ************************************************************************/ 807 static void 808 ixgbe_attach(device_t parent, device_t dev, void *aux) 809 { 810 struct adapter *adapter; 811 struct ixgbe_hw *hw; 812 int error = -1; 813 u32 ctrl_ext; 814 u16 high, low, nvmreg; 815 pcireg_t id, subid; 816 const ixgbe_vendor_info_t *ent; 817 struct pci_attach_args *pa = aux; 818 bool unsupported_sfp = false; 819 const char *str; 820 char wqname[MAXCOMLEN]; 821 char buf[256]; 822 823 INIT_DEBUGOUT("ixgbe_attach: begin"); 824 825 /* Allocate, clear, and link in our adapter structure */ 826 adapter = device_private(dev); 827 adapter->hw.back = adapter; 828 adapter->dev = dev; 829 hw = &adapter->hw; 830 adapter->osdep.pc = pa->pa_pc; 831 adapter->osdep.tag = pa->pa_tag; 832 if (pci_dma64_available(pa)) 833 adapter->osdep.dmat = pa->pa_dmat64; 834 else 835 adapter->osdep.dmat = pa->pa_dmat; 836 adapter->osdep.attached = false; 837 adapter->osdep.detaching = false; 838 839 ent = ixgbe_lookup(pa); 840 841 KASSERT(ent != NULL); 842 843 aprint_normal(": %s, Version - %s\n", 844 ixgbe_strings[ent->index], ixgbe_driver_version); 845 846 /* Core Lock Init */ 847 IXGBE_CORE_LOCK_INIT(adapter, device_xname(dev)); 848 849 /* Set up the timer callout and workqueue */ 850 callout_init(&adapter->timer, IXGBE_CALLOUT_FLAGS); 851 snprintf(wqname, sizeof(wqname), "%s-timer", device_xname(dev)); 852 error = workqueue_create(&adapter->timer_wq, wqname, 853 ixgbe_handle_timer, adapter, IXGBE_WORKQUEUE_PRI, IPL_NET, 854 IXGBE_TASKLET_WQ_FLAGS); 855 if (error) { 856 aprint_error_dev(dev, 857 "could not create timer workqueue (%d)\n", error); 858 goto err_out; 859 } 860 861 /* Determine hardware revision */ 862 id = pci_conf_read(pa->pa_pc, pa->pa_tag, PCI_ID_REG); 863 subid = pci_conf_read(pa->pa_pc, pa->pa_tag, PCI_SUBSYS_ID_REG); 864 865 hw->vendor_id = PCI_VENDOR(id); 866 hw->device_id = PCI_PRODUCT(id); 867 hw->revision_id = 868 PCI_REVISION(pci_conf_read(pa->pa_pc, pa->pa_tag, PCI_CLASS_REG)); 869 hw->subsystem_vendor_id = PCI_SUBSYS_VENDOR(subid); 870 hw->subsystem_device_id = PCI_SUBSYS_ID(subid); 871 872 /* Set quirk flags */ 873 ixgbe_quirks(adapter); 874 875 /* 876 * Make sure BUSMASTER is set 877 */ 878 ixgbe_pci_enable_busmaster(pa->pa_pc, pa->pa_tag); 879 880 /* Do base PCI setup - map BAR0 */ 881 if (ixgbe_allocate_pci_resources(adapter, pa)) { 882 aprint_error_dev(dev, "Allocation of PCI resources failed\n"); 883 error = ENXIO; 884 goto err_out; 885 } 886 887 /* let hardware know driver is loaded */ 888 ctrl_ext = IXGBE_READ_REG(hw, IXGBE_CTRL_EXT); 889 ctrl_ext |= IXGBE_CTRL_EXT_DRV_LOAD; 890 IXGBE_WRITE_REG(hw, IXGBE_CTRL_EXT, ctrl_ext); 891 892 /* 893 * Initialize the shared code 894 */ 895 if (ixgbe_init_shared_code(hw) != 0) { 896 aprint_error_dev(dev, "Unable to initialize the shared code\n"); 897 error = ENXIO; 898 goto err_out; 899 } 900 901 switch (hw->mac.type) { 902 case ixgbe_mac_82598EB: 903 str = "82598EB"; 904 break; 905 case ixgbe_mac_82599EB: 906 str = "82599EB"; 907 break; 908 case ixgbe_mac_X540: 909 str = "X540"; 910 break; 911 case ixgbe_mac_X550: 912 str = "X550"; 913 break; 914 case ixgbe_mac_X550EM_x: 915 str = "X550EM X"; 916 break; 917 case ixgbe_mac_X550EM_a: 918 str = "X550EM A"; 919 break; 920 default: 921 str = "Unknown"; 922 break; 923 } 924 aprint_normal_dev(dev, "device %s\n", str); 925 926 hw->allow_unsupported_sfp = allow_unsupported_sfp; 927 928 /* Pick up the 82599 settings */ 929 if (hw->mac.type != ixgbe_mac_82598EB) 930 hw->phy.smart_speed = ixgbe_smart_speed; 931 932 /* Set the right number of segments */ 933 KASSERT(IXGBE_82599_SCATTER_MAX >= IXGBE_SCATTER_DEFAULT); 934 adapter->num_segs = IXGBE_SCATTER_DEFAULT; 935 936 /* Ensure SW/FW semaphore is free */ 937 ixgbe_init_swfw_semaphore(hw); 938 939 hw->mac.ops.set_lan_id(hw); 940 ixgbe_init_device_features(adapter); 941 942 if (ixgbe_configure_interrupts(adapter)) { 943 error = ENXIO; 944 goto err_out; 945 } 946 947 /* Allocate multicast array memory. */ 948 adapter->mta = malloc(sizeof(*adapter->mta) * 949 MAX_NUM_MULTICAST_ADDRESSES, M_DEVBUF, M_WAITOK); 950 951 /* Enable WoL (if supported) */ 952 ixgbe_check_wol_support(adapter); 953 954 /* Register for VLAN events */ 955 ether_set_vlan_cb(&adapter->osdep.ec, ixgbe_vlan_cb); 956 957 /* Verify adapter fan is still functional (if applicable) */ 958 if (adapter->feat_en & IXGBE_FEATURE_FAN_FAIL) { 959 u32 esdp = IXGBE_READ_REG(hw, IXGBE_ESDP); 960 ixgbe_check_fan_failure(adapter, esdp, FALSE); 961 } 962 963 /* Set an initial default flow control value */ 964 hw->fc.requested_mode = ixgbe_flow_control; 965 966 /* Do descriptor calc and sanity checks */ 967 if (((ixgbe_txd * sizeof(union ixgbe_adv_tx_desc)) % DBA_ALIGN) != 0 || 968 ixgbe_txd < MIN_TXD || ixgbe_txd > MAX_TXD) { 969 aprint_error_dev(dev, "TXD config issue, using default!\n"); 970 adapter->num_tx_desc = DEFAULT_TXD; 971 } else 972 adapter->num_tx_desc = ixgbe_txd; 973 974 if (((ixgbe_rxd * sizeof(union ixgbe_adv_rx_desc)) % DBA_ALIGN) != 0 || 975 ixgbe_rxd < MIN_RXD || ixgbe_rxd > MAX_RXD) { 976 aprint_error_dev(dev, "RXD config issue, using default!\n"); 977 adapter->num_rx_desc = DEFAULT_RXD; 978 } else 979 adapter->num_rx_desc = ixgbe_rxd; 980 981 /* Sysctls for limiting the amount of work done in the taskqueues */ 982 adapter->rx_process_limit 983 = (ixgbe_rx_process_limit <= adapter->num_rx_desc) 984 ? ixgbe_rx_process_limit : adapter->num_rx_desc; 985 adapter->tx_process_limit 986 = (ixgbe_tx_process_limit <= adapter->num_tx_desc) 987 ? ixgbe_tx_process_limit : adapter->num_tx_desc; 988 989 /* Set default high limit of copying mbuf in rxeof */ 990 adapter->rx_copy_len = IXGBE_RX_COPY_LEN_MAX; 991 992 /* Allocate our TX/RX Queues */ 993 if (ixgbe_allocate_queues(adapter)) { 994 error = ENOMEM; 995 goto err_out; 996 } 997 998 hw->phy.reset_if_overtemp = TRUE; 999 error = ixgbe_reset_hw(hw); 1000 hw->phy.reset_if_overtemp = FALSE; 1001 if (error == IXGBE_ERR_SFP_NOT_PRESENT) 1002 error = IXGBE_SUCCESS; 1003 else if (error == IXGBE_ERR_SFP_NOT_SUPPORTED) { 1004 aprint_error_dev(dev, "Unsupported SFP+ module detected!\n"); 1005 unsupported_sfp = true; 1006 error = IXGBE_SUCCESS; 1007 } else if (error) { 1008 aprint_error_dev(dev, 1009 "Hardware initialization failed(error = %d)\n", error); 1010 error = EIO; 1011 goto err_late; 1012 } 1013 1014 /* Make sure we have a good EEPROM before we read from it */ 1015 if (ixgbe_validate_eeprom_checksum(&adapter->hw, NULL) < 0) { 1016 aprint_error_dev(dev, "The EEPROM Checksum Is Not Valid\n"); 1017 error = EIO; 1018 goto err_late; 1019 } 1020 1021 aprint_normal("%s:", device_xname(dev)); 1022 /* NVM Image Version */ 1023 high = low = 0; 1024 switch (hw->mac.type) { 1025 case ixgbe_mac_82598EB: 1026 /* 1027 * Print version from the dev starter version (0x29). The 1028 * location is the same as newer device's IXGBE_NVM_MAP_VER. 1029 */ 1030 hw->eeprom.ops.read(hw, IXGBE_NVM_MAP_VER, &nvmreg); 1031 if (nvmreg == 0xffff) 1032 break; 1033 high = (nvmreg >> 12) & 0x0f; 1034 low = (nvmreg >> 4) & 0xff; 1035 id = nvmreg & 0x0f; 1036 /* 1037 * The following output might not be correct. Some 82598 cards 1038 * have 0x1070 or 0x2090. 82598 spec update notes about 2.9.0. 1039 */ 1040 aprint_normal(" NVM Image Version %u.%u.%u,", high, low, id); 1041 break; 1042 case ixgbe_mac_X540: 1043 case ixgbe_mac_X550EM_a: 1044 hw->eeprom.ops.read(hw, IXGBE_NVM_IMAGE_VER, &nvmreg); 1045 if (nvmreg == 0xffff) 1046 break; 1047 high = (nvmreg >> 12) & 0x0f; 1048 low = (nvmreg >> 4) & 0xff; 1049 id = nvmreg & 0x0f; 1050 aprint_normal(" NVM Image Version %u.", high); 1051 if (hw->mac.type == ixgbe_mac_X540) 1052 str = "%x"; 1053 else 1054 str = "%02x"; 1055 aprint_normal(str, low); 1056 aprint_normal(" ID 0x%x,", id); 1057 break; 1058 case ixgbe_mac_X550EM_x: 1059 case ixgbe_mac_X550: 1060 hw->eeprom.ops.read(hw, IXGBE_NVM_IMAGE_VER, &nvmreg); 1061 if (nvmreg == 0xffff) 1062 break; 1063 high = (nvmreg >> 12) & 0x0f; 1064 low = nvmreg & 0xff; 1065 aprint_normal(" NVM Image Version %u.%02x,", high, low); 1066 break; 1067 default: 1068 break; 1069 } 1070 hw->eeprom.nvm_image_ver_high = high; 1071 hw->eeprom.nvm_image_ver_low = low; 1072 1073 /* PHY firmware revision */ 1074 switch (hw->mac.type) { 1075 case ixgbe_mac_X540: 1076 case ixgbe_mac_X550: 1077 hw->eeprom.ops.read(hw, IXGBE_PHYFW_REV, &nvmreg); 1078 if (nvmreg == 0xffff) 1079 break; 1080 high = (nvmreg >> 12) & 0x0f; 1081 low = (nvmreg >> 4) & 0xff; 1082 id = nvmreg & 0x000f; 1083 aprint_normal(" PHY FW Revision %u.", high); 1084 if (hw->mac.type == ixgbe_mac_X540) 1085 str = "%x"; 1086 else 1087 str = "%02x"; 1088 aprint_normal(str, low); 1089 aprint_normal(" ID 0x%x,", id); 1090 break; 1091 default: 1092 break; 1093 } 1094 1095 /* NVM Map version & OEM NVM Image version */ 1096 switch (hw->mac.type) { 1097 case ixgbe_mac_X550: 1098 case ixgbe_mac_X550EM_x: 1099 case ixgbe_mac_X550EM_a: 1100 hw->eeprom.ops.read(hw, IXGBE_NVM_MAP_VER, &nvmreg); 1101 if (nvmreg != 0xffff) { 1102 high = (nvmreg >> 12) & 0x0f; 1103 low = nvmreg & 0x00ff; 1104 aprint_normal(" NVM Map version %u.%02x,", high, low); 1105 } 1106 hw->eeprom.ops.read(hw, IXGBE_OEM_NVM_IMAGE_VER, &nvmreg); 1107 if (nvmreg != 0xffff) { 1108 high = (nvmreg >> 12) & 0x0f; 1109 low = nvmreg & 0x00ff; 1110 aprint_verbose(" OEM NVM Image version %u.%02x,", high, 1111 low); 1112 } 1113 break; 1114 default: 1115 break; 1116 } 1117 1118 /* Print the ETrackID */ 1119 hw->eeprom.ops.read(hw, IXGBE_ETRACKID_H, &high); 1120 hw->eeprom.ops.read(hw, IXGBE_ETRACKID_L, &low); 1121 aprint_normal(" ETrackID %08x\n", ((uint32_t)high << 16) | low); 1122 1123 /* Printed Board Assembly number */ 1124 error = ixgbe_read_pba_string(hw, buf, IXGBE_PBANUM_LENGTH); 1125 aprint_normal_dev(dev, "PBA number %s\n", error ? "unknown" : buf); 1126 1127 if (adapter->feat_en & IXGBE_FEATURE_MSIX) { 1128 error = ixgbe_allocate_msix(adapter, pa); 1129 if (error) { 1130 /* Free allocated queue structures first */ 1131 ixgbe_free_queues(adapter); 1132 1133 /* Fallback to legacy interrupt */ 1134 adapter->feat_en &= ~IXGBE_FEATURE_MSIX; 1135 if (adapter->feat_cap & IXGBE_FEATURE_MSI) 1136 adapter->feat_en |= IXGBE_FEATURE_MSI; 1137 adapter->num_queues = 1; 1138 1139 /* Allocate our TX/RX Queues again */ 1140 if (ixgbe_allocate_queues(adapter)) { 1141 error = ENOMEM; 1142 goto err_out; 1143 } 1144 } 1145 } 1146 1147 /* Recovery mode */ 1148 switch (adapter->hw.mac.type) { 1149 case ixgbe_mac_X550: 1150 case ixgbe_mac_X550EM_x: 1151 case ixgbe_mac_X550EM_a: 1152 /* >= 2.00 */ 1153 if (hw->eeprom.nvm_image_ver_high >= 2) { 1154 adapter->feat_cap |= IXGBE_FEATURE_RECOVERY_MODE; 1155 adapter->feat_en |= IXGBE_FEATURE_RECOVERY_MODE; 1156 } 1157 break; 1158 default: 1159 break; 1160 } 1161 1162 if ((adapter->feat_en & IXGBE_FEATURE_MSIX) == 0) 1163 error = ixgbe_allocate_legacy(adapter, pa); 1164 if (error) 1165 goto err_late; 1166 1167 /* Tasklets for Link, SFP, Multispeed Fiber and Flow Director */ 1168 mutex_init(&(adapter)->admin_mtx, MUTEX_DEFAULT, IPL_NET); 1169 snprintf(wqname, sizeof(wqname), "%s-admin", device_xname(dev)); 1170 error = workqueue_create(&adapter->admin_wq, wqname, 1171 ixgbe_handle_admin, adapter, IXGBE_WORKQUEUE_PRI, IPL_NET, 1172 IXGBE_TASKLET_WQ_FLAGS); 1173 if (error) { 1174 aprint_error_dev(dev, 1175 "could not create admin workqueue (%d)\n", error); 1176 goto err_out; 1177 } 1178 1179 error = ixgbe_start_hw(hw); 1180 switch (error) { 1181 case IXGBE_ERR_EEPROM_VERSION: 1182 aprint_error_dev(dev, "This device is a pre-production adapter/" 1183 "LOM. Please be aware there may be issues associated " 1184 "with your hardware.\nIf you are experiencing problems " 1185 "please contact your Intel or hardware representative " 1186 "who provided you with this hardware.\n"); 1187 break; 1188 default: 1189 break; 1190 } 1191 1192 /* Setup OS specific network interface */ 1193 if (ixgbe_setup_interface(dev, adapter) != 0) 1194 goto err_late; 1195 1196 /* 1197 * Print PHY ID only for copper PHY. On device which has SFP(+) cage 1198 * and a module is inserted, phy.id is not MII PHY id but SFF 8024 ID. 1199 */ 1200 if (hw->phy.media_type == ixgbe_media_type_copper) { 1201 uint16_t id1, id2; 1202 int oui, model, rev; 1203 char descr[MII_MAX_DESCR_LEN]; 1204 1205 id1 = hw->phy.id >> 16; 1206 id2 = hw->phy.id & 0xffff; 1207 oui = MII_OUI(id1, id2); 1208 model = MII_MODEL(id2); 1209 rev = MII_REV(id2); 1210 mii_get_descr(descr, sizeof(descr), oui, model); 1211 if (descr[0]) 1212 aprint_normal_dev(dev, "PHY: %s, rev. %d\n", 1213 descr, rev); 1214 else 1215 aprint_normal_dev(dev, 1216 "PHY OUI 0x%06x, model 0x%04x, rev. %d\n", 1217 oui, model, rev); 1218 } 1219 1220 /* Enable EEE power saving */ 1221 if (adapter->feat_cap & IXGBE_FEATURE_EEE) 1222 hw->mac.ops.setup_eee(hw, 1223 adapter->feat_en & IXGBE_FEATURE_EEE); 1224 1225 /* Enable power to the phy. */ 1226 if (!unsupported_sfp) { 1227 /* Enable the optics for 82599 SFP+ fiber */ 1228 ixgbe_enable_tx_laser(hw); 1229 1230 /* 1231 * XXX Currently, ixgbe_set_phy_power() supports only copper 1232 * PHY, so it's not required to test with !unsupported_sfp. 1233 */ 1234 ixgbe_set_phy_power(hw, TRUE); 1235 } 1236 1237 /* Initialize statistics */ 1238 ixgbe_update_stats_counters(adapter); 1239 1240 /* Check PCIE slot type/speed/width */ 1241 ixgbe_get_slot_info(adapter); 1242 1243 /* 1244 * Do time init and sysctl init here, but 1245 * only on the first port of a bypass adapter. 1246 */ 1247 ixgbe_bypass_init(adapter); 1248 1249 /* Set an initial dmac value */ 1250 adapter->dmac = 0; 1251 /* Set initial advertised speeds (if applicable) */ 1252 adapter->advertise = ixgbe_get_default_advertise(adapter); 1253 1254 if (adapter->feat_cap & IXGBE_FEATURE_SRIOV) 1255 ixgbe_define_iov_schemas(dev, &error); 1256 1257 /* Add sysctls */ 1258 ixgbe_add_device_sysctls(adapter); 1259 ixgbe_add_hw_stats(adapter); 1260 1261 /* For Netmap */ 1262 adapter->init_locked = ixgbe_init_locked; 1263 adapter->stop_locked = ixgbe_stop_locked; 1264 1265 if (adapter->feat_en & IXGBE_FEATURE_NETMAP) 1266 ixgbe_netmap_attach(adapter); 1267 1268 snprintb(buf, sizeof(buf), IXGBE_FEATURE_FLAGS, adapter->feat_cap); 1269 aprint_verbose_dev(dev, "feature cap %s\n", buf); 1270 snprintb(buf, sizeof(buf), IXGBE_FEATURE_FLAGS, adapter->feat_en); 1271 aprint_verbose_dev(dev, "feature ena %s\n", buf); 1272 1273 if (pmf_device_register(dev, ixgbe_suspend, ixgbe_resume)) 1274 pmf_class_network_register(dev, adapter->ifp); 1275 else 1276 aprint_error_dev(dev, "couldn't establish power handler\n"); 1277 1278 /* Init recovery mode timer and state variable */ 1279 if (adapter->feat_en & IXGBE_FEATURE_RECOVERY_MODE) { 1280 adapter->recovery_mode = 0; 1281 1282 /* Set up the timer callout */ 1283 callout_init(&adapter->recovery_mode_timer, 1284 IXGBE_CALLOUT_FLAGS); 1285 snprintf(wqname, sizeof(wqname), "%s-recovery", 1286 device_xname(dev)); 1287 error = workqueue_create(&adapter->recovery_mode_timer_wq, 1288 wqname, ixgbe_handle_recovery_mode_timer, adapter, 1289 IXGBE_WORKQUEUE_PRI, IPL_NET, IXGBE_TASKLET_WQ_FLAGS); 1290 if (error) { 1291 aprint_error_dev(dev, "could not create " 1292 "recovery_mode_timer workqueue (%d)\n", error); 1293 goto err_out; 1294 } 1295 1296 /* Start the task */ 1297 callout_reset(&adapter->recovery_mode_timer, hz, 1298 ixgbe_recovery_mode_timer, adapter); 1299 } 1300 1301 INIT_DEBUGOUT("ixgbe_attach: end"); 1302 adapter->osdep.attached = true; 1303 1304 return; 1305 1306 err_late: 1307 ixgbe_free_queues(adapter); 1308 err_out: 1309 ctrl_ext = IXGBE_READ_REG(&adapter->hw, IXGBE_CTRL_EXT); 1310 ctrl_ext &= ~IXGBE_CTRL_EXT_DRV_LOAD; 1311 IXGBE_WRITE_REG(&adapter->hw, IXGBE_CTRL_EXT, ctrl_ext); 1312 ixgbe_free_deferred_handlers(adapter); 1313 ixgbe_free_pci_resources(adapter); 1314 if (adapter->mta != NULL) 1315 free(adapter->mta, M_DEVBUF); 1316 mutex_destroy(&(adapter)->admin_mtx); /* XXX appropriate order? */ 1317 IXGBE_CORE_LOCK_DESTROY(adapter); 1318 1319 return; 1320 } /* ixgbe_attach */ 1321 1322 /************************************************************************ 1323 * ixgbe_check_wol_support 1324 * 1325 * Checks whether the adapter's ports are capable of 1326 * Wake On LAN by reading the adapter's NVM. 1327 * 1328 * Sets each port's hw->wol_enabled value depending 1329 * on the value read here. 1330 ************************************************************************/ 1331 static void 1332 ixgbe_check_wol_support(struct adapter *adapter) 1333 { 1334 struct ixgbe_hw *hw = &adapter->hw; 1335 u16 dev_caps = 0; 1336 1337 /* Find out WoL support for port */ 1338 adapter->wol_support = hw->wol_enabled = 0; 1339 ixgbe_get_device_caps(hw, &dev_caps); 1340 if ((dev_caps & IXGBE_DEVICE_CAPS_WOL_PORT0_1) || 1341 ((dev_caps & IXGBE_DEVICE_CAPS_WOL_PORT0) && 1342 hw->bus.func == 0)) 1343 adapter->wol_support = hw->wol_enabled = 1; 1344 1345 /* Save initial wake up filter configuration */ 1346 adapter->wufc = IXGBE_READ_REG(hw, IXGBE_WUFC); 1347 1348 return; 1349 } /* ixgbe_check_wol_support */ 1350 1351 /************************************************************************ 1352 * ixgbe_setup_interface 1353 * 1354 * Setup networking device structure and register an interface. 1355 ************************************************************************/ 1356 static int 1357 ixgbe_setup_interface(device_t dev, struct adapter *adapter) 1358 { 1359 struct ethercom *ec = &adapter->osdep.ec; 1360 struct ifnet *ifp; 1361 1362 INIT_DEBUGOUT("ixgbe_setup_interface: begin"); 1363 1364 ifp = adapter->ifp = &ec->ec_if; 1365 strlcpy(ifp->if_xname, device_xname(dev), IFNAMSIZ); 1366 ifp->if_baudrate = IF_Gbps(10); 1367 ifp->if_init = ixgbe_init; 1368 ifp->if_stop = ixgbe_ifstop; 1369 ifp->if_softc = adapter; 1370 ifp->if_flags = IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST; 1371 #ifdef IXGBE_MPSAFE 1372 ifp->if_extflags = IFEF_MPSAFE; 1373 #endif 1374 ifp->if_ioctl = ixgbe_ioctl; 1375 #if __FreeBSD_version >= 1100045 1376 /* TSO parameters */ 1377 ifp->if_hw_tsomax = 65518; 1378 ifp->if_hw_tsomaxsegcount = IXGBE_82599_SCATTER; 1379 ifp->if_hw_tsomaxsegsize = 2048; 1380 #endif 1381 if (adapter->feat_en & IXGBE_FEATURE_LEGACY_TX) { 1382 #if 0 1383 ixgbe_start_locked = ixgbe_legacy_start_locked; 1384 #endif 1385 } else { 1386 ifp->if_transmit = ixgbe_mq_start; 1387 #if 0 1388 ixgbe_start_locked = ixgbe_mq_start_locked; 1389 #endif 1390 } 1391 ifp->if_start = ixgbe_legacy_start; 1392 IFQ_SET_MAXLEN(&ifp->if_snd, adapter->num_tx_desc - 2); 1393 IFQ_SET_READY(&ifp->if_snd); 1394 1395 if_initialize(ifp); 1396 adapter->ipq = if_percpuq_create(&adapter->osdep.ec.ec_if); 1397 ether_ifattach(ifp, adapter->hw.mac.addr); 1398 aprint_normal_dev(dev, "Ethernet address %s\n", 1399 ether_sprintf(adapter->hw.mac.addr)); 1400 /* 1401 * We use per TX queue softint, so if_deferred_start_init() isn't 1402 * used. 1403 */ 1404 ether_set_ifflags_cb(ec, ixgbe_ifflags_cb); 1405 1406 adapter->max_frame_size = ifp->if_mtu + ETHER_HDR_LEN + ETHER_CRC_LEN; 1407 1408 /* 1409 * Tell the upper layer(s) we support long frames. 1410 */ 1411 ifp->if_hdrlen = sizeof(struct ether_vlan_header); 1412 1413 /* Set capability flags */ 1414 ifp->if_capabilities |= IFCAP_RXCSUM 1415 | IFCAP_TXCSUM 1416 | IFCAP_TSOv4 1417 | IFCAP_TSOv6; 1418 ifp->if_capenable = 0; 1419 1420 ec->ec_capabilities |= ETHERCAP_VLAN_HWTAGGING 1421 | ETHERCAP_VLAN_HWCSUM 1422 | ETHERCAP_JUMBO_MTU 1423 | ETHERCAP_VLAN_MTU; 1424 1425 /* Enable the above capabilities by default */ 1426 ec->ec_capenable = ec->ec_capabilities; 1427 1428 /* 1429 * Don't turn this on by default, if vlans are 1430 * created on another pseudo device (eg. lagg) 1431 * then vlan events are not passed thru, breaking 1432 * operation, but with HW FILTER off it works. If 1433 * using vlans directly on the ixgbe driver you can 1434 * enable this and get full hardware tag filtering. 1435 */ 1436 ec->ec_capabilities |= ETHERCAP_VLAN_HWFILTER; 1437 1438 /* 1439 * Specify the media types supported by this adapter and register 1440 * callbacks to update media and link information 1441 */ 1442 ec->ec_ifmedia = &adapter->media; 1443 ifmedia_init_with_lock(&adapter->media, IFM_IMASK, ixgbe_media_change, 1444 ixgbe_media_status, &adapter->core_mtx); 1445 1446 adapter->phy_layer = ixgbe_get_supported_physical_layer(&adapter->hw); 1447 ixgbe_add_media_types(adapter); 1448 1449 /* Set autoselect media by default */ 1450 ifmedia_set(&adapter->media, IFM_ETHER | IFM_AUTO); 1451 1452 if_register(ifp); 1453 1454 return (0); 1455 } /* ixgbe_setup_interface */ 1456 1457 /************************************************************************ 1458 * ixgbe_add_media_types 1459 ************************************************************************/ 1460 static void 1461 ixgbe_add_media_types(struct adapter *adapter) 1462 { 1463 struct ixgbe_hw *hw = &adapter->hw; 1464 u64 layer; 1465 1466 layer = adapter->phy_layer; 1467 1468 #define ADD(mm, dd) \ 1469 ifmedia_add(&adapter->media, IFM_ETHER | (mm), (dd), NULL); 1470 1471 ADD(IFM_NONE, 0); 1472 1473 /* Media types with matching NetBSD media defines */ 1474 if (layer & IXGBE_PHYSICAL_LAYER_10GBASE_T) { 1475 ADD(IFM_10G_T | IFM_FDX, 0); 1476 } 1477 if (layer & IXGBE_PHYSICAL_LAYER_1000BASE_T) { 1478 ADD(IFM_1000_T | IFM_FDX, 0); 1479 } 1480 if (layer & IXGBE_PHYSICAL_LAYER_100BASE_TX) { 1481 ADD(IFM_100_TX | IFM_FDX, 0); 1482 } 1483 if (layer & IXGBE_PHYSICAL_LAYER_10BASE_T) { 1484 ADD(IFM_10_T | IFM_FDX, 0); 1485 } 1486 1487 if (layer & IXGBE_PHYSICAL_LAYER_SFP_PLUS_CU || 1488 layer & IXGBE_PHYSICAL_LAYER_SFP_ACTIVE_DA) { 1489 ADD(IFM_10G_TWINAX | IFM_FDX, 0); 1490 } 1491 1492 if (layer & IXGBE_PHYSICAL_LAYER_10GBASE_LR) { 1493 ADD(IFM_10G_LR | IFM_FDX, 0); 1494 if (hw->phy.multispeed_fiber) { 1495 ADD(IFM_1000_LX | IFM_FDX, 0); 1496 } 1497 } 1498 if (layer & IXGBE_PHYSICAL_LAYER_10GBASE_SR) { 1499 ADD(IFM_10G_SR | IFM_FDX, 0); 1500 if (hw->phy.multispeed_fiber) { 1501 ADD(IFM_1000_SX | IFM_FDX, 0); 1502 } 1503 } else if (layer & IXGBE_PHYSICAL_LAYER_1000BASE_SX) { 1504 ADD(IFM_1000_SX | IFM_FDX, 0); 1505 } 1506 if (layer & IXGBE_PHYSICAL_LAYER_10GBASE_CX4) { 1507 ADD(IFM_10G_CX4 | IFM_FDX, 0); 1508 } 1509 1510 if (layer & IXGBE_PHYSICAL_LAYER_10GBASE_KR) { 1511 ADD(IFM_10G_KR | IFM_FDX, 0); 1512 } 1513 if (layer & IXGBE_PHYSICAL_LAYER_10GBASE_KX4) { 1514 ADD(IFM_10G_KX4 | IFM_FDX, 0); 1515 } 1516 if (layer & IXGBE_PHYSICAL_LAYER_1000BASE_KX) { 1517 ADD(IFM_1000_KX | IFM_FDX, 0); 1518 } 1519 if (layer & IXGBE_PHYSICAL_LAYER_2500BASE_KX) { 1520 ADD(IFM_2500_KX | IFM_FDX, 0); 1521 } 1522 if (layer & IXGBE_PHYSICAL_LAYER_2500BASE_T) { 1523 ADD(IFM_2500_T | IFM_FDX, 0); 1524 } 1525 if (layer & IXGBE_PHYSICAL_LAYER_5GBASE_T) { 1526 ADD(IFM_5000_T | IFM_FDX, 0); 1527 } 1528 if (layer & IXGBE_PHYSICAL_LAYER_1000BASE_BX) 1529 ADD(IFM_1000_BX10 | IFM_FDX, 0); 1530 /* XXX no ifmedia_set? */ 1531 1532 ADD(IFM_AUTO, 0); 1533 1534 #undef ADD 1535 } /* ixgbe_add_media_types */ 1536 1537 /************************************************************************ 1538 * ixgbe_is_sfp 1539 ************************************************************************/ 1540 static inline bool 1541 ixgbe_is_sfp(struct ixgbe_hw *hw) 1542 { 1543 switch (hw->mac.type) { 1544 case ixgbe_mac_82598EB: 1545 if (hw->phy.type == ixgbe_phy_nl) 1546 return (TRUE); 1547 return (FALSE); 1548 case ixgbe_mac_82599EB: 1549 case ixgbe_mac_X550EM_x: 1550 case ixgbe_mac_X550EM_a: 1551 switch (hw->mac.ops.get_media_type(hw)) { 1552 case ixgbe_media_type_fiber: 1553 case ixgbe_media_type_fiber_qsfp: 1554 return (TRUE); 1555 default: 1556 return (FALSE); 1557 } 1558 default: 1559 return (FALSE); 1560 } 1561 } /* ixgbe_is_sfp */ 1562 1563 static void 1564 ixgbe_schedule_admin_tasklet(struct adapter *adapter) 1565 { 1566 1567 KASSERT(mutex_owned(&adapter->admin_mtx)); 1568 1569 if (__predict_true(adapter->osdep.detaching == false)) { 1570 if (adapter->admin_pending == 0) 1571 workqueue_enqueue(adapter->admin_wq, 1572 &adapter->admin_wc, NULL); 1573 adapter->admin_pending = 1; 1574 } 1575 } 1576 1577 /************************************************************************ 1578 * ixgbe_config_link 1579 ************************************************************************/ 1580 static void 1581 ixgbe_config_link(struct adapter *adapter) 1582 { 1583 struct ixgbe_hw *hw = &adapter->hw; 1584 u32 autoneg, err = 0; 1585 u32 task_requests = 0; 1586 bool sfp, negotiate = false; 1587 1588 sfp = ixgbe_is_sfp(hw); 1589 1590 if (sfp) { 1591 if (hw->phy.multispeed_fiber) { 1592 ixgbe_enable_tx_laser(hw); 1593 task_requests |= IXGBE_REQUEST_TASK_MSF_WOI; 1594 } 1595 task_requests |= IXGBE_REQUEST_TASK_MOD_WOI; 1596 1597 mutex_enter(&adapter->admin_mtx); 1598 adapter->task_requests |= task_requests; 1599 ixgbe_schedule_admin_tasklet(adapter); 1600 mutex_exit(&adapter->admin_mtx); 1601 } else { 1602 struct ifmedia *ifm = &adapter->media; 1603 1604 if (hw->mac.ops.check_link) 1605 err = ixgbe_check_link(hw, &adapter->link_speed, 1606 &adapter->link_up, FALSE); 1607 if (err) 1608 return; 1609 1610 /* 1611 * Check if it's the first call. If it's the first call, 1612 * get value for auto negotiation. 1613 */ 1614 autoneg = hw->phy.autoneg_advertised; 1615 if ((IFM_SUBTYPE(ifm->ifm_cur->ifm_media) != IFM_NONE) 1616 && ((!autoneg) && (hw->mac.ops.get_link_capabilities))) 1617 err = hw->mac.ops.get_link_capabilities(hw, &autoneg, 1618 &negotiate); 1619 if (err) 1620 return; 1621 if (hw->mac.ops.setup_link) 1622 err = hw->mac.ops.setup_link(hw, autoneg, 1623 adapter->link_up); 1624 } 1625 1626 } /* ixgbe_config_link */ 1627 1628 /************************************************************************ 1629 * ixgbe_update_stats_counters - Update board statistics counters. 1630 ************************************************************************/ 1631 static void 1632 ixgbe_update_stats_counters(struct adapter *adapter) 1633 { 1634 struct ifnet *ifp = adapter->ifp; 1635 struct ixgbe_hw *hw = &adapter->hw; 1636 struct ixgbe_hw_stats *stats = &adapter->stats.pf; 1637 u32 missed_rx = 0, bprc, lxontxc, lxofftxc; 1638 u64 total, total_missed_rx = 0; 1639 uint64_t crcerrs, illerrc, rlec, ruc, rfc, roc, rjc; 1640 unsigned int queue_counters; 1641 int i; 1642 1643 IXGBE_EVC_REGADD2(hw, stats, IXGBE_CRCERRS, crcerrs); 1644 IXGBE_EVC_REGADD2(hw, stats, IXGBE_ILLERRC, illerrc); 1645 1646 IXGBE_EVC_REGADD(hw, stats, IXGBE_ERRBC, errbc); 1647 IXGBE_EVC_REGADD(hw, stats, IXGBE_MSPDC, mspdc); 1648 if (hw->mac.type >= ixgbe_mac_X550) 1649 IXGBE_EVC_REGADD(hw, stats, IXGBE_MBSDC, mbsdc); 1650 1651 /* 16 registers exist */ 1652 queue_counters = uimin(__arraycount(stats->qprc), adapter->num_queues); 1653 for (i = 0; i < queue_counters; i++) { 1654 IXGBE_EVC_REGADD(hw, stats, IXGBE_QPRC(i), qprc[i]); 1655 IXGBE_EVC_REGADD(hw, stats, IXGBE_QPTC(i), qptc[i]); 1656 if (hw->mac.type >= ixgbe_mac_82599EB) 1657 IXGBE_EVC_REGADD(hw, stats, IXGBE_QPRDC(i), qprdc[i]); 1658 } 1659 1660 /* 8 registers exist */ 1661 for (i = 0; i < IXGBE_TC_COUNTER_NUM; i++) { 1662 uint32_t mp; 1663 1664 /* MPC */ 1665 mp = IXGBE_READ_REG(hw, IXGBE_MPC(i)); 1666 /* global total per queue */ 1667 IXGBE_EVC_ADD(&stats->mpc[i], mp); 1668 /* running comprehensive total for stats display */ 1669 total_missed_rx += mp; 1670 1671 if (hw->mac.type == ixgbe_mac_82598EB) 1672 IXGBE_EVC_REGADD(hw, stats, IXGBE_RNBC(i), rnbc[i]); 1673 1674 IXGBE_EVC_REGADD(hw, stats, IXGBE_PXONTXC(i), pxontxc[i]); 1675 IXGBE_EVC_REGADD(hw, stats, IXGBE_PXOFFTXC(i), pxofftxc[i]); 1676 if (hw->mac.type >= ixgbe_mac_82599EB) { 1677 IXGBE_EVC_REGADD(hw, stats, IXGBE_PXONRXCNT(i), pxonrxc[i]); 1678 IXGBE_EVC_REGADD(hw, stats, IXGBE_PXOFFRXCNT(i), pxoffrxc[i]); 1679 IXGBE_EVC_REGADD(hw, stats, IXGBE_PXON2OFFCNT(i), 1680 pxon2offc[i]); 1681 } else { 1682 IXGBE_EVC_REGADD(hw, stats, IXGBE_PXONRXC(i), pxonrxc[i]); 1683 IXGBE_EVC_REGADD(hw, stats, IXGBE_PXOFFRXC(i), pxoffrxc[i]); 1684 } 1685 } 1686 IXGBE_EVC_ADD(&stats->mpctotal, total_missed_rx); 1687 1688 /* Document says M[LR]FC are valid when link is up and 10Gbps */ 1689 if ((adapter->link_active == LINK_STATE_UP) 1690 && (adapter->link_speed == IXGBE_LINK_SPEED_10GB_FULL)) { 1691 IXGBE_EVC_REGADD(hw, stats, IXGBE_MLFC, mlfc); 1692 IXGBE_EVC_REGADD(hw, stats, IXGBE_MRFC, mrfc); 1693 } 1694 IXGBE_EVC_REGADD2(hw, stats, IXGBE_RLEC, rlec); 1695 1696 /* Hardware workaround, gprc counts missed packets */ 1697 IXGBE_EVC_ADD(&stats->gprc, 1698 IXGBE_READ_REG(hw, IXGBE_GPRC) - missed_rx); 1699 1700 IXGBE_EVC_REGADD2(hw, stats, IXGBE_LXONTXC, lxontxc); 1701 IXGBE_EVC_REGADD2(hw, stats, IXGBE_LXOFFTXC, lxofftxc); 1702 total = lxontxc + lxofftxc; 1703 1704 if (hw->mac.type != ixgbe_mac_82598EB) { 1705 IXGBE_EVC_ADD(&stats->gorc, IXGBE_READ_REG(hw, IXGBE_GORCL) + 1706 ((u64)IXGBE_READ_REG(hw, IXGBE_GORCH) << 32)); 1707 IXGBE_EVC_ADD(&stats->gotc, IXGBE_READ_REG(hw, IXGBE_GOTCL) + 1708 ((u64)IXGBE_READ_REG(hw, IXGBE_GOTCH) << 32) 1709 - total * ETHER_MIN_LEN); 1710 IXGBE_EVC_ADD(&stats->tor, IXGBE_READ_REG(hw, IXGBE_TORL) + 1711 ((u64)IXGBE_READ_REG(hw, IXGBE_TORH) << 32)); 1712 IXGBE_EVC_REGADD(hw, stats, IXGBE_LXONRXCNT, lxonrxc); 1713 IXGBE_EVC_REGADD(hw, stats, IXGBE_LXOFFRXCNT, lxoffrxc); 1714 } else { 1715 IXGBE_EVC_REGADD(hw, stats, IXGBE_LXONRXC, lxonrxc); 1716 IXGBE_EVC_REGADD(hw, stats, IXGBE_LXOFFRXC, lxoffrxc); 1717 /* 82598 only has a counter in the high register */ 1718 IXGBE_EVC_REGADD(hw, stats, IXGBE_GORCH, gorc); 1719 IXGBE_EVC_ADD(&stats->gotc, IXGBE_READ_REG(hw, IXGBE_GOTCH) 1720 - total * ETHER_MIN_LEN); 1721 IXGBE_EVC_REGADD(hw, stats, IXGBE_TORH, tor); 1722 } 1723 1724 /* 1725 * Workaround: mprc hardware is incorrectly counting 1726 * broadcasts, so for now we subtract those. 1727 */ 1728 IXGBE_EVC_REGADD2(hw, stats, IXGBE_BPRC, bprc); 1729 IXGBE_EVC_ADD(&stats->mprc, IXGBE_READ_REG(hw, IXGBE_MPRC) 1730 - ((hw->mac.type == ixgbe_mac_82598EB) ? bprc : 0)); 1731 1732 IXGBE_EVC_REGADD(hw, stats, IXGBE_PRC64, prc64); 1733 IXGBE_EVC_REGADD(hw, stats, IXGBE_PRC127, prc127); 1734 IXGBE_EVC_REGADD(hw, stats, IXGBE_PRC255, prc255); 1735 IXGBE_EVC_REGADD(hw, stats, IXGBE_PRC511, prc511); 1736 IXGBE_EVC_REGADD(hw, stats, IXGBE_PRC1023, prc1023); 1737 IXGBE_EVC_REGADD(hw, stats, IXGBE_PRC1522, prc1522); 1738 1739 IXGBE_EVC_ADD(&stats->gptc, IXGBE_READ_REG(hw, IXGBE_GPTC) - total); 1740 IXGBE_EVC_ADD(&stats->mptc, IXGBE_READ_REG(hw, IXGBE_MPTC) - total); 1741 IXGBE_EVC_ADD(&stats->ptc64, IXGBE_READ_REG(hw, IXGBE_PTC64) - total); 1742 1743 IXGBE_EVC_REGADD2(hw, stats, IXGBE_RUC, ruc); 1744 IXGBE_EVC_REGADD2(hw, stats, IXGBE_RFC, rfc); 1745 IXGBE_EVC_REGADD2(hw, stats, IXGBE_ROC, roc); 1746 IXGBE_EVC_REGADD2(hw, stats, IXGBE_RJC, rjc); 1747 1748 IXGBE_EVC_REGADD(hw, stats, IXGBE_MNGPRC, mngprc); 1749 IXGBE_EVC_REGADD(hw, stats, IXGBE_MNGPDC, mngpdc); 1750 IXGBE_EVC_REGADD(hw, stats, IXGBE_MNGPTC, mngptc); 1751 IXGBE_EVC_REGADD(hw, stats, IXGBE_TPR, tpr); 1752 IXGBE_EVC_REGADD(hw, stats, IXGBE_TPT, tpt); 1753 IXGBE_EVC_REGADD(hw, stats, IXGBE_PTC127, ptc127); 1754 IXGBE_EVC_REGADD(hw, stats, IXGBE_PTC255, ptc255); 1755 IXGBE_EVC_REGADD(hw, stats, IXGBE_PTC511, ptc511); 1756 IXGBE_EVC_REGADD(hw, stats, IXGBE_PTC1023, ptc1023); 1757 IXGBE_EVC_REGADD(hw, stats, IXGBE_PTC1522, ptc1522); 1758 IXGBE_EVC_REGADD(hw, stats, IXGBE_BPTC, bptc); 1759 IXGBE_EVC_REGADD(hw, stats, IXGBE_XEC, xec); 1760 IXGBE_EVC_REGADD(hw, stats, IXGBE_FCCRC, fccrc); 1761 IXGBE_EVC_REGADD(hw, stats, IXGBE_FCLAST, fclast); 1762 /* Only read FCOE on 82599 */ 1763 if (hw->mac.type != ixgbe_mac_82598EB) { 1764 IXGBE_EVC_REGADD(hw, stats, IXGBE_FCOERPDC, fcoerpdc); 1765 IXGBE_EVC_REGADD(hw, stats, IXGBE_FCOEPRC, fcoeprc); 1766 IXGBE_EVC_REGADD(hw, stats, IXGBE_FCOEPTC, fcoeptc); 1767 IXGBE_EVC_REGADD(hw, stats, IXGBE_FCOEDWRC, fcoedwrc); 1768 IXGBE_EVC_REGADD(hw, stats, IXGBE_FCOEDWTC, fcoedwtc); 1769 } 1770 1771 /* 1772 * Fill out the OS statistics structure. Only RX errors are required 1773 * here because all TX counters are incremented in the TX path and 1774 * normal RX counters are prepared in ether_input(). 1775 */ 1776 net_stat_ref_t nsr = IF_STAT_GETREF(ifp); 1777 if_statadd_ref(nsr, if_iqdrops, total_missed_rx); 1778 1779 /* 1780 * Aggregate following types of errors as RX errors: 1781 * - CRC error count, 1782 * - illegal byte error count, 1783 * - length error count, 1784 * - undersized packets count, 1785 * - fragmented packets count, 1786 * - oversized packets count, 1787 * - jabber count. 1788 */ 1789 if_statadd_ref(nsr, if_ierrors, 1790 crcerrs + illerrc + rlec + ruc + rfc + roc + rjc); 1791 1792 IF_STAT_PUTREF(ifp); 1793 } /* ixgbe_update_stats_counters */ 1794 1795 /************************************************************************ 1796 * ixgbe_add_hw_stats 1797 * 1798 * Add sysctl variables, one per statistic, to the system. 1799 ************************************************************************/ 1800 static void 1801 ixgbe_add_hw_stats(struct adapter *adapter) 1802 { 1803 device_t dev = adapter->dev; 1804 const struct sysctlnode *rnode, *cnode; 1805 struct sysctllog **log = &adapter->sysctllog; 1806 struct tx_ring *txr = adapter->tx_rings; 1807 struct rx_ring *rxr = adapter->rx_rings; 1808 struct ixgbe_hw *hw = &adapter->hw; 1809 struct ixgbe_hw_stats *stats = &adapter->stats.pf; 1810 const char *xname = device_xname(dev); 1811 int i; 1812 1813 /* Driver Statistics */ 1814 evcnt_attach_dynamic(&adapter->efbig_tx_dma_setup, EVCNT_TYPE_MISC, 1815 NULL, xname, "Driver tx dma soft fail EFBIG"); 1816 evcnt_attach_dynamic(&adapter->mbuf_defrag_failed, EVCNT_TYPE_MISC, 1817 NULL, xname, "m_defrag() failed"); 1818 evcnt_attach_dynamic(&adapter->efbig2_tx_dma_setup, EVCNT_TYPE_MISC, 1819 NULL, xname, "Driver tx dma hard fail EFBIG"); 1820 evcnt_attach_dynamic(&adapter->einval_tx_dma_setup, EVCNT_TYPE_MISC, 1821 NULL, xname, "Driver tx dma hard fail EINVAL"); 1822 evcnt_attach_dynamic(&adapter->other_tx_dma_setup, EVCNT_TYPE_MISC, 1823 NULL, xname, "Driver tx dma hard fail other"); 1824 evcnt_attach_dynamic(&adapter->eagain_tx_dma_setup, EVCNT_TYPE_MISC, 1825 NULL, xname, "Driver tx dma soft fail EAGAIN"); 1826 evcnt_attach_dynamic(&adapter->enomem_tx_dma_setup, EVCNT_TYPE_MISC, 1827 NULL, xname, "Driver tx dma soft fail ENOMEM"); 1828 evcnt_attach_dynamic(&adapter->watchdog_events, EVCNT_TYPE_MISC, 1829 NULL, xname, "Watchdog timeouts"); 1830 evcnt_attach_dynamic(&adapter->tso_err, EVCNT_TYPE_MISC, 1831 NULL, xname, "TSO errors"); 1832 evcnt_attach_dynamic(&adapter->admin_irqev, EVCNT_TYPE_INTR, 1833 NULL, xname, "Admin MSI-X IRQ Handled"); 1834 evcnt_attach_dynamic(&adapter->link_workev, EVCNT_TYPE_INTR, 1835 NULL, xname, "Link event"); 1836 evcnt_attach_dynamic(&adapter->mod_workev, EVCNT_TYPE_INTR, 1837 NULL, xname, "SFP+ module event"); 1838 evcnt_attach_dynamic(&adapter->msf_workev, EVCNT_TYPE_INTR, 1839 NULL, xname, "Multispeed event"); 1840 evcnt_attach_dynamic(&adapter->phy_workev, EVCNT_TYPE_INTR, 1841 NULL, xname, "External PHY event"); 1842 1843 /* Max number of traffic class is 8 */ 1844 KASSERT(IXGBE_DCB_MAX_TRAFFIC_CLASS == 8); 1845 for (i = 0; i < IXGBE_TC_COUNTER_NUM; i++) { 1846 snprintf(adapter->tcs[i].evnamebuf, 1847 sizeof(adapter->tcs[i].evnamebuf), "%s tc%d", 1848 xname, i); 1849 if (i < __arraycount(stats->mpc)) { 1850 evcnt_attach_dynamic(&stats->mpc[i], 1851 EVCNT_TYPE_MISC, NULL, adapter->tcs[i].evnamebuf, 1852 "RX Missed Packet Count"); 1853 if (hw->mac.type == ixgbe_mac_82598EB) 1854 evcnt_attach_dynamic(&stats->rnbc[i], 1855 EVCNT_TYPE_MISC, NULL, 1856 adapter->tcs[i].evnamebuf, 1857 "Receive No Buffers"); 1858 } 1859 if (i < __arraycount(stats->pxontxc)) { 1860 evcnt_attach_dynamic(&stats->pxontxc[i], 1861 EVCNT_TYPE_MISC, NULL, adapter->tcs[i].evnamebuf, 1862 "pxontxc"); 1863 evcnt_attach_dynamic(&stats->pxonrxc[i], 1864 EVCNT_TYPE_MISC, NULL, adapter->tcs[i].evnamebuf, 1865 "pxonrxc"); 1866 evcnt_attach_dynamic(&stats->pxofftxc[i], 1867 EVCNT_TYPE_MISC, NULL, adapter->tcs[i].evnamebuf, 1868 "pxofftxc"); 1869 evcnt_attach_dynamic(&stats->pxoffrxc[i], 1870 EVCNT_TYPE_MISC, NULL, adapter->tcs[i].evnamebuf, 1871 "pxoffrxc"); 1872 if (hw->mac.type >= ixgbe_mac_82599EB) 1873 evcnt_attach_dynamic(&stats->pxon2offc[i], 1874 EVCNT_TYPE_MISC, NULL, 1875 adapter->tcs[i].evnamebuf, 1876 "pxon2offc"); 1877 } 1878 } 1879 1880 for (i = 0; i < adapter->num_queues; i++, rxr++, txr++) { 1881 #ifdef LRO 1882 struct lro_ctrl *lro = &rxr->lro; 1883 #endif /* LRO */ 1884 1885 snprintf(adapter->queues[i].evnamebuf, 1886 sizeof(adapter->queues[i].evnamebuf), "%s q%d", 1887 xname, i); 1888 snprintf(adapter->queues[i].namebuf, 1889 sizeof(adapter->queues[i].namebuf), "q%d", i); 1890 1891 if ((rnode = ixgbe_sysctl_instance(adapter)) == NULL) { 1892 aprint_error_dev(dev, "could not create sysctl root\n"); 1893 break; 1894 } 1895 1896 if (sysctl_createv(log, 0, &rnode, &rnode, 1897 0, CTLTYPE_NODE, 1898 adapter->queues[i].namebuf, SYSCTL_DESCR("Queue Name"), 1899 NULL, 0, NULL, 0, CTL_CREATE, CTL_EOL) != 0) 1900 break; 1901 1902 if (sysctl_createv(log, 0, &rnode, &cnode, 1903 CTLFLAG_READWRITE, CTLTYPE_INT, 1904 "interrupt_rate", SYSCTL_DESCR("Interrupt Rate"), 1905 ixgbe_sysctl_interrupt_rate_handler, 0, 1906 (void *)&adapter->queues[i], 0, CTL_CREATE, CTL_EOL) != 0) 1907 break; 1908 1909 if (sysctl_createv(log, 0, &rnode, &cnode, 1910 CTLFLAG_READONLY, CTLTYPE_INT, 1911 "txd_head", SYSCTL_DESCR("Transmit Descriptor Head"), 1912 ixgbe_sysctl_tdh_handler, 0, (void *)txr, 1913 0, CTL_CREATE, CTL_EOL) != 0) 1914 break; 1915 1916 if (sysctl_createv(log, 0, &rnode, &cnode, 1917 CTLFLAG_READONLY, CTLTYPE_INT, 1918 "txd_tail", SYSCTL_DESCR("Transmit Descriptor Tail"), 1919 ixgbe_sysctl_tdt_handler, 0, (void *)txr, 1920 0, CTL_CREATE, CTL_EOL) != 0) 1921 break; 1922 1923 evcnt_attach_dynamic(&adapter->queues[i].irqs, EVCNT_TYPE_INTR, 1924 NULL, adapter->queues[i].evnamebuf, "IRQs on queue"); 1925 evcnt_attach_dynamic(&adapter->queues[i].handleq, 1926 EVCNT_TYPE_MISC, NULL, adapter->queues[i].evnamebuf, 1927 "Handled queue in softint"); 1928 evcnt_attach_dynamic(&adapter->queues[i].req, EVCNT_TYPE_MISC, 1929 NULL, adapter->queues[i].evnamebuf, "Requeued in softint"); 1930 evcnt_attach_dynamic(&txr->tso_tx, EVCNT_TYPE_MISC, 1931 NULL, adapter->queues[i].evnamebuf, "TSO"); 1932 evcnt_attach_dynamic(&txr->no_desc_avail, EVCNT_TYPE_MISC, 1933 NULL, adapter->queues[i].evnamebuf, 1934 "TX Queue No Descriptor Available"); 1935 evcnt_attach_dynamic(&txr->total_packets, EVCNT_TYPE_MISC, 1936 NULL, adapter->queues[i].evnamebuf, 1937 "Queue Packets Transmitted"); 1938 #ifndef IXGBE_LEGACY_TX 1939 evcnt_attach_dynamic(&txr->pcq_drops, EVCNT_TYPE_MISC, 1940 NULL, adapter->queues[i].evnamebuf, 1941 "Packets dropped in pcq"); 1942 #endif 1943 1944 if (sysctl_createv(log, 0, &rnode, &cnode, 1945 CTLFLAG_READONLY, CTLTYPE_INT, "rxd_nxck", 1946 SYSCTL_DESCR("Receive Descriptor next to check"), 1947 ixgbe_sysctl_next_to_check_handler, 0, (void *)rxr, 0, 1948 CTL_CREATE, CTL_EOL) != 0) 1949 break; 1950 1951 if (sysctl_createv(log, 0, &rnode, &cnode, 1952 CTLFLAG_READONLY, CTLTYPE_INT, "rxd_nxrf", 1953 SYSCTL_DESCR("Receive Descriptor next to refresh"), 1954 ixgbe_sysctl_next_to_refresh_handler, 0, (void *)rxr, 0, 1955 CTL_CREATE, CTL_EOL) != 0) 1956 break; 1957 1958 if (sysctl_createv(log, 0, &rnode, &cnode, 1959 CTLFLAG_READONLY, CTLTYPE_INT, "rxd_head", 1960 SYSCTL_DESCR("Receive Descriptor Head"), 1961 ixgbe_sysctl_rdh_handler, 0, (void *)rxr, 0, 1962 CTL_CREATE, CTL_EOL) != 0) 1963 break; 1964 1965 if (sysctl_createv(log, 0, &rnode, &cnode, 1966 CTLFLAG_READONLY, CTLTYPE_INT, "rxd_tail", 1967 SYSCTL_DESCR("Receive Descriptor Tail"), 1968 ixgbe_sysctl_rdt_handler, 0, (void *)rxr, 0, 1969 CTL_CREATE, CTL_EOL) != 0) 1970 break; 1971 1972 if (i < __arraycount(stats->qprc)) { 1973 evcnt_attach_dynamic(&stats->qprc[i], EVCNT_TYPE_MISC, 1974 NULL, adapter->queues[i].evnamebuf, "qprc"); 1975 evcnt_attach_dynamic(&stats->qptc[i], EVCNT_TYPE_MISC, 1976 NULL, adapter->queues[i].evnamebuf, "qptc"); 1977 evcnt_attach_dynamic(&stats->qbrc[i], EVCNT_TYPE_MISC, 1978 NULL, adapter->queues[i].evnamebuf, "qbrc"); 1979 evcnt_attach_dynamic(&stats->qbtc[i], EVCNT_TYPE_MISC, 1980 NULL, adapter->queues[i].evnamebuf, "qbtc"); 1981 if (hw->mac.type >= ixgbe_mac_82599EB) 1982 evcnt_attach_dynamic(&stats->qprdc[i], 1983 EVCNT_TYPE_MISC, NULL, 1984 adapter->queues[i].evnamebuf, "qprdc"); 1985 } 1986 1987 evcnt_attach_dynamic(&rxr->rx_packets, EVCNT_TYPE_MISC, 1988 NULL, adapter->queues[i].evnamebuf, 1989 "Queue Packets Received"); 1990 evcnt_attach_dynamic(&rxr->rx_bytes, EVCNT_TYPE_MISC, 1991 NULL, adapter->queues[i].evnamebuf, 1992 "Queue Bytes Received"); 1993 evcnt_attach_dynamic(&rxr->rx_copies, EVCNT_TYPE_MISC, 1994 NULL, adapter->queues[i].evnamebuf, "Copied RX Frames"); 1995 evcnt_attach_dynamic(&rxr->no_mbuf, EVCNT_TYPE_MISC, 1996 NULL, adapter->queues[i].evnamebuf, "Rx no mbuf"); 1997 evcnt_attach_dynamic(&rxr->rx_discarded, EVCNT_TYPE_MISC, 1998 NULL, adapter->queues[i].evnamebuf, "Rx discarded"); 1999 #ifdef LRO 2000 SYSCTL_ADD_INT(ctx, queue_list, OID_AUTO, "lro_queued", 2001 CTLFLAG_RD, &lro->lro_queued, 0, 2002 "LRO Queued"); 2003 SYSCTL_ADD_INT(ctx, queue_list, OID_AUTO, "lro_flushed", 2004 CTLFLAG_RD, &lro->lro_flushed, 0, 2005 "LRO Flushed"); 2006 #endif /* LRO */ 2007 } 2008 2009 /* MAC stats get their own sub node */ 2010 2011 snprintf(stats->namebuf, 2012 sizeof(stats->namebuf), "%s MAC Statistics", xname); 2013 2014 evcnt_attach_dynamic(&stats->ipcs, EVCNT_TYPE_MISC, NULL, 2015 stats->namebuf, "rx csum offload - IP"); 2016 evcnt_attach_dynamic(&stats->l4cs, EVCNT_TYPE_MISC, NULL, 2017 stats->namebuf, "rx csum offload - L4"); 2018 evcnt_attach_dynamic(&stats->ipcs_bad, EVCNT_TYPE_MISC, NULL, 2019 stats->namebuf, "rx csum offload - IP bad"); 2020 evcnt_attach_dynamic(&stats->l4cs_bad, EVCNT_TYPE_MISC, NULL, 2021 stats->namebuf, "rx csum offload - L4 bad"); 2022 evcnt_attach_dynamic(&stats->intzero, EVCNT_TYPE_MISC, NULL, 2023 stats->namebuf, "Interrupt conditions zero"); 2024 evcnt_attach_dynamic(&stats->legint, EVCNT_TYPE_MISC, NULL, 2025 stats->namebuf, "Legacy interrupts"); 2026 2027 evcnt_attach_dynamic(&stats->crcerrs, EVCNT_TYPE_MISC, NULL, 2028 stats->namebuf, "CRC Errors"); 2029 evcnt_attach_dynamic(&stats->illerrc, EVCNT_TYPE_MISC, NULL, 2030 stats->namebuf, "Illegal Byte Errors"); 2031 evcnt_attach_dynamic(&stats->errbc, EVCNT_TYPE_MISC, NULL, 2032 stats->namebuf, "Byte Errors"); 2033 evcnt_attach_dynamic(&stats->mspdc, EVCNT_TYPE_MISC, NULL, 2034 stats->namebuf, "MAC Short Packets Discarded"); 2035 if (hw->mac.type >= ixgbe_mac_X550) 2036 evcnt_attach_dynamic(&stats->mbsdc, EVCNT_TYPE_MISC, NULL, 2037 stats->namebuf, "Bad SFD"); 2038 evcnt_attach_dynamic(&stats->mpctotal, EVCNT_TYPE_MISC, NULL, 2039 stats->namebuf, "Total Packets Missed"); 2040 evcnt_attach_dynamic(&stats->mlfc, EVCNT_TYPE_MISC, NULL, 2041 stats->namebuf, "MAC Local Faults"); 2042 evcnt_attach_dynamic(&stats->mrfc, EVCNT_TYPE_MISC, NULL, 2043 stats->namebuf, "MAC Remote Faults"); 2044 evcnt_attach_dynamic(&stats->rlec, EVCNT_TYPE_MISC, NULL, 2045 stats->namebuf, "Receive Length Errors"); 2046 evcnt_attach_dynamic(&stats->lxontxc, EVCNT_TYPE_MISC, NULL, 2047 stats->namebuf, "Link XON Transmitted"); 2048 evcnt_attach_dynamic(&stats->lxonrxc, EVCNT_TYPE_MISC, NULL, 2049 stats->namebuf, "Link XON Received"); 2050 evcnt_attach_dynamic(&stats->lxofftxc, EVCNT_TYPE_MISC, NULL, 2051 stats->namebuf, "Link XOFF Transmitted"); 2052 evcnt_attach_dynamic(&stats->lxoffrxc, EVCNT_TYPE_MISC, NULL, 2053 stats->namebuf, "Link XOFF Received"); 2054 2055 /* Packet Reception Stats */ 2056 evcnt_attach_dynamic(&stats->tor, EVCNT_TYPE_MISC, NULL, 2057 stats->namebuf, "Total Octets Received"); 2058 evcnt_attach_dynamic(&stats->gorc, EVCNT_TYPE_MISC, NULL, 2059 stats->namebuf, "Good Octets Received"); 2060 evcnt_attach_dynamic(&stats->tpr, EVCNT_TYPE_MISC, NULL, 2061 stats->namebuf, "Total Packets Received"); 2062 evcnt_attach_dynamic(&stats->gprc, EVCNT_TYPE_MISC, NULL, 2063 stats->namebuf, "Good Packets Received"); 2064 evcnt_attach_dynamic(&stats->mprc, EVCNT_TYPE_MISC, NULL, 2065 stats->namebuf, "Multicast Packets Received"); 2066 evcnt_attach_dynamic(&stats->bprc, EVCNT_TYPE_MISC, NULL, 2067 stats->namebuf, "Broadcast Packets Received"); 2068 evcnt_attach_dynamic(&stats->prc64, EVCNT_TYPE_MISC, NULL, 2069 stats->namebuf, "64 byte frames received "); 2070 evcnt_attach_dynamic(&stats->prc127, EVCNT_TYPE_MISC, NULL, 2071 stats->namebuf, "65-127 byte frames received"); 2072 evcnt_attach_dynamic(&stats->prc255, EVCNT_TYPE_MISC, NULL, 2073 stats->namebuf, "128-255 byte frames received"); 2074 evcnt_attach_dynamic(&stats->prc511, EVCNT_TYPE_MISC, NULL, 2075 stats->namebuf, "256-511 byte frames received"); 2076 evcnt_attach_dynamic(&stats->prc1023, EVCNT_TYPE_MISC, NULL, 2077 stats->namebuf, "512-1023 byte frames received"); 2078 evcnt_attach_dynamic(&stats->prc1522, EVCNT_TYPE_MISC, NULL, 2079 stats->namebuf, "1023-1522 byte frames received"); 2080 evcnt_attach_dynamic(&stats->ruc, EVCNT_TYPE_MISC, NULL, 2081 stats->namebuf, "Receive Undersized"); 2082 evcnt_attach_dynamic(&stats->rfc, EVCNT_TYPE_MISC, NULL, 2083 stats->namebuf, "Fragmented Packets Received "); 2084 evcnt_attach_dynamic(&stats->roc, EVCNT_TYPE_MISC, NULL, 2085 stats->namebuf, "Oversized Packets Received"); 2086 evcnt_attach_dynamic(&stats->rjc, EVCNT_TYPE_MISC, NULL, 2087 stats->namebuf, "Received Jabber"); 2088 evcnt_attach_dynamic(&stats->mngprc, EVCNT_TYPE_MISC, NULL, 2089 stats->namebuf, "Management Packets Received"); 2090 evcnt_attach_dynamic(&stats->mngpdc, EVCNT_TYPE_MISC, NULL, 2091 stats->namebuf, "Management Packets Dropped"); 2092 evcnt_attach_dynamic(&stats->xec, EVCNT_TYPE_MISC, NULL, 2093 stats->namebuf, "Checksum Errors"); 2094 2095 /* Packet Transmission Stats */ 2096 evcnt_attach_dynamic(&stats->gotc, EVCNT_TYPE_MISC, NULL, 2097 stats->namebuf, "Good Octets Transmitted"); 2098 evcnt_attach_dynamic(&stats->tpt, EVCNT_TYPE_MISC, NULL, 2099 stats->namebuf, "Total Packets Transmitted"); 2100 evcnt_attach_dynamic(&stats->gptc, EVCNT_TYPE_MISC, NULL, 2101 stats->namebuf, "Good Packets Transmitted"); 2102 evcnt_attach_dynamic(&stats->bptc, EVCNT_TYPE_MISC, NULL, 2103 stats->namebuf, "Broadcast Packets Transmitted"); 2104 evcnt_attach_dynamic(&stats->mptc, EVCNT_TYPE_MISC, NULL, 2105 stats->namebuf, "Multicast Packets Transmitted"); 2106 evcnt_attach_dynamic(&stats->mngptc, EVCNT_TYPE_MISC, NULL, 2107 stats->namebuf, "Management Packets Transmitted"); 2108 evcnt_attach_dynamic(&stats->ptc64, EVCNT_TYPE_MISC, NULL, 2109 stats->namebuf, "64 byte frames transmitted "); 2110 evcnt_attach_dynamic(&stats->ptc127, EVCNT_TYPE_MISC, NULL, 2111 stats->namebuf, "65-127 byte frames transmitted"); 2112 evcnt_attach_dynamic(&stats->ptc255, EVCNT_TYPE_MISC, NULL, 2113 stats->namebuf, "128-255 byte frames transmitted"); 2114 evcnt_attach_dynamic(&stats->ptc511, EVCNT_TYPE_MISC, NULL, 2115 stats->namebuf, "256-511 byte frames transmitted"); 2116 evcnt_attach_dynamic(&stats->ptc1023, EVCNT_TYPE_MISC, NULL, 2117 stats->namebuf, "512-1023 byte frames transmitted"); 2118 evcnt_attach_dynamic(&stats->ptc1522, EVCNT_TYPE_MISC, NULL, 2119 stats->namebuf, "1024-1522 byte frames transmitted"); 2120 } /* ixgbe_add_hw_stats */ 2121 2122 static void 2123 ixgbe_clear_evcnt(struct adapter *adapter) 2124 { 2125 struct tx_ring *txr = adapter->tx_rings; 2126 struct rx_ring *rxr = adapter->rx_rings; 2127 struct ixgbe_hw *hw = &adapter->hw; 2128 struct ixgbe_hw_stats *stats = &adapter->stats.pf; 2129 int i; 2130 2131 IXGBE_EVC_STORE(&adapter->efbig_tx_dma_setup, 0); 2132 IXGBE_EVC_STORE(&adapter->mbuf_defrag_failed, 0); 2133 IXGBE_EVC_STORE(&adapter->efbig2_tx_dma_setup, 0); 2134 IXGBE_EVC_STORE(&adapter->einval_tx_dma_setup, 0); 2135 IXGBE_EVC_STORE(&adapter->other_tx_dma_setup, 0); 2136 IXGBE_EVC_STORE(&adapter->eagain_tx_dma_setup, 0); 2137 IXGBE_EVC_STORE(&adapter->enomem_tx_dma_setup, 0); 2138 IXGBE_EVC_STORE(&adapter->tso_err, 0); 2139 IXGBE_EVC_STORE(&adapter->watchdog_events, 0); 2140 IXGBE_EVC_STORE(&adapter->admin_irqev, 0); 2141 IXGBE_EVC_STORE(&adapter->link_workev, 0); 2142 IXGBE_EVC_STORE(&adapter->mod_workev, 0); 2143 IXGBE_EVC_STORE(&adapter->msf_workev, 0); 2144 IXGBE_EVC_STORE(&adapter->phy_workev, 0); 2145 2146 for (i = 0; i < IXGBE_TC_COUNTER_NUM; i++) { 2147 if (i < __arraycount(stats->mpc)) { 2148 IXGBE_EVC_STORE(&stats->mpc[i], 0); 2149 if (hw->mac.type == ixgbe_mac_82598EB) 2150 IXGBE_EVC_STORE(&stats->rnbc[i], 0); 2151 } 2152 if (i < __arraycount(stats->pxontxc)) { 2153 IXGBE_EVC_STORE(&stats->pxontxc[i], 0); 2154 IXGBE_EVC_STORE(&stats->pxonrxc[i], 0); 2155 IXGBE_EVC_STORE(&stats->pxofftxc[i], 0); 2156 IXGBE_EVC_STORE(&stats->pxoffrxc[i], 0); 2157 if (hw->mac.type >= ixgbe_mac_82599EB) 2158 IXGBE_EVC_STORE(&stats->pxon2offc[i], 0); 2159 } 2160 } 2161 2162 txr = adapter->tx_rings; 2163 for (i = 0; i < adapter->num_queues; i++, rxr++, txr++) { 2164 IXGBE_EVC_STORE(&adapter->queues[i].irqs, 0); 2165 IXGBE_EVC_STORE(&adapter->queues[i].handleq, 0); 2166 IXGBE_EVC_STORE(&adapter->queues[i].req, 0); 2167 IXGBE_EVC_STORE(&txr->no_desc_avail, 0); 2168 IXGBE_EVC_STORE(&txr->total_packets, 0); 2169 IXGBE_EVC_STORE(&txr->tso_tx, 0); 2170 #ifndef IXGBE_LEGACY_TX 2171 IXGBE_EVC_STORE(&txr->pcq_drops, 0); 2172 #endif 2173 txr->q_efbig_tx_dma_setup = 0; 2174 txr->q_mbuf_defrag_failed = 0; 2175 txr->q_efbig2_tx_dma_setup = 0; 2176 txr->q_einval_tx_dma_setup = 0; 2177 txr->q_other_tx_dma_setup = 0; 2178 txr->q_eagain_tx_dma_setup = 0; 2179 txr->q_enomem_tx_dma_setup = 0; 2180 txr->q_tso_err = 0; 2181 2182 if (i < __arraycount(stats->qprc)) { 2183 IXGBE_EVC_STORE(&stats->qprc[i], 0); 2184 IXGBE_EVC_STORE(&stats->qptc[i], 0); 2185 IXGBE_EVC_STORE(&stats->qbrc[i], 0); 2186 IXGBE_EVC_STORE(&stats->qbtc[i], 0); 2187 if (hw->mac.type >= ixgbe_mac_82599EB) 2188 IXGBE_EVC_STORE(&stats->qprdc[i], 0); 2189 } 2190 2191 IXGBE_EVC_STORE(&rxr->rx_packets, 0); 2192 IXGBE_EVC_STORE(&rxr->rx_bytes, 0); 2193 IXGBE_EVC_STORE(&rxr->rx_copies, 0); 2194 IXGBE_EVC_STORE(&rxr->no_mbuf, 0); 2195 IXGBE_EVC_STORE(&rxr->rx_discarded, 0); 2196 } 2197 IXGBE_EVC_STORE(&stats->ipcs, 0); 2198 IXGBE_EVC_STORE(&stats->l4cs, 0); 2199 IXGBE_EVC_STORE(&stats->ipcs_bad, 0); 2200 IXGBE_EVC_STORE(&stats->l4cs_bad, 0); 2201 IXGBE_EVC_STORE(&stats->intzero, 0); 2202 IXGBE_EVC_STORE(&stats->legint, 0); 2203 IXGBE_EVC_STORE(&stats->crcerrs, 0); 2204 IXGBE_EVC_STORE(&stats->illerrc, 0); 2205 IXGBE_EVC_STORE(&stats->errbc, 0); 2206 IXGBE_EVC_STORE(&stats->mspdc, 0); 2207 if (hw->mac.type >= ixgbe_mac_X550) 2208 IXGBE_EVC_STORE(&stats->mbsdc, 0); 2209 IXGBE_EVC_STORE(&stats->mpctotal, 0); 2210 IXGBE_EVC_STORE(&stats->mlfc, 0); 2211 IXGBE_EVC_STORE(&stats->mrfc, 0); 2212 IXGBE_EVC_STORE(&stats->rlec, 0); 2213 IXGBE_EVC_STORE(&stats->lxontxc, 0); 2214 IXGBE_EVC_STORE(&stats->lxonrxc, 0); 2215 IXGBE_EVC_STORE(&stats->lxofftxc, 0); 2216 IXGBE_EVC_STORE(&stats->lxoffrxc, 0); 2217 2218 /* Packet Reception Stats */ 2219 IXGBE_EVC_STORE(&stats->tor, 0); 2220 IXGBE_EVC_STORE(&stats->gorc, 0); 2221 IXGBE_EVC_STORE(&stats->tpr, 0); 2222 IXGBE_EVC_STORE(&stats->gprc, 0); 2223 IXGBE_EVC_STORE(&stats->mprc, 0); 2224 IXGBE_EVC_STORE(&stats->bprc, 0); 2225 IXGBE_EVC_STORE(&stats->prc64, 0); 2226 IXGBE_EVC_STORE(&stats->prc127, 0); 2227 IXGBE_EVC_STORE(&stats->prc255, 0); 2228 IXGBE_EVC_STORE(&stats->prc511, 0); 2229 IXGBE_EVC_STORE(&stats->prc1023, 0); 2230 IXGBE_EVC_STORE(&stats->prc1522, 0); 2231 IXGBE_EVC_STORE(&stats->ruc, 0); 2232 IXGBE_EVC_STORE(&stats->rfc, 0); 2233 IXGBE_EVC_STORE(&stats->roc, 0); 2234 IXGBE_EVC_STORE(&stats->rjc, 0); 2235 IXGBE_EVC_STORE(&stats->mngprc, 0); 2236 IXGBE_EVC_STORE(&stats->mngpdc, 0); 2237 IXGBE_EVC_STORE(&stats->xec, 0); 2238 2239 /* Packet Transmission Stats */ 2240 IXGBE_EVC_STORE(&stats->gotc, 0); 2241 IXGBE_EVC_STORE(&stats->tpt, 0); 2242 IXGBE_EVC_STORE(&stats->gptc, 0); 2243 IXGBE_EVC_STORE(&stats->bptc, 0); 2244 IXGBE_EVC_STORE(&stats->mptc, 0); 2245 IXGBE_EVC_STORE(&stats->mngptc, 0); 2246 IXGBE_EVC_STORE(&stats->ptc64, 0); 2247 IXGBE_EVC_STORE(&stats->ptc127, 0); 2248 IXGBE_EVC_STORE(&stats->ptc255, 0); 2249 IXGBE_EVC_STORE(&stats->ptc511, 0); 2250 IXGBE_EVC_STORE(&stats->ptc1023, 0); 2251 IXGBE_EVC_STORE(&stats->ptc1522, 0); 2252 } 2253 2254 /************************************************************************ 2255 * ixgbe_sysctl_tdh_handler - Transmit Descriptor Head handler function 2256 * 2257 * Retrieves the TDH value from the hardware 2258 ************************************************************************/ 2259 static int 2260 ixgbe_sysctl_tdh_handler(SYSCTLFN_ARGS) 2261 { 2262 struct sysctlnode node = *rnode; 2263 struct tx_ring *txr = (struct tx_ring *)node.sysctl_data; 2264 struct adapter *adapter; 2265 uint32_t val; 2266 2267 if (!txr) 2268 return (0); 2269 2270 adapter = txr->adapter; 2271 if (ixgbe_fw_recovery_mode_swflag(adapter)) 2272 return (EPERM); 2273 2274 val = IXGBE_READ_REG(&adapter->hw, IXGBE_TDH(txr->me)); 2275 node.sysctl_data = &val; 2276 return sysctl_lookup(SYSCTLFN_CALL(&node)); 2277 } /* ixgbe_sysctl_tdh_handler */ 2278 2279 /************************************************************************ 2280 * ixgbe_sysctl_tdt_handler - Transmit Descriptor Tail handler function 2281 * 2282 * Retrieves the TDT value from the hardware 2283 ************************************************************************/ 2284 static int 2285 ixgbe_sysctl_tdt_handler(SYSCTLFN_ARGS) 2286 { 2287 struct sysctlnode node = *rnode; 2288 struct tx_ring *txr = (struct tx_ring *)node.sysctl_data; 2289 struct adapter *adapter; 2290 uint32_t val; 2291 2292 if (!txr) 2293 return (0); 2294 2295 adapter = txr->adapter; 2296 if (ixgbe_fw_recovery_mode_swflag(adapter)) 2297 return (EPERM); 2298 2299 val = IXGBE_READ_REG(&adapter->hw, IXGBE_TDT(txr->me)); 2300 node.sysctl_data = &val; 2301 return sysctl_lookup(SYSCTLFN_CALL(&node)); 2302 } /* ixgbe_sysctl_tdt_handler */ 2303 2304 /************************************************************************ 2305 * ixgbe_sysctl_next_to_check_handler - Receive Descriptor next to check 2306 * handler function 2307 * 2308 * Retrieves the next_to_check value 2309 ************************************************************************/ 2310 static int 2311 ixgbe_sysctl_next_to_check_handler(SYSCTLFN_ARGS) 2312 { 2313 struct sysctlnode node = *rnode; 2314 struct rx_ring *rxr = (struct rx_ring *)node.sysctl_data; 2315 struct adapter *adapter; 2316 uint32_t val; 2317 2318 if (!rxr) 2319 return (0); 2320 2321 adapter = rxr->adapter; 2322 if (ixgbe_fw_recovery_mode_swflag(adapter)) 2323 return (EPERM); 2324 2325 val = rxr->next_to_check; 2326 node.sysctl_data = &val; 2327 return sysctl_lookup(SYSCTLFN_CALL(&node)); 2328 } /* ixgbe_sysctl_next_to_check_handler */ 2329 2330 /************************************************************************ 2331 * ixgbe_sysctl_next_to_refresh_handler - Receive Descriptor next to check 2332 * handler function 2333 * 2334 * Retrieves the next_to_refresh value 2335 ************************************************************************/ 2336 static int 2337 ixgbe_sysctl_next_to_refresh_handler(SYSCTLFN_ARGS) 2338 { 2339 struct sysctlnode node = *rnode; 2340 struct rx_ring *rxr = (struct rx_ring *)node.sysctl_data; 2341 struct adapter *adapter; 2342 uint32_t val; 2343 2344 if (!rxr) 2345 return (0); 2346 2347 adapter = rxr->adapter; 2348 if (ixgbe_fw_recovery_mode_swflag(adapter)) 2349 return (EPERM); 2350 2351 val = rxr->next_to_refresh; 2352 node.sysctl_data = &val; 2353 return sysctl_lookup(SYSCTLFN_CALL(&node)); 2354 } /* ixgbe_sysctl_next_to_refresh_handler */ 2355 2356 /************************************************************************ 2357 * ixgbe_sysctl_rdh_handler - Receive Descriptor Head handler function 2358 * 2359 * Retrieves the RDH value from the hardware 2360 ************************************************************************/ 2361 static int 2362 ixgbe_sysctl_rdh_handler(SYSCTLFN_ARGS) 2363 { 2364 struct sysctlnode node = *rnode; 2365 struct rx_ring *rxr = (struct rx_ring *)node.sysctl_data; 2366 struct adapter *adapter; 2367 uint32_t val; 2368 2369 if (!rxr) 2370 return (0); 2371 2372 adapter = rxr->adapter; 2373 if (ixgbe_fw_recovery_mode_swflag(adapter)) 2374 return (EPERM); 2375 2376 val = IXGBE_READ_REG(&adapter->hw, IXGBE_RDH(rxr->me)); 2377 node.sysctl_data = &val; 2378 return sysctl_lookup(SYSCTLFN_CALL(&node)); 2379 } /* ixgbe_sysctl_rdh_handler */ 2380 2381 /************************************************************************ 2382 * ixgbe_sysctl_rdt_handler - Receive Descriptor Tail handler function 2383 * 2384 * Retrieves the RDT value from the hardware 2385 ************************************************************************/ 2386 static int 2387 ixgbe_sysctl_rdt_handler(SYSCTLFN_ARGS) 2388 { 2389 struct sysctlnode node = *rnode; 2390 struct rx_ring *rxr = (struct rx_ring *)node.sysctl_data; 2391 struct adapter *adapter; 2392 uint32_t val; 2393 2394 if (!rxr) 2395 return (0); 2396 2397 adapter = rxr->adapter; 2398 if (ixgbe_fw_recovery_mode_swflag(adapter)) 2399 return (EPERM); 2400 2401 val = IXGBE_READ_REG(&adapter->hw, IXGBE_RDT(rxr->me)); 2402 node.sysctl_data = &val; 2403 return sysctl_lookup(SYSCTLFN_CALL(&node)); 2404 } /* ixgbe_sysctl_rdt_handler */ 2405 2406 static int 2407 ixgbe_vlan_cb(struct ethercom *ec, uint16_t vid, bool set) 2408 { 2409 struct ifnet *ifp = &ec->ec_if; 2410 struct adapter *adapter = ifp->if_softc; 2411 int rv; 2412 2413 if (set) 2414 rv = ixgbe_register_vlan(adapter, vid); 2415 else 2416 rv = ixgbe_unregister_vlan(adapter, vid); 2417 2418 if (rv != 0) 2419 return rv; 2420 2421 /* 2422 * Control VLAN HW tagging when ec_nvlan is changed from 1 to 0 2423 * or 0 to 1. 2424 */ 2425 if ((set && (ec->ec_nvlans == 1)) || (!set && (ec->ec_nvlans == 0))) 2426 ixgbe_setup_vlan_hw_tagging(adapter); 2427 2428 return rv; 2429 } 2430 2431 /************************************************************************ 2432 * ixgbe_register_vlan 2433 * 2434 * Run via vlan config EVENT, it enables us to use the 2435 * HW Filter table since we can get the vlan id. This 2436 * just creates the entry in the soft version of the 2437 * VFTA, init will repopulate the real table. 2438 ************************************************************************/ 2439 static int 2440 ixgbe_register_vlan(struct adapter *adapter, u16 vtag) 2441 { 2442 u16 index, bit; 2443 int error; 2444 2445 if ((vtag == 0) || (vtag > 4095)) /* Invalid */ 2446 return EINVAL; 2447 2448 IXGBE_CORE_LOCK(adapter); 2449 index = (vtag >> 5) & 0x7F; 2450 bit = vtag & 0x1F; 2451 adapter->shadow_vfta[index] |= ((u32)1 << bit); 2452 error = adapter->hw.mac.ops.set_vfta(&adapter->hw, vtag, 0, true, 2453 true); 2454 IXGBE_CORE_UNLOCK(adapter); 2455 if (error != 0) 2456 error = EACCES; 2457 2458 return error; 2459 } /* ixgbe_register_vlan */ 2460 2461 /************************************************************************ 2462 * ixgbe_unregister_vlan 2463 * 2464 * Run via vlan unconfig EVENT, remove our entry in the soft vfta. 2465 ************************************************************************/ 2466 static int 2467 ixgbe_unregister_vlan(struct adapter *adapter, u16 vtag) 2468 { 2469 u16 index, bit; 2470 int error; 2471 2472 if ((vtag == 0) || (vtag > 4095)) /* Invalid */ 2473 return EINVAL; 2474 2475 IXGBE_CORE_LOCK(adapter); 2476 index = (vtag >> 5) & 0x7F; 2477 bit = vtag & 0x1F; 2478 adapter->shadow_vfta[index] &= ~((u32)1 << bit); 2479 error = adapter->hw.mac.ops.set_vfta(&adapter->hw, vtag, 0, false, 2480 true); 2481 IXGBE_CORE_UNLOCK(adapter); 2482 if (error != 0) 2483 error = EACCES; 2484 2485 return error; 2486 } /* ixgbe_unregister_vlan */ 2487 2488 static void 2489 ixgbe_setup_vlan_hw_tagging(struct adapter *adapter) 2490 { 2491 struct ethercom *ec = &adapter->osdep.ec; 2492 struct ixgbe_hw *hw = &adapter->hw; 2493 struct rx_ring *rxr; 2494 u32 ctrl; 2495 int i; 2496 bool hwtagging; 2497 2498 /* Enable HW tagging only if any vlan is attached */ 2499 hwtagging = (ec->ec_capenable & ETHERCAP_VLAN_HWTAGGING) 2500 && VLAN_ATTACHED(ec); 2501 2502 /* Setup the queues for vlans */ 2503 for (i = 0; i < adapter->num_queues; i++) { 2504 rxr = &adapter->rx_rings[i]; 2505 /* 2506 * On 82599 and later, the VLAN enable is per/queue in RXDCTL. 2507 */ 2508 if (hw->mac.type != ixgbe_mac_82598EB) { 2509 ctrl = IXGBE_READ_REG(hw, IXGBE_RXDCTL(rxr->me)); 2510 if (hwtagging) 2511 ctrl |= IXGBE_RXDCTL_VME; 2512 else 2513 ctrl &= ~IXGBE_RXDCTL_VME; 2514 IXGBE_WRITE_REG(hw, IXGBE_RXDCTL(rxr->me), ctrl); 2515 } 2516 rxr->vtag_strip = hwtagging ? TRUE : FALSE; 2517 } 2518 2519 /* VLAN hw tagging for 82598 */ 2520 if (hw->mac.type == ixgbe_mac_82598EB) { 2521 ctrl = IXGBE_READ_REG(hw, IXGBE_VLNCTRL); 2522 if (hwtagging) 2523 ctrl |= IXGBE_VLNCTRL_VME; 2524 else 2525 ctrl &= ~IXGBE_VLNCTRL_VME; 2526 IXGBE_WRITE_REG(hw, IXGBE_VLNCTRL, ctrl); 2527 } 2528 } /* ixgbe_setup_vlan_hw_tagging */ 2529 2530 static void 2531 ixgbe_setup_vlan_hw_support(struct adapter *adapter) 2532 { 2533 struct ethercom *ec = &adapter->osdep.ec; 2534 struct ixgbe_hw *hw = &adapter->hw; 2535 int i; 2536 u32 ctrl; 2537 struct vlanid_list *vlanidp; 2538 2539 /* 2540 * This function is called from both if_init and ifflags_cb() 2541 * on NetBSD. 2542 */ 2543 2544 /* 2545 * Part 1: 2546 * Setup VLAN HW tagging 2547 */ 2548 ixgbe_setup_vlan_hw_tagging(adapter); 2549 2550 /* 2551 * Part 2: 2552 * Setup VLAN HW filter 2553 */ 2554 /* Cleanup shadow_vfta */ 2555 for (i = 0; i < IXGBE_VFTA_SIZE; i++) 2556 adapter->shadow_vfta[i] = 0; 2557 /* Generate shadow_vfta from ec_vids */ 2558 ETHER_LOCK(ec); 2559 SIMPLEQ_FOREACH(vlanidp, &ec->ec_vids, vid_list) { 2560 uint32_t idx; 2561 2562 idx = vlanidp->vid / 32; 2563 KASSERT(idx < IXGBE_VFTA_SIZE); 2564 adapter->shadow_vfta[idx] |= (u32)1 << (vlanidp->vid % 32); 2565 } 2566 ETHER_UNLOCK(ec); 2567 for (i = 0; i < IXGBE_VFTA_SIZE; i++) 2568 IXGBE_WRITE_REG(hw, IXGBE_VFTA(i), adapter->shadow_vfta[i]); 2569 2570 ctrl = IXGBE_READ_REG(hw, IXGBE_VLNCTRL); 2571 /* Enable the Filter Table if enabled */ 2572 if (ec->ec_capenable & ETHERCAP_VLAN_HWFILTER) 2573 ctrl |= IXGBE_VLNCTRL_VFE; 2574 else 2575 ctrl &= ~IXGBE_VLNCTRL_VFE; 2576 IXGBE_WRITE_REG(hw, IXGBE_VLNCTRL, ctrl); 2577 } /* ixgbe_setup_vlan_hw_support */ 2578 2579 /************************************************************************ 2580 * ixgbe_get_slot_info 2581 * 2582 * Get the width and transaction speed of 2583 * the slot this adapter is plugged into. 2584 ************************************************************************/ 2585 static void 2586 ixgbe_get_slot_info(struct adapter *adapter) 2587 { 2588 device_t dev = adapter->dev; 2589 struct ixgbe_hw *hw = &adapter->hw; 2590 u32 offset; 2591 u16 link; 2592 int bus_info_valid = TRUE; 2593 2594 /* Some devices are behind an internal bridge */ 2595 switch (hw->device_id) { 2596 case IXGBE_DEV_ID_82599_SFP_SF_QP: 2597 case IXGBE_DEV_ID_82599_QSFP_SF_QP: 2598 goto get_parent_info; 2599 default: 2600 break; 2601 } 2602 2603 ixgbe_get_bus_info(hw); 2604 2605 /* 2606 * Some devices don't use PCI-E, but there is no need 2607 * to display "Unknown" for bus speed and width. 2608 */ 2609 switch (hw->mac.type) { 2610 case ixgbe_mac_X550EM_x: 2611 case ixgbe_mac_X550EM_a: 2612 return; 2613 default: 2614 goto display; 2615 } 2616 2617 get_parent_info: 2618 /* 2619 * For the Quad port adapter we need to parse back 2620 * up the PCI tree to find the speed of the expansion 2621 * slot into which this adapter is plugged. A bit more work. 2622 */ 2623 dev = device_parent(device_parent(dev)); 2624 #if 0 2625 #ifdef IXGBE_DEBUG 2626 device_printf(dev, "parent pcib = %x,%x,%x\n", pci_get_bus(dev), 2627 pci_get_slot(dev), pci_get_function(dev)); 2628 #endif 2629 dev = device_parent(device_parent(dev)); 2630 #ifdef IXGBE_DEBUG 2631 device_printf(dev, "slot pcib = %x,%x,%x\n", pci_get_bus(dev), 2632 pci_get_slot(dev), pci_get_function(dev)); 2633 #endif 2634 #endif 2635 /* Now get the PCI Express Capabilities offset */ 2636 if (pci_get_capability(adapter->osdep.pc, adapter->osdep.tag, 2637 PCI_CAP_PCIEXPRESS, &offset, NULL)) { 2638 /* 2639 * Hmm...can't get PCI-Express capabilities. 2640 * Falling back to default method. 2641 */ 2642 bus_info_valid = FALSE; 2643 ixgbe_get_bus_info(hw); 2644 goto display; 2645 } 2646 /* ...and read the Link Status Register */ 2647 link = pci_conf_read(adapter->osdep.pc, adapter->osdep.tag, 2648 offset + PCIE_LCSR) >> 16; 2649 ixgbe_set_pci_config_data_generic(hw, link); 2650 2651 display: 2652 device_printf(dev, "PCI Express Bus: Speed %s Width %s\n", 2653 ((hw->bus.speed == ixgbe_bus_speed_8000) ? "8.0GT/s" : 2654 (hw->bus.speed == ixgbe_bus_speed_5000) ? "5.0GT/s" : 2655 (hw->bus.speed == ixgbe_bus_speed_2500) ? "2.5GT/s" : 2656 "Unknown"), 2657 ((hw->bus.width == ixgbe_bus_width_pcie_x8) ? "x8" : 2658 (hw->bus.width == ixgbe_bus_width_pcie_x4) ? "x4" : 2659 (hw->bus.width == ixgbe_bus_width_pcie_x1) ? "x1" : 2660 "Unknown")); 2661 2662 if (bus_info_valid) { 2663 if ((hw->device_id != IXGBE_DEV_ID_82599_SFP_SF_QP) && 2664 ((hw->bus.width <= ixgbe_bus_width_pcie_x4) && 2665 (hw->bus.speed == ixgbe_bus_speed_2500))) { 2666 device_printf(dev, "PCI-Express bandwidth available" 2667 " for this card\n is not sufficient for" 2668 " optimal performance.\n"); 2669 device_printf(dev, "For optimal performance a x8 " 2670 "PCIE, or x4 PCIE Gen2 slot is required.\n"); 2671 } 2672 if ((hw->device_id == IXGBE_DEV_ID_82599_SFP_SF_QP) && 2673 ((hw->bus.width <= ixgbe_bus_width_pcie_x8) && 2674 (hw->bus.speed < ixgbe_bus_speed_8000))) { 2675 device_printf(dev, "PCI-Express bandwidth available" 2676 " for this card\n is not sufficient for" 2677 " optimal performance.\n"); 2678 device_printf(dev, "For optimal performance a x8 " 2679 "PCIE Gen3 slot is required.\n"); 2680 } 2681 } else 2682 device_printf(dev, "Unable to determine slot speed/width. The speed/width reported are that of the internal switch.\n"); 2683 2684 return; 2685 } /* ixgbe_get_slot_info */ 2686 2687 /************************************************************************ 2688 * ixgbe_enable_queue - MSI-X Interrupt Handlers and Tasklets 2689 ************************************************************************/ 2690 static inline void 2691 ixgbe_enable_queue(struct adapter *adapter, u32 vector) 2692 { 2693 struct ixgbe_hw *hw = &adapter->hw; 2694 struct ix_queue *que = &adapter->queues[vector]; 2695 u64 queue = 1ULL << vector; 2696 u32 mask; 2697 2698 mutex_enter(&que->dc_mtx); 2699 if (que->disabled_count > 0 && --que->disabled_count > 0) 2700 goto out; 2701 2702 if (hw->mac.type == ixgbe_mac_82598EB) { 2703 mask = (IXGBE_EIMS_RTX_QUEUE & queue); 2704 IXGBE_WRITE_REG(hw, IXGBE_EIMS, mask); 2705 } else { 2706 mask = (queue & 0xFFFFFFFF); 2707 if (mask) 2708 IXGBE_WRITE_REG(hw, IXGBE_EIMS_EX(0), mask); 2709 mask = (queue >> 32); 2710 if (mask) 2711 IXGBE_WRITE_REG(hw, IXGBE_EIMS_EX(1), mask); 2712 } 2713 out: 2714 mutex_exit(&que->dc_mtx); 2715 } /* ixgbe_enable_queue */ 2716 2717 /************************************************************************ 2718 * ixgbe_disable_queue_internal 2719 ************************************************************************/ 2720 static inline void 2721 ixgbe_disable_queue_internal(struct adapter *adapter, u32 vector, bool nestok) 2722 { 2723 struct ixgbe_hw *hw = &adapter->hw; 2724 struct ix_queue *que = &adapter->queues[vector]; 2725 u64 queue = 1ULL << vector; 2726 u32 mask; 2727 2728 mutex_enter(&que->dc_mtx); 2729 2730 if (que->disabled_count > 0) { 2731 if (nestok) 2732 que->disabled_count++; 2733 goto out; 2734 } 2735 que->disabled_count++; 2736 2737 if (hw->mac.type == ixgbe_mac_82598EB) { 2738 mask = (IXGBE_EIMS_RTX_QUEUE & queue); 2739 IXGBE_WRITE_REG(hw, IXGBE_EIMC, mask); 2740 } else { 2741 mask = (queue & 0xFFFFFFFF); 2742 if (mask) 2743 IXGBE_WRITE_REG(hw, IXGBE_EIMC_EX(0), mask); 2744 mask = (queue >> 32); 2745 if (mask) 2746 IXGBE_WRITE_REG(hw, IXGBE_EIMC_EX(1), mask); 2747 } 2748 out: 2749 mutex_exit(&que->dc_mtx); 2750 } /* ixgbe_disable_queue_internal */ 2751 2752 /************************************************************************ 2753 * ixgbe_disable_queue 2754 ************************************************************************/ 2755 static inline void 2756 ixgbe_disable_queue(struct adapter *adapter, u32 vector) 2757 { 2758 2759 ixgbe_disable_queue_internal(adapter, vector, true); 2760 } /* ixgbe_disable_queue */ 2761 2762 /************************************************************************ 2763 * ixgbe_sched_handle_que - schedule deferred packet processing 2764 ************************************************************************/ 2765 static inline void 2766 ixgbe_sched_handle_que(struct adapter *adapter, struct ix_queue *que) 2767 { 2768 2769 if (que->txrx_use_workqueue) { 2770 /* 2771 * adapter->que_wq is bound to each CPU instead of 2772 * each NIC queue to reduce workqueue kthread. As we 2773 * should consider about interrupt affinity in this 2774 * function, the workqueue kthread must be WQ_PERCPU. 2775 * If create WQ_PERCPU workqueue kthread for each NIC 2776 * queue, that number of created workqueue kthread is 2777 * (number of used NIC queue) * (number of CPUs) = 2778 * (number of CPUs) ^ 2 most often. 2779 * 2780 * The same NIC queue's interrupts are avoided by 2781 * masking the queue's interrupt. And different 2782 * NIC queue's interrupts use different struct work 2783 * (que->wq_cookie). So, "enqueued flag" to avoid 2784 * twice workqueue_enqueue() is not required . 2785 */ 2786 workqueue_enqueue(adapter->que_wq, &que->wq_cookie, curcpu()); 2787 } else { 2788 softint_schedule(que->que_si); 2789 } 2790 } 2791 2792 /************************************************************************ 2793 * ixgbe_msix_que - MSI-X Queue Interrupt Service routine 2794 ************************************************************************/ 2795 static int 2796 ixgbe_msix_que(void *arg) 2797 { 2798 struct ix_queue *que = arg; 2799 struct adapter *adapter = que->adapter; 2800 struct ifnet *ifp = adapter->ifp; 2801 struct tx_ring *txr = que->txr; 2802 struct rx_ring *rxr = que->rxr; 2803 bool more; 2804 u32 newitr = 0; 2805 2806 /* Protect against spurious interrupts */ 2807 if ((ifp->if_flags & IFF_RUNNING) == 0) 2808 return 0; 2809 2810 ixgbe_disable_queue(adapter, que->msix); 2811 IXGBE_EVC_ADD(&que->irqs, 1); 2812 2813 /* 2814 * Don't change "que->txrx_use_workqueue" from this point to avoid 2815 * flip-flopping softint/workqueue mode in one deferred processing. 2816 */ 2817 que->txrx_use_workqueue = adapter->txrx_use_workqueue; 2818 2819 #ifdef __NetBSD__ 2820 /* Don't run ixgbe_rxeof in interrupt context */ 2821 more = true; 2822 #else 2823 more = ixgbe_rxeof(que); 2824 #endif 2825 2826 IXGBE_TX_LOCK(txr); 2827 ixgbe_txeof(txr); 2828 IXGBE_TX_UNLOCK(txr); 2829 2830 /* Do AIM now? */ 2831 2832 if (adapter->enable_aim == false) 2833 goto no_calc; 2834 /* 2835 * Do Adaptive Interrupt Moderation: 2836 * - Write out last calculated setting 2837 * - Calculate based on average size over 2838 * the last interval. 2839 */ 2840 if (que->eitr_setting) 2841 ixgbe_eitr_write(adapter, que->msix, que->eitr_setting); 2842 2843 que->eitr_setting = 0; 2844 2845 /* Idle, do nothing */ 2846 if ((txr->bytes == 0) && (rxr->bytes == 0)) 2847 goto no_calc; 2848 2849 if ((txr->bytes) && (txr->packets)) 2850 newitr = txr->bytes/txr->packets; 2851 if ((rxr->bytes) && (rxr->packets)) 2852 newitr = uimax(newitr, (rxr->bytes / rxr->packets)); 2853 newitr += 24; /* account for hardware frame, crc */ 2854 2855 /* set an upper boundary */ 2856 newitr = uimin(newitr, 3000); 2857 2858 /* Be nice to the mid range */ 2859 if ((newitr > 300) && (newitr < 1200)) 2860 newitr = (newitr / 3); 2861 else 2862 newitr = (newitr / 2); 2863 2864 /* 2865 * When RSC is used, ITR interval must be larger than RSC_DELAY. 2866 * Currently, we use 2us for RSC_DELAY. The minimum value is always 2867 * greater than 2us on 100M (and 10M?(not documented)), but it's not 2868 * on 1G and higher. 2869 */ 2870 if ((adapter->link_speed != IXGBE_LINK_SPEED_100_FULL) 2871 && (adapter->link_speed != IXGBE_LINK_SPEED_10_FULL)) { 2872 if (newitr < IXGBE_MIN_RSC_EITR_10G1G) 2873 newitr = IXGBE_MIN_RSC_EITR_10G1G; 2874 } 2875 2876 /* save for next interrupt */ 2877 que->eitr_setting = newitr; 2878 2879 /* Reset state */ 2880 txr->bytes = 0; 2881 txr->packets = 0; 2882 rxr->bytes = 0; 2883 rxr->packets = 0; 2884 2885 no_calc: 2886 if (more) 2887 ixgbe_sched_handle_que(adapter, que); 2888 else 2889 ixgbe_enable_queue(adapter, que->msix); 2890 2891 return 1; 2892 } /* ixgbe_msix_que */ 2893 2894 /************************************************************************ 2895 * ixgbe_media_status - Media Ioctl callback 2896 * 2897 * Called whenever the user queries the status of 2898 * the interface using ifconfig. 2899 ************************************************************************/ 2900 static void 2901 ixgbe_media_status(struct ifnet *ifp, struct ifmediareq *ifmr) 2902 { 2903 struct adapter *adapter = ifp->if_softc; 2904 struct ixgbe_hw *hw = &adapter->hw; 2905 int layer; 2906 2907 INIT_DEBUGOUT("ixgbe_media_status: begin"); 2908 ixgbe_update_link_status(adapter); 2909 2910 ifmr->ifm_status = IFM_AVALID; 2911 ifmr->ifm_active = IFM_ETHER; 2912 2913 if (adapter->link_active != LINK_STATE_UP) { 2914 ifmr->ifm_active |= IFM_NONE; 2915 return; 2916 } 2917 2918 ifmr->ifm_status |= IFM_ACTIVE; 2919 layer = adapter->phy_layer; 2920 2921 if (layer & IXGBE_PHYSICAL_LAYER_10GBASE_T || 2922 layer & IXGBE_PHYSICAL_LAYER_5GBASE_T || 2923 layer & IXGBE_PHYSICAL_LAYER_2500BASE_T || 2924 layer & IXGBE_PHYSICAL_LAYER_1000BASE_T || 2925 layer & IXGBE_PHYSICAL_LAYER_100BASE_TX || 2926 layer & IXGBE_PHYSICAL_LAYER_10BASE_T) 2927 switch (adapter->link_speed) { 2928 case IXGBE_LINK_SPEED_10GB_FULL: 2929 ifmr->ifm_active |= IFM_10G_T | IFM_FDX; 2930 break; 2931 case IXGBE_LINK_SPEED_5GB_FULL: 2932 ifmr->ifm_active |= IFM_5000_T | IFM_FDX; 2933 break; 2934 case IXGBE_LINK_SPEED_2_5GB_FULL: 2935 ifmr->ifm_active |= IFM_2500_T | IFM_FDX; 2936 break; 2937 case IXGBE_LINK_SPEED_1GB_FULL: 2938 ifmr->ifm_active |= IFM_1000_T | IFM_FDX; 2939 break; 2940 case IXGBE_LINK_SPEED_100_FULL: 2941 ifmr->ifm_active |= IFM_100_TX | IFM_FDX; 2942 break; 2943 case IXGBE_LINK_SPEED_10_FULL: 2944 ifmr->ifm_active |= IFM_10_T | IFM_FDX; 2945 break; 2946 } 2947 if (layer & IXGBE_PHYSICAL_LAYER_SFP_PLUS_CU || 2948 layer & IXGBE_PHYSICAL_LAYER_SFP_ACTIVE_DA) 2949 switch (adapter->link_speed) { 2950 case IXGBE_LINK_SPEED_10GB_FULL: 2951 ifmr->ifm_active |= IFM_10G_TWINAX | IFM_FDX; 2952 break; 2953 } 2954 if (layer & IXGBE_PHYSICAL_LAYER_10GBASE_LR) 2955 switch (adapter->link_speed) { 2956 case IXGBE_LINK_SPEED_10GB_FULL: 2957 ifmr->ifm_active |= IFM_10G_LR | IFM_FDX; 2958 break; 2959 case IXGBE_LINK_SPEED_1GB_FULL: 2960 ifmr->ifm_active |= IFM_1000_LX | IFM_FDX; 2961 break; 2962 } 2963 if (layer & IXGBE_PHYSICAL_LAYER_10GBASE_LRM) 2964 switch (adapter->link_speed) { 2965 case IXGBE_LINK_SPEED_10GB_FULL: 2966 ifmr->ifm_active |= IFM_10G_LRM | IFM_FDX; 2967 break; 2968 case IXGBE_LINK_SPEED_1GB_FULL: 2969 ifmr->ifm_active |= IFM_1000_LX | IFM_FDX; 2970 break; 2971 } 2972 if (layer & IXGBE_PHYSICAL_LAYER_10GBASE_SR || 2973 layer & IXGBE_PHYSICAL_LAYER_1000BASE_SX) 2974 switch (adapter->link_speed) { 2975 case IXGBE_LINK_SPEED_10GB_FULL: 2976 ifmr->ifm_active |= IFM_10G_SR | IFM_FDX; 2977 break; 2978 case IXGBE_LINK_SPEED_1GB_FULL: 2979 ifmr->ifm_active |= IFM_1000_SX | IFM_FDX; 2980 break; 2981 } 2982 if (layer & IXGBE_PHYSICAL_LAYER_10GBASE_CX4) 2983 switch (adapter->link_speed) { 2984 case IXGBE_LINK_SPEED_10GB_FULL: 2985 ifmr->ifm_active |= IFM_10G_CX4 | IFM_FDX; 2986 break; 2987 } 2988 /* 2989 * XXX: These need to use the proper media types once 2990 * they're added. 2991 */ 2992 if (layer & IXGBE_PHYSICAL_LAYER_10GBASE_KR) 2993 switch (adapter->link_speed) { 2994 case IXGBE_LINK_SPEED_10GB_FULL: 2995 ifmr->ifm_active |= IFM_10G_KR | IFM_FDX; 2996 break; 2997 case IXGBE_LINK_SPEED_2_5GB_FULL: 2998 ifmr->ifm_active |= IFM_2500_KX | IFM_FDX; 2999 break; 3000 case IXGBE_LINK_SPEED_1GB_FULL: 3001 ifmr->ifm_active |= IFM_1000_KX | IFM_FDX; 3002 break; 3003 } 3004 else if (layer & IXGBE_PHYSICAL_LAYER_10GBASE_KX4 || 3005 layer & IXGBE_PHYSICAL_LAYER_2500BASE_KX || 3006 layer & IXGBE_PHYSICAL_LAYER_1000BASE_KX) 3007 switch (adapter->link_speed) { 3008 case IXGBE_LINK_SPEED_10GB_FULL: 3009 ifmr->ifm_active |= IFM_10G_KX4 | IFM_FDX; 3010 break; 3011 case IXGBE_LINK_SPEED_2_5GB_FULL: 3012 ifmr->ifm_active |= IFM_2500_KX | IFM_FDX; 3013 break; 3014 case IXGBE_LINK_SPEED_1GB_FULL: 3015 ifmr->ifm_active |= IFM_1000_KX | IFM_FDX; 3016 break; 3017 } 3018 3019 /* If nothing is recognized... */ 3020 #if 0 3021 if (IFM_SUBTYPE(ifmr->ifm_active) == 0) 3022 ifmr->ifm_active |= IFM_UNKNOWN; 3023 #endif 3024 3025 ifp->if_baudrate = ifmedia_baudrate(ifmr->ifm_active); 3026 3027 /* Display current flow control setting used on link */ 3028 if (hw->fc.current_mode == ixgbe_fc_rx_pause || 3029 hw->fc.current_mode == ixgbe_fc_full) 3030 ifmr->ifm_active |= IFM_ETH_RXPAUSE; 3031 if (hw->fc.current_mode == ixgbe_fc_tx_pause || 3032 hw->fc.current_mode == ixgbe_fc_full) 3033 ifmr->ifm_active |= IFM_ETH_TXPAUSE; 3034 3035 return; 3036 } /* ixgbe_media_status */ 3037 3038 /************************************************************************ 3039 * ixgbe_media_change - Media Ioctl callback 3040 * 3041 * Called when the user changes speed/duplex using 3042 * media/mediopt option with ifconfig. 3043 ************************************************************************/ 3044 static int 3045 ixgbe_media_change(struct ifnet *ifp) 3046 { 3047 struct adapter *adapter = ifp->if_softc; 3048 struct ifmedia *ifm = &adapter->media; 3049 struct ixgbe_hw *hw = &adapter->hw; 3050 ixgbe_link_speed speed = 0; 3051 ixgbe_link_speed link_caps = 0; 3052 bool negotiate = false; 3053 s32 err = IXGBE_NOT_IMPLEMENTED; 3054 3055 INIT_DEBUGOUT("ixgbe_media_change: begin"); 3056 3057 if (IFM_TYPE(ifm->ifm_media) != IFM_ETHER) 3058 return (EINVAL); 3059 3060 if (hw->phy.media_type == ixgbe_media_type_backplane) 3061 return (EPERM); 3062 3063 /* 3064 * We don't actually need to check against the supported 3065 * media types of the adapter; ifmedia will take care of 3066 * that for us. 3067 */ 3068 switch (IFM_SUBTYPE(ifm->ifm_media)) { 3069 case IFM_AUTO: 3070 err = hw->mac.ops.get_link_capabilities(hw, &link_caps, 3071 &negotiate); 3072 if (err != IXGBE_SUCCESS) { 3073 device_printf(adapter->dev, "Unable to determine " 3074 "supported advertise speeds\n"); 3075 return (ENODEV); 3076 } 3077 speed |= link_caps; 3078 break; 3079 case IFM_10G_T: 3080 case IFM_10G_LRM: 3081 case IFM_10G_LR: 3082 case IFM_10G_TWINAX: 3083 case IFM_10G_SR: 3084 case IFM_10G_CX4: 3085 case IFM_10G_KR: 3086 case IFM_10G_KX4: 3087 speed |= IXGBE_LINK_SPEED_10GB_FULL; 3088 break; 3089 case IFM_5000_T: 3090 speed |= IXGBE_LINK_SPEED_5GB_FULL; 3091 break; 3092 case IFM_2500_T: 3093 case IFM_2500_KX: 3094 speed |= IXGBE_LINK_SPEED_2_5GB_FULL; 3095 break; 3096 case IFM_1000_T: 3097 case IFM_1000_LX: 3098 case IFM_1000_SX: 3099 case IFM_1000_KX: 3100 speed |= IXGBE_LINK_SPEED_1GB_FULL; 3101 break; 3102 case IFM_100_TX: 3103 speed |= IXGBE_LINK_SPEED_100_FULL; 3104 break; 3105 case IFM_10_T: 3106 speed |= IXGBE_LINK_SPEED_10_FULL; 3107 break; 3108 case IFM_NONE: 3109 break; 3110 default: 3111 goto invalid; 3112 } 3113 3114 hw->mac.autotry_restart = TRUE; 3115 hw->mac.ops.setup_link(hw, speed, TRUE); 3116 adapter->advertise = 0; 3117 if (IFM_SUBTYPE(ifm->ifm_media) != IFM_AUTO) { 3118 if ((speed & IXGBE_LINK_SPEED_10GB_FULL) != 0) 3119 adapter->advertise |= 1 << 2; 3120 if ((speed & IXGBE_LINK_SPEED_1GB_FULL) != 0) 3121 adapter->advertise |= 1 << 1; 3122 if ((speed & IXGBE_LINK_SPEED_100_FULL) != 0) 3123 adapter->advertise |= 1 << 0; 3124 if ((speed & IXGBE_LINK_SPEED_10_FULL) != 0) 3125 adapter->advertise |= 1 << 3; 3126 if ((speed & IXGBE_LINK_SPEED_2_5GB_FULL) != 0) 3127 adapter->advertise |= 1 << 4; 3128 if ((speed & IXGBE_LINK_SPEED_5GB_FULL) != 0) 3129 adapter->advertise |= 1 << 5; 3130 } 3131 3132 return (0); 3133 3134 invalid: 3135 device_printf(adapter->dev, "Invalid media type!\n"); 3136 3137 return (EINVAL); 3138 } /* ixgbe_media_change */ 3139 3140 /************************************************************************ 3141 * ixgbe_msix_admin - Link status change ISR (MSI/MSI-X) 3142 ************************************************************************/ 3143 static int 3144 ixgbe_msix_admin(void *arg) 3145 { 3146 struct adapter *adapter = arg; 3147 struct ixgbe_hw *hw = &adapter->hw; 3148 u32 eicr; 3149 u32 eims_orig; 3150 u32 eims_disable = 0; 3151 3152 IXGBE_EVC_ADD(&adapter->admin_irqev, 1); 3153 3154 eims_orig = IXGBE_READ_REG(hw, IXGBE_EIMS); 3155 /* Pause other interrupts */ 3156 IXGBE_WRITE_REG(hw, IXGBE_EIMC, IXGBE_MSIX_OTHER_CLEAR_MASK); 3157 3158 /* 3159 * First get the cause. 3160 * 3161 * The specifications of 82598, 82599, X540 and X550 say EICS register 3162 * is write only. However, Linux says it is a workaround for silicon 3163 * errata to read EICS instead of EICR to get interrupt cause. 3164 * At least, reading EICR clears lower 16bits of EIMS on 82598. 3165 */ 3166 eicr = IXGBE_READ_REG(hw, IXGBE_EICS); 3167 /* Be sure the queue bits are not cleared */ 3168 eicr &= ~IXGBE_EICR_RTX_QUEUE; 3169 /* Clear all OTHER interrupts with write */ 3170 IXGBE_WRITE_REG(hw, IXGBE_EICR, eicr); 3171 3172 ixgbe_intr_admin_common(adapter, eicr, &eims_disable); 3173 3174 /* Re-enable some OTHER interrupts */ 3175 IXGBE_WRITE_REG(hw, IXGBE_EIMS, eims_orig & ~eims_disable); 3176 3177 return 1; 3178 } /* ixgbe_msix_admin */ 3179 3180 static void 3181 ixgbe_intr_admin_common(struct adapter *adapter, u32 eicr, u32 *eims_disable) 3182 { 3183 struct ixgbe_hw *hw = &adapter->hw; 3184 u32 task_requests = 0; 3185 s32 retval; 3186 3187 /* Link status change */ 3188 if (eicr & IXGBE_EICR_LSC) { 3189 task_requests |= IXGBE_REQUEST_TASK_LSC; 3190 *eims_disable |= IXGBE_EIMS_LSC; 3191 } 3192 3193 if (ixgbe_is_sfp(hw)) { 3194 u32 eicr_mask; 3195 3196 /* Pluggable optics-related interrupt */ 3197 if (hw->mac.type >= ixgbe_mac_X540) 3198 eicr_mask = IXGBE_EICR_GPI_SDP0_X540; 3199 else 3200 eicr_mask = IXGBE_EICR_GPI_SDP2_BY_MAC(hw); 3201 3202 /* 3203 * An interrupt might not arrive when a module is inserted. 3204 * When an link status change interrupt occurred and the driver 3205 * still regard SFP as unplugged, issue the module softint 3206 * and then issue LSC interrupt. 3207 */ 3208 if ((eicr & eicr_mask) 3209 || ((hw->phy.sfp_type == ixgbe_sfp_type_not_present) 3210 && (eicr & IXGBE_EICR_LSC))) { 3211 task_requests |= IXGBE_REQUEST_TASK_MOD; 3212 *eims_disable |= IXGBE_EIMS_LSC; 3213 } 3214 3215 if ((hw->mac.type == ixgbe_mac_82599EB) && 3216 (eicr & IXGBE_EICR_GPI_SDP1_BY_MAC(hw))) { 3217 task_requests |= IXGBE_REQUEST_TASK_MSF; 3218 *eims_disable |= IXGBE_EIMS_GPI_SDP1_BY_MAC(hw); 3219 } 3220 } 3221 3222 if (adapter->hw.mac.type != ixgbe_mac_82598EB) { 3223 #ifdef IXGBE_FDIR 3224 if ((adapter->feat_en & IXGBE_FEATURE_FDIR) && 3225 (eicr & IXGBE_EICR_FLOW_DIR)) { 3226 if (!atomic_cas_uint(&adapter->fdir_reinit, 0, 1)) { 3227 task_requests |= IXGBE_REQUEST_TASK_FDIR; 3228 /* Disable the interrupt */ 3229 *eims_disable |= IXGBE_EIMS_FLOW_DIR; 3230 } 3231 } 3232 #endif 3233 3234 if (eicr & IXGBE_EICR_ECC) { 3235 if (ratecheck(&adapter->lasterr_time, 3236 &ixgbe_errlog_intrvl)) 3237 device_printf(adapter->dev, 3238 "CRITICAL: ECC ERROR!! Please Reboot!!\n"); 3239 } 3240 3241 /* Check for over temp condition */ 3242 if (adapter->feat_en & IXGBE_FEATURE_TEMP_SENSOR) { 3243 switch (adapter->hw.mac.type) { 3244 case ixgbe_mac_X550EM_a: 3245 if (!(eicr & IXGBE_EICR_GPI_SDP0_X550EM_a)) 3246 break; 3247 retval = hw->phy.ops.check_overtemp(hw); 3248 if (retval != IXGBE_ERR_OVERTEMP) 3249 break; 3250 if (ratecheck(&adapter->lasterr_time, 3251 &ixgbe_errlog_intrvl)) { 3252 device_printf(adapter->dev, 3253 "CRITICAL: OVER TEMP!! " 3254 "PHY IS SHUT DOWN!!\n"); 3255 device_printf(adapter->dev, 3256 "System shutdown required!\n"); 3257 } 3258 break; 3259 default: 3260 if (!(eicr & IXGBE_EICR_TS)) 3261 break; 3262 retval = hw->phy.ops.check_overtemp(hw); 3263 if (retval != IXGBE_ERR_OVERTEMP) 3264 break; 3265 if (ratecheck(&adapter->lasterr_time, 3266 &ixgbe_errlog_intrvl)) { 3267 device_printf(adapter->dev, 3268 "CRITICAL: OVER TEMP!! " 3269 "PHY IS SHUT DOWN!!\n"); 3270 device_printf(adapter->dev, 3271 "System shutdown required!\n"); 3272 } 3273 break; 3274 } 3275 } 3276 3277 /* Check for VF message */ 3278 if ((adapter->feat_en & IXGBE_FEATURE_SRIOV) && 3279 (eicr & IXGBE_EICR_MAILBOX)) { 3280 task_requests |= IXGBE_REQUEST_TASK_MBX; 3281 *eims_disable |= IXGBE_EIMS_MAILBOX; 3282 } 3283 } 3284 3285 /* Check for fan failure */ 3286 if (adapter->feat_en & IXGBE_FEATURE_FAN_FAIL) 3287 ixgbe_check_fan_failure(adapter, eicr, true); 3288 3289 /* External PHY interrupt */ 3290 if ((hw->phy.type == ixgbe_phy_x550em_ext_t) && 3291 (eicr & IXGBE_EICR_GPI_SDP0_X540)) { 3292 task_requests |= IXGBE_REQUEST_TASK_PHY; 3293 *eims_disable |= IXGBE_EICR_GPI_SDP0_X540; 3294 } 3295 3296 if (task_requests != 0) { 3297 mutex_enter(&adapter->admin_mtx); 3298 adapter->task_requests |= task_requests; 3299 ixgbe_schedule_admin_tasklet(adapter); 3300 mutex_exit(&adapter->admin_mtx); 3301 } 3302 3303 } 3304 3305 static void 3306 ixgbe_eitr_write(struct adapter *adapter, uint32_t index, uint32_t itr) 3307 { 3308 3309 if (adapter->hw.mac.type == ixgbe_mac_82598EB) 3310 itr |= itr << 16; 3311 else 3312 itr |= IXGBE_EITR_CNT_WDIS; 3313 3314 IXGBE_WRITE_REG(&adapter->hw, IXGBE_EITR(index), itr); 3315 } 3316 3317 3318 /************************************************************************ 3319 * ixgbe_sysctl_interrupt_rate_handler 3320 ************************************************************************/ 3321 static int 3322 ixgbe_sysctl_interrupt_rate_handler(SYSCTLFN_ARGS) 3323 { 3324 struct sysctlnode node = *rnode; 3325 struct ix_queue *que = (struct ix_queue *)node.sysctl_data; 3326 struct adapter *adapter; 3327 uint32_t reg, usec, rate; 3328 int error; 3329 3330 if (que == NULL) 3331 return 0; 3332 3333 adapter = que->adapter; 3334 if (ixgbe_fw_recovery_mode_swflag(adapter)) 3335 return (EPERM); 3336 3337 reg = IXGBE_READ_REG(&adapter->hw, IXGBE_EITR(que->msix)); 3338 usec = ((reg & 0x0FF8) >> 3); 3339 if (usec > 0) 3340 rate = 500000 / usec; 3341 else 3342 rate = 0; 3343 node.sysctl_data = &rate; 3344 error = sysctl_lookup(SYSCTLFN_CALL(&node)); 3345 if (error || newp == NULL) 3346 return error; 3347 reg &= ~0xfff; /* default, no limitation */ 3348 if (rate > 0 && rate < 500000) { 3349 if (rate < 1000) 3350 rate = 1000; 3351 reg |= ((4000000 / rate) & 0xff8); 3352 /* 3353 * When RSC is used, ITR interval must be larger than 3354 * RSC_DELAY. Currently, we use 2us for RSC_DELAY. 3355 * The minimum value is always greater than 2us on 100M 3356 * (and 10M?(not documented)), but it's not on 1G and higher. 3357 */ 3358 if ((adapter->link_speed != IXGBE_LINK_SPEED_100_FULL) 3359 && (adapter->link_speed != IXGBE_LINK_SPEED_10_FULL)) { 3360 if ((adapter->num_queues > 1) 3361 && (reg < IXGBE_MIN_RSC_EITR_10G1G)) 3362 return EINVAL; 3363 } 3364 ixgbe_max_interrupt_rate = rate; 3365 } else 3366 ixgbe_max_interrupt_rate = 0; 3367 ixgbe_eitr_write(adapter, que->msix, reg); 3368 3369 return (0); 3370 } /* ixgbe_sysctl_interrupt_rate_handler */ 3371 3372 const struct sysctlnode * 3373 ixgbe_sysctl_instance(struct adapter *adapter) 3374 { 3375 const char *dvname; 3376 struct sysctllog **log; 3377 int rc; 3378 const struct sysctlnode *rnode; 3379 3380 if (adapter->sysctltop != NULL) 3381 return adapter->sysctltop; 3382 3383 log = &adapter->sysctllog; 3384 dvname = device_xname(adapter->dev); 3385 3386 if ((rc = sysctl_createv(log, 0, NULL, &rnode, 3387 0, CTLTYPE_NODE, dvname, 3388 SYSCTL_DESCR("ixgbe information and settings"), 3389 NULL, 0, NULL, 0, CTL_HW, CTL_CREATE, CTL_EOL)) != 0) 3390 goto err; 3391 3392 return rnode; 3393 err: 3394 device_printf(adapter->dev, 3395 "%s: sysctl_createv failed, rc = %d\n", __func__, rc); 3396 return NULL; 3397 } 3398 3399 /************************************************************************ 3400 * ixgbe_add_device_sysctls 3401 ************************************************************************/ 3402 static void 3403 ixgbe_add_device_sysctls(struct adapter *adapter) 3404 { 3405 device_t dev = adapter->dev; 3406 struct ixgbe_hw *hw = &adapter->hw; 3407 struct sysctllog **log; 3408 const struct sysctlnode *rnode, *cnode; 3409 3410 log = &adapter->sysctllog; 3411 3412 if ((rnode = ixgbe_sysctl_instance(adapter)) == NULL) { 3413 aprint_error_dev(dev, "could not create sysctl root\n"); 3414 return; 3415 } 3416 3417 if (sysctl_createv(log, 0, &rnode, &cnode, 3418 CTLFLAG_READWRITE, CTLTYPE_INT, 3419 "debug", SYSCTL_DESCR("Debug Info"), 3420 ixgbe_sysctl_debug, 0, (void *)adapter, 0, CTL_CREATE, CTL_EOL) 3421 != 0) 3422 aprint_error_dev(dev, "could not create sysctl\n"); 3423 3424 if (sysctl_createv(log, 0, &rnode, &cnode, 3425 CTLFLAG_READWRITE, CTLTYPE_INT, 3426 "rx_copy_len", SYSCTL_DESCR("RX Copy Length"), 3427 ixgbe_sysctl_rx_copy_len, 0, 3428 (void *)adapter, 0, CTL_CREATE, CTL_EOL) != 0) 3429 aprint_error_dev(dev, "could not create sysctl\n"); 3430 3431 if (sysctl_createv(log, 0, &rnode, &cnode, 3432 CTLFLAG_READONLY, CTLTYPE_INT, 3433 "num_tx_desc", SYSCTL_DESCR("Number of TX descriptors"), 3434 NULL, 0, &adapter->num_tx_desc, 0, CTL_CREATE, CTL_EOL) != 0) 3435 aprint_error_dev(dev, "could not create sysctl\n"); 3436 3437 if (sysctl_createv(log, 0, &rnode, &cnode, 3438 CTLFLAG_READONLY, CTLTYPE_INT, 3439 "num_rx_desc", SYSCTL_DESCR("Number of RX descriptors"), 3440 NULL, 0, &adapter->num_rx_desc, 0, CTL_CREATE, CTL_EOL) != 0) 3441 aprint_error_dev(dev, "could not create sysctl\n"); 3442 3443 if (sysctl_createv(log, 0, &rnode, &cnode, 3444 CTLFLAG_READWRITE, CTLTYPE_INT, "rx_process_limit", 3445 SYSCTL_DESCR("max number of RX packets to process"), 3446 ixgbe_sysctl_rx_process_limit, 0, (void *)adapter, 0, CTL_CREATE, 3447 CTL_EOL) != 0) 3448 aprint_error_dev(dev, "could not create sysctl\n"); 3449 3450 if (sysctl_createv(log, 0, &rnode, &cnode, 3451 CTLFLAG_READWRITE, CTLTYPE_INT, "tx_process_limit", 3452 SYSCTL_DESCR("max number of TX packets to process"), 3453 ixgbe_sysctl_tx_process_limit, 0, (void *)adapter, 0, CTL_CREATE, 3454 CTL_EOL) != 0) 3455 aprint_error_dev(dev, "could not create sysctl\n"); 3456 3457 if (sysctl_createv(log, 0, &rnode, &cnode, 3458 CTLFLAG_READONLY, CTLTYPE_INT, 3459 "num_queues", SYSCTL_DESCR("Number of queues"), 3460 NULL, 0, &adapter->num_queues, 0, CTL_CREATE, CTL_EOL) != 0) 3461 aprint_error_dev(dev, "could not create sysctl\n"); 3462 3463 /* Sysctls for all devices */ 3464 if (sysctl_createv(log, 0, &rnode, &cnode, CTLFLAG_READWRITE, 3465 CTLTYPE_INT, "fc", SYSCTL_DESCR(IXGBE_SYSCTL_DESC_SET_FC), 3466 ixgbe_sysctl_flowcntl, 0, (void *)adapter, 0, CTL_CREATE, 3467 CTL_EOL) != 0) 3468 aprint_error_dev(dev, "could not create sysctl\n"); 3469 3470 adapter->enable_aim = ixgbe_enable_aim; 3471 if (sysctl_createv(log, 0, &rnode, &cnode, CTLFLAG_READWRITE, 3472 CTLTYPE_BOOL, "enable_aim", SYSCTL_DESCR("Interrupt Moderation"), 3473 NULL, 0, &adapter->enable_aim, 0, CTL_CREATE, CTL_EOL) != 0) 3474 aprint_error_dev(dev, "could not create sysctl\n"); 3475 3476 if (sysctl_createv(log, 0, &rnode, &cnode, 3477 CTLFLAG_READWRITE, CTLTYPE_INT, 3478 "advertise_speed", SYSCTL_DESCR(IXGBE_SYSCTL_DESC_ADV_SPEED), 3479 ixgbe_sysctl_advertise, 0, (void *)adapter, 0, CTL_CREATE, 3480 CTL_EOL) != 0) 3481 aprint_error_dev(dev, "could not create sysctl\n"); 3482 3483 /* 3484 * If each "que->txrx_use_workqueue" is changed in sysctl handler, 3485 * it causesflip-flopping softint/workqueue mode in one deferred 3486 * processing. Therefore, preempt_disable()/preempt_enable() are 3487 * required in ixgbe_sched_handle_que() to avoid 3488 * KASSERT(ixgbe_sched_handle_que()) in softint_schedule(). 3489 * I think changing "que->txrx_use_workqueue" in interrupt handler 3490 * is lighter than doing preempt_disable()/preempt_enable() in every 3491 * ixgbe_sched_handle_que(). 3492 */ 3493 adapter->txrx_use_workqueue = ixgbe_txrx_workqueue; 3494 if (sysctl_createv(log, 0, &rnode, &cnode, CTLFLAG_READWRITE, 3495 CTLTYPE_BOOL, "txrx_workqueue", 3496 SYSCTL_DESCR("Use workqueue for packet processing"), 3497 NULL, 0, &adapter->txrx_use_workqueue, 0, CTL_CREATE, 3498 CTL_EOL) != 0) 3499 aprint_error_dev(dev, "could not create sysctl\n"); 3500 3501 #ifdef IXGBE_DEBUG 3502 /* testing sysctls (for all devices) */ 3503 if (sysctl_createv(log, 0, &rnode, &cnode, CTLFLAG_READWRITE, 3504 CTLTYPE_INT, "power_state", SYSCTL_DESCR("PCI Power State"), 3505 ixgbe_sysctl_power_state, 0, (void *)adapter, 0, CTL_CREATE, 3506 CTL_EOL) != 0) 3507 aprint_error_dev(dev, "could not create sysctl\n"); 3508 3509 if (sysctl_createv(log, 0, &rnode, &cnode, CTLFLAG_READONLY, 3510 CTLTYPE_STRING, "print_rss_config", 3511 SYSCTL_DESCR("Prints RSS Configuration"), 3512 ixgbe_sysctl_print_rss_config, 0, (void *)adapter, 0, CTL_CREATE, 3513 CTL_EOL) != 0) 3514 aprint_error_dev(dev, "could not create sysctl\n"); 3515 #endif 3516 /* for X550 series devices */ 3517 if (hw->mac.type >= ixgbe_mac_X550) 3518 if (sysctl_createv(log, 0, &rnode, &cnode, CTLFLAG_READWRITE, 3519 CTLTYPE_INT, "dmac", SYSCTL_DESCR("DMA Coalesce"), 3520 ixgbe_sysctl_dmac, 0, (void *)adapter, 0, CTL_CREATE, 3521 CTL_EOL) != 0) 3522 aprint_error_dev(dev, "could not create sysctl\n"); 3523 3524 /* for WoL-capable devices */ 3525 if (adapter->wol_support) { 3526 if (sysctl_createv(log, 0, &rnode, &cnode, CTLFLAG_READWRITE, 3527 CTLTYPE_BOOL, "wol_enable", 3528 SYSCTL_DESCR("Enable/Disable Wake on LAN"), 3529 ixgbe_sysctl_wol_enable, 0, (void *)adapter, 0, CTL_CREATE, 3530 CTL_EOL) != 0) 3531 aprint_error_dev(dev, "could not create sysctl\n"); 3532 3533 if (sysctl_createv(log, 0, &rnode, &cnode, CTLFLAG_READWRITE, 3534 CTLTYPE_INT, "wufc", 3535 SYSCTL_DESCR("Enable/Disable Wake Up Filters"), 3536 ixgbe_sysctl_wufc, 0, (void *)adapter, 0, CTL_CREATE, 3537 CTL_EOL) != 0) 3538 aprint_error_dev(dev, "could not create sysctl\n"); 3539 } 3540 3541 /* for X552/X557-AT devices */ 3542 if (hw->device_id == IXGBE_DEV_ID_X550EM_X_10G_T) { 3543 const struct sysctlnode *phy_node; 3544 3545 if (sysctl_createv(log, 0, &rnode, &phy_node, 0, CTLTYPE_NODE, 3546 "phy", SYSCTL_DESCR("External PHY sysctls"), 3547 NULL, 0, NULL, 0, CTL_CREATE, CTL_EOL) != 0) { 3548 aprint_error_dev(dev, "could not create sysctl\n"); 3549 return; 3550 } 3551 3552 if (sysctl_createv(log, 0, &phy_node, &cnode, CTLFLAG_READONLY, 3553 CTLTYPE_INT, "temp", 3554 SYSCTL_DESCR("Current External PHY Temperature (Celsius)"), 3555 ixgbe_sysctl_phy_temp, 0, (void *)adapter, 0, CTL_CREATE, 3556 CTL_EOL) != 0) 3557 aprint_error_dev(dev, "could not create sysctl\n"); 3558 3559 if (sysctl_createv(log, 0, &phy_node, &cnode, CTLFLAG_READONLY, 3560 CTLTYPE_INT, "overtemp_occurred", 3561 SYSCTL_DESCR( 3562 "External PHY High Temperature Event Occurred"), 3563 ixgbe_sysctl_phy_overtemp_occurred, 0, (void *)adapter, 0, 3564 CTL_CREATE, CTL_EOL) != 0) 3565 aprint_error_dev(dev, "could not create sysctl\n"); 3566 } 3567 3568 if ((hw->mac.type == ixgbe_mac_X550EM_a) 3569 && (hw->phy.type == ixgbe_phy_fw)) 3570 if (sysctl_createv(log, 0, &rnode, &cnode, CTLFLAG_READWRITE, 3571 CTLTYPE_BOOL, "force_10_100_autonego", 3572 SYSCTL_DESCR("Force autonego on 10M and 100M"), 3573 NULL, 0, &hw->phy.force_10_100_autonego, 0, 3574 CTL_CREATE, CTL_EOL) != 0) 3575 aprint_error_dev(dev, "could not create sysctl\n"); 3576 3577 if (adapter->feat_cap & IXGBE_FEATURE_EEE) { 3578 if (sysctl_createv(log, 0, &rnode, &cnode, CTLFLAG_READWRITE, 3579 CTLTYPE_INT, "eee_state", 3580 SYSCTL_DESCR("EEE Power Save State"), 3581 ixgbe_sysctl_eee_state, 0, (void *)adapter, 0, CTL_CREATE, 3582 CTL_EOL) != 0) 3583 aprint_error_dev(dev, "could not create sysctl\n"); 3584 } 3585 } /* ixgbe_add_device_sysctls */ 3586 3587 /************************************************************************ 3588 * ixgbe_allocate_pci_resources 3589 ************************************************************************/ 3590 static int 3591 ixgbe_allocate_pci_resources(struct adapter *adapter, 3592 const struct pci_attach_args *pa) 3593 { 3594 pcireg_t memtype, csr; 3595 device_t dev = adapter->dev; 3596 bus_addr_t addr; 3597 int flags; 3598 3599 memtype = pci_mapreg_type(pa->pa_pc, pa->pa_tag, PCI_BAR(0)); 3600 switch (memtype) { 3601 case PCI_MAPREG_TYPE_MEM | PCI_MAPREG_MEM_TYPE_32BIT: 3602 case PCI_MAPREG_TYPE_MEM | PCI_MAPREG_MEM_TYPE_64BIT: 3603 adapter->osdep.mem_bus_space_tag = pa->pa_memt; 3604 if (pci_mapreg_info(pa->pa_pc, pa->pa_tag, PCI_BAR(0), 3605 memtype, &addr, &adapter->osdep.mem_size, &flags) != 0) 3606 goto map_err; 3607 if ((flags & BUS_SPACE_MAP_PREFETCHABLE) != 0) { 3608 aprint_normal_dev(dev, "clearing prefetchable bit\n"); 3609 flags &= ~BUS_SPACE_MAP_PREFETCHABLE; 3610 } 3611 if (bus_space_map(adapter->osdep.mem_bus_space_tag, addr, 3612 adapter->osdep.mem_size, flags, 3613 &adapter->osdep.mem_bus_space_handle) != 0) { 3614 map_err: 3615 adapter->osdep.mem_size = 0; 3616 aprint_error_dev(dev, "unable to map BAR0\n"); 3617 return ENXIO; 3618 } 3619 /* 3620 * Enable address decoding for memory range in case BIOS or 3621 * UEFI don't set it. 3622 */ 3623 csr = pci_conf_read(pa->pa_pc, pa->pa_tag, 3624 PCI_COMMAND_STATUS_REG); 3625 csr |= PCI_COMMAND_MEM_ENABLE; 3626 pci_conf_write(pa->pa_pc, pa->pa_tag, PCI_COMMAND_STATUS_REG, 3627 csr); 3628 break; 3629 default: 3630 aprint_error_dev(dev, "unexpected type on BAR0\n"); 3631 return ENXIO; 3632 } 3633 3634 return (0); 3635 } /* ixgbe_allocate_pci_resources */ 3636 3637 static void 3638 ixgbe_free_deferred_handlers(struct adapter *adapter) 3639 { 3640 struct ix_queue *que = adapter->queues; 3641 struct tx_ring *txr = adapter->tx_rings; 3642 int i; 3643 3644 for (i = 0; i < adapter->num_queues; i++, que++, txr++) { 3645 if (!(adapter->feat_en & IXGBE_FEATURE_LEGACY_TX)) { 3646 if (txr->txr_si != NULL) 3647 softint_disestablish(txr->txr_si); 3648 } 3649 if (que->que_si != NULL) 3650 softint_disestablish(que->que_si); 3651 } 3652 if (adapter->txr_wq != NULL) 3653 workqueue_destroy(adapter->txr_wq); 3654 if (adapter->txr_wq_enqueued != NULL) 3655 percpu_free(adapter->txr_wq_enqueued, sizeof(u_int)); 3656 if (adapter->que_wq != NULL) 3657 workqueue_destroy(adapter->que_wq); 3658 3659 if (adapter->admin_wq != NULL) { 3660 workqueue_destroy(adapter->admin_wq); 3661 adapter->admin_wq = NULL; 3662 } 3663 if (adapter->timer_wq != NULL) { 3664 workqueue_destroy(adapter->timer_wq); 3665 adapter->timer_wq = NULL; 3666 } 3667 if (adapter->recovery_mode_timer_wq != NULL) { 3668 /* 3669 * ixgbe_ifstop() doesn't call the workqueue_wait() for 3670 * the recovery_mode_timer workqueue, so call it here. 3671 */ 3672 workqueue_wait(adapter->recovery_mode_timer_wq, 3673 &adapter->recovery_mode_timer_wc); 3674 atomic_store_relaxed(&adapter->recovery_mode_timer_pending, 0); 3675 workqueue_destroy(adapter->recovery_mode_timer_wq); 3676 adapter->recovery_mode_timer_wq = NULL; 3677 } 3678 } /* ixgbe_free_deferred_handlers */ 3679 3680 /************************************************************************ 3681 * ixgbe_detach - Device removal routine 3682 * 3683 * Called when the driver is being removed. 3684 * Stops the adapter and deallocates all the resources 3685 * that were allocated for driver operation. 3686 * 3687 * return 0 on success, positive on failure 3688 ************************************************************************/ 3689 static int 3690 ixgbe_detach(device_t dev, int flags) 3691 { 3692 struct adapter *adapter = device_private(dev); 3693 struct rx_ring *rxr = adapter->rx_rings; 3694 struct tx_ring *txr = adapter->tx_rings; 3695 struct ixgbe_hw *hw = &adapter->hw; 3696 struct ixgbe_hw_stats *stats = &adapter->stats.pf; 3697 u32 ctrl_ext; 3698 int i; 3699 3700 INIT_DEBUGOUT("ixgbe_detach: begin"); 3701 if (adapter->osdep.attached == false) 3702 return 0; 3703 3704 if (ixgbe_pci_iov_detach(dev) != 0) { 3705 device_printf(dev, "SR-IOV in use; detach first.\n"); 3706 return (EBUSY); 3707 } 3708 3709 if (VLAN_ATTACHED(&adapter->osdep.ec) && 3710 (flags & (DETACH_SHUTDOWN | DETACH_FORCE)) == 0) { 3711 aprint_error_dev(dev, "VLANs in use, detach first\n"); 3712 return (EBUSY); 3713 } 3714 3715 ether_ifdetach(adapter->ifp); 3716 3717 adapter->osdep.detaching = true; 3718 /* 3719 * Stop the interface. ixgbe_setup_low_power_mode() calls 3720 * ixgbe_ifstop(), so it's not required to call ixgbe_ifstop() 3721 * directly. 3722 */ 3723 ixgbe_setup_low_power_mode(adapter); 3724 3725 callout_halt(&adapter->timer, NULL); 3726 if (adapter->feat_en & IXGBE_FEATURE_RECOVERY_MODE) 3727 callout_halt(&adapter->recovery_mode_timer, NULL); 3728 3729 workqueue_wait(adapter->admin_wq, &adapter->admin_wc); 3730 atomic_store_relaxed(&adapter->admin_pending, 0); 3731 workqueue_wait(adapter->timer_wq, &adapter->timer_wc); 3732 atomic_store_relaxed(&adapter->timer_pending, 0); 3733 3734 pmf_device_deregister(dev); 3735 3736 ixgbe_free_deferred_handlers(adapter); 3737 3738 /* let hardware know driver is unloading */ 3739 ctrl_ext = IXGBE_READ_REG(&adapter->hw, IXGBE_CTRL_EXT); 3740 ctrl_ext &= ~IXGBE_CTRL_EXT_DRV_LOAD; 3741 IXGBE_WRITE_REG(&adapter->hw, IXGBE_CTRL_EXT, ctrl_ext); 3742 3743 if (adapter->feat_en & IXGBE_FEATURE_NETMAP) 3744 netmap_detach(adapter->ifp); 3745 3746 ixgbe_free_pci_resources(adapter); 3747 #if 0 /* XXX the NetBSD port is probably missing something here */ 3748 bus_generic_detach(dev); 3749 #endif 3750 if_detach(adapter->ifp); 3751 ifmedia_fini(&adapter->media); 3752 if_percpuq_destroy(adapter->ipq); 3753 3754 sysctl_teardown(&adapter->sysctllog); 3755 evcnt_detach(&adapter->efbig_tx_dma_setup); 3756 evcnt_detach(&adapter->mbuf_defrag_failed); 3757 evcnt_detach(&adapter->efbig2_tx_dma_setup); 3758 evcnt_detach(&adapter->einval_tx_dma_setup); 3759 evcnt_detach(&adapter->other_tx_dma_setup); 3760 evcnt_detach(&adapter->eagain_tx_dma_setup); 3761 evcnt_detach(&adapter->enomem_tx_dma_setup); 3762 evcnt_detach(&adapter->watchdog_events); 3763 evcnt_detach(&adapter->tso_err); 3764 evcnt_detach(&adapter->admin_irqev); 3765 evcnt_detach(&adapter->link_workev); 3766 evcnt_detach(&adapter->mod_workev); 3767 evcnt_detach(&adapter->msf_workev); 3768 evcnt_detach(&adapter->phy_workev); 3769 3770 for (i = 0; i < IXGBE_TC_COUNTER_NUM; i++) { 3771 if (i < __arraycount(stats->mpc)) { 3772 evcnt_detach(&stats->mpc[i]); 3773 if (hw->mac.type == ixgbe_mac_82598EB) 3774 evcnt_detach(&stats->rnbc[i]); 3775 } 3776 if (i < __arraycount(stats->pxontxc)) { 3777 evcnt_detach(&stats->pxontxc[i]); 3778 evcnt_detach(&stats->pxonrxc[i]); 3779 evcnt_detach(&stats->pxofftxc[i]); 3780 evcnt_detach(&stats->pxoffrxc[i]); 3781 if (hw->mac.type >= ixgbe_mac_82599EB) 3782 evcnt_detach(&stats->pxon2offc[i]); 3783 } 3784 } 3785 3786 txr = adapter->tx_rings; 3787 for (i = 0; i < adapter->num_queues; i++, rxr++, txr++) { 3788 evcnt_detach(&adapter->queues[i].irqs); 3789 evcnt_detach(&adapter->queues[i].handleq); 3790 evcnt_detach(&adapter->queues[i].req); 3791 evcnt_detach(&txr->no_desc_avail); 3792 evcnt_detach(&txr->total_packets); 3793 evcnt_detach(&txr->tso_tx); 3794 #ifndef IXGBE_LEGACY_TX 3795 evcnt_detach(&txr->pcq_drops); 3796 #endif 3797 3798 if (i < __arraycount(stats->qprc)) { 3799 evcnt_detach(&stats->qprc[i]); 3800 evcnt_detach(&stats->qptc[i]); 3801 evcnt_detach(&stats->qbrc[i]); 3802 evcnt_detach(&stats->qbtc[i]); 3803 if (hw->mac.type >= ixgbe_mac_82599EB) 3804 evcnt_detach(&stats->qprdc[i]); 3805 } 3806 3807 evcnt_detach(&rxr->rx_packets); 3808 evcnt_detach(&rxr->rx_bytes); 3809 evcnt_detach(&rxr->rx_copies); 3810 evcnt_detach(&rxr->no_mbuf); 3811 evcnt_detach(&rxr->rx_discarded); 3812 } 3813 evcnt_detach(&stats->ipcs); 3814 evcnt_detach(&stats->l4cs); 3815 evcnt_detach(&stats->ipcs_bad); 3816 evcnt_detach(&stats->l4cs_bad); 3817 evcnt_detach(&stats->intzero); 3818 evcnt_detach(&stats->legint); 3819 evcnt_detach(&stats->crcerrs); 3820 evcnt_detach(&stats->illerrc); 3821 evcnt_detach(&stats->errbc); 3822 evcnt_detach(&stats->mspdc); 3823 if (hw->mac.type >= ixgbe_mac_X550) 3824 evcnt_detach(&stats->mbsdc); 3825 evcnt_detach(&stats->mpctotal); 3826 evcnt_detach(&stats->mlfc); 3827 evcnt_detach(&stats->mrfc); 3828 evcnt_detach(&stats->rlec); 3829 evcnt_detach(&stats->lxontxc); 3830 evcnt_detach(&stats->lxonrxc); 3831 evcnt_detach(&stats->lxofftxc); 3832 evcnt_detach(&stats->lxoffrxc); 3833 3834 /* Packet Reception Stats */ 3835 evcnt_detach(&stats->tor); 3836 evcnt_detach(&stats->gorc); 3837 evcnt_detach(&stats->tpr); 3838 evcnt_detach(&stats->gprc); 3839 evcnt_detach(&stats->mprc); 3840 evcnt_detach(&stats->bprc); 3841 evcnt_detach(&stats->prc64); 3842 evcnt_detach(&stats->prc127); 3843 evcnt_detach(&stats->prc255); 3844 evcnt_detach(&stats->prc511); 3845 evcnt_detach(&stats->prc1023); 3846 evcnt_detach(&stats->prc1522); 3847 evcnt_detach(&stats->ruc); 3848 evcnt_detach(&stats->rfc); 3849 evcnt_detach(&stats->roc); 3850 evcnt_detach(&stats->rjc); 3851 evcnt_detach(&stats->mngprc); 3852 evcnt_detach(&stats->mngpdc); 3853 evcnt_detach(&stats->xec); 3854 3855 /* Packet Transmission Stats */ 3856 evcnt_detach(&stats->gotc); 3857 evcnt_detach(&stats->tpt); 3858 evcnt_detach(&stats->gptc); 3859 evcnt_detach(&stats->bptc); 3860 evcnt_detach(&stats->mptc); 3861 evcnt_detach(&stats->mngptc); 3862 evcnt_detach(&stats->ptc64); 3863 evcnt_detach(&stats->ptc127); 3864 evcnt_detach(&stats->ptc255); 3865 evcnt_detach(&stats->ptc511); 3866 evcnt_detach(&stats->ptc1023); 3867 evcnt_detach(&stats->ptc1522); 3868 3869 ixgbe_free_queues(adapter); 3870 free(adapter->mta, M_DEVBUF); 3871 3872 mutex_destroy(&adapter->admin_mtx); /* XXX appropriate order? */ 3873 IXGBE_CORE_LOCK_DESTROY(adapter); 3874 3875 return (0); 3876 } /* ixgbe_detach */ 3877 3878 /************************************************************************ 3879 * ixgbe_setup_low_power_mode - LPLU/WoL preparation 3880 * 3881 * Prepare the adapter/port for LPLU and/or WoL 3882 ************************************************************************/ 3883 static int 3884 ixgbe_setup_low_power_mode(struct adapter *adapter) 3885 { 3886 struct ixgbe_hw *hw = &adapter->hw; 3887 device_t dev = adapter->dev; 3888 struct ifnet *ifp = adapter->ifp; 3889 s32 error = 0; 3890 3891 /* Limit power management flow to X550EM baseT */ 3892 if (hw->device_id == IXGBE_DEV_ID_X550EM_X_10G_T && 3893 hw->phy.ops.enter_lplu) { 3894 /* X550EM baseT adapters need a special LPLU flow */ 3895 hw->phy.reset_disable = true; 3896 ixgbe_ifstop(ifp, 1); 3897 error = hw->phy.ops.enter_lplu(hw); 3898 if (error) 3899 device_printf(dev, 3900 "Error entering LPLU: %d\n", error); 3901 hw->phy.reset_disable = false; 3902 } else { 3903 /* Just stop for other adapters */ 3904 ixgbe_ifstop(ifp, 1); 3905 } 3906 3907 IXGBE_CORE_LOCK(adapter); 3908 3909 if (!hw->wol_enabled) { 3910 ixgbe_set_phy_power(hw, FALSE); 3911 IXGBE_WRITE_REG(hw, IXGBE_WUFC, 0); 3912 IXGBE_WRITE_REG(hw, IXGBE_WUC, 0); 3913 } else { 3914 /* Turn off support for APM wakeup. (Using ACPI instead) */ 3915 IXGBE_WRITE_REG(hw, IXGBE_GRC_BY_MAC(hw), 3916 IXGBE_READ_REG(hw, IXGBE_GRC_BY_MAC(hw)) & ~(u32)2); 3917 3918 /* 3919 * Clear Wake Up Status register to prevent any previous wakeup 3920 * events from waking us up immediately after we suspend. 3921 */ 3922 IXGBE_WRITE_REG(hw, IXGBE_WUS, 0xffffffff); 3923 3924 /* 3925 * Program the Wakeup Filter Control register with user filter 3926 * settings 3927 */ 3928 IXGBE_WRITE_REG(hw, IXGBE_WUFC, adapter->wufc); 3929 3930 /* Enable wakeups and power management in Wakeup Control */ 3931 IXGBE_WRITE_REG(hw, IXGBE_WUC, 3932 IXGBE_WUC_WKEN | IXGBE_WUC_PME_EN); 3933 3934 } 3935 3936 IXGBE_CORE_UNLOCK(adapter); 3937 3938 return error; 3939 } /* ixgbe_setup_low_power_mode */ 3940 3941 /************************************************************************ 3942 * ixgbe_shutdown - Shutdown entry point 3943 ************************************************************************/ 3944 #if 0 /* XXX NetBSD ought to register something like this through pmf(9) */ 3945 static int 3946 ixgbe_shutdown(device_t dev) 3947 { 3948 struct adapter *adapter = device_private(dev); 3949 int error = 0; 3950 3951 INIT_DEBUGOUT("ixgbe_shutdown: begin"); 3952 3953 error = ixgbe_setup_low_power_mode(adapter); 3954 3955 return (error); 3956 } /* ixgbe_shutdown */ 3957 #endif 3958 3959 /************************************************************************ 3960 * ixgbe_suspend 3961 * 3962 * From D0 to D3 3963 ************************************************************************/ 3964 static bool 3965 ixgbe_suspend(device_t dev, const pmf_qual_t *qual) 3966 { 3967 struct adapter *adapter = device_private(dev); 3968 int error = 0; 3969 3970 INIT_DEBUGOUT("ixgbe_suspend: begin"); 3971 3972 error = ixgbe_setup_low_power_mode(adapter); 3973 3974 return (error); 3975 } /* ixgbe_suspend */ 3976 3977 /************************************************************************ 3978 * ixgbe_resume 3979 * 3980 * From D3 to D0 3981 ************************************************************************/ 3982 static bool 3983 ixgbe_resume(device_t dev, const pmf_qual_t *qual) 3984 { 3985 struct adapter *adapter = device_private(dev); 3986 struct ifnet *ifp = adapter->ifp; 3987 struct ixgbe_hw *hw = &adapter->hw; 3988 u32 wus; 3989 3990 INIT_DEBUGOUT("ixgbe_resume: begin"); 3991 3992 IXGBE_CORE_LOCK(adapter); 3993 3994 /* Read & clear WUS register */ 3995 wus = IXGBE_READ_REG(hw, IXGBE_WUS); 3996 if (wus) 3997 device_printf(dev, "Woken up by (WUS): %#010x\n", 3998 IXGBE_READ_REG(hw, IXGBE_WUS)); 3999 IXGBE_WRITE_REG(hw, IXGBE_WUS, 0xffffffff); 4000 /* And clear WUFC until next low-power transition */ 4001 IXGBE_WRITE_REG(hw, IXGBE_WUFC, 0); 4002 4003 /* 4004 * Required after D3->D0 transition; 4005 * will re-advertise all previous advertised speeds 4006 */ 4007 if (ifp->if_flags & IFF_UP) 4008 ixgbe_init_locked(adapter); 4009 4010 IXGBE_CORE_UNLOCK(adapter); 4011 4012 return true; 4013 } /* ixgbe_resume */ 4014 4015 /* 4016 * Set the various hardware offload abilities. 4017 * 4018 * This takes the ifnet's if_capenable flags (e.g. set by the user using 4019 * ifconfig) and indicates to the OS via the ifnet's if_hwassist field what 4020 * mbuf offload flags the driver will understand. 4021 */ 4022 static void 4023 ixgbe_set_if_hwassist(struct adapter *adapter) 4024 { 4025 /* XXX */ 4026 } 4027 4028 /************************************************************************ 4029 * ixgbe_init_locked - Init entry point 4030 * 4031 * Used in two ways: It is used by the stack as an init 4032 * entry point in network interface structure. It is also 4033 * used by the driver as a hw/sw initialization routine to 4034 * get to a consistent state. 4035 * 4036 * return 0 on success, positive on failure 4037 ************************************************************************/ 4038 static void 4039 ixgbe_init_locked(struct adapter *adapter) 4040 { 4041 struct ifnet *ifp = adapter->ifp; 4042 device_t dev = adapter->dev; 4043 struct ixgbe_hw *hw = &adapter->hw; 4044 struct ix_queue *que; 4045 struct tx_ring *txr; 4046 struct rx_ring *rxr; 4047 u32 txdctl, mhadd; 4048 u32 rxdctl, rxctrl; 4049 u32 ctrl_ext; 4050 bool unsupported_sfp = false; 4051 int i, j, error; 4052 4053 /* XXX check IFF_UP and IFF_RUNNING, power-saving state! */ 4054 4055 KASSERT(mutex_owned(&adapter->core_mtx)); 4056 INIT_DEBUGOUT("ixgbe_init_locked: begin"); 4057 4058 hw->need_unsupported_sfp_recovery = false; 4059 hw->adapter_stopped = FALSE; 4060 ixgbe_stop_adapter(hw); 4061 callout_stop(&adapter->timer); 4062 if (adapter->feat_en & IXGBE_FEATURE_RECOVERY_MODE) 4063 callout_stop(&adapter->recovery_mode_timer); 4064 for (i = 0, que = adapter->queues; i < adapter->num_queues; i++, que++) 4065 que->disabled_count = 0; 4066 4067 /* XXX I moved this here from the SIOCSIFMTU case in ixgbe_ioctl(). */ 4068 adapter->max_frame_size = 4069 ifp->if_mtu + ETHER_HDR_LEN + ETHER_CRC_LEN; 4070 4071 /* Queue indices may change with IOV mode */ 4072 ixgbe_align_all_queue_indices(adapter); 4073 4074 /* reprogram the RAR[0] in case user changed it. */ 4075 ixgbe_set_rar(hw, 0, hw->mac.addr, adapter->pool, IXGBE_RAH_AV); 4076 4077 /* Get the latest mac address, User can use a LAA */ 4078 memcpy(hw->mac.addr, CLLADDR(ifp->if_sadl), 4079 IXGBE_ETH_LENGTH_OF_ADDRESS); 4080 ixgbe_set_rar(hw, 0, hw->mac.addr, adapter->pool, 1); 4081 hw->addr_ctrl.rar_used_count = 1; 4082 4083 /* Set hardware offload abilities from ifnet flags */ 4084 ixgbe_set_if_hwassist(adapter); 4085 4086 /* Prepare transmit descriptors and buffers */ 4087 if (ixgbe_setup_transmit_structures(adapter)) { 4088 device_printf(dev, "Could not setup transmit structures\n"); 4089 ixgbe_stop_locked(adapter); 4090 return; 4091 } 4092 4093 ixgbe_init_hw(hw); 4094 4095 ixgbe_initialize_iov(adapter); 4096 4097 ixgbe_initialize_transmit_units(adapter); 4098 4099 /* Setup Multicast table */ 4100 ixgbe_set_rxfilter(adapter); 4101 4102 /* Use fixed buffer size, even for jumbo frames */ 4103 adapter->rx_mbuf_sz = MCLBYTES; 4104 4105 /* Prepare receive descriptors and buffers */ 4106 error = ixgbe_setup_receive_structures(adapter); 4107 if (error) { 4108 device_printf(dev, 4109 "Could not setup receive structures (err = %d)\n", error); 4110 ixgbe_stop_locked(adapter); 4111 return; 4112 } 4113 4114 /* Configure RX settings */ 4115 ixgbe_initialize_receive_units(adapter); 4116 4117 /* Initialize variable holding task enqueue requests interrupts */ 4118 adapter->task_requests = 0; 4119 4120 /* Enable SDP & MSI-X interrupts based on adapter */ 4121 ixgbe_config_gpie(adapter); 4122 4123 /* Set MTU size */ 4124 if (ifp->if_mtu > ETHERMTU) { 4125 /* aka IXGBE_MAXFRS on 82599 and newer */ 4126 mhadd = IXGBE_READ_REG(hw, IXGBE_MHADD); 4127 mhadd &= ~IXGBE_MHADD_MFS_MASK; 4128 mhadd |= adapter->max_frame_size << IXGBE_MHADD_MFS_SHIFT; 4129 IXGBE_WRITE_REG(hw, IXGBE_MHADD, mhadd); 4130 } 4131 4132 /* Now enable all the queues */ 4133 for (i = 0; i < adapter->num_queues; i++) { 4134 txr = &adapter->tx_rings[i]; 4135 txdctl = IXGBE_READ_REG(hw, IXGBE_TXDCTL(txr->me)); 4136 txdctl |= IXGBE_TXDCTL_ENABLE; 4137 /* Set WTHRESH to 8, burst writeback */ 4138 txdctl |= IXGBE_TX_WTHRESH << IXGBE_TXDCTL_WTHRESH_SHIFT; 4139 /* 4140 * When the internal queue falls below PTHRESH (32), 4141 * start prefetching as long as there are at least 4142 * HTHRESH (1) buffers ready. The values are taken 4143 * from the Intel linux driver 3.8.21. 4144 * Prefetching enables tx line rate even with 1 queue. 4145 */ 4146 txdctl |= (32 << 0) | (1 << 8); 4147 IXGBE_WRITE_REG(hw, IXGBE_TXDCTL(txr->me), txdctl); 4148 } 4149 4150 for (i = 0; i < adapter->num_queues; i++) { 4151 rxr = &adapter->rx_rings[i]; 4152 rxdctl = IXGBE_READ_REG(hw, IXGBE_RXDCTL(rxr->me)); 4153 if (hw->mac.type == ixgbe_mac_82598EB) { 4154 /* 4155 * PTHRESH = 21 4156 * HTHRESH = 4 4157 * WTHRESH = 8 4158 */ 4159 rxdctl &= ~0x3FFFFF; 4160 rxdctl |= 0x080420; 4161 } 4162 rxdctl |= IXGBE_RXDCTL_ENABLE; 4163 IXGBE_WRITE_REG(hw, IXGBE_RXDCTL(rxr->me), rxdctl); 4164 for (j = 0; j < 10; j++) { 4165 if (IXGBE_READ_REG(hw, IXGBE_RXDCTL(rxr->me)) & 4166 IXGBE_RXDCTL_ENABLE) 4167 break; 4168 else 4169 msec_delay(1); 4170 } 4171 IXGBE_WRITE_BARRIER(hw); 4172 4173 /* 4174 * In netmap mode, we must preserve the buffers made 4175 * available to userspace before the if_init() 4176 * (this is true by default on the TX side, because 4177 * init makes all buffers available to userspace). 4178 * 4179 * netmap_reset() and the device specific routines 4180 * (e.g. ixgbe_setup_receive_rings()) map these 4181 * buffers at the end of the NIC ring, so here we 4182 * must set the RDT (tail) register to make sure 4183 * they are not overwritten. 4184 * 4185 * In this driver the NIC ring starts at RDH = 0, 4186 * RDT points to the last slot available for reception (?), 4187 * so RDT = num_rx_desc - 1 means the whole ring is available. 4188 */ 4189 #ifdef DEV_NETMAP 4190 if ((adapter->feat_en & IXGBE_FEATURE_NETMAP) && 4191 (ifp->if_capenable & IFCAP_NETMAP)) { 4192 struct netmap_adapter *na = NA(adapter->ifp); 4193 struct netmap_kring *kring = na->rx_rings[i]; 4194 int t = na->num_rx_desc - 1 - nm_kr_rxspace(kring); 4195 4196 IXGBE_WRITE_REG(hw, IXGBE_RDT(rxr->me), t); 4197 } else 4198 #endif /* DEV_NETMAP */ 4199 IXGBE_WRITE_REG(hw, IXGBE_RDT(rxr->me), 4200 adapter->num_rx_desc - 1); 4201 } 4202 4203 /* Enable Receive engine */ 4204 rxctrl = IXGBE_READ_REG(hw, IXGBE_RXCTRL); 4205 if (hw->mac.type == ixgbe_mac_82598EB) 4206 rxctrl |= IXGBE_RXCTRL_DMBYPS; 4207 rxctrl |= IXGBE_RXCTRL_RXEN; 4208 ixgbe_enable_rx_dma(hw, rxctrl); 4209 4210 callout_reset(&adapter->timer, hz, ixgbe_local_timer, adapter); 4211 atomic_store_relaxed(&adapter->timer_pending, 0); 4212 if (adapter->feat_en & IXGBE_FEATURE_RECOVERY_MODE) 4213 callout_reset(&adapter->recovery_mode_timer, hz, 4214 ixgbe_recovery_mode_timer, adapter); 4215 4216 /* Set up MSI/MSI-X routing */ 4217 if (adapter->feat_en & IXGBE_FEATURE_MSIX) { 4218 ixgbe_configure_ivars(adapter); 4219 /* Set up auto-mask */ 4220 if (hw->mac.type == ixgbe_mac_82598EB) 4221 IXGBE_WRITE_REG(hw, IXGBE_EIAM, IXGBE_EICS_RTX_QUEUE); 4222 else { 4223 IXGBE_WRITE_REG(hw, IXGBE_EIAM_EX(0), 0xFFFFFFFF); 4224 IXGBE_WRITE_REG(hw, IXGBE_EIAM_EX(1), 0xFFFFFFFF); 4225 } 4226 } else { /* Simple settings for Legacy/MSI */ 4227 ixgbe_set_ivar(adapter, 0, 0, 0); 4228 ixgbe_set_ivar(adapter, 0, 0, 1); 4229 IXGBE_WRITE_REG(hw, IXGBE_EIAM, IXGBE_EICS_RTX_QUEUE); 4230 } 4231 4232 ixgbe_init_fdir(adapter); 4233 4234 /* 4235 * Check on any SFP devices that 4236 * need to be kick-started 4237 */ 4238 if (hw->phy.type == ixgbe_phy_none) { 4239 error = hw->phy.ops.identify(hw); 4240 if (error == IXGBE_ERR_SFP_NOT_SUPPORTED) 4241 unsupported_sfp = true; 4242 } else if (hw->phy.type == ixgbe_phy_sfp_unsupported) 4243 unsupported_sfp = true; 4244 4245 if (unsupported_sfp) 4246 device_printf(dev, 4247 "Unsupported SFP+ module type was detected.\n"); 4248 4249 /* Set moderation on the Link interrupt */ 4250 ixgbe_eitr_write(adapter, adapter->vector, IXGBE_LINK_ITR); 4251 4252 /* Enable EEE power saving */ 4253 if (adapter->feat_cap & IXGBE_FEATURE_EEE) 4254 hw->mac.ops.setup_eee(hw, 4255 adapter->feat_en & IXGBE_FEATURE_EEE); 4256 4257 /* Enable power to the phy. */ 4258 if (!unsupported_sfp) { 4259 ixgbe_set_phy_power(hw, TRUE); 4260 4261 /* Config/Enable Link */ 4262 ixgbe_config_link(adapter); 4263 } 4264 4265 /* Hardware Packet Buffer & Flow Control setup */ 4266 ixgbe_config_delay_values(adapter); 4267 4268 /* Initialize the FC settings */ 4269 ixgbe_start_hw(hw); 4270 4271 /* Set up VLAN support and filter */ 4272 ixgbe_setup_vlan_hw_support(adapter); 4273 4274 /* Setup DMA Coalescing */ 4275 ixgbe_config_dmac(adapter); 4276 4277 /* OK to schedule workqueues. */ 4278 adapter->schedule_wqs_ok = true; 4279 4280 /* And now turn on interrupts */ 4281 ixgbe_enable_intr(adapter); 4282 4283 /* Enable the use of the MBX by the VF's */ 4284 if (adapter->feat_en & IXGBE_FEATURE_SRIOV) { 4285 ctrl_ext = IXGBE_READ_REG(hw, IXGBE_CTRL_EXT); 4286 ctrl_ext |= IXGBE_CTRL_EXT_PFRSTD; 4287 IXGBE_WRITE_REG(hw, IXGBE_CTRL_EXT, ctrl_ext); 4288 } 4289 4290 /* Update saved flags. See ixgbe_ifflags_cb() */ 4291 adapter->if_flags = ifp->if_flags; 4292 adapter->ec_capenable = adapter->osdep.ec.ec_capenable; 4293 4294 /* Now inform the stack we're ready */ 4295 ifp->if_flags |= IFF_RUNNING; 4296 4297 return; 4298 } /* ixgbe_init_locked */ 4299 4300 /************************************************************************ 4301 * ixgbe_init 4302 ************************************************************************/ 4303 static int 4304 ixgbe_init(struct ifnet *ifp) 4305 { 4306 struct adapter *adapter = ifp->if_softc; 4307 4308 IXGBE_CORE_LOCK(adapter); 4309 ixgbe_init_locked(adapter); 4310 IXGBE_CORE_UNLOCK(adapter); 4311 4312 return 0; /* XXX ixgbe_init_locked cannot fail? really? */ 4313 } /* ixgbe_init */ 4314 4315 /************************************************************************ 4316 * ixgbe_set_ivar 4317 * 4318 * Setup the correct IVAR register for a particular MSI-X interrupt 4319 * (yes this is all very magic and confusing :) 4320 * - entry is the register array entry 4321 * - vector is the MSI-X vector for this queue 4322 * - type is RX/TX/MISC 4323 ************************************************************************/ 4324 static void 4325 ixgbe_set_ivar(struct adapter *adapter, u8 entry, u8 vector, s8 type) 4326 { 4327 struct ixgbe_hw *hw = &adapter->hw; 4328 u32 ivar, index; 4329 4330 vector |= IXGBE_IVAR_ALLOC_VAL; 4331 4332 switch (hw->mac.type) { 4333 case ixgbe_mac_82598EB: 4334 if (type == -1) 4335 entry = IXGBE_IVAR_OTHER_CAUSES_INDEX; 4336 else 4337 entry += (type * 64); 4338 index = (entry >> 2) & 0x1F; 4339 ivar = IXGBE_READ_REG(hw, IXGBE_IVAR(index)); 4340 ivar &= ~(0xffUL << (8 * (entry & 0x3))); 4341 ivar |= ((u32)vector << (8 * (entry & 0x3))); 4342 IXGBE_WRITE_REG(&adapter->hw, IXGBE_IVAR(index), ivar); 4343 break; 4344 case ixgbe_mac_82599EB: 4345 case ixgbe_mac_X540: 4346 case ixgbe_mac_X550: 4347 case ixgbe_mac_X550EM_x: 4348 case ixgbe_mac_X550EM_a: 4349 if (type == -1) { /* MISC IVAR */ 4350 index = (entry & 1) * 8; 4351 ivar = IXGBE_READ_REG(hw, IXGBE_IVAR_MISC); 4352 ivar &= ~(0xffUL << index); 4353 ivar |= ((u32)vector << index); 4354 IXGBE_WRITE_REG(hw, IXGBE_IVAR_MISC, ivar); 4355 } else { /* RX/TX IVARS */ 4356 index = (16 * (entry & 1)) + (8 * type); 4357 ivar = IXGBE_READ_REG(hw, IXGBE_IVAR(entry >> 1)); 4358 ivar &= ~(0xffUL << index); 4359 ivar |= ((u32)vector << index); 4360 IXGBE_WRITE_REG(hw, IXGBE_IVAR(entry >> 1), ivar); 4361 } 4362 break; 4363 default: 4364 break; 4365 } 4366 } /* ixgbe_set_ivar */ 4367 4368 /************************************************************************ 4369 * ixgbe_configure_ivars 4370 ************************************************************************/ 4371 static void 4372 ixgbe_configure_ivars(struct adapter *adapter) 4373 { 4374 struct ix_queue *que = adapter->queues; 4375 u32 newitr; 4376 4377 if (ixgbe_max_interrupt_rate > 0) 4378 newitr = (4000000 / ixgbe_max_interrupt_rate) & 0x0FF8; 4379 else { 4380 /* 4381 * Disable DMA coalescing if interrupt moderation is 4382 * disabled. 4383 */ 4384 adapter->dmac = 0; 4385 newitr = 0; 4386 } 4387 4388 for (int i = 0; i < adapter->num_queues; i++, que++) { 4389 struct rx_ring *rxr = &adapter->rx_rings[i]; 4390 struct tx_ring *txr = &adapter->tx_rings[i]; 4391 /* First the RX queue entry */ 4392 ixgbe_set_ivar(adapter, rxr->me, que->msix, 0); 4393 /* ... and the TX */ 4394 ixgbe_set_ivar(adapter, txr->me, que->msix, 1); 4395 /* Set an Initial EITR value */ 4396 ixgbe_eitr_write(adapter, que->msix, newitr); 4397 /* 4398 * To eliminate influence of the previous state. 4399 * At this point, Tx/Rx interrupt handler 4400 * (ixgbe_msix_que()) cannot be called, so both 4401 * IXGBE_TX_LOCK and IXGBE_RX_LOCK are not required. 4402 */ 4403 que->eitr_setting = 0; 4404 } 4405 4406 /* For the Link interrupt */ 4407 ixgbe_set_ivar(adapter, 1, adapter->vector, -1); 4408 } /* ixgbe_configure_ivars */ 4409 4410 /************************************************************************ 4411 * ixgbe_config_gpie 4412 ************************************************************************/ 4413 static void 4414 ixgbe_config_gpie(struct adapter *adapter) 4415 { 4416 struct ixgbe_hw *hw = &adapter->hw; 4417 u32 gpie; 4418 4419 gpie = IXGBE_READ_REG(hw, IXGBE_GPIE); 4420 4421 if (adapter->feat_en & IXGBE_FEATURE_MSIX) { 4422 /* Enable Enhanced MSI-X mode */ 4423 gpie |= IXGBE_GPIE_MSIX_MODE 4424 | IXGBE_GPIE_EIAME 4425 | IXGBE_GPIE_PBA_SUPPORT 4426 | IXGBE_GPIE_OCD; 4427 } 4428 4429 /* Fan Failure Interrupt */ 4430 if (adapter->feat_en & IXGBE_FEATURE_FAN_FAIL) 4431 gpie |= IXGBE_SDP1_GPIEN; 4432 4433 /* Thermal Sensor Interrupt */ 4434 if (adapter->feat_en & IXGBE_FEATURE_TEMP_SENSOR) 4435 gpie |= IXGBE_SDP0_GPIEN_X540; 4436 4437 /* Link detection */ 4438 switch (hw->mac.type) { 4439 case ixgbe_mac_82599EB: 4440 gpie |= IXGBE_SDP1_GPIEN | IXGBE_SDP2_GPIEN; 4441 break; 4442 case ixgbe_mac_X550EM_x: 4443 case ixgbe_mac_X550EM_a: 4444 gpie |= IXGBE_SDP0_GPIEN_X540; 4445 break; 4446 default: 4447 break; 4448 } 4449 4450 IXGBE_WRITE_REG(hw, IXGBE_GPIE, gpie); 4451 4452 } /* ixgbe_config_gpie */ 4453 4454 /************************************************************************ 4455 * ixgbe_config_delay_values 4456 * 4457 * Requires adapter->max_frame_size to be set. 4458 ************************************************************************/ 4459 static void 4460 ixgbe_config_delay_values(struct adapter *adapter) 4461 { 4462 struct ixgbe_hw *hw = &adapter->hw; 4463 u32 rxpb, frame, size, tmp; 4464 4465 frame = adapter->max_frame_size; 4466 4467 /* Calculate High Water */ 4468 switch (hw->mac.type) { 4469 case ixgbe_mac_X540: 4470 case ixgbe_mac_X550: 4471 case ixgbe_mac_X550EM_x: 4472 case ixgbe_mac_X550EM_a: 4473 tmp = IXGBE_DV_X540(frame, frame); 4474 break; 4475 default: 4476 tmp = IXGBE_DV(frame, frame); 4477 break; 4478 } 4479 size = IXGBE_BT2KB(tmp); 4480 rxpb = IXGBE_READ_REG(hw, IXGBE_RXPBSIZE(0)) >> 10; 4481 hw->fc.high_water[0] = rxpb - size; 4482 4483 /* Now calculate Low Water */ 4484 switch (hw->mac.type) { 4485 case ixgbe_mac_X540: 4486 case ixgbe_mac_X550: 4487 case ixgbe_mac_X550EM_x: 4488 case ixgbe_mac_X550EM_a: 4489 tmp = IXGBE_LOW_DV_X540(frame); 4490 break; 4491 default: 4492 tmp = IXGBE_LOW_DV(frame); 4493 break; 4494 } 4495 hw->fc.low_water[0] = IXGBE_BT2KB(tmp); 4496 4497 hw->fc.pause_time = IXGBE_FC_PAUSE; 4498 hw->fc.send_xon = TRUE; 4499 } /* ixgbe_config_delay_values */ 4500 4501 /************************************************************************ 4502 * ixgbe_set_rxfilter - Multicast Update 4503 * 4504 * Called whenever multicast address list is updated. 4505 ************************************************************************/ 4506 static void 4507 ixgbe_set_rxfilter(struct adapter *adapter) 4508 { 4509 struct ixgbe_mc_addr *mta; 4510 struct ifnet *ifp = adapter->ifp; 4511 u8 *update_ptr; 4512 int mcnt = 0; 4513 u32 fctrl; 4514 struct ethercom *ec = &adapter->osdep.ec; 4515 struct ether_multi *enm; 4516 struct ether_multistep step; 4517 4518 KASSERT(mutex_owned(&adapter->core_mtx)); 4519 IOCTL_DEBUGOUT("ixgbe_set_rxfilter: begin"); 4520 4521 mta = adapter->mta; 4522 bzero(mta, sizeof(*mta) * MAX_NUM_MULTICAST_ADDRESSES); 4523 4524 ETHER_LOCK(ec); 4525 ec->ec_flags &= ~ETHER_F_ALLMULTI; 4526 ETHER_FIRST_MULTI(step, ec, enm); 4527 while (enm != NULL) { 4528 if ((mcnt == MAX_NUM_MULTICAST_ADDRESSES) || 4529 (memcmp(enm->enm_addrlo, enm->enm_addrhi, 4530 ETHER_ADDR_LEN) != 0)) { 4531 ec->ec_flags |= ETHER_F_ALLMULTI; 4532 break; 4533 } 4534 bcopy(enm->enm_addrlo, 4535 mta[mcnt].addr, IXGBE_ETH_LENGTH_OF_ADDRESS); 4536 mta[mcnt].vmdq = adapter->pool; 4537 mcnt++; 4538 ETHER_NEXT_MULTI(step, enm); 4539 } 4540 4541 fctrl = IXGBE_READ_REG(&adapter->hw, IXGBE_FCTRL); 4542 if (ifp->if_flags & IFF_PROMISC) 4543 fctrl |= (IXGBE_FCTRL_UPE | IXGBE_FCTRL_MPE); 4544 else if (ec->ec_flags & ETHER_F_ALLMULTI) { 4545 fctrl |= IXGBE_FCTRL_MPE; 4546 fctrl &= ~IXGBE_FCTRL_UPE; 4547 } else 4548 fctrl &= ~(IXGBE_FCTRL_UPE | IXGBE_FCTRL_MPE); 4549 4550 IXGBE_WRITE_REG(&adapter->hw, IXGBE_FCTRL, fctrl); 4551 4552 /* Update multicast filter entries only when it's not ALLMULTI */ 4553 if ((ec->ec_flags & ETHER_F_ALLMULTI) == 0) { 4554 ETHER_UNLOCK(ec); 4555 update_ptr = (u8 *)mta; 4556 ixgbe_update_mc_addr_list(&adapter->hw, update_ptr, mcnt, 4557 ixgbe_mc_array_itr, TRUE); 4558 } else 4559 ETHER_UNLOCK(ec); 4560 } /* ixgbe_set_rxfilter */ 4561 4562 /************************************************************************ 4563 * ixgbe_mc_array_itr 4564 * 4565 * An iterator function needed by the multicast shared code. 4566 * It feeds the shared code routine the addresses in the 4567 * array of ixgbe_set_rxfilter() one by one. 4568 ************************************************************************/ 4569 static u8 * 4570 ixgbe_mc_array_itr(struct ixgbe_hw *hw, u8 **update_ptr, u32 *vmdq) 4571 { 4572 struct ixgbe_mc_addr *mta; 4573 4574 mta = (struct ixgbe_mc_addr *)*update_ptr; 4575 *vmdq = mta->vmdq; 4576 4577 *update_ptr = (u8*)(mta + 1); 4578 4579 return (mta->addr); 4580 } /* ixgbe_mc_array_itr */ 4581 4582 /************************************************************************ 4583 * ixgbe_local_timer - Timer routine 4584 * 4585 * Checks for link status, updates statistics, 4586 * and runs the watchdog check. 4587 ************************************************************************/ 4588 static void 4589 ixgbe_local_timer(void *arg) 4590 { 4591 struct adapter *adapter = arg; 4592 4593 if (adapter->schedule_wqs_ok) { 4594 if (atomic_cas_uint(&adapter->timer_pending, 0, 1) == 0) 4595 workqueue_enqueue(adapter->timer_wq, 4596 &adapter->timer_wc, NULL); 4597 } 4598 } 4599 4600 static void 4601 ixgbe_handle_timer(struct work *wk, void *context) 4602 { 4603 struct adapter *adapter = context; 4604 struct ixgbe_hw *hw = &adapter->hw; 4605 device_t dev = adapter->dev; 4606 struct ix_queue *que = adapter->queues; 4607 u64 queues = 0; 4608 u64 v0, v1, v2, v3, v4, v5, v6, v7; 4609 int hung = 0; 4610 int i; 4611 4612 IXGBE_CORE_LOCK(adapter); 4613 4614 /* Check for pluggable optics */ 4615 if (ixgbe_is_sfp(hw)) { 4616 bool sched_mod_task = false; 4617 4618 if (hw->mac.type == ixgbe_mac_82598EB) { 4619 /* 4620 * On 82598EB, SFP+'s MOD_ABS pin is not connected to 4621 * any GPIO(SDP). So just schedule TASK_MOD. 4622 */ 4623 sched_mod_task = true; 4624 } else { 4625 bool was_full, is_full; 4626 4627 was_full = 4628 hw->phy.sfp_type != ixgbe_sfp_type_not_present; 4629 is_full = ixgbe_sfp_cage_full(hw); 4630 4631 /* Do probe if cage state changed */ 4632 if (was_full ^ is_full) 4633 sched_mod_task = true; 4634 } 4635 if (sched_mod_task) { 4636 mutex_enter(&adapter->admin_mtx); 4637 adapter->task_requests |= IXGBE_REQUEST_TASK_MOD_WOI; 4638 ixgbe_schedule_admin_tasklet(adapter); 4639 mutex_exit(&adapter->admin_mtx); 4640 } 4641 } 4642 4643 ixgbe_update_link_status(adapter); 4644 ixgbe_update_stats_counters(adapter); 4645 4646 /* Update some event counters */ 4647 v0 = v1 = v2 = v3 = v4 = v5 = v6 = v7 = 0; 4648 que = adapter->queues; 4649 for (i = 0; i < adapter->num_queues; i++, que++) { 4650 struct tx_ring *txr = que->txr; 4651 4652 v0 += txr->q_efbig_tx_dma_setup; 4653 v1 += txr->q_mbuf_defrag_failed; 4654 v2 += txr->q_efbig2_tx_dma_setup; 4655 v3 += txr->q_einval_tx_dma_setup; 4656 v4 += txr->q_other_tx_dma_setup; 4657 v5 += txr->q_eagain_tx_dma_setup; 4658 v6 += txr->q_enomem_tx_dma_setup; 4659 v7 += txr->q_tso_err; 4660 } 4661 IXGBE_EVC_STORE(&adapter->efbig_tx_dma_setup, v0); 4662 IXGBE_EVC_STORE(&adapter->mbuf_defrag_failed, v1); 4663 IXGBE_EVC_STORE(&adapter->efbig2_tx_dma_setup, v2); 4664 IXGBE_EVC_STORE(&adapter->einval_tx_dma_setup, v3); 4665 IXGBE_EVC_STORE(&adapter->other_tx_dma_setup, v4); 4666 IXGBE_EVC_STORE(&adapter->eagain_tx_dma_setup, v5); 4667 IXGBE_EVC_STORE(&adapter->enomem_tx_dma_setup, v6); 4668 IXGBE_EVC_STORE(&adapter->tso_err, v7); 4669 4670 /* 4671 * Check the TX queues status 4672 * - mark hung queues so we don't schedule on them 4673 * - watchdog only if all queues show hung 4674 */ 4675 que = adapter->queues; 4676 for (i = 0; i < adapter->num_queues; i++, que++) { 4677 /* Keep track of queues with work for soft irq */ 4678 if (que->txr->busy) 4679 queues |= 1ULL << que->me; 4680 /* 4681 * Each time txeof runs without cleaning, but there 4682 * are uncleaned descriptors it increments busy. If 4683 * we get to the MAX we declare it hung. 4684 */ 4685 if (que->busy == IXGBE_QUEUE_HUNG) { 4686 ++hung; 4687 /* Mark the queue as inactive */ 4688 adapter->active_queues &= ~(1ULL << que->me); 4689 continue; 4690 } else { 4691 /* Check if we've come back from hung */ 4692 if ((adapter->active_queues & (1ULL << que->me)) == 0) 4693 adapter->active_queues |= 1ULL << que->me; 4694 } 4695 if (que->busy >= IXGBE_MAX_TX_BUSY) { 4696 device_printf(dev, 4697 "Warning queue %d appears to be hung!\n", i); 4698 que->txr->busy = IXGBE_QUEUE_HUNG; 4699 ++hung; 4700 } 4701 } 4702 4703 /* Only truly watchdog if all queues show hung */ 4704 if (hung == adapter->num_queues) 4705 goto watchdog; 4706 #if 0 /* XXX Avoid unexpectedly disabling interrupt forever (PR#53294) */ 4707 else if (queues != 0) { /* Force an IRQ on queues with work */ 4708 que = adapter->queues; 4709 for (i = 0; i < adapter->num_queues; i++, que++) { 4710 mutex_enter(&que->dc_mtx); 4711 if (que->disabled_count == 0) 4712 ixgbe_rearm_queues(adapter, 4713 queues & ((u64)1 << i)); 4714 mutex_exit(&que->dc_mtx); 4715 } 4716 } 4717 #endif 4718 4719 atomic_store_relaxed(&adapter->timer_pending, 0); 4720 IXGBE_CORE_UNLOCK(adapter); 4721 callout_reset(&adapter->timer, hz, ixgbe_local_timer, adapter); 4722 return; 4723 4724 watchdog: 4725 device_printf(adapter->dev, "Watchdog timeout -- resetting\n"); 4726 adapter->ifp->if_flags &= ~IFF_RUNNING; 4727 IXGBE_EVC_ADD(&adapter->watchdog_events, 1); 4728 ixgbe_init_locked(adapter); 4729 IXGBE_CORE_UNLOCK(adapter); 4730 } /* ixgbe_handle_timer */ 4731 4732 /************************************************************************ 4733 * ixgbe_recovery_mode_timer - Recovery mode timer routine 4734 ************************************************************************/ 4735 static void 4736 ixgbe_recovery_mode_timer(void *arg) 4737 { 4738 struct adapter *adapter = arg; 4739 4740 if (__predict_true(adapter->osdep.detaching == false)) { 4741 if (atomic_cas_uint(&adapter->recovery_mode_timer_pending, 4742 0, 1) == 0) { 4743 workqueue_enqueue(adapter->recovery_mode_timer_wq, 4744 &adapter->recovery_mode_timer_wc, NULL); 4745 } 4746 } 4747 } 4748 4749 static void 4750 ixgbe_handle_recovery_mode_timer(struct work *wk, void *context) 4751 { 4752 struct adapter *adapter = context; 4753 struct ixgbe_hw *hw = &adapter->hw; 4754 4755 IXGBE_CORE_LOCK(adapter); 4756 if (ixgbe_fw_recovery_mode(hw)) { 4757 if (atomic_cas_uint(&adapter->recovery_mode, 0, 1)) { 4758 /* Firmware error detected, entering recovery mode */ 4759 device_printf(adapter->dev, "Firmware recovery mode detected. Limiting functionality. Refer to the Intel(R) Ethernet Adapters and Devices User Guide for details on firmware recovery mode.\n"); 4760 4761 if (hw->adapter_stopped == FALSE) 4762 ixgbe_stop_locked(adapter); 4763 } 4764 } else 4765 atomic_cas_uint(&adapter->recovery_mode, 1, 0); 4766 4767 atomic_store_relaxed(&adapter->recovery_mode_timer_pending, 0); 4768 callout_reset(&adapter->recovery_mode_timer, hz, 4769 ixgbe_recovery_mode_timer, adapter); 4770 IXGBE_CORE_UNLOCK(adapter); 4771 } /* ixgbe_handle_recovery_mode_timer */ 4772 4773 /************************************************************************ 4774 * ixgbe_handle_mod - Tasklet for SFP module interrupts 4775 * bool int_en: true if it's called when the interrupt is enabled. 4776 ************************************************************************/ 4777 static void 4778 ixgbe_handle_mod(void *context, bool int_en) 4779 { 4780 struct adapter *adapter = context; 4781 struct ixgbe_hw *hw = &adapter->hw; 4782 device_t dev = adapter->dev; 4783 enum ixgbe_sfp_type last_sfp_type; 4784 u32 err; 4785 bool last_unsupported_sfp_recovery; 4786 4787 KASSERT(mutex_owned(&adapter->core_mtx)); 4788 4789 last_sfp_type = hw->phy.sfp_type; 4790 last_unsupported_sfp_recovery = hw->need_unsupported_sfp_recovery; 4791 IXGBE_EVC_ADD(&adapter->mod_workev, 1); 4792 if (adapter->hw.need_crosstalk_fix) { 4793 if ((hw->mac.type != ixgbe_mac_82598EB) && 4794 !ixgbe_sfp_cage_full(hw)) 4795 goto out; 4796 } 4797 4798 err = hw->phy.ops.identify_sfp(hw); 4799 if (err == IXGBE_ERR_SFP_NOT_SUPPORTED) { 4800 if (last_unsupported_sfp_recovery == false) 4801 device_printf(dev, 4802 "Unsupported SFP+ module type was detected.\n"); 4803 goto out; 4804 } 4805 4806 if (hw->need_unsupported_sfp_recovery) { 4807 device_printf(dev, "Recovering from unsupported SFP\n"); 4808 /* 4809 * We could recover the status by calling setup_sfp(), 4810 * setup_link() and some others. It's complex and might not 4811 * work correctly on some unknown cases. To avoid such type of 4812 * problem, call ixgbe_init_locked(). It's simple and safe 4813 * approach. 4814 */ 4815 ixgbe_init_locked(adapter); 4816 } else if ((hw->phy.sfp_type != ixgbe_sfp_type_not_present) && 4817 (hw->phy.sfp_type != last_sfp_type)) { 4818 /* A module is inserted and changed. */ 4819 4820 if (hw->mac.type == ixgbe_mac_82598EB) 4821 err = hw->phy.ops.reset(hw); 4822 else { 4823 err = hw->mac.ops.setup_sfp(hw); 4824 hw->phy.sfp_setup_needed = FALSE; 4825 } 4826 if (err == IXGBE_ERR_SFP_NOT_SUPPORTED) { 4827 device_printf(dev, 4828 "Setup failure - unsupported SFP+ module type.\n"); 4829 goto out; 4830 } 4831 } 4832 4833 out: 4834 /* get_supported_phy_layer will call hw->phy.ops.identify_sfp() */ 4835 adapter->phy_layer = ixgbe_get_supported_physical_layer(hw); 4836 4837 /* Adjust media types shown in ifconfig */ 4838 IXGBE_CORE_UNLOCK(adapter); 4839 ifmedia_removeall(&adapter->media); 4840 ixgbe_add_media_types(adapter); 4841 ifmedia_set(&adapter->media, IFM_ETHER | IFM_AUTO); 4842 IXGBE_CORE_LOCK(adapter); 4843 4844 /* 4845 * Don't schedule MSF event if the chip is 82598. 82598 doesn't support 4846 * MSF. At least, calling ixgbe_handle_msf on 82598 DA makes the link 4847 * flap because the function calls setup_link(). 4848 */ 4849 if (hw->mac.type != ixgbe_mac_82598EB) { 4850 mutex_enter(&adapter->admin_mtx); 4851 if (int_en) 4852 adapter->task_requests |= IXGBE_REQUEST_TASK_MSF; 4853 else 4854 adapter->task_requests |= IXGBE_REQUEST_TASK_MSF_WOI; 4855 mutex_exit(&adapter->admin_mtx); 4856 } 4857 4858 /* 4859 * Don't call ixgbe_schedule_admin_tasklet() because we are on 4860 * the workqueue now. 4861 */ 4862 } /* ixgbe_handle_mod */ 4863 4864 4865 /************************************************************************ 4866 * ixgbe_handle_msf - Tasklet for MSF (multispeed fiber) interrupts 4867 ************************************************************************/ 4868 static void 4869 ixgbe_handle_msf(void *context) 4870 { 4871 struct adapter *adapter = context; 4872 struct ixgbe_hw *hw = &adapter->hw; 4873 u32 autoneg; 4874 bool negotiate; 4875 4876 KASSERT(mutex_owned(&adapter->core_mtx)); 4877 4878 IXGBE_EVC_ADD(&adapter->msf_workev, 1); 4879 4880 autoneg = hw->phy.autoneg_advertised; 4881 if ((!autoneg) && (hw->mac.ops.get_link_capabilities)) 4882 hw->mac.ops.get_link_capabilities(hw, &autoneg, &negotiate); 4883 if (hw->mac.ops.setup_link) 4884 hw->mac.ops.setup_link(hw, autoneg, TRUE); 4885 } /* ixgbe_handle_msf */ 4886 4887 /************************************************************************ 4888 * ixgbe_handle_phy - Tasklet for external PHY interrupts 4889 ************************************************************************/ 4890 static void 4891 ixgbe_handle_phy(void *context) 4892 { 4893 struct adapter *adapter = context; 4894 struct ixgbe_hw *hw = &adapter->hw; 4895 int error; 4896 4897 KASSERT(mutex_owned(&adapter->core_mtx)); 4898 4899 IXGBE_EVC_ADD(&adapter->phy_workev, 1); 4900 error = hw->phy.ops.handle_lasi(hw); 4901 if (error == IXGBE_ERR_OVERTEMP) 4902 device_printf(adapter->dev, 4903 "CRITICAL: EXTERNAL PHY OVER TEMP!! " 4904 " PHY will downshift to lower power state!\n"); 4905 else if (error) 4906 device_printf(adapter->dev, 4907 "Error handling LASI interrupt: %d\n", error); 4908 } /* ixgbe_handle_phy */ 4909 4910 static void 4911 ixgbe_handle_admin(struct work *wk, void *context) 4912 { 4913 struct adapter *adapter = context; 4914 struct ifnet *ifp = adapter->ifp; 4915 struct ixgbe_hw *hw = &adapter->hw; 4916 u32 task_requests; 4917 u32 eims_enable = 0; 4918 4919 mutex_enter(&adapter->admin_mtx); 4920 adapter->admin_pending = 0; 4921 task_requests = adapter->task_requests; 4922 adapter->task_requests = 0; 4923 mutex_exit(&adapter->admin_mtx); 4924 4925 /* 4926 * Hold the IFNET_LOCK across this entire call. This will 4927 * prevent additional changes to adapter->phy_layer 4928 * and serialize calls to this tasklet. We cannot hold the 4929 * CORE_LOCK while calling into the ifmedia functions as 4930 * they call ifmedia_lock() and the lock is CORE_LOCK. 4931 */ 4932 IFNET_LOCK(ifp); 4933 IXGBE_CORE_LOCK(adapter); 4934 if ((task_requests & IXGBE_REQUEST_TASK_LSC) != 0) { 4935 ixgbe_handle_link(adapter); 4936 eims_enable |= IXGBE_EIMS_LSC; 4937 } 4938 if ((task_requests & IXGBE_REQUEST_TASK_MOD_WOI) != 0) { 4939 ixgbe_handle_mod(adapter, false); 4940 } 4941 if ((task_requests & IXGBE_REQUEST_TASK_MOD) != 0) { 4942 ixgbe_handle_mod(adapter, true); 4943 if (hw->mac.type >= ixgbe_mac_X540) 4944 eims_enable |= IXGBE_EICR_GPI_SDP0_X540; 4945 else 4946 eims_enable |= IXGBE_EICR_GPI_SDP2_BY_MAC(hw); 4947 } 4948 if ((task_requests 4949 & (IXGBE_REQUEST_TASK_MSF_WOI | IXGBE_REQUEST_TASK_MSF)) != 0) { 4950 ixgbe_handle_msf(adapter); 4951 if (((task_requests & IXGBE_REQUEST_TASK_MSF) != 0) && 4952 (hw->mac.type == ixgbe_mac_82599EB)) 4953 eims_enable |= IXGBE_EIMS_GPI_SDP1_BY_MAC(hw); 4954 } 4955 if ((task_requests & IXGBE_REQUEST_TASK_PHY) != 0) { 4956 ixgbe_handle_phy(adapter); 4957 eims_enable |= IXGBE_EICR_GPI_SDP0_X540; 4958 } 4959 if ((task_requests & IXGBE_REQUEST_TASK_FDIR) != 0) { 4960 ixgbe_reinit_fdir(adapter); 4961 eims_enable |= IXGBE_EIMS_FLOW_DIR; 4962 } 4963 #if 0 /* notyet */ 4964 if ((task_requests & IXGBE_REQUEST_TASK_MBX) != 0) { 4965 ixgbe_handle_mbx(adapter); 4966 eims_enable |= IXGBE_EIMS_MAILBOX; 4967 } 4968 #endif 4969 IXGBE_WRITE_REG(hw, IXGBE_EIMS, eims_enable); 4970 4971 IXGBE_CORE_UNLOCK(adapter); 4972 IFNET_UNLOCK(ifp); 4973 } /* ixgbe_handle_admin */ 4974 4975 static void 4976 ixgbe_ifstop(struct ifnet *ifp, int disable) 4977 { 4978 struct adapter *adapter = ifp->if_softc; 4979 4980 IXGBE_CORE_LOCK(adapter); 4981 ixgbe_stop_locked(adapter); 4982 IXGBE_CORE_UNLOCK(adapter); 4983 4984 workqueue_wait(adapter->timer_wq, &adapter->timer_wc); 4985 atomic_store_relaxed(&adapter->timer_pending, 0); 4986 } 4987 4988 /************************************************************************ 4989 * ixgbe_stop_locked - Stop the hardware 4990 * 4991 * Disables all traffic on the adapter by issuing a 4992 * global reset on the MAC and deallocates TX/RX buffers. 4993 ************************************************************************/ 4994 static void 4995 ixgbe_stop_locked(void *arg) 4996 { 4997 struct ifnet *ifp; 4998 struct adapter *adapter = arg; 4999 struct ixgbe_hw *hw = &adapter->hw; 5000 5001 ifp = adapter->ifp; 5002 5003 KASSERT(mutex_owned(&adapter->core_mtx)); 5004 5005 INIT_DEBUGOUT("ixgbe_stop_locked: begin\n"); 5006 ixgbe_disable_intr(adapter); 5007 callout_stop(&adapter->timer); 5008 5009 /* Don't schedule workqueues. */ 5010 adapter->schedule_wqs_ok = false; 5011 5012 /* Let the stack know...*/ 5013 ifp->if_flags &= ~IFF_RUNNING; 5014 5015 ixgbe_reset_hw(hw); 5016 hw->adapter_stopped = FALSE; 5017 ixgbe_stop_adapter(hw); 5018 if (hw->mac.type == ixgbe_mac_82599EB) 5019 ixgbe_stop_mac_link_on_d3_82599(hw); 5020 /* Turn off the laser - noop with no optics */ 5021 ixgbe_disable_tx_laser(hw); 5022 5023 /* Update the stack */ 5024 adapter->link_up = FALSE; 5025 ixgbe_update_link_status(adapter); 5026 5027 /* reprogram the RAR[0] in case user changed it. */ 5028 ixgbe_set_rar(&adapter->hw, 0, adapter->hw.mac.addr, 0, IXGBE_RAH_AV); 5029 5030 return; 5031 } /* ixgbe_stop_locked */ 5032 5033 /************************************************************************ 5034 * ixgbe_update_link_status - Update OS on link state 5035 * 5036 * Note: Only updates the OS on the cached link state. 5037 * The real check of the hardware only happens with 5038 * a link interrupt. 5039 ************************************************************************/ 5040 static void 5041 ixgbe_update_link_status(struct adapter *adapter) 5042 { 5043 struct ifnet *ifp = adapter->ifp; 5044 device_t dev = adapter->dev; 5045 struct ixgbe_hw *hw = &adapter->hw; 5046 5047 KASSERT(mutex_owned(&adapter->core_mtx)); 5048 5049 if (adapter->link_up) { 5050 if (adapter->link_active != LINK_STATE_UP) { 5051 /* 5052 * To eliminate influence of the previous state 5053 * in the same way as ixgbe_init_locked(). 5054 */ 5055 struct ix_queue *que = adapter->queues; 5056 for (int i = 0; i < adapter->num_queues; i++, que++) 5057 que->eitr_setting = 0; 5058 5059 if (adapter->link_speed == IXGBE_LINK_SPEED_10GB_FULL){ 5060 /* 5061 * Discard count for both MAC Local Fault and 5062 * Remote Fault because those registers are 5063 * valid only when the link speed is up and 5064 * 10Gbps. 5065 */ 5066 IXGBE_READ_REG(hw, IXGBE_MLFC); 5067 IXGBE_READ_REG(hw, IXGBE_MRFC); 5068 } 5069 5070 if (bootverbose) { 5071 const char *bpsmsg; 5072 5073 switch (adapter->link_speed) { 5074 case IXGBE_LINK_SPEED_10GB_FULL: 5075 bpsmsg = "10 Gbps"; 5076 break; 5077 case IXGBE_LINK_SPEED_5GB_FULL: 5078 bpsmsg = "5 Gbps"; 5079 break; 5080 case IXGBE_LINK_SPEED_2_5GB_FULL: 5081 bpsmsg = "2.5 Gbps"; 5082 break; 5083 case IXGBE_LINK_SPEED_1GB_FULL: 5084 bpsmsg = "1 Gbps"; 5085 break; 5086 case IXGBE_LINK_SPEED_100_FULL: 5087 bpsmsg = "100 Mbps"; 5088 break; 5089 case IXGBE_LINK_SPEED_10_FULL: 5090 bpsmsg = "10 Mbps"; 5091 break; 5092 default: 5093 bpsmsg = "unknown speed"; 5094 break; 5095 } 5096 device_printf(dev, "Link is up %s %s \n", 5097 bpsmsg, "Full Duplex"); 5098 } 5099 adapter->link_active = LINK_STATE_UP; 5100 /* Update any Flow Control changes */ 5101 ixgbe_fc_enable(&adapter->hw); 5102 /* Update DMA coalescing config */ 5103 ixgbe_config_dmac(adapter); 5104 if_link_state_change(ifp, LINK_STATE_UP); 5105 5106 if (adapter->feat_en & IXGBE_FEATURE_SRIOV) 5107 ixgbe_ping_all_vfs(adapter); 5108 } 5109 } else { 5110 /* 5111 * Do it when link active changes to DOWN. i.e. 5112 * a) LINK_STATE_UNKNOWN -> LINK_STATE_DOWN 5113 * b) LINK_STATE_UP -> LINK_STATE_DOWN 5114 */ 5115 if (adapter->link_active != LINK_STATE_DOWN) { 5116 if (bootverbose) 5117 device_printf(dev, "Link is Down\n"); 5118 if_link_state_change(ifp, LINK_STATE_DOWN); 5119 adapter->link_active = LINK_STATE_DOWN; 5120 if (adapter->feat_en & IXGBE_FEATURE_SRIOV) 5121 ixgbe_ping_all_vfs(adapter); 5122 ixgbe_drain_all(adapter); 5123 } 5124 } 5125 } /* ixgbe_update_link_status */ 5126 5127 /************************************************************************ 5128 * ixgbe_config_dmac - Configure DMA Coalescing 5129 ************************************************************************/ 5130 static void 5131 ixgbe_config_dmac(struct adapter *adapter) 5132 { 5133 struct ixgbe_hw *hw = &adapter->hw; 5134 struct ixgbe_dmac_config *dcfg = &hw->mac.dmac_config; 5135 5136 if (hw->mac.type < ixgbe_mac_X550 || !hw->mac.ops.dmac_config) 5137 return; 5138 5139 if (dcfg->watchdog_timer ^ adapter->dmac || 5140 dcfg->link_speed ^ adapter->link_speed) { 5141 dcfg->watchdog_timer = adapter->dmac; 5142 dcfg->fcoe_en = false; 5143 dcfg->link_speed = adapter->link_speed; 5144 dcfg->num_tcs = 1; 5145 5146 INIT_DEBUGOUT2("dmac settings: watchdog %d, link speed %d\n", 5147 dcfg->watchdog_timer, dcfg->link_speed); 5148 5149 hw->mac.ops.dmac_config(hw); 5150 } 5151 } /* ixgbe_config_dmac */ 5152 5153 /************************************************************************ 5154 * ixgbe_enable_intr 5155 ************************************************************************/ 5156 static void 5157 ixgbe_enable_intr(struct adapter *adapter) 5158 { 5159 struct ixgbe_hw *hw = &adapter->hw; 5160 struct ix_queue *que = adapter->queues; 5161 u32 mask, fwsm; 5162 5163 mask = (IXGBE_EIMS_ENABLE_MASK & ~IXGBE_EIMS_RTX_QUEUE); 5164 5165 switch (adapter->hw.mac.type) { 5166 case ixgbe_mac_82599EB: 5167 mask |= IXGBE_EIMS_ECC; 5168 /* Temperature sensor on some adapters */ 5169 mask |= IXGBE_EIMS_GPI_SDP0; 5170 /* SFP+ (RX_LOS_N & MOD_ABS_N) */ 5171 mask |= IXGBE_EIMS_GPI_SDP1; 5172 mask |= IXGBE_EIMS_GPI_SDP2; 5173 break; 5174 case ixgbe_mac_X540: 5175 /* Detect if Thermal Sensor is enabled */ 5176 fwsm = IXGBE_READ_REG(hw, IXGBE_FWSM); 5177 if (fwsm & IXGBE_FWSM_TS_ENABLED) 5178 mask |= IXGBE_EIMS_TS; 5179 mask |= IXGBE_EIMS_ECC; 5180 break; 5181 case ixgbe_mac_X550: 5182 /* MAC thermal sensor is automatically enabled */ 5183 mask |= IXGBE_EIMS_TS; 5184 mask |= IXGBE_EIMS_ECC; 5185 break; 5186 case ixgbe_mac_X550EM_x: 5187 case ixgbe_mac_X550EM_a: 5188 /* Some devices use SDP0 for important information */ 5189 if (hw->device_id == IXGBE_DEV_ID_X550EM_X_SFP || 5190 hw->device_id == IXGBE_DEV_ID_X550EM_A_SFP || 5191 hw->device_id == IXGBE_DEV_ID_X550EM_A_SFP_N || 5192 hw->device_id == IXGBE_DEV_ID_X550EM_X_10G_T) 5193 mask |= IXGBE_EIMS_GPI_SDP0_BY_MAC(hw); 5194 if (hw->phy.type == ixgbe_phy_x550em_ext_t) 5195 mask |= IXGBE_EICR_GPI_SDP0_X540; 5196 mask |= IXGBE_EIMS_ECC; 5197 break; 5198 default: 5199 break; 5200 } 5201 5202 /* Enable Fan Failure detection */ 5203 if (adapter->feat_en & IXGBE_FEATURE_FAN_FAIL) 5204 mask |= IXGBE_EIMS_GPI_SDP1; 5205 /* Enable SR-IOV */ 5206 if (adapter->feat_en & IXGBE_FEATURE_SRIOV) 5207 mask |= IXGBE_EIMS_MAILBOX; 5208 /* Enable Flow Director */ 5209 if (adapter->feat_en & IXGBE_FEATURE_FDIR) 5210 mask |= IXGBE_EIMS_FLOW_DIR; 5211 5212 IXGBE_WRITE_REG(hw, IXGBE_EIMS, mask); 5213 5214 /* With MSI-X we use auto clear */ 5215 if (adapter->msix_mem) { 5216 /* 5217 * We use auto clear for RTX_QUEUE only. Don't use other 5218 * interrupts (e.g. link interrupt). BTW, we don't use 5219 * TCP_TIMER interrupt itself. 5220 */ 5221 IXGBE_WRITE_REG(hw, IXGBE_EIAC, IXGBE_EIMS_RTX_QUEUE); 5222 } 5223 5224 /* 5225 * Now enable all queues, this is done separately to 5226 * allow for handling the extended (beyond 32) MSI-X 5227 * vectors that can be used by 82599 5228 */ 5229 for (int i = 0; i < adapter->num_queues; i++, que++) 5230 ixgbe_enable_queue(adapter, que->msix); 5231 5232 IXGBE_WRITE_FLUSH(hw); 5233 5234 } /* ixgbe_enable_intr */ 5235 5236 /************************************************************************ 5237 * ixgbe_disable_intr_internal 5238 ************************************************************************/ 5239 static void 5240 ixgbe_disable_intr_internal(struct adapter *adapter, bool nestok) 5241 { 5242 struct ix_queue *que = adapter->queues; 5243 5244 /* disable interrupts other than queues */ 5245 IXGBE_WRITE_REG(&adapter->hw, IXGBE_EIMC, ~IXGBE_EIMC_RTX_QUEUE); 5246 5247 if (adapter->msix_mem) 5248 IXGBE_WRITE_REG(&adapter->hw, IXGBE_EIAC, 0); 5249 5250 for (int i = 0; i < adapter->num_queues; i++, que++) 5251 ixgbe_disable_queue_internal(adapter, que->msix, nestok); 5252 5253 IXGBE_WRITE_FLUSH(&adapter->hw); 5254 5255 } /* ixgbe_do_disable_intr_internal */ 5256 5257 /************************************************************************ 5258 * ixgbe_disable_intr 5259 ************************************************************************/ 5260 static void 5261 ixgbe_disable_intr(struct adapter *adapter) 5262 { 5263 5264 ixgbe_disable_intr_internal(adapter, true); 5265 } /* ixgbe_disable_intr */ 5266 5267 /************************************************************************ 5268 * ixgbe_ensure_disabled_intr 5269 ************************************************************************/ 5270 void 5271 ixgbe_ensure_disabled_intr(struct adapter *adapter) 5272 { 5273 5274 ixgbe_disable_intr_internal(adapter, false); 5275 } /* ixgbe_ensure_disabled_intr */ 5276 5277 /************************************************************************ 5278 * ixgbe_legacy_irq - Legacy Interrupt Service routine 5279 ************************************************************************/ 5280 static int 5281 ixgbe_legacy_irq(void *arg) 5282 { 5283 struct ix_queue *que = arg; 5284 struct adapter *adapter = que->adapter; 5285 struct ixgbe_hw *hw = &adapter->hw; 5286 struct ifnet *ifp = adapter->ifp; 5287 struct tx_ring *txr = adapter->tx_rings; 5288 u32 eicr; 5289 u32 eims_orig; 5290 u32 eims_enable = 0; 5291 u32 eims_disable = 0; 5292 5293 eims_orig = IXGBE_READ_REG(hw, IXGBE_EIMS); 5294 /* 5295 * Silicon errata #26 on 82598. Disable all interrupts before reading 5296 * EICR. 5297 */ 5298 IXGBE_WRITE_REG(hw, IXGBE_EIMC, IXGBE_IRQ_CLEAR_MASK); 5299 5300 /* Read and clear EICR */ 5301 eicr = IXGBE_READ_REG(hw, IXGBE_EICR); 5302 5303 if (eicr == 0) { 5304 IXGBE_EVC_ADD(&adapter->stats.pf.intzero, 1); 5305 IXGBE_WRITE_REG(hw, IXGBE_EIMS, eims_orig); 5306 return 0; 5307 } 5308 IXGBE_EVC_ADD(&adapter->stats.pf.legint, 1); 5309 5310 /* Queue (0) intr */ 5311 if (((ifp->if_flags & IFF_RUNNING) != 0) && 5312 (eicr & IXGBE_EIMC_RTX_QUEUE) != 0) { 5313 IXGBE_EVC_ADD(&que->irqs, 1); 5314 5315 /* 5316 * The same as ixgbe_msix_que() about 5317 * "que->txrx_use_workqueue". 5318 */ 5319 que->txrx_use_workqueue = adapter->txrx_use_workqueue; 5320 5321 IXGBE_TX_LOCK(txr); 5322 ixgbe_txeof(txr); 5323 #ifdef notyet 5324 if (!ixgbe_ring_empty(ifp, txr->br)) 5325 ixgbe_start_locked(ifp, txr); 5326 #endif 5327 IXGBE_TX_UNLOCK(txr); 5328 5329 IXGBE_EVC_ADD(&que->req, 1); 5330 ixgbe_sched_handle_que(adapter, que); 5331 /* Disable queue 0 interrupt */ 5332 eims_disable |= 1UL << 0; 5333 5334 } else 5335 eims_enable |= IXGBE_EIMC_RTX_QUEUE; 5336 5337 ixgbe_intr_admin_common(adapter, eicr, &eims_disable); 5338 5339 /* Re-enable some interrupts */ 5340 IXGBE_WRITE_REG(hw, IXGBE_EIMS, 5341 (eims_orig & ~eims_disable) | eims_enable); 5342 5343 return 1; 5344 } /* ixgbe_legacy_irq */ 5345 5346 /************************************************************************ 5347 * ixgbe_free_pciintr_resources 5348 ************************************************************************/ 5349 static void 5350 ixgbe_free_pciintr_resources(struct adapter *adapter) 5351 { 5352 struct ix_queue *que = adapter->queues; 5353 int rid; 5354 5355 /* 5356 * Release all msix queue resources: 5357 */ 5358 for (int i = 0; i < adapter->num_queues; i++, que++) { 5359 if (que->res != NULL) { 5360 pci_intr_disestablish(adapter->osdep.pc, 5361 adapter->osdep.ihs[i]); 5362 adapter->osdep.ihs[i] = NULL; 5363 } 5364 } 5365 5366 /* Clean the Legacy or Link interrupt last */ 5367 if (adapter->vector) /* we are doing MSIX */ 5368 rid = adapter->vector; 5369 else 5370 rid = 0; 5371 5372 if (adapter->osdep.ihs[rid] != NULL) { 5373 pci_intr_disestablish(adapter->osdep.pc, 5374 adapter->osdep.ihs[rid]); 5375 adapter->osdep.ihs[rid] = NULL; 5376 } 5377 5378 if (adapter->osdep.intrs != NULL) { 5379 pci_intr_release(adapter->osdep.pc, adapter->osdep.intrs, 5380 adapter->osdep.nintrs); 5381 adapter->osdep.intrs = NULL; 5382 } 5383 } /* ixgbe_free_pciintr_resources */ 5384 5385 /************************************************************************ 5386 * ixgbe_free_pci_resources 5387 ************************************************************************/ 5388 static void 5389 ixgbe_free_pci_resources(struct adapter *adapter) 5390 { 5391 5392 ixgbe_free_pciintr_resources(adapter); 5393 5394 if (adapter->osdep.mem_size != 0) { 5395 bus_space_unmap(adapter->osdep.mem_bus_space_tag, 5396 adapter->osdep.mem_bus_space_handle, 5397 adapter->osdep.mem_size); 5398 } 5399 5400 } /* ixgbe_free_pci_resources */ 5401 5402 /************************************************************************ 5403 * ixgbe_sysctl_flowcntl 5404 * 5405 * SYSCTL wrapper around setting Flow Control 5406 ************************************************************************/ 5407 static int 5408 ixgbe_sysctl_flowcntl(SYSCTLFN_ARGS) 5409 { 5410 struct sysctlnode node = *rnode; 5411 struct adapter *adapter = (struct adapter *)node.sysctl_data; 5412 int error, fc; 5413 5414 if (ixgbe_fw_recovery_mode_swflag(adapter)) 5415 return (EPERM); 5416 5417 fc = adapter->hw.fc.current_mode; 5418 node.sysctl_data = &fc; 5419 error = sysctl_lookup(SYSCTLFN_CALL(&node)); 5420 if (error != 0 || newp == NULL) 5421 return error; 5422 5423 /* Don't bother if it's not changed */ 5424 if (fc == adapter->hw.fc.current_mode) 5425 return (0); 5426 5427 return ixgbe_set_flowcntl(adapter, fc); 5428 } /* ixgbe_sysctl_flowcntl */ 5429 5430 /************************************************************************ 5431 * ixgbe_set_flowcntl - Set flow control 5432 * 5433 * Flow control values: 5434 * 0 - off 5435 * 1 - rx pause 5436 * 2 - tx pause 5437 * 3 - full 5438 ************************************************************************/ 5439 static int 5440 ixgbe_set_flowcntl(struct adapter *adapter, int fc) 5441 { 5442 switch (fc) { 5443 case ixgbe_fc_rx_pause: 5444 case ixgbe_fc_tx_pause: 5445 case ixgbe_fc_full: 5446 adapter->hw.fc.requested_mode = fc; 5447 if (adapter->num_queues > 1) 5448 ixgbe_disable_rx_drop(adapter); 5449 break; 5450 case ixgbe_fc_none: 5451 adapter->hw.fc.requested_mode = ixgbe_fc_none; 5452 if (adapter->num_queues > 1) 5453 ixgbe_enable_rx_drop(adapter); 5454 break; 5455 default: 5456 return (EINVAL); 5457 } 5458 5459 #if 0 /* XXX NetBSD */ 5460 /* Don't autoneg if forcing a value */ 5461 adapter->hw.fc.disable_fc_autoneg = TRUE; 5462 #endif 5463 ixgbe_fc_enable(&adapter->hw); 5464 5465 return (0); 5466 } /* ixgbe_set_flowcntl */ 5467 5468 /************************************************************************ 5469 * ixgbe_enable_rx_drop 5470 * 5471 * Enable the hardware to drop packets when the buffer is 5472 * full. This is useful with multiqueue, so that no single 5473 * queue being full stalls the entire RX engine. We only 5474 * enable this when Multiqueue is enabled AND Flow Control 5475 * is disabled. 5476 ************************************************************************/ 5477 static void 5478 ixgbe_enable_rx_drop(struct adapter *adapter) 5479 { 5480 struct ixgbe_hw *hw = &adapter->hw; 5481 struct rx_ring *rxr; 5482 u32 srrctl; 5483 5484 for (int i = 0; i < adapter->num_queues; i++) { 5485 rxr = &adapter->rx_rings[i]; 5486 srrctl = IXGBE_READ_REG(hw, IXGBE_SRRCTL(rxr->me)); 5487 srrctl |= IXGBE_SRRCTL_DROP_EN; 5488 IXGBE_WRITE_REG(hw, IXGBE_SRRCTL(rxr->me), srrctl); 5489 } 5490 5491 /* enable drop for each vf */ 5492 for (int i = 0; i < adapter->num_vfs; i++) { 5493 IXGBE_WRITE_REG(hw, IXGBE_QDE, 5494 (IXGBE_QDE_WRITE | (i << IXGBE_QDE_IDX_SHIFT) | 5495 IXGBE_QDE_ENABLE)); 5496 } 5497 } /* ixgbe_enable_rx_drop */ 5498 5499 /************************************************************************ 5500 * ixgbe_disable_rx_drop 5501 ************************************************************************/ 5502 static void 5503 ixgbe_disable_rx_drop(struct adapter *adapter) 5504 { 5505 struct ixgbe_hw *hw = &adapter->hw; 5506 struct rx_ring *rxr; 5507 u32 srrctl; 5508 5509 for (int i = 0; i < adapter->num_queues; i++) { 5510 rxr = &adapter->rx_rings[i]; 5511 srrctl = IXGBE_READ_REG(hw, IXGBE_SRRCTL(rxr->me)); 5512 srrctl &= ~IXGBE_SRRCTL_DROP_EN; 5513 IXGBE_WRITE_REG(hw, IXGBE_SRRCTL(rxr->me), srrctl); 5514 } 5515 5516 /* disable drop for each vf */ 5517 for (int i = 0; i < adapter->num_vfs; i++) { 5518 IXGBE_WRITE_REG(hw, IXGBE_QDE, 5519 (IXGBE_QDE_WRITE | (i << IXGBE_QDE_IDX_SHIFT))); 5520 } 5521 } /* ixgbe_disable_rx_drop */ 5522 5523 /************************************************************************ 5524 * ixgbe_sysctl_advertise 5525 * 5526 * SYSCTL wrapper around setting advertised speed 5527 ************************************************************************/ 5528 static int 5529 ixgbe_sysctl_advertise(SYSCTLFN_ARGS) 5530 { 5531 struct sysctlnode node = *rnode; 5532 struct adapter *adapter = (struct adapter *)node.sysctl_data; 5533 int error = 0, advertise; 5534 5535 if (ixgbe_fw_recovery_mode_swflag(adapter)) 5536 return (EPERM); 5537 5538 advertise = adapter->advertise; 5539 node.sysctl_data = &advertise; 5540 error = sysctl_lookup(SYSCTLFN_CALL(&node)); 5541 if (error != 0 || newp == NULL) 5542 return error; 5543 5544 return ixgbe_set_advertise(adapter, advertise); 5545 } /* ixgbe_sysctl_advertise */ 5546 5547 /************************************************************************ 5548 * ixgbe_set_advertise - Control advertised link speed 5549 * 5550 * Flags: 5551 * 0x00 - Default (all capable link speed) 5552 * 0x1 - advertise 100 Mb 5553 * 0x2 - advertise 1G 5554 * 0x4 - advertise 10G 5555 * 0x8 - advertise 10 Mb (yes, Mb) 5556 * 0x10 - advertise 2.5G 5557 * 0x20 - advertise 5G 5558 ************************************************************************/ 5559 static int 5560 ixgbe_set_advertise(struct adapter *adapter, int advertise) 5561 { 5562 device_t dev; 5563 struct ixgbe_hw *hw; 5564 ixgbe_link_speed speed = 0; 5565 ixgbe_link_speed link_caps = 0; 5566 s32 err = IXGBE_NOT_IMPLEMENTED; 5567 bool negotiate = FALSE; 5568 5569 /* Checks to validate new value */ 5570 if (adapter->advertise == advertise) /* no change */ 5571 return (0); 5572 5573 dev = adapter->dev; 5574 hw = &adapter->hw; 5575 5576 /* No speed changes for backplane media */ 5577 if (hw->phy.media_type == ixgbe_media_type_backplane) 5578 return (ENODEV); 5579 5580 if (!((hw->phy.media_type == ixgbe_media_type_copper) || 5581 (hw->phy.multispeed_fiber))) { 5582 device_printf(dev, 5583 "Advertised speed can only be set on copper or " 5584 "multispeed fiber media types.\n"); 5585 return (EINVAL); 5586 } 5587 5588 if (advertise < 0x0 || advertise > 0x3f) { 5589 device_printf(dev, "Invalid advertised speed; valid modes are 0x0 through 0x3f\n"); 5590 return (EINVAL); 5591 } 5592 5593 if (hw->mac.ops.get_link_capabilities) { 5594 err = hw->mac.ops.get_link_capabilities(hw, &link_caps, 5595 &negotiate); 5596 if (err != IXGBE_SUCCESS) { 5597 device_printf(dev, "Unable to determine supported advertise speeds\n"); 5598 return (ENODEV); 5599 } 5600 } 5601 5602 /* Set new value and report new advertised mode */ 5603 if (advertise & 0x1) { 5604 if (!(link_caps & IXGBE_LINK_SPEED_100_FULL)) { 5605 device_printf(dev, "Interface does not support 100Mb advertised speed\n"); 5606 return (EINVAL); 5607 } 5608 speed |= IXGBE_LINK_SPEED_100_FULL; 5609 } 5610 if (advertise & 0x2) { 5611 if (!(link_caps & IXGBE_LINK_SPEED_1GB_FULL)) { 5612 device_printf(dev, "Interface does not support 1Gb advertised speed\n"); 5613 return (EINVAL); 5614 } 5615 speed |= IXGBE_LINK_SPEED_1GB_FULL; 5616 } 5617 if (advertise & 0x4) { 5618 if (!(link_caps & IXGBE_LINK_SPEED_10GB_FULL)) { 5619 device_printf(dev, "Interface does not support 10Gb advertised speed\n"); 5620 return (EINVAL); 5621 } 5622 speed |= IXGBE_LINK_SPEED_10GB_FULL; 5623 } 5624 if (advertise & 0x8) { 5625 if (!(link_caps & IXGBE_LINK_SPEED_10_FULL)) { 5626 device_printf(dev, "Interface does not support 10Mb advertised speed\n"); 5627 return (EINVAL); 5628 } 5629 speed |= IXGBE_LINK_SPEED_10_FULL; 5630 } 5631 if (advertise & 0x10) { 5632 if (!(link_caps & IXGBE_LINK_SPEED_2_5GB_FULL)) { 5633 device_printf(dev, "Interface does not support 2.5Gb advertised speed\n"); 5634 return (EINVAL); 5635 } 5636 speed |= IXGBE_LINK_SPEED_2_5GB_FULL; 5637 } 5638 if (advertise & 0x20) { 5639 if (!(link_caps & IXGBE_LINK_SPEED_5GB_FULL)) { 5640 device_printf(dev, "Interface does not support 5Gb advertised speed\n"); 5641 return (EINVAL); 5642 } 5643 speed |= IXGBE_LINK_SPEED_5GB_FULL; 5644 } 5645 if (advertise == 0) 5646 speed = link_caps; /* All capable link speed */ 5647 5648 hw->mac.autotry_restart = TRUE; 5649 hw->mac.ops.setup_link(hw, speed, TRUE); 5650 adapter->advertise = advertise; 5651 5652 return (0); 5653 } /* ixgbe_set_advertise */ 5654 5655 /************************************************************************ 5656 * ixgbe_get_default_advertise - Get default advertised speed settings 5657 * 5658 * Formatted for sysctl usage. 5659 * Flags: 5660 * 0x1 - advertise 100 Mb 5661 * 0x2 - advertise 1G 5662 * 0x4 - advertise 10G 5663 * 0x8 - advertise 10 Mb (yes, Mb) 5664 * 0x10 - advertise 2.5G 5665 * 0x20 - advertise 5G 5666 ************************************************************************/ 5667 static int 5668 ixgbe_get_default_advertise(struct adapter *adapter) 5669 { 5670 struct ixgbe_hw *hw = &adapter->hw; 5671 int speed; 5672 ixgbe_link_speed link_caps = 0; 5673 s32 err; 5674 bool negotiate = FALSE; 5675 5676 /* 5677 * Advertised speed means nothing unless it's copper or 5678 * multi-speed fiber 5679 */ 5680 if (!(hw->phy.media_type == ixgbe_media_type_copper) && 5681 !(hw->phy.multispeed_fiber)) 5682 return (0); 5683 5684 err = hw->mac.ops.get_link_capabilities(hw, &link_caps, &negotiate); 5685 if (err != IXGBE_SUCCESS) 5686 return (0); 5687 5688 speed = 5689 ((link_caps & IXGBE_LINK_SPEED_10GB_FULL) ? 0x4 : 0) | 5690 ((link_caps & IXGBE_LINK_SPEED_5GB_FULL) ? 0x20 : 0) | 5691 ((link_caps & IXGBE_LINK_SPEED_2_5GB_FULL) ? 0x10 : 0) | 5692 ((link_caps & IXGBE_LINK_SPEED_1GB_FULL) ? 0x2 : 0) | 5693 ((link_caps & IXGBE_LINK_SPEED_100_FULL) ? 0x1 : 0) | 5694 ((link_caps & IXGBE_LINK_SPEED_10_FULL) ? 0x8 : 0); 5695 5696 return speed; 5697 } /* ixgbe_get_default_advertise */ 5698 5699 /************************************************************************ 5700 * ixgbe_sysctl_dmac - Manage DMA Coalescing 5701 * 5702 * Control values: 5703 * 0/1 - off / on (use default value of 1000) 5704 * 5705 * Legal timer values are: 5706 * 50,100,250,500,1000,2000,5000,10000 5707 * 5708 * Turning off interrupt moderation will also turn this off. 5709 ************************************************************************/ 5710 static int 5711 ixgbe_sysctl_dmac(SYSCTLFN_ARGS) 5712 { 5713 struct sysctlnode node = *rnode; 5714 struct adapter *adapter = (struct adapter *)node.sysctl_data; 5715 struct ifnet *ifp = adapter->ifp; 5716 int error; 5717 int newval; 5718 5719 if (ixgbe_fw_recovery_mode_swflag(adapter)) 5720 return (EPERM); 5721 5722 newval = adapter->dmac; 5723 node.sysctl_data = &newval; 5724 error = sysctl_lookup(SYSCTLFN_CALL(&node)); 5725 if ((error) || (newp == NULL)) 5726 return (error); 5727 5728 switch (newval) { 5729 case 0: 5730 /* Disabled */ 5731 adapter->dmac = 0; 5732 break; 5733 case 1: 5734 /* Enable and use default */ 5735 adapter->dmac = 1000; 5736 break; 5737 case 50: 5738 case 100: 5739 case 250: 5740 case 500: 5741 case 1000: 5742 case 2000: 5743 case 5000: 5744 case 10000: 5745 /* Legal values - allow */ 5746 adapter->dmac = newval; 5747 break; 5748 default: 5749 /* Do nothing, illegal value */ 5750 return (EINVAL); 5751 } 5752 5753 /* Re-initialize hardware if it's already running */ 5754 if (ifp->if_flags & IFF_RUNNING) 5755 if_init(ifp); 5756 5757 return (0); 5758 } 5759 5760 #ifdef IXGBE_DEBUG 5761 /************************************************************************ 5762 * ixgbe_sysctl_power_state 5763 * 5764 * Sysctl to test power states 5765 * Values: 5766 * 0 - set device to D0 5767 * 3 - set device to D3 5768 * (none) - get current device power state 5769 ************************************************************************/ 5770 static int 5771 ixgbe_sysctl_power_state(SYSCTLFN_ARGS) 5772 { 5773 #ifdef notyet 5774 struct sysctlnode node = *rnode; 5775 struct adapter *adapter = (struct adapter *)node.sysctl_data; 5776 device_t dev = adapter->dev; 5777 int curr_ps, new_ps, error = 0; 5778 5779 if (ixgbe_fw_recovery_mode_swflag(adapter)) 5780 return (EPERM); 5781 5782 curr_ps = new_ps = pci_get_powerstate(dev); 5783 5784 error = sysctl_lookup(SYSCTLFN_CALL(&node)); 5785 if ((error) || (req->newp == NULL)) 5786 return (error); 5787 5788 if (new_ps == curr_ps) 5789 return (0); 5790 5791 if (new_ps == 3 && curr_ps == 0) 5792 error = DEVICE_SUSPEND(dev); 5793 else if (new_ps == 0 && curr_ps == 3) 5794 error = DEVICE_RESUME(dev); 5795 else 5796 return (EINVAL); 5797 5798 device_printf(dev, "New state: %d\n", pci_get_powerstate(dev)); 5799 5800 return (error); 5801 #else 5802 return 0; 5803 #endif 5804 } /* ixgbe_sysctl_power_state */ 5805 #endif 5806 5807 /************************************************************************ 5808 * ixgbe_sysctl_wol_enable 5809 * 5810 * Sysctl to enable/disable the WoL capability, 5811 * if supported by the adapter. 5812 * 5813 * Values: 5814 * 0 - disabled 5815 * 1 - enabled 5816 ************************************************************************/ 5817 static int 5818 ixgbe_sysctl_wol_enable(SYSCTLFN_ARGS) 5819 { 5820 struct sysctlnode node = *rnode; 5821 struct adapter *adapter = (struct adapter *)node.sysctl_data; 5822 struct ixgbe_hw *hw = &adapter->hw; 5823 bool new_wol_enabled; 5824 int error = 0; 5825 5826 /* 5827 * It's not required to check recovery mode because this function never 5828 * touches hardware. 5829 */ 5830 new_wol_enabled = hw->wol_enabled; 5831 node.sysctl_data = &new_wol_enabled; 5832 error = sysctl_lookup(SYSCTLFN_CALL(&node)); 5833 if ((error) || (newp == NULL)) 5834 return (error); 5835 if (new_wol_enabled == hw->wol_enabled) 5836 return (0); 5837 5838 if (new_wol_enabled && !adapter->wol_support) 5839 return (ENODEV); 5840 else 5841 hw->wol_enabled = new_wol_enabled; 5842 5843 return (0); 5844 } /* ixgbe_sysctl_wol_enable */ 5845 5846 /************************************************************************ 5847 * ixgbe_sysctl_wufc - Wake Up Filter Control 5848 * 5849 * Sysctl to enable/disable the types of packets that the 5850 * adapter will wake up on upon receipt. 5851 * Flags: 5852 * 0x1 - Link Status Change 5853 * 0x2 - Magic Packet 5854 * 0x4 - Direct Exact 5855 * 0x8 - Directed Multicast 5856 * 0x10 - Broadcast 5857 * 0x20 - ARP/IPv4 Request Packet 5858 * 0x40 - Direct IPv4 Packet 5859 * 0x80 - Direct IPv6 Packet 5860 * 5861 * Settings not listed above will cause the sysctl to return an error. 5862 ************************************************************************/ 5863 static int 5864 ixgbe_sysctl_wufc(SYSCTLFN_ARGS) 5865 { 5866 struct sysctlnode node = *rnode; 5867 struct adapter *adapter = (struct adapter *)node.sysctl_data; 5868 int error = 0; 5869 u32 new_wufc; 5870 5871 /* 5872 * It's not required to check recovery mode because this function never 5873 * touches hardware. 5874 */ 5875 new_wufc = adapter->wufc; 5876 node.sysctl_data = &new_wufc; 5877 error = sysctl_lookup(SYSCTLFN_CALL(&node)); 5878 if ((error) || (newp == NULL)) 5879 return (error); 5880 if (new_wufc == adapter->wufc) 5881 return (0); 5882 5883 if (new_wufc & 0xffffff00) 5884 return (EINVAL); 5885 5886 new_wufc &= 0xff; 5887 new_wufc |= (0xffffff & adapter->wufc); 5888 adapter->wufc = new_wufc; 5889 5890 return (0); 5891 } /* ixgbe_sysctl_wufc */ 5892 5893 #ifdef IXGBE_DEBUG 5894 /************************************************************************ 5895 * ixgbe_sysctl_print_rss_config 5896 ************************************************************************/ 5897 static int 5898 ixgbe_sysctl_print_rss_config(SYSCTLFN_ARGS) 5899 { 5900 #ifdef notyet 5901 struct sysctlnode node = *rnode; 5902 struct adapter *adapter = (struct adapter *)node.sysctl_data; 5903 struct ixgbe_hw *hw = &adapter->hw; 5904 device_t dev = adapter->dev; 5905 struct sbuf *buf; 5906 int error = 0, reta_size; 5907 u32 reg; 5908 5909 if (ixgbe_fw_recovery_mode_swflag(adapter)) 5910 return (EPERM); 5911 5912 buf = sbuf_new_for_sysctl(NULL, NULL, 128, req); 5913 if (!buf) { 5914 device_printf(dev, "Could not allocate sbuf for output.\n"); 5915 return (ENOMEM); 5916 } 5917 5918 // TODO: use sbufs to make a string to print out 5919 /* Set multiplier for RETA setup and table size based on MAC */ 5920 switch (adapter->hw.mac.type) { 5921 case ixgbe_mac_X550: 5922 case ixgbe_mac_X550EM_x: 5923 case ixgbe_mac_X550EM_a: 5924 reta_size = 128; 5925 break; 5926 default: 5927 reta_size = 32; 5928 break; 5929 } 5930 5931 /* Print out the redirection table */ 5932 sbuf_cat(buf, "\n"); 5933 for (int i = 0; i < reta_size; i++) { 5934 if (i < 32) { 5935 reg = IXGBE_READ_REG(hw, IXGBE_RETA(i)); 5936 sbuf_printf(buf, "RETA(%2d): 0x%08x\n", i, reg); 5937 } else { 5938 reg = IXGBE_READ_REG(hw, IXGBE_ERETA(i - 32)); 5939 sbuf_printf(buf, "ERETA(%2d): 0x%08x\n", i - 32, reg); 5940 } 5941 } 5942 5943 // TODO: print more config 5944 5945 error = sbuf_finish(buf); 5946 if (error) 5947 device_printf(dev, "Error finishing sbuf: %d\n", error); 5948 5949 sbuf_delete(buf); 5950 #endif 5951 return (0); 5952 } /* ixgbe_sysctl_print_rss_config */ 5953 #endif /* IXGBE_DEBUG */ 5954 5955 /************************************************************************ 5956 * ixgbe_sysctl_phy_temp - Retrieve temperature of PHY 5957 * 5958 * For X552/X557-AT devices using an external PHY 5959 ************************************************************************/ 5960 static int 5961 ixgbe_sysctl_phy_temp(SYSCTLFN_ARGS) 5962 { 5963 struct sysctlnode node = *rnode; 5964 struct adapter *adapter = (struct adapter *)node.sysctl_data; 5965 struct ixgbe_hw *hw = &adapter->hw; 5966 int val; 5967 u16 reg; 5968 int error; 5969 5970 if (ixgbe_fw_recovery_mode_swflag(adapter)) 5971 return (EPERM); 5972 5973 if (hw->device_id != IXGBE_DEV_ID_X550EM_X_10G_T) { 5974 device_printf(adapter->dev, 5975 "Device has no supported external thermal sensor.\n"); 5976 return (ENODEV); 5977 } 5978 5979 if (hw->phy.ops.read_reg(hw, IXGBE_PHY_CURRENT_TEMP, 5980 IXGBE_MDIO_VENDOR_SPECIFIC_1_DEV_TYPE, ®)) { 5981 device_printf(adapter->dev, 5982 "Error reading from PHY's current temperature register\n"); 5983 return (EAGAIN); 5984 } 5985 5986 node.sysctl_data = &val; 5987 5988 /* Shift temp for output */ 5989 val = reg >> 8; 5990 5991 error = sysctl_lookup(SYSCTLFN_CALL(&node)); 5992 if ((error) || (newp == NULL)) 5993 return (error); 5994 5995 return (0); 5996 } /* ixgbe_sysctl_phy_temp */ 5997 5998 /************************************************************************ 5999 * ixgbe_sysctl_phy_overtemp_occurred 6000 * 6001 * Reports (directly from the PHY) whether the current PHY 6002 * temperature is over the overtemp threshold. 6003 ************************************************************************/ 6004 static int 6005 ixgbe_sysctl_phy_overtemp_occurred(SYSCTLFN_ARGS) 6006 { 6007 struct sysctlnode node = *rnode; 6008 struct adapter *adapter = (struct adapter *)node.sysctl_data; 6009 struct ixgbe_hw *hw = &adapter->hw; 6010 int val, error; 6011 u16 reg; 6012 6013 if (ixgbe_fw_recovery_mode_swflag(adapter)) 6014 return (EPERM); 6015 6016 if (hw->device_id != IXGBE_DEV_ID_X550EM_X_10G_T) { 6017 device_printf(adapter->dev, 6018 "Device has no supported external thermal sensor.\n"); 6019 return (ENODEV); 6020 } 6021 6022 if (hw->phy.ops.read_reg(hw, IXGBE_PHY_OVERTEMP_STATUS, 6023 IXGBE_MDIO_VENDOR_SPECIFIC_1_DEV_TYPE, ®)) { 6024 device_printf(adapter->dev, 6025 "Error reading from PHY's temperature status register\n"); 6026 return (EAGAIN); 6027 } 6028 6029 node.sysctl_data = &val; 6030 6031 /* Get occurrence bit */ 6032 val = !!(reg & 0x4000); 6033 6034 error = sysctl_lookup(SYSCTLFN_CALL(&node)); 6035 if ((error) || (newp == NULL)) 6036 return (error); 6037 6038 return (0); 6039 } /* ixgbe_sysctl_phy_overtemp_occurred */ 6040 6041 /************************************************************************ 6042 * ixgbe_sysctl_eee_state 6043 * 6044 * Sysctl to set EEE power saving feature 6045 * Values: 6046 * 0 - disable EEE 6047 * 1 - enable EEE 6048 * (none) - get current device EEE state 6049 ************************************************************************/ 6050 static int 6051 ixgbe_sysctl_eee_state(SYSCTLFN_ARGS) 6052 { 6053 struct sysctlnode node = *rnode; 6054 struct adapter *adapter = (struct adapter *)node.sysctl_data; 6055 struct ifnet *ifp = adapter->ifp; 6056 device_t dev = adapter->dev; 6057 int curr_eee, new_eee, error = 0; 6058 s32 retval; 6059 6060 if (ixgbe_fw_recovery_mode_swflag(adapter)) 6061 return (EPERM); 6062 6063 curr_eee = new_eee = !!(adapter->feat_en & IXGBE_FEATURE_EEE); 6064 node.sysctl_data = &new_eee; 6065 error = sysctl_lookup(SYSCTLFN_CALL(&node)); 6066 if ((error) || (newp == NULL)) 6067 return (error); 6068 6069 /* Nothing to do */ 6070 if (new_eee == curr_eee) 6071 return (0); 6072 6073 /* Not supported */ 6074 if (!(adapter->feat_cap & IXGBE_FEATURE_EEE)) 6075 return (EINVAL); 6076 6077 /* Bounds checking */ 6078 if ((new_eee < 0) || (new_eee > 1)) 6079 return (EINVAL); 6080 6081 retval = ixgbe_setup_eee(&adapter->hw, new_eee); 6082 if (retval) { 6083 device_printf(dev, "Error in EEE setup: 0x%08X\n", retval); 6084 return (EINVAL); 6085 } 6086 6087 /* Restart auto-neg */ 6088 if_init(ifp); 6089 6090 device_printf(dev, "New EEE state: %d\n", new_eee); 6091 6092 /* Cache new value */ 6093 if (new_eee) 6094 adapter->feat_en |= IXGBE_FEATURE_EEE; 6095 else 6096 adapter->feat_en &= ~IXGBE_FEATURE_EEE; 6097 6098 return (error); 6099 } /* ixgbe_sysctl_eee_state */ 6100 6101 #define PRINTQS(adapter, regname) \ 6102 do { \ 6103 struct ixgbe_hw *_hw = &(adapter)->hw; \ 6104 int _i; \ 6105 \ 6106 printf("%s: %s", device_xname((adapter)->dev), #regname); \ 6107 for (_i = 0; _i < (adapter)->num_queues; _i++) { \ 6108 printf((_i == 0) ? "\t" : " "); \ 6109 printf("%08x", IXGBE_READ_REG(_hw, \ 6110 IXGBE_##regname(_i))); \ 6111 } \ 6112 printf("\n"); \ 6113 } while (0) 6114 6115 /************************************************************************ 6116 * ixgbe_print_debug_info 6117 * 6118 * Called only when em_display_debug_stats is enabled. 6119 * Provides a way to take a look at important statistics 6120 * maintained by the driver and hardware. 6121 ************************************************************************/ 6122 static void 6123 ixgbe_print_debug_info(struct adapter *adapter) 6124 { 6125 device_t dev = adapter->dev; 6126 struct ixgbe_hw *hw = &adapter->hw; 6127 int table_size; 6128 int i; 6129 6130 switch (adapter->hw.mac.type) { 6131 case ixgbe_mac_X550: 6132 case ixgbe_mac_X550EM_x: 6133 case ixgbe_mac_X550EM_a: 6134 table_size = 128; 6135 break; 6136 default: 6137 table_size = 32; 6138 break; 6139 } 6140 6141 device_printf(dev, "[E]RETA:\n"); 6142 for (i = 0; i < table_size; i++) { 6143 if (i < 32) 6144 printf("%02x: %08x\n", i, IXGBE_READ_REG(hw, 6145 IXGBE_RETA(i))); 6146 else 6147 printf("%02x: %08x\n", i, IXGBE_READ_REG(hw, 6148 IXGBE_ERETA(i - 32))); 6149 } 6150 6151 device_printf(dev, "queue:"); 6152 for (i = 0; i < adapter->num_queues; i++) { 6153 printf((i == 0) ? "\t" : " "); 6154 printf("%8d", i); 6155 } 6156 printf("\n"); 6157 PRINTQS(adapter, RDBAL); 6158 PRINTQS(adapter, RDBAH); 6159 PRINTQS(adapter, RDLEN); 6160 PRINTQS(adapter, SRRCTL); 6161 PRINTQS(adapter, RDH); 6162 PRINTQS(adapter, RDT); 6163 PRINTQS(adapter, RXDCTL); 6164 6165 device_printf(dev, "RQSMR:"); 6166 for (i = 0; i < adapter->num_queues / 4; i++) { 6167 printf((i == 0) ? "\t" : " "); 6168 printf("%08x", IXGBE_READ_REG(hw, IXGBE_RQSMR(i))); 6169 } 6170 printf("\n"); 6171 6172 device_printf(dev, "disabled_count:"); 6173 for (i = 0; i < adapter->num_queues; i++) { 6174 printf((i == 0) ? "\t" : " "); 6175 printf("%8d", adapter->queues[i].disabled_count); 6176 } 6177 printf("\n"); 6178 6179 device_printf(dev, "EIMS:\t%08x\n", IXGBE_READ_REG(hw, IXGBE_EIMS)); 6180 if (hw->mac.type != ixgbe_mac_82598EB) { 6181 device_printf(dev, "EIMS_EX(0):\t%08x\n", 6182 IXGBE_READ_REG(hw, IXGBE_EIMS_EX(0))); 6183 device_printf(dev, "EIMS_EX(1):\t%08x\n", 6184 IXGBE_READ_REG(hw, IXGBE_EIMS_EX(1))); 6185 } 6186 device_printf(dev, "EIAM:\t%08x\n", IXGBE_READ_REG(hw, IXGBE_EIAM)); 6187 device_printf(dev, "EIAC:\t%08x\n", IXGBE_READ_REG(hw, IXGBE_EIAC)); 6188 } /* ixgbe_print_debug_info */ 6189 6190 /************************************************************************ 6191 * ixgbe_sysctl_debug 6192 ************************************************************************/ 6193 static int 6194 ixgbe_sysctl_debug(SYSCTLFN_ARGS) 6195 { 6196 struct sysctlnode node = *rnode; 6197 struct adapter *adapter = (struct adapter *)node.sysctl_data; 6198 int error, result = 0; 6199 6200 if (ixgbe_fw_recovery_mode_swflag(adapter)) 6201 return (EPERM); 6202 6203 node.sysctl_data = &result; 6204 error = sysctl_lookup(SYSCTLFN_CALL(&node)); 6205 6206 if (error || newp == NULL) 6207 return error; 6208 6209 if (result == 1) 6210 ixgbe_print_debug_info(adapter); 6211 6212 return 0; 6213 } /* ixgbe_sysctl_debug */ 6214 6215 /************************************************************************ 6216 * ixgbe_sysctl_rx_copy_len 6217 ************************************************************************/ 6218 static int 6219 ixgbe_sysctl_rx_copy_len(SYSCTLFN_ARGS) 6220 { 6221 struct sysctlnode node = *rnode; 6222 struct adapter *adapter = (struct adapter *)node.sysctl_data; 6223 int error; 6224 int result = adapter->rx_copy_len; 6225 6226 node.sysctl_data = &result; 6227 error = sysctl_lookup(SYSCTLFN_CALL(&node)); 6228 6229 if (error || newp == NULL) 6230 return error; 6231 6232 if ((result < 0) || (result > IXGBE_RX_COPY_LEN_MAX)) 6233 return EINVAL; 6234 6235 adapter->rx_copy_len = result; 6236 6237 return 0; 6238 } /* ixgbe_sysctl_rx_copy_len */ 6239 6240 /************************************************************************ 6241 * ixgbe_sysctl_tx_process_limit 6242 ************************************************************************/ 6243 static int 6244 ixgbe_sysctl_tx_process_limit(SYSCTLFN_ARGS) 6245 { 6246 struct sysctlnode node = *rnode; 6247 struct adapter *adapter = (struct adapter *)node.sysctl_data; 6248 int error; 6249 int result = adapter->tx_process_limit; 6250 6251 node.sysctl_data = &result; 6252 error = sysctl_lookup(SYSCTLFN_CALL(&node)); 6253 6254 if (error || newp == NULL) 6255 return error; 6256 6257 if ((result <= 0) || (result > adapter->num_tx_desc)) 6258 return EINVAL; 6259 6260 adapter->tx_process_limit = result; 6261 6262 return 0; 6263 } /* ixgbe_sysctl_tx_process_limit */ 6264 6265 /************************************************************************ 6266 * ixgbe_sysctl_rx_process_limit 6267 ************************************************************************/ 6268 static int 6269 ixgbe_sysctl_rx_process_limit(SYSCTLFN_ARGS) 6270 { 6271 struct sysctlnode node = *rnode; 6272 struct adapter *adapter = (struct adapter *)node.sysctl_data; 6273 int error; 6274 int result = adapter->rx_process_limit; 6275 6276 node.sysctl_data = &result; 6277 error = sysctl_lookup(SYSCTLFN_CALL(&node)); 6278 6279 if (error || newp == NULL) 6280 return error; 6281 6282 if ((result <= 0) || (result > adapter->num_rx_desc)) 6283 return EINVAL; 6284 6285 adapter->rx_process_limit = result; 6286 6287 return 0; 6288 } /* ixgbe_sysctl_rx_process_limit */ 6289 6290 /************************************************************************ 6291 * ixgbe_init_device_features 6292 ************************************************************************/ 6293 static void 6294 ixgbe_init_device_features(struct adapter *adapter) 6295 { 6296 adapter->feat_cap = IXGBE_FEATURE_NETMAP 6297 | IXGBE_FEATURE_RSS 6298 | IXGBE_FEATURE_MSI 6299 | IXGBE_FEATURE_MSIX 6300 | IXGBE_FEATURE_LEGACY_IRQ 6301 | IXGBE_FEATURE_LEGACY_TX; 6302 6303 /* Set capabilities first... */ 6304 switch (adapter->hw.mac.type) { 6305 case ixgbe_mac_82598EB: 6306 if (adapter->hw.device_id == IXGBE_DEV_ID_82598AT) 6307 adapter->feat_cap |= IXGBE_FEATURE_FAN_FAIL; 6308 break; 6309 case ixgbe_mac_X540: 6310 adapter->feat_cap |= IXGBE_FEATURE_SRIOV; 6311 adapter->feat_cap |= IXGBE_FEATURE_FDIR; 6312 if ((adapter->hw.device_id == IXGBE_DEV_ID_X540_BYPASS) && 6313 (adapter->hw.bus.func == 0)) 6314 adapter->feat_cap |= IXGBE_FEATURE_BYPASS; 6315 break; 6316 case ixgbe_mac_X550: 6317 /* 6318 * IXGBE_FEATURE_RECOVERY_MODE will be set after reading 6319 * NVM Image version. 6320 */ 6321 adapter->feat_cap |= IXGBE_FEATURE_TEMP_SENSOR; 6322 adapter->feat_cap |= IXGBE_FEATURE_SRIOV; 6323 adapter->feat_cap |= IXGBE_FEATURE_FDIR; 6324 break; 6325 case ixgbe_mac_X550EM_x: 6326 /* 6327 * IXGBE_FEATURE_RECOVERY_MODE will be set after reading 6328 * NVM Image version. 6329 */ 6330 adapter->feat_cap |= IXGBE_FEATURE_SRIOV; 6331 adapter->feat_cap |= IXGBE_FEATURE_FDIR; 6332 break; 6333 case ixgbe_mac_X550EM_a: 6334 /* 6335 * IXGBE_FEATURE_RECOVERY_MODE will be set after reading 6336 * NVM Image version. 6337 */ 6338 adapter->feat_cap |= IXGBE_FEATURE_SRIOV; 6339 adapter->feat_cap |= IXGBE_FEATURE_FDIR; 6340 adapter->feat_cap &= ~IXGBE_FEATURE_LEGACY_IRQ; 6341 if ((adapter->hw.device_id == IXGBE_DEV_ID_X550EM_A_1G_T) || 6342 (adapter->hw.device_id == IXGBE_DEV_ID_X550EM_A_1G_T_L)) { 6343 adapter->feat_cap |= IXGBE_FEATURE_TEMP_SENSOR; 6344 adapter->feat_cap |= IXGBE_FEATURE_EEE; 6345 } 6346 break; 6347 case ixgbe_mac_82599EB: 6348 adapter->feat_cap |= IXGBE_FEATURE_SRIOV; 6349 adapter->feat_cap |= IXGBE_FEATURE_FDIR; 6350 if ((adapter->hw.device_id == IXGBE_DEV_ID_82599_BYPASS) && 6351 (adapter->hw.bus.func == 0)) 6352 adapter->feat_cap |= IXGBE_FEATURE_BYPASS; 6353 if (adapter->hw.device_id == IXGBE_DEV_ID_82599_QSFP_SF_QP) 6354 adapter->feat_cap &= ~IXGBE_FEATURE_LEGACY_IRQ; 6355 break; 6356 default: 6357 break; 6358 } 6359 6360 /* Enabled by default... */ 6361 /* Fan failure detection */ 6362 if (adapter->feat_cap & IXGBE_FEATURE_FAN_FAIL) 6363 adapter->feat_en |= IXGBE_FEATURE_FAN_FAIL; 6364 /* Netmap */ 6365 if (adapter->feat_cap & IXGBE_FEATURE_NETMAP) 6366 adapter->feat_en |= IXGBE_FEATURE_NETMAP; 6367 /* EEE */ 6368 if (adapter->feat_cap & IXGBE_FEATURE_EEE) 6369 adapter->feat_en |= IXGBE_FEATURE_EEE; 6370 /* Thermal Sensor */ 6371 if (adapter->feat_cap & IXGBE_FEATURE_TEMP_SENSOR) 6372 adapter->feat_en |= IXGBE_FEATURE_TEMP_SENSOR; 6373 /* 6374 * Recovery mode: 6375 * NetBSD: IXGBE_FEATURE_RECOVERY_MODE will be controlled after reading 6376 * NVM Image version. 6377 */ 6378 6379 /* Enabled via global sysctl... */ 6380 /* Flow Director */ 6381 if (ixgbe_enable_fdir) { 6382 if (adapter->feat_cap & IXGBE_FEATURE_FDIR) 6383 adapter->feat_en |= IXGBE_FEATURE_FDIR; 6384 else 6385 device_printf(adapter->dev, "Device does not support Flow Director. Leaving disabled."); 6386 } 6387 /* Legacy (single queue) transmit */ 6388 if ((adapter->feat_cap & IXGBE_FEATURE_LEGACY_TX) && 6389 ixgbe_enable_legacy_tx) 6390 adapter->feat_en |= IXGBE_FEATURE_LEGACY_TX; 6391 /* 6392 * Message Signal Interrupts - Extended (MSI-X) 6393 * Normal MSI is only enabled if MSI-X calls fail. 6394 */ 6395 if (!ixgbe_enable_msix) 6396 adapter->feat_cap &= ~IXGBE_FEATURE_MSIX; 6397 /* Receive-Side Scaling (RSS) */ 6398 if ((adapter->feat_cap & IXGBE_FEATURE_RSS) && ixgbe_enable_rss) 6399 adapter->feat_en |= IXGBE_FEATURE_RSS; 6400 6401 /* Disable features with unmet dependencies... */ 6402 /* No MSI-X */ 6403 if (!(adapter->feat_cap & IXGBE_FEATURE_MSIX)) { 6404 adapter->feat_cap &= ~IXGBE_FEATURE_RSS; 6405 adapter->feat_cap &= ~IXGBE_FEATURE_SRIOV; 6406 adapter->feat_en &= ~IXGBE_FEATURE_RSS; 6407 adapter->feat_en &= ~IXGBE_FEATURE_SRIOV; 6408 } 6409 } /* ixgbe_init_device_features */ 6410 6411 /************************************************************************ 6412 * ixgbe_probe - Device identification routine 6413 * 6414 * Determines if the driver should be loaded on 6415 * adapter based on its PCI vendor/device ID. 6416 * 6417 * return BUS_PROBE_DEFAULT on success, positive on failure 6418 ************************************************************************/ 6419 static int 6420 ixgbe_probe(device_t dev, cfdata_t cf, void *aux) 6421 { 6422 const struct pci_attach_args *pa = aux; 6423 6424 return (ixgbe_lookup(pa) != NULL) ? 1 : 0; 6425 } 6426 6427 static const ixgbe_vendor_info_t * 6428 ixgbe_lookup(const struct pci_attach_args *pa) 6429 { 6430 const ixgbe_vendor_info_t *ent; 6431 pcireg_t subid; 6432 6433 INIT_DEBUGOUT("ixgbe_lookup: begin"); 6434 6435 if (PCI_VENDOR(pa->pa_id) != IXGBE_INTEL_VENDOR_ID) 6436 return NULL; 6437 6438 subid = pci_conf_read(pa->pa_pc, pa->pa_tag, PCI_SUBSYS_ID_REG); 6439 6440 for (ent = ixgbe_vendor_info_array; ent->vendor_id != 0; ent++) { 6441 if ((PCI_VENDOR(pa->pa_id) == ent->vendor_id) && 6442 (PCI_PRODUCT(pa->pa_id) == ent->device_id) && 6443 ((PCI_SUBSYS_VENDOR(subid) == ent->subvendor_id) || 6444 (ent->subvendor_id == 0)) && 6445 ((PCI_SUBSYS_ID(subid) == ent->subdevice_id) || 6446 (ent->subdevice_id == 0))) { 6447 return ent; 6448 } 6449 } 6450 return NULL; 6451 } 6452 6453 static int 6454 ixgbe_ifflags_cb(struct ethercom *ec) 6455 { 6456 struct ifnet *ifp = &ec->ec_if; 6457 struct adapter *adapter = ifp->if_softc; 6458 u_short change; 6459 int rv = 0; 6460 6461 IXGBE_CORE_LOCK(adapter); 6462 6463 change = ifp->if_flags ^ adapter->if_flags; 6464 if (change != 0) 6465 adapter->if_flags = ifp->if_flags; 6466 6467 if ((change & ~(IFF_CANTCHANGE | IFF_DEBUG)) != 0) { 6468 rv = ENETRESET; 6469 goto out; 6470 } else if ((change & IFF_PROMISC) != 0) 6471 ixgbe_set_rxfilter(adapter); 6472 6473 /* Check for ec_capenable. */ 6474 change = ec->ec_capenable ^ adapter->ec_capenable; 6475 adapter->ec_capenable = ec->ec_capenable; 6476 if ((change & ~(ETHERCAP_VLAN_MTU | ETHERCAP_VLAN_HWTAGGING 6477 | ETHERCAP_VLAN_HWFILTER)) != 0) { 6478 rv = ENETRESET; 6479 goto out; 6480 } 6481 6482 /* 6483 * Special handling is not required for ETHERCAP_VLAN_MTU. 6484 * MAXFRS(MHADD) does not include the 4bytes of the VLAN header. 6485 */ 6486 6487 /* Set up VLAN support and filter */ 6488 if ((change & (ETHERCAP_VLAN_HWTAGGING | ETHERCAP_VLAN_HWFILTER)) != 0) 6489 ixgbe_setup_vlan_hw_support(adapter); 6490 6491 out: 6492 IXGBE_CORE_UNLOCK(adapter); 6493 6494 return rv; 6495 } 6496 6497 /************************************************************************ 6498 * ixgbe_ioctl - Ioctl entry point 6499 * 6500 * Called when the user wants to configure the interface. 6501 * 6502 * return 0 on success, positive on failure 6503 ************************************************************************/ 6504 static int 6505 ixgbe_ioctl(struct ifnet *ifp, u_long command, void *data) 6506 { 6507 struct adapter *adapter = ifp->if_softc; 6508 struct ixgbe_hw *hw = &adapter->hw; 6509 struct ifcapreq *ifcr = data; 6510 struct ifreq *ifr = data; 6511 int error = 0; 6512 int l4csum_en; 6513 const int l4csum = IFCAP_CSUM_TCPv4_Rx | IFCAP_CSUM_UDPv4_Rx | 6514 IFCAP_CSUM_TCPv6_Rx | IFCAP_CSUM_UDPv6_Rx; 6515 6516 if (ixgbe_fw_recovery_mode_swflag(adapter)) 6517 return (EPERM); 6518 6519 switch (command) { 6520 case SIOCSIFFLAGS: 6521 IOCTL_DEBUGOUT("ioctl: SIOCSIFFLAGS (Set Interface Flags)"); 6522 break; 6523 case SIOCADDMULTI: 6524 case SIOCDELMULTI: 6525 IOCTL_DEBUGOUT("ioctl: SIOC(ADD|DEL)MULTI"); 6526 break; 6527 case SIOCSIFMEDIA: 6528 case SIOCGIFMEDIA: 6529 IOCTL_DEBUGOUT("ioctl: SIOCxIFMEDIA (Get/Set Interface Media)"); 6530 break; 6531 case SIOCSIFCAP: 6532 IOCTL_DEBUGOUT("ioctl: SIOCSIFCAP (Set Capabilities)"); 6533 break; 6534 case SIOCSIFMTU: 6535 IOCTL_DEBUGOUT("ioctl: SIOCSIFMTU (Set Interface MTU)"); 6536 break; 6537 #ifdef __NetBSD__ 6538 case SIOCINITIFADDR: 6539 IOCTL_DEBUGOUT("ioctl: SIOCINITIFADDR"); 6540 break; 6541 case SIOCGIFFLAGS: 6542 IOCTL_DEBUGOUT("ioctl: SIOCGIFFLAGS"); 6543 break; 6544 case SIOCGIFAFLAG_IN: 6545 IOCTL_DEBUGOUT("ioctl: SIOCGIFAFLAG_IN"); 6546 break; 6547 case SIOCGIFADDR: 6548 IOCTL_DEBUGOUT("ioctl: SIOCGIFADDR"); 6549 break; 6550 case SIOCGIFMTU: 6551 IOCTL_DEBUGOUT("ioctl: SIOCGIFMTU (Get Interface MTU)"); 6552 break; 6553 case SIOCGIFCAP: 6554 IOCTL_DEBUGOUT("ioctl: SIOCGIFCAP (Get IF cap)"); 6555 break; 6556 case SIOCGETHERCAP: 6557 IOCTL_DEBUGOUT("ioctl: SIOCGETHERCAP (Get ethercap)"); 6558 break; 6559 case SIOCGLIFADDR: 6560 IOCTL_DEBUGOUT("ioctl: SIOCGLIFADDR (Get Interface addr)"); 6561 break; 6562 case SIOCZIFDATA: 6563 IOCTL_DEBUGOUT("ioctl: SIOCZIFDATA (Zero counter)"); 6564 hw->mac.ops.clear_hw_cntrs(hw); 6565 ixgbe_clear_evcnt(adapter); 6566 break; 6567 case SIOCAIFADDR: 6568 IOCTL_DEBUGOUT("ioctl: SIOCAIFADDR (add/chg IF alias)"); 6569 break; 6570 #endif 6571 default: 6572 IOCTL_DEBUGOUT1("ioctl: UNKNOWN (0x%X)", (int)command); 6573 break; 6574 } 6575 6576 switch (command) { 6577 case SIOCGI2C: 6578 { 6579 struct ixgbe_i2c_req i2c; 6580 6581 IOCTL_DEBUGOUT("ioctl: SIOCGI2C (Get I2C Data)"); 6582 error = copyin(ifr->ifr_data, &i2c, sizeof(i2c)); 6583 if (error != 0) 6584 break; 6585 if (i2c.dev_addr != 0xA0 && i2c.dev_addr != 0xA2) { 6586 error = EINVAL; 6587 break; 6588 } 6589 if (i2c.len > sizeof(i2c.data)) { 6590 error = EINVAL; 6591 break; 6592 } 6593 6594 hw->phy.ops.read_i2c_byte(hw, i2c.offset, 6595 i2c.dev_addr, i2c.data); 6596 error = copyout(&i2c, ifr->ifr_data, sizeof(i2c)); 6597 break; 6598 } 6599 case SIOCSIFCAP: 6600 /* Layer-4 Rx checksum offload has to be turned on and 6601 * off as a unit. 6602 */ 6603 l4csum_en = ifcr->ifcr_capenable & l4csum; 6604 if (l4csum_en != l4csum && l4csum_en != 0) 6605 return EINVAL; 6606 /*FALLTHROUGH*/ 6607 case SIOCADDMULTI: 6608 case SIOCDELMULTI: 6609 case SIOCSIFFLAGS: 6610 case SIOCSIFMTU: 6611 default: 6612 if ((error = ether_ioctl(ifp, command, data)) != ENETRESET) 6613 return error; 6614 if ((ifp->if_flags & IFF_RUNNING) == 0) 6615 ; 6616 else if (command == SIOCSIFCAP || command == SIOCSIFMTU) { 6617 IXGBE_CORE_LOCK(adapter); 6618 if ((ifp->if_flags & IFF_RUNNING) != 0) 6619 ixgbe_init_locked(adapter); 6620 ixgbe_recalculate_max_frame(adapter); 6621 IXGBE_CORE_UNLOCK(adapter); 6622 } else if (command == SIOCADDMULTI || command == SIOCDELMULTI) { 6623 /* 6624 * Multicast list has changed; set the hardware filter 6625 * accordingly. 6626 */ 6627 IXGBE_CORE_LOCK(adapter); 6628 ixgbe_disable_intr(adapter); 6629 ixgbe_set_rxfilter(adapter); 6630 ixgbe_enable_intr(adapter); 6631 IXGBE_CORE_UNLOCK(adapter); 6632 } 6633 return 0; 6634 } 6635 6636 return error; 6637 } /* ixgbe_ioctl */ 6638 6639 /************************************************************************ 6640 * ixgbe_check_fan_failure 6641 ************************************************************************/ 6642 static int 6643 ixgbe_check_fan_failure(struct adapter *adapter, u32 reg, bool in_interrupt) 6644 { 6645 u32 mask; 6646 6647 mask = (in_interrupt) ? IXGBE_EICR_GPI_SDP1_BY_MAC(&adapter->hw) : 6648 IXGBE_ESDP_SDP1; 6649 6650 if ((reg & mask) == 0) 6651 return IXGBE_SUCCESS; 6652 6653 /* 6654 * Use ratecheck() just in case interrupt occur frequently. 6655 * When EXPX9501AT's fan stopped, interrupt occurred only once, 6656 * an red LED on the board turned on and link never up until 6657 * power off. 6658 */ 6659 if (ratecheck(&adapter->lasterr_time, &ixgbe_errlog_intrvl)) 6660 device_printf(adapter->dev, 6661 "\nCRITICAL: FAN FAILURE!! REPLACE IMMEDIATELY!!\n"); 6662 6663 return IXGBE_ERR_FAN_FAILURE; 6664 } /* ixgbe_check_fan_failure */ 6665 6666 /************************************************************************ 6667 * ixgbe_handle_que 6668 ************************************************************************/ 6669 static void 6670 ixgbe_handle_que(void *context) 6671 { 6672 struct ix_queue *que = context; 6673 struct adapter *adapter = que->adapter; 6674 struct tx_ring *txr = que->txr; 6675 struct ifnet *ifp = adapter->ifp; 6676 bool more = false; 6677 6678 IXGBE_EVC_ADD(&que->handleq, 1); 6679 6680 if (ifp->if_flags & IFF_RUNNING) { 6681 more = ixgbe_rxeof(que); 6682 IXGBE_TX_LOCK(txr); 6683 more |= ixgbe_txeof(txr); 6684 if (!(adapter->feat_en & IXGBE_FEATURE_LEGACY_TX)) 6685 if (!ixgbe_mq_ring_empty(ifp, txr->txr_interq)) 6686 ixgbe_mq_start_locked(ifp, txr); 6687 /* Only for queue 0 */ 6688 /* NetBSD still needs this for CBQ */ 6689 if ((&adapter->queues[0] == que) 6690 && (!ixgbe_legacy_ring_empty(ifp, NULL))) 6691 ixgbe_legacy_start_locked(ifp, txr); 6692 IXGBE_TX_UNLOCK(txr); 6693 } 6694 6695 if (more) { 6696 IXGBE_EVC_ADD(&que->req, 1); 6697 ixgbe_sched_handle_que(adapter, que); 6698 } else if (que->res != NULL) { 6699 /* MSIX: Re-enable this interrupt */ 6700 ixgbe_enable_queue(adapter, que->msix); 6701 } else { 6702 /* INTx or MSI */ 6703 ixgbe_enable_queue(adapter, 0); 6704 } 6705 6706 return; 6707 } /* ixgbe_handle_que */ 6708 6709 /************************************************************************ 6710 * ixgbe_handle_que_work 6711 ************************************************************************/ 6712 static void 6713 ixgbe_handle_que_work(struct work *wk, void *context) 6714 { 6715 struct ix_queue *que = container_of(wk, struct ix_queue, wq_cookie); 6716 6717 /* 6718 * "enqueued flag" is not required here. 6719 * See ixgbe_msix_que(). 6720 */ 6721 ixgbe_handle_que(que); 6722 } 6723 6724 /************************************************************************ 6725 * ixgbe_allocate_legacy - Setup the Legacy or MSI Interrupt handler 6726 ************************************************************************/ 6727 static int 6728 ixgbe_allocate_legacy(struct adapter *adapter, 6729 const struct pci_attach_args *pa) 6730 { 6731 device_t dev = adapter->dev; 6732 struct ix_queue *que = adapter->queues; 6733 struct tx_ring *txr = adapter->tx_rings; 6734 int counts[PCI_INTR_TYPE_SIZE]; 6735 pci_intr_type_t intr_type, max_type; 6736 char intrbuf[PCI_INTRSTR_LEN]; 6737 char wqname[MAXCOMLEN]; 6738 const char *intrstr = NULL; 6739 int defertx_error = 0, error; 6740 6741 /* We allocate a single interrupt resource */ 6742 max_type = PCI_INTR_TYPE_MSI; 6743 counts[PCI_INTR_TYPE_MSIX] = 0; 6744 counts[PCI_INTR_TYPE_MSI] = 6745 (adapter->feat_en & IXGBE_FEATURE_MSI) ? 1 : 0; 6746 /* Check not feat_en but feat_cap to fallback to INTx */ 6747 counts[PCI_INTR_TYPE_INTX] = 6748 (adapter->feat_cap & IXGBE_FEATURE_LEGACY_IRQ) ? 1 : 0; 6749 6750 alloc_retry: 6751 if (pci_intr_alloc(pa, &adapter->osdep.intrs, counts, max_type) != 0) { 6752 aprint_error_dev(dev, "couldn't alloc interrupt\n"); 6753 return ENXIO; 6754 } 6755 adapter->osdep.nintrs = 1; 6756 intrstr = pci_intr_string(adapter->osdep.pc, adapter->osdep.intrs[0], 6757 intrbuf, sizeof(intrbuf)); 6758 adapter->osdep.ihs[0] = pci_intr_establish_xname(adapter->osdep.pc, 6759 adapter->osdep.intrs[0], IPL_NET, ixgbe_legacy_irq, que, 6760 device_xname(dev)); 6761 intr_type = pci_intr_type(adapter->osdep.pc, adapter->osdep.intrs[0]); 6762 if (adapter->osdep.ihs[0] == NULL) { 6763 aprint_error_dev(dev,"unable to establish %s\n", 6764 (intr_type == PCI_INTR_TYPE_MSI) ? "MSI" : "INTx"); 6765 pci_intr_release(adapter->osdep.pc, adapter->osdep.intrs, 1); 6766 adapter->osdep.intrs = NULL; 6767 switch (intr_type) { 6768 case PCI_INTR_TYPE_MSI: 6769 /* The next try is for INTx: Disable MSI */ 6770 max_type = PCI_INTR_TYPE_INTX; 6771 counts[PCI_INTR_TYPE_INTX] = 1; 6772 adapter->feat_en &= ~IXGBE_FEATURE_MSI; 6773 if (adapter->feat_cap & IXGBE_FEATURE_LEGACY_IRQ) { 6774 adapter->feat_en |= IXGBE_FEATURE_LEGACY_IRQ; 6775 goto alloc_retry; 6776 } else 6777 break; 6778 case PCI_INTR_TYPE_INTX: 6779 default: 6780 /* See below */ 6781 break; 6782 } 6783 } 6784 if (intr_type == PCI_INTR_TYPE_INTX) { 6785 adapter->feat_en &= ~IXGBE_FEATURE_MSI; 6786 adapter->feat_en |= IXGBE_FEATURE_LEGACY_IRQ; 6787 } 6788 if (adapter->osdep.ihs[0] == NULL) { 6789 aprint_error_dev(dev, 6790 "couldn't establish interrupt%s%s\n", 6791 intrstr ? " at " : "", intrstr ? intrstr : ""); 6792 pci_intr_release(adapter->osdep.pc, adapter->osdep.intrs, 1); 6793 adapter->osdep.intrs = NULL; 6794 return ENXIO; 6795 } 6796 aprint_normal_dev(dev, "interrupting at %s\n", intrstr); 6797 /* 6798 * Try allocating a fast interrupt and the associated deferred 6799 * processing contexts. 6800 */ 6801 if (!(adapter->feat_en & IXGBE_FEATURE_LEGACY_TX)) { 6802 txr->txr_si = 6803 softint_establish(SOFTINT_NET | IXGBE_SOFTINT_FLAGS, 6804 ixgbe_deferred_mq_start, txr); 6805 6806 snprintf(wqname, sizeof(wqname), "%sdeferTx", 6807 device_xname(dev)); 6808 defertx_error = workqueue_create(&adapter->txr_wq, wqname, 6809 ixgbe_deferred_mq_start_work, adapter, IXGBE_WORKQUEUE_PRI, 6810 IPL_NET, IXGBE_WORKQUEUE_FLAGS); 6811 adapter->txr_wq_enqueued = percpu_alloc(sizeof(u_int)); 6812 } 6813 que->que_si = softint_establish(SOFTINT_NET | IXGBE_SOFTINT_FLAGS, 6814 ixgbe_handle_que, que); 6815 snprintf(wqname, sizeof(wqname), "%sTxRx", device_xname(dev)); 6816 error = workqueue_create(&adapter->que_wq, wqname, 6817 ixgbe_handle_que_work, adapter, IXGBE_WORKQUEUE_PRI, IPL_NET, 6818 IXGBE_WORKQUEUE_FLAGS); 6819 6820 if ((!(adapter->feat_en & IXGBE_FEATURE_LEGACY_TX) 6821 && ((txr->txr_si == NULL) || defertx_error != 0)) 6822 || (que->que_si == NULL) || error != 0) { 6823 aprint_error_dev(dev, 6824 "could not establish software interrupts\n"); 6825 6826 return ENXIO; 6827 } 6828 /* For simplicity in the handlers */ 6829 adapter->active_queues = IXGBE_EIMS_ENABLE_MASK; 6830 6831 return (0); 6832 } /* ixgbe_allocate_legacy */ 6833 6834 /************************************************************************ 6835 * ixgbe_allocate_msix - Setup MSI-X Interrupt resources and handlers 6836 ************************************************************************/ 6837 static int 6838 ixgbe_allocate_msix(struct adapter *adapter, const struct pci_attach_args *pa) 6839 { 6840 device_t dev = adapter->dev; 6841 struct ix_queue *que = adapter->queues; 6842 struct tx_ring *txr = adapter->tx_rings; 6843 pci_chipset_tag_t pc; 6844 char intrbuf[PCI_INTRSTR_LEN]; 6845 char intr_xname[32]; 6846 char wqname[MAXCOMLEN]; 6847 const char *intrstr = NULL; 6848 int error, vector = 0; 6849 int cpu_id = 0; 6850 kcpuset_t *affinity; 6851 #ifdef RSS 6852 unsigned int rss_buckets = 0; 6853 kcpuset_t cpu_mask; 6854 #endif 6855 6856 pc = adapter->osdep.pc; 6857 #ifdef RSS 6858 /* 6859 * If we're doing RSS, the number of queues needs to 6860 * match the number of RSS buckets that are configured. 6861 * 6862 * + If there's more queues than RSS buckets, we'll end 6863 * up with queues that get no traffic. 6864 * 6865 * + If there's more RSS buckets than queues, we'll end 6866 * up having multiple RSS buckets map to the same queue, 6867 * so there'll be some contention. 6868 */ 6869 rss_buckets = rss_getnumbuckets(); 6870 if ((adapter->feat_en & IXGBE_FEATURE_RSS) && 6871 (adapter->num_queues != rss_buckets)) { 6872 device_printf(dev, 6873 "%s: number of queues (%d) != number of RSS buckets (%d)" 6874 "; performance will be impacted.\n", 6875 __func__, adapter->num_queues, rss_buckets); 6876 } 6877 #endif 6878 6879 adapter->osdep.nintrs = adapter->num_queues + 1; 6880 if (pci_msix_alloc_exact(pa, &adapter->osdep.intrs, 6881 adapter->osdep.nintrs) != 0) { 6882 aprint_error_dev(dev, 6883 "failed to allocate MSI-X interrupt\n"); 6884 return (ENXIO); 6885 } 6886 6887 kcpuset_create(&affinity, false); 6888 for (int i = 0; i < adapter->num_queues; i++, vector++, que++, txr++) { 6889 snprintf(intr_xname, sizeof(intr_xname), "%s TXRX%d", 6890 device_xname(dev), i); 6891 intrstr = pci_intr_string(pc, adapter->osdep.intrs[i], intrbuf, 6892 sizeof(intrbuf)); 6893 #ifdef IXGBE_MPSAFE 6894 pci_intr_setattr(pc, &adapter->osdep.intrs[i], PCI_INTR_MPSAFE, 6895 true); 6896 #endif 6897 /* Set the handler function */ 6898 que->res = adapter->osdep.ihs[i] = pci_intr_establish_xname(pc, 6899 adapter->osdep.intrs[i], IPL_NET, ixgbe_msix_que, que, 6900 intr_xname); 6901 if (que->res == NULL) { 6902 aprint_error_dev(dev, 6903 "Failed to register QUE handler\n"); 6904 error = ENXIO; 6905 goto err_out; 6906 } 6907 que->msix = vector; 6908 adapter->active_queues |= 1ULL << que->msix; 6909 6910 if (adapter->feat_en & IXGBE_FEATURE_RSS) { 6911 #ifdef RSS 6912 /* 6913 * The queue ID is used as the RSS layer bucket ID. 6914 * We look up the queue ID -> RSS CPU ID and select 6915 * that. 6916 */ 6917 cpu_id = rss_getcpu(i % rss_getnumbuckets()); 6918 CPU_SETOF(cpu_id, &cpu_mask); 6919 #endif 6920 } else { 6921 /* 6922 * Bind the MSI-X vector, and thus the 6923 * rings to the corresponding CPU. 6924 * 6925 * This just happens to match the default RSS 6926 * round-robin bucket -> queue -> CPU allocation. 6927 */ 6928 if (adapter->num_queues > 1) 6929 cpu_id = i; 6930 } 6931 /* Round-robin affinity */ 6932 kcpuset_zero(affinity); 6933 kcpuset_set(affinity, cpu_id % ncpu); 6934 error = interrupt_distribute(adapter->osdep.ihs[i], affinity, 6935 NULL); 6936 aprint_normal_dev(dev, "for TX/RX, interrupting at %s", 6937 intrstr); 6938 if (error == 0) { 6939 #if 1 /* def IXGBE_DEBUG */ 6940 #ifdef RSS 6941 aprintf_normal(", bound RSS bucket %d to CPU %d", i, 6942 cpu_id % ncpu); 6943 #else 6944 aprint_normal(", bound queue %d to cpu %d", i, 6945 cpu_id % ncpu); 6946 #endif 6947 #endif /* IXGBE_DEBUG */ 6948 } 6949 aprint_normal("\n"); 6950 6951 if (!(adapter->feat_en & IXGBE_FEATURE_LEGACY_TX)) { 6952 txr->txr_si = softint_establish( 6953 SOFTINT_NET | IXGBE_SOFTINT_FLAGS, 6954 ixgbe_deferred_mq_start, txr); 6955 if (txr->txr_si == NULL) { 6956 aprint_error_dev(dev, 6957 "couldn't establish software interrupt\n"); 6958 error = ENXIO; 6959 goto err_out; 6960 } 6961 } 6962 que->que_si 6963 = softint_establish(SOFTINT_NET | IXGBE_SOFTINT_FLAGS, 6964 ixgbe_handle_que, que); 6965 if (que->que_si == NULL) { 6966 aprint_error_dev(dev, 6967 "couldn't establish software interrupt\n"); 6968 error = ENXIO; 6969 goto err_out; 6970 } 6971 } 6972 snprintf(wqname, sizeof(wqname), "%sdeferTx", device_xname(dev)); 6973 error = workqueue_create(&adapter->txr_wq, wqname, 6974 ixgbe_deferred_mq_start_work, adapter, IXGBE_WORKQUEUE_PRI, IPL_NET, 6975 IXGBE_WORKQUEUE_FLAGS); 6976 if (error) { 6977 aprint_error_dev(dev, 6978 "couldn't create workqueue for deferred Tx\n"); 6979 goto err_out; 6980 } 6981 adapter->txr_wq_enqueued = percpu_alloc(sizeof(u_int)); 6982 6983 snprintf(wqname, sizeof(wqname), "%sTxRx", device_xname(dev)); 6984 error = workqueue_create(&adapter->que_wq, wqname, 6985 ixgbe_handle_que_work, adapter, IXGBE_WORKQUEUE_PRI, IPL_NET, 6986 IXGBE_WORKQUEUE_FLAGS); 6987 if (error) { 6988 aprint_error_dev(dev, "couldn't create workqueue for Tx/Rx\n"); 6989 goto err_out; 6990 } 6991 6992 /* and Link */ 6993 cpu_id++; 6994 snprintf(intr_xname, sizeof(intr_xname), "%s link", device_xname(dev)); 6995 adapter->vector = vector; 6996 intrstr = pci_intr_string(pc, adapter->osdep.intrs[vector], intrbuf, 6997 sizeof(intrbuf)); 6998 #ifdef IXGBE_MPSAFE 6999 pci_intr_setattr(pc, &adapter->osdep.intrs[vector], PCI_INTR_MPSAFE, 7000 true); 7001 #endif 7002 /* Set the link handler function */ 7003 adapter->osdep.ihs[vector] = pci_intr_establish_xname(pc, 7004 adapter->osdep.intrs[vector], IPL_NET, ixgbe_msix_admin, adapter, 7005 intr_xname); 7006 if (adapter->osdep.ihs[vector] == NULL) { 7007 aprint_error_dev(dev, "Failed to register LINK handler\n"); 7008 error = ENXIO; 7009 goto err_out; 7010 } 7011 /* Round-robin affinity */ 7012 kcpuset_zero(affinity); 7013 kcpuset_set(affinity, cpu_id % ncpu); 7014 error = interrupt_distribute(adapter->osdep.ihs[vector], affinity, 7015 NULL); 7016 7017 aprint_normal_dev(dev, 7018 "for link, interrupting at %s", intrstr); 7019 if (error == 0) 7020 aprint_normal(", affinity to cpu %d\n", cpu_id % ncpu); 7021 else 7022 aprint_normal("\n"); 7023 7024 kcpuset_destroy(affinity); 7025 aprint_normal_dev(dev, 7026 "Using MSI-X interrupts with %d vectors\n", vector + 1); 7027 7028 return (0); 7029 7030 err_out: 7031 kcpuset_destroy(affinity); 7032 ixgbe_free_deferred_handlers(adapter); 7033 ixgbe_free_pciintr_resources(adapter); 7034 return (error); 7035 } /* ixgbe_allocate_msix */ 7036 7037 /************************************************************************ 7038 * ixgbe_configure_interrupts 7039 * 7040 * Setup MSI-X, MSI, or legacy interrupts (in that order). 7041 * This will also depend on user settings. 7042 ************************************************************************/ 7043 static int 7044 ixgbe_configure_interrupts(struct adapter *adapter) 7045 { 7046 device_t dev = adapter->dev; 7047 struct ixgbe_mac_info *mac = &adapter->hw.mac; 7048 int want, queues, msgs; 7049 7050 /* Default to 1 queue if MSI-X setup fails */ 7051 adapter->num_queues = 1; 7052 7053 /* Override by tuneable */ 7054 if (!(adapter->feat_cap & IXGBE_FEATURE_MSIX)) 7055 goto msi; 7056 7057 /* 7058 * NetBSD only: Use single vector MSI when number of CPU is 1 to save 7059 * interrupt slot. 7060 */ 7061 if (ncpu == 1) 7062 goto msi; 7063 7064 /* First try MSI-X */ 7065 msgs = pci_msix_count(adapter->osdep.pc, adapter->osdep.tag); 7066 msgs = MIN(msgs, IXG_MAX_NINTR); 7067 if (msgs < 2) 7068 goto msi; 7069 7070 adapter->msix_mem = (void *)1; /* XXX */ 7071 7072 /* Figure out a reasonable auto config value */ 7073 queues = (ncpu > (msgs - 1)) ? (msgs - 1) : ncpu; 7074 7075 #ifdef RSS 7076 /* If we're doing RSS, clamp at the number of RSS buckets */ 7077 if (adapter->feat_en & IXGBE_FEATURE_RSS) 7078 queues = uimin(queues, rss_getnumbuckets()); 7079 #endif 7080 if (ixgbe_num_queues > queues) { 7081 aprint_error_dev(adapter->dev, "ixgbe_num_queues (%d) is too large, using reduced amount (%d).\n", ixgbe_num_queues, queues); 7082 ixgbe_num_queues = queues; 7083 } 7084 7085 if (ixgbe_num_queues != 0) 7086 queues = ixgbe_num_queues; 7087 else 7088 queues = uimin(queues, 7089 uimin(mac->max_tx_queues, mac->max_rx_queues)); 7090 7091 /* reflect correct sysctl value */ 7092 ixgbe_num_queues = queues; 7093 7094 /* 7095 * Want one vector (RX/TX pair) per queue 7096 * plus an additional for Link. 7097 */ 7098 want = queues + 1; 7099 if (msgs >= want) 7100 msgs = want; 7101 else { 7102 aprint_error_dev(dev, "MSI-X Configuration Problem, " 7103 "%d vectors but %d queues wanted!\n", 7104 msgs, want); 7105 goto msi; 7106 } 7107 adapter->num_queues = queues; 7108 adapter->feat_en |= IXGBE_FEATURE_MSIX; 7109 return (0); 7110 7111 /* 7112 * MSI-X allocation failed or provided us with 7113 * less vectors than needed. Free MSI-X resources 7114 * and we'll try enabling MSI. 7115 */ 7116 msi: 7117 /* Without MSI-X, some features are no longer supported */ 7118 adapter->feat_cap &= ~IXGBE_FEATURE_RSS; 7119 adapter->feat_en &= ~IXGBE_FEATURE_RSS; 7120 adapter->feat_cap &= ~IXGBE_FEATURE_SRIOV; 7121 adapter->feat_en &= ~IXGBE_FEATURE_SRIOV; 7122 7123 msgs = pci_msi_count(adapter->osdep.pc, adapter->osdep.tag); 7124 adapter->msix_mem = NULL; /* XXX */ 7125 if (msgs > 1) 7126 msgs = 1; 7127 if (msgs != 0) { 7128 msgs = 1; 7129 adapter->feat_en |= IXGBE_FEATURE_MSI; 7130 return (0); 7131 } 7132 7133 if (!(adapter->feat_cap & IXGBE_FEATURE_LEGACY_IRQ)) { 7134 aprint_error_dev(dev, 7135 "Device does not support legacy interrupts.\n"); 7136 return 1; 7137 } 7138 7139 adapter->feat_en |= IXGBE_FEATURE_LEGACY_IRQ; 7140 7141 return (0); 7142 } /* ixgbe_configure_interrupts */ 7143 7144 7145 /************************************************************************ 7146 * ixgbe_handle_link - Tasklet for MSI-X Link interrupts 7147 * 7148 * Done outside of interrupt context since the driver might sleep 7149 ************************************************************************/ 7150 static void 7151 ixgbe_handle_link(void *context) 7152 { 7153 struct adapter *adapter = context; 7154 struct ixgbe_hw *hw = &adapter->hw; 7155 7156 KASSERT(mutex_owned(&adapter->core_mtx)); 7157 7158 IXGBE_EVC_ADD(&adapter->link_workev, 1); 7159 ixgbe_check_link(hw, &adapter->link_speed, &adapter->link_up, 0); 7160 ixgbe_update_link_status(adapter); 7161 7162 /* Re-enable link interrupts */ 7163 IXGBE_WRITE_REG(hw, IXGBE_EIMS, IXGBE_EIMS_LSC); 7164 } /* ixgbe_handle_link */ 7165 7166 #if 0 7167 /************************************************************************ 7168 * ixgbe_rearm_queues 7169 ************************************************************************/ 7170 static __inline void 7171 ixgbe_rearm_queues(struct adapter *adapter, u64 queues) 7172 { 7173 u32 mask; 7174 7175 switch (adapter->hw.mac.type) { 7176 case ixgbe_mac_82598EB: 7177 mask = (IXGBE_EIMS_RTX_QUEUE & queues); 7178 IXGBE_WRITE_REG(&adapter->hw, IXGBE_EICS, mask); 7179 break; 7180 case ixgbe_mac_82599EB: 7181 case ixgbe_mac_X540: 7182 case ixgbe_mac_X550: 7183 case ixgbe_mac_X550EM_x: 7184 case ixgbe_mac_X550EM_a: 7185 mask = (queues & 0xFFFFFFFF); 7186 IXGBE_WRITE_REG(&adapter->hw, IXGBE_EICS_EX(0), mask); 7187 mask = (queues >> 32); 7188 IXGBE_WRITE_REG(&adapter->hw, IXGBE_EICS_EX(1), mask); 7189 break; 7190 default: 7191 break; 7192 } 7193 } /* ixgbe_rearm_queues */ 7194 #endif 7195
Indexes created Fri Sep 26 08:10:20 GMT 2025