ixgbe.c revision 1.315
1 /* $NetBSD: ixgbe.c,v 1.315 2022/05/30 05:07:38 msaitoh Exp $ */ 2 3 /****************************************************************************** 4 5 Copyright (c) 2001-2017, Intel Corporation 6 All rights reserved. 7 8 Redistribution and use in source and binary forms, with or without 9 modification, are permitted provided that the following conditions are met: 10 11 1. Redistributions of source code must retain the above copyright notice, 12 this list of conditions and the following disclaimer. 13 14 2. Redistributions in binary form must reproduce the above copyright 15 notice, this list of conditions and the following disclaimer in the 16 documentation and/or other materials provided with the distribution. 17 18 3. Neither the name of the Intel Corporation nor the names of its 19 contributors may be used to endorse or promote products derived from 20 this software without specific prior written permission. 21 22 THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 23 AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 24 IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 25 ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE 26 LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 27 CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 28 SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 29 INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 30 CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 31 ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 32 POSSIBILITY OF SUCH DAMAGE. 33 34 ******************************************************************************/ 35 /*$FreeBSD: head/sys/dev/ixgbe/if_ix.c 331224 2018-03-19 20:55:05Z erj $*/ 36 37 /* 38 * Copyright (c) 2011 The NetBSD Foundation, Inc. 39 * All rights reserved. 40 * 41 * This code is derived from software contributed to The NetBSD Foundation 42 * by Coyote Point Systems, Inc. 43 * 44 * Redistribution and use in source and binary forms, with or without 45 * modification, are permitted provided that the following conditions 46 * are met: 47 * 1. Redistributions of source code must retain the above copyright 48 * notice, this list of conditions and the following disclaimer. 49 * 2. Redistributions in binary form must reproduce the above copyright 50 * notice, this list of conditions and the following disclaimer in the 51 * documentation and/or other materials provided with the distribution. 52 * 53 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS 54 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED 55 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 56 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS 57 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 58 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 59 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 60 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 61 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 62 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 63 * POSSIBILITY OF SUCH DAMAGE. 64 */ 65 66 #include <sys/cdefs.h> 67 __KERNEL_RCSID(0, "$NetBSD: ixgbe.c,v 1.315 2022/05/30 05:07:38 msaitoh Exp $"); 68 69 #ifdef _KERNEL_OPT 70 #include "opt_inet.h" 71 #include "opt_inet6.h" 72 #include "opt_net_mpsafe.h" 73 #endif 74 75 #include "ixgbe.h" 76 #include "ixgbe_phy.h" 77 #include "ixgbe_sriov.h" 78 79 #include <sys/cprng.h> 80 #include <dev/mii/mii.h> 81 #include <dev/mii/miivar.h> 82 83 /************************************************************************ 84 * Driver version 85 ************************************************************************/ 86 static const char ixgbe_driver_version[] = "4.0.1-k"; 87 /* XXX NetBSD: + 3.3.24 */ 88 89 /************************************************************************ 90 * PCI Device ID Table 91 * 92 * Used by probe to select devices to load on 93 * Last field stores an index into ixgbe_strings 94 * Last entry must be all 0s 95 * 96 * { Vendor ID, Device ID, SubVendor ID, SubDevice ID, String Index } 97 ************************************************************************/ 98 static const ixgbe_vendor_info_t ixgbe_vendor_info_array[] = 99 { 100 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598AF_DUAL_PORT, 0, 0, 0}, 101 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598AF_SINGLE_PORT, 0, 0, 0}, 102 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598EB_CX4, 0, 0, 0}, 103 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598AT, 0, 0, 0}, 104 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598AT2, 0, 0, 0}, 105 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598, 0, 0, 0}, 106 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598_BX, 0, 0, 0}, 107 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598_DA_DUAL_PORT, 0, 0, 0}, 108 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598_CX4_DUAL_PORT, 0, 0, 0}, 109 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598EB_XF_LR, 0, 0, 0}, 110 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598_SR_DUAL_PORT_EM, 0, 0, 0}, 111 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598EB_SFP_LOM, 0, 0, 0}, 112 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_KR, 0, 0, 0}, 113 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_KX4, 0, 0, 0}, 114 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_KX4_MEZZ, 0, 0, 0}, 115 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_SFP, 0, 0, 0}, 116 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_SFP_EM, 0, 0, 0}, 117 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_XAUI_LOM, 0, 0, 0}, 118 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_CX4, 0, 0, 0}, 119 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_T3_LOM, 0, 0, 0}, 120 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_COMBO_BACKPLANE, 0, 0, 0}, 121 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_BACKPLANE_FCOE, 0, 0, 0}, 122 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_SFP_SF2, 0, 0, 0}, 123 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_SFP_FCOE, 0, 0, 0}, 124 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599EN_SFP, 0, 0, 0}, 125 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_SFP_SF_QP, 0, 0, 0}, 126 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_QSFP_SF_QP, 0, 0, 0}, 127 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_X540T, 0, 0, 0}, 128 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_X540T1, 0, 0, 0}, 129 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_X550T, 0, 0, 0}, 130 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_X550T1, 0, 0, 0}, 131 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_X550EM_X_KR, 0, 0, 0}, 132 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_X550EM_X_KX4, 0, 0, 0}, 133 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_X550EM_X_10G_T, 0, 0, 0}, 134 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_X550EM_X_1G_T, 0, 0, 0}, 135 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_X550EM_X_SFP, 0, 0, 0}, 136 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_X550EM_X_XFI, 0, 0, 0}, 137 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_X550EM_A_KR, 0, 0, 0}, 138 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_X550EM_A_KR_L, 0, 0, 0}, 139 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_X550EM_A_QSFP, 0, 0, 0}, 140 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_X550EM_A_QSFP_N, 0, 0, 0}, 141 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_X550EM_A_SFP, 0, 0, 0}, 142 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_X550EM_A_SFP_N, 0, 0, 0}, 143 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_X550EM_A_SGMII, 0, 0, 0}, 144 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_X550EM_A_SGMII_L, 0, 0, 0}, 145 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_X550EM_A_10G_T, 0, 0, 0}, 146 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_X550EM_A_1G_T, 0, 0, 0}, 147 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_X550EM_A_1G_T_L, 0, 0, 0}, 148 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_X540_BYPASS, 0, 0, 0}, 149 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_BYPASS, 0, 0, 0}, 150 /* required last entry */ 151 {0, 0, 0, 0, 0} 152 }; 153 154 /************************************************************************ 155 * Table of branding strings 156 ************************************************************************/ 157 static const char *ixgbe_strings[] = { 158 "Intel(R) PRO/10GbE PCI-Express Network Driver" 159 }; 160 161 /************************************************************************ 162 * Function prototypes 163 ************************************************************************/ 164 static int ixgbe_probe(device_t, cfdata_t, void *); 165 static void ixgbe_quirks(struct adapter *); 166 static void ixgbe_attach(device_t, device_t, void *); 167 static int ixgbe_detach(device_t, int); 168 #if 0 169 static int ixgbe_shutdown(device_t); 170 #endif 171 static bool ixgbe_suspend(device_t, const pmf_qual_t *); 172 static bool ixgbe_resume(device_t, const pmf_qual_t *); 173 static int ixgbe_ifflags_cb(struct ethercom *); 174 static int ixgbe_ioctl(struct ifnet *, u_long, void *); 175 static int ixgbe_init(struct ifnet *); 176 static void ixgbe_init_locked(struct adapter *); 177 static void ixgbe_ifstop(struct ifnet *, int); 178 static void ixgbe_stop_locked(void *); 179 static void ixgbe_init_device_features(struct adapter *); 180 static int ixgbe_check_fan_failure(struct adapter *, u32, bool); 181 static void ixgbe_add_media_types(struct adapter *); 182 static void ixgbe_media_status(struct ifnet *, struct ifmediareq *); 183 static int ixgbe_media_change(struct ifnet *); 184 static int ixgbe_allocate_pci_resources(struct adapter *, 185 const struct pci_attach_args *); 186 static void ixgbe_free_deferred_handlers(struct adapter *); 187 static void ixgbe_get_slot_info(struct adapter *); 188 static int ixgbe_allocate_msix(struct adapter *, 189 const struct pci_attach_args *); 190 static int ixgbe_allocate_legacy(struct adapter *, 191 const struct pci_attach_args *); 192 static int ixgbe_configure_interrupts(struct adapter *); 193 static void ixgbe_free_pciintr_resources(struct adapter *); 194 static void ixgbe_free_pci_resources(struct adapter *); 195 static void ixgbe_local_timer(void *); 196 static void ixgbe_handle_timer(struct work *, void *); 197 static void ixgbe_recovery_mode_timer(void *); 198 static void ixgbe_handle_recovery_mode_timer(struct work *, void *); 199 static int ixgbe_setup_interface(device_t, struct adapter *); 200 static void ixgbe_config_gpie(struct adapter *); 201 static void ixgbe_config_dmac(struct adapter *); 202 static void ixgbe_config_delay_values(struct adapter *); 203 static void ixgbe_schedule_admin_tasklet(struct adapter *); 204 static void ixgbe_config_link(struct adapter *); 205 static void ixgbe_check_wol_support(struct adapter *); 206 static int ixgbe_setup_low_power_mode(struct adapter *); 207 #if 0 208 static void ixgbe_rearm_queues(struct adapter *, u64); 209 #endif 210 211 static void ixgbe_initialize_transmit_units(struct adapter *); 212 static void ixgbe_initialize_receive_units(struct adapter *); 213 static void ixgbe_enable_rx_drop(struct adapter *); 214 static void ixgbe_disable_rx_drop(struct adapter *); 215 static void ixgbe_initialize_rss_mapping(struct adapter *); 216 217 static void ixgbe_enable_intr(struct adapter *); 218 static void ixgbe_disable_intr(struct adapter *); 219 static void ixgbe_update_stats_counters(struct adapter *); 220 static void ixgbe_set_rxfilter(struct adapter *); 221 static void ixgbe_update_link_status(struct adapter *); 222 static void ixgbe_set_ivar(struct adapter *, u8, u8, s8); 223 static void ixgbe_configure_ivars(struct adapter *); 224 static u8 * ixgbe_mc_array_itr(struct ixgbe_hw *, u8 **, u32 *); 225 static void ixgbe_eitr_write(struct adapter *, uint32_t, uint32_t); 226 227 static void ixgbe_setup_vlan_hw_tagging(struct adapter *); 228 static void ixgbe_setup_vlan_hw_support(struct adapter *); 229 static int ixgbe_vlan_cb(struct ethercom *, uint16_t, bool); 230 static int ixgbe_register_vlan(struct adapter *, u16); 231 static int ixgbe_unregister_vlan(struct adapter *, u16); 232 233 static void ixgbe_add_device_sysctls(struct adapter *); 234 static void ixgbe_add_hw_stats(struct adapter *); 235 static void ixgbe_clear_evcnt(struct adapter *); 236 static int ixgbe_set_flowcntl(struct adapter *, int); 237 static int ixgbe_set_advertise(struct adapter *, int); 238 static int ixgbe_get_default_advertise(struct adapter *); 239 240 /* Sysctl handlers */ 241 static int ixgbe_sysctl_flowcntl(SYSCTLFN_PROTO); 242 static int ixgbe_sysctl_advertise(SYSCTLFN_PROTO); 243 static int ixgbe_sysctl_interrupt_rate_handler(SYSCTLFN_PROTO); 244 static int ixgbe_sysctl_dmac(SYSCTLFN_PROTO); 245 static int ixgbe_sysctl_phy_temp(SYSCTLFN_PROTO); 246 static int ixgbe_sysctl_phy_overtemp_occurred(SYSCTLFN_PROTO); 247 #ifdef IXGBE_DEBUG 248 static int ixgbe_sysctl_power_state(SYSCTLFN_PROTO); 249 static int ixgbe_sysctl_print_rss_config(SYSCTLFN_PROTO); 250 #endif 251 static int ixgbe_sysctl_next_to_check_handler(SYSCTLFN_PROTO); 252 static int ixgbe_sysctl_next_to_refresh_handler(SYSCTLFN_PROTO); 253 static int ixgbe_sysctl_rdh_handler(SYSCTLFN_PROTO); 254 static int ixgbe_sysctl_rdt_handler(SYSCTLFN_PROTO); 255 static int ixgbe_sysctl_tdt_handler(SYSCTLFN_PROTO); 256 static int ixgbe_sysctl_tdh_handler(SYSCTLFN_PROTO); 257 static int ixgbe_sysctl_eee_state(SYSCTLFN_PROTO); 258 static int ixgbe_sysctl_debug(SYSCTLFN_PROTO); 259 static int ixgbe_sysctl_rx_copy_len(SYSCTLFN_PROTO); 260 static int ixgbe_sysctl_tx_process_limit(SYSCTLFN_PROTO); 261 static int ixgbe_sysctl_rx_process_limit(SYSCTLFN_PROTO); 262 static int ixgbe_sysctl_wol_enable(SYSCTLFN_PROTO); 263 static int ixgbe_sysctl_wufc(SYSCTLFN_PROTO); 264 265 /* Interrupt functions */ 266 static int ixgbe_msix_que(void *); 267 static int ixgbe_msix_admin(void *); 268 static void ixgbe_intr_admin_common(struct adapter *, u32, u32 *); 269 static int ixgbe_legacy_irq(void *); 270 271 /* Event handlers running on workqueue */ 272 static void ixgbe_handle_que(void *); 273 static void ixgbe_handle_link(void *); 274 static void ixgbe_handle_msf(void *); 275 static void ixgbe_handle_mod(void *, bool); 276 static void ixgbe_handle_phy(void *); 277 278 /* Deferred workqueue handlers */ 279 static void ixgbe_handle_admin(struct work *, void *); 280 static void ixgbe_handle_que_work(struct work *, void *); 281 282 static const ixgbe_vendor_info_t *ixgbe_lookup(const struct pci_attach_args *); 283 284 /************************************************************************ 285 * NetBSD Device Interface Entry Points 286 ************************************************************************/ 287 CFATTACH_DECL3_NEW(ixg, sizeof(struct adapter), 288 ixgbe_probe, ixgbe_attach, ixgbe_detach, NULL, NULL, NULL, 289 DVF_DETACH_SHUTDOWN); 290 291 #if 0 292 devclass_t ix_devclass; 293 DRIVER_MODULE(ix, pci, ix_driver, ix_devclass, 0, 0); 294 295 MODULE_DEPEND(ix, pci, 1, 1, 1); 296 MODULE_DEPEND(ix, ether, 1, 1, 1); 297 #ifdef DEV_NETMAP 298 MODULE_DEPEND(ix, netmap, 1, 1, 1); 299 #endif 300 #endif 301 302 /* 303 * TUNEABLE PARAMETERS: 304 */ 305 306 /* 307 * AIM: Adaptive Interrupt Moderation 308 * which means that the interrupt rate 309 * is varied over time based on the 310 * traffic for that interrupt vector 311 */ 312 static bool ixgbe_enable_aim = true; 313 #define SYSCTL_INT(_a1, _a2, _a3, _a4, _a5, _a6, _a7) 314 SYSCTL_INT(_hw_ix, OID_AUTO, enable_aim, CTLFLAG_RDTUN, &ixgbe_enable_aim, 0, 315 "Enable adaptive interrupt moderation"); 316 317 static int ixgbe_max_interrupt_rate = (4000000 / IXGBE_LOW_LATENCY); 318 SYSCTL_INT(_hw_ix, OID_AUTO, max_interrupt_rate, CTLFLAG_RDTUN, 319 &ixgbe_max_interrupt_rate, 0, "Maximum interrupts per second"); 320 321 /* How many packets rxeof tries to clean at a time */ 322 static int ixgbe_rx_process_limit = 256; 323 SYSCTL_INT(_hw_ix, OID_AUTO, rx_process_limit, CTLFLAG_RDTUN, 324 &ixgbe_rx_process_limit, 0, "Maximum number of received packets to process at a time, -1 means unlimited"); 325 326 /* How many packets txeof tries to clean at a time */ 327 static int ixgbe_tx_process_limit = 256; 328 SYSCTL_INT(_hw_ix, OID_AUTO, tx_process_limit, CTLFLAG_RDTUN, 329 &ixgbe_tx_process_limit, 0, 330 "Maximum number of sent packets to process at a time, -1 means unlimited"); 331 332 /* Flow control setting, default to full */ 333 static int ixgbe_flow_control = ixgbe_fc_full; 334 SYSCTL_INT(_hw_ix, OID_AUTO, flow_control, CTLFLAG_RDTUN, 335 &ixgbe_flow_control, 0, "Default flow control used for all adapters"); 336 337 /* Which packet processing uses workqueue or softint */ 338 static bool ixgbe_txrx_workqueue = false; 339 340 /* 341 * Smart speed setting, default to on 342 * this only works as a compile option 343 * right now as its during attach, set 344 * this to 'ixgbe_smart_speed_off' to 345 * disable. 346 */ 347 static int ixgbe_smart_speed = ixgbe_smart_speed_on; 348 349 /* 350 * MSI-X should be the default for best performance, 351 * but this allows it to be forced off for testing. 352 */ 353 static int ixgbe_enable_msix = 1; 354 SYSCTL_INT(_hw_ix, OID_AUTO, enable_msix, CTLFLAG_RDTUN, &ixgbe_enable_msix, 0, 355 "Enable MSI-X interrupts"); 356 357 /* 358 * Number of Queues, can be set to 0, 359 * it then autoconfigures based on the 360 * number of cpus with a max of 8. This 361 * can be overridden manually here. 362 */ 363 static int ixgbe_num_queues = 0; 364 SYSCTL_INT(_hw_ix, OID_AUTO, num_queues, CTLFLAG_RDTUN, &ixgbe_num_queues, 0, 365 "Number of queues to configure, 0 indicates autoconfigure"); 366 367 /* 368 * Number of TX descriptors per ring, 369 * setting higher than RX as this seems 370 * the better performing choice. 371 */ 372 static int ixgbe_txd = PERFORM_TXD; 373 SYSCTL_INT(_hw_ix, OID_AUTO, txd, CTLFLAG_RDTUN, &ixgbe_txd, 0, 374 "Number of transmit descriptors per queue"); 375 376 /* Number of RX descriptors per ring */ 377 static int ixgbe_rxd = PERFORM_RXD; 378 SYSCTL_INT(_hw_ix, OID_AUTO, rxd, CTLFLAG_RDTUN, &ixgbe_rxd, 0, 379 "Number of receive descriptors per queue"); 380 381 /* 382 * Defining this on will allow the use 383 * of unsupported SFP+ modules, note that 384 * doing so you are on your own :) 385 */ 386 static int allow_unsupported_sfp = false; 387 #define TUNABLE_INT(__x, __y) 388 TUNABLE_INT("hw.ix.unsupported_sfp", &allow_unsupported_sfp); 389 390 /* 391 * Not sure if Flow Director is fully baked, 392 * so we'll default to turning it off. 393 */ 394 static int ixgbe_enable_fdir = 0; 395 SYSCTL_INT(_hw_ix, OID_AUTO, enable_fdir, CTLFLAG_RDTUN, &ixgbe_enable_fdir, 0, 396 "Enable Flow Director"); 397 398 /* Legacy Transmit (single queue) */ 399 static int ixgbe_enable_legacy_tx = 0; 400 SYSCTL_INT(_hw_ix, OID_AUTO, enable_legacy_tx, CTLFLAG_RDTUN, 401 &ixgbe_enable_legacy_tx, 0, "Enable Legacy TX flow"); 402 403 /* Receive-Side Scaling */ 404 static int ixgbe_enable_rss = 1; 405 SYSCTL_INT(_hw_ix, OID_AUTO, enable_rss, CTLFLAG_RDTUN, &ixgbe_enable_rss, 0, 406 "Enable Receive-Side Scaling (RSS)"); 407 408 #if 0 409 static int (*ixgbe_start_locked)(struct ifnet *, struct tx_ring *); 410 static int (*ixgbe_ring_empty)(struct ifnet *, pcq_t *); 411 #endif 412 413 #ifdef NET_MPSAFE 414 #define IXGBE_MPSAFE 1 415 #define IXGBE_CALLOUT_FLAGS CALLOUT_MPSAFE 416 #define IXGBE_SOFTINT_FLAGS SOFTINT_MPSAFE 417 #define IXGBE_WORKQUEUE_FLAGS WQ_PERCPU | WQ_MPSAFE 418 #define IXGBE_TASKLET_WQ_FLAGS WQ_MPSAFE 419 #else 420 #define IXGBE_CALLOUT_FLAGS 0 421 #define IXGBE_SOFTINT_FLAGS 0 422 #define IXGBE_WORKQUEUE_FLAGS WQ_PERCPU 423 #define IXGBE_TASKLET_WQ_FLAGS 0 424 #endif 425 #define IXGBE_WORKQUEUE_PRI PRI_SOFTNET 426 427 /* Interval between reports of errors */ 428 static const struct timeval ixgbe_errlog_intrvl = { 60, 0 }; /* 60s */ 429 430 /************************************************************************ 431 * ixgbe_initialize_rss_mapping 432 ************************************************************************/ 433 static void 434 ixgbe_initialize_rss_mapping(struct adapter *adapter) 435 { 436 struct ixgbe_hw *hw = &adapter->hw; 437 u32 reta = 0, mrqc, rss_key[10]; 438 int queue_id, table_size, index_mult; 439 int i, j; 440 u32 rss_hash_config; 441 442 /* force use default RSS key. */ 443 #ifdef __NetBSD__ 444 rss_getkey((uint8_t *) &rss_key); 445 #else 446 if (adapter->feat_en & IXGBE_FEATURE_RSS) { 447 /* Fetch the configured RSS key */ 448 rss_getkey((uint8_t *) &rss_key); 449 } else { 450 /* set up random bits */ 451 cprng_fast(&rss_key, sizeof(rss_key)); 452 } 453 #endif 454 455 /* Set multiplier for RETA setup and table size based on MAC */ 456 index_mult = 0x1; 457 table_size = 128; 458 switch (adapter->hw.mac.type) { 459 case ixgbe_mac_82598EB: 460 index_mult = 0x11; 461 break; 462 case ixgbe_mac_X550: 463 case ixgbe_mac_X550EM_x: 464 case ixgbe_mac_X550EM_a: 465 table_size = 512; 466 break; 467 default: 468 break; 469 } 470 471 /* Set up the redirection table */ 472 for (i = 0, j = 0; i < table_size; i++, j++) { 473 if (j == adapter->num_queues) 474 j = 0; 475 476 if (adapter->feat_en & IXGBE_FEATURE_RSS) { 477 /* 478 * Fetch the RSS bucket id for the given indirection 479 * entry. Cap it at the number of configured buckets 480 * (which is num_queues.) 481 */ 482 queue_id = rss_get_indirection_to_bucket(i); 483 queue_id = queue_id % adapter->num_queues; 484 } else 485 queue_id = (j * index_mult); 486 487 /* 488 * The low 8 bits are for hash value (n+0); 489 * The next 8 bits are for hash value (n+1), etc. 490 */ 491 reta = reta >> 8; 492 reta = reta | (((uint32_t) queue_id) << 24); 493 if ((i & 3) == 3) { 494 if (i < 128) 495 IXGBE_WRITE_REG(hw, IXGBE_RETA(i >> 2), reta); 496 else 497 IXGBE_WRITE_REG(hw, IXGBE_ERETA((i >> 2) - 32), 498 reta); 499 reta = 0; 500 } 501 } 502 503 /* Now fill our hash function seeds */ 504 for (i = 0; i < 10; i++) 505 IXGBE_WRITE_REG(hw, IXGBE_RSSRK(i), rss_key[i]); 506 507 /* Perform hash on these packet types */ 508 if (adapter->feat_en & IXGBE_FEATURE_RSS) 509 rss_hash_config = rss_gethashconfig(); 510 else { 511 /* 512 * Disable UDP - IP fragments aren't currently being handled 513 * and so we end up with a mix of 2-tuple and 4-tuple 514 * traffic. 515 */ 516 rss_hash_config = RSS_HASHTYPE_RSS_IPV4 517 | RSS_HASHTYPE_RSS_TCP_IPV4 518 | RSS_HASHTYPE_RSS_IPV6 519 | RSS_HASHTYPE_RSS_TCP_IPV6 520 | RSS_HASHTYPE_RSS_IPV6_EX 521 | RSS_HASHTYPE_RSS_TCP_IPV6_EX; 522 } 523 524 mrqc = IXGBE_MRQC_RSSEN; 525 if (rss_hash_config & RSS_HASHTYPE_RSS_IPV4) 526 mrqc |= IXGBE_MRQC_RSS_FIELD_IPV4; 527 if (rss_hash_config & RSS_HASHTYPE_RSS_TCP_IPV4) 528 mrqc |= IXGBE_MRQC_RSS_FIELD_IPV4_TCP; 529 if (rss_hash_config & RSS_HASHTYPE_RSS_IPV6) 530 mrqc |= IXGBE_MRQC_RSS_FIELD_IPV6; 531 if (rss_hash_config & RSS_HASHTYPE_RSS_TCP_IPV6) 532 mrqc |= IXGBE_MRQC_RSS_FIELD_IPV6_TCP; 533 if (rss_hash_config & RSS_HASHTYPE_RSS_IPV6_EX) 534 mrqc |= IXGBE_MRQC_RSS_FIELD_IPV6_EX; 535 if (rss_hash_config & RSS_HASHTYPE_RSS_TCP_IPV6_EX) 536 mrqc |= IXGBE_MRQC_RSS_FIELD_IPV6_EX_TCP; 537 if (rss_hash_config & RSS_HASHTYPE_RSS_UDP_IPV4) 538 mrqc |= IXGBE_MRQC_RSS_FIELD_IPV4_UDP; 539 if (rss_hash_config & RSS_HASHTYPE_RSS_UDP_IPV6) 540 mrqc |= IXGBE_MRQC_RSS_FIELD_IPV6_UDP; 541 if (rss_hash_config & RSS_HASHTYPE_RSS_UDP_IPV6_EX) 542 mrqc |= IXGBE_MRQC_RSS_FIELD_IPV6_EX_UDP; 543 mrqc |= ixgbe_get_mrqc(adapter->iov_mode); 544 IXGBE_WRITE_REG(hw, IXGBE_MRQC, mrqc); 545 } /* ixgbe_initialize_rss_mapping */ 546 547 /************************************************************************ 548 * ixgbe_initialize_receive_units - Setup receive registers and features. 549 ************************************************************************/ 550 #define BSIZEPKT_ROUNDUP ((1<<IXGBE_SRRCTL_BSIZEPKT_SHIFT)-1) 551 552 static void 553 ixgbe_initialize_receive_units(struct adapter *adapter) 554 { 555 struct rx_ring *rxr = adapter->rx_rings; 556 struct ixgbe_hw *hw = &adapter->hw; 557 struct ifnet *ifp = adapter->ifp; 558 int i, j; 559 u32 bufsz, fctrl, srrctl, rxcsum; 560 u32 hlreg; 561 562 /* 563 * Make sure receives are disabled while 564 * setting up the descriptor ring 565 */ 566 ixgbe_disable_rx(hw); 567 568 /* Enable broadcasts */ 569 fctrl = IXGBE_READ_REG(hw, IXGBE_FCTRL); 570 fctrl |= IXGBE_FCTRL_BAM; 571 if (adapter->hw.mac.type == ixgbe_mac_82598EB) { 572 fctrl |= IXGBE_FCTRL_DPF; 573 fctrl |= IXGBE_FCTRL_PMCF; 574 } 575 IXGBE_WRITE_REG(hw, IXGBE_FCTRL, fctrl); 576 577 /* Set for Jumbo Frames? */ 578 hlreg = IXGBE_READ_REG(hw, IXGBE_HLREG0); 579 if (ifp->if_mtu > ETHERMTU) 580 hlreg |= IXGBE_HLREG0_JUMBOEN; 581 else 582 hlreg &= ~IXGBE_HLREG0_JUMBOEN; 583 584 #ifdef DEV_NETMAP 585 /* CRC stripping is conditional in Netmap */ 586 if ((adapter->feat_en & IXGBE_FEATURE_NETMAP) && 587 (ifp->if_capenable & IFCAP_NETMAP) && 588 !ix_crcstrip) 589 hlreg &= ~IXGBE_HLREG0_RXCRCSTRP; 590 else 591 #endif /* DEV_NETMAP */ 592 hlreg |= IXGBE_HLREG0_RXCRCSTRP; 593 594 IXGBE_WRITE_REG(hw, IXGBE_HLREG0, hlreg); 595 596 bufsz = (adapter->rx_mbuf_sz + BSIZEPKT_ROUNDUP) >> 597 IXGBE_SRRCTL_BSIZEPKT_SHIFT; 598 599 for (i = 0; i < adapter->num_queues; i++, rxr++) { 600 u64 rdba = rxr->rxdma.dma_paddr; 601 u32 reg; 602 int regnum = i / 4; /* 1 register per 4 queues */ 603 int regshift = i % 4; /* 4 bits per 1 queue */ 604 j = rxr->me; 605 606 /* Setup the Base and Length of the Rx Descriptor Ring */ 607 IXGBE_WRITE_REG(hw, IXGBE_RDBAL(j), 608 (rdba & 0x00000000ffffffffULL)); 609 IXGBE_WRITE_REG(hw, IXGBE_RDBAH(j), (rdba >> 32)); 610 IXGBE_WRITE_REG(hw, IXGBE_RDLEN(j), 611 adapter->num_rx_desc * sizeof(union ixgbe_adv_rx_desc)); 612 613 /* Set up the SRRCTL register */ 614 srrctl = IXGBE_READ_REG(hw, IXGBE_SRRCTL(j)); 615 srrctl &= ~IXGBE_SRRCTL_BSIZEHDR_MASK; 616 srrctl &= ~IXGBE_SRRCTL_BSIZEPKT_MASK; 617 srrctl |= bufsz; 618 srrctl |= IXGBE_SRRCTL_DESCTYPE_ADV_ONEBUF; 619 620 /* Set RQSMR (Receive Queue Statistic Mapping) register */ 621 reg = IXGBE_READ_REG(hw, IXGBE_RQSMR(regnum)); 622 reg &= ~(0x000000ffUL << (regshift * 8)); 623 reg |= i << (regshift * 8); 624 IXGBE_WRITE_REG(hw, IXGBE_RQSMR(regnum), reg); 625 626 /* 627 * Set DROP_EN iff we have no flow control and >1 queue. 628 * Note that srrctl was cleared shortly before during reset, 629 * so we do not need to clear the bit, but do it just in case 630 * this code is moved elsewhere. 631 */ 632 if (adapter->num_queues > 1 && 633 adapter->hw.fc.requested_mode == ixgbe_fc_none) { 634 srrctl |= IXGBE_SRRCTL_DROP_EN; 635 } else { 636 srrctl &= ~IXGBE_SRRCTL_DROP_EN; 637 } 638 639 IXGBE_WRITE_REG(hw, IXGBE_SRRCTL(j), srrctl); 640 641 /* Setup the HW Rx Head and Tail Descriptor Pointers */ 642 IXGBE_WRITE_REG(hw, IXGBE_RDH(j), 0); 643 IXGBE_WRITE_REG(hw, IXGBE_RDT(j), 0); 644 645 /* Set the driver rx tail address */ 646 rxr->tail = IXGBE_RDT(rxr->me); 647 } 648 649 if (adapter->hw.mac.type != ixgbe_mac_82598EB) { 650 u32 psrtype = IXGBE_PSRTYPE_TCPHDR 651 | IXGBE_PSRTYPE_UDPHDR 652 | IXGBE_PSRTYPE_IPV4HDR 653 | IXGBE_PSRTYPE_IPV6HDR; 654 IXGBE_WRITE_REG(hw, IXGBE_PSRTYPE(0), psrtype); 655 } 656 657 rxcsum = IXGBE_READ_REG(hw, IXGBE_RXCSUM); 658 659 ixgbe_initialize_rss_mapping(adapter); 660 661 if (adapter->num_queues > 1) { 662 /* RSS and RX IPP Checksum are mutually exclusive */ 663 rxcsum |= IXGBE_RXCSUM_PCSD; 664 } 665 666 if (ifp->if_capenable & IFCAP_RXCSUM) 667 rxcsum |= IXGBE_RXCSUM_PCSD; 668 669 /* This is useful for calculating UDP/IP fragment checksums */ 670 if (!(rxcsum & IXGBE_RXCSUM_PCSD)) 671 rxcsum |= IXGBE_RXCSUM_IPPCSE; 672 673 IXGBE_WRITE_REG(hw, IXGBE_RXCSUM, rxcsum); 674 675 } /* ixgbe_initialize_receive_units */ 676 677 /************************************************************************ 678 * ixgbe_initialize_transmit_units - Enable transmit units. 679 ************************************************************************/ 680 static void 681 ixgbe_initialize_transmit_units(struct adapter *adapter) 682 { 683 struct tx_ring *txr = adapter->tx_rings; 684 struct ixgbe_hw *hw = &adapter->hw; 685 int i; 686 687 INIT_DEBUGOUT("ixgbe_initialize_transmit_units"); 688 689 /* Setup the Base and Length of the Tx Descriptor Ring */ 690 for (i = 0; i < adapter->num_queues; i++, txr++) { 691 u64 tdba = txr->txdma.dma_paddr; 692 u32 txctrl = 0; 693 u32 tqsmreg, reg; 694 int regnum = i / 4; /* 1 register per 4 queues */ 695 int regshift = i % 4; /* 4 bits per 1 queue */ 696 int j = txr->me; 697 698 IXGBE_WRITE_REG(hw, IXGBE_TDBAL(j), 699 (tdba & 0x00000000ffffffffULL)); 700 IXGBE_WRITE_REG(hw, IXGBE_TDBAH(j), (tdba >> 32)); 701 IXGBE_WRITE_REG(hw, IXGBE_TDLEN(j), 702 adapter->num_tx_desc * sizeof(union ixgbe_adv_tx_desc)); 703 704 /* 705 * Set TQSMR (Transmit Queue Statistic Mapping) register. 706 * Register location is different between 82598 and others. 707 */ 708 if (adapter->hw.mac.type == ixgbe_mac_82598EB) 709 tqsmreg = IXGBE_TQSMR(regnum); 710 else 711 tqsmreg = IXGBE_TQSM(regnum); 712 reg = IXGBE_READ_REG(hw, tqsmreg); 713 reg &= ~(0x000000ffUL << (regshift * 8)); 714 reg |= i << (regshift * 8); 715 IXGBE_WRITE_REG(hw, tqsmreg, reg); 716 717 /* Setup the HW Tx Head and Tail descriptor pointers */ 718 IXGBE_WRITE_REG(hw, IXGBE_TDH(j), 0); 719 IXGBE_WRITE_REG(hw, IXGBE_TDT(j), 0); 720 721 /* Cache the tail address */ 722 txr->tail = IXGBE_TDT(j); 723 724 txr->txr_no_space = false; 725 726 /* Disable Head Writeback */ 727 /* 728 * Note: for X550 series devices, these registers are actually 729 * prefixed with TPH_ instead of DCA_, but the addresses and 730 * fields remain the same. 731 */ 732 switch (hw->mac.type) { 733 case ixgbe_mac_82598EB: 734 txctrl = IXGBE_READ_REG(hw, IXGBE_DCA_TXCTRL(j)); 735 break; 736 default: 737 txctrl = IXGBE_READ_REG(hw, IXGBE_DCA_TXCTRL_82599(j)); 738 break; 739 } 740 txctrl &= ~IXGBE_DCA_TXCTRL_DESC_WRO_EN; 741 switch (hw->mac.type) { 742 case ixgbe_mac_82598EB: 743 IXGBE_WRITE_REG(hw, IXGBE_DCA_TXCTRL(j), txctrl); 744 break; 745 default: 746 IXGBE_WRITE_REG(hw, IXGBE_DCA_TXCTRL_82599(j), txctrl); 747 break; 748 } 749 750 } 751 752 if (hw->mac.type != ixgbe_mac_82598EB) { 753 u32 dmatxctl, rttdcs; 754 755 dmatxctl = IXGBE_READ_REG(hw, IXGBE_DMATXCTL); 756 dmatxctl |= IXGBE_DMATXCTL_TE; 757 IXGBE_WRITE_REG(hw, IXGBE_DMATXCTL, dmatxctl); 758 /* Disable arbiter to set MTQC */ 759 rttdcs = IXGBE_READ_REG(hw, IXGBE_RTTDCS); 760 rttdcs |= IXGBE_RTTDCS_ARBDIS; 761 IXGBE_WRITE_REG(hw, IXGBE_RTTDCS, rttdcs); 762 IXGBE_WRITE_REG(hw, IXGBE_MTQC, 763 ixgbe_get_mtqc(adapter->iov_mode)); 764 rttdcs &= ~IXGBE_RTTDCS_ARBDIS; 765 IXGBE_WRITE_REG(hw, IXGBE_RTTDCS, rttdcs); 766 } 767 768 return; 769 } /* ixgbe_initialize_transmit_units */ 770 771 static void 772 ixgbe_quirks(struct adapter *adapter) 773 { 774 device_t dev = adapter->dev; 775 struct ixgbe_hw *hw = &adapter->hw; 776 const char *vendor, *product; 777 778 if (hw->device_id == IXGBE_DEV_ID_X550EM_A_SFP_N) { 779 /* 780 * Quirk for inverted logic of SFP+'s MOD_ABS on GIGABYTE 781 * MA10-ST0. 782 */ 783 vendor = pmf_get_platform("system-vendor"); 784 product = pmf_get_platform("system-product"); 785 786 if ((vendor == NULL) || (product == NULL)) 787 return; 788 789 if ((strcmp(vendor, "GIGABYTE") == 0) && 790 (strcmp(product, "MA10-ST0") == 0)) { 791 aprint_verbose_dev(dev, 792 "Enable SFP+ MOD_ABS inverse quirk\n"); 793 adapter->hw.quirks |= IXGBE_QUIRK_MOD_ABS_INVERT; 794 } 795 } 796 } 797 798 /************************************************************************ 799 * ixgbe_attach - Device initialization routine 800 * 801 * Called when the driver is being loaded. 802 * Identifies the type of hardware, allocates all resources 803 * and initializes the hardware. 804 * 805 * return 0 on success, positive on failure 806 ************************************************************************/ 807 static void 808 ixgbe_attach(device_t parent, device_t dev, void *aux) 809 { 810 struct adapter *adapter; 811 struct ixgbe_hw *hw; 812 int error = -1; 813 u32 ctrl_ext; 814 u16 high, low, nvmreg; 815 pcireg_t id, subid; 816 const ixgbe_vendor_info_t *ent; 817 struct pci_attach_args *pa = aux; 818 bool unsupported_sfp = false; 819 const char *str; 820 char wqname[MAXCOMLEN]; 821 char buf[256]; 822 823 INIT_DEBUGOUT("ixgbe_attach: begin"); 824 825 /* Allocate, clear, and link in our adapter structure */ 826 adapter = device_private(dev); 827 adapter->hw.back = adapter; 828 adapter->dev = dev; 829 hw = &adapter->hw; 830 adapter->osdep.pc = pa->pa_pc; 831 adapter->osdep.tag = pa->pa_tag; 832 if (pci_dma64_available(pa)) 833 adapter->osdep.dmat = pa->pa_dmat64; 834 else 835 adapter->osdep.dmat = pa->pa_dmat; 836 adapter->osdep.attached = false; 837 adapter->osdep.detaching = false; 838 839 ent = ixgbe_lookup(pa); 840 841 KASSERT(ent != NULL); 842 843 aprint_normal(": %s, Version - %s\n", 844 ixgbe_strings[ent->index], ixgbe_driver_version); 845 846 /* Core Lock Init */ 847 IXGBE_CORE_LOCK_INIT(adapter, device_xname(dev)); 848 849 /* Set up the timer callout and workqueue */ 850 callout_init(&adapter->timer, IXGBE_CALLOUT_FLAGS); 851 snprintf(wqname, sizeof(wqname), "%s-timer", device_xname(dev)); 852 error = workqueue_create(&adapter->timer_wq, wqname, 853 ixgbe_handle_timer, adapter, IXGBE_WORKQUEUE_PRI, IPL_NET, 854 IXGBE_TASKLET_WQ_FLAGS); 855 if (error) { 856 aprint_error_dev(dev, 857 "could not create timer workqueue (%d)\n", error); 858 goto err_out; 859 } 860 861 /* Determine hardware revision */ 862 id = pci_conf_read(pa->pa_pc, pa->pa_tag, PCI_ID_REG); 863 subid = pci_conf_read(pa->pa_pc, pa->pa_tag, PCI_SUBSYS_ID_REG); 864 865 hw->vendor_id = PCI_VENDOR(id); 866 hw->device_id = PCI_PRODUCT(id); 867 hw->revision_id = 868 PCI_REVISION(pci_conf_read(pa->pa_pc, pa->pa_tag, PCI_CLASS_REG)); 869 hw->subsystem_vendor_id = PCI_SUBSYS_VENDOR(subid); 870 hw->subsystem_device_id = PCI_SUBSYS_ID(subid); 871 872 /* Set quirk flags */ 873 ixgbe_quirks(adapter); 874 875 /* 876 * Make sure BUSMASTER is set 877 */ 878 ixgbe_pci_enable_busmaster(pa->pa_pc, pa->pa_tag); 879 880 /* Do base PCI setup - map BAR0 */ 881 if (ixgbe_allocate_pci_resources(adapter, pa)) { 882 aprint_error_dev(dev, "Allocation of PCI resources failed\n"); 883 error = ENXIO; 884 goto err_out; 885 } 886 887 /* let hardware know driver is loaded */ 888 ctrl_ext = IXGBE_READ_REG(hw, IXGBE_CTRL_EXT); 889 ctrl_ext |= IXGBE_CTRL_EXT_DRV_LOAD; 890 IXGBE_WRITE_REG(hw, IXGBE_CTRL_EXT, ctrl_ext); 891 892 /* 893 * Initialize the shared code 894 */ 895 if (ixgbe_init_shared_code(hw) != 0) { 896 aprint_error_dev(dev, "Unable to initialize the shared code\n"); 897 error = ENXIO; 898 goto err_out; 899 } 900 901 switch (hw->mac.type) { 902 case ixgbe_mac_82598EB: 903 str = "82598EB"; 904 break; 905 case ixgbe_mac_82599EB: 906 str = "82599EB"; 907 break; 908 case ixgbe_mac_X540: 909 str = "X540"; 910 break; 911 case ixgbe_mac_X550: 912 str = "X550"; 913 break; 914 case ixgbe_mac_X550EM_x: 915 str = "X550EM X"; 916 break; 917 case ixgbe_mac_X550EM_a: 918 str = "X550EM A"; 919 break; 920 default: 921 str = "Unknown"; 922 break; 923 } 924 aprint_normal_dev(dev, "device %s\n", str); 925 926 hw->allow_unsupported_sfp = allow_unsupported_sfp; 927 928 /* Pick up the 82599 settings */ 929 if (hw->mac.type != ixgbe_mac_82598EB) 930 hw->phy.smart_speed = ixgbe_smart_speed; 931 932 /* Set the right number of segments */ 933 KASSERT(IXGBE_82599_SCATTER_MAX >= IXGBE_SCATTER_DEFAULT); 934 adapter->num_segs = IXGBE_SCATTER_DEFAULT; 935 936 /* Ensure SW/FW semaphore is free */ 937 ixgbe_init_swfw_semaphore(hw); 938 939 hw->mac.ops.set_lan_id(hw); 940 ixgbe_init_device_features(adapter); 941 942 if (ixgbe_configure_interrupts(adapter)) { 943 error = ENXIO; 944 goto err_out; 945 } 946 947 /* Allocate multicast array memory. */ 948 adapter->mta = malloc(sizeof(*adapter->mta) * 949 MAX_NUM_MULTICAST_ADDRESSES, M_DEVBUF, M_WAITOK); 950 951 /* Enable WoL (if supported) */ 952 ixgbe_check_wol_support(adapter); 953 954 /* Register for VLAN events */ 955 ether_set_vlan_cb(&adapter->osdep.ec, ixgbe_vlan_cb); 956 957 /* Verify adapter fan is still functional (if applicable) */ 958 if (adapter->feat_en & IXGBE_FEATURE_FAN_FAIL) { 959 u32 esdp = IXGBE_READ_REG(hw, IXGBE_ESDP); 960 ixgbe_check_fan_failure(adapter, esdp, FALSE); 961 } 962 963 /* Set an initial default flow control value */ 964 hw->fc.requested_mode = ixgbe_flow_control; 965 966 /* Do descriptor calc and sanity checks */ 967 if (((ixgbe_txd * sizeof(union ixgbe_adv_tx_desc)) % DBA_ALIGN) != 0 || 968 ixgbe_txd < MIN_TXD || ixgbe_txd > MAX_TXD) { 969 aprint_error_dev(dev, "TXD config issue, using default!\n"); 970 adapter->num_tx_desc = DEFAULT_TXD; 971 } else 972 adapter->num_tx_desc = ixgbe_txd; 973 974 if (((ixgbe_rxd * sizeof(union ixgbe_adv_rx_desc)) % DBA_ALIGN) != 0 || 975 ixgbe_rxd < MIN_RXD || ixgbe_rxd > MAX_RXD) { 976 aprint_error_dev(dev, "RXD config issue, using default!\n"); 977 adapter->num_rx_desc = DEFAULT_RXD; 978 } else 979 adapter->num_rx_desc = ixgbe_rxd; 980 981 /* Sysctls for limiting the amount of work done in the taskqueues */ 982 adapter->rx_process_limit 983 = (ixgbe_rx_process_limit <= adapter->num_rx_desc) 984 ? ixgbe_rx_process_limit : adapter->num_rx_desc; 985 adapter->tx_process_limit 986 = (ixgbe_tx_process_limit <= adapter->num_tx_desc) 987 ? ixgbe_tx_process_limit : adapter->num_tx_desc; 988 989 /* Set default high limit of copying mbuf in rxeof */ 990 adapter->rx_copy_len = IXGBE_RX_COPY_LEN_MAX; 991 992 /* Allocate our TX/RX Queues */ 993 if (ixgbe_allocate_queues(adapter)) { 994 error = ENOMEM; 995 goto err_out; 996 } 997 998 hw->phy.reset_if_overtemp = TRUE; 999 error = ixgbe_reset_hw(hw); 1000 hw->phy.reset_if_overtemp = FALSE; 1001 if (error == IXGBE_ERR_SFP_NOT_PRESENT) 1002 error = IXGBE_SUCCESS; 1003 else if (error == IXGBE_ERR_SFP_NOT_SUPPORTED) { 1004 aprint_error_dev(dev, "Unsupported SFP+ module detected!\n"); 1005 unsupported_sfp = true; 1006 error = IXGBE_SUCCESS; 1007 } else if (error) { 1008 aprint_error_dev(dev, 1009 "Hardware initialization failed(error = %d)\n", error); 1010 error = EIO; 1011 goto err_late; 1012 } 1013 1014 /* Make sure we have a good EEPROM before we read from it */ 1015 if (ixgbe_validate_eeprom_checksum(&adapter->hw, NULL) < 0) { 1016 aprint_error_dev(dev, "The EEPROM Checksum Is Not Valid\n"); 1017 error = EIO; 1018 goto err_late; 1019 } 1020 1021 aprint_normal("%s:", device_xname(dev)); 1022 /* NVM Image Version */ 1023 high = low = 0; 1024 switch (hw->mac.type) { 1025 case ixgbe_mac_82598EB: 1026 /* 1027 * Print version from the dev starter version (0x29). The 1028 * location is the same as newer device's IXGBE_NVM_MAP_VER. 1029 */ 1030 hw->eeprom.ops.read(hw, IXGBE_NVM_MAP_VER, &nvmreg); 1031 if (nvmreg == 0xffff) 1032 break; 1033 high = (nvmreg >> 12) & 0x0f; 1034 low = (nvmreg >> 4) & 0xff; 1035 id = nvmreg & 0x0f; 1036 /* 1037 * The following output might not be correct. Some 82598 cards 1038 * have 0x1070 or 0x2090. 82598 spec update notes about 2.9.0. 1039 */ 1040 aprint_normal(" NVM Image Version %u.%u.%u,", high, low, id); 1041 break; 1042 case ixgbe_mac_X540: 1043 case ixgbe_mac_X550EM_a: 1044 hw->eeprom.ops.read(hw, IXGBE_NVM_IMAGE_VER, &nvmreg); 1045 if (nvmreg == 0xffff) 1046 break; 1047 high = (nvmreg >> 12) & 0x0f; 1048 low = (nvmreg >> 4) & 0xff; 1049 id = nvmreg & 0x0f; 1050 aprint_normal(" NVM Image Version %u.", high); 1051 if (hw->mac.type == ixgbe_mac_X540) 1052 str = "%x"; 1053 else 1054 str = "%02x"; 1055 aprint_normal(str, low); 1056 aprint_normal(" ID 0x%x,", id); 1057 break; 1058 case ixgbe_mac_X550EM_x: 1059 case ixgbe_mac_X550: 1060 hw->eeprom.ops.read(hw, IXGBE_NVM_IMAGE_VER, &nvmreg); 1061 if (nvmreg == 0xffff) 1062 break; 1063 high = (nvmreg >> 12) & 0x0f; 1064 low = nvmreg & 0xff; 1065 aprint_normal(" NVM Image Version %u.%02x,", high, low); 1066 break; 1067 default: 1068 break; 1069 } 1070 hw->eeprom.nvm_image_ver_high = high; 1071 hw->eeprom.nvm_image_ver_low = low; 1072 1073 /* PHY firmware revision */ 1074 switch (hw->mac.type) { 1075 case ixgbe_mac_X540: 1076 case ixgbe_mac_X550: 1077 hw->eeprom.ops.read(hw, IXGBE_PHYFW_REV, &nvmreg); 1078 if (nvmreg == 0xffff) 1079 break; 1080 high = (nvmreg >> 12) & 0x0f; 1081 low = (nvmreg >> 4) & 0xff; 1082 id = nvmreg & 0x000f; 1083 aprint_normal(" PHY FW Revision %u.", high); 1084 if (hw->mac.type == ixgbe_mac_X540) 1085 str = "%x"; 1086 else 1087 str = "%02x"; 1088 aprint_normal(str, low); 1089 aprint_normal(" ID 0x%x,", id); 1090 break; 1091 default: 1092 break; 1093 } 1094 1095 /* NVM Map version & OEM NVM Image version */ 1096 switch (hw->mac.type) { 1097 case ixgbe_mac_X550: 1098 case ixgbe_mac_X550EM_x: 1099 case ixgbe_mac_X550EM_a: 1100 hw->eeprom.ops.read(hw, IXGBE_NVM_MAP_VER, &nvmreg); 1101 if (nvmreg != 0xffff) { 1102 high = (nvmreg >> 12) & 0x0f; 1103 low = nvmreg & 0x00ff; 1104 aprint_normal(" NVM Map version %u.%02x,", high, low); 1105 } 1106 hw->eeprom.ops.read(hw, IXGBE_OEM_NVM_IMAGE_VER, &nvmreg); 1107 if (nvmreg != 0xffff) { 1108 high = (nvmreg >> 12) & 0x0f; 1109 low = nvmreg & 0x00ff; 1110 aprint_verbose(" OEM NVM Image version %u.%02x,", high, 1111 low); 1112 } 1113 break; 1114 default: 1115 break; 1116 } 1117 1118 /* Print the ETrackID */ 1119 hw->eeprom.ops.read(hw, IXGBE_ETRACKID_H, &high); 1120 hw->eeprom.ops.read(hw, IXGBE_ETRACKID_L, &low); 1121 aprint_normal(" ETrackID %08x\n", ((uint32_t)high << 16) | low); 1122 1123 /* Printed Board Assembly number */ 1124 error = ixgbe_read_pba_string(hw, buf, IXGBE_PBANUM_LENGTH); 1125 aprint_normal_dev(dev, "PBA number %s\n", error ? "unknown" : buf); 1126 1127 if (adapter->feat_en & IXGBE_FEATURE_MSIX) { 1128 error = ixgbe_allocate_msix(adapter, pa); 1129 if (error) { 1130 /* Free allocated queue structures first */ 1131 ixgbe_free_queues(adapter); 1132 1133 /* Fallback to legacy interrupt */ 1134 if (adapter->feat_cap & IXGBE_FEATURE_MSI) 1135 adapter->feat_en |= IXGBE_FEATURE_MSI; 1136 adapter->num_queues = 1; 1137 1138 /* Allocate our TX/RX Queues again */ 1139 if (ixgbe_allocate_queues(adapter)) { 1140 error = ENOMEM; 1141 goto err_out; 1142 } 1143 } 1144 } 1145 1146 /* Recovery mode */ 1147 switch (adapter->hw.mac.type) { 1148 case ixgbe_mac_X550: 1149 case ixgbe_mac_X550EM_x: 1150 case ixgbe_mac_X550EM_a: 1151 /* >= 2.00 */ 1152 if (hw->eeprom.nvm_image_ver_high >= 2) { 1153 adapter->feat_cap |= IXGBE_FEATURE_RECOVERY_MODE; 1154 adapter->feat_en |= IXGBE_FEATURE_RECOVERY_MODE; 1155 } 1156 break; 1157 default: 1158 break; 1159 } 1160 1161 if ((adapter->feat_en & IXGBE_FEATURE_MSIX) == 0) 1162 error = ixgbe_allocate_legacy(adapter, pa); 1163 if (error) 1164 goto err_late; 1165 1166 /* Tasklets for Link, SFP, Multispeed Fiber and Flow Director */ 1167 mutex_init(&(adapter)->admin_mtx, MUTEX_DEFAULT, IPL_NET); 1168 snprintf(wqname, sizeof(wqname), "%s-admin", device_xname(dev)); 1169 error = workqueue_create(&adapter->admin_wq, wqname, 1170 ixgbe_handle_admin, adapter, IXGBE_WORKQUEUE_PRI, IPL_NET, 1171 IXGBE_TASKLET_WQ_FLAGS); 1172 if (error) { 1173 aprint_error_dev(dev, 1174 "could not create admin workqueue (%d)\n", error); 1175 goto err_out; 1176 } 1177 1178 error = ixgbe_start_hw(hw); 1179 switch (error) { 1180 case IXGBE_ERR_EEPROM_VERSION: 1181 aprint_error_dev(dev, "This device is a pre-production adapter/" 1182 "LOM. Please be aware there may be issues associated " 1183 "with your hardware.\nIf you are experiencing problems " 1184 "please contact your Intel or hardware representative " 1185 "who provided you with this hardware.\n"); 1186 break; 1187 default: 1188 break; 1189 } 1190 1191 /* Setup OS specific network interface */ 1192 if (ixgbe_setup_interface(dev, adapter) != 0) 1193 goto err_late; 1194 1195 /* 1196 * Print PHY ID only for copper PHY. On device which has SFP(+) cage 1197 * and a module is inserted, phy.id is not MII PHY id but SFF 8024 ID. 1198 */ 1199 if (hw->phy.media_type == ixgbe_media_type_copper) { 1200 uint16_t id1, id2; 1201 int oui, model, rev; 1202 char descr[MII_MAX_DESCR_LEN]; 1203 1204 id1 = hw->phy.id >> 16; 1205 id2 = hw->phy.id & 0xffff; 1206 oui = MII_OUI(id1, id2); 1207 model = MII_MODEL(id2); 1208 rev = MII_REV(id2); 1209 mii_get_descr(descr, sizeof(descr), oui, model); 1210 if (descr[0]) 1211 aprint_normal_dev(dev, "PHY: %s, rev. %d\n", 1212 descr, rev); 1213 else 1214 aprint_normal_dev(dev, 1215 "PHY OUI 0x%06x, model 0x%04x, rev. %d\n", 1216 oui, model, rev); 1217 } 1218 1219 /* Enable EEE power saving */ 1220 if (adapter->feat_cap & IXGBE_FEATURE_EEE) 1221 hw->mac.ops.setup_eee(hw, 1222 adapter->feat_en & IXGBE_FEATURE_EEE); 1223 1224 /* Enable power to the phy. */ 1225 if (!unsupported_sfp) { 1226 /* Enable the optics for 82599 SFP+ fiber */ 1227 ixgbe_enable_tx_laser(hw); 1228 1229 /* 1230 * XXX Currently, ixgbe_set_phy_power() supports only copper 1231 * PHY, so it's not required to test with !unsupported_sfp. 1232 */ 1233 ixgbe_set_phy_power(hw, TRUE); 1234 } 1235 1236 /* Initialize statistics */ 1237 ixgbe_update_stats_counters(adapter); 1238 1239 /* Check PCIE slot type/speed/width */ 1240 ixgbe_get_slot_info(adapter); 1241 1242 /* 1243 * Do time init and sysctl init here, but 1244 * only on the first port of a bypass adapter. 1245 */ 1246 ixgbe_bypass_init(adapter); 1247 1248 /* Set an initial dmac value */ 1249 adapter->dmac = 0; 1250 /* Set initial advertised speeds (if applicable) */ 1251 adapter->advertise = ixgbe_get_default_advertise(adapter); 1252 1253 if (adapter->feat_cap & IXGBE_FEATURE_SRIOV) 1254 ixgbe_define_iov_schemas(dev, &error); 1255 1256 /* Add sysctls */ 1257 ixgbe_add_device_sysctls(adapter); 1258 ixgbe_add_hw_stats(adapter); 1259 1260 /* For Netmap */ 1261 adapter->init_locked = ixgbe_init_locked; 1262 adapter->stop_locked = ixgbe_stop_locked; 1263 1264 if (adapter->feat_en & IXGBE_FEATURE_NETMAP) 1265 ixgbe_netmap_attach(adapter); 1266 1267 snprintb(buf, sizeof(buf), IXGBE_FEATURE_FLAGS, adapter->feat_cap); 1268 aprint_verbose_dev(dev, "feature cap %s\n", buf); 1269 snprintb(buf, sizeof(buf), IXGBE_FEATURE_FLAGS, adapter->feat_en); 1270 aprint_verbose_dev(dev, "feature ena %s\n", buf); 1271 1272 if (pmf_device_register(dev, ixgbe_suspend, ixgbe_resume)) 1273 pmf_class_network_register(dev, adapter->ifp); 1274 else 1275 aprint_error_dev(dev, "couldn't establish power handler\n"); 1276 1277 /* Init recovery mode timer and state variable */ 1278 if (adapter->feat_en & IXGBE_FEATURE_RECOVERY_MODE) { 1279 adapter->recovery_mode = 0; 1280 1281 /* Set up the timer callout */ 1282 callout_init(&adapter->recovery_mode_timer, 1283 IXGBE_CALLOUT_FLAGS); 1284 snprintf(wqname, sizeof(wqname), "%s-recovery", 1285 device_xname(dev)); 1286 error = workqueue_create(&adapter->recovery_mode_timer_wq, 1287 wqname, ixgbe_handle_recovery_mode_timer, adapter, 1288 IXGBE_WORKQUEUE_PRI, IPL_NET, IXGBE_TASKLET_WQ_FLAGS); 1289 if (error) { 1290 aprint_error_dev(dev, "could not create " 1291 "recovery_mode_timer workqueue (%d)\n", error); 1292 goto err_out; 1293 } 1294 1295 /* Start the task */ 1296 callout_reset(&adapter->recovery_mode_timer, hz, 1297 ixgbe_recovery_mode_timer, adapter); 1298 } 1299 1300 INIT_DEBUGOUT("ixgbe_attach: end"); 1301 adapter->osdep.attached = true; 1302 1303 return; 1304 1305 err_late: 1306 ixgbe_free_queues(adapter); 1307 err_out: 1308 ctrl_ext = IXGBE_READ_REG(&adapter->hw, IXGBE_CTRL_EXT); 1309 ctrl_ext &= ~IXGBE_CTRL_EXT_DRV_LOAD; 1310 IXGBE_WRITE_REG(&adapter->hw, IXGBE_CTRL_EXT, ctrl_ext); 1311 ixgbe_free_deferred_handlers(adapter); 1312 ixgbe_free_pci_resources(adapter); 1313 if (adapter->mta != NULL) 1314 free(adapter->mta, M_DEVBUF); 1315 mutex_destroy(&(adapter)->admin_mtx); /* XXX appropriate order? */ 1316 IXGBE_CORE_LOCK_DESTROY(adapter); 1317 1318 return; 1319 } /* ixgbe_attach */ 1320 1321 /************************************************************************ 1322 * ixgbe_check_wol_support 1323 * 1324 * Checks whether the adapter's ports are capable of 1325 * Wake On LAN by reading the adapter's NVM. 1326 * 1327 * Sets each port's hw->wol_enabled value depending 1328 * on the value read here. 1329 ************************************************************************/ 1330 static void 1331 ixgbe_check_wol_support(struct adapter *adapter) 1332 { 1333 struct ixgbe_hw *hw = &adapter->hw; 1334 u16 dev_caps = 0; 1335 1336 /* Find out WoL support for port */ 1337 adapter->wol_support = hw->wol_enabled = 0; 1338 ixgbe_get_device_caps(hw, &dev_caps); 1339 if ((dev_caps & IXGBE_DEVICE_CAPS_WOL_PORT0_1) || 1340 ((dev_caps & IXGBE_DEVICE_CAPS_WOL_PORT0) && 1341 hw->bus.func == 0)) 1342 adapter->wol_support = hw->wol_enabled = 1; 1343 1344 /* Save initial wake up filter configuration */ 1345 adapter->wufc = IXGBE_READ_REG(hw, IXGBE_WUFC); 1346 1347 return; 1348 } /* ixgbe_check_wol_support */ 1349 1350 /************************************************************************ 1351 * ixgbe_setup_interface 1352 * 1353 * Setup networking device structure and register an interface. 1354 ************************************************************************/ 1355 static int 1356 ixgbe_setup_interface(device_t dev, struct adapter *adapter) 1357 { 1358 struct ethercom *ec = &adapter->osdep.ec; 1359 struct ifnet *ifp; 1360 1361 INIT_DEBUGOUT("ixgbe_setup_interface: begin"); 1362 1363 ifp = adapter->ifp = &ec->ec_if; 1364 strlcpy(ifp->if_xname, device_xname(dev), IFNAMSIZ); 1365 ifp->if_baudrate = IF_Gbps(10); 1366 ifp->if_init = ixgbe_init; 1367 ifp->if_stop = ixgbe_ifstop; 1368 ifp->if_softc = adapter; 1369 ifp->if_flags = IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST; 1370 #ifdef IXGBE_MPSAFE 1371 ifp->if_extflags = IFEF_MPSAFE; 1372 #endif 1373 ifp->if_ioctl = ixgbe_ioctl; 1374 #if __FreeBSD_version >= 1100045 1375 /* TSO parameters */ 1376 ifp->if_hw_tsomax = 65518; 1377 ifp->if_hw_tsomaxsegcount = IXGBE_82599_SCATTER; 1378 ifp->if_hw_tsomaxsegsize = 2048; 1379 #endif 1380 if (adapter->feat_en & IXGBE_FEATURE_LEGACY_TX) { 1381 #if 0 1382 ixgbe_start_locked = ixgbe_legacy_start_locked; 1383 #endif 1384 } else { 1385 ifp->if_transmit = ixgbe_mq_start; 1386 #if 0 1387 ixgbe_start_locked = ixgbe_mq_start_locked; 1388 #endif 1389 } 1390 ifp->if_start = ixgbe_legacy_start; 1391 IFQ_SET_MAXLEN(&ifp->if_snd, adapter->num_tx_desc - 2); 1392 IFQ_SET_READY(&ifp->if_snd); 1393 1394 if_initialize(ifp); 1395 adapter->ipq = if_percpuq_create(&adapter->osdep.ec.ec_if); 1396 ether_ifattach(ifp, adapter->hw.mac.addr); 1397 aprint_normal_dev(dev, "Ethernet address %s\n", 1398 ether_sprintf(adapter->hw.mac.addr)); 1399 /* 1400 * We use per TX queue softint, so if_deferred_start_init() isn't 1401 * used. 1402 */ 1403 ether_set_ifflags_cb(ec, ixgbe_ifflags_cb); 1404 1405 adapter->max_frame_size = ifp->if_mtu + ETHER_HDR_LEN + ETHER_CRC_LEN; 1406 1407 /* 1408 * Tell the upper layer(s) we support long frames. 1409 */ 1410 ifp->if_hdrlen = sizeof(struct ether_vlan_header); 1411 1412 /* Set capability flags */ 1413 ifp->if_capabilities |= IFCAP_RXCSUM 1414 | IFCAP_TXCSUM 1415 | IFCAP_TSOv4 1416 | IFCAP_TSOv6; 1417 ifp->if_capenable = 0; 1418 1419 ec->ec_capabilities |= ETHERCAP_VLAN_HWTAGGING 1420 | ETHERCAP_VLAN_HWCSUM 1421 | ETHERCAP_JUMBO_MTU 1422 | ETHERCAP_VLAN_MTU; 1423 1424 /* Enable the above capabilities by default */ 1425 ec->ec_capenable = ec->ec_capabilities; 1426 1427 /* 1428 * Don't turn this on by default, if vlans are 1429 * created on another pseudo device (eg. lagg) 1430 * then vlan events are not passed thru, breaking 1431 * operation, but with HW FILTER off it works. If 1432 * using vlans directly on the ixgbe driver you can 1433 * enable this and get full hardware tag filtering. 1434 */ 1435 ec->ec_capabilities |= ETHERCAP_VLAN_HWFILTER; 1436 1437 /* 1438 * Specify the media types supported by this adapter and register 1439 * callbacks to update media and link information 1440 */ 1441 ec->ec_ifmedia = &adapter->media; 1442 ifmedia_init_with_lock(&adapter->media, IFM_IMASK, ixgbe_media_change, 1443 ixgbe_media_status, &adapter->core_mtx); 1444 1445 adapter->phy_layer = ixgbe_get_supported_physical_layer(&adapter->hw); 1446 ixgbe_add_media_types(adapter); 1447 1448 /* Set autoselect media by default */ 1449 ifmedia_set(&adapter->media, IFM_ETHER | IFM_AUTO); 1450 1451 if_register(ifp); 1452 1453 return (0); 1454 } /* ixgbe_setup_interface */ 1455 1456 /************************************************************************ 1457 * ixgbe_add_media_types 1458 ************************************************************************/ 1459 static void 1460 ixgbe_add_media_types(struct adapter *adapter) 1461 { 1462 struct ixgbe_hw *hw = &adapter->hw; 1463 u64 layer; 1464 1465 layer = adapter->phy_layer; 1466 1467 #define ADD(mm, dd) \ 1468 ifmedia_add(&adapter->media, IFM_ETHER | (mm), (dd), NULL); 1469 1470 ADD(IFM_NONE, 0); 1471 1472 /* Media types with matching NetBSD media defines */ 1473 if (layer & IXGBE_PHYSICAL_LAYER_10GBASE_T) { 1474 ADD(IFM_10G_T | IFM_FDX, 0); 1475 } 1476 if (layer & IXGBE_PHYSICAL_LAYER_1000BASE_T) { 1477 ADD(IFM_1000_T | IFM_FDX, 0); 1478 } 1479 if (layer & IXGBE_PHYSICAL_LAYER_100BASE_TX) { 1480 ADD(IFM_100_TX | IFM_FDX, 0); 1481 } 1482 if (layer & IXGBE_PHYSICAL_LAYER_10BASE_T) { 1483 ADD(IFM_10_T | IFM_FDX, 0); 1484 } 1485 1486 if (layer & IXGBE_PHYSICAL_LAYER_SFP_PLUS_CU || 1487 layer & IXGBE_PHYSICAL_LAYER_SFP_ACTIVE_DA) { 1488 ADD(IFM_10G_TWINAX | IFM_FDX, 0); 1489 } 1490 1491 if (layer & IXGBE_PHYSICAL_LAYER_10GBASE_LR) { 1492 ADD(IFM_10G_LR | IFM_FDX, 0); 1493 if (hw->phy.multispeed_fiber) { 1494 ADD(IFM_1000_LX | IFM_FDX, 0); 1495 } 1496 } 1497 if (layer & IXGBE_PHYSICAL_LAYER_10GBASE_SR) { 1498 ADD(IFM_10G_SR | IFM_FDX, 0); 1499 if (hw->phy.multispeed_fiber) { 1500 ADD(IFM_1000_SX | IFM_FDX, 0); 1501 } 1502 } else if (layer & IXGBE_PHYSICAL_LAYER_1000BASE_SX) { 1503 ADD(IFM_1000_SX | IFM_FDX, 0); 1504 } 1505 if (layer & IXGBE_PHYSICAL_LAYER_10GBASE_CX4) { 1506 ADD(IFM_10G_CX4 | IFM_FDX, 0); 1507 } 1508 1509 if (layer & IXGBE_PHYSICAL_LAYER_10GBASE_KR) { 1510 ADD(IFM_10G_KR | IFM_FDX, 0); 1511 } 1512 if (layer & IXGBE_PHYSICAL_LAYER_10GBASE_KX4) { 1513 ADD(IFM_10G_KX4 | IFM_FDX, 0); 1514 } 1515 if (layer & IXGBE_PHYSICAL_LAYER_1000BASE_KX) { 1516 ADD(IFM_1000_KX | IFM_FDX, 0); 1517 } 1518 if (layer & IXGBE_PHYSICAL_LAYER_2500BASE_KX) { 1519 ADD(IFM_2500_KX | IFM_FDX, 0); 1520 } 1521 if (layer & IXGBE_PHYSICAL_LAYER_2500BASE_T) { 1522 ADD(IFM_2500_T | IFM_FDX, 0); 1523 } 1524 if (layer & IXGBE_PHYSICAL_LAYER_5GBASE_T) { 1525 ADD(IFM_5000_T | IFM_FDX, 0); 1526 } 1527 if (layer & IXGBE_PHYSICAL_LAYER_1000BASE_BX) 1528 ADD(IFM_1000_BX10 | IFM_FDX, 0); 1529 /* XXX no ifmedia_set? */ 1530 1531 ADD(IFM_AUTO, 0); 1532 1533 #undef ADD 1534 } /* ixgbe_add_media_types */ 1535 1536 /************************************************************************ 1537 * ixgbe_is_sfp 1538 ************************************************************************/ 1539 static inline bool 1540 ixgbe_is_sfp(struct ixgbe_hw *hw) 1541 { 1542 switch (hw->mac.type) { 1543 case ixgbe_mac_82598EB: 1544 if (hw->phy.type == ixgbe_phy_nl) 1545 return (TRUE); 1546 return (FALSE); 1547 case ixgbe_mac_82599EB: 1548 case ixgbe_mac_X550EM_x: 1549 case ixgbe_mac_X550EM_a: 1550 switch (hw->mac.ops.get_media_type(hw)) { 1551 case ixgbe_media_type_fiber: 1552 case ixgbe_media_type_fiber_qsfp: 1553 return (TRUE); 1554 default: 1555 return (FALSE); 1556 } 1557 default: 1558 return (FALSE); 1559 } 1560 } /* ixgbe_is_sfp */ 1561 1562 static void 1563 ixgbe_schedule_admin_tasklet(struct adapter *adapter) 1564 { 1565 1566 KASSERT(mutex_owned(&adapter->admin_mtx)); 1567 1568 if (__predict_true(adapter->osdep.detaching == false)) { 1569 if (adapter->admin_pending == 0) 1570 workqueue_enqueue(adapter->admin_wq, 1571 &adapter->admin_wc, NULL); 1572 adapter->admin_pending = 1; 1573 } 1574 } 1575 1576 /************************************************************************ 1577 * ixgbe_config_link 1578 ************************************************************************/ 1579 static void 1580 ixgbe_config_link(struct adapter *adapter) 1581 { 1582 struct ixgbe_hw *hw = &adapter->hw; 1583 u32 autoneg, err = 0; 1584 u32 task_requests = 0; 1585 bool sfp, negotiate = false; 1586 1587 sfp = ixgbe_is_sfp(hw); 1588 1589 if (sfp) { 1590 if (hw->phy.multispeed_fiber) { 1591 ixgbe_enable_tx_laser(hw); 1592 task_requests |= IXGBE_REQUEST_TASK_MSF_WOI; 1593 } 1594 task_requests |= IXGBE_REQUEST_TASK_MOD_WOI; 1595 1596 mutex_enter(&adapter->admin_mtx); 1597 adapter->task_requests |= task_requests; 1598 ixgbe_schedule_admin_tasklet(adapter); 1599 mutex_exit(&adapter->admin_mtx); 1600 } else { 1601 struct ifmedia *ifm = &adapter->media; 1602 1603 if (hw->mac.ops.check_link) 1604 err = ixgbe_check_link(hw, &adapter->link_speed, 1605 &adapter->link_up, FALSE); 1606 if (err) 1607 return; 1608 1609 /* 1610 * Check if it's the first call. If it's the first call, 1611 * get value for auto negotiation. 1612 */ 1613 autoneg = hw->phy.autoneg_advertised; 1614 if ((IFM_SUBTYPE(ifm->ifm_cur->ifm_media) != IFM_NONE) 1615 && ((!autoneg) && (hw->mac.ops.get_link_capabilities))) 1616 err = hw->mac.ops.get_link_capabilities(hw, &autoneg, 1617 &negotiate); 1618 if (err) 1619 return; 1620 if (hw->mac.ops.setup_link) 1621 err = hw->mac.ops.setup_link(hw, autoneg, 1622 adapter->link_up); 1623 } 1624 1625 } /* ixgbe_config_link */ 1626 1627 /************************************************************************ 1628 * ixgbe_update_stats_counters - Update board statistics counters. 1629 ************************************************************************/ 1630 static void 1631 ixgbe_update_stats_counters(struct adapter *adapter) 1632 { 1633 struct ifnet *ifp = adapter->ifp; 1634 struct ixgbe_hw *hw = &adapter->hw; 1635 struct ixgbe_hw_stats *stats = &adapter->stats.pf; 1636 u32 missed_rx = 0, bprc, lxontxc, lxofftxc; 1637 u64 total, total_missed_rx = 0; 1638 uint64_t crcerrs, illerrc, rlec, ruc, rfc, roc, rjc; 1639 unsigned int queue_counters; 1640 int i; 1641 1642 IXGBE_EVC_REGADD2(hw, stats, IXGBE_CRCERRS, crcerrs); 1643 IXGBE_EVC_REGADD2(hw, stats, IXGBE_ILLERRC, illerrc); 1644 1645 IXGBE_EVC_REGADD(hw, stats, IXGBE_ERRBC, errbc); 1646 IXGBE_EVC_REGADD(hw, stats, IXGBE_MSPDC, mspdc); 1647 if (hw->mac.type >= ixgbe_mac_X550) 1648 IXGBE_EVC_REGADD(hw, stats, IXGBE_MBSDC, mbsdc); 1649 1650 /* 16 registers exist */ 1651 queue_counters = uimin(__arraycount(stats->qprc), adapter->num_queues); 1652 for (i = 0; i < queue_counters; i++) { 1653 IXGBE_EVC_REGADD(hw, stats, IXGBE_QPRC(i), qprc[i]); 1654 IXGBE_EVC_REGADD(hw, stats, IXGBE_QPTC(i), qptc[i]); 1655 if (hw->mac.type >= ixgbe_mac_82599EB) 1656 IXGBE_EVC_REGADD(hw, stats, IXGBE_QPRDC(i), qprdc[i]); 1657 } 1658 1659 /* 8 registers exist */ 1660 for (i = 0; i < IXGBE_TC_COUNTER_NUM; i++) { 1661 uint32_t mp; 1662 1663 /* MPC */ 1664 mp = IXGBE_READ_REG(hw, IXGBE_MPC(i)); 1665 /* global total per queue */ 1666 IXGBE_EVC_ADD(&stats->mpc[i], mp); 1667 /* running comprehensive total for stats display */ 1668 total_missed_rx += mp; 1669 1670 if (hw->mac.type == ixgbe_mac_82598EB) 1671 IXGBE_EVC_REGADD(hw, stats, IXGBE_RNBC(i), rnbc[i]); 1672 1673 IXGBE_EVC_REGADD(hw, stats, IXGBE_PXONTXC(i), pxontxc[i]); 1674 IXGBE_EVC_REGADD(hw, stats, IXGBE_PXOFFTXC(i), pxofftxc[i]); 1675 if (hw->mac.type >= ixgbe_mac_82599EB) { 1676 IXGBE_EVC_REGADD(hw, stats, IXGBE_PXONRXCNT(i), pxonrxc[i]); 1677 IXGBE_EVC_REGADD(hw, stats, IXGBE_PXOFFRXCNT(i), pxoffrxc[i]); 1678 IXGBE_EVC_REGADD(hw, stats, IXGBE_PXON2OFFCNT(i), 1679 pxon2offc[i]); 1680 } else { 1681 IXGBE_EVC_REGADD(hw, stats, IXGBE_PXONRXC(i), pxonrxc[i]); 1682 IXGBE_EVC_REGADD(hw, stats, IXGBE_PXOFFRXC(i), pxoffrxc[i]); 1683 } 1684 } 1685 IXGBE_EVC_ADD(&stats->mpctotal, total_missed_rx); 1686 1687 /* Document says M[LR]FC are valid when link is up and 10Gbps */ 1688 if ((adapter->link_active == LINK_STATE_UP) 1689 && (adapter->link_speed == IXGBE_LINK_SPEED_10GB_FULL)) { 1690 IXGBE_EVC_REGADD(hw, stats, IXGBE_MLFC, mlfc); 1691 IXGBE_EVC_REGADD(hw, stats, IXGBE_MRFC, mrfc); 1692 } 1693 IXGBE_EVC_REGADD2(hw, stats, IXGBE_RLEC, rlec); 1694 1695 /* Hardware workaround, gprc counts missed packets */ 1696 IXGBE_EVC_ADD(&stats->gprc, 1697 IXGBE_READ_REG(hw, IXGBE_GPRC) - missed_rx); 1698 1699 IXGBE_EVC_REGADD2(hw, stats, IXGBE_LXONTXC, lxontxc); 1700 IXGBE_EVC_REGADD2(hw, stats, IXGBE_LXOFFTXC, lxofftxc); 1701 total = lxontxc + lxofftxc; 1702 1703 if (hw->mac.type != ixgbe_mac_82598EB) { 1704 IXGBE_EVC_ADD(&stats->gorc, IXGBE_READ_REG(hw, IXGBE_GORCL) + 1705 ((u64)IXGBE_READ_REG(hw, IXGBE_GORCH) << 32)); 1706 IXGBE_EVC_ADD(&stats->gotc, IXGBE_READ_REG(hw, IXGBE_GOTCL) + 1707 ((u64)IXGBE_READ_REG(hw, IXGBE_GOTCH) << 32) 1708 - total * ETHER_MIN_LEN); 1709 IXGBE_EVC_ADD(&stats->tor, IXGBE_READ_REG(hw, IXGBE_TORL) + 1710 ((u64)IXGBE_READ_REG(hw, IXGBE_TORH) << 32)); 1711 IXGBE_EVC_REGADD(hw, stats, IXGBE_LXONRXCNT, lxonrxc); 1712 IXGBE_EVC_REGADD(hw, stats, IXGBE_LXOFFRXCNT, lxoffrxc); 1713 } else { 1714 IXGBE_EVC_REGADD(hw, stats, IXGBE_LXONRXC, lxonrxc); 1715 IXGBE_EVC_REGADD(hw, stats, IXGBE_LXOFFRXC, lxoffrxc); 1716 /* 82598 only has a counter in the high register */ 1717 IXGBE_EVC_REGADD(hw, stats, IXGBE_GORCH, gorc); 1718 IXGBE_EVC_ADD(&stats->gotc, IXGBE_READ_REG(hw, IXGBE_GOTCH) 1719 - total * ETHER_MIN_LEN); 1720 IXGBE_EVC_REGADD(hw, stats, IXGBE_TORH, tor); 1721 } 1722 1723 /* 1724 * Workaround: mprc hardware is incorrectly counting 1725 * broadcasts, so for now we subtract those. 1726 */ 1727 IXGBE_EVC_REGADD2(hw, stats, IXGBE_BPRC, bprc); 1728 IXGBE_EVC_ADD(&stats->mprc, IXGBE_READ_REG(hw, IXGBE_MPRC) 1729 - ((hw->mac.type == ixgbe_mac_82598EB) ? bprc : 0)); 1730 1731 IXGBE_EVC_REGADD(hw, stats, IXGBE_PRC64, prc64); 1732 IXGBE_EVC_REGADD(hw, stats, IXGBE_PRC127, prc127); 1733 IXGBE_EVC_REGADD(hw, stats, IXGBE_PRC255, prc255); 1734 IXGBE_EVC_REGADD(hw, stats, IXGBE_PRC511, prc511); 1735 IXGBE_EVC_REGADD(hw, stats, IXGBE_PRC1023, prc1023); 1736 IXGBE_EVC_REGADD(hw, stats, IXGBE_PRC1522, prc1522); 1737 1738 IXGBE_EVC_ADD(&stats->gptc, IXGBE_READ_REG(hw, IXGBE_GPTC) - total); 1739 IXGBE_EVC_ADD(&stats->mptc, IXGBE_READ_REG(hw, IXGBE_MPTC) - total); 1740 IXGBE_EVC_ADD(&stats->ptc64, IXGBE_READ_REG(hw, IXGBE_PTC64) - total); 1741 1742 IXGBE_EVC_REGADD2(hw, stats, IXGBE_RUC, ruc); 1743 IXGBE_EVC_REGADD2(hw, stats, IXGBE_RFC, rfc); 1744 IXGBE_EVC_REGADD2(hw, stats, IXGBE_ROC, roc); 1745 IXGBE_EVC_REGADD2(hw, stats, IXGBE_RJC, rjc); 1746 1747 IXGBE_EVC_REGADD(hw, stats, IXGBE_MNGPRC, mngprc); 1748 IXGBE_EVC_REGADD(hw, stats, IXGBE_MNGPDC, mngpdc); 1749 IXGBE_EVC_REGADD(hw, stats, IXGBE_MNGPTC, mngptc); 1750 IXGBE_EVC_REGADD(hw, stats, IXGBE_TPR, tpr); 1751 IXGBE_EVC_REGADD(hw, stats, IXGBE_TPT, tpt); 1752 IXGBE_EVC_REGADD(hw, stats, IXGBE_PTC127, ptc127); 1753 IXGBE_EVC_REGADD(hw, stats, IXGBE_PTC255, ptc255); 1754 IXGBE_EVC_REGADD(hw, stats, IXGBE_PTC511, ptc511); 1755 IXGBE_EVC_REGADD(hw, stats, IXGBE_PTC1023, ptc1023); 1756 IXGBE_EVC_REGADD(hw, stats, IXGBE_PTC1522, ptc1522); 1757 IXGBE_EVC_REGADD(hw, stats, IXGBE_BPTC, bptc); 1758 IXGBE_EVC_REGADD(hw, stats, IXGBE_XEC, xec); 1759 IXGBE_EVC_REGADD(hw, stats, IXGBE_FCCRC, fccrc); 1760 IXGBE_EVC_REGADD(hw, stats, IXGBE_FCLAST, fclast); 1761 /* Only read FCOE on 82599 */ 1762 if (hw->mac.type != ixgbe_mac_82598EB) { 1763 IXGBE_EVC_REGADD(hw, stats, IXGBE_FCOERPDC, fcoerpdc); 1764 IXGBE_EVC_REGADD(hw, stats, IXGBE_FCOEPRC, fcoeprc); 1765 IXGBE_EVC_REGADD(hw, stats, IXGBE_FCOEPTC, fcoeptc); 1766 IXGBE_EVC_REGADD(hw, stats, IXGBE_FCOEDWRC, fcoedwrc); 1767 IXGBE_EVC_REGADD(hw, stats, IXGBE_FCOEDWTC, fcoedwtc); 1768 } 1769 1770 /* 1771 * Fill out the OS statistics structure. Only RX errors are required 1772 * here because all TX counters are incremented in the TX path and 1773 * normal RX counters are prepared in ether_input(). 1774 */ 1775 net_stat_ref_t nsr = IF_STAT_GETREF(ifp); 1776 if_statadd_ref(nsr, if_iqdrops, total_missed_rx); 1777 1778 /* 1779 * Aggregate following types of errors as RX errors: 1780 * - CRC error count, 1781 * - illegal byte error count, 1782 * - length error count, 1783 * - undersized packets count, 1784 * - fragmented packets count, 1785 * - oversized packets count, 1786 * - jabber count. 1787 */ 1788 if_statadd_ref(nsr, if_ierrors, 1789 crcerrs + illerrc + rlec + ruc + rfc + roc + rjc); 1790 1791 IF_STAT_PUTREF(ifp); 1792 } /* ixgbe_update_stats_counters */ 1793 1794 /************************************************************************ 1795 * ixgbe_add_hw_stats 1796 * 1797 * Add sysctl variables, one per statistic, to the system. 1798 ************************************************************************/ 1799 static void 1800 ixgbe_add_hw_stats(struct adapter *adapter) 1801 { 1802 device_t dev = adapter->dev; 1803 const struct sysctlnode *rnode, *cnode; 1804 struct sysctllog **log = &adapter->sysctllog; 1805 struct tx_ring *txr = adapter->tx_rings; 1806 struct rx_ring *rxr = adapter->rx_rings; 1807 struct ixgbe_hw *hw = &adapter->hw; 1808 struct ixgbe_hw_stats *stats = &adapter->stats.pf; 1809 const char *xname = device_xname(dev); 1810 int i; 1811 1812 /* Driver Statistics */ 1813 evcnt_attach_dynamic(&adapter->efbig_tx_dma_setup, EVCNT_TYPE_MISC, 1814 NULL, xname, "Driver tx dma soft fail EFBIG"); 1815 evcnt_attach_dynamic(&adapter->mbuf_defrag_failed, EVCNT_TYPE_MISC, 1816 NULL, xname, "m_defrag() failed"); 1817 evcnt_attach_dynamic(&adapter->efbig2_tx_dma_setup, EVCNT_TYPE_MISC, 1818 NULL, xname, "Driver tx dma hard fail EFBIG"); 1819 evcnt_attach_dynamic(&adapter->einval_tx_dma_setup, EVCNT_TYPE_MISC, 1820 NULL, xname, "Driver tx dma hard fail EINVAL"); 1821 evcnt_attach_dynamic(&adapter->other_tx_dma_setup, EVCNT_TYPE_MISC, 1822 NULL, xname, "Driver tx dma hard fail other"); 1823 evcnt_attach_dynamic(&adapter->eagain_tx_dma_setup, EVCNT_TYPE_MISC, 1824 NULL, xname, "Driver tx dma soft fail EAGAIN"); 1825 evcnt_attach_dynamic(&adapter->enomem_tx_dma_setup, EVCNT_TYPE_MISC, 1826 NULL, xname, "Driver tx dma soft fail ENOMEM"); 1827 evcnt_attach_dynamic(&adapter->watchdog_events, EVCNT_TYPE_MISC, 1828 NULL, xname, "Watchdog timeouts"); 1829 evcnt_attach_dynamic(&adapter->tso_err, EVCNT_TYPE_MISC, 1830 NULL, xname, "TSO errors"); 1831 evcnt_attach_dynamic(&adapter->admin_irqev, EVCNT_TYPE_INTR, 1832 NULL, xname, "Admin MSI-X IRQ Handled"); 1833 evcnt_attach_dynamic(&adapter->link_workev, EVCNT_TYPE_INTR, 1834 NULL, xname, "Link event"); 1835 evcnt_attach_dynamic(&adapter->mod_workev, EVCNT_TYPE_INTR, 1836 NULL, xname, "SFP+ module event"); 1837 evcnt_attach_dynamic(&adapter->msf_workev, EVCNT_TYPE_INTR, 1838 NULL, xname, "Multispeed event"); 1839 evcnt_attach_dynamic(&adapter->phy_workev, EVCNT_TYPE_INTR, 1840 NULL, xname, "External PHY event"); 1841 1842 /* Max number of traffic class is 8 */ 1843 KASSERT(IXGBE_DCB_MAX_TRAFFIC_CLASS == 8); 1844 for (i = 0; i < IXGBE_TC_COUNTER_NUM; i++) { 1845 snprintf(adapter->tcs[i].evnamebuf, 1846 sizeof(adapter->tcs[i].evnamebuf), "%s tc%d", 1847 xname, i); 1848 if (i < __arraycount(stats->mpc)) { 1849 evcnt_attach_dynamic(&stats->mpc[i], 1850 EVCNT_TYPE_MISC, NULL, adapter->tcs[i].evnamebuf, 1851 "RX Missed Packet Count"); 1852 if (hw->mac.type == ixgbe_mac_82598EB) 1853 evcnt_attach_dynamic(&stats->rnbc[i], 1854 EVCNT_TYPE_MISC, NULL, 1855 adapter->tcs[i].evnamebuf, 1856 "Receive No Buffers"); 1857 } 1858 if (i < __arraycount(stats->pxontxc)) { 1859 evcnt_attach_dynamic(&stats->pxontxc[i], 1860 EVCNT_TYPE_MISC, NULL, adapter->tcs[i].evnamebuf, 1861 "pxontxc"); 1862 evcnt_attach_dynamic(&stats->pxonrxc[i], 1863 EVCNT_TYPE_MISC, NULL, adapter->tcs[i].evnamebuf, 1864 "pxonrxc"); 1865 evcnt_attach_dynamic(&stats->pxofftxc[i], 1866 EVCNT_TYPE_MISC, NULL, adapter->tcs[i].evnamebuf, 1867 "pxofftxc"); 1868 evcnt_attach_dynamic(&stats->pxoffrxc[i], 1869 EVCNT_TYPE_MISC, NULL, adapter->tcs[i].evnamebuf, 1870 "pxoffrxc"); 1871 if (hw->mac.type >= ixgbe_mac_82599EB) 1872 evcnt_attach_dynamic(&stats->pxon2offc[i], 1873 EVCNT_TYPE_MISC, NULL, 1874 adapter->tcs[i].evnamebuf, 1875 "pxon2offc"); 1876 } 1877 } 1878 1879 for (i = 0; i < adapter->num_queues; i++, rxr++, txr++) { 1880 #ifdef LRO 1881 struct lro_ctrl *lro = &rxr->lro; 1882 #endif /* LRO */ 1883 1884 snprintf(adapter->queues[i].evnamebuf, 1885 sizeof(adapter->queues[i].evnamebuf), "%s q%d", 1886 xname, i); 1887 snprintf(adapter->queues[i].namebuf, 1888 sizeof(adapter->queues[i].namebuf), "q%d", i); 1889 1890 if ((rnode = ixgbe_sysctl_instance(adapter)) == NULL) { 1891 aprint_error_dev(dev, "could not create sysctl root\n"); 1892 break; 1893 } 1894 1895 if (sysctl_createv(log, 0, &rnode, &rnode, 1896 0, CTLTYPE_NODE, 1897 adapter->queues[i].namebuf, SYSCTL_DESCR("Queue Name"), 1898 NULL, 0, NULL, 0, CTL_CREATE, CTL_EOL) != 0) 1899 break; 1900 1901 if (sysctl_createv(log, 0, &rnode, &cnode, 1902 CTLFLAG_READWRITE, CTLTYPE_INT, 1903 "interrupt_rate", SYSCTL_DESCR("Interrupt Rate"), 1904 ixgbe_sysctl_interrupt_rate_handler, 0, 1905 (void *)&adapter->queues[i], 0, CTL_CREATE, CTL_EOL) != 0) 1906 break; 1907 1908 if (sysctl_createv(log, 0, &rnode, &cnode, 1909 CTLFLAG_READONLY, CTLTYPE_INT, 1910 "txd_head", SYSCTL_DESCR("Transmit Descriptor Head"), 1911 ixgbe_sysctl_tdh_handler, 0, (void *)txr, 1912 0, CTL_CREATE, CTL_EOL) != 0) 1913 break; 1914 1915 if (sysctl_createv(log, 0, &rnode, &cnode, 1916 CTLFLAG_READONLY, CTLTYPE_INT, 1917 "txd_tail", SYSCTL_DESCR("Transmit Descriptor Tail"), 1918 ixgbe_sysctl_tdt_handler, 0, (void *)txr, 1919 0, CTL_CREATE, CTL_EOL) != 0) 1920 break; 1921 1922 evcnt_attach_dynamic(&adapter->queues[i].irqs, EVCNT_TYPE_INTR, 1923 NULL, adapter->queues[i].evnamebuf, "IRQs on queue"); 1924 evcnt_attach_dynamic(&adapter->queues[i].handleq, 1925 EVCNT_TYPE_MISC, NULL, adapter->queues[i].evnamebuf, 1926 "Handled queue in softint"); 1927 evcnt_attach_dynamic(&adapter->queues[i].req, EVCNT_TYPE_MISC, 1928 NULL, adapter->queues[i].evnamebuf, "Requeued in softint"); 1929 evcnt_attach_dynamic(&txr->tso_tx, EVCNT_TYPE_MISC, 1930 NULL, adapter->queues[i].evnamebuf, "TSO"); 1931 evcnt_attach_dynamic(&txr->no_desc_avail, EVCNT_TYPE_MISC, 1932 NULL, adapter->queues[i].evnamebuf, 1933 "TX Queue No Descriptor Available"); 1934 evcnt_attach_dynamic(&txr->total_packets, EVCNT_TYPE_MISC, 1935 NULL, adapter->queues[i].evnamebuf, 1936 "Queue Packets Transmitted"); 1937 #ifndef IXGBE_LEGACY_TX 1938 evcnt_attach_dynamic(&txr->pcq_drops, EVCNT_TYPE_MISC, 1939 NULL, adapter->queues[i].evnamebuf, 1940 "Packets dropped in pcq"); 1941 #endif 1942 1943 if (sysctl_createv(log, 0, &rnode, &cnode, 1944 CTLFLAG_READONLY, CTLTYPE_INT, "rxd_nxck", 1945 SYSCTL_DESCR("Receive Descriptor next to check"), 1946 ixgbe_sysctl_next_to_check_handler, 0, (void *)rxr, 0, 1947 CTL_CREATE, CTL_EOL) != 0) 1948 break; 1949 1950 if (sysctl_createv(log, 0, &rnode, &cnode, 1951 CTLFLAG_READONLY, CTLTYPE_INT, "rxd_nxrf", 1952 SYSCTL_DESCR("Receive Descriptor next to refresh"), 1953 ixgbe_sysctl_next_to_refresh_handler, 0, (void *)rxr, 0, 1954 CTL_CREATE, CTL_EOL) != 0) 1955 break; 1956 1957 if (sysctl_createv(log, 0, &rnode, &cnode, 1958 CTLFLAG_READONLY, CTLTYPE_INT, "rxd_head", 1959 SYSCTL_DESCR("Receive Descriptor Head"), 1960 ixgbe_sysctl_rdh_handler, 0, (void *)rxr, 0, 1961 CTL_CREATE, CTL_EOL) != 0) 1962 break; 1963 1964 if (sysctl_createv(log, 0, &rnode, &cnode, 1965 CTLFLAG_READONLY, CTLTYPE_INT, "rxd_tail", 1966 SYSCTL_DESCR("Receive Descriptor Tail"), 1967 ixgbe_sysctl_rdt_handler, 0, (void *)rxr, 0, 1968 CTL_CREATE, CTL_EOL) != 0) 1969 break; 1970 1971 if (i < __arraycount(stats->qprc)) { 1972 evcnt_attach_dynamic(&stats->qprc[i], EVCNT_TYPE_MISC, 1973 NULL, adapter->queues[i].evnamebuf, "qprc"); 1974 evcnt_attach_dynamic(&stats->qptc[i], EVCNT_TYPE_MISC, 1975 NULL, adapter->queues[i].evnamebuf, "qptc"); 1976 evcnt_attach_dynamic(&stats->qbrc[i], EVCNT_TYPE_MISC, 1977 NULL, adapter->queues[i].evnamebuf, "qbrc"); 1978 evcnt_attach_dynamic(&stats->qbtc[i], EVCNT_TYPE_MISC, 1979 NULL, adapter->queues[i].evnamebuf, "qbtc"); 1980 if (hw->mac.type >= ixgbe_mac_82599EB) 1981 evcnt_attach_dynamic(&stats->qprdc[i], 1982 EVCNT_TYPE_MISC, NULL, 1983 adapter->queues[i].evnamebuf, "qprdc"); 1984 } 1985 1986 evcnt_attach_dynamic(&rxr->rx_packets, EVCNT_TYPE_MISC, 1987 NULL, adapter->queues[i].evnamebuf, 1988 "Queue Packets Received"); 1989 evcnt_attach_dynamic(&rxr->rx_bytes, EVCNT_TYPE_MISC, 1990 NULL, adapter->queues[i].evnamebuf, 1991 "Queue Bytes Received"); 1992 evcnt_attach_dynamic(&rxr->rx_copies, EVCNT_TYPE_MISC, 1993 NULL, adapter->queues[i].evnamebuf, "Copied RX Frames"); 1994 evcnt_attach_dynamic(&rxr->no_mbuf, EVCNT_TYPE_MISC, 1995 NULL, adapter->queues[i].evnamebuf, "Rx no mbuf"); 1996 evcnt_attach_dynamic(&rxr->rx_discarded, EVCNT_TYPE_MISC, 1997 NULL, adapter->queues[i].evnamebuf, "Rx discarded"); 1998 #ifdef LRO 1999 SYSCTL_ADD_INT(ctx, queue_list, OID_AUTO, "lro_queued", 2000 CTLFLAG_RD, &lro->lro_queued, 0, 2001 "LRO Queued"); 2002 SYSCTL_ADD_INT(ctx, queue_list, OID_AUTO, "lro_flushed", 2003 CTLFLAG_RD, &lro->lro_flushed, 0, 2004 "LRO Flushed"); 2005 #endif /* LRO */ 2006 } 2007 2008 /* MAC stats get their own sub node */ 2009 2010 snprintf(stats->namebuf, 2011 sizeof(stats->namebuf), "%s MAC Statistics", xname); 2012 2013 evcnt_attach_dynamic(&stats->ipcs, EVCNT_TYPE_MISC, NULL, 2014 stats->namebuf, "rx csum offload - IP"); 2015 evcnt_attach_dynamic(&stats->l4cs, EVCNT_TYPE_MISC, NULL, 2016 stats->namebuf, "rx csum offload - L4"); 2017 evcnt_attach_dynamic(&stats->ipcs_bad, EVCNT_TYPE_MISC, NULL, 2018 stats->namebuf, "rx csum offload - IP bad"); 2019 evcnt_attach_dynamic(&stats->l4cs_bad, EVCNT_TYPE_MISC, NULL, 2020 stats->namebuf, "rx csum offload - L4 bad"); 2021 evcnt_attach_dynamic(&stats->intzero, EVCNT_TYPE_MISC, NULL, 2022 stats->namebuf, "Interrupt conditions zero"); 2023 evcnt_attach_dynamic(&stats->legint, EVCNT_TYPE_MISC, NULL, 2024 stats->namebuf, "Legacy interrupts"); 2025 2026 evcnt_attach_dynamic(&stats->crcerrs, EVCNT_TYPE_MISC, NULL, 2027 stats->namebuf, "CRC Errors"); 2028 evcnt_attach_dynamic(&stats->illerrc, EVCNT_TYPE_MISC, NULL, 2029 stats->namebuf, "Illegal Byte Errors"); 2030 evcnt_attach_dynamic(&stats->errbc, EVCNT_TYPE_MISC, NULL, 2031 stats->namebuf, "Byte Errors"); 2032 evcnt_attach_dynamic(&stats->mspdc, EVCNT_TYPE_MISC, NULL, 2033 stats->namebuf, "MAC Short Packets Discarded"); 2034 if (hw->mac.type >= ixgbe_mac_X550) 2035 evcnt_attach_dynamic(&stats->mbsdc, EVCNT_TYPE_MISC, NULL, 2036 stats->namebuf, "Bad SFD"); 2037 evcnt_attach_dynamic(&stats->mpctotal, EVCNT_TYPE_MISC, NULL, 2038 stats->namebuf, "Total Packets Missed"); 2039 evcnt_attach_dynamic(&stats->mlfc, EVCNT_TYPE_MISC, NULL, 2040 stats->namebuf, "MAC Local Faults"); 2041 evcnt_attach_dynamic(&stats->mrfc, EVCNT_TYPE_MISC, NULL, 2042 stats->namebuf, "MAC Remote Faults"); 2043 evcnt_attach_dynamic(&stats->rlec, EVCNT_TYPE_MISC, NULL, 2044 stats->namebuf, "Receive Length Errors"); 2045 evcnt_attach_dynamic(&stats->lxontxc, EVCNT_TYPE_MISC, NULL, 2046 stats->namebuf, "Link XON Transmitted"); 2047 evcnt_attach_dynamic(&stats->lxonrxc, EVCNT_TYPE_MISC, NULL, 2048 stats->namebuf, "Link XON Received"); 2049 evcnt_attach_dynamic(&stats->lxofftxc, EVCNT_TYPE_MISC, NULL, 2050 stats->namebuf, "Link XOFF Transmitted"); 2051 evcnt_attach_dynamic(&stats->lxoffrxc, EVCNT_TYPE_MISC, NULL, 2052 stats->namebuf, "Link XOFF Received"); 2053 2054 /* Packet Reception Stats */ 2055 evcnt_attach_dynamic(&stats->tor, EVCNT_TYPE_MISC, NULL, 2056 stats->namebuf, "Total Octets Received"); 2057 evcnt_attach_dynamic(&stats->gorc, EVCNT_TYPE_MISC, NULL, 2058 stats->namebuf, "Good Octets Received"); 2059 evcnt_attach_dynamic(&stats->tpr, EVCNT_TYPE_MISC, NULL, 2060 stats->namebuf, "Total Packets Received"); 2061 evcnt_attach_dynamic(&stats->gprc, EVCNT_TYPE_MISC, NULL, 2062 stats->namebuf, "Good Packets Received"); 2063 evcnt_attach_dynamic(&stats->mprc, EVCNT_TYPE_MISC, NULL, 2064 stats->namebuf, "Multicast Packets Received"); 2065 evcnt_attach_dynamic(&stats->bprc, EVCNT_TYPE_MISC, NULL, 2066 stats->namebuf, "Broadcast Packets Received"); 2067 evcnt_attach_dynamic(&stats->prc64, EVCNT_TYPE_MISC, NULL, 2068 stats->namebuf, "64 byte frames received "); 2069 evcnt_attach_dynamic(&stats->prc127, EVCNT_TYPE_MISC, NULL, 2070 stats->namebuf, "65-127 byte frames received"); 2071 evcnt_attach_dynamic(&stats->prc255, EVCNT_TYPE_MISC, NULL, 2072 stats->namebuf, "128-255 byte frames received"); 2073 evcnt_attach_dynamic(&stats->prc511, EVCNT_TYPE_MISC, NULL, 2074 stats->namebuf, "256-511 byte frames received"); 2075 evcnt_attach_dynamic(&stats->prc1023, EVCNT_TYPE_MISC, NULL, 2076 stats->namebuf, "512-1023 byte frames received"); 2077 evcnt_attach_dynamic(&stats->prc1522, EVCNT_TYPE_MISC, NULL, 2078 stats->namebuf, "1023-1522 byte frames received"); 2079 evcnt_attach_dynamic(&stats->ruc, EVCNT_TYPE_MISC, NULL, 2080 stats->namebuf, "Receive Undersized"); 2081 evcnt_attach_dynamic(&stats->rfc, EVCNT_TYPE_MISC, NULL, 2082 stats->namebuf, "Fragmented Packets Received "); 2083 evcnt_attach_dynamic(&stats->roc, EVCNT_TYPE_MISC, NULL, 2084 stats->namebuf, "Oversized Packets Received"); 2085 evcnt_attach_dynamic(&stats->rjc, EVCNT_TYPE_MISC, NULL, 2086 stats->namebuf, "Received Jabber"); 2087 evcnt_attach_dynamic(&stats->mngprc, EVCNT_TYPE_MISC, NULL, 2088 stats->namebuf, "Management Packets Received"); 2089 evcnt_attach_dynamic(&stats->mngpdc, EVCNT_TYPE_MISC, NULL, 2090 stats->namebuf, "Management Packets Dropped"); 2091 evcnt_attach_dynamic(&stats->xec, EVCNT_TYPE_MISC, NULL, 2092 stats->namebuf, "Checksum Errors"); 2093 2094 /* Packet Transmission Stats */ 2095 evcnt_attach_dynamic(&stats->gotc, EVCNT_TYPE_MISC, NULL, 2096 stats->namebuf, "Good Octets Transmitted"); 2097 evcnt_attach_dynamic(&stats->tpt, EVCNT_TYPE_MISC, NULL, 2098 stats->namebuf, "Total Packets Transmitted"); 2099 evcnt_attach_dynamic(&stats->gptc, EVCNT_TYPE_MISC, NULL, 2100 stats->namebuf, "Good Packets Transmitted"); 2101 evcnt_attach_dynamic(&stats->bptc, EVCNT_TYPE_MISC, NULL, 2102 stats->namebuf, "Broadcast Packets Transmitted"); 2103 evcnt_attach_dynamic(&stats->mptc, EVCNT_TYPE_MISC, NULL, 2104 stats->namebuf, "Multicast Packets Transmitted"); 2105 evcnt_attach_dynamic(&stats->mngptc, EVCNT_TYPE_MISC, NULL, 2106 stats->namebuf, "Management Packets Transmitted"); 2107 evcnt_attach_dynamic(&stats->ptc64, EVCNT_TYPE_MISC, NULL, 2108 stats->namebuf, "64 byte frames transmitted "); 2109 evcnt_attach_dynamic(&stats->ptc127, EVCNT_TYPE_MISC, NULL, 2110 stats->namebuf, "65-127 byte frames transmitted"); 2111 evcnt_attach_dynamic(&stats->ptc255, EVCNT_TYPE_MISC, NULL, 2112 stats->namebuf, "128-255 byte frames transmitted"); 2113 evcnt_attach_dynamic(&stats->ptc511, EVCNT_TYPE_MISC, NULL, 2114 stats->namebuf, "256-511 byte frames transmitted"); 2115 evcnt_attach_dynamic(&stats->ptc1023, EVCNT_TYPE_MISC, NULL, 2116 stats->namebuf, "512-1023 byte frames transmitted"); 2117 evcnt_attach_dynamic(&stats->ptc1522, EVCNT_TYPE_MISC, NULL, 2118 stats->namebuf, "1024-1522 byte frames transmitted"); 2119 } /* ixgbe_add_hw_stats */ 2120 2121 static void 2122 ixgbe_clear_evcnt(struct adapter *adapter) 2123 { 2124 struct tx_ring *txr = adapter->tx_rings; 2125 struct rx_ring *rxr = adapter->rx_rings; 2126 struct ixgbe_hw *hw = &adapter->hw; 2127 struct ixgbe_hw_stats *stats = &adapter->stats.pf; 2128 int i; 2129 2130 IXGBE_EVC_STORE(&adapter->efbig_tx_dma_setup, 0); 2131 IXGBE_EVC_STORE(&adapter->mbuf_defrag_failed, 0); 2132 IXGBE_EVC_STORE(&adapter->efbig2_tx_dma_setup, 0); 2133 IXGBE_EVC_STORE(&adapter->einval_tx_dma_setup, 0); 2134 IXGBE_EVC_STORE(&adapter->other_tx_dma_setup, 0); 2135 IXGBE_EVC_STORE(&adapter->eagain_tx_dma_setup, 0); 2136 IXGBE_EVC_STORE(&adapter->enomem_tx_dma_setup, 0); 2137 IXGBE_EVC_STORE(&adapter->tso_err, 0); 2138 IXGBE_EVC_STORE(&adapter->watchdog_events, 0); 2139 IXGBE_EVC_STORE(&adapter->admin_irqev, 0); 2140 IXGBE_EVC_STORE(&adapter->link_workev, 0); 2141 IXGBE_EVC_STORE(&adapter->mod_workev, 0); 2142 IXGBE_EVC_STORE(&adapter->msf_workev, 0); 2143 IXGBE_EVC_STORE(&adapter->phy_workev, 0); 2144 2145 for (i = 0; i < IXGBE_TC_COUNTER_NUM; i++) { 2146 if (i < __arraycount(stats->mpc)) { 2147 IXGBE_EVC_STORE(&stats->mpc[i], 0); 2148 if (hw->mac.type == ixgbe_mac_82598EB) 2149 IXGBE_EVC_STORE(&stats->rnbc[i], 0); 2150 } 2151 if (i < __arraycount(stats->pxontxc)) { 2152 IXGBE_EVC_STORE(&stats->pxontxc[i], 0); 2153 IXGBE_EVC_STORE(&stats->pxonrxc[i], 0); 2154 IXGBE_EVC_STORE(&stats->pxofftxc[i], 0); 2155 IXGBE_EVC_STORE(&stats->pxoffrxc[i], 0); 2156 if (hw->mac.type >= ixgbe_mac_82599EB) 2157 IXGBE_EVC_STORE(&stats->pxon2offc[i], 0); 2158 } 2159 } 2160 2161 txr = adapter->tx_rings; 2162 for (i = 0; i < adapter->num_queues; i++, rxr++, txr++) { 2163 IXGBE_EVC_STORE(&adapter->queues[i].irqs, 0); 2164 IXGBE_EVC_STORE(&adapter->queues[i].handleq, 0); 2165 IXGBE_EVC_STORE(&adapter->queues[i].req, 0); 2166 IXGBE_EVC_STORE(&txr->no_desc_avail, 0); 2167 IXGBE_EVC_STORE(&txr->total_packets, 0); 2168 IXGBE_EVC_STORE(&txr->tso_tx, 0); 2169 #ifndef IXGBE_LEGACY_TX 2170 IXGBE_EVC_STORE(&txr->pcq_drops, 0); 2171 #endif 2172 txr->q_efbig_tx_dma_setup = 0; 2173 txr->q_mbuf_defrag_failed = 0; 2174 txr->q_efbig2_tx_dma_setup = 0; 2175 txr->q_einval_tx_dma_setup = 0; 2176 txr->q_other_tx_dma_setup = 0; 2177 txr->q_eagain_tx_dma_setup = 0; 2178 txr->q_enomem_tx_dma_setup = 0; 2179 txr->q_tso_err = 0; 2180 2181 if (i < __arraycount(stats->qprc)) { 2182 IXGBE_EVC_STORE(&stats->qprc[i], 0); 2183 IXGBE_EVC_STORE(&stats->qptc[i], 0); 2184 IXGBE_EVC_STORE(&stats->qbrc[i], 0); 2185 IXGBE_EVC_STORE(&stats->qbtc[i], 0); 2186 if (hw->mac.type >= ixgbe_mac_82599EB) 2187 IXGBE_EVC_STORE(&stats->qprdc[i], 0); 2188 } 2189 2190 IXGBE_EVC_STORE(&rxr->rx_packets, 0); 2191 IXGBE_EVC_STORE(&rxr->rx_bytes, 0); 2192 IXGBE_EVC_STORE(&rxr->rx_copies, 0); 2193 IXGBE_EVC_STORE(&rxr->no_mbuf, 0); 2194 IXGBE_EVC_STORE(&rxr->rx_discarded, 0); 2195 } 2196 IXGBE_EVC_STORE(&stats->ipcs, 0); 2197 IXGBE_EVC_STORE(&stats->l4cs, 0); 2198 IXGBE_EVC_STORE(&stats->ipcs_bad, 0); 2199 IXGBE_EVC_STORE(&stats->l4cs_bad, 0); 2200 IXGBE_EVC_STORE(&stats->intzero, 0); 2201 IXGBE_EVC_STORE(&stats->legint, 0); 2202 IXGBE_EVC_STORE(&stats->crcerrs, 0); 2203 IXGBE_EVC_STORE(&stats->illerrc, 0); 2204 IXGBE_EVC_STORE(&stats->errbc, 0); 2205 IXGBE_EVC_STORE(&stats->mspdc, 0); 2206 if (hw->mac.type >= ixgbe_mac_X550) 2207 IXGBE_EVC_STORE(&stats->mbsdc, 0); 2208 IXGBE_EVC_STORE(&stats->mpctotal, 0); 2209 IXGBE_EVC_STORE(&stats->mlfc, 0); 2210 IXGBE_EVC_STORE(&stats->mrfc, 0); 2211 IXGBE_EVC_STORE(&stats->rlec, 0); 2212 IXGBE_EVC_STORE(&stats->lxontxc, 0); 2213 IXGBE_EVC_STORE(&stats->lxonrxc, 0); 2214 IXGBE_EVC_STORE(&stats->lxofftxc, 0); 2215 IXGBE_EVC_STORE(&stats->lxoffrxc, 0); 2216 2217 /* Packet Reception Stats */ 2218 IXGBE_EVC_STORE(&stats->tor, 0); 2219 IXGBE_EVC_STORE(&stats->gorc, 0); 2220 IXGBE_EVC_STORE(&stats->tpr, 0); 2221 IXGBE_EVC_STORE(&stats->gprc, 0); 2222 IXGBE_EVC_STORE(&stats->mprc, 0); 2223 IXGBE_EVC_STORE(&stats->bprc, 0); 2224 IXGBE_EVC_STORE(&stats->prc64, 0); 2225 IXGBE_EVC_STORE(&stats->prc127, 0); 2226 IXGBE_EVC_STORE(&stats->prc255, 0); 2227 IXGBE_EVC_STORE(&stats->prc511, 0); 2228 IXGBE_EVC_STORE(&stats->prc1023, 0); 2229 IXGBE_EVC_STORE(&stats->prc1522, 0); 2230 IXGBE_EVC_STORE(&stats->ruc, 0); 2231 IXGBE_EVC_STORE(&stats->rfc, 0); 2232 IXGBE_EVC_STORE(&stats->roc, 0); 2233 IXGBE_EVC_STORE(&stats->rjc, 0); 2234 IXGBE_EVC_STORE(&stats->mngprc, 0); 2235 IXGBE_EVC_STORE(&stats->mngpdc, 0); 2236 IXGBE_EVC_STORE(&stats->xec, 0); 2237 2238 /* Packet Transmission Stats */ 2239 IXGBE_EVC_STORE(&stats->gotc, 0); 2240 IXGBE_EVC_STORE(&stats->tpt, 0); 2241 IXGBE_EVC_STORE(&stats->gptc, 0); 2242 IXGBE_EVC_STORE(&stats->bptc, 0); 2243 IXGBE_EVC_STORE(&stats->mptc, 0); 2244 IXGBE_EVC_STORE(&stats->mngptc, 0); 2245 IXGBE_EVC_STORE(&stats->ptc64, 0); 2246 IXGBE_EVC_STORE(&stats->ptc127, 0); 2247 IXGBE_EVC_STORE(&stats->ptc255, 0); 2248 IXGBE_EVC_STORE(&stats->ptc511, 0); 2249 IXGBE_EVC_STORE(&stats->ptc1023, 0); 2250 IXGBE_EVC_STORE(&stats->ptc1522, 0); 2251 } 2252 2253 /************************************************************************ 2254 * ixgbe_sysctl_tdh_handler - Transmit Descriptor Head handler function 2255 * 2256 * Retrieves the TDH value from the hardware 2257 ************************************************************************/ 2258 static int 2259 ixgbe_sysctl_tdh_handler(SYSCTLFN_ARGS) 2260 { 2261 struct sysctlnode node = *rnode; 2262 struct tx_ring *txr = (struct tx_ring *)node.sysctl_data; 2263 struct adapter *adapter; 2264 uint32_t val; 2265 2266 if (!txr) 2267 return (0); 2268 2269 adapter = txr->adapter; 2270 if (ixgbe_fw_recovery_mode_swflag(adapter)) 2271 return (EPERM); 2272 2273 val = IXGBE_READ_REG(&adapter->hw, IXGBE_TDH(txr->me)); 2274 node.sysctl_data = &val; 2275 return sysctl_lookup(SYSCTLFN_CALL(&node)); 2276 } /* ixgbe_sysctl_tdh_handler */ 2277 2278 /************************************************************************ 2279 * ixgbe_sysctl_tdt_handler - Transmit Descriptor Tail handler function 2280 * 2281 * Retrieves the TDT value from the hardware 2282 ************************************************************************/ 2283 static int 2284 ixgbe_sysctl_tdt_handler(SYSCTLFN_ARGS) 2285 { 2286 struct sysctlnode node = *rnode; 2287 struct tx_ring *txr = (struct tx_ring *)node.sysctl_data; 2288 struct adapter *adapter; 2289 uint32_t val; 2290 2291 if (!txr) 2292 return (0); 2293 2294 adapter = txr->adapter; 2295 if (ixgbe_fw_recovery_mode_swflag(adapter)) 2296 return (EPERM); 2297 2298 val = IXGBE_READ_REG(&adapter->hw, IXGBE_TDT(txr->me)); 2299 node.sysctl_data = &val; 2300 return sysctl_lookup(SYSCTLFN_CALL(&node)); 2301 } /* ixgbe_sysctl_tdt_handler */ 2302 2303 /************************************************************************ 2304 * ixgbe_sysctl_next_to_check_handler - Receive Descriptor next to check 2305 * handler function 2306 * 2307 * Retrieves the next_to_check value 2308 ************************************************************************/ 2309 static int 2310 ixgbe_sysctl_next_to_check_handler(SYSCTLFN_ARGS) 2311 { 2312 struct sysctlnode node = *rnode; 2313 struct rx_ring *rxr = (struct rx_ring *)node.sysctl_data; 2314 struct adapter *adapter; 2315 uint32_t val; 2316 2317 if (!rxr) 2318 return (0); 2319 2320 adapter = rxr->adapter; 2321 if (ixgbe_fw_recovery_mode_swflag(adapter)) 2322 return (EPERM); 2323 2324 val = rxr->next_to_check; 2325 node.sysctl_data = &val; 2326 return sysctl_lookup(SYSCTLFN_CALL(&node)); 2327 } /* ixgbe_sysctl_next_to_check_handler */ 2328 2329 /************************************************************************ 2330 * ixgbe_sysctl_next_to_refresh_handler - Receive Descriptor next to check 2331 * handler function 2332 * 2333 * Retrieves the next_to_refresh value 2334 ************************************************************************/ 2335 static int 2336 ixgbe_sysctl_next_to_refresh_handler(SYSCTLFN_ARGS) 2337 { 2338 struct sysctlnode node = *rnode; 2339 struct rx_ring *rxr = (struct rx_ring *)node.sysctl_data; 2340 struct adapter *adapter; 2341 uint32_t val; 2342 2343 if (!rxr) 2344 return (0); 2345 2346 adapter = rxr->adapter; 2347 if (ixgbe_fw_recovery_mode_swflag(adapter)) 2348 return (EPERM); 2349 2350 val = rxr->next_to_refresh; 2351 node.sysctl_data = &val; 2352 return sysctl_lookup(SYSCTLFN_CALL(&node)); 2353 } /* ixgbe_sysctl_next_to_refresh_handler */ 2354 2355 /************************************************************************ 2356 * ixgbe_sysctl_rdh_handler - Receive Descriptor Head handler function 2357 * 2358 * Retrieves the RDH value from the hardware 2359 ************************************************************************/ 2360 static int 2361 ixgbe_sysctl_rdh_handler(SYSCTLFN_ARGS) 2362 { 2363 struct sysctlnode node = *rnode; 2364 struct rx_ring *rxr = (struct rx_ring *)node.sysctl_data; 2365 struct adapter *adapter; 2366 uint32_t val; 2367 2368 if (!rxr) 2369 return (0); 2370 2371 adapter = rxr->adapter; 2372 if (ixgbe_fw_recovery_mode_swflag(adapter)) 2373 return (EPERM); 2374 2375 val = IXGBE_READ_REG(&adapter->hw, IXGBE_RDH(rxr->me)); 2376 node.sysctl_data = &val; 2377 return sysctl_lookup(SYSCTLFN_CALL(&node)); 2378 } /* ixgbe_sysctl_rdh_handler */ 2379 2380 /************************************************************************ 2381 * ixgbe_sysctl_rdt_handler - Receive Descriptor Tail handler function 2382 * 2383 * Retrieves the RDT value from the hardware 2384 ************************************************************************/ 2385 static int 2386 ixgbe_sysctl_rdt_handler(SYSCTLFN_ARGS) 2387 { 2388 struct sysctlnode node = *rnode; 2389 struct rx_ring *rxr = (struct rx_ring *)node.sysctl_data; 2390 struct adapter *adapter; 2391 uint32_t val; 2392 2393 if (!rxr) 2394 return (0); 2395 2396 adapter = rxr->adapter; 2397 if (ixgbe_fw_recovery_mode_swflag(adapter)) 2398 return (EPERM); 2399 2400 val = IXGBE_READ_REG(&adapter->hw, IXGBE_RDT(rxr->me)); 2401 node.sysctl_data = &val; 2402 return sysctl_lookup(SYSCTLFN_CALL(&node)); 2403 } /* ixgbe_sysctl_rdt_handler */ 2404 2405 static int 2406 ixgbe_vlan_cb(struct ethercom *ec, uint16_t vid, bool set) 2407 { 2408 struct ifnet *ifp = &ec->ec_if; 2409 struct adapter *adapter = ifp->if_softc; 2410 int rv; 2411 2412 if (set) 2413 rv = ixgbe_register_vlan(adapter, vid); 2414 else 2415 rv = ixgbe_unregister_vlan(adapter, vid); 2416 2417 if (rv != 0) 2418 return rv; 2419 2420 /* 2421 * Control VLAN HW tagging when ec_nvlan is changed from 1 to 0 2422 * or 0 to 1. 2423 */ 2424 if ((set && (ec->ec_nvlans == 1)) || (!set && (ec->ec_nvlans == 0))) 2425 ixgbe_setup_vlan_hw_tagging(adapter); 2426 2427 return rv; 2428 } 2429 2430 /************************************************************************ 2431 * ixgbe_register_vlan 2432 * 2433 * Run via vlan config EVENT, it enables us to use the 2434 * HW Filter table since we can get the vlan id. This 2435 * just creates the entry in the soft version of the 2436 * VFTA, init will repopulate the real table. 2437 ************************************************************************/ 2438 static int 2439 ixgbe_register_vlan(struct adapter *adapter, u16 vtag) 2440 { 2441 u16 index, bit; 2442 int error; 2443 2444 if ((vtag == 0) || (vtag > 4095)) /* Invalid */ 2445 return EINVAL; 2446 2447 IXGBE_CORE_LOCK(adapter); 2448 index = (vtag >> 5) & 0x7F; 2449 bit = vtag & 0x1F; 2450 adapter->shadow_vfta[index] |= ((u32)1 << bit); 2451 error = adapter->hw.mac.ops.set_vfta(&adapter->hw, vtag, 0, true, 2452 true); 2453 IXGBE_CORE_UNLOCK(adapter); 2454 if (error != 0) 2455 error = EACCES; 2456 2457 return error; 2458 } /* ixgbe_register_vlan */ 2459 2460 /************************************************************************ 2461 * ixgbe_unregister_vlan 2462 * 2463 * Run via vlan unconfig EVENT, remove our entry in the soft vfta. 2464 ************************************************************************/ 2465 static int 2466 ixgbe_unregister_vlan(struct adapter *adapter, u16 vtag) 2467 { 2468 u16 index, bit; 2469 int error; 2470 2471 if ((vtag == 0) || (vtag > 4095)) /* Invalid */ 2472 return EINVAL; 2473 2474 IXGBE_CORE_LOCK(adapter); 2475 index = (vtag >> 5) & 0x7F; 2476 bit = vtag & 0x1F; 2477 adapter->shadow_vfta[index] &= ~((u32)1 << bit); 2478 error = adapter->hw.mac.ops.set_vfta(&adapter->hw, vtag, 0, false, 2479 true); 2480 IXGBE_CORE_UNLOCK(adapter); 2481 if (error != 0) 2482 error = EACCES; 2483 2484 return error; 2485 } /* ixgbe_unregister_vlan */ 2486 2487 static void 2488 ixgbe_setup_vlan_hw_tagging(struct adapter *adapter) 2489 { 2490 struct ethercom *ec = &adapter->osdep.ec; 2491 struct ixgbe_hw *hw = &adapter->hw; 2492 struct rx_ring *rxr; 2493 u32 ctrl; 2494 int i; 2495 bool hwtagging; 2496 2497 /* Enable HW tagging only if any vlan is attached */ 2498 hwtagging = (ec->ec_capenable & ETHERCAP_VLAN_HWTAGGING) 2499 && VLAN_ATTACHED(ec); 2500 2501 /* Setup the queues for vlans */ 2502 for (i = 0; i < adapter->num_queues; i++) { 2503 rxr = &adapter->rx_rings[i]; 2504 /* 2505 * On 82599 and later, the VLAN enable is per/queue in RXDCTL. 2506 */ 2507 if (hw->mac.type != ixgbe_mac_82598EB) { 2508 ctrl = IXGBE_READ_REG(hw, IXGBE_RXDCTL(rxr->me)); 2509 if (hwtagging) 2510 ctrl |= IXGBE_RXDCTL_VME; 2511 else 2512 ctrl &= ~IXGBE_RXDCTL_VME; 2513 IXGBE_WRITE_REG(hw, IXGBE_RXDCTL(rxr->me), ctrl); 2514 } 2515 rxr->vtag_strip = hwtagging ? TRUE : FALSE; 2516 } 2517 2518 /* VLAN hw tagging for 82598 */ 2519 if (hw->mac.type == ixgbe_mac_82598EB) { 2520 ctrl = IXGBE_READ_REG(hw, IXGBE_VLNCTRL); 2521 if (hwtagging) 2522 ctrl |= IXGBE_VLNCTRL_VME; 2523 else 2524 ctrl &= ~IXGBE_VLNCTRL_VME; 2525 IXGBE_WRITE_REG(hw, IXGBE_VLNCTRL, ctrl); 2526 } 2527 } /* ixgbe_setup_vlan_hw_tagging */ 2528 2529 static void 2530 ixgbe_setup_vlan_hw_support(struct adapter *adapter) 2531 { 2532 struct ethercom *ec = &adapter->osdep.ec; 2533 struct ixgbe_hw *hw = &adapter->hw; 2534 int i; 2535 u32 ctrl; 2536 struct vlanid_list *vlanidp; 2537 2538 /* 2539 * This function is called from both if_init and ifflags_cb() 2540 * on NetBSD. 2541 */ 2542 2543 /* 2544 * Part 1: 2545 * Setup VLAN HW tagging 2546 */ 2547 ixgbe_setup_vlan_hw_tagging(adapter); 2548 2549 /* 2550 * Part 2: 2551 * Setup VLAN HW filter 2552 */ 2553 /* Cleanup shadow_vfta */ 2554 for (i = 0; i < IXGBE_VFTA_SIZE; i++) 2555 adapter->shadow_vfta[i] = 0; 2556 /* Generate shadow_vfta from ec_vids */ 2557 ETHER_LOCK(ec); 2558 SIMPLEQ_FOREACH(vlanidp, &ec->ec_vids, vid_list) { 2559 uint32_t idx; 2560 2561 idx = vlanidp->vid / 32; 2562 KASSERT(idx < IXGBE_VFTA_SIZE); 2563 adapter->shadow_vfta[idx] |= (u32)1 << (vlanidp->vid % 32); 2564 } 2565 ETHER_UNLOCK(ec); 2566 for (i = 0; i < IXGBE_VFTA_SIZE; i++) 2567 IXGBE_WRITE_REG(hw, IXGBE_VFTA(i), adapter->shadow_vfta[i]); 2568 2569 ctrl = IXGBE_READ_REG(hw, IXGBE_VLNCTRL); 2570 /* Enable the Filter Table if enabled */ 2571 if (ec->ec_capenable & ETHERCAP_VLAN_HWFILTER) 2572 ctrl |= IXGBE_VLNCTRL_VFE; 2573 else 2574 ctrl &= ~IXGBE_VLNCTRL_VFE; 2575 IXGBE_WRITE_REG(hw, IXGBE_VLNCTRL, ctrl); 2576 } /* ixgbe_setup_vlan_hw_support */ 2577 2578 /************************************************************************ 2579 * ixgbe_get_slot_info 2580 * 2581 * Get the width and transaction speed of 2582 * the slot this adapter is plugged into. 2583 ************************************************************************/ 2584 static void 2585 ixgbe_get_slot_info(struct adapter *adapter) 2586 { 2587 device_t dev = adapter->dev; 2588 struct ixgbe_hw *hw = &adapter->hw; 2589 u32 offset; 2590 u16 link; 2591 int bus_info_valid = TRUE; 2592 2593 /* Some devices are behind an internal bridge */ 2594 switch (hw->device_id) { 2595 case IXGBE_DEV_ID_82599_SFP_SF_QP: 2596 case IXGBE_DEV_ID_82599_QSFP_SF_QP: 2597 goto get_parent_info; 2598 default: 2599 break; 2600 } 2601 2602 ixgbe_get_bus_info(hw); 2603 2604 /* 2605 * Some devices don't use PCI-E, but there is no need 2606 * to display "Unknown" for bus speed and width. 2607 */ 2608 switch (hw->mac.type) { 2609 case ixgbe_mac_X550EM_x: 2610 case ixgbe_mac_X550EM_a: 2611 return; 2612 default: 2613 goto display; 2614 } 2615 2616 get_parent_info: 2617 /* 2618 * For the Quad port adapter we need to parse back 2619 * up the PCI tree to find the speed of the expansion 2620 * slot into which this adapter is plugged. A bit more work. 2621 */ 2622 dev = device_parent(device_parent(dev)); 2623 #if 0 2624 #ifdef IXGBE_DEBUG 2625 device_printf(dev, "parent pcib = %x,%x,%x\n", pci_get_bus(dev), 2626 pci_get_slot(dev), pci_get_function(dev)); 2627 #endif 2628 dev = device_parent(device_parent(dev)); 2629 #ifdef IXGBE_DEBUG 2630 device_printf(dev, "slot pcib = %x,%x,%x\n", pci_get_bus(dev), 2631 pci_get_slot(dev), pci_get_function(dev)); 2632 #endif 2633 #endif 2634 /* Now get the PCI Express Capabilities offset */ 2635 if (pci_get_capability(adapter->osdep.pc, adapter->osdep.tag, 2636 PCI_CAP_PCIEXPRESS, &offset, NULL)) { 2637 /* 2638 * Hmm...can't get PCI-Express capabilities. 2639 * Falling back to default method. 2640 */ 2641 bus_info_valid = FALSE; 2642 ixgbe_get_bus_info(hw); 2643 goto display; 2644 } 2645 /* ...and read the Link Status Register */ 2646 link = pci_conf_read(adapter->osdep.pc, adapter->osdep.tag, 2647 offset + PCIE_LCSR) >> 16; 2648 ixgbe_set_pci_config_data_generic(hw, link); 2649 2650 display: 2651 device_printf(dev, "PCI Express Bus: Speed %s Width %s\n", 2652 ((hw->bus.speed == ixgbe_bus_speed_8000) ? "8.0GT/s" : 2653 (hw->bus.speed == ixgbe_bus_speed_5000) ? "5.0GT/s" : 2654 (hw->bus.speed == ixgbe_bus_speed_2500) ? "2.5GT/s" : 2655 "Unknown"), 2656 ((hw->bus.width == ixgbe_bus_width_pcie_x8) ? "x8" : 2657 (hw->bus.width == ixgbe_bus_width_pcie_x4) ? "x4" : 2658 (hw->bus.width == ixgbe_bus_width_pcie_x1) ? "x1" : 2659 "Unknown")); 2660 2661 if (bus_info_valid) { 2662 if ((hw->device_id != IXGBE_DEV_ID_82599_SFP_SF_QP) && 2663 ((hw->bus.width <= ixgbe_bus_width_pcie_x4) && 2664 (hw->bus.speed == ixgbe_bus_speed_2500))) { 2665 device_printf(dev, "PCI-Express bandwidth available" 2666 " for this card\n is not sufficient for" 2667 " optimal performance.\n"); 2668 device_printf(dev, "For optimal performance a x8 " 2669 "PCIE, or x4 PCIE Gen2 slot is required.\n"); 2670 } 2671 if ((hw->device_id == IXGBE_DEV_ID_82599_SFP_SF_QP) && 2672 ((hw->bus.width <= ixgbe_bus_width_pcie_x8) && 2673 (hw->bus.speed < ixgbe_bus_speed_8000))) { 2674 device_printf(dev, "PCI-Express bandwidth available" 2675 " for this card\n is not sufficient for" 2676 " optimal performance.\n"); 2677 device_printf(dev, "For optimal performance a x8 " 2678 "PCIE Gen3 slot is required.\n"); 2679 } 2680 } else 2681 device_printf(dev, "Unable to determine slot speed/width. The speed/width reported are that of the internal switch.\n"); 2682 2683 return; 2684 } /* ixgbe_get_slot_info */ 2685 2686 /************************************************************************ 2687 * ixgbe_enable_queue - MSI-X Interrupt Handlers and Tasklets 2688 ************************************************************************/ 2689 static inline void 2690 ixgbe_enable_queue(struct adapter *adapter, u32 vector) 2691 { 2692 struct ixgbe_hw *hw = &adapter->hw; 2693 struct ix_queue *que = &adapter->queues[vector]; 2694 u64 queue = 1ULL << vector; 2695 u32 mask; 2696 2697 mutex_enter(&que->dc_mtx); 2698 if (que->disabled_count > 0 && --que->disabled_count > 0) 2699 goto out; 2700 2701 if (hw->mac.type == ixgbe_mac_82598EB) { 2702 mask = (IXGBE_EIMS_RTX_QUEUE & queue); 2703 IXGBE_WRITE_REG(hw, IXGBE_EIMS, mask); 2704 } else { 2705 mask = (queue & 0xFFFFFFFF); 2706 if (mask) 2707 IXGBE_WRITE_REG(hw, IXGBE_EIMS_EX(0), mask); 2708 mask = (queue >> 32); 2709 if (mask) 2710 IXGBE_WRITE_REG(hw, IXGBE_EIMS_EX(1), mask); 2711 } 2712 out: 2713 mutex_exit(&que->dc_mtx); 2714 } /* ixgbe_enable_queue */ 2715 2716 /************************************************************************ 2717 * ixgbe_disable_queue_internal 2718 ************************************************************************/ 2719 static inline void 2720 ixgbe_disable_queue_internal(struct adapter *adapter, u32 vector, bool nestok) 2721 { 2722 struct ixgbe_hw *hw = &adapter->hw; 2723 struct ix_queue *que = &adapter->queues[vector]; 2724 u64 queue = 1ULL << vector; 2725 u32 mask; 2726 2727 mutex_enter(&que->dc_mtx); 2728 2729 if (que->disabled_count > 0) { 2730 if (nestok) 2731 que->disabled_count++; 2732 goto out; 2733 } 2734 que->disabled_count++; 2735 2736 if (hw->mac.type == ixgbe_mac_82598EB) { 2737 mask = (IXGBE_EIMS_RTX_QUEUE & queue); 2738 IXGBE_WRITE_REG(hw, IXGBE_EIMC, mask); 2739 } else { 2740 mask = (queue & 0xFFFFFFFF); 2741 if (mask) 2742 IXGBE_WRITE_REG(hw, IXGBE_EIMC_EX(0), mask); 2743 mask = (queue >> 32); 2744 if (mask) 2745 IXGBE_WRITE_REG(hw, IXGBE_EIMC_EX(1), mask); 2746 } 2747 out: 2748 mutex_exit(&que->dc_mtx); 2749 } /* ixgbe_disable_queue_internal */ 2750 2751 /************************************************************************ 2752 * ixgbe_disable_queue 2753 ************************************************************************/ 2754 static inline void 2755 ixgbe_disable_queue(struct adapter *adapter, u32 vector) 2756 { 2757 2758 ixgbe_disable_queue_internal(adapter, vector, true); 2759 } /* ixgbe_disable_queue */ 2760 2761 /************************************************************************ 2762 * ixgbe_sched_handle_que - schedule deferred packet processing 2763 ************************************************************************/ 2764 static inline void 2765 ixgbe_sched_handle_que(struct adapter *adapter, struct ix_queue *que) 2766 { 2767 2768 if (que->txrx_use_workqueue) { 2769 /* 2770 * adapter->que_wq is bound to each CPU instead of 2771 * each NIC queue to reduce workqueue kthread. As we 2772 * should consider about interrupt affinity in this 2773 * function, the workqueue kthread must be WQ_PERCPU. 2774 * If create WQ_PERCPU workqueue kthread for each NIC 2775 * queue, that number of created workqueue kthread is 2776 * (number of used NIC queue) * (number of CPUs) = 2777 * (number of CPUs) ^ 2 most often. 2778 * 2779 * The same NIC queue's interrupts are avoided by 2780 * masking the queue's interrupt. And different 2781 * NIC queue's interrupts use different struct work 2782 * (que->wq_cookie). So, "enqueued flag" to avoid 2783 * twice workqueue_enqueue() is not required . 2784 */ 2785 workqueue_enqueue(adapter->que_wq, &que->wq_cookie, curcpu()); 2786 } else { 2787 softint_schedule(que->que_si); 2788 } 2789 } 2790 2791 /************************************************************************ 2792 * ixgbe_msix_que - MSI-X Queue Interrupt Service routine 2793 ************************************************************************/ 2794 static int 2795 ixgbe_msix_que(void *arg) 2796 { 2797 struct ix_queue *que = arg; 2798 struct adapter *adapter = que->adapter; 2799 struct ifnet *ifp = adapter->ifp; 2800 struct tx_ring *txr = que->txr; 2801 struct rx_ring *rxr = que->rxr; 2802 bool more; 2803 u32 newitr = 0; 2804 2805 /* Protect against spurious interrupts */ 2806 if ((ifp->if_flags & IFF_RUNNING) == 0) 2807 return 0; 2808 2809 ixgbe_disable_queue(adapter, que->msix); 2810 IXGBE_EVC_ADD(&que->irqs, 1); 2811 2812 /* 2813 * Don't change "que->txrx_use_workqueue" from this point to avoid 2814 * flip-flopping softint/workqueue mode in one deferred processing. 2815 */ 2816 que->txrx_use_workqueue = adapter->txrx_use_workqueue; 2817 2818 #ifdef __NetBSD__ 2819 /* Don't run ixgbe_rxeof in interrupt context */ 2820 more = true; 2821 #else 2822 more = ixgbe_rxeof(que); 2823 #endif 2824 2825 IXGBE_TX_LOCK(txr); 2826 ixgbe_txeof(txr); 2827 IXGBE_TX_UNLOCK(txr); 2828 2829 /* Do AIM now? */ 2830 2831 if (adapter->enable_aim == false) 2832 goto no_calc; 2833 /* 2834 * Do Adaptive Interrupt Moderation: 2835 * - Write out last calculated setting 2836 * - Calculate based on average size over 2837 * the last interval. 2838 */ 2839 if (que->eitr_setting) 2840 ixgbe_eitr_write(adapter, que->msix, que->eitr_setting); 2841 2842 que->eitr_setting = 0; 2843 2844 /* Idle, do nothing */ 2845 if ((txr->bytes == 0) && (rxr->bytes == 0)) 2846 goto no_calc; 2847 2848 if ((txr->bytes) && (txr->packets)) 2849 newitr = txr->bytes/txr->packets; 2850 if ((rxr->bytes) && (rxr->packets)) 2851 newitr = uimax(newitr, (rxr->bytes / rxr->packets)); 2852 newitr += 24; /* account for hardware frame, crc */ 2853 2854 /* set an upper boundary */ 2855 newitr = uimin(newitr, 3000); 2856 2857 /* Be nice to the mid range */ 2858 if ((newitr > 300) && (newitr < 1200)) 2859 newitr = (newitr / 3); 2860 else 2861 newitr = (newitr / 2); 2862 2863 /* 2864 * When RSC is used, ITR interval must be larger than RSC_DELAY. 2865 * Currently, we use 2us for RSC_DELAY. The minimum value is always 2866 * greater than 2us on 100M (and 10M?(not documented)), but it's not 2867 * on 1G and higher. 2868 */ 2869 if ((adapter->link_speed != IXGBE_LINK_SPEED_100_FULL) 2870 && (adapter->link_speed != IXGBE_LINK_SPEED_10_FULL)) { 2871 if (newitr < IXGBE_MIN_RSC_EITR_10G1G) 2872 newitr = IXGBE_MIN_RSC_EITR_10G1G; 2873 } 2874 2875 /* save for next interrupt */ 2876 que->eitr_setting = newitr; 2877 2878 /* Reset state */ 2879 txr->bytes = 0; 2880 txr->packets = 0; 2881 rxr->bytes = 0; 2882 rxr->packets = 0; 2883 2884 no_calc: 2885 if (more) 2886 ixgbe_sched_handle_que(adapter, que); 2887 else 2888 ixgbe_enable_queue(adapter, que->msix); 2889 2890 return 1; 2891 } /* ixgbe_msix_que */ 2892 2893 /************************************************************************ 2894 * ixgbe_media_status - Media Ioctl callback 2895 * 2896 * Called whenever the user queries the status of 2897 * the interface using ifconfig. 2898 ************************************************************************/ 2899 static void 2900 ixgbe_media_status(struct ifnet *ifp, struct ifmediareq *ifmr) 2901 { 2902 struct adapter *adapter = ifp->if_softc; 2903 struct ixgbe_hw *hw = &adapter->hw; 2904 int layer; 2905 2906 INIT_DEBUGOUT("ixgbe_media_status: begin"); 2907 ixgbe_update_link_status(adapter); 2908 2909 ifmr->ifm_status = IFM_AVALID; 2910 ifmr->ifm_active = IFM_ETHER; 2911 2912 if (adapter->link_active != LINK_STATE_UP) { 2913 ifmr->ifm_active |= IFM_NONE; 2914 return; 2915 } 2916 2917 ifmr->ifm_status |= IFM_ACTIVE; 2918 layer = adapter->phy_layer; 2919 2920 if (layer & IXGBE_PHYSICAL_LAYER_10GBASE_T || 2921 layer & IXGBE_PHYSICAL_LAYER_5GBASE_T || 2922 layer & IXGBE_PHYSICAL_LAYER_2500BASE_T || 2923 layer & IXGBE_PHYSICAL_LAYER_1000BASE_T || 2924 layer & IXGBE_PHYSICAL_LAYER_100BASE_TX || 2925 layer & IXGBE_PHYSICAL_LAYER_10BASE_T) 2926 switch (adapter->link_speed) { 2927 case IXGBE_LINK_SPEED_10GB_FULL: 2928 ifmr->ifm_active |= IFM_10G_T | IFM_FDX; 2929 break; 2930 case IXGBE_LINK_SPEED_5GB_FULL: 2931 ifmr->ifm_active |= IFM_5000_T | IFM_FDX; 2932 break; 2933 case IXGBE_LINK_SPEED_2_5GB_FULL: 2934 ifmr->ifm_active |= IFM_2500_T | IFM_FDX; 2935 break; 2936 case IXGBE_LINK_SPEED_1GB_FULL: 2937 ifmr->ifm_active |= IFM_1000_T | IFM_FDX; 2938 break; 2939 case IXGBE_LINK_SPEED_100_FULL: 2940 ifmr->ifm_active |= IFM_100_TX | IFM_FDX; 2941 break; 2942 case IXGBE_LINK_SPEED_10_FULL: 2943 ifmr->ifm_active |= IFM_10_T | IFM_FDX; 2944 break; 2945 } 2946 if (layer & IXGBE_PHYSICAL_LAYER_SFP_PLUS_CU || 2947 layer & IXGBE_PHYSICAL_LAYER_SFP_ACTIVE_DA) 2948 switch (adapter->link_speed) { 2949 case IXGBE_LINK_SPEED_10GB_FULL: 2950 ifmr->ifm_active |= IFM_10G_TWINAX | IFM_FDX; 2951 break; 2952 } 2953 if (layer & IXGBE_PHYSICAL_LAYER_10GBASE_LR) 2954 switch (adapter->link_speed) { 2955 case IXGBE_LINK_SPEED_10GB_FULL: 2956 ifmr->ifm_active |= IFM_10G_LR | IFM_FDX; 2957 break; 2958 case IXGBE_LINK_SPEED_1GB_FULL: 2959 ifmr->ifm_active |= IFM_1000_LX | IFM_FDX; 2960 break; 2961 } 2962 if (layer & IXGBE_PHYSICAL_LAYER_10GBASE_LRM) 2963 switch (adapter->link_speed) { 2964 case IXGBE_LINK_SPEED_10GB_FULL: 2965 ifmr->ifm_active |= IFM_10G_LRM | IFM_FDX; 2966 break; 2967 case IXGBE_LINK_SPEED_1GB_FULL: 2968 ifmr->ifm_active |= IFM_1000_LX | IFM_FDX; 2969 break; 2970 } 2971 if (layer & IXGBE_PHYSICAL_LAYER_10GBASE_SR || 2972 layer & IXGBE_PHYSICAL_LAYER_1000BASE_SX) 2973 switch (adapter->link_speed) { 2974 case IXGBE_LINK_SPEED_10GB_FULL: 2975 ifmr->ifm_active |= IFM_10G_SR | IFM_FDX; 2976 break; 2977 case IXGBE_LINK_SPEED_1GB_FULL: 2978 ifmr->ifm_active |= IFM_1000_SX | IFM_FDX; 2979 break; 2980 } 2981 if (layer & IXGBE_PHYSICAL_LAYER_10GBASE_CX4) 2982 switch (adapter->link_speed) { 2983 case IXGBE_LINK_SPEED_10GB_FULL: 2984 ifmr->ifm_active |= IFM_10G_CX4 | IFM_FDX; 2985 break; 2986 } 2987 /* 2988 * XXX: These need to use the proper media types once 2989 * they're added. 2990 */ 2991 if (layer & IXGBE_PHYSICAL_LAYER_10GBASE_KR) 2992 switch (adapter->link_speed) { 2993 case IXGBE_LINK_SPEED_10GB_FULL: 2994 ifmr->ifm_active |= IFM_10G_KR | IFM_FDX; 2995 break; 2996 case IXGBE_LINK_SPEED_2_5GB_FULL: 2997 ifmr->ifm_active |= IFM_2500_KX | IFM_FDX; 2998 break; 2999 case IXGBE_LINK_SPEED_1GB_FULL: 3000 ifmr->ifm_active |= IFM_1000_KX | IFM_FDX; 3001 break; 3002 } 3003 else if (layer & IXGBE_PHYSICAL_LAYER_10GBASE_KX4 || 3004 layer & IXGBE_PHYSICAL_LAYER_2500BASE_KX || 3005 layer & IXGBE_PHYSICAL_LAYER_1000BASE_KX) 3006 switch (adapter->link_speed) { 3007 case IXGBE_LINK_SPEED_10GB_FULL: 3008 ifmr->ifm_active |= IFM_10G_KX4 | IFM_FDX; 3009 break; 3010 case IXGBE_LINK_SPEED_2_5GB_FULL: 3011 ifmr->ifm_active |= IFM_2500_KX | IFM_FDX; 3012 break; 3013 case IXGBE_LINK_SPEED_1GB_FULL: 3014 ifmr->ifm_active |= IFM_1000_KX | IFM_FDX; 3015 break; 3016 } 3017 3018 /* If nothing is recognized... */ 3019 #if 0 3020 if (IFM_SUBTYPE(ifmr->ifm_active) == 0) 3021 ifmr->ifm_active |= IFM_UNKNOWN; 3022 #endif 3023 3024 ifp->if_baudrate = ifmedia_baudrate(ifmr->ifm_active); 3025 3026 /* Display current flow control setting used on link */ 3027 if (hw->fc.current_mode == ixgbe_fc_rx_pause || 3028 hw->fc.current_mode == ixgbe_fc_full) 3029 ifmr->ifm_active |= IFM_ETH_RXPAUSE; 3030 if (hw->fc.current_mode == ixgbe_fc_tx_pause || 3031 hw->fc.current_mode == ixgbe_fc_full) 3032 ifmr->ifm_active |= IFM_ETH_TXPAUSE; 3033 3034 return; 3035 } /* ixgbe_media_status */ 3036 3037 /************************************************************************ 3038 * ixgbe_media_change - Media Ioctl callback 3039 * 3040 * Called when the user changes speed/duplex using 3041 * media/mediopt option with ifconfig. 3042 ************************************************************************/ 3043 static int 3044 ixgbe_media_change(struct ifnet *ifp) 3045 { 3046 struct adapter *adapter = ifp->if_softc; 3047 struct ifmedia *ifm = &adapter->media; 3048 struct ixgbe_hw *hw = &adapter->hw; 3049 ixgbe_link_speed speed = 0; 3050 ixgbe_link_speed link_caps = 0; 3051 bool negotiate = false; 3052 s32 err = IXGBE_NOT_IMPLEMENTED; 3053 3054 INIT_DEBUGOUT("ixgbe_media_change: begin"); 3055 3056 if (IFM_TYPE(ifm->ifm_media) != IFM_ETHER) 3057 return (EINVAL); 3058 3059 if (hw->phy.media_type == ixgbe_media_type_backplane) 3060 return (EPERM); 3061 3062 /* 3063 * We don't actually need to check against the supported 3064 * media types of the adapter; ifmedia will take care of 3065 * that for us. 3066 */ 3067 switch (IFM_SUBTYPE(ifm->ifm_media)) { 3068 case IFM_AUTO: 3069 err = hw->mac.ops.get_link_capabilities(hw, &link_caps, 3070 &negotiate); 3071 if (err != IXGBE_SUCCESS) { 3072 device_printf(adapter->dev, "Unable to determine " 3073 "supported advertise speeds\n"); 3074 return (ENODEV); 3075 } 3076 speed |= link_caps; 3077 break; 3078 case IFM_10G_T: 3079 case IFM_10G_LRM: 3080 case IFM_10G_LR: 3081 case IFM_10G_TWINAX: 3082 case IFM_10G_SR: 3083 case IFM_10G_CX4: 3084 case IFM_10G_KR: 3085 case IFM_10G_KX4: 3086 speed |= IXGBE_LINK_SPEED_10GB_FULL; 3087 break; 3088 case IFM_5000_T: 3089 speed |= IXGBE_LINK_SPEED_5GB_FULL; 3090 break; 3091 case IFM_2500_T: 3092 case IFM_2500_KX: 3093 speed |= IXGBE_LINK_SPEED_2_5GB_FULL; 3094 break; 3095 case IFM_1000_T: 3096 case IFM_1000_LX: 3097 case IFM_1000_SX: 3098 case IFM_1000_KX: 3099 speed |= IXGBE_LINK_SPEED_1GB_FULL; 3100 break; 3101 case IFM_100_TX: 3102 speed |= IXGBE_LINK_SPEED_100_FULL; 3103 break; 3104 case IFM_10_T: 3105 speed |= IXGBE_LINK_SPEED_10_FULL; 3106 break; 3107 case IFM_NONE: 3108 break; 3109 default: 3110 goto invalid; 3111 } 3112 3113 hw->mac.autotry_restart = TRUE; 3114 hw->mac.ops.setup_link(hw, speed, TRUE); 3115 adapter->advertise = 0; 3116 if (IFM_SUBTYPE(ifm->ifm_media) != IFM_AUTO) { 3117 if ((speed & IXGBE_LINK_SPEED_10GB_FULL) != 0) 3118 adapter->advertise |= 1 << 2; 3119 if ((speed & IXGBE_LINK_SPEED_1GB_FULL) != 0) 3120 adapter->advertise |= 1 << 1; 3121 if ((speed & IXGBE_LINK_SPEED_100_FULL) != 0) 3122 adapter->advertise |= 1 << 0; 3123 if ((speed & IXGBE_LINK_SPEED_10_FULL) != 0) 3124 adapter->advertise |= 1 << 3; 3125 if ((speed & IXGBE_LINK_SPEED_2_5GB_FULL) != 0) 3126 adapter->advertise |= 1 << 4; 3127 if ((speed & IXGBE_LINK_SPEED_5GB_FULL) != 0) 3128 adapter->advertise |= 1 << 5; 3129 } 3130 3131 return (0); 3132 3133 invalid: 3134 device_printf(adapter->dev, "Invalid media type!\n"); 3135 3136 return (EINVAL); 3137 } /* ixgbe_media_change */ 3138 3139 /************************************************************************ 3140 * ixgbe_msix_admin - Link status change ISR (MSI/MSI-X) 3141 ************************************************************************/ 3142 static int 3143 ixgbe_msix_admin(void *arg) 3144 { 3145 struct adapter *adapter = arg; 3146 struct ixgbe_hw *hw = &adapter->hw; 3147 u32 eicr; 3148 u32 eims_orig; 3149 u32 eims_disable = 0; 3150 3151 IXGBE_EVC_ADD(&adapter->admin_irqev, 1); 3152 3153 eims_orig = IXGBE_READ_REG(hw, IXGBE_EIMS); 3154 /* Pause other interrupts */ 3155 IXGBE_WRITE_REG(hw, IXGBE_EIMC, IXGBE_MSIX_OTHER_CLEAR_MASK); 3156 3157 /* 3158 * First get the cause. 3159 * 3160 * The specifications of 82598, 82599, X540 and X550 say EICS register 3161 * is write only. However, Linux says it is a workaround for silicon 3162 * errata to read EICS instead of EICR to get interrupt cause. 3163 * At least, reading EICR clears lower 16bits of EIMS on 82598. 3164 */ 3165 eicr = IXGBE_READ_REG(hw, IXGBE_EICS); 3166 /* Be sure the queue bits are not cleared */ 3167 eicr &= ~IXGBE_EICR_RTX_QUEUE; 3168 /* Clear all OTHER interrupts with write */ 3169 IXGBE_WRITE_REG(hw, IXGBE_EICR, eicr); 3170 3171 ixgbe_intr_admin_common(adapter, eicr, &eims_disable); 3172 3173 /* Re-enable some OTHER interrupts */ 3174 IXGBE_WRITE_REG(hw, IXGBE_EIMS, eims_orig & ~eims_disable); 3175 3176 return 1; 3177 } /* ixgbe_msix_admin */ 3178 3179 static void 3180 ixgbe_intr_admin_common(struct adapter *adapter, u32 eicr, u32 *eims_disable) 3181 { 3182 struct ixgbe_hw *hw = &adapter->hw; 3183 u32 task_requests = 0; 3184 s32 retval; 3185 3186 /* Link status change */ 3187 if (eicr & IXGBE_EICR_LSC) { 3188 task_requests |= IXGBE_REQUEST_TASK_LSC; 3189 *eims_disable |= IXGBE_EIMS_LSC; 3190 } 3191 3192 if (ixgbe_is_sfp(hw)) { 3193 u32 eicr_mask; 3194 3195 /* Pluggable optics-related interrupt */ 3196 if (hw->mac.type >= ixgbe_mac_X540) 3197 eicr_mask = IXGBE_EICR_GPI_SDP0_X540; 3198 else 3199 eicr_mask = IXGBE_EICR_GPI_SDP2_BY_MAC(hw); 3200 3201 /* 3202 * An interrupt might not arrive when a module is inserted. 3203 * When an link status change interrupt occurred and the driver 3204 * still regard SFP as unplugged, issue the module softint 3205 * and then issue LSC interrupt. 3206 */ 3207 if ((eicr & eicr_mask) 3208 || ((hw->phy.sfp_type == ixgbe_sfp_type_not_present) 3209 && (eicr & IXGBE_EICR_LSC))) { 3210 task_requests |= IXGBE_REQUEST_TASK_MOD; 3211 *eims_disable |= IXGBE_EIMS_LSC; 3212 } 3213 3214 if ((hw->mac.type == ixgbe_mac_82599EB) && 3215 (eicr & IXGBE_EICR_GPI_SDP1_BY_MAC(hw))) { 3216 task_requests |= IXGBE_REQUEST_TASK_MSF; 3217 *eims_disable |= IXGBE_EIMS_GPI_SDP1_BY_MAC(hw); 3218 } 3219 } 3220 3221 if (adapter->hw.mac.type != ixgbe_mac_82598EB) { 3222 #ifdef IXGBE_FDIR 3223 if ((adapter->feat_en & IXGBE_FEATURE_FDIR) && 3224 (eicr & IXGBE_EICR_FLOW_DIR)) { 3225 if (!atomic_cas_uint(&adapter->fdir_reinit, 0, 1)) { 3226 task_requests |= IXGBE_REQUEST_TASK_FDIR; 3227 /* Disable the interrupt */ 3228 *eims_disable |= IXGBE_EIMS_FLOW_DIR; 3229 } 3230 } 3231 #endif 3232 3233 if (eicr & IXGBE_EICR_ECC) { 3234 if (ratecheck(&adapter->lasterr_time, 3235 &ixgbe_errlog_intrvl)) 3236 device_printf(adapter->dev, 3237 "CRITICAL: ECC ERROR!! Please Reboot!!\n"); 3238 } 3239 3240 /* Check for over temp condition */ 3241 if (adapter->feat_en & IXGBE_FEATURE_TEMP_SENSOR) { 3242 switch (adapter->hw.mac.type) { 3243 case ixgbe_mac_X550EM_a: 3244 if (!(eicr & IXGBE_EICR_GPI_SDP0_X550EM_a)) 3245 break; 3246 retval = hw->phy.ops.check_overtemp(hw); 3247 if (retval != IXGBE_ERR_OVERTEMP) 3248 break; 3249 if (ratecheck(&adapter->lasterr_time, 3250 &ixgbe_errlog_intrvl)) { 3251 device_printf(adapter->dev, 3252 "CRITICAL: OVER TEMP!! " 3253 "PHY IS SHUT DOWN!!\n"); 3254 device_printf(adapter->dev, 3255 "System shutdown required!\n"); 3256 } 3257 break; 3258 default: 3259 if (!(eicr & IXGBE_EICR_TS)) 3260 break; 3261 retval = hw->phy.ops.check_overtemp(hw); 3262 if (retval != IXGBE_ERR_OVERTEMP) 3263 break; 3264 if (ratecheck(&adapter->lasterr_time, 3265 &ixgbe_errlog_intrvl)) { 3266 device_printf(adapter->dev, 3267 "CRITICAL: OVER TEMP!! " 3268 "PHY IS SHUT DOWN!!\n"); 3269 device_printf(adapter->dev, 3270 "System shutdown required!\n"); 3271 } 3272 break; 3273 } 3274 } 3275 3276 /* Check for VF message */ 3277 if ((adapter->feat_en & IXGBE_FEATURE_SRIOV) && 3278 (eicr & IXGBE_EICR_MAILBOX)) { 3279 task_requests |= IXGBE_REQUEST_TASK_MBX; 3280 *eims_disable |= IXGBE_EIMS_MAILBOX; 3281 } 3282 } 3283 3284 /* Check for fan failure */ 3285 if (adapter->feat_en & IXGBE_FEATURE_FAN_FAIL) 3286 ixgbe_check_fan_failure(adapter, eicr, true); 3287 3288 /* External PHY interrupt */ 3289 if ((hw->phy.type == ixgbe_phy_x550em_ext_t) && 3290 (eicr & IXGBE_EICR_GPI_SDP0_X540)) { 3291 task_requests |= IXGBE_REQUEST_TASK_PHY; 3292 *eims_disable |= IXGBE_EICR_GPI_SDP0_X540; 3293 } 3294 3295 if (task_requests != 0) { 3296 mutex_enter(&adapter->admin_mtx); 3297 adapter->task_requests |= task_requests; 3298 ixgbe_schedule_admin_tasklet(adapter); 3299 mutex_exit(&adapter->admin_mtx); 3300 } 3301 3302 } 3303 3304 static void 3305 ixgbe_eitr_write(struct adapter *adapter, uint32_t index, uint32_t itr) 3306 { 3307 3308 if (adapter->hw.mac.type == ixgbe_mac_82598EB) 3309 itr |= itr << 16; 3310 else 3311 itr |= IXGBE_EITR_CNT_WDIS; 3312 3313 IXGBE_WRITE_REG(&adapter->hw, IXGBE_EITR(index), itr); 3314 } 3315 3316 3317 /************************************************************************ 3318 * ixgbe_sysctl_interrupt_rate_handler 3319 ************************************************************************/ 3320 static int 3321 ixgbe_sysctl_interrupt_rate_handler(SYSCTLFN_ARGS) 3322 { 3323 struct sysctlnode node = *rnode; 3324 struct ix_queue *que = (struct ix_queue *)node.sysctl_data; 3325 struct adapter *adapter; 3326 uint32_t reg, usec, rate; 3327 int error; 3328 3329 if (que == NULL) 3330 return 0; 3331 3332 adapter = que->adapter; 3333 if (ixgbe_fw_recovery_mode_swflag(adapter)) 3334 return (EPERM); 3335 3336 reg = IXGBE_READ_REG(&adapter->hw, IXGBE_EITR(que->msix)); 3337 usec = ((reg & 0x0FF8) >> 3); 3338 if (usec > 0) 3339 rate = 500000 / usec; 3340 else 3341 rate = 0; 3342 node.sysctl_data = &rate; 3343 error = sysctl_lookup(SYSCTLFN_CALL(&node)); 3344 if (error || newp == NULL) 3345 return error; 3346 reg &= ~0xfff; /* default, no limitation */ 3347 if (rate > 0 && rate < 500000) { 3348 if (rate < 1000) 3349 rate = 1000; 3350 reg |= ((4000000 / rate) & 0xff8); 3351 /* 3352 * When RSC is used, ITR interval must be larger than 3353 * RSC_DELAY. Currently, we use 2us for RSC_DELAY. 3354 * The minimum value is always greater than 2us on 100M 3355 * (and 10M?(not documented)), but it's not on 1G and higher. 3356 */ 3357 if ((adapter->link_speed != IXGBE_LINK_SPEED_100_FULL) 3358 && (adapter->link_speed != IXGBE_LINK_SPEED_10_FULL)) { 3359 if ((adapter->num_queues > 1) 3360 && (reg < IXGBE_MIN_RSC_EITR_10G1G)) 3361 return EINVAL; 3362 } 3363 ixgbe_max_interrupt_rate = rate; 3364 } else 3365 ixgbe_max_interrupt_rate = 0; 3366 ixgbe_eitr_write(adapter, que->msix, reg); 3367 3368 return (0); 3369 } /* ixgbe_sysctl_interrupt_rate_handler */ 3370 3371 const struct sysctlnode * 3372 ixgbe_sysctl_instance(struct adapter *adapter) 3373 { 3374 const char *dvname; 3375 struct sysctllog **log; 3376 int rc; 3377 const struct sysctlnode *rnode; 3378 3379 if (adapter->sysctltop != NULL) 3380 return adapter->sysctltop; 3381 3382 log = &adapter->sysctllog; 3383 dvname = device_xname(adapter->dev); 3384 3385 if ((rc = sysctl_createv(log, 0, NULL, &rnode, 3386 0, CTLTYPE_NODE, dvname, 3387 SYSCTL_DESCR("ixgbe information and settings"), 3388 NULL, 0, NULL, 0, CTL_HW, CTL_CREATE, CTL_EOL)) != 0) 3389 goto err; 3390 3391 return rnode; 3392 err: 3393 device_printf(adapter->dev, 3394 "%s: sysctl_createv failed, rc = %d\n", __func__, rc); 3395 return NULL; 3396 } 3397 3398 /************************************************************************ 3399 * ixgbe_add_device_sysctls 3400 ************************************************************************/ 3401 static void 3402 ixgbe_add_device_sysctls(struct adapter *adapter) 3403 { 3404 device_t dev = adapter->dev; 3405 struct ixgbe_hw *hw = &adapter->hw; 3406 struct sysctllog **log; 3407 const struct sysctlnode *rnode, *cnode; 3408 3409 log = &adapter->sysctllog; 3410 3411 if ((rnode = ixgbe_sysctl_instance(adapter)) == NULL) { 3412 aprint_error_dev(dev, "could not create sysctl root\n"); 3413 return; 3414 } 3415 3416 if (sysctl_createv(log, 0, &rnode, &cnode, 3417 CTLFLAG_READWRITE, CTLTYPE_INT, 3418 "debug", SYSCTL_DESCR("Debug Info"), 3419 ixgbe_sysctl_debug, 0, (void *)adapter, 0, CTL_CREATE, CTL_EOL) 3420 != 0) 3421 aprint_error_dev(dev, "could not create sysctl\n"); 3422 3423 if (sysctl_createv(log, 0, &rnode, &cnode, 3424 CTLFLAG_READWRITE, CTLTYPE_INT, 3425 "rx_copy_len", SYSCTL_DESCR("RX Copy Length"), 3426 ixgbe_sysctl_rx_copy_len, 0, 3427 (void *)adapter, 0, CTL_CREATE, CTL_EOL) != 0) 3428 aprint_error_dev(dev, "could not create sysctl\n"); 3429 3430 if (sysctl_createv(log, 0, &rnode, &cnode, 3431 CTLFLAG_READONLY, CTLTYPE_INT, 3432 "num_tx_desc", SYSCTL_DESCR("Number of TX descriptors"), 3433 NULL, 0, &adapter->num_tx_desc, 0, CTL_CREATE, CTL_EOL) != 0) 3434 aprint_error_dev(dev, "could not create sysctl\n"); 3435 3436 if (sysctl_createv(log, 0, &rnode, &cnode, 3437 CTLFLAG_READONLY, CTLTYPE_INT, 3438 "num_rx_desc", SYSCTL_DESCR("Number of RX descriptors"), 3439 NULL, 0, &adapter->num_rx_desc, 0, CTL_CREATE, CTL_EOL) != 0) 3440 aprint_error_dev(dev, "could not create sysctl\n"); 3441 3442 if (sysctl_createv(log, 0, &rnode, &cnode, 3443 CTLFLAG_READWRITE, CTLTYPE_INT, "rx_process_limit", 3444 SYSCTL_DESCR("max number of RX packets to process"), 3445 ixgbe_sysctl_rx_process_limit, 0, (void *)adapter, 0, CTL_CREATE, 3446 CTL_EOL) != 0) 3447 aprint_error_dev(dev, "could not create sysctl\n"); 3448 3449 if (sysctl_createv(log, 0, &rnode, &cnode, 3450 CTLFLAG_READWRITE, CTLTYPE_INT, "tx_process_limit", 3451 SYSCTL_DESCR("max number of TX packets to process"), 3452 ixgbe_sysctl_tx_process_limit, 0, (void *)adapter, 0, CTL_CREATE, 3453 CTL_EOL) != 0) 3454 aprint_error_dev(dev, "could not create sysctl\n"); 3455 3456 if (sysctl_createv(log, 0, &rnode, &cnode, 3457 CTLFLAG_READONLY, CTLTYPE_INT, 3458 "num_queues", SYSCTL_DESCR("Number of queues"), 3459 NULL, 0, &adapter->num_queues, 0, CTL_CREATE, CTL_EOL) != 0) 3460 aprint_error_dev(dev, "could not create sysctl\n"); 3461 3462 /* Sysctls for all devices */ 3463 if (sysctl_createv(log, 0, &rnode, &cnode, CTLFLAG_READWRITE, 3464 CTLTYPE_INT, "fc", SYSCTL_DESCR(IXGBE_SYSCTL_DESC_SET_FC), 3465 ixgbe_sysctl_flowcntl, 0, (void *)adapter, 0, CTL_CREATE, 3466 CTL_EOL) != 0) 3467 aprint_error_dev(dev, "could not create sysctl\n"); 3468 3469 adapter->enable_aim = ixgbe_enable_aim; 3470 if (sysctl_createv(log, 0, &rnode, &cnode, CTLFLAG_READWRITE, 3471 CTLTYPE_BOOL, "enable_aim", SYSCTL_DESCR("Interrupt Moderation"), 3472 NULL, 0, &adapter->enable_aim, 0, CTL_CREATE, CTL_EOL) != 0) 3473 aprint_error_dev(dev, "could not create sysctl\n"); 3474 3475 if (sysctl_createv(log, 0, &rnode, &cnode, 3476 CTLFLAG_READWRITE, CTLTYPE_INT, 3477 "advertise_speed", SYSCTL_DESCR(IXGBE_SYSCTL_DESC_ADV_SPEED), 3478 ixgbe_sysctl_advertise, 0, (void *)adapter, 0, CTL_CREATE, 3479 CTL_EOL) != 0) 3480 aprint_error_dev(dev, "could not create sysctl\n"); 3481 3482 /* 3483 * If each "que->txrx_use_workqueue" is changed in sysctl handler, 3484 * it causesflip-flopping softint/workqueue mode in one deferred 3485 * processing. Therefore, preempt_disable()/preempt_enable() are 3486 * required in ixgbe_sched_handle_que() to avoid 3487 * KASSERT(ixgbe_sched_handle_que()) in softint_schedule(). 3488 * I think changing "que->txrx_use_workqueue" in interrupt handler 3489 * is lighter than doing preempt_disable()/preempt_enable() in every 3490 * ixgbe_sched_handle_que(). 3491 */ 3492 adapter->txrx_use_workqueue = ixgbe_txrx_workqueue; 3493 if (sysctl_createv(log, 0, &rnode, &cnode, CTLFLAG_READWRITE, 3494 CTLTYPE_BOOL, "txrx_workqueue", 3495 SYSCTL_DESCR("Use workqueue for packet processing"), 3496 NULL, 0, &adapter->txrx_use_workqueue, 0, CTL_CREATE, 3497 CTL_EOL) != 0) 3498 aprint_error_dev(dev, "could not create sysctl\n"); 3499 3500 #ifdef IXGBE_DEBUG 3501 /* testing sysctls (for all devices) */ 3502 if (sysctl_createv(log, 0, &rnode, &cnode, CTLFLAG_READWRITE, 3503 CTLTYPE_INT, "power_state", SYSCTL_DESCR("PCI Power State"), 3504 ixgbe_sysctl_power_state, 0, (void *)adapter, 0, CTL_CREATE, 3505 CTL_EOL) != 0) 3506 aprint_error_dev(dev, "could not create sysctl\n"); 3507 3508 if (sysctl_createv(log, 0, &rnode, &cnode, CTLFLAG_READONLY, 3509 CTLTYPE_STRING, "print_rss_config", 3510 SYSCTL_DESCR("Prints RSS Configuration"), 3511 ixgbe_sysctl_print_rss_config, 0, (void *)adapter, 0, CTL_CREATE, 3512 CTL_EOL) != 0) 3513 aprint_error_dev(dev, "could not create sysctl\n"); 3514 #endif 3515 /* for X550 series devices */ 3516 if (hw->mac.type >= ixgbe_mac_X550) 3517 if (sysctl_createv(log, 0, &rnode, &cnode, CTLFLAG_READWRITE, 3518 CTLTYPE_INT, "dmac", SYSCTL_DESCR("DMA Coalesce"), 3519 ixgbe_sysctl_dmac, 0, (void *)adapter, 0, CTL_CREATE, 3520 CTL_EOL) != 0) 3521 aprint_error_dev(dev, "could not create sysctl\n"); 3522 3523 /* for WoL-capable devices */ 3524 if (adapter->wol_support) { 3525 if (sysctl_createv(log, 0, &rnode, &cnode, CTLFLAG_READWRITE, 3526 CTLTYPE_BOOL, "wol_enable", 3527 SYSCTL_DESCR("Enable/Disable Wake on LAN"), 3528 ixgbe_sysctl_wol_enable, 0, (void *)adapter, 0, CTL_CREATE, 3529 CTL_EOL) != 0) 3530 aprint_error_dev(dev, "could not create sysctl\n"); 3531 3532 if (sysctl_createv(log, 0, &rnode, &cnode, CTLFLAG_READWRITE, 3533 CTLTYPE_INT, "wufc", 3534 SYSCTL_DESCR("Enable/Disable Wake Up Filters"), 3535 ixgbe_sysctl_wufc, 0, (void *)adapter, 0, CTL_CREATE, 3536 CTL_EOL) != 0) 3537 aprint_error_dev(dev, "could not create sysctl\n"); 3538 } 3539 3540 /* for X552/X557-AT devices */ 3541 if (hw->device_id == IXGBE_DEV_ID_X550EM_X_10G_T) { 3542 const struct sysctlnode *phy_node; 3543 3544 if (sysctl_createv(log, 0, &rnode, &phy_node, 0, CTLTYPE_NODE, 3545 "phy", SYSCTL_DESCR("External PHY sysctls"), 3546 NULL, 0, NULL, 0, CTL_CREATE, CTL_EOL) != 0) { 3547 aprint_error_dev(dev, "could not create sysctl\n"); 3548 return; 3549 } 3550 3551 if (sysctl_createv(log, 0, &phy_node, &cnode, CTLFLAG_READONLY, 3552 CTLTYPE_INT, "temp", 3553 SYSCTL_DESCR("Current External PHY Temperature (Celsius)"), 3554 ixgbe_sysctl_phy_temp, 0, (void *)adapter, 0, CTL_CREATE, 3555 CTL_EOL) != 0) 3556 aprint_error_dev(dev, "could not create sysctl\n"); 3557 3558 if (sysctl_createv(log, 0, &phy_node, &cnode, CTLFLAG_READONLY, 3559 CTLTYPE_INT, "overtemp_occurred", 3560 SYSCTL_DESCR( 3561 "External PHY High Temperature Event Occurred"), 3562 ixgbe_sysctl_phy_overtemp_occurred, 0, (void *)adapter, 0, 3563 CTL_CREATE, CTL_EOL) != 0) 3564 aprint_error_dev(dev, "could not create sysctl\n"); 3565 } 3566 3567 if ((hw->mac.type == ixgbe_mac_X550EM_a) 3568 && (hw->phy.type == ixgbe_phy_fw)) 3569 if (sysctl_createv(log, 0, &rnode, &cnode, CTLFLAG_READWRITE, 3570 CTLTYPE_BOOL, "force_10_100_autonego", 3571 SYSCTL_DESCR("Force autonego on 10M and 100M"), 3572 NULL, 0, &hw->phy.force_10_100_autonego, 0, 3573 CTL_CREATE, CTL_EOL) != 0) 3574 aprint_error_dev(dev, "could not create sysctl\n"); 3575 3576 if (adapter->feat_cap & IXGBE_FEATURE_EEE) { 3577 if (sysctl_createv(log, 0, &rnode, &cnode, CTLFLAG_READWRITE, 3578 CTLTYPE_INT, "eee_state", 3579 SYSCTL_DESCR("EEE Power Save State"), 3580 ixgbe_sysctl_eee_state, 0, (void *)adapter, 0, CTL_CREATE, 3581 CTL_EOL) != 0) 3582 aprint_error_dev(dev, "could not create sysctl\n"); 3583 } 3584 } /* ixgbe_add_device_sysctls */ 3585 3586 /************************************************************************ 3587 * ixgbe_allocate_pci_resources 3588 ************************************************************************/ 3589 static int 3590 ixgbe_allocate_pci_resources(struct adapter *adapter, 3591 const struct pci_attach_args *pa) 3592 { 3593 pcireg_t memtype, csr; 3594 device_t dev = adapter->dev; 3595 bus_addr_t addr; 3596 int flags; 3597 3598 memtype = pci_mapreg_type(pa->pa_pc, pa->pa_tag, PCI_BAR(0)); 3599 switch (memtype) { 3600 case PCI_MAPREG_TYPE_MEM | PCI_MAPREG_MEM_TYPE_32BIT: 3601 case PCI_MAPREG_TYPE_MEM | PCI_MAPREG_MEM_TYPE_64BIT: 3602 adapter->osdep.mem_bus_space_tag = pa->pa_memt; 3603 if (pci_mapreg_info(pa->pa_pc, pa->pa_tag, PCI_BAR(0), 3604 memtype, &addr, &adapter->osdep.mem_size, &flags) != 0) 3605 goto map_err; 3606 if ((flags & BUS_SPACE_MAP_PREFETCHABLE) != 0) { 3607 aprint_normal_dev(dev, "clearing prefetchable bit\n"); 3608 flags &= ~BUS_SPACE_MAP_PREFETCHABLE; 3609 } 3610 if (bus_space_map(adapter->osdep.mem_bus_space_tag, addr, 3611 adapter->osdep.mem_size, flags, 3612 &adapter->osdep.mem_bus_space_handle) != 0) { 3613 map_err: 3614 adapter->osdep.mem_size = 0; 3615 aprint_error_dev(dev, "unable to map BAR0\n"); 3616 return ENXIO; 3617 } 3618 /* 3619 * Enable address decoding for memory range in case BIOS or 3620 * UEFI don't set it. 3621 */ 3622 csr = pci_conf_read(pa->pa_pc, pa->pa_tag, 3623 PCI_COMMAND_STATUS_REG); 3624 csr |= PCI_COMMAND_MEM_ENABLE; 3625 pci_conf_write(pa->pa_pc, pa->pa_tag, PCI_COMMAND_STATUS_REG, 3626 csr); 3627 break; 3628 default: 3629 aprint_error_dev(dev, "unexpected type on BAR0\n"); 3630 return ENXIO; 3631 } 3632 3633 return (0); 3634 } /* ixgbe_allocate_pci_resources */ 3635 3636 static void 3637 ixgbe_free_deferred_handlers(struct adapter *adapter) 3638 { 3639 struct ix_queue *que = adapter->queues; 3640 struct tx_ring *txr = adapter->tx_rings; 3641 int i; 3642 3643 for (i = 0; i < adapter->num_queues; i++, que++, txr++) { 3644 if (!(adapter->feat_en & IXGBE_FEATURE_LEGACY_TX)) { 3645 if (txr->txr_si != NULL) 3646 softint_disestablish(txr->txr_si); 3647 } 3648 if (que->que_si != NULL) 3649 softint_disestablish(que->que_si); 3650 } 3651 if (adapter->txr_wq != NULL) 3652 workqueue_destroy(adapter->txr_wq); 3653 if (adapter->txr_wq_enqueued != NULL) 3654 percpu_free(adapter->txr_wq_enqueued, sizeof(u_int)); 3655 if (adapter->que_wq != NULL) 3656 workqueue_destroy(adapter->que_wq); 3657 3658 if (adapter->admin_wq != NULL) { 3659 workqueue_destroy(adapter->admin_wq); 3660 adapter->admin_wq = NULL; 3661 } 3662 if (adapter->timer_wq != NULL) { 3663 workqueue_destroy(adapter->timer_wq); 3664 adapter->timer_wq = NULL; 3665 } 3666 if (adapter->recovery_mode_timer_wq != NULL) { 3667 /* 3668 * ixgbe_ifstop() doesn't call the workqueue_wait() for 3669 * the recovery_mode_timer workqueue, so call it here. 3670 */ 3671 workqueue_wait(adapter->recovery_mode_timer_wq, 3672 &adapter->recovery_mode_timer_wc); 3673 atomic_store_relaxed(&adapter->recovery_mode_timer_pending, 0); 3674 workqueue_destroy(adapter->recovery_mode_timer_wq); 3675 adapter->recovery_mode_timer_wq = NULL; 3676 } 3677 } /* ixgbe_free_deferred_handlers */ 3678 3679 /************************************************************************ 3680 * ixgbe_detach - Device removal routine 3681 * 3682 * Called when the driver is being removed. 3683 * Stops the adapter and deallocates all the resources 3684 * that were allocated for driver operation. 3685 * 3686 * return 0 on success, positive on failure 3687 ************************************************************************/ 3688 static int 3689 ixgbe_detach(device_t dev, int flags) 3690 { 3691 struct adapter *adapter = device_private(dev); 3692 struct rx_ring *rxr = adapter->rx_rings; 3693 struct tx_ring *txr = adapter->tx_rings; 3694 struct ixgbe_hw *hw = &adapter->hw; 3695 struct ixgbe_hw_stats *stats = &adapter->stats.pf; 3696 u32 ctrl_ext; 3697 int i; 3698 3699 INIT_DEBUGOUT("ixgbe_detach: begin"); 3700 if (adapter->osdep.attached == false) 3701 return 0; 3702 3703 if (ixgbe_pci_iov_detach(dev) != 0) { 3704 device_printf(dev, "SR-IOV in use; detach first.\n"); 3705 return (EBUSY); 3706 } 3707 3708 if (VLAN_ATTACHED(&adapter->osdep.ec) && 3709 (flags & (DETACH_SHUTDOWN | DETACH_FORCE)) == 0) { 3710 aprint_error_dev(dev, "VLANs in use, detach first\n"); 3711 return (EBUSY); 3712 } 3713 3714 ether_ifdetach(adapter->ifp); 3715 3716 adapter->osdep.detaching = true; 3717 /* 3718 * Stop the interface. ixgbe_setup_low_power_mode() calls 3719 * ixgbe_ifstop(), so it's not required to call ixgbe_ifstop() 3720 * directly. 3721 */ 3722 ixgbe_setup_low_power_mode(adapter); 3723 3724 callout_halt(&adapter->timer, NULL); 3725 if (adapter->feat_en & IXGBE_FEATURE_RECOVERY_MODE) 3726 callout_halt(&adapter->recovery_mode_timer, NULL); 3727 3728 workqueue_wait(adapter->admin_wq, &adapter->admin_wc); 3729 atomic_store_relaxed(&adapter->admin_pending, 0); 3730 workqueue_wait(adapter->timer_wq, &adapter->timer_wc); 3731 atomic_store_relaxed(&adapter->timer_pending, 0); 3732 3733 pmf_device_deregister(dev); 3734 3735 ixgbe_free_deferred_handlers(adapter); 3736 3737 /* let hardware know driver is unloading */ 3738 ctrl_ext = IXGBE_READ_REG(&adapter->hw, IXGBE_CTRL_EXT); 3739 ctrl_ext &= ~IXGBE_CTRL_EXT_DRV_LOAD; 3740 IXGBE_WRITE_REG(&adapter->hw, IXGBE_CTRL_EXT, ctrl_ext); 3741 3742 if (adapter->feat_en & IXGBE_FEATURE_NETMAP) 3743 netmap_detach(adapter->ifp); 3744 3745 ixgbe_free_pci_resources(adapter); 3746 #if 0 /* XXX the NetBSD port is probably missing something here */ 3747 bus_generic_detach(dev); 3748 #endif 3749 if_detach(adapter->ifp); 3750 ifmedia_fini(&adapter->media); 3751 if_percpuq_destroy(adapter->ipq); 3752 3753 sysctl_teardown(&adapter->sysctllog); 3754 evcnt_detach(&adapter->efbig_tx_dma_setup); 3755 evcnt_detach(&adapter->mbuf_defrag_failed); 3756 evcnt_detach(&adapter->efbig2_tx_dma_setup); 3757 evcnt_detach(&adapter->einval_tx_dma_setup); 3758 evcnt_detach(&adapter->other_tx_dma_setup); 3759 evcnt_detach(&adapter->eagain_tx_dma_setup); 3760 evcnt_detach(&adapter->enomem_tx_dma_setup); 3761 evcnt_detach(&adapter->watchdog_events); 3762 evcnt_detach(&adapter->tso_err); 3763 evcnt_detach(&adapter->admin_irqev); 3764 evcnt_detach(&adapter->link_workev); 3765 evcnt_detach(&adapter->mod_workev); 3766 evcnt_detach(&adapter->msf_workev); 3767 evcnt_detach(&adapter->phy_workev); 3768 3769 for (i = 0; i < IXGBE_TC_COUNTER_NUM; i++) { 3770 if (i < __arraycount(stats->mpc)) { 3771 evcnt_detach(&stats->mpc[i]); 3772 if (hw->mac.type == ixgbe_mac_82598EB) 3773 evcnt_detach(&stats->rnbc[i]); 3774 } 3775 if (i < __arraycount(stats->pxontxc)) { 3776 evcnt_detach(&stats->pxontxc[i]); 3777 evcnt_detach(&stats->pxonrxc[i]); 3778 evcnt_detach(&stats->pxofftxc[i]); 3779 evcnt_detach(&stats->pxoffrxc[i]); 3780 if (hw->mac.type >= ixgbe_mac_82599EB) 3781 evcnt_detach(&stats->pxon2offc[i]); 3782 } 3783 } 3784 3785 txr = adapter->tx_rings; 3786 for (i = 0; i < adapter->num_queues; i++, rxr++, txr++) { 3787 evcnt_detach(&adapter->queues[i].irqs); 3788 evcnt_detach(&adapter->queues[i].handleq); 3789 evcnt_detach(&adapter->queues[i].req); 3790 evcnt_detach(&txr->no_desc_avail); 3791 evcnt_detach(&txr->total_packets); 3792 evcnt_detach(&txr->tso_tx); 3793 #ifndef IXGBE_LEGACY_TX 3794 evcnt_detach(&txr->pcq_drops); 3795 #endif 3796 3797 if (i < __arraycount(stats->qprc)) { 3798 evcnt_detach(&stats->qprc[i]); 3799 evcnt_detach(&stats->qptc[i]); 3800 evcnt_detach(&stats->qbrc[i]); 3801 evcnt_detach(&stats->qbtc[i]); 3802 if (hw->mac.type >= ixgbe_mac_82599EB) 3803 evcnt_detach(&stats->qprdc[i]); 3804 } 3805 3806 evcnt_detach(&rxr->rx_packets); 3807 evcnt_detach(&rxr->rx_bytes); 3808 evcnt_detach(&rxr->rx_copies); 3809 evcnt_detach(&rxr->no_mbuf); 3810 evcnt_detach(&rxr->rx_discarded); 3811 } 3812 evcnt_detach(&stats->ipcs); 3813 evcnt_detach(&stats->l4cs); 3814 evcnt_detach(&stats->ipcs_bad); 3815 evcnt_detach(&stats->l4cs_bad); 3816 evcnt_detach(&stats->intzero); 3817 evcnt_detach(&stats->legint); 3818 evcnt_detach(&stats->crcerrs); 3819 evcnt_detach(&stats->illerrc); 3820 evcnt_detach(&stats->errbc); 3821 evcnt_detach(&stats->mspdc); 3822 if (hw->mac.type >= ixgbe_mac_X550) 3823 evcnt_detach(&stats->mbsdc); 3824 evcnt_detach(&stats->mpctotal); 3825 evcnt_detach(&stats->mlfc); 3826 evcnt_detach(&stats->mrfc); 3827 evcnt_detach(&stats->rlec); 3828 evcnt_detach(&stats->lxontxc); 3829 evcnt_detach(&stats->lxonrxc); 3830 evcnt_detach(&stats->lxofftxc); 3831 evcnt_detach(&stats->lxoffrxc); 3832 3833 /* Packet Reception Stats */ 3834 evcnt_detach(&stats->tor); 3835 evcnt_detach(&stats->gorc); 3836 evcnt_detach(&stats->tpr); 3837 evcnt_detach(&stats->gprc); 3838 evcnt_detach(&stats->mprc); 3839 evcnt_detach(&stats->bprc); 3840 evcnt_detach(&stats->prc64); 3841 evcnt_detach(&stats->prc127); 3842 evcnt_detach(&stats->prc255); 3843 evcnt_detach(&stats->prc511); 3844 evcnt_detach(&stats->prc1023); 3845 evcnt_detach(&stats->prc1522); 3846 evcnt_detach(&stats->ruc); 3847 evcnt_detach(&stats->rfc); 3848 evcnt_detach(&stats->roc); 3849 evcnt_detach(&stats->rjc); 3850 evcnt_detach(&stats->mngprc); 3851 evcnt_detach(&stats->mngpdc); 3852 evcnt_detach(&stats->xec); 3853 3854 /* Packet Transmission Stats */ 3855 evcnt_detach(&stats->gotc); 3856 evcnt_detach(&stats->tpt); 3857 evcnt_detach(&stats->gptc); 3858 evcnt_detach(&stats->bptc); 3859 evcnt_detach(&stats->mptc); 3860 evcnt_detach(&stats->mngptc); 3861 evcnt_detach(&stats->ptc64); 3862 evcnt_detach(&stats->ptc127); 3863 evcnt_detach(&stats->ptc255); 3864 evcnt_detach(&stats->ptc511); 3865 evcnt_detach(&stats->ptc1023); 3866 evcnt_detach(&stats->ptc1522); 3867 3868 ixgbe_free_queues(adapter); 3869 free(adapter->mta, M_DEVBUF); 3870 3871 mutex_destroy(&adapter->admin_mtx); /* XXX appropriate order? */ 3872 IXGBE_CORE_LOCK_DESTROY(adapter); 3873 3874 return (0); 3875 } /* ixgbe_detach */ 3876 3877 /************************************************************************ 3878 * ixgbe_setup_low_power_mode - LPLU/WoL preparation 3879 * 3880 * Prepare the adapter/port for LPLU and/or WoL 3881 ************************************************************************/ 3882 static int 3883 ixgbe_setup_low_power_mode(struct adapter *adapter) 3884 { 3885 struct ixgbe_hw *hw = &adapter->hw; 3886 device_t dev = adapter->dev; 3887 struct ifnet *ifp = adapter->ifp; 3888 s32 error = 0; 3889 3890 /* Limit power management flow to X550EM baseT */ 3891 if (hw->device_id == IXGBE_DEV_ID_X550EM_X_10G_T && 3892 hw->phy.ops.enter_lplu) { 3893 /* X550EM baseT adapters need a special LPLU flow */ 3894 hw->phy.reset_disable = true; 3895 ixgbe_ifstop(ifp, 1); 3896 error = hw->phy.ops.enter_lplu(hw); 3897 if (error) 3898 device_printf(dev, 3899 "Error entering LPLU: %d\n", error); 3900 hw->phy.reset_disable = false; 3901 } else { 3902 /* Just stop for other adapters */ 3903 ixgbe_ifstop(ifp, 1); 3904 } 3905 3906 IXGBE_CORE_LOCK(adapter); 3907 3908 if (!hw->wol_enabled) { 3909 ixgbe_set_phy_power(hw, FALSE); 3910 IXGBE_WRITE_REG(hw, IXGBE_WUFC, 0); 3911 IXGBE_WRITE_REG(hw, IXGBE_WUC, 0); 3912 } else { 3913 /* Turn off support for APM wakeup. (Using ACPI instead) */ 3914 IXGBE_WRITE_REG(hw, IXGBE_GRC_BY_MAC(hw), 3915 IXGBE_READ_REG(hw, IXGBE_GRC_BY_MAC(hw)) & ~(u32)2); 3916 3917 /* 3918 * Clear Wake Up Status register to prevent any previous wakeup 3919 * events from waking us up immediately after we suspend. 3920 */ 3921 IXGBE_WRITE_REG(hw, IXGBE_WUS, 0xffffffff); 3922 3923 /* 3924 * Program the Wakeup Filter Control register with user filter 3925 * settings 3926 */ 3927 IXGBE_WRITE_REG(hw, IXGBE_WUFC, adapter->wufc); 3928 3929 /* Enable wakeups and power management in Wakeup Control */ 3930 IXGBE_WRITE_REG(hw, IXGBE_WUC, 3931 IXGBE_WUC_WKEN | IXGBE_WUC_PME_EN); 3932 3933 } 3934 3935 IXGBE_CORE_UNLOCK(adapter); 3936 3937 return error; 3938 } /* ixgbe_setup_low_power_mode */ 3939 3940 /************************************************************************ 3941 * ixgbe_shutdown - Shutdown entry point 3942 ************************************************************************/ 3943 #if 0 /* XXX NetBSD ought to register something like this through pmf(9) */ 3944 static int 3945 ixgbe_shutdown(device_t dev) 3946 { 3947 struct adapter *adapter = device_private(dev); 3948 int error = 0; 3949 3950 INIT_DEBUGOUT("ixgbe_shutdown: begin"); 3951 3952 error = ixgbe_setup_low_power_mode(adapter); 3953 3954 return (error); 3955 } /* ixgbe_shutdown */ 3956 #endif 3957 3958 /************************************************************************ 3959 * ixgbe_suspend 3960 * 3961 * From D0 to D3 3962 ************************************************************************/ 3963 static bool 3964 ixgbe_suspend(device_t dev, const pmf_qual_t *qual) 3965 { 3966 struct adapter *adapter = device_private(dev); 3967 int error = 0; 3968 3969 INIT_DEBUGOUT("ixgbe_suspend: begin"); 3970 3971 error = ixgbe_setup_low_power_mode(adapter); 3972 3973 return (error); 3974 } /* ixgbe_suspend */ 3975 3976 /************************************************************************ 3977 * ixgbe_resume 3978 * 3979 * From D3 to D0 3980 ************************************************************************/ 3981 static bool 3982 ixgbe_resume(device_t dev, const pmf_qual_t *qual) 3983 { 3984 struct adapter *adapter = device_private(dev); 3985 struct ifnet *ifp = adapter->ifp; 3986 struct ixgbe_hw *hw = &adapter->hw; 3987 u32 wus; 3988 3989 INIT_DEBUGOUT("ixgbe_resume: begin"); 3990 3991 IXGBE_CORE_LOCK(adapter); 3992 3993 /* Read & clear WUS register */ 3994 wus = IXGBE_READ_REG(hw, IXGBE_WUS); 3995 if (wus) 3996 device_printf(dev, "Woken up by (WUS): %#010x\n", 3997 IXGBE_READ_REG(hw, IXGBE_WUS)); 3998 IXGBE_WRITE_REG(hw, IXGBE_WUS, 0xffffffff); 3999 /* And clear WUFC until next low-power transition */ 4000 IXGBE_WRITE_REG(hw, IXGBE_WUFC, 0); 4001 4002 /* 4003 * Required after D3->D0 transition; 4004 * will re-advertise all previous advertised speeds 4005 */ 4006 if (ifp->if_flags & IFF_UP) 4007 ixgbe_init_locked(adapter); 4008 4009 IXGBE_CORE_UNLOCK(adapter); 4010 4011 return true; 4012 } /* ixgbe_resume */ 4013 4014 /* 4015 * Set the various hardware offload abilities. 4016 * 4017 * This takes the ifnet's if_capenable flags (e.g. set by the user using 4018 * ifconfig) and indicates to the OS via the ifnet's if_hwassist field what 4019 * mbuf offload flags the driver will understand. 4020 */ 4021 static void 4022 ixgbe_set_if_hwassist(struct adapter *adapter) 4023 { 4024 /* XXX */ 4025 } 4026 4027 /************************************************************************ 4028 * ixgbe_init_locked - Init entry point 4029 * 4030 * Used in two ways: It is used by the stack as an init 4031 * entry point in network interface structure. It is also 4032 * used by the driver as a hw/sw initialization routine to 4033 * get to a consistent state. 4034 * 4035 * return 0 on success, positive on failure 4036 ************************************************************************/ 4037 static void 4038 ixgbe_init_locked(struct adapter *adapter) 4039 { 4040 struct ifnet *ifp = adapter->ifp; 4041 device_t dev = adapter->dev; 4042 struct ixgbe_hw *hw = &adapter->hw; 4043 struct ix_queue *que; 4044 struct tx_ring *txr; 4045 struct rx_ring *rxr; 4046 u32 txdctl, mhadd; 4047 u32 rxdctl, rxctrl; 4048 u32 ctrl_ext; 4049 bool unsupported_sfp = false; 4050 int i, j, error; 4051 4052 /* XXX check IFF_UP and IFF_RUNNING, power-saving state! */ 4053 4054 KASSERT(mutex_owned(&adapter->core_mtx)); 4055 INIT_DEBUGOUT("ixgbe_init_locked: begin"); 4056 4057 hw->need_unsupported_sfp_recovery = false; 4058 hw->adapter_stopped = FALSE; 4059 ixgbe_stop_adapter(hw); 4060 callout_stop(&adapter->timer); 4061 if (adapter->feat_en & IXGBE_FEATURE_RECOVERY_MODE) 4062 callout_stop(&adapter->recovery_mode_timer); 4063 for (i = 0, que = adapter->queues; i < adapter->num_queues; i++, que++) 4064 que->disabled_count = 0; 4065 4066 /* XXX I moved this here from the SIOCSIFMTU case in ixgbe_ioctl(). */ 4067 adapter->max_frame_size = 4068 ifp->if_mtu + ETHER_HDR_LEN + ETHER_CRC_LEN; 4069 4070 /* Queue indices may change with IOV mode */ 4071 ixgbe_align_all_queue_indices(adapter); 4072 4073 /* reprogram the RAR[0] in case user changed it. */ 4074 ixgbe_set_rar(hw, 0, hw->mac.addr, adapter->pool, IXGBE_RAH_AV); 4075 4076 /* Get the latest mac address, User can use a LAA */ 4077 memcpy(hw->mac.addr, CLLADDR(ifp->if_sadl), 4078 IXGBE_ETH_LENGTH_OF_ADDRESS); 4079 ixgbe_set_rar(hw, 0, hw->mac.addr, adapter->pool, 1); 4080 hw->addr_ctrl.rar_used_count = 1; 4081 4082 /* Set hardware offload abilities from ifnet flags */ 4083 ixgbe_set_if_hwassist(adapter); 4084 4085 /* Prepare transmit descriptors and buffers */ 4086 if (ixgbe_setup_transmit_structures(adapter)) { 4087 device_printf(dev, "Could not setup transmit structures\n"); 4088 ixgbe_stop_locked(adapter); 4089 return; 4090 } 4091 4092 ixgbe_init_hw(hw); 4093 4094 ixgbe_initialize_iov(adapter); 4095 4096 ixgbe_initialize_transmit_units(adapter); 4097 4098 /* Setup Multicast table */ 4099 ixgbe_set_rxfilter(adapter); 4100 4101 /* Use fixed buffer size, even for jumbo frames */ 4102 adapter->rx_mbuf_sz = MCLBYTES; 4103 4104 /* Prepare receive descriptors and buffers */ 4105 error = ixgbe_setup_receive_structures(adapter); 4106 if (error) { 4107 device_printf(dev, 4108 "Could not setup receive structures (err = %d)\n", error); 4109 ixgbe_stop_locked(adapter); 4110 return; 4111 } 4112 4113 /* Configure RX settings */ 4114 ixgbe_initialize_receive_units(adapter); 4115 4116 /* Initialize variable holding task enqueue requests interrupts */ 4117 adapter->task_requests = 0; 4118 4119 /* Enable SDP & MSI-X interrupts based on adapter */ 4120 ixgbe_config_gpie(adapter); 4121 4122 /* Set MTU size */ 4123 if (ifp->if_mtu > ETHERMTU) { 4124 /* aka IXGBE_MAXFRS on 82599 and newer */ 4125 mhadd = IXGBE_READ_REG(hw, IXGBE_MHADD); 4126 mhadd &= ~IXGBE_MHADD_MFS_MASK; 4127 mhadd |= adapter->max_frame_size << IXGBE_MHADD_MFS_SHIFT; 4128 IXGBE_WRITE_REG(hw, IXGBE_MHADD, mhadd); 4129 } 4130 4131 /* Now enable all the queues */ 4132 for (i = 0; i < adapter->num_queues; i++) { 4133 txr = &adapter->tx_rings[i]; 4134 txdctl = IXGBE_READ_REG(hw, IXGBE_TXDCTL(txr->me)); 4135 txdctl |= IXGBE_TXDCTL_ENABLE; 4136 /* Set WTHRESH to 8, burst writeback */ 4137 txdctl |= IXGBE_TX_WTHRESH << IXGBE_TXDCTL_WTHRESH_SHIFT; 4138 /* 4139 * When the internal queue falls below PTHRESH (32), 4140 * start prefetching as long as there are at least 4141 * HTHRESH (1) buffers ready. The values are taken 4142 * from the Intel linux driver 3.8.21. 4143 * Prefetching enables tx line rate even with 1 queue. 4144 */ 4145 txdctl |= (32 << 0) | (1 << 8); 4146 IXGBE_WRITE_REG(hw, IXGBE_TXDCTL(txr->me), txdctl); 4147 } 4148 4149 for (i = 0; i < adapter->num_queues; i++) { 4150 rxr = &adapter->rx_rings[i]; 4151 rxdctl = IXGBE_READ_REG(hw, IXGBE_RXDCTL(rxr->me)); 4152 if (hw->mac.type == ixgbe_mac_82598EB) { 4153 /* 4154 * PTHRESH = 21 4155 * HTHRESH = 4 4156 * WTHRESH = 8 4157 */ 4158 rxdctl &= ~0x3FFFFF; 4159 rxdctl |= 0x080420; 4160 } 4161 rxdctl |= IXGBE_RXDCTL_ENABLE; 4162 IXGBE_WRITE_REG(hw, IXGBE_RXDCTL(rxr->me), rxdctl); 4163 for (j = 0; j < 10; j++) { 4164 if (IXGBE_READ_REG(hw, IXGBE_RXDCTL(rxr->me)) & 4165 IXGBE_RXDCTL_ENABLE) 4166 break; 4167 else 4168 msec_delay(1); 4169 } 4170 IXGBE_WRITE_BARRIER(hw); 4171 4172 /* 4173 * In netmap mode, we must preserve the buffers made 4174 * available to userspace before the if_init() 4175 * (this is true by default on the TX side, because 4176 * init makes all buffers available to userspace). 4177 * 4178 * netmap_reset() and the device specific routines 4179 * (e.g. ixgbe_setup_receive_rings()) map these 4180 * buffers at the end of the NIC ring, so here we 4181 * must set the RDT (tail) register to make sure 4182 * they are not overwritten. 4183 * 4184 * In this driver the NIC ring starts at RDH = 0, 4185 * RDT points to the last slot available for reception (?), 4186 * so RDT = num_rx_desc - 1 means the whole ring is available. 4187 */ 4188 #ifdef DEV_NETMAP 4189 if ((adapter->feat_en & IXGBE_FEATURE_NETMAP) && 4190 (ifp->if_capenable & IFCAP_NETMAP)) { 4191 struct netmap_adapter *na = NA(adapter->ifp); 4192 struct netmap_kring *kring = na->rx_rings[i]; 4193 int t = na->num_rx_desc - 1 - nm_kr_rxspace(kring); 4194 4195 IXGBE_WRITE_REG(hw, IXGBE_RDT(rxr->me), t); 4196 } else 4197 #endif /* DEV_NETMAP */ 4198 IXGBE_WRITE_REG(hw, IXGBE_RDT(rxr->me), 4199 adapter->num_rx_desc - 1); 4200 } 4201 4202 /* Enable Receive engine */ 4203 rxctrl = IXGBE_READ_REG(hw, IXGBE_RXCTRL); 4204 if (hw->mac.type == ixgbe_mac_82598EB) 4205 rxctrl |= IXGBE_RXCTRL_DMBYPS; 4206 rxctrl |= IXGBE_RXCTRL_RXEN; 4207 ixgbe_enable_rx_dma(hw, rxctrl); 4208 4209 callout_reset(&adapter->timer, hz, ixgbe_local_timer, adapter); 4210 atomic_store_relaxed(&adapter->timer_pending, 0); 4211 if (adapter->feat_en & IXGBE_FEATURE_RECOVERY_MODE) 4212 callout_reset(&adapter->recovery_mode_timer, hz, 4213 ixgbe_recovery_mode_timer, adapter); 4214 4215 /* Set up MSI/MSI-X routing */ 4216 if (adapter->feat_en & IXGBE_FEATURE_MSIX) { 4217 ixgbe_configure_ivars(adapter); 4218 /* Set up auto-mask */ 4219 if (hw->mac.type == ixgbe_mac_82598EB) 4220 IXGBE_WRITE_REG(hw, IXGBE_EIAM, IXGBE_EICS_RTX_QUEUE); 4221 else { 4222 IXGBE_WRITE_REG(hw, IXGBE_EIAM_EX(0), 0xFFFFFFFF); 4223 IXGBE_WRITE_REG(hw, IXGBE_EIAM_EX(1), 0xFFFFFFFF); 4224 } 4225 } else { /* Simple settings for Legacy/MSI */ 4226 ixgbe_set_ivar(adapter, 0, 0, 0); 4227 ixgbe_set_ivar(adapter, 0, 0, 1); 4228 IXGBE_WRITE_REG(hw, IXGBE_EIAM, IXGBE_EICS_RTX_QUEUE); 4229 } 4230 4231 ixgbe_init_fdir(adapter); 4232 4233 /* 4234 * Check on any SFP devices that 4235 * need to be kick-started 4236 */ 4237 if (hw->phy.type == ixgbe_phy_none) { 4238 error = hw->phy.ops.identify(hw); 4239 if (error == IXGBE_ERR_SFP_NOT_SUPPORTED) 4240 unsupported_sfp = true; 4241 } else if (hw->phy.type == ixgbe_phy_sfp_unsupported) 4242 unsupported_sfp = true; 4243 4244 if (unsupported_sfp) 4245 device_printf(dev, 4246 "Unsupported SFP+ module type was detected.\n"); 4247 4248 /* Set moderation on the Link interrupt */ 4249 ixgbe_eitr_write(adapter, adapter->vector, IXGBE_LINK_ITR); 4250 4251 /* Enable EEE power saving */ 4252 if (adapter->feat_cap & IXGBE_FEATURE_EEE) 4253 hw->mac.ops.setup_eee(hw, 4254 adapter->feat_en & IXGBE_FEATURE_EEE); 4255 4256 /* Enable power to the phy. */ 4257 if (!unsupported_sfp) { 4258 ixgbe_set_phy_power(hw, TRUE); 4259 4260 /* Config/Enable Link */ 4261 ixgbe_config_link(adapter); 4262 } 4263 4264 /* Hardware Packet Buffer & Flow Control setup */ 4265 ixgbe_config_delay_values(adapter); 4266 4267 /* Initialize the FC settings */ 4268 ixgbe_start_hw(hw); 4269 4270 /* Set up VLAN support and filter */ 4271 ixgbe_setup_vlan_hw_support(adapter); 4272 4273 /* Setup DMA Coalescing */ 4274 ixgbe_config_dmac(adapter); 4275 4276 /* OK to schedule workqueues. */ 4277 adapter->schedule_wqs_ok = true; 4278 4279 /* And now turn on interrupts */ 4280 ixgbe_enable_intr(adapter); 4281 4282 /* Enable the use of the MBX by the VF's */ 4283 if (adapter->feat_en & IXGBE_FEATURE_SRIOV) { 4284 ctrl_ext = IXGBE_READ_REG(hw, IXGBE_CTRL_EXT); 4285 ctrl_ext |= IXGBE_CTRL_EXT_PFRSTD; 4286 IXGBE_WRITE_REG(hw, IXGBE_CTRL_EXT, ctrl_ext); 4287 } 4288 4289 /* Update saved flags. See ixgbe_ifflags_cb() */ 4290 adapter->if_flags = ifp->if_flags; 4291 adapter->ec_capenable = adapter->osdep.ec.ec_capenable; 4292 4293 /* Now inform the stack we're ready */ 4294 ifp->if_flags |= IFF_RUNNING; 4295 4296 return; 4297 } /* ixgbe_init_locked */ 4298 4299 /************************************************************************ 4300 * ixgbe_init 4301 ************************************************************************/ 4302 static int 4303 ixgbe_init(struct ifnet *ifp) 4304 { 4305 struct adapter *adapter = ifp->if_softc; 4306 4307 IXGBE_CORE_LOCK(adapter); 4308 ixgbe_init_locked(adapter); 4309 IXGBE_CORE_UNLOCK(adapter); 4310 4311 return 0; /* XXX ixgbe_init_locked cannot fail? really? */ 4312 } /* ixgbe_init */ 4313 4314 /************************************************************************ 4315 * ixgbe_set_ivar 4316 * 4317 * Setup the correct IVAR register for a particular MSI-X interrupt 4318 * (yes this is all very magic and confusing :) 4319 * - entry is the register array entry 4320 * - vector is the MSI-X vector for this queue 4321 * - type is RX/TX/MISC 4322 ************************************************************************/ 4323 static void 4324 ixgbe_set_ivar(struct adapter *adapter, u8 entry, u8 vector, s8 type) 4325 { 4326 struct ixgbe_hw *hw = &adapter->hw; 4327 u32 ivar, index; 4328 4329 vector |= IXGBE_IVAR_ALLOC_VAL; 4330 4331 switch (hw->mac.type) { 4332 case ixgbe_mac_82598EB: 4333 if (type == -1) 4334 entry = IXGBE_IVAR_OTHER_CAUSES_INDEX; 4335 else 4336 entry += (type * 64); 4337 index = (entry >> 2) & 0x1F; 4338 ivar = IXGBE_READ_REG(hw, IXGBE_IVAR(index)); 4339 ivar &= ~(0xffUL << (8 * (entry & 0x3))); 4340 ivar |= ((u32)vector << (8 * (entry & 0x3))); 4341 IXGBE_WRITE_REG(&adapter->hw, IXGBE_IVAR(index), ivar); 4342 break; 4343 case ixgbe_mac_82599EB: 4344 case ixgbe_mac_X540: 4345 case ixgbe_mac_X550: 4346 case ixgbe_mac_X550EM_x: 4347 case ixgbe_mac_X550EM_a: 4348 if (type == -1) { /* MISC IVAR */ 4349 index = (entry & 1) * 8; 4350 ivar = IXGBE_READ_REG(hw, IXGBE_IVAR_MISC); 4351 ivar &= ~(0xffUL << index); 4352 ivar |= ((u32)vector << index); 4353 IXGBE_WRITE_REG(hw, IXGBE_IVAR_MISC, ivar); 4354 } else { /* RX/TX IVARS */ 4355 index = (16 * (entry & 1)) + (8 * type); 4356 ivar = IXGBE_READ_REG(hw, IXGBE_IVAR(entry >> 1)); 4357 ivar &= ~(0xffUL << index); 4358 ivar |= ((u32)vector << index); 4359 IXGBE_WRITE_REG(hw, IXGBE_IVAR(entry >> 1), ivar); 4360 } 4361 break; 4362 default: 4363 break; 4364 } 4365 } /* ixgbe_set_ivar */ 4366 4367 /************************************************************************ 4368 * ixgbe_configure_ivars 4369 ************************************************************************/ 4370 static void 4371 ixgbe_configure_ivars(struct adapter *adapter) 4372 { 4373 struct ix_queue *que = adapter->queues; 4374 u32 newitr; 4375 4376 if (ixgbe_max_interrupt_rate > 0) 4377 newitr = (4000000 / ixgbe_max_interrupt_rate) & 0x0FF8; 4378 else { 4379 /* 4380 * Disable DMA coalescing if interrupt moderation is 4381 * disabled. 4382 */ 4383 adapter->dmac = 0; 4384 newitr = 0; 4385 } 4386 4387 for (int i = 0; i < adapter->num_queues; i++, que++) { 4388 struct rx_ring *rxr = &adapter->rx_rings[i]; 4389 struct tx_ring *txr = &adapter->tx_rings[i]; 4390 /* First the RX queue entry */ 4391 ixgbe_set_ivar(adapter, rxr->me, que->msix, 0); 4392 /* ... and the TX */ 4393 ixgbe_set_ivar(adapter, txr->me, que->msix, 1); 4394 /* Set an Initial EITR value */ 4395 ixgbe_eitr_write(adapter, que->msix, newitr); 4396 /* 4397 * To eliminate influence of the previous state. 4398 * At this point, Tx/Rx interrupt handler 4399 * (ixgbe_msix_que()) cannot be called, so both 4400 * IXGBE_TX_LOCK and IXGBE_RX_LOCK are not required. 4401 */ 4402 que->eitr_setting = 0; 4403 } 4404 4405 /* For the Link interrupt */ 4406 ixgbe_set_ivar(adapter, 1, adapter->vector, -1); 4407 } /* ixgbe_configure_ivars */ 4408 4409 /************************************************************************ 4410 * ixgbe_config_gpie 4411 ************************************************************************/ 4412 static void 4413 ixgbe_config_gpie(struct adapter *adapter) 4414 { 4415 struct ixgbe_hw *hw = &adapter->hw; 4416 u32 gpie; 4417 4418 gpie = IXGBE_READ_REG(hw, IXGBE_GPIE); 4419 4420 if (adapter->feat_en & IXGBE_FEATURE_MSIX) { 4421 /* Enable Enhanced MSI-X mode */ 4422 gpie |= IXGBE_GPIE_MSIX_MODE 4423 | IXGBE_GPIE_EIAME 4424 | IXGBE_GPIE_PBA_SUPPORT 4425 | IXGBE_GPIE_OCD; 4426 } 4427 4428 /* Fan Failure Interrupt */ 4429 if (adapter->feat_en & IXGBE_FEATURE_FAN_FAIL) 4430 gpie |= IXGBE_SDP1_GPIEN; 4431 4432 /* Thermal Sensor Interrupt */ 4433 if (adapter->feat_en & IXGBE_FEATURE_TEMP_SENSOR) 4434 gpie |= IXGBE_SDP0_GPIEN_X540; 4435 4436 /* Link detection */ 4437 switch (hw->mac.type) { 4438 case ixgbe_mac_82599EB: 4439 gpie |= IXGBE_SDP1_GPIEN | IXGBE_SDP2_GPIEN; 4440 break; 4441 case ixgbe_mac_X550EM_x: 4442 case ixgbe_mac_X550EM_a: 4443 gpie |= IXGBE_SDP0_GPIEN_X540; 4444 break; 4445 default: 4446 break; 4447 } 4448 4449 IXGBE_WRITE_REG(hw, IXGBE_GPIE, gpie); 4450 4451 } /* ixgbe_config_gpie */ 4452 4453 /************************************************************************ 4454 * ixgbe_config_delay_values 4455 * 4456 * Requires adapter->max_frame_size to be set. 4457 ************************************************************************/ 4458 static void 4459 ixgbe_config_delay_values(struct adapter *adapter) 4460 { 4461 struct ixgbe_hw *hw = &adapter->hw; 4462 u32 rxpb, frame, size, tmp; 4463 4464 frame = adapter->max_frame_size; 4465 4466 /* Calculate High Water */ 4467 switch (hw->mac.type) { 4468 case ixgbe_mac_X540: 4469 case ixgbe_mac_X550: 4470 case ixgbe_mac_X550EM_x: 4471 case ixgbe_mac_X550EM_a: 4472 tmp = IXGBE_DV_X540(frame, frame); 4473 break; 4474 default: 4475 tmp = IXGBE_DV(frame, frame); 4476 break; 4477 } 4478 size = IXGBE_BT2KB(tmp); 4479 rxpb = IXGBE_READ_REG(hw, IXGBE_RXPBSIZE(0)) >> 10; 4480 hw->fc.high_water[0] = rxpb - size; 4481 4482 /* Now calculate Low Water */ 4483 switch (hw->mac.type) { 4484 case ixgbe_mac_X540: 4485 case ixgbe_mac_X550: 4486 case ixgbe_mac_X550EM_x: 4487 case ixgbe_mac_X550EM_a: 4488 tmp = IXGBE_LOW_DV_X540(frame); 4489 break; 4490 default: 4491 tmp = IXGBE_LOW_DV(frame); 4492 break; 4493 } 4494 hw->fc.low_water[0] = IXGBE_BT2KB(tmp); 4495 4496 hw->fc.pause_time = IXGBE_FC_PAUSE; 4497 hw->fc.send_xon = TRUE; 4498 } /* ixgbe_config_delay_values */ 4499 4500 /************************************************************************ 4501 * ixgbe_set_rxfilter - Multicast Update 4502 * 4503 * Called whenever multicast address list is updated. 4504 ************************************************************************/ 4505 static void 4506 ixgbe_set_rxfilter(struct adapter *adapter) 4507 { 4508 struct ixgbe_mc_addr *mta; 4509 struct ifnet *ifp = adapter->ifp; 4510 u8 *update_ptr; 4511 int mcnt = 0; 4512 u32 fctrl; 4513 struct ethercom *ec = &adapter->osdep.ec; 4514 struct ether_multi *enm; 4515 struct ether_multistep step; 4516 4517 KASSERT(mutex_owned(&adapter->core_mtx)); 4518 IOCTL_DEBUGOUT("ixgbe_set_rxfilter: begin"); 4519 4520 mta = adapter->mta; 4521 bzero(mta, sizeof(*mta) * MAX_NUM_MULTICAST_ADDRESSES); 4522 4523 ETHER_LOCK(ec); 4524 ec->ec_flags &= ~ETHER_F_ALLMULTI; 4525 ETHER_FIRST_MULTI(step, ec, enm); 4526 while (enm != NULL) { 4527 if ((mcnt == MAX_NUM_MULTICAST_ADDRESSES) || 4528 (memcmp(enm->enm_addrlo, enm->enm_addrhi, 4529 ETHER_ADDR_LEN) != 0)) { 4530 ec->ec_flags |= ETHER_F_ALLMULTI; 4531 break; 4532 } 4533 bcopy(enm->enm_addrlo, 4534 mta[mcnt].addr, IXGBE_ETH_LENGTH_OF_ADDRESS); 4535 mta[mcnt].vmdq = adapter->pool; 4536 mcnt++; 4537 ETHER_NEXT_MULTI(step, enm); 4538 } 4539 4540 fctrl = IXGBE_READ_REG(&adapter->hw, IXGBE_FCTRL); 4541 if (ifp->if_flags & IFF_PROMISC) 4542 fctrl |= (IXGBE_FCTRL_UPE | IXGBE_FCTRL_MPE); 4543 else if (ec->ec_flags & ETHER_F_ALLMULTI) { 4544 fctrl |= IXGBE_FCTRL_MPE; 4545 fctrl &= ~IXGBE_FCTRL_UPE; 4546 } else 4547 fctrl &= ~(IXGBE_FCTRL_UPE | IXGBE_FCTRL_MPE); 4548 4549 IXGBE_WRITE_REG(&adapter->hw, IXGBE_FCTRL, fctrl); 4550 4551 /* Update multicast filter entries only when it's not ALLMULTI */ 4552 if ((ec->ec_flags & ETHER_F_ALLMULTI) == 0) { 4553 ETHER_UNLOCK(ec); 4554 update_ptr = (u8 *)mta; 4555 ixgbe_update_mc_addr_list(&adapter->hw, update_ptr, mcnt, 4556 ixgbe_mc_array_itr, TRUE); 4557 } else 4558 ETHER_UNLOCK(ec); 4559 } /* ixgbe_set_rxfilter */ 4560 4561 /************************************************************************ 4562 * ixgbe_mc_array_itr 4563 * 4564 * An iterator function needed by the multicast shared code. 4565 * It feeds the shared code routine the addresses in the 4566 * array of ixgbe_set_rxfilter() one by one. 4567 ************************************************************************/ 4568 static u8 * 4569 ixgbe_mc_array_itr(struct ixgbe_hw *hw, u8 **update_ptr, u32 *vmdq) 4570 { 4571 struct ixgbe_mc_addr *mta; 4572 4573 mta = (struct ixgbe_mc_addr *)*update_ptr; 4574 *vmdq = mta->vmdq; 4575 4576 *update_ptr = (u8*)(mta + 1); 4577 4578 return (mta->addr); 4579 } /* ixgbe_mc_array_itr */ 4580 4581 /************************************************************************ 4582 * ixgbe_local_timer - Timer routine 4583 * 4584 * Checks for link status, updates statistics, 4585 * and runs the watchdog check. 4586 ************************************************************************/ 4587 static void 4588 ixgbe_local_timer(void *arg) 4589 { 4590 struct adapter *adapter = arg; 4591 4592 if (adapter->schedule_wqs_ok) { 4593 if (atomic_cas_uint(&adapter->timer_pending, 0, 1) == 0) 4594 workqueue_enqueue(adapter->timer_wq, 4595 &adapter->timer_wc, NULL); 4596 } 4597 } 4598 4599 static void 4600 ixgbe_handle_timer(struct work *wk, void *context) 4601 { 4602 struct adapter *adapter = context; 4603 struct ixgbe_hw *hw = &adapter->hw; 4604 device_t dev = adapter->dev; 4605 struct ix_queue *que = adapter->queues; 4606 u64 queues = 0; 4607 u64 v0, v1, v2, v3, v4, v5, v6, v7; 4608 int hung = 0; 4609 int i; 4610 4611 IXGBE_CORE_LOCK(adapter); 4612 4613 /* Check for pluggable optics */ 4614 if (ixgbe_is_sfp(hw)) { 4615 bool sched_mod_task = false; 4616 4617 if (hw->mac.type == ixgbe_mac_82598EB) { 4618 /* 4619 * On 82598EB, SFP+'s MOD_ABS pin is not connected to 4620 * any GPIO(SDP). So just schedule TASK_MOD. 4621 */ 4622 sched_mod_task = true; 4623 } else { 4624 bool was_full, is_full; 4625 4626 was_full = 4627 hw->phy.sfp_type != ixgbe_sfp_type_not_present; 4628 is_full = ixgbe_sfp_cage_full(hw); 4629 4630 /* Do probe if cage state changed */ 4631 if (was_full ^ is_full) 4632 sched_mod_task = true; 4633 } 4634 if (sched_mod_task) { 4635 mutex_enter(&adapter->admin_mtx); 4636 adapter->task_requests |= IXGBE_REQUEST_TASK_MOD_WOI; 4637 ixgbe_schedule_admin_tasklet(adapter); 4638 mutex_exit(&adapter->admin_mtx); 4639 } 4640 } 4641 4642 ixgbe_update_link_status(adapter); 4643 ixgbe_update_stats_counters(adapter); 4644 4645 /* Update some event counters */ 4646 v0 = v1 = v2 = v3 = v4 = v5 = v6 = v7 = 0; 4647 que = adapter->queues; 4648 for (i = 0; i < adapter->num_queues; i++, que++) { 4649 struct tx_ring *txr = que->txr; 4650 4651 v0 += txr->q_efbig_tx_dma_setup; 4652 v1 += txr->q_mbuf_defrag_failed; 4653 v2 += txr->q_efbig2_tx_dma_setup; 4654 v3 += txr->q_einval_tx_dma_setup; 4655 v4 += txr->q_other_tx_dma_setup; 4656 v5 += txr->q_eagain_tx_dma_setup; 4657 v6 += txr->q_enomem_tx_dma_setup; 4658 v7 += txr->q_tso_err; 4659 } 4660 IXGBE_EVC_STORE(&adapter->efbig_tx_dma_setup, v0); 4661 IXGBE_EVC_STORE(&adapter->mbuf_defrag_failed, v1); 4662 IXGBE_EVC_STORE(&adapter->efbig2_tx_dma_setup, v2); 4663 IXGBE_EVC_STORE(&adapter->einval_tx_dma_setup, v3); 4664 IXGBE_EVC_STORE(&adapter->other_tx_dma_setup, v4); 4665 IXGBE_EVC_STORE(&adapter->eagain_tx_dma_setup, v5); 4666 IXGBE_EVC_STORE(&adapter->enomem_tx_dma_setup, v6); 4667 IXGBE_EVC_STORE(&adapter->tso_err, v7); 4668 4669 /* 4670 * Check the TX queues status 4671 * - mark hung queues so we don't schedule on them 4672 * - watchdog only if all queues show hung 4673 */ 4674 que = adapter->queues; 4675 for (i = 0; i < adapter->num_queues; i++, que++) { 4676 /* Keep track of queues with work for soft irq */ 4677 if (que->txr->busy) 4678 queues |= 1ULL << que->me; 4679 /* 4680 * Each time txeof runs without cleaning, but there 4681 * are uncleaned descriptors it increments busy. If 4682 * we get to the MAX we declare it hung. 4683 */ 4684 if (que->busy == IXGBE_QUEUE_HUNG) { 4685 ++hung; 4686 /* Mark the queue as inactive */ 4687 adapter->active_queues &= ~(1ULL << que->me); 4688 continue; 4689 } else { 4690 /* Check if we've come back from hung */ 4691 if ((adapter->active_queues & (1ULL << que->me)) == 0) 4692 adapter->active_queues |= 1ULL << que->me; 4693 } 4694 if (que->busy >= IXGBE_MAX_TX_BUSY) { 4695 device_printf(dev, 4696 "Warning queue %d appears to be hung!\n", i); 4697 que->txr->busy = IXGBE_QUEUE_HUNG; 4698 ++hung; 4699 } 4700 } 4701 4702 /* Only truly watchdog if all queues show hung */ 4703 if (hung == adapter->num_queues) 4704 goto watchdog; 4705 #if 0 /* XXX Avoid unexpectedly disabling interrupt forever (PR#53294) */ 4706 else if (queues != 0) { /* Force an IRQ on queues with work */ 4707 que = adapter->queues; 4708 for (i = 0; i < adapter->num_queues; i++, que++) { 4709 mutex_enter(&que->dc_mtx); 4710 if (que->disabled_count == 0) 4711 ixgbe_rearm_queues(adapter, 4712 queues & ((u64)1 << i)); 4713 mutex_exit(&que->dc_mtx); 4714 } 4715 } 4716 #endif 4717 4718 atomic_store_relaxed(&adapter->timer_pending, 0); 4719 IXGBE_CORE_UNLOCK(adapter); 4720 callout_reset(&adapter->timer, hz, ixgbe_local_timer, adapter); 4721 return; 4722 4723 watchdog: 4724 device_printf(adapter->dev, "Watchdog timeout -- resetting\n"); 4725 adapter->ifp->if_flags &= ~IFF_RUNNING; 4726 IXGBE_EVC_ADD(&adapter->watchdog_events, 1); 4727 ixgbe_init_locked(adapter); 4728 IXGBE_CORE_UNLOCK(adapter); 4729 } /* ixgbe_handle_timer */ 4730 4731 /************************************************************************ 4732 * ixgbe_recovery_mode_timer - Recovery mode timer routine 4733 ************************************************************************/ 4734 static void 4735 ixgbe_recovery_mode_timer(void *arg) 4736 { 4737 struct adapter *adapter = arg; 4738 4739 if (__predict_true(adapter->osdep.detaching == false)) { 4740 if (atomic_cas_uint(&adapter->recovery_mode_timer_pending, 4741 0, 1) == 0) { 4742 workqueue_enqueue(adapter->recovery_mode_timer_wq, 4743 &adapter->recovery_mode_timer_wc, NULL); 4744 } 4745 } 4746 } 4747 4748 static void 4749 ixgbe_handle_recovery_mode_timer(struct work *wk, void *context) 4750 { 4751 struct adapter *adapter = context; 4752 struct ixgbe_hw *hw = &adapter->hw; 4753 4754 IXGBE_CORE_LOCK(adapter); 4755 if (ixgbe_fw_recovery_mode(hw)) { 4756 if (atomic_cas_uint(&adapter->recovery_mode, 0, 1)) { 4757 /* Firmware error detected, entering recovery mode */ 4758 device_printf(adapter->dev, "Firmware recovery mode detected. Limiting functionality. Refer to the Intel(R) Ethernet Adapters and Devices User Guide for details on firmware recovery mode.\n"); 4759 4760 if (hw->adapter_stopped == FALSE) 4761 ixgbe_stop_locked(adapter); 4762 } 4763 } else 4764 atomic_cas_uint(&adapter->recovery_mode, 1, 0); 4765 4766 atomic_store_relaxed(&adapter->recovery_mode_timer_pending, 0); 4767 callout_reset(&adapter->recovery_mode_timer, hz, 4768 ixgbe_recovery_mode_timer, adapter); 4769 IXGBE_CORE_UNLOCK(adapter); 4770 } /* ixgbe_handle_recovery_mode_timer */ 4771 4772 /************************************************************************ 4773 * ixgbe_handle_mod - Tasklet for SFP module interrupts 4774 * bool int_en: true if it's called when the interrupt is enabled. 4775 ************************************************************************/ 4776 static void 4777 ixgbe_handle_mod(void *context, bool int_en) 4778 { 4779 struct adapter *adapter = context; 4780 struct ixgbe_hw *hw = &adapter->hw; 4781 device_t dev = adapter->dev; 4782 enum ixgbe_sfp_type last_sfp_type; 4783 u32 err; 4784 bool last_unsupported_sfp_recovery; 4785 4786 KASSERT(mutex_owned(&adapter->core_mtx)); 4787 4788 last_sfp_type = hw->phy.sfp_type; 4789 last_unsupported_sfp_recovery = hw->need_unsupported_sfp_recovery; 4790 IXGBE_EVC_ADD(&adapter->mod_workev, 1); 4791 if (adapter->hw.need_crosstalk_fix) { 4792 if ((hw->mac.type != ixgbe_mac_82598EB) && 4793 !ixgbe_sfp_cage_full(hw)) 4794 goto out; 4795 } 4796 4797 err = hw->phy.ops.identify_sfp(hw); 4798 if (err == IXGBE_ERR_SFP_NOT_SUPPORTED) { 4799 if (last_unsupported_sfp_recovery == false) 4800 device_printf(dev, 4801 "Unsupported SFP+ module type was detected.\n"); 4802 goto out; 4803 } 4804 4805 if (hw->need_unsupported_sfp_recovery) { 4806 device_printf(dev, "Recovering from unsupported SFP\n"); 4807 /* 4808 * We could recover the status by calling setup_sfp(), 4809 * setup_link() and some others. It's complex and might not 4810 * work correctly on some unknown cases. To avoid such type of 4811 * problem, call ixgbe_init_locked(). It's simple and safe 4812 * approach. 4813 */ 4814 ixgbe_init_locked(adapter); 4815 } else if ((hw->phy.sfp_type != ixgbe_sfp_type_not_present) && 4816 (hw->phy.sfp_type != last_sfp_type)) { 4817 /* A module is inserted and changed. */ 4818 4819 if (hw->mac.type == ixgbe_mac_82598EB) 4820 err = hw->phy.ops.reset(hw); 4821 else { 4822 err = hw->mac.ops.setup_sfp(hw); 4823 hw->phy.sfp_setup_needed = FALSE; 4824 } 4825 if (err == IXGBE_ERR_SFP_NOT_SUPPORTED) { 4826 device_printf(dev, 4827 "Setup failure - unsupported SFP+ module type.\n"); 4828 goto out; 4829 } 4830 } 4831 4832 out: 4833 /* get_supported_phy_layer will call hw->phy.ops.identify_sfp() */ 4834 adapter->phy_layer = ixgbe_get_supported_physical_layer(hw); 4835 4836 /* Adjust media types shown in ifconfig */ 4837 IXGBE_CORE_UNLOCK(adapter); 4838 ifmedia_removeall(&adapter->media); 4839 ixgbe_add_media_types(adapter); 4840 ifmedia_set(&adapter->media, IFM_ETHER | IFM_AUTO); 4841 IXGBE_CORE_LOCK(adapter); 4842 4843 /* 4844 * Don't schedule MSF event if the chip is 82598. 82598 doesn't support 4845 * MSF. At least, calling ixgbe_handle_msf on 82598 DA makes the link 4846 * flap because the function calls setup_link(). 4847 */ 4848 if (hw->mac.type != ixgbe_mac_82598EB) { 4849 mutex_enter(&adapter->admin_mtx); 4850 if (int_en) 4851 adapter->task_requests |= IXGBE_REQUEST_TASK_MSF; 4852 else 4853 adapter->task_requests |= IXGBE_REQUEST_TASK_MSF_WOI; 4854 mutex_exit(&adapter->admin_mtx); 4855 } 4856 4857 /* 4858 * Don't call ixgbe_schedule_admin_tasklet() because we are on 4859 * the workqueue now. 4860 */ 4861 } /* ixgbe_handle_mod */ 4862 4863 4864 /************************************************************************ 4865 * ixgbe_handle_msf - Tasklet for MSF (multispeed fiber) interrupts 4866 ************************************************************************/ 4867 static void 4868 ixgbe_handle_msf(void *context) 4869 { 4870 struct adapter *adapter = context; 4871 struct ixgbe_hw *hw = &adapter->hw; 4872 u32 autoneg; 4873 bool negotiate; 4874 4875 KASSERT(mutex_owned(&adapter->core_mtx)); 4876 4877 IXGBE_EVC_ADD(&adapter->msf_workev, 1); 4878 4879 autoneg = hw->phy.autoneg_advertised; 4880 if ((!autoneg) && (hw->mac.ops.get_link_capabilities)) 4881 hw->mac.ops.get_link_capabilities(hw, &autoneg, &negotiate); 4882 if (hw->mac.ops.setup_link) 4883 hw->mac.ops.setup_link(hw, autoneg, TRUE); 4884 } /* ixgbe_handle_msf */ 4885 4886 /************************************************************************ 4887 * ixgbe_handle_phy - Tasklet for external PHY interrupts 4888 ************************************************************************/ 4889 static void 4890 ixgbe_handle_phy(void *context) 4891 { 4892 struct adapter *adapter = context; 4893 struct ixgbe_hw *hw = &adapter->hw; 4894 int error; 4895 4896 KASSERT(mutex_owned(&adapter->core_mtx)); 4897 4898 IXGBE_EVC_ADD(&adapter->phy_workev, 1); 4899 error = hw->phy.ops.handle_lasi(hw); 4900 if (error == IXGBE_ERR_OVERTEMP) 4901 device_printf(adapter->dev, 4902 "CRITICAL: EXTERNAL PHY OVER TEMP!! " 4903 " PHY will downshift to lower power state!\n"); 4904 else if (error) 4905 device_printf(adapter->dev, 4906 "Error handling LASI interrupt: %d\n", error); 4907 } /* ixgbe_handle_phy */ 4908 4909 static void 4910 ixgbe_handle_admin(struct work *wk, void *context) 4911 { 4912 struct adapter *adapter = context; 4913 struct ifnet *ifp = adapter->ifp; 4914 struct ixgbe_hw *hw = &adapter->hw; 4915 u32 task_requests; 4916 u32 eims_enable = 0; 4917 4918 mutex_enter(&adapter->admin_mtx); 4919 adapter->admin_pending = 0; 4920 task_requests = adapter->task_requests; 4921 adapter->task_requests = 0; 4922 mutex_exit(&adapter->admin_mtx); 4923 4924 /* 4925 * Hold the IFNET_LOCK across this entire call. This will 4926 * prevent additional changes to adapter->phy_layer 4927 * and serialize calls to this tasklet. We cannot hold the 4928 * CORE_LOCK while calling into the ifmedia functions as 4929 * they call ifmedia_lock() and the lock is CORE_LOCK. 4930 */ 4931 IFNET_LOCK(ifp); 4932 IXGBE_CORE_LOCK(adapter); 4933 if ((task_requests & IXGBE_REQUEST_TASK_LSC) != 0) { 4934 ixgbe_handle_link(adapter); 4935 eims_enable |= IXGBE_EIMS_LSC; 4936 } 4937 if ((task_requests & IXGBE_REQUEST_TASK_MOD_WOI) != 0) { 4938 ixgbe_handle_mod(adapter, false); 4939 } 4940 if ((task_requests & IXGBE_REQUEST_TASK_MOD) != 0) { 4941 ixgbe_handle_mod(adapter, true); 4942 if (hw->mac.type >= ixgbe_mac_X540) 4943 eims_enable |= IXGBE_EICR_GPI_SDP0_X540; 4944 else 4945 eims_enable |= IXGBE_EICR_GPI_SDP2_BY_MAC(hw); 4946 } 4947 if ((task_requests 4948 & (IXGBE_REQUEST_TASK_MSF_WOI | IXGBE_REQUEST_TASK_MSF)) != 0) { 4949 ixgbe_handle_msf(adapter); 4950 if (((task_requests & IXGBE_REQUEST_TASK_MSF) != 0) && 4951 (hw->mac.type == ixgbe_mac_82599EB)) 4952 eims_enable |= IXGBE_EIMS_GPI_SDP1_BY_MAC(hw); 4953 } 4954 if ((task_requests & IXGBE_REQUEST_TASK_PHY) != 0) { 4955 ixgbe_handle_phy(adapter); 4956 eims_enable |= IXGBE_EICR_GPI_SDP0_X540; 4957 } 4958 if ((task_requests & IXGBE_REQUEST_TASK_FDIR) != 0) { 4959 ixgbe_reinit_fdir(adapter); 4960 eims_enable |= IXGBE_EIMS_FLOW_DIR; 4961 } 4962 #if 0 /* notyet */ 4963 if ((task_requests & IXGBE_REQUEST_TASK_MBX) != 0) { 4964 ixgbe_handle_mbx(adapter); 4965 eims_enable |= IXGBE_EIMS_MAILBOX; 4966 } 4967 #endif 4968 IXGBE_WRITE_REG(hw, IXGBE_EIMS, eims_enable); 4969 4970 IXGBE_CORE_UNLOCK(adapter); 4971 IFNET_UNLOCK(ifp); 4972 } /* ixgbe_handle_admin */ 4973 4974 static void 4975 ixgbe_ifstop(struct ifnet *ifp, int disable) 4976 { 4977 struct adapter *adapter = ifp->if_softc; 4978 4979 IXGBE_CORE_LOCK(adapter); 4980 ixgbe_stop_locked(adapter); 4981 IXGBE_CORE_UNLOCK(adapter); 4982 4983 workqueue_wait(adapter->timer_wq, &adapter->timer_wc); 4984 atomic_store_relaxed(&adapter->timer_pending, 0); 4985 } 4986 4987 /************************************************************************ 4988 * ixgbe_stop_locked - Stop the hardware 4989 * 4990 * Disables all traffic on the adapter by issuing a 4991 * global reset on the MAC and deallocates TX/RX buffers. 4992 ************************************************************************/ 4993 static void 4994 ixgbe_stop_locked(void *arg) 4995 { 4996 struct ifnet *ifp; 4997 struct adapter *adapter = arg; 4998 struct ixgbe_hw *hw = &adapter->hw; 4999 5000 ifp = adapter->ifp; 5001 5002 KASSERT(mutex_owned(&adapter->core_mtx)); 5003 5004 INIT_DEBUGOUT("ixgbe_stop_locked: begin\n"); 5005 ixgbe_disable_intr(adapter); 5006 callout_stop(&adapter->timer); 5007 5008 /* Don't schedule workqueues. */ 5009 adapter->schedule_wqs_ok = false; 5010 5011 /* Let the stack know...*/ 5012 ifp->if_flags &= ~IFF_RUNNING; 5013 5014 ixgbe_reset_hw(hw); 5015 hw->adapter_stopped = FALSE; 5016 ixgbe_stop_adapter(hw); 5017 if (hw->mac.type == ixgbe_mac_82599EB) 5018 ixgbe_stop_mac_link_on_d3_82599(hw); 5019 /* Turn off the laser - noop with no optics */ 5020 ixgbe_disable_tx_laser(hw); 5021 5022 /* Update the stack */ 5023 adapter->link_up = FALSE; 5024 ixgbe_update_link_status(adapter); 5025 5026 /* reprogram the RAR[0] in case user changed it. */ 5027 ixgbe_set_rar(&adapter->hw, 0, adapter->hw.mac.addr, 0, IXGBE_RAH_AV); 5028 5029 return; 5030 } /* ixgbe_stop_locked */ 5031 5032 /************************************************************************ 5033 * ixgbe_update_link_status - Update OS on link state 5034 * 5035 * Note: Only updates the OS on the cached link state. 5036 * The real check of the hardware only happens with 5037 * a link interrupt. 5038 ************************************************************************/ 5039 static void 5040 ixgbe_update_link_status(struct adapter *adapter) 5041 { 5042 struct ifnet *ifp = adapter->ifp; 5043 device_t dev = adapter->dev; 5044 struct ixgbe_hw *hw = &adapter->hw; 5045 5046 KASSERT(mutex_owned(&adapter->core_mtx)); 5047 5048 if (adapter->link_up) { 5049 if (adapter->link_active != LINK_STATE_UP) { 5050 /* 5051 * To eliminate influence of the previous state 5052 * in the same way as ixgbe_init_locked(). 5053 */ 5054 struct ix_queue *que = adapter->queues; 5055 for (int i = 0; i < adapter->num_queues; i++, que++) 5056 que->eitr_setting = 0; 5057 5058 if (adapter->link_speed == IXGBE_LINK_SPEED_10GB_FULL){ 5059 /* 5060 * Discard count for both MAC Local Fault and 5061 * Remote Fault because those registers are 5062 * valid only when the link speed is up and 5063 * 10Gbps. 5064 */ 5065 IXGBE_READ_REG(hw, IXGBE_MLFC); 5066 IXGBE_READ_REG(hw, IXGBE_MRFC); 5067 } 5068 5069 if (bootverbose) { 5070 const char *bpsmsg; 5071 5072 switch (adapter->link_speed) { 5073 case IXGBE_LINK_SPEED_10GB_FULL: 5074 bpsmsg = "10 Gbps"; 5075 break; 5076 case IXGBE_LINK_SPEED_5GB_FULL: 5077 bpsmsg = "5 Gbps"; 5078 break; 5079 case IXGBE_LINK_SPEED_2_5GB_FULL: 5080 bpsmsg = "2.5 Gbps"; 5081 break; 5082 case IXGBE_LINK_SPEED_1GB_FULL: 5083 bpsmsg = "1 Gbps"; 5084 break; 5085 case IXGBE_LINK_SPEED_100_FULL: 5086 bpsmsg = "100 Mbps"; 5087 break; 5088 case IXGBE_LINK_SPEED_10_FULL: 5089 bpsmsg = "10 Mbps"; 5090 break; 5091 default: 5092 bpsmsg = "unknown speed"; 5093 break; 5094 } 5095 device_printf(dev, "Link is up %s %s \n", 5096 bpsmsg, "Full Duplex"); 5097 } 5098 adapter->link_active = LINK_STATE_UP; 5099 /* Update any Flow Control changes */ 5100 ixgbe_fc_enable(&adapter->hw); 5101 /* Update DMA coalescing config */ 5102 ixgbe_config_dmac(adapter); 5103 if_link_state_change(ifp, LINK_STATE_UP); 5104 5105 if (adapter->feat_en & IXGBE_FEATURE_SRIOV) 5106 ixgbe_ping_all_vfs(adapter); 5107 } 5108 } else { 5109 /* 5110 * Do it when link active changes to DOWN. i.e. 5111 * a) LINK_STATE_UNKNOWN -> LINK_STATE_DOWN 5112 * b) LINK_STATE_UP -> LINK_STATE_DOWN 5113 */ 5114 if (adapter->link_active != LINK_STATE_DOWN) { 5115 if (bootverbose) 5116 device_printf(dev, "Link is Down\n"); 5117 if_link_state_change(ifp, LINK_STATE_DOWN); 5118 adapter->link_active = LINK_STATE_DOWN; 5119 if (adapter->feat_en & IXGBE_FEATURE_SRIOV) 5120 ixgbe_ping_all_vfs(adapter); 5121 ixgbe_drain_all(adapter); 5122 } 5123 } 5124 } /* ixgbe_update_link_status */ 5125 5126 /************************************************************************ 5127 * ixgbe_config_dmac - Configure DMA Coalescing 5128 ************************************************************************/ 5129 static void 5130 ixgbe_config_dmac(struct adapter *adapter) 5131 { 5132 struct ixgbe_hw *hw = &adapter->hw; 5133 struct ixgbe_dmac_config *dcfg = &hw->mac.dmac_config; 5134 5135 if (hw->mac.type < ixgbe_mac_X550 || !hw->mac.ops.dmac_config) 5136 return; 5137 5138 if (dcfg->watchdog_timer ^ adapter->dmac || 5139 dcfg->link_speed ^ adapter->link_speed) { 5140 dcfg->watchdog_timer = adapter->dmac; 5141 dcfg->fcoe_en = false; 5142 dcfg->link_speed = adapter->link_speed; 5143 dcfg->num_tcs = 1; 5144 5145 INIT_DEBUGOUT2("dmac settings: watchdog %d, link speed %d\n", 5146 dcfg->watchdog_timer, dcfg->link_speed); 5147 5148 hw->mac.ops.dmac_config(hw); 5149 } 5150 } /* ixgbe_config_dmac */ 5151 5152 /************************************************************************ 5153 * ixgbe_enable_intr 5154 ************************************************************************/ 5155 static void 5156 ixgbe_enable_intr(struct adapter *adapter) 5157 { 5158 struct ixgbe_hw *hw = &adapter->hw; 5159 struct ix_queue *que = adapter->queues; 5160 u32 mask, fwsm; 5161 5162 mask = (IXGBE_EIMS_ENABLE_MASK & ~IXGBE_EIMS_RTX_QUEUE); 5163 5164 switch (adapter->hw.mac.type) { 5165 case ixgbe_mac_82599EB: 5166 mask |= IXGBE_EIMS_ECC; 5167 /* Temperature sensor on some adapters */ 5168 mask |= IXGBE_EIMS_GPI_SDP0; 5169 /* SFP+ (RX_LOS_N & MOD_ABS_N) */ 5170 mask |= IXGBE_EIMS_GPI_SDP1; 5171 mask |= IXGBE_EIMS_GPI_SDP2; 5172 break; 5173 case ixgbe_mac_X540: 5174 /* Detect if Thermal Sensor is enabled */ 5175 fwsm = IXGBE_READ_REG(hw, IXGBE_FWSM); 5176 if (fwsm & IXGBE_FWSM_TS_ENABLED) 5177 mask |= IXGBE_EIMS_TS; 5178 mask |= IXGBE_EIMS_ECC; 5179 break; 5180 case ixgbe_mac_X550: 5181 /* MAC thermal sensor is automatically enabled */ 5182 mask |= IXGBE_EIMS_TS; 5183 mask |= IXGBE_EIMS_ECC; 5184 break; 5185 case ixgbe_mac_X550EM_x: 5186 case ixgbe_mac_X550EM_a: 5187 /* Some devices use SDP0 for important information */ 5188 if (hw->device_id == IXGBE_DEV_ID_X550EM_X_SFP || 5189 hw->device_id == IXGBE_DEV_ID_X550EM_A_SFP || 5190 hw->device_id == IXGBE_DEV_ID_X550EM_A_SFP_N || 5191 hw->device_id == IXGBE_DEV_ID_X550EM_X_10G_T) 5192 mask |= IXGBE_EIMS_GPI_SDP0_BY_MAC(hw); 5193 if (hw->phy.type == ixgbe_phy_x550em_ext_t) 5194 mask |= IXGBE_EICR_GPI_SDP0_X540; 5195 mask |= IXGBE_EIMS_ECC; 5196 break; 5197 default: 5198 break; 5199 } 5200 5201 /* Enable Fan Failure detection */ 5202 if (adapter->feat_en & IXGBE_FEATURE_FAN_FAIL) 5203 mask |= IXGBE_EIMS_GPI_SDP1; 5204 /* Enable SR-IOV */ 5205 if (adapter->feat_en & IXGBE_FEATURE_SRIOV) 5206 mask |= IXGBE_EIMS_MAILBOX; 5207 /* Enable Flow Director */ 5208 if (adapter->feat_en & IXGBE_FEATURE_FDIR) 5209 mask |= IXGBE_EIMS_FLOW_DIR; 5210 5211 IXGBE_WRITE_REG(hw, IXGBE_EIMS, mask); 5212 5213 /* With MSI-X we use auto clear */ 5214 if ((adapter->feat_en & IXGBE_FEATURE_MSIX) != 0) { 5215 /* 5216 * We use auto clear for RTX_QUEUE only. Don't use other 5217 * interrupts (e.g. link interrupt). BTW, we don't use 5218 * TCP_TIMER interrupt itself. 5219 */ 5220 IXGBE_WRITE_REG(hw, IXGBE_EIAC, IXGBE_EIMS_RTX_QUEUE); 5221 } 5222 5223 /* 5224 * Now enable all queues, this is done separately to 5225 * allow for handling the extended (beyond 32) MSI-X 5226 * vectors that can be used by 82599 5227 */ 5228 for (int i = 0; i < adapter->num_queues; i++, que++) 5229 ixgbe_enable_queue(adapter, que->msix); 5230 5231 IXGBE_WRITE_FLUSH(hw); 5232 5233 } /* ixgbe_enable_intr */ 5234 5235 /************************************************************************ 5236 * ixgbe_disable_intr_internal 5237 ************************************************************************/ 5238 static void 5239 ixgbe_disable_intr_internal(struct adapter *adapter, bool nestok) 5240 { 5241 struct ix_queue *que = adapter->queues; 5242 5243 /* disable interrupts other than queues */ 5244 IXGBE_WRITE_REG(&adapter->hw, IXGBE_EIMC, ~IXGBE_EIMC_RTX_QUEUE); 5245 5246 if ((adapter->feat_en & IXGBE_FEATURE_MSIX) != 0) 5247 IXGBE_WRITE_REG(&adapter->hw, IXGBE_EIAC, 0); 5248 5249 for (int i = 0; i < adapter->num_queues; i++, que++) 5250 ixgbe_disable_queue_internal(adapter, que->msix, nestok); 5251 5252 IXGBE_WRITE_FLUSH(&adapter->hw); 5253 5254 } /* ixgbe_do_disable_intr_internal */ 5255 5256 /************************************************************************ 5257 * ixgbe_disable_intr 5258 ************************************************************************/ 5259 static void 5260 ixgbe_disable_intr(struct adapter *adapter) 5261 { 5262 5263 ixgbe_disable_intr_internal(adapter, true); 5264 } /* ixgbe_disable_intr */ 5265 5266 /************************************************************************ 5267 * ixgbe_ensure_disabled_intr 5268 ************************************************************************/ 5269 void 5270 ixgbe_ensure_disabled_intr(struct adapter *adapter) 5271 { 5272 5273 ixgbe_disable_intr_internal(adapter, false); 5274 } /* ixgbe_ensure_disabled_intr */ 5275 5276 /************************************************************************ 5277 * ixgbe_legacy_irq - Legacy Interrupt Service routine 5278 ************************************************************************/ 5279 static int 5280 ixgbe_legacy_irq(void *arg) 5281 { 5282 struct ix_queue *que = arg; 5283 struct adapter *adapter = que->adapter; 5284 struct ixgbe_hw *hw = &adapter->hw; 5285 struct ifnet *ifp = adapter->ifp; 5286 struct tx_ring *txr = adapter->tx_rings; 5287 u32 eicr; 5288 u32 eims_orig; 5289 u32 eims_enable = 0; 5290 u32 eims_disable = 0; 5291 5292 eims_orig = IXGBE_READ_REG(hw, IXGBE_EIMS); 5293 /* 5294 * Silicon errata #26 on 82598. Disable all interrupts before reading 5295 * EICR. 5296 */ 5297 IXGBE_WRITE_REG(hw, IXGBE_EIMC, IXGBE_IRQ_CLEAR_MASK); 5298 5299 /* Read and clear EICR */ 5300 eicr = IXGBE_READ_REG(hw, IXGBE_EICR); 5301 5302 if (eicr == 0) { 5303 IXGBE_EVC_ADD(&adapter->stats.pf.intzero, 1); 5304 IXGBE_WRITE_REG(hw, IXGBE_EIMS, eims_orig); 5305 return 0; 5306 } 5307 IXGBE_EVC_ADD(&adapter->stats.pf.legint, 1); 5308 5309 /* Queue (0) intr */ 5310 if (((ifp->if_flags & IFF_RUNNING) != 0) && 5311 (eicr & IXGBE_EIMC_RTX_QUEUE) != 0) { 5312 IXGBE_EVC_ADD(&que->irqs, 1); 5313 5314 /* 5315 * The same as ixgbe_msix_que() about 5316 * "que->txrx_use_workqueue". 5317 */ 5318 que->txrx_use_workqueue = adapter->txrx_use_workqueue; 5319 5320 IXGBE_TX_LOCK(txr); 5321 ixgbe_txeof(txr); 5322 #ifdef notyet 5323 if (!ixgbe_ring_empty(ifp, txr->br)) 5324 ixgbe_start_locked(ifp, txr); 5325 #endif 5326 IXGBE_TX_UNLOCK(txr); 5327 5328 IXGBE_EVC_ADD(&que->req, 1); 5329 ixgbe_sched_handle_que(adapter, que); 5330 /* Disable queue 0 interrupt */ 5331 eims_disable |= 1UL << 0; 5332 5333 } else 5334 eims_enable |= IXGBE_EIMC_RTX_QUEUE; 5335 5336 ixgbe_intr_admin_common(adapter, eicr, &eims_disable); 5337 5338 /* Re-enable some interrupts */ 5339 IXGBE_WRITE_REG(hw, IXGBE_EIMS, 5340 (eims_orig & ~eims_disable) | eims_enable); 5341 5342 return 1; 5343 } /* ixgbe_legacy_irq */ 5344 5345 /************************************************************************ 5346 * ixgbe_free_pciintr_resources 5347 ************************************************************************/ 5348 static void 5349 ixgbe_free_pciintr_resources(struct adapter *adapter) 5350 { 5351 struct ix_queue *que = adapter->queues; 5352 int rid; 5353 5354 /* 5355 * Release all msix queue resources: 5356 */ 5357 for (int i = 0; i < adapter->num_queues; i++, que++) { 5358 if (que->res != NULL) { 5359 pci_intr_disestablish(adapter->osdep.pc, 5360 adapter->osdep.ihs[i]); 5361 adapter->osdep.ihs[i] = NULL; 5362 } 5363 } 5364 5365 /* Clean the Legacy or Link interrupt last */ 5366 if (adapter->vector) /* we are doing MSIX */ 5367 rid = adapter->vector; 5368 else 5369 rid = 0; 5370 5371 if (adapter->osdep.ihs[rid] != NULL) { 5372 pci_intr_disestablish(adapter->osdep.pc, 5373 adapter->osdep.ihs[rid]); 5374 adapter->osdep.ihs[rid] = NULL; 5375 } 5376 5377 if (adapter->osdep.intrs != NULL) { 5378 pci_intr_release(adapter->osdep.pc, adapter->osdep.intrs, 5379 adapter->osdep.nintrs); 5380 adapter->osdep.intrs = NULL; 5381 } 5382 } /* ixgbe_free_pciintr_resources */ 5383 5384 /************************************************************************ 5385 * ixgbe_free_pci_resources 5386 ************************************************************************/ 5387 static void 5388 ixgbe_free_pci_resources(struct adapter *adapter) 5389 { 5390 5391 ixgbe_free_pciintr_resources(adapter); 5392 5393 if (adapter->osdep.mem_size != 0) { 5394 bus_space_unmap(adapter->osdep.mem_bus_space_tag, 5395 adapter->osdep.mem_bus_space_handle, 5396 adapter->osdep.mem_size); 5397 } 5398 5399 } /* ixgbe_free_pci_resources */ 5400 5401 /************************************************************************ 5402 * ixgbe_sysctl_flowcntl 5403 * 5404 * SYSCTL wrapper around setting Flow Control 5405 ************************************************************************/ 5406 static int 5407 ixgbe_sysctl_flowcntl(SYSCTLFN_ARGS) 5408 { 5409 struct sysctlnode node = *rnode; 5410 struct adapter *adapter = (struct adapter *)node.sysctl_data; 5411 int error, fc; 5412 5413 if (ixgbe_fw_recovery_mode_swflag(adapter)) 5414 return (EPERM); 5415 5416 fc = adapter->hw.fc.current_mode; 5417 node.sysctl_data = &fc; 5418 error = sysctl_lookup(SYSCTLFN_CALL(&node)); 5419 if (error != 0 || newp == NULL) 5420 return error; 5421 5422 /* Don't bother if it's not changed */ 5423 if (fc == adapter->hw.fc.current_mode) 5424 return (0); 5425 5426 return ixgbe_set_flowcntl(adapter, fc); 5427 } /* ixgbe_sysctl_flowcntl */ 5428 5429 /************************************************************************ 5430 * ixgbe_set_flowcntl - Set flow control 5431 * 5432 * Flow control values: 5433 * 0 - off 5434 * 1 - rx pause 5435 * 2 - tx pause 5436 * 3 - full 5437 ************************************************************************/ 5438 static int 5439 ixgbe_set_flowcntl(struct adapter *adapter, int fc) 5440 { 5441 switch (fc) { 5442 case ixgbe_fc_rx_pause: 5443 case ixgbe_fc_tx_pause: 5444 case ixgbe_fc_full: 5445 adapter->hw.fc.requested_mode = fc; 5446 if (adapter->num_queues > 1) 5447 ixgbe_disable_rx_drop(adapter); 5448 break; 5449 case ixgbe_fc_none: 5450 adapter->hw.fc.requested_mode = ixgbe_fc_none; 5451 if (adapter->num_queues > 1) 5452 ixgbe_enable_rx_drop(adapter); 5453 break; 5454 default: 5455 return (EINVAL); 5456 } 5457 5458 #if 0 /* XXX NetBSD */ 5459 /* Don't autoneg if forcing a value */ 5460 adapter->hw.fc.disable_fc_autoneg = TRUE; 5461 #endif 5462 ixgbe_fc_enable(&adapter->hw); 5463 5464 return (0); 5465 } /* ixgbe_set_flowcntl */ 5466 5467 /************************************************************************ 5468 * ixgbe_enable_rx_drop 5469 * 5470 * Enable the hardware to drop packets when the buffer is 5471 * full. This is useful with multiqueue, so that no single 5472 * queue being full stalls the entire RX engine. We only 5473 * enable this when Multiqueue is enabled AND Flow Control 5474 * is disabled. 5475 ************************************************************************/ 5476 static void 5477 ixgbe_enable_rx_drop(struct adapter *adapter) 5478 { 5479 struct ixgbe_hw *hw = &adapter->hw; 5480 struct rx_ring *rxr; 5481 u32 srrctl; 5482 5483 for (int i = 0; i < adapter->num_queues; i++) { 5484 rxr = &adapter->rx_rings[i]; 5485 srrctl = IXGBE_READ_REG(hw, IXGBE_SRRCTL(rxr->me)); 5486 srrctl |= IXGBE_SRRCTL_DROP_EN; 5487 IXGBE_WRITE_REG(hw, IXGBE_SRRCTL(rxr->me), srrctl); 5488 } 5489 5490 /* enable drop for each vf */ 5491 for (int i = 0; i < adapter->num_vfs; i++) { 5492 IXGBE_WRITE_REG(hw, IXGBE_QDE, 5493 (IXGBE_QDE_WRITE | (i << IXGBE_QDE_IDX_SHIFT) | 5494 IXGBE_QDE_ENABLE)); 5495 } 5496 } /* ixgbe_enable_rx_drop */ 5497 5498 /************************************************************************ 5499 * ixgbe_disable_rx_drop 5500 ************************************************************************/ 5501 static void 5502 ixgbe_disable_rx_drop(struct adapter *adapter) 5503 { 5504 struct ixgbe_hw *hw = &adapter->hw; 5505 struct rx_ring *rxr; 5506 u32 srrctl; 5507 5508 for (int i = 0; i < adapter->num_queues; i++) { 5509 rxr = &adapter->rx_rings[i]; 5510 srrctl = IXGBE_READ_REG(hw, IXGBE_SRRCTL(rxr->me)); 5511 srrctl &= ~IXGBE_SRRCTL_DROP_EN; 5512 IXGBE_WRITE_REG(hw, IXGBE_SRRCTL(rxr->me), srrctl); 5513 } 5514 5515 /* disable drop for each vf */ 5516 for (int i = 0; i < adapter->num_vfs; i++) { 5517 IXGBE_WRITE_REG(hw, IXGBE_QDE, 5518 (IXGBE_QDE_WRITE | (i << IXGBE_QDE_IDX_SHIFT))); 5519 } 5520 } /* ixgbe_disable_rx_drop */ 5521 5522 /************************************************************************ 5523 * ixgbe_sysctl_advertise 5524 * 5525 * SYSCTL wrapper around setting advertised speed 5526 ************************************************************************/ 5527 static int 5528 ixgbe_sysctl_advertise(SYSCTLFN_ARGS) 5529 { 5530 struct sysctlnode node = *rnode; 5531 struct adapter *adapter = (struct adapter *)node.sysctl_data; 5532 int error = 0, advertise; 5533 5534 if (ixgbe_fw_recovery_mode_swflag(adapter)) 5535 return (EPERM); 5536 5537 advertise = adapter->advertise; 5538 node.sysctl_data = &advertise; 5539 error = sysctl_lookup(SYSCTLFN_CALL(&node)); 5540 if (error != 0 || newp == NULL) 5541 return error; 5542 5543 return ixgbe_set_advertise(adapter, advertise); 5544 } /* ixgbe_sysctl_advertise */ 5545 5546 /************************************************************************ 5547 * ixgbe_set_advertise - Control advertised link speed 5548 * 5549 * Flags: 5550 * 0x00 - Default (all capable link speed) 5551 * 0x1 - advertise 100 Mb 5552 * 0x2 - advertise 1G 5553 * 0x4 - advertise 10G 5554 * 0x8 - advertise 10 Mb (yes, Mb) 5555 * 0x10 - advertise 2.5G 5556 * 0x20 - advertise 5G 5557 ************************************************************************/ 5558 static int 5559 ixgbe_set_advertise(struct adapter *adapter, int advertise) 5560 { 5561 device_t dev; 5562 struct ixgbe_hw *hw; 5563 ixgbe_link_speed speed = 0; 5564 ixgbe_link_speed link_caps = 0; 5565 s32 err = IXGBE_NOT_IMPLEMENTED; 5566 bool negotiate = FALSE; 5567 5568 /* Checks to validate new value */ 5569 if (adapter->advertise == advertise) /* no change */ 5570 return (0); 5571 5572 dev = adapter->dev; 5573 hw = &adapter->hw; 5574 5575 /* No speed changes for backplane media */ 5576 if (hw->phy.media_type == ixgbe_media_type_backplane) 5577 return (ENODEV); 5578 5579 if (!((hw->phy.media_type == ixgbe_media_type_copper) || 5580 (hw->phy.multispeed_fiber))) { 5581 device_printf(dev, 5582 "Advertised speed can only be set on copper or " 5583 "multispeed fiber media types.\n"); 5584 return (EINVAL); 5585 } 5586 5587 if (advertise < 0x0 || advertise > 0x3f) { 5588 device_printf(dev, "Invalid advertised speed; valid modes are 0x0 through 0x3f\n"); 5589 return (EINVAL); 5590 } 5591 5592 if (hw->mac.ops.get_link_capabilities) { 5593 err = hw->mac.ops.get_link_capabilities(hw, &link_caps, 5594 &negotiate); 5595 if (err != IXGBE_SUCCESS) { 5596 device_printf(dev, "Unable to determine supported advertise speeds\n"); 5597 return (ENODEV); 5598 } 5599 } 5600 5601 /* Set new value and report new advertised mode */ 5602 if (advertise & 0x1) { 5603 if (!(link_caps & IXGBE_LINK_SPEED_100_FULL)) { 5604 device_printf(dev, "Interface does not support 100Mb advertised speed\n"); 5605 return (EINVAL); 5606 } 5607 speed |= IXGBE_LINK_SPEED_100_FULL; 5608 } 5609 if (advertise & 0x2) { 5610 if (!(link_caps & IXGBE_LINK_SPEED_1GB_FULL)) { 5611 device_printf(dev, "Interface does not support 1Gb advertised speed\n"); 5612 return (EINVAL); 5613 } 5614 speed |= IXGBE_LINK_SPEED_1GB_FULL; 5615 } 5616 if (advertise & 0x4) { 5617 if (!(link_caps & IXGBE_LINK_SPEED_10GB_FULL)) { 5618 device_printf(dev, "Interface does not support 10Gb advertised speed\n"); 5619 return (EINVAL); 5620 } 5621 speed |= IXGBE_LINK_SPEED_10GB_FULL; 5622 } 5623 if (advertise & 0x8) { 5624 if (!(link_caps & IXGBE_LINK_SPEED_10_FULL)) { 5625 device_printf(dev, "Interface does not support 10Mb advertised speed\n"); 5626 return (EINVAL); 5627 } 5628 speed |= IXGBE_LINK_SPEED_10_FULL; 5629 } 5630 if (advertise & 0x10) { 5631 if (!(link_caps & IXGBE_LINK_SPEED_2_5GB_FULL)) { 5632 device_printf(dev, "Interface does not support 2.5Gb advertised speed\n"); 5633 return (EINVAL); 5634 } 5635 speed |= IXGBE_LINK_SPEED_2_5GB_FULL; 5636 } 5637 if (advertise & 0x20) { 5638 if (!(link_caps & IXGBE_LINK_SPEED_5GB_FULL)) { 5639 device_printf(dev, "Interface does not support 5Gb advertised speed\n"); 5640 return (EINVAL); 5641 } 5642 speed |= IXGBE_LINK_SPEED_5GB_FULL; 5643 } 5644 if (advertise == 0) 5645 speed = link_caps; /* All capable link speed */ 5646 5647 hw->mac.autotry_restart = TRUE; 5648 hw->mac.ops.setup_link(hw, speed, TRUE); 5649 adapter->advertise = advertise; 5650 5651 return (0); 5652 } /* ixgbe_set_advertise */ 5653 5654 /************************************************************************ 5655 * ixgbe_get_default_advertise - Get default advertised speed settings 5656 * 5657 * Formatted for sysctl usage. 5658 * Flags: 5659 * 0x1 - advertise 100 Mb 5660 * 0x2 - advertise 1G 5661 * 0x4 - advertise 10G 5662 * 0x8 - advertise 10 Mb (yes, Mb) 5663 * 0x10 - advertise 2.5G 5664 * 0x20 - advertise 5G 5665 ************************************************************************/ 5666 static int 5667 ixgbe_get_default_advertise(struct adapter *adapter) 5668 { 5669 struct ixgbe_hw *hw = &adapter->hw; 5670 int speed; 5671 ixgbe_link_speed link_caps = 0; 5672 s32 err; 5673 bool negotiate = FALSE; 5674 5675 /* 5676 * Advertised speed means nothing unless it's copper or 5677 * multi-speed fiber 5678 */ 5679 if (!(hw->phy.media_type == ixgbe_media_type_copper) && 5680 !(hw->phy.multispeed_fiber)) 5681 return (0); 5682 5683 err = hw->mac.ops.get_link_capabilities(hw, &link_caps, &negotiate); 5684 if (err != IXGBE_SUCCESS) 5685 return (0); 5686 5687 speed = 5688 ((link_caps & IXGBE_LINK_SPEED_10GB_FULL) ? 0x4 : 0) | 5689 ((link_caps & IXGBE_LINK_SPEED_5GB_FULL) ? 0x20 : 0) | 5690 ((link_caps & IXGBE_LINK_SPEED_2_5GB_FULL) ? 0x10 : 0) | 5691 ((link_caps & IXGBE_LINK_SPEED_1GB_FULL) ? 0x2 : 0) | 5692 ((link_caps & IXGBE_LINK_SPEED_100_FULL) ? 0x1 : 0) | 5693 ((link_caps & IXGBE_LINK_SPEED_10_FULL) ? 0x8 : 0); 5694 5695 return speed; 5696 } /* ixgbe_get_default_advertise */ 5697 5698 /************************************************************************ 5699 * ixgbe_sysctl_dmac - Manage DMA Coalescing 5700 * 5701 * Control values: 5702 * 0/1 - off / on (use default value of 1000) 5703 * 5704 * Legal timer values are: 5705 * 50,100,250,500,1000,2000,5000,10000 5706 * 5707 * Turning off interrupt moderation will also turn this off. 5708 ************************************************************************/ 5709 static int 5710 ixgbe_sysctl_dmac(SYSCTLFN_ARGS) 5711 { 5712 struct sysctlnode node = *rnode; 5713 struct adapter *adapter = (struct adapter *)node.sysctl_data; 5714 struct ifnet *ifp = adapter->ifp; 5715 int error; 5716 int newval; 5717 5718 if (ixgbe_fw_recovery_mode_swflag(adapter)) 5719 return (EPERM); 5720 5721 newval = adapter->dmac; 5722 node.sysctl_data = &newval; 5723 error = sysctl_lookup(SYSCTLFN_CALL(&node)); 5724 if ((error) || (newp == NULL)) 5725 return (error); 5726 5727 switch (newval) { 5728 case 0: 5729 /* Disabled */ 5730 adapter->dmac = 0; 5731 break; 5732 case 1: 5733 /* Enable and use default */ 5734 adapter->dmac = 1000; 5735 break; 5736 case 50: 5737 case 100: 5738 case 250: 5739 case 500: 5740 case 1000: 5741 case 2000: 5742 case 5000: 5743 case 10000: 5744 /* Legal values - allow */ 5745 adapter->dmac = newval; 5746 break; 5747 default: 5748 /* Do nothing, illegal value */ 5749 return (EINVAL); 5750 } 5751 5752 /* Re-initialize hardware if it's already running */ 5753 if (ifp->if_flags & IFF_RUNNING) 5754 if_init(ifp); 5755 5756 return (0); 5757 } 5758 5759 #ifdef IXGBE_DEBUG 5760 /************************************************************************ 5761 * ixgbe_sysctl_power_state 5762 * 5763 * Sysctl to test power states 5764 * Values: 5765 * 0 - set device to D0 5766 * 3 - set device to D3 5767 * (none) - get current device power state 5768 ************************************************************************/ 5769 static int 5770 ixgbe_sysctl_power_state(SYSCTLFN_ARGS) 5771 { 5772 #ifdef notyet 5773 struct sysctlnode node = *rnode; 5774 struct adapter *adapter = (struct adapter *)node.sysctl_data; 5775 device_t dev = adapter->dev; 5776 int curr_ps, new_ps, error = 0; 5777 5778 if (ixgbe_fw_recovery_mode_swflag(adapter)) 5779 return (EPERM); 5780 5781 curr_ps = new_ps = pci_get_powerstate(dev); 5782 5783 error = sysctl_lookup(SYSCTLFN_CALL(&node)); 5784 if ((error) || (req->newp == NULL)) 5785 return (error); 5786 5787 if (new_ps == curr_ps) 5788 return (0); 5789 5790 if (new_ps == 3 && curr_ps == 0) 5791 error = DEVICE_SUSPEND(dev); 5792 else if (new_ps == 0 && curr_ps == 3) 5793 error = DEVICE_RESUME(dev); 5794 else 5795 return (EINVAL); 5796 5797 device_printf(dev, "New state: %d\n", pci_get_powerstate(dev)); 5798 5799 return (error); 5800 #else 5801 return 0; 5802 #endif 5803 } /* ixgbe_sysctl_power_state */ 5804 #endif 5805 5806 /************************************************************************ 5807 * ixgbe_sysctl_wol_enable 5808 * 5809 * Sysctl to enable/disable the WoL capability, 5810 * if supported by the adapter. 5811 * 5812 * Values: 5813 * 0 - disabled 5814 * 1 - enabled 5815 ************************************************************************/ 5816 static int 5817 ixgbe_sysctl_wol_enable(SYSCTLFN_ARGS) 5818 { 5819 struct sysctlnode node = *rnode; 5820 struct adapter *adapter = (struct adapter *)node.sysctl_data; 5821 struct ixgbe_hw *hw = &adapter->hw; 5822 bool new_wol_enabled; 5823 int error = 0; 5824 5825 /* 5826 * It's not required to check recovery mode because this function never 5827 * touches hardware. 5828 */ 5829 new_wol_enabled = hw->wol_enabled; 5830 node.sysctl_data = &new_wol_enabled; 5831 error = sysctl_lookup(SYSCTLFN_CALL(&node)); 5832 if ((error) || (newp == NULL)) 5833 return (error); 5834 if (new_wol_enabled == hw->wol_enabled) 5835 return (0); 5836 5837 if (new_wol_enabled && !adapter->wol_support) 5838 return (ENODEV); 5839 else 5840 hw->wol_enabled = new_wol_enabled; 5841 5842 return (0); 5843 } /* ixgbe_sysctl_wol_enable */ 5844 5845 /************************************************************************ 5846 * ixgbe_sysctl_wufc - Wake Up Filter Control 5847 * 5848 * Sysctl to enable/disable the types of packets that the 5849 * adapter will wake up on upon receipt. 5850 * Flags: 5851 * 0x1 - Link Status Change 5852 * 0x2 - Magic Packet 5853 * 0x4 - Direct Exact 5854 * 0x8 - Directed Multicast 5855 * 0x10 - Broadcast 5856 * 0x20 - ARP/IPv4 Request Packet 5857 * 0x40 - Direct IPv4 Packet 5858 * 0x80 - Direct IPv6 Packet 5859 * 5860 * Settings not listed above will cause the sysctl to return an error. 5861 ************************************************************************/ 5862 static int 5863 ixgbe_sysctl_wufc(SYSCTLFN_ARGS) 5864 { 5865 struct sysctlnode node = *rnode; 5866 struct adapter *adapter = (struct adapter *)node.sysctl_data; 5867 int error = 0; 5868 u32 new_wufc; 5869 5870 /* 5871 * It's not required to check recovery mode because this function never 5872 * touches hardware. 5873 */ 5874 new_wufc = adapter->wufc; 5875 node.sysctl_data = &new_wufc; 5876 error = sysctl_lookup(SYSCTLFN_CALL(&node)); 5877 if ((error) || (newp == NULL)) 5878 return (error); 5879 if (new_wufc == adapter->wufc) 5880 return (0); 5881 5882 if (new_wufc & 0xffffff00) 5883 return (EINVAL); 5884 5885 new_wufc &= 0xff; 5886 new_wufc |= (0xffffff & adapter->wufc); 5887 adapter->wufc = new_wufc; 5888 5889 return (0); 5890 } /* ixgbe_sysctl_wufc */ 5891 5892 #ifdef IXGBE_DEBUG 5893 /************************************************************************ 5894 * ixgbe_sysctl_print_rss_config 5895 ************************************************************************/ 5896 static int 5897 ixgbe_sysctl_print_rss_config(SYSCTLFN_ARGS) 5898 { 5899 #ifdef notyet 5900 struct sysctlnode node = *rnode; 5901 struct adapter *adapter = (struct adapter *)node.sysctl_data; 5902 struct ixgbe_hw *hw = &adapter->hw; 5903 device_t dev = adapter->dev; 5904 struct sbuf *buf; 5905 int error = 0, reta_size; 5906 u32 reg; 5907 5908 if (ixgbe_fw_recovery_mode_swflag(adapter)) 5909 return (EPERM); 5910 5911 buf = sbuf_new_for_sysctl(NULL, NULL, 128, req); 5912 if (!buf) { 5913 device_printf(dev, "Could not allocate sbuf for output.\n"); 5914 return (ENOMEM); 5915 } 5916 5917 // TODO: use sbufs to make a string to print out 5918 /* Set multiplier for RETA setup and table size based on MAC */ 5919 switch (adapter->hw.mac.type) { 5920 case ixgbe_mac_X550: 5921 case ixgbe_mac_X550EM_x: 5922 case ixgbe_mac_X550EM_a: 5923 reta_size = 128; 5924 break; 5925 default: 5926 reta_size = 32; 5927 break; 5928 } 5929 5930 /* Print out the redirection table */ 5931 sbuf_cat(buf, "\n"); 5932 for (int i = 0; i < reta_size; i++) { 5933 if (i < 32) { 5934 reg = IXGBE_READ_REG(hw, IXGBE_RETA(i)); 5935 sbuf_printf(buf, "RETA(%2d): 0x%08x\n", i, reg); 5936 } else { 5937 reg = IXGBE_READ_REG(hw, IXGBE_ERETA(i - 32)); 5938 sbuf_printf(buf, "ERETA(%2d): 0x%08x\n", i - 32, reg); 5939 } 5940 } 5941 5942 // TODO: print more config 5943 5944 error = sbuf_finish(buf); 5945 if (error) 5946 device_printf(dev, "Error finishing sbuf: %d\n", error); 5947 5948 sbuf_delete(buf); 5949 #endif 5950 return (0); 5951 } /* ixgbe_sysctl_print_rss_config */ 5952 #endif /* IXGBE_DEBUG */ 5953 5954 /************************************************************************ 5955 * ixgbe_sysctl_phy_temp - Retrieve temperature of PHY 5956 * 5957 * For X552/X557-AT devices using an external PHY 5958 ************************************************************************/ 5959 static int 5960 ixgbe_sysctl_phy_temp(SYSCTLFN_ARGS) 5961 { 5962 struct sysctlnode node = *rnode; 5963 struct adapter *adapter = (struct adapter *)node.sysctl_data; 5964 struct ixgbe_hw *hw = &adapter->hw; 5965 int val; 5966 u16 reg; 5967 int error; 5968 5969 if (ixgbe_fw_recovery_mode_swflag(adapter)) 5970 return (EPERM); 5971 5972 if (hw->device_id != IXGBE_DEV_ID_X550EM_X_10G_T) { 5973 device_printf(adapter->dev, 5974 "Device has no supported external thermal sensor.\n"); 5975 return (ENODEV); 5976 } 5977 5978 if (hw->phy.ops.read_reg(hw, IXGBE_PHY_CURRENT_TEMP, 5979 IXGBE_MDIO_VENDOR_SPECIFIC_1_DEV_TYPE, ®)) { 5980 device_printf(adapter->dev, 5981 "Error reading from PHY's current temperature register\n"); 5982 return (EAGAIN); 5983 } 5984 5985 node.sysctl_data = &val; 5986 5987 /* Shift temp for output */ 5988 val = reg >> 8; 5989 5990 error = sysctl_lookup(SYSCTLFN_CALL(&node)); 5991 if ((error) || (newp == NULL)) 5992 return (error); 5993 5994 return (0); 5995 } /* ixgbe_sysctl_phy_temp */ 5996 5997 /************************************************************************ 5998 * ixgbe_sysctl_phy_overtemp_occurred 5999 * 6000 * Reports (directly from the PHY) whether the current PHY 6001 * temperature is over the overtemp threshold. 6002 ************************************************************************/ 6003 static int 6004 ixgbe_sysctl_phy_overtemp_occurred(SYSCTLFN_ARGS) 6005 { 6006 struct sysctlnode node = *rnode; 6007 struct adapter *adapter = (struct adapter *)node.sysctl_data; 6008 struct ixgbe_hw *hw = &adapter->hw; 6009 int val, error; 6010 u16 reg; 6011 6012 if (ixgbe_fw_recovery_mode_swflag(adapter)) 6013 return (EPERM); 6014 6015 if (hw->device_id != IXGBE_DEV_ID_X550EM_X_10G_T) { 6016 device_printf(adapter->dev, 6017 "Device has no supported external thermal sensor.\n"); 6018 return (ENODEV); 6019 } 6020 6021 if (hw->phy.ops.read_reg(hw, IXGBE_PHY_OVERTEMP_STATUS, 6022 IXGBE_MDIO_VENDOR_SPECIFIC_1_DEV_TYPE, ®)) { 6023 device_printf(adapter->dev, 6024 "Error reading from PHY's temperature status register\n"); 6025 return (EAGAIN); 6026 } 6027 6028 node.sysctl_data = &val; 6029 6030 /* Get occurrence bit */ 6031 val = !!(reg & 0x4000); 6032 6033 error = sysctl_lookup(SYSCTLFN_CALL(&node)); 6034 if ((error) || (newp == NULL)) 6035 return (error); 6036 6037 return (0); 6038 } /* ixgbe_sysctl_phy_overtemp_occurred */ 6039 6040 /************************************************************************ 6041 * ixgbe_sysctl_eee_state 6042 * 6043 * Sysctl to set EEE power saving feature 6044 * Values: 6045 * 0 - disable EEE 6046 * 1 - enable EEE 6047 * (none) - get current device EEE state 6048 ************************************************************************/ 6049 static int 6050 ixgbe_sysctl_eee_state(SYSCTLFN_ARGS) 6051 { 6052 struct sysctlnode node = *rnode; 6053 struct adapter *adapter = (struct adapter *)node.sysctl_data; 6054 struct ifnet *ifp = adapter->ifp; 6055 device_t dev = adapter->dev; 6056 int curr_eee, new_eee, error = 0; 6057 s32 retval; 6058 6059 if (ixgbe_fw_recovery_mode_swflag(adapter)) 6060 return (EPERM); 6061 6062 curr_eee = new_eee = !!(adapter->feat_en & IXGBE_FEATURE_EEE); 6063 node.sysctl_data = &new_eee; 6064 error = sysctl_lookup(SYSCTLFN_CALL(&node)); 6065 if ((error) || (newp == NULL)) 6066 return (error); 6067 6068 /* Nothing to do */ 6069 if (new_eee == curr_eee) 6070 return (0); 6071 6072 /* Not supported */ 6073 if (!(adapter->feat_cap & IXGBE_FEATURE_EEE)) 6074 return (EINVAL); 6075 6076 /* Bounds checking */ 6077 if ((new_eee < 0) || (new_eee > 1)) 6078 return (EINVAL); 6079 6080 retval = ixgbe_setup_eee(&adapter->hw, new_eee); 6081 if (retval) { 6082 device_printf(dev, "Error in EEE setup: 0x%08X\n", retval); 6083 return (EINVAL); 6084 } 6085 6086 /* Restart auto-neg */ 6087 if_init(ifp); 6088 6089 device_printf(dev, "New EEE state: %d\n", new_eee); 6090 6091 /* Cache new value */ 6092 if (new_eee) 6093 adapter->feat_en |= IXGBE_FEATURE_EEE; 6094 else 6095 adapter->feat_en &= ~IXGBE_FEATURE_EEE; 6096 6097 return (error); 6098 } /* ixgbe_sysctl_eee_state */ 6099 6100 #define PRINTQS(adapter, regname) \ 6101 do { \ 6102 struct ixgbe_hw *_hw = &(adapter)->hw; \ 6103 int _i; \ 6104 \ 6105 printf("%s: %s", device_xname((adapter)->dev), #regname); \ 6106 for (_i = 0; _i < (adapter)->num_queues; _i++) { \ 6107 printf((_i == 0) ? "\t" : " "); \ 6108 printf("%08x", IXGBE_READ_REG(_hw, \ 6109 IXGBE_##regname(_i))); \ 6110 } \ 6111 printf("\n"); \ 6112 } while (0) 6113 6114 /************************************************************************ 6115 * ixgbe_print_debug_info 6116 * 6117 * Called only when em_display_debug_stats is enabled. 6118 * Provides a way to take a look at important statistics 6119 * maintained by the driver and hardware. 6120 ************************************************************************/ 6121 static void 6122 ixgbe_print_debug_info(struct adapter *adapter) 6123 { 6124 device_t dev = adapter->dev; 6125 struct ixgbe_hw *hw = &adapter->hw; 6126 int table_size; 6127 int i; 6128 6129 switch (adapter->hw.mac.type) { 6130 case ixgbe_mac_X550: 6131 case ixgbe_mac_X550EM_x: 6132 case ixgbe_mac_X550EM_a: 6133 table_size = 128; 6134 break; 6135 default: 6136 table_size = 32; 6137 break; 6138 } 6139 6140 device_printf(dev, "[E]RETA:\n"); 6141 for (i = 0; i < table_size; i++) { 6142 if (i < 32) 6143 printf("%02x: %08x\n", i, IXGBE_READ_REG(hw, 6144 IXGBE_RETA(i))); 6145 else 6146 printf("%02x: %08x\n", i, IXGBE_READ_REG(hw, 6147 IXGBE_ERETA(i - 32))); 6148 } 6149 6150 device_printf(dev, "queue:"); 6151 for (i = 0; i < adapter->num_queues; i++) { 6152 printf((i == 0) ? "\t" : " "); 6153 printf("%8d", i); 6154 } 6155 printf("\n"); 6156 PRINTQS(adapter, RDBAL); 6157 PRINTQS(adapter, RDBAH); 6158 PRINTQS(adapter, RDLEN); 6159 PRINTQS(adapter, SRRCTL); 6160 PRINTQS(adapter, RDH); 6161 PRINTQS(adapter, RDT); 6162 PRINTQS(adapter, RXDCTL); 6163 6164 device_printf(dev, "RQSMR:"); 6165 for (i = 0; i < adapter->num_queues / 4; i++) { 6166 printf((i == 0) ? "\t" : " "); 6167 printf("%08x", IXGBE_READ_REG(hw, IXGBE_RQSMR(i))); 6168 } 6169 printf("\n"); 6170 6171 device_printf(dev, "disabled_count:"); 6172 for (i = 0; i < adapter->num_queues; i++) { 6173 printf((i == 0) ? "\t" : " "); 6174 printf("%8d", adapter->queues[i].disabled_count); 6175 } 6176 printf("\n"); 6177 6178 device_printf(dev, "EIMS:\t%08x\n", IXGBE_READ_REG(hw, IXGBE_EIMS)); 6179 if (hw->mac.type != ixgbe_mac_82598EB) { 6180 device_printf(dev, "EIMS_EX(0):\t%08x\n", 6181 IXGBE_READ_REG(hw, IXGBE_EIMS_EX(0))); 6182 device_printf(dev, "EIMS_EX(1):\t%08x\n", 6183 IXGBE_READ_REG(hw, IXGBE_EIMS_EX(1))); 6184 } 6185 device_printf(dev, "EIAM:\t%08x\n", IXGBE_READ_REG(hw, IXGBE_EIAM)); 6186 device_printf(dev, "EIAC:\t%08x\n", IXGBE_READ_REG(hw, IXGBE_EIAC)); 6187 } /* ixgbe_print_debug_info */ 6188 6189 /************************************************************************ 6190 * ixgbe_sysctl_debug 6191 ************************************************************************/ 6192 static int 6193 ixgbe_sysctl_debug(SYSCTLFN_ARGS) 6194 { 6195 struct sysctlnode node = *rnode; 6196 struct adapter *adapter = (struct adapter *)node.sysctl_data; 6197 int error, result = 0; 6198 6199 if (ixgbe_fw_recovery_mode_swflag(adapter)) 6200 return (EPERM); 6201 6202 node.sysctl_data = &result; 6203 error = sysctl_lookup(SYSCTLFN_CALL(&node)); 6204 6205 if (error || newp == NULL) 6206 return error; 6207 6208 if (result == 1) 6209 ixgbe_print_debug_info(adapter); 6210 6211 return 0; 6212 } /* ixgbe_sysctl_debug */ 6213 6214 /************************************************************************ 6215 * ixgbe_sysctl_rx_copy_len 6216 ************************************************************************/ 6217 static int 6218 ixgbe_sysctl_rx_copy_len(SYSCTLFN_ARGS) 6219 { 6220 struct sysctlnode node = *rnode; 6221 struct adapter *adapter = (struct adapter *)node.sysctl_data; 6222 int error; 6223 int result = adapter->rx_copy_len; 6224 6225 node.sysctl_data = &result; 6226 error = sysctl_lookup(SYSCTLFN_CALL(&node)); 6227 6228 if (error || newp == NULL) 6229 return error; 6230 6231 if ((result < 0) || (result > IXGBE_RX_COPY_LEN_MAX)) 6232 return EINVAL; 6233 6234 adapter->rx_copy_len = result; 6235 6236 return 0; 6237 } /* ixgbe_sysctl_rx_copy_len */ 6238 6239 /************************************************************************ 6240 * ixgbe_sysctl_tx_process_limit 6241 ************************************************************************/ 6242 static int 6243 ixgbe_sysctl_tx_process_limit(SYSCTLFN_ARGS) 6244 { 6245 struct sysctlnode node = *rnode; 6246 struct adapter *adapter = (struct adapter *)node.sysctl_data; 6247 int error; 6248 int result = adapter->tx_process_limit; 6249 6250 node.sysctl_data = &result; 6251 error = sysctl_lookup(SYSCTLFN_CALL(&node)); 6252 6253 if (error || newp == NULL) 6254 return error; 6255 6256 if ((result <= 0) || (result > adapter->num_tx_desc)) 6257 return EINVAL; 6258 6259 adapter->tx_process_limit = result; 6260 6261 return 0; 6262 } /* ixgbe_sysctl_tx_process_limit */ 6263 6264 /************************************************************************ 6265 * ixgbe_sysctl_rx_process_limit 6266 ************************************************************************/ 6267 static int 6268 ixgbe_sysctl_rx_process_limit(SYSCTLFN_ARGS) 6269 { 6270 struct sysctlnode node = *rnode; 6271 struct adapter *adapter = (struct adapter *)node.sysctl_data; 6272 int error; 6273 int result = adapter->rx_process_limit; 6274 6275 node.sysctl_data = &result; 6276 error = sysctl_lookup(SYSCTLFN_CALL(&node)); 6277 6278 if (error || newp == NULL) 6279 return error; 6280 6281 if ((result <= 0) || (result > adapter->num_rx_desc)) 6282 return EINVAL; 6283 6284 adapter->rx_process_limit = result; 6285 6286 return 0; 6287 } /* ixgbe_sysctl_rx_process_limit */ 6288 6289 /************************************************************************ 6290 * ixgbe_init_device_features 6291 ************************************************************************/ 6292 static void 6293 ixgbe_init_device_features(struct adapter *adapter) 6294 { 6295 adapter->feat_cap = IXGBE_FEATURE_NETMAP 6296 | IXGBE_FEATURE_RSS 6297 | IXGBE_FEATURE_MSI 6298 | IXGBE_FEATURE_MSIX 6299 | IXGBE_FEATURE_LEGACY_IRQ 6300 | IXGBE_FEATURE_LEGACY_TX; 6301 6302 /* Set capabilities first... */ 6303 switch (adapter->hw.mac.type) { 6304 case ixgbe_mac_82598EB: 6305 if (adapter->hw.device_id == IXGBE_DEV_ID_82598AT) 6306 adapter->feat_cap |= IXGBE_FEATURE_FAN_FAIL; 6307 break; 6308 case ixgbe_mac_X540: 6309 adapter->feat_cap |= IXGBE_FEATURE_SRIOV; 6310 adapter->feat_cap |= IXGBE_FEATURE_FDIR; 6311 if ((adapter->hw.device_id == IXGBE_DEV_ID_X540_BYPASS) && 6312 (adapter->hw.bus.func == 0)) 6313 adapter->feat_cap |= IXGBE_FEATURE_BYPASS; 6314 break; 6315 case ixgbe_mac_X550: 6316 /* 6317 * IXGBE_FEATURE_RECOVERY_MODE will be set after reading 6318 * NVM Image version. 6319 */ 6320 adapter->feat_cap |= IXGBE_FEATURE_TEMP_SENSOR; 6321 adapter->feat_cap |= IXGBE_FEATURE_SRIOV; 6322 adapter->feat_cap |= IXGBE_FEATURE_FDIR; 6323 break; 6324 case ixgbe_mac_X550EM_x: 6325 /* 6326 * IXGBE_FEATURE_RECOVERY_MODE will be set after reading 6327 * NVM Image version. 6328 */ 6329 adapter->feat_cap |= IXGBE_FEATURE_SRIOV; 6330 adapter->feat_cap |= IXGBE_FEATURE_FDIR; 6331 break; 6332 case ixgbe_mac_X550EM_a: 6333 /* 6334 * IXGBE_FEATURE_RECOVERY_MODE will be set after reading 6335 * NVM Image version. 6336 */ 6337 adapter->feat_cap |= IXGBE_FEATURE_SRIOV; 6338 adapter->feat_cap |= IXGBE_FEATURE_FDIR; 6339 adapter->feat_cap &= ~IXGBE_FEATURE_LEGACY_IRQ; 6340 if ((adapter->hw.device_id == IXGBE_DEV_ID_X550EM_A_1G_T) || 6341 (adapter->hw.device_id == IXGBE_DEV_ID_X550EM_A_1G_T_L)) { 6342 adapter->feat_cap |= IXGBE_FEATURE_TEMP_SENSOR; 6343 adapter->feat_cap |= IXGBE_FEATURE_EEE; 6344 } 6345 break; 6346 case ixgbe_mac_82599EB: 6347 adapter->feat_cap |= IXGBE_FEATURE_SRIOV; 6348 adapter->feat_cap |= IXGBE_FEATURE_FDIR; 6349 if ((adapter->hw.device_id == IXGBE_DEV_ID_82599_BYPASS) && 6350 (adapter->hw.bus.func == 0)) 6351 adapter->feat_cap |= IXGBE_FEATURE_BYPASS; 6352 if (adapter->hw.device_id == IXGBE_DEV_ID_82599_QSFP_SF_QP) 6353 adapter->feat_cap &= ~IXGBE_FEATURE_LEGACY_IRQ; 6354 break; 6355 default: 6356 break; 6357 } 6358 6359 /* Enabled by default... */ 6360 /* Fan failure detection */ 6361 if (adapter->feat_cap & IXGBE_FEATURE_FAN_FAIL) 6362 adapter->feat_en |= IXGBE_FEATURE_FAN_FAIL; 6363 /* Netmap */ 6364 if (adapter->feat_cap & IXGBE_FEATURE_NETMAP) 6365 adapter->feat_en |= IXGBE_FEATURE_NETMAP; 6366 /* EEE */ 6367 if (adapter->feat_cap & IXGBE_FEATURE_EEE) 6368 adapter->feat_en |= IXGBE_FEATURE_EEE; 6369 /* Thermal Sensor */ 6370 if (adapter->feat_cap & IXGBE_FEATURE_TEMP_SENSOR) 6371 adapter->feat_en |= IXGBE_FEATURE_TEMP_SENSOR; 6372 /* 6373 * Recovery mode: 6374 * NetBSD: IXGBE_FEATURE_RECOVERY_MODE will be controlled after reading 6375 * NVM Image version. 6376 */ 6377 6378 /* Enabled via global sysctl... */ 6379 /* Flow Director */ 6380 if (ixgbe_enable_fdir) { 6381 if (adapter->feat_cap & IXGBE_FEATURE_FDIR) 6382 adapter->feat_en |= IXGBE_FEATURE_FDIR; 6383 else 6384 device_printf(adapter->dev, "Device does not support Flow Director. Leaving disabled."); 6385 } 6386 /* Legacy (single queue) transmit */ 6387 if ((adapter->feat_cap & IXGBE_FEATURE_LEGACY_TX) && 6388 ixgbe_enable_legacy_tx) 6389 adapter->feat_en |= IXGBE_FEATURE_LEGACY_TX; 6390 /* 6391 * Message Signal Interrupts - Extended (MSI-X) 6392 * Normal MSI is only enabled if MSI-X calls fail. 6393 */ 6394 if (!ixgbe_enable_msix) 6395 adapter->feat_cap &= ~IXGBE_FEATURE_MSIX; 6396 /* Receive-Side Scaling (RSS) */ 6397 if ((adapter->feat_cap & IXGBE_FEATURE_RSS) && ixgbe_enable_rss) 6398 adapter->feat_en |= IXGBE_FEATURE_RSS; 6399 6400 /* Disable features with unmet dependencies... */ 6401 /* No MSI-X */ 6402 if (!(adapter->feat_cap & IXGBE_FEATURE_MSIX)) { 6403 adapter->feat_cap &= ~IXGBE_FEATURE_RSS; 6404 adapter->feat_cap &= ~IXGBE_FEATURE_SRIOV; 6405 adapter->feat_en &= ~IXGBE_FEATURE_RSS; 6406 adapter->feat_en &= ~IXGBE_FEATURE_SRIOV; 6407 } 6408 } /* ixgbe_init_device_features */ 6409 6410 /************************************************************************ 6411 * ixgbe_probe - Device identification routine 6412 * 6413 * Determines if the driver should be loaded on 6414 * adapter based on its PCI vendor/device ID. 6415 * 6416 * return BUS_PROBE_DEFAULT on success, positive on failure 6417 ************************************************************************/ 6418 static int 6419 ixgbe_probe(device_t dev, cfdata_t cf, void *aux) 6420 { 6421 const struct pci_attach_args *pa = aux; 6422 6423 return (ixgbe_lookup(pa) != NULL) ? 1 : 0; 6424 } 6425 6426 static const ixgbe_vendor_info_t * 6427 ixgbe_lookup(const struct pci_attach_args *pa) 6428 { 6429 const ixgbe_vendor_info_t *ent; 6430 pcireg_t subid; 6431 6432 INIT_DEBUGOUT("ixgbe_lookup: begin"); 6433 6434 if (PCI_VENDOR(pa->pa_id) != IXGBE_INTEL_VENDOR_ID) 6435 return NULL; 6436 6437 subid = pci_conf_read(pa->pa_pc, pa->pa_tag, PCI_SUBSYS_ID_REG); 6438 6439 for (ent = ixgbe_vendor_info_array; ent->vendor_id != 0; ent++) { 6440 if ((PCI_VENDOR(pa->pa_id) == ent->vendor_id) && 6441 (PCI_PRODUCT(pa->pa_id) == ent->device_id) && 6442 ((PCI_SUBSYS_VENDOR(subid) == ent->subvendor_id) || 6443 (ent->subvendor_id == 0)) && 6444 ((PCI_SUBSYS_ID(subid) == ent->subdevice_id) || 6445 (ent->subdevice_id == 0))) { 6446 return ent; 6447 } 6448 } 6449 return NULL; 6450 } 6451 6452 static int 6453 ixgbe_ifflags_cb(struct ethercom *ec) 6454 { 6455 struct ifnet *ifp = &ec->ec_if; 6456 struct adapter *adapter = ifp->if_softc; 6457 u_short change; 6458 int rv = 0; 6459 6460 IXGBE_CORE_LOCK(adapter); 6461 6462 change = ifp->if_flags ^ adapter->if_flags; 6463 if (change != 0) 6464 adapter->if_flags = ifp->if_flags; 6465 6466 if ((change & ~(IFF_CANTCHANGE | IFF_DEBUG)) != 0) { 6467 rv = ENETRESET; 6468 goto out; 6469 } else if ((change & IFF_PROMISC) != 0) 6470 ixgbe_set_rxfilter(adapter); 6471 6472 /* Check for ec_capenable. */ 6473 change = ec->ec_capenable ^ adapter->ec_capenable; 6474 adapter->ec_capenable = ec->ec_capenable; 6475 if ((change & ~(ETHERCAP_VLAN_MTU | ETHERCAP_VLAN_HWTAGGING 6476 | ETHERCAP_VLAN_HWFILTER)) != 0) { 6477 rv = ENETRESET; 6478 goto out; 6479 } 6480 6481 /* 6482 * Special handling is not required for ETHERCAP_VLAN_MTU. 6483 * MAXFRS(MHADD) does not include the 4bytes of the VLAN header. 6484 */ 6485 6486 /* Set up VLAN support and filter */ 6487 if ((change & (ETHERCAP_VLAN_HWTAGGING | ETHERCAP_VLAN_HWFILTER)) != 0) 6488 ixgbe_setup_vlan_hw_support(adapter); 6489 6490 out: 6491 IXGBE_CORE_UNLOCK(adapter); 6492 6493 return rv; 6494 } 6495 6496 /************************************************************************ 6497 * ixgbe_ioctl - Ioctl entry point 6498 * 6499 * Called when the user wants to configure the interface. 6500 * 6501 * return 0 on success, positive on failure 6502 ************************************************************************/ 6503 static int 6504 ixgbe_ioctl(struct ifnet *ifp, u_long command, void *data) 6505 { 6506 struct adapter *adapter = ifp->if_softc; 6507 struct ixgbe_hw *hw = &adapter->hw; 6508 struct ifcapreq *ifcr = data; 6509 struct ifreq *ifr = data; 6510 int error = 0; 6511 int l4csum_en; 6512 const int l4csum = IFCAP_CSUM_TCPv4_Rx | IFCAP_CSUM_UDPv4_Rx | 6513 IFCAP_CSUM_TCPv6_Rx | IFCAP_CSUM_UDPv6_Rx; 6514 6515 if (ixgbe_fw_recovery_mode_swflag(adapter)) 6516 return (EPERM); 6517 6518 switch (command) { 6519 case SIOCSIFFLAGS: 6520 IOCTL_DEBUGOUT("ioctl: SIOCSIFFLAGS (Set Interface Flags)"); 6521 break; 6522 case SIOCADDMULTI: 6523 case SIOCDELMULTI: 6524 IOCTL_DEBUGOUT("ioctl: SIOC(ADD|DEL)MULTI"); 6525 break; 6526 case SIOCSIFMEDIA: 6527 case SIOCGIFMEDIA: 6528 IOCTL_DEBUGOUT("ioctl: SIOCxIFMEDIA (Get/Set Interface Media)"); 6529 break; 6530 case SIOCSIFCAP: 6531 IOCTL_DEBUGOUT("ioctl: SIOCSIFCAP (Set Capabilities)"); 6532 break; 6533 case SIOCSIFMTU: 6534 IOCTL_DEBUGOUT("ioctl: SIOCSIFMTU (Set Interface MTU)"); 6535 break; 6536 #ifdef __NetBSD__ 6537 case SIOCINITIFADDR: 6538 IOCTL_DEBUGOUT("ioctl: SIOCINITIFADDR"); 6539 break; 6540 case SIOCGIFFLAGS: 6541 IOCTL_DEBUGOUT("ioctl: SIOCGIFFLAGS"); 6542 break; 6543 case SIOCGIFAFLAG_IN: 6544 IOCTL_DEBUGOUT("ioctl: SIOCGIFAFLAG_IN"); 6545 break; 6546 case SIOCGIFADDR: 6547 IOCTL_DEBUGOUT("ioctl: SIOCGIFADDR"); 6548 break; 6549 case SIOCGIFMTU: 6550 IOCTL_DEBUGOUT("ioctl: SIOCGIFMTU (Get Interface MTU)"); 6551 break; 6552 case SIOCGIFCAP: 6553 IOCTL_DEBUGOUT("ioctl: SIOCGIFCAP (Get IF cap)"); 6554 break; 6555 case SIOCGETHERCAP: 6556 IOCTL_DEBUGOUT("ioctl: SIOCGETHERCAP (Get ethercap)"); 6557 break; 6558 case SIOCGLIFADDR: 6559 IOCTL_DEBUGOUT("ioctl: SIOCGLIFADDR (Get Interface addr)"); 6560 break; 6561 case SIOCZIFDATA: 6562 IOCTL_DEBUGOUT("ioctl: SIOCZIFDATA (Zero counter)"); 6563 hw->mac.ops.clear_hw_cntrs(hw); 6564 ixgbe_clear_evcnt(adapter); 6565 break; 6566 case SIOCAIFADDR: 6567 IOCTL_DEBUGOUT("ioctl: SIOCAIFADDR (add/chg IF alias)"); 6568 break; 6569 #endif 6570 default: 6571 IOCTL_DEBUGOUT1("ioctl: UNKNOWN (0x%X)", (int)command); 6572 break; 6573 } 6574 6575 switch (command) { 6576 case SIOCGI2C: 6577 { 6578 struct ixgbe_i2c_req i2c; 6579 6580 IOCTL_DEBUGOUT("ioctl: SIOCGI2C (Get I2C Data)"); 6581 error = copyin(ifr->ifr_data, &i2c, sizeof(i2c)); 6582 if (error != 0) 6583 break; 6584 if (i2c.dev_addr != 0xA0 && i2c.dev_addr != 0xA2) { 6585 error = EINVAL; 6586 break; 6587 } 6588 if (i2c.len > sizeof(i2c.data)) { 6589 error = EINVAL; 6590 break; 6591 } 6592 6593 hw->phy.ops.read_i2c_byte(hw, i2c.offset, 6594 i2c.dev_addr, i2c.data); 6595 error = copyout(&i2c, ifr->ifr_data, sizeof(i2c)); 6596 break; 6597 } 6598 case SIOCSIFCAP: 6599 /* Layer-4 Rx checksum offload has to be turned on and 6600 * off as a unit. 6601 */ 6602 l4csum_en = ifcr->ifcr_capenable & l4csum; 6603 if (l4csum_en != l4csum && l4csum_en != 0) 6604 return EINVAL; 6605 /*FALLTHROUGH*/ 6606 case SIOCADDMULTI: 6607 case SIOCDELMULTI: 6608 case SIOCSIFFLAGS: 6609 case SIOCSIFMTU: 6610 default: 6611 if ((error = ether_ioctl(ifp, command, data)) != ENETRESET) 6612 return error; 6613 if ((ifp->if_flags & IFF_RUNNING) == 0) 6614 ; 6615 else if (command == SIOCSIFCAP || command == SIOCSIFMTU) { 6616 IXGBE_CORE_LOCK(adapter); 6617 if ((ifp->if_flags & IFF_RUNNING) != 0) 6618 ixgbe_init_locked(adapter); 6619 ixgbe_recalculate_max_frame(adapter); 6620 IXGBE_CORE_UNLOCK(adapter); 6621 } else if (command == SIOCADDMULTI || command == SIOCDELMULTI) { 6622 /* 6623 * Multicast list has changed; set the hardware filter 6624 * accordingly. 6625 */ 6626 IXGBE_CORE_LOCK(adapter); 6627 ixgbe_disable_intr(adapter); 6628 ixgbe_set_rxfilter(adapter); 6629 ixgbe_enable_intr(adapter); 6630 IXGBE_CORE_UNLOCK(adapter); 6631 } 6632 return 0; 6633 } 6634 6635 return error; 6636 } /* ixgbe_ioctl */ 6637 6638 /************************************************************************ 6639 * ixgbe_check_fan_failure 6640 ************************************************************************/ 6641 static int 6642 ixgbe_check_fan_failure(struct adapter *adapter, u32 reg, bool in_interrupt) 6643 { 6644 u32 mask; 6645 6646 mask = (in_interrupt) ? IXGBE_EICR_GPI_SDP1_BY_MAC(&adapter->hw) : 6647 IXGBE_ESDP_SDP1; 6648 6649 if ((reg & mask) == 0) 6650 return IXGBE_SUCCESS; 6651 6652 /* 6653 * Use ratecheck() just in case interrupt occur frequently. 6654 * When EXPX9501AT's fan stopped, interrupt occurred only once, 6655 * an red LED on the board turned on and link never up until 6656 * power off. 6657 */ 6658 if (ratecheck(&adapter->lasterr_time, &ixgbe_errlog_intrvl)) 6659 device_printf(adapter->dev, 6660 "\nCRITICAL: FAN FAILURE!! REPLACE IMMEDIATELY!!\n"); 6661 6662 return IXGBE_ERR_FAN_FAILURE; 6663 } /* ixgbe_check_fan_failure */ 6664 6665 /************************************************************************ 6666 * ixgbe_handle_que 6667 ************************************************************************/ 6668 static void 6669 ixgbe_handle_que(void *context) 6670 { 6671 struct ix_queue *que = context; 6672 struct adapter *adapter = que->adapter; 6673 struct tx_ring *txr = que->txr; 6674 struct ifnet *ifp = adapter->ifp; 6675 bool more = false; 6676 6677 IXGBE_EVC_ADD(&que->handleq, 1); 6678 6679 if (ifp->if_flags & IFF_RUNNING) { 6680 more = ixgbe_rxeof(que); 6681 IXGBE_TX_LOCK(txr); 6682 more |= ixgbe_txeof(txr); 6683 if (!(adapter->feat_en & IXGBE_FEATURE_LEGACY_TX)) 6684 if (!ixgbe_mq_ring_empty(ifp, txr->txr_interq)) 6685 ixgbe_mq_start_locked(ifp, txr); 6686 /* Only for queue 0 */ 6687 /* NetBSD still needs this for CBQ */ 6688 if ((&adapter->queues[0] == que) 6689 && (!ixgbe_legacy_ring_empty(ifp, NULL))) 6690 ixgbe_legacy_start_locked(ifp, txr); 6691 IXGBE_TX_UNLOCK(txr); 6692 } 6693 6694 if (more) { 6695 IXGBE_EVC_ADD(&que->req, 1); 6696 ixgbe_sched_handle_que(adapter, que); 6697 } else if (que->res != NULL) { 6698 /* MSIX: Re-enable this interrupt */ 6699 ixgbe_enable_queue(adapter, que->msix); 6700 } else { 6701 /* INTx or MSI */ 6702 ixgbe_enable_queue(adapter, 0); 6703 } 6704 6705 return; 6706 } /* ixgbe_handle_que */ 6707 6708 /************************************************************************ 6709 * ixgbe_handle_que_work 6710 ************************************************************************/ 6711 static void 6712 ixgbe_handle_que_work(struct work *wk, void *context) 6713 { 6714 struct ix_queue *que = container_of(wk, struct ix_queue, wq_cookie); 6715 6716 /* 6717 * "enqueued flag" is not required here. 6718 * See ixgbe_msix_que(). 6719 */ 6720 ixgbe_handle_que(que); 6721 } 6722 6723 /************************************************************************ 6724 * ixgbe_allocate_legacy - Setup the Legacy or MSI Interrupt handler 6725 ************************************************************************/ 6726 static int 6727 ixgbe_allocate_legacy(struct adapter *adapter, 6728 const struct pci_attach_args *pa) 6729 { 6730 device_t dev = adapter->dev; 6731 struct ix_queue *que = adapter->queues; 6732 struct tx_ring *txr = adapter->tx_rings; 6733 int counts[PCI_INTR_TYPE_SIZE]; 6734 pci_intr_type_t intr_type, max_type; 6735 char intrbuf[PCI_INTRSTR_LEN]; 6736 char wqname[MAXCOMLEN]; 6737 const char *intrstr = NULL; 6738 int defertx_error = 0, error; 6739 6740 /* We allocate a single interrupt resource */ 6741 max_type = PCI_INTR_TYPE_MSI; 6742 counts[PCI_INTR_TYPE_MSIX] = 0; 6743 counts[PCI_INTR_TYPE_MSI] = 6744 (adapter->feat_en & IXGBE_FEATURE_MSI) ? 1 : 0; 6745 /* Check not feat_en but feat_cap to fallback to INTx */ 6746 counts[PCI_INTR_TYPE_INTX] = 6747 (adapter->feat_cap & IXGBE_FEATURE_LEGACY_IRQ) ? 1 : 0; 6748 6749 alloc_retry: 6750 if (pci_intr_alloc(pa, &adapter->osdep.intrs, counts, max_type) != 0) { 6751 aprint_error_dev(dev, "couldn't alloc interrupt\n"); 6752 return ENXIO; 6753 } 6754 adapter->osdep.nintrs = 1; 6755 intrstr = pci_intr_string(adapter->osdep.pc, adapter->osdep.intrs[0], 6756 intrbuf, sizeof(intrbuf)); 6757 adapter->osdep.ihs[0] = pci_intr_establish_xname(adapter->osdep.pc, 6758 adapter->osdep.intrs[0], IPL_NET, ixgbe_legacy_irq, que, 6759 device_xname(dev)); 6760 intr_type = pci_intr_type(adapter->osdep.pc, adapter->osdep.intrs[0]); 6761 if (adapter->osdep.ihs[0] == NULL) { 6762 aprint_error_dev(dev,"unable to establish %s\n", 6763 (intr_type == PCI_INTR_TYPE_MSI) ? "MSI" : "INTx"); 6764 pci_intr_release(adapter->osdep.pc, adapter->osdep.intrs, 1); 6765 adapter->osdep.intrs = NULL; 6766 switch (intr_type) { 6767 case PCI_INTR_TYPE_MSI: 6768 /* The next try is for INTx: Disable MSI */ 6769 max_type = PCI_INTR_TYPE_INTX; 6770 counts[PCI_INTR_TYPE_INTX] = 1; 6771 adapter->feat_en &= ~IXGBE_FEATURE_MSI; 6772 if (adapter->feat_cap & IXGBE_FEATURE_LEGACY_IRQ) { 6773 adapter->feat_en |= IXGBE_FEATURE_LEGACY_IRQ; 6774 goto alloc_retry; 6775 } else 6776 break; 6777 case PCI_INTR_TYPE_INTX: 6778 default: 6779 /* See below */ 6780 break; 6781 } 6782 } 6783 if (intr_type == PCI_INTR_TYPE_INTX) { 6784 adapter->feat_en &= ~IXGBE_FEATURE_MSI; 6785 adapter->feat_en |= IXGBE_FEATURE_LEGACY_IRQ; 6786 } 6787 if (adapter->osdep.ihs[0] == NULL) { 6788 aprint_error_dev(dev, 6789 "couldn't establish interrupt%s%s\n", 6790 intrstr ? " at " : "", intrstr ? intrstr : ""); 6791 pci_intr_release(adapter->osdep.pc, adapter->osdep.intrs, 1); 6792 adapter->osdep.intrs = NULL; 6793 return ENXIO; 6794 } 6795 aprint_normal_dev(dev, "interrupting at %s\n", intrstr); 6796 /* 6797 * Try allocating a fast interrupt and the associated deferred 6798 * processing contexts. 6799 */ 6800 if (!(adapter->feat_en & IXGBE_FEATURE_LEGACY_TX)) { 6801 txr->txr_si = 6802 softint_establish(SOFTINT_NET | IXGBE_SOFTINT_FLAGS, 6803 ixgbe_deferred_mq_start, txr); 6804 6805 snprintf(wqname, sizeof(wqname), "%sdeferTx", 6806 device_xname(dev)); 6807 defertx_error = workqueue_create(&adapter->txr_wq, wqname, 6808 ixgbe_deferred_mq_start_work, adapter, IXGBE_WORKQUEUE_PRI, 6809 IPL_NET, IXGBE_WORKQUEUE_FLAGS); 6810 adapter->txr_wq_enqueued = percpu_alloc(sizeof(u_int)); 6811 } 6812 que->que_si = softint_establish(SOFTINT_NET | IXGBE_SOFTINT_FLAGS, 6813 ixgbe_handle_que, que); 6814 snprintf(wqname, sizeof(wqname), "%sTxRx", device_xname(dev)); 6815 error = workqueue_create(&adapter->que_wq, wqname, 6816 ixgbe_handle_que_work, adapter, IXGBE_WORKQUEUE_PRI, IPL_NET, 6817 IXGBE_WORKQUEUE_FLAGS); 6818 6819 if ((!(adapter->feat_en & IXGBE_FEATURE_LEGACY_TX) 6820 && ((txr->txr_si == NULL) || defertx_error != 0)) 6821 || (que->que_si == NULL) || error != 0) { 6822 aprint_error_dev(dev, 6823 "could not establish software interrupts\n"); 6824 6825 return ENXIO; 6826 } 6827 /* For simplicity in the handlers */ 6828 adapter->active_queues = IXGBE_EIMS_ENABLE_MASK; 6829 6830 return (0); 6831 } /* ixgbe_allocate_legacy */ 6832 6833 /************************************************************************ 6834 * ixgbe_allocate_msix - Setup MSI-X Interrupt resources and handlers 6835 ************************************************************************/ 6836 static int 6837 ixgbe_allocate_msix(struct adapter *adapter, const struct pci_attach_args *pa) 6838 { 6839 device_t dev = adapter->dev; 6840 struct ix_queue *que = adapter->queues; 6841 struct tx_ring *txr = adapter->tx_rings; 6842 pci_chipset_tag_t pc; 6843 char intrbuf[PCI_INTRSTR_LEN]; 6844 char intr_xname[32]; 6845 char wqname[MAXCOMLEN]; 6846 const char *intrstr = NULL; 6847 int error, vector = 0; 6848 int cpu_id = 0; 6849 kcpuset_t *affinity; 6850 #ifdef RSS 6851 unsigned int rss_buckets = 0; 6852 kcpuset_t cpu_mask; 6853 #endif 6854 6855 pc = adapter->osdep.pc; 6856 #ifdef RSS 6857 /* 6858 * If we're doing RSS, the number of queues needs to 6859 * match the number of RSS buckets that are configured. 6860 * 6861 * + If there's more queues than RSS buckets, we'll end 6862 * up with queues that get no traffic. 6863 * 6864 * + If there's more RSS buckets than queues, we'll end 6865 * up having multiple RSS buckets map to the same queue, 6866 * so there'll be some contention. 6867 */ 6868 rss_buckets = rss_getnumbuckets(); 6869 if ((adapter->feat_en & IXGBE_FEATURE_RSS) && 6870 (adapter->num_queues != rss_buckets)) { 6871 device_printf(dev, 6872 "%s: number of queues (%d) != number of RSS buckets (%d)" 6873 "; performance will be impacted.\n", 6874 __func__, adapter->num_queues, rss_buckets); 6875 } 6876 #endif 6877 6878 adapter->osdep.nintrs = adapter->num_queues + 1; 6879 if (pci_msix_alloc_exact(pa, &adapter->osdep.intrs, 6880 adapter->osdep.nintrs) != 0) { 6881 aprint_error_dev(dev, 6882 "failed to allocate MSI-X interrupt\n"); 6883 adapter->feat_en &= ~IXGBE_FEATURE_MSIX; 6884 return (ENXIO); 6885 } 6886 6887 kcpuset_create(&affinity, false); 6888 for (int i = 0; i < adapter->num_queues; i++, vector++, que++, txr++) { 6889 snprintf(intr_xname, sizeof(intr_xname), "%s TXRX%d", 6890 device_xname(dev), i); 6891 intrstr = pci_intr_string(pc, adapter->osdep.intrs[i], intrbuf, 6892 sizeof(intrbuf)); 6893 #ifdef IXGBE_MPSAFE 6894 pci_intr_setattr(pc, &adapter->osdep.intrs[i], PCI_INTR_MPSAFE, 6895 true); 6896 #endif 6897 /* Set the handler function */ 6898 que->res = adapter->osdep.ihs[i] = pci_intr_establish_xname(pc, 6899 adapter->osdep.intrs[i], IPL_NET, ixgbe_msix_que, que, 6900 intr_xname); 6901 if (que->res == NULL) { 6902 aprint_error_dev(dev, 6903 "Failed to register QUE handler\n"); 6904 error = ENXIO; 6905 goto err_out; 6906 } 6907 que->msix = vector; 6908 adapter->active_queues |= 1ULL << que->msix; 6909 6910 if (adapter->feat_en & IXGBE_FEATURE_RSS) { 6911 #ifdef RSS 6912 /* 6913 * The queue ID is used as the RSS layer bucket ID. 6914 * We look up the queue ID -> RSS CPU ID and select 6915 * that. 6916 */ 6917 cpu_id = rss_getcpu(i % rss_getnumbuckets()); 6918 CPU_SETOF(cpu_id, &cpu_mask); 6919 #endif 6920 } else { 6921 /* 6922 * Bind the MSI-X vector, and thus the 6923 * rings to the corresponding CPU. 6924 * 6925 * This just happens to match the default RSS 6926 * round-robin bucket -> queue -> CPU allocation. 6927 */ 6928 if (adapter->num_queues > 1) 6929 cpu_id = i; 6930 } 6931 /* Round-robin affinity */ 6932 kcpuset_zero(affinity); 6933 kcpuset_set(affinity, cpu_id % ncpu); 6934 error = interrupt_distribute(adapter->osdep.ihs[i], affinity, 6935 NULL); 6936 aprint_normal_dev(dev, "for TX/RX, interrupting at %s", 6937 intrstr); 6938 if (error == 0) { 6939 #if 1 /* def IXGBE_DEBUG */ 6940 #ifdef RSS 6941 aprintf_normal(", bound RSS bucket %d to CPU %d", i, 6942 cpu_id % ncpu); 6943 #else 6944 aprint_normal(", bound queue %d to cpu %d", i, 6945 cpu_id % ncpu); 6946 #endif 6947 #endif /* IXGBE_DEBUG */ 6948 } 6949 aprint_normal("\n"); 6950 6951 if (!(adapter->feat_en & IXGBE_FEATURE_LEGACY_TX)) { 6952 txr->txr_si = softint_establish( 6953 SOFTINT_NET | IXGBE_SOFTINT_FLAGS, 6954 ixgbe_deferred_mq_start, txr); 6955 if (txr->txr_si == NULL) { 6956 aprint_error_dev(dev, 6957 "couldn't establish software interrupt\n"); 6958 error = ENXIO; 6959 goto err_out; 6960 } 6961 } 6962 que->que_si 6963 = softint_establish(SOFTINT_NET | IXGBE_SOFTINT_FLAGS, 6964 ixgbe_handle_que, que); 6965 if (que->que_si == NULL) { 6966 aprint_error_dev(dev, 6967 "couldn't establish software interrupt\n"); 6968 error = ENXIO; 6969 goto err_out; 6970 } 6971 } 6972 snprintf(wqname, sizeof(wqname), "%sdeferTx", device_xname(dev)); 6973 error = workqueue_create(&adapter->txr_wq, wqname, 6974 ixgbe_deferred_mq_start_work, adapter, IXGBE_WORKQUEUE_PRI, IPL_NET, 6975 IXGBE_WORKQUEUE_FLAGS); 6976 if (error) { 6977 aprint_error_dev(dev, 6978 "couldn't create workqueue for deferred Tx\n"); 6979 goto err_out; 6980 } 6981 adapter->txr_wq_enqueued = percpu_alloc(sizeof(u_int)); 6982 6983 snprintf(wqname, sizeof(wqname), "%sTxRx", device_xname(dev)); 6984 error = workqueue_create(&adapter->que_wq, wqname, 6985 ixgbe_handle_que_work, adapter, IXGBE_WORKQUEUE_PRI, IPL_NET, 6986 IXGBE_WORKQUEUE_FLAGS); 6987 if (error) { 6988 aprint_error_dev(dev, "couldn't create workqueue for Tx/Rx\n"); 6989 goto err_out; 6990 } 6991 6992 /* and Link */ 6993 cpu_id++; 6994 snprintf(intr_xname, sizeof(intr_xname), "%s link", device_xname(dev)); 6995 adapter->vector = vector; 6996 intrstr = pci_intr_string(pc, adapter->osdep.intrs[vector], intrbuf, 6997 sizeof(intrbuf)); 6998 #ifdef IXGBE_MPSAFE 6999 pci_intr_setattr(pc, &adapter->osdep.intrs[vector], PCI_INTR_MPSAFE, 7000 true); 7001 #endif 7002 /* Set the link handler function */ 7003 adapter->osdep.ihs[vector] = pci_intr_establish_xname(pc, 7004 adapter->osdep.intrs[vector], IPL_NET, ixgbe_msix_admin, adapter, 7005 intr_xname); 7006 if (adapter->osdep.ihs[vector] == NULL) { 7007 aprint_error_dev(dev, "Failed to register LINK handler\n"); 7008 error = ENXIO; 7009 goto err_out; 7010 } 7011 /* Round-robin affinity */ 7012 kcpuset_zero(affinity); 7013 kcpuset_set(affinity, cpu_id % ncpu); 7014 error = interrupt_distribute(adapter->osdep.ihs[vector], affinity, 7015 NULL); 7016 7017 aprint_normal_dev(dev, 7018 "for link, interrupting at %s", intrstr); 7019 if (error == 0) 7020 aprint_normal(", affinity to cpu %d\n", cpu_id % ncpu); 7021 else 7022 aprint_normal("\n"); 7023 7024 kcpuset_destroy(affinity); 7025 aprint_normal_dev(dev, 7026 "Using MSI-X interrupts with %d vectors\n", vector + 1); 7027 7028 return (0); 7029 7030 err_out: 7031 kcpuset_destroy(affinity); 7032 ixgbe_free_deferred_handlers(adapter); 7033 ixgbe_free_pciintr_resources(adapter); 7034 return (error); 7035 } /* ixgbe_allocate_msix */ 7036 7037 /************************************************************************ 7038 * ixgbe_configure_interrupts 7039 * 7040 * Setup MSI-X, MSI, or legacy interrupts (in that order). 7041 * This will also depend on user settings. 7042 ************************************************************************/ 7043 static int 7044 ixgbe_configure_interrupts(struct adapter *adapter) 7045 { 7046 device_t dev = adapter->dev; 7047 struct ixgbe_mac_info *mac = &adapter->hw.mac; 7048 int want, queues, msgs; 7049 7050 /* Default to 1 queue if MSI-X setup fails */ 7051 adapter->num_queues = 1; 7052 7053 /* Override by tuneable */ 7054 if (!(adapter->feat_cap & IXGBE_FEATURE_MSIX)) 7055 goto msi; 7056 7057 /* 7058 * NetBSD only: Use single vector MSI when number of CPU is 1 to save 7059 * interrupt slot. 7060 */ 7061 if (ncpu == 1) 7062 goto msi; 7063 7064 /* First try MSI-X */ 7065 msgs = pci_msix_count(adapter->osdep.pc, adapter->osdep.tag); 7066 msgs = MIN(msgs, IXG_MAX_NINTR); 7067 if (msgs < 2) 7068 goto msi; 7069 7070 adapter->feat_en |= IXGBE_FEATURE_MSIX; 7071 7072 /* Figure out a reasonable auto config value */ 7073 queues = (ncpu > (msgs - 1)) ? (msgs - 1) : ncpu; 7074 7075 #ifdef RSS 7076 /* If we're doing RSS, clamp at the number of RSS buckets */ 7077 if (adapter->feat_en & IXGBE_FEATURE_RSS) 7078 queues = uimin(queues, rss_getnumbuckets()); 7079 #endif 7080 if (ixgbe_num_queues > queues) { 7081 aprint_error_dev(adapter->dev, "ixgbe_num_queues (%d) is too large, using reduced amount (%d).\n", ixgbe_num_queues, queues); 7082 ixgbe_num_queues = queues; 7083 } 7084 7085 if (ixgbe_num_queues != 0) 7086 queues = ixgbe_num_queues; 7087 else 7088 queues = uimin(queues, 7089 uimin(mac->max_tx_queues, mac->max_rx_queues)); 7090 7091 /* reflect correct sysctl value */ 7092 ixgbe_num_queues = queues; 7093 7094 /* 7095 * Want one vector (RX/TX pair) per queue 7096 * plus an additional for Link. 7097 */ 7098 want = queues + 1; 7099 if (msgs >= want) 7100 msgs = want; 7101 else { 7102 aprint_error_dev(dev, "MSI-X Configuration Problem, " 7103 "%d vectors but %d queues wanted!\n", 7104 msgs, want); 7105 goto msi; 7106 } 7107 adapter->num_queues = queues; 7108 adapter->feat_en |= IXGBE_FEATURE_MSIX; 7109 return (0); 7110 7111 /* 7112 * MSI-X allocation failed or provided us with 7113 * less vectors than needed. Free MSI-X resources 7114 * and we'll try enabling MSI. 7115 */ 7116 msi: 7117 /* Without MSI-X, some features are no longer supported */ 7118 adapter->feat_cap &= ~IXGBE_FEATURE_RSS; 7119 adapter->feat_en &= ~IXGBE_FEATURE_RSS; 7120 adapter->feat_cap &= ~IXGBE_FEATURE_SRIOV; 7121 adapter->feat_en &= ~IXGBE_FEATURE_SRIOV; 7122 7123 msgs = pci_msi_count(adapter->osdep.pc, adapter->osdep.tag); 7124 adapter->feat_en &= ~IXGBE_FEATURE_MSIX; 7125 if (msgs > 1) 7126 msgs = 1; 7127 if (msgs != 0) { 7128 msgs = 1; 7129 adapter->feat_en |= IXGBE_FEATURE_MSI; 7130 return (0); 7131 } 7132 7133 if (!(adapter->feat_cap & IXGBE_FEATURE_LEGACY_IRQ)) { 7134 aprint_error_dev(dev, 7135 "Device does not support legacy interrupts.\n"); 7136 return 1; 7137 } 7138 7139 adapter->feat_en |= IXGBE_FEATURE_LEGACY_IRQ; 7140 7141 return (0); 7142 } /* ixgbe_configure_interrupts */ 7143 7144 7145 /************************************************************************ 7146 * ixgbe_handle_link - Tasklet for MSI-X Link interrupts 7147 * 7148 * Done outside of interrupt context since the driver might sleep 7149 ************************************************************************/ 7150 static void 7151 ixgbe_handle_link(void *context) 7152 { 7153 struct adapter *adapter = context; 7154 struct ixgbe_hw *hw = &adapter->hw; 7155 7156 KASSERT(mutex_owned(&adapter->core_mtx)); 7157 7158 IXGBE_EVC_ADD(&adapter->link_workev, 1); 7159 ixgbe_check_link(hw, &adapter->link_speed, &adapter->link_up, 0); 7160 ixgbe_update_link_status(adapter); 7161 7162 /* Re-enable link interrupts */ 7163 IXGBE_WRITE_REG(hw, IXGBE_EIMS, IXGBE_EIMS_LSC); 7164 } /* ixgbe_handle_link */ 7165 7166 #if 0 7167 /************************************************************************ 7168 * ixgbe_rearm_queues 7169 ************************************************************************/ 7170 static __inline void 7171 ixgbe_rearm_queues(struct adapter *adapter, u64 queues) 7172 { 7173 u32 mask; 7174 7175 switch (adapter->hw.mac.type) { 7176 case ixgbe_mac_82598EB: 7177 mask = (IXGBE_EIMS_RTX_QUEUE & queues); 7178 IXGBE_WRITE_REG(&adapter->hw, IXGBE_EICS, mask); 7179 break; 7180 case ixgbe_mac_82599EB: 7181 case ixgbe_mac_X540: 7182 case ixgbe_mac_X550: 7183 case ixgbe_mac_X550EM_x: 7184 case ixgbe_mac_X550EM_a: 7185 mask = (queues & 0xFFFFFFFF); 7186 IXGBE_WRITE_REG(&adapter->hw, IXGBE_EICS_EX(0), mask); 7187 mask = (queues >> 32); 7188 IXGBE_WRITE_REG(&adapter->hw, IXGBE_EICS_EX(1), mask); 7189 break; 7190 default: 7191 break; 7192 } 7193 } /* ixgbe_rearm_queues */ 7194 #endif 7195
Indexes created Tue Sep 30 07:10:03 GMT 2025