ixgbe.c revision 1.329
1 /* $NetBSD: ixgbe.c,v 1.329 2023/09/13 08:05:23 msaitoh Exp $ */ 2 3 /****************************************************************************** 4 5 Copyright (c) 2001-2017, Intel Corporation 6 All rights reserved. 7 8 Redistribution and use in source and binary forms, with or without 9 modification, are permitted provided that the following conditions are met: 10 11 1. Redistributions of source code must retain the above copyright notice, 12 this list of conditions and the following disclaimer. 13 14 2. Redistributions in binary form must reproduce the above copyright 15 notice, this list of conditions and the following disclaimer in the 16 documentation and/or other materials provided with the distribution. 17 18 3. Neither the name of the Intel Corporation nor the names of its 19 contributors may be used to endorse or promote products derived from 20 this software without specific prior written permission. 21 22 THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 23 AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 24 IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 25 ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE 26 LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 27 CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 28 SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 29 INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 30 CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 31 ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 32 POSSIBILITY OF SUCH DAMAGE. 33 34 ******************************************************************************/ 35 /*$FreeBSD: head/sys/dev/ixgbe/if_ix.c 331224 2018-03-19 20:55:05Z erj $*/ 36 37 /* 38 * Copyright (c) 2011 The NetBSD Foundation, Inc. 39 * All rights reserved. 40 * 41 * This code is derived from software contributed to The NetBSD Foundation 42 * by Coyote Point Systems, Inc. 43 * 44 * Redistribution and use in source and binary forms, with or without 45 * modification, are permitted provided that the following conditions 46 * are met: 47 * 1. Redistributions of source code must retain the above copyright 48 * notice, this list of conditions and the following disclaimer. 49 * 2. Redistributions in binary form must reproduce the above copyright 50 * notice, this list of conditions and the following disclaimer in the 51 * documentation and/or other materials provided with the distribution. 52 * 53 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS 54 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED 55 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 56 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS 57 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 58 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 59 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 60 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 61 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 62 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 63 * POSSIBILITY OF SUCH DAMAGE. 64 */ 65 66 #include <sys/cdefs.h> 67 __KERNEL_RCSID(0, "$NetBSD: ixgbe.c,v 1.329 2023/09/13 08:05:23 msaitoh Exp $"); 68 69 #ifdef _KERNEL_OPT 70 #include "opt_inet.h" 71 #include "opt_inet6.h" 72 #include "opt_net_mpsafe.h" 73 #endif 74 75 #include "ixgbe.h" 76 #include "ixgbe_phy.h" 77 #include "ixgbe_sriov.h" 78 79 #include <sys/cprng.h> 80 #include <dev/mii/mii.h> 81 #include <dev/mii/miivar.h> 82 83 /************************************************************************ 84 * Driver version 85 ************************************************************************/ 86 static const char ixgbe_driver_version[] = "4.0.1-k"; 87 /* XXX NetBSD: + 3.3.24 */ 88 89 /************************************************************************ 90 * PCI Device ID Table 91 * 92 * Used by probe to select devices to load on 93 * Last field stores an index into ixgbe_strings 94 * Last entry must be all 0s 95 * 96 * { Vendor ID, Device ID, SubVendor ID, SubDevice ID, String Index } 97 ************************************************************************/ 98 static const ixgbe_vendor_info_t ixgbe_vendor_info_array[] = 99 { 100 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598AF_DUAL_PORT, 0, 0, 0}, 101 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598AF_SINGLE_PORT, 0, 0, 0}, 102 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598EB_CX4, 0, 0, 0}, 103 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598AT, 0, 0, 0}, 104 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598AT2, 0, 0, 0}, 105 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598, 0, 0, 0}, 106 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598_BX, 0, 0, 0}, 107 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598_DA_DUAL_PORT, 0, 0, 0}, 108 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598_CX4_DUAL_PORT, 0, 0, 0}, 109 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598EB_XF_LR, 0, 0, 0}, 110 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598_SR_DUAL_PORT_EM, 0, 0, 0}, 111 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598EB_SFP_LOM, 0, 0, 0}, 112 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_KR, 0, 0, 0}, 113 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_KX4, 0, 0, 0}, 114 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_KX4_MEZZ, 0, 0, 0}, 115 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_SFP, 0, 0, 0}, 116 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_SFP_EM, 0, 0, 0}, 117 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_XAUI_LOM, 0, 0, 0}, 118 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_CX4, 0, 0, 0}, 119 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_T3_LOM, 0, 0, 0}, 120 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_COMBO_BACKPLANE, 0, 0, 0}, 121 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_BACKPLANE_FCOE, 0, 0, 0}, 122 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_SFP_SF2, 0, 0, 0}, 123 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_SFP_FCOE, 0, 0, 0}, 124 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599EN_SFP, 0, 0, 0}, 125 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_SFP_SF_QP, 0, 0, 0}, 126 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_QSFP_SF_QP, 0, 0, 0}, 127 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_X540T, 0, 0, 0}, 128 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_X540T1, 0, 0, 0}, 129 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_X550T, 0, 0, 0}, 130 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_X550T1, 0, 0, 0}, 131 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_X550EM_X_KR, 0, 0, 0}, 132 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_X550EM_X_KX4, 0, 0, 0}, 133 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_X550EM_X_10G_T, 0, 0, 0}, 134 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_X550EM_X_1G_T, 0, 0, 0}, 135 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_X550EM_X_SFP, 0, 0, 0}, 136 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_X550EM_X_XFI, 0, 0, 0}, 137 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_X550EM_A_KR, 0, 0, 0}, 138 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_X550EM_A_KR_L, 0, 0, 0}, 139 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_X550EM_A_QSFP, 0, 0, 0}, 140 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_X550EM_A_QSFP_N, 0, 0, 0}, 141 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_X550EM_A_SFP, 0, 0, 0}, 142 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_X550EM_A_SFP_N, 0, 0, 0}, 143 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_X550EM_A_SGMII, 0, 0, 0}, 144 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_X550EM_A_SGMII_L, 0, 0, 0}, 145 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_X550EM_A_10G_T, 0, 0, 0}, 146 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_X550EM_A_1G_T, 0, 0, 0}, 147 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_X550EM_A_1G_T_L, 0, 0, 0}, 148 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_X540_BYPASS, 0, 0, 0}, 149 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_BYPASS, 0, 0, 0}, 150 /* required last entry */ 151 {0, 0, 0, 0, 0} 152 }; 153 154 /************************************************************************ 155 * Table of branding strings 156 ************************************************************************/ 157 static const char *ixgbe_strings[] = { 158 "Intel(R) PRO/10GbE PCI-Express Network Driver" 159 }; 160 161 /************************************************************************ 162 * Function prototypes 163 ************************************************************************/ 164 static int ixgbe_probe(device_t, cfdata_t, void *); 165 static void ixgbe_quirks(struct adapter *); 166 static void ixgbe_attach(device_t, device_t, void *); 167 static int ixgbe_detach(device_t, int); 168 #if 0 169 static int ixgbe_shutdown(device_t); 170 #endif 171 static bool ixgbe_suspend(device_t, const pmf_qual_t *); 172 static bool ixgbe_resume(device_t, const pmf_qual_t *); 173 static int ixgbe_ifflags_cb(struct ethercom *); 174 static int ixgbe_ioctl(struct ifnet *, u_long, void *); 175 static int ixgbe_init(struct ifnet *); 176 static void ixgbe_init_locked(struct adapter *); 177 static void ixgbe_ifstop(struct ifnet *, int); 178 static void ixgbe_stop_locked(void *); 179 static void ixgbe_init_device_features(struct adapter *); 180 static int ixgbe_check_fan_failure(struct adapter *, u32, bool); 181 static void ixgbe_add_media_types(struct adapter *); 182 static void ixgbe_media_status(struct ifnet *, struct ifmediareq *); 183 static int ixgbe_media_change(struct ifnet *); 184 static int ixgbe_allocate_pci_resources(struct adapter *, 185 const struct pci_attach_args *); 186 static void ixgbe_free_deferred_handlers(struct adapter *); 187 static void ixgbe_get_slot_info(struct adapter *); 188 static int ixgbe_allocate_msix(struct adapter *, 189 const struct pci_attach_args *); 190 static int ixgbe_allocate_legacy(struct adapter *, 191 const struct pci_attach_args *); 192 static int ixgbe_configure_interrupts(struct adapter *); 193 static void ixgbe_free_pciintr_resources(struct adapter *); 194 static void ixgbe_free_pci_resources(struct adapter *); 195 static void ixgbe_local_timer(void *); 196 static void ixgbe_handle_timer(struct work *, void *); 197 static void ixgbe_recovery_mode_timer(void *); 198 static void ixgbe_handle_recovery_mode_timer(struct work *, void *); 199 static int ixgbe_setup_interface(device_t, struct adapter *); 200 static void ixgbe_config_gpie(struct adapter *); 201 static void ixgbe_config_dmac(struct adapter *); 202 static void ixgbe_config_delay_values(struct adapter *); 203 static void ixgbe_schedule_admin_tasklet(struct adapter *); 204 static void ixgbe_config_link(struct adapter *); 205 static void ixgbe_check_wol_support(struct adapter *); 206 static int ixgbe_setup_low_power_mode(struct adapter *); 207 #if 0 208 static void ixgbe_rearm_queues(struct adapter *, u64); 209 #endif 210 211 static void ixgbe_initialize_transmit_units(struct adapter *); 212 static void ixgbe_initialize_receive_units(struct adapter *); 213 static void ixgbe_enable_rx_drop(struct adapter *); 214 static void ixgbe_disable_rx_drop(struct adapter *); 215 static void ixgbe_initialize_rss_mapping(struct adapter *); 216 217 static void ixgbe_enable_intr(struct adapter *); 218 static void ixgbe_disable_intr(struct adapter *); 219 static void ixgbe_update_stats_counters(struct adapter *); 220 static void ixgbe_set_rxfilter(struct adapter *); 221 static void ixgbe_update_link_status(struct adapter *); 222 static void ixgbe_set_ivar(struct adapter *, u8, u8, s8); 223 static void ixgbe_configure_ivars(struct adapter *); 224 static u8 * ixgbe_mc_array_itr(struct ixgbe_hw *, u8 **, u32 *); 225 static void ixgbe_eitr_write(struct adapter *, uint32_t, uint32_t); 226 227 static void ixgbe_setup_vlan_hw_tagging(struct adapter *); 228 static void ixgbe_setup_vlan_hw_support(struct adapter *); 229 static int ixgbe_vlan_cb(struct ethercom *, uint16_t, bool); 230 static int ixgbe_register_vlan(struct adapter *, u16); 231 static int ixgbe_unregister_vlan(struct adapter *, u16); 232 233 static void ixgbe_add_device_sysctls(struct adapter *); 234 static void ixgbe_add_hw_stats(struct adapter *); 235 static void ixgbe_clear_evcnt(struct adapter *); 236 static int ixgbe_set_flowcntl(struct adapter *, int); 237 static int ixgbe_set_advertise(struct adapter *, int); 238 static int ixgbe_get_default_advertise(struct adapter *); 239 240 /* Sysctl handlers */ 241 static int ixgbe_sysctl_flowcntl(SYSCTLFN_PROTO); 242 static int ixgbe_sysctl_advertise(SYSCTLFN_PROTO); 243 static int ixgbe_sysctl_interrupt_rate_handler(SYSCTLFN_PROTO); 244 static int ixgbe_sysctl_dmac(SYSCTLFN_PROTO); 245 static int ixgbe_sysctl_phy_temp(SYSCTLFN_PROTO); 246 static int ixgbe_sysctl_phy_overtemp_occurred(SYSCTLFN_PROTO); 247 #ifdef IXGBE_DEBUG 248 static int ixgbe_sysctl_power_state(SYSCTLFN_PROTO); 249 static int ixgbe_sysctl_print_rss_config(SYSCTLFN_PROTO); 250 #endif 251 static int ixgbe_sysctl_next_to_check_handler(SYSCTLFN_PROTO); 252 static int ixgbe_sysctl_next_to_refresh_handler(SYSCTLFN_PROTO); 253 static int ixgbe_sysctl_rdh_handler(SYSCTLFN_PROTO); 254 static int ixgbe_sysctl_rdt_handler(SYSCTLFN_PROTO); 255 static int ixgbe_sysctl_tdt_handler(SYSCTLFN_PROTO); 256 static int ixgbe_sysctl_tdh_handler(SYSCTLFN_PROTO); 257 static int ixgbe_sysctl_eee_state(SYSCTLFN_PROTO); 258 static int ixgbe_sysctl_debug(SYSCTLFN_PROTO); 259 static int ixgbe_sysctl_rx_copy_len(SYSCTLFN_PROTO); 260 static int ixgbe_sysctl_tx_process_limit(SYSCTLFN_PROTO); 261 static int ixgbe_sysctl_rx_process_limit(SYSCTLFN_PROTO); 262 static int ixgbe_sysctl_wol_enable(SYSCTLFN_PROTO); 263 static int ixgbe_sysctl_wufc(SYSCTLFN_PROTO); 264 265 /* Interrupt functions */ 266 static int ixgbe_msix_que(void *); 267 static int ixgbe_msix_admin(void *); 268 static void ixgbe_intr_admin_common(struct adapter *, u32, u32 *); 269 static int ixgbe_legacy_irq(void *); 270 271 /* Event handlers running on workqueue */ 272 static void ixgbe_handle_que(void *); 273 static void ixgbe_handle_link(void *); 274 static void ixgbe_handle_msf(void *); 275 static void ixgbe_handle_mod(void *, bool); 276 static void ixgbe_handle_phy(void *); 277 278 /* Deferred workqueue handlers */ 279 static void ixgbe_handle_admin(struct work *, void *); 280 static void ixgbe_handle_que_work(struct work *, void *); 281 282 static const ixgbe_vendor_info_t *ixgbe_lookup(const struct pci_attach_args *); 283 284 /************************************************************************ 285 * NetBSD Device Interface Entry Points 286 ************************************************************************/ 287 CFATTACH_DECL3_NEW(ixg, sizeof(struct adapter), 288 ixgbe_probe, ixgbe_attach, ixgbe_detach, NULL, NULL, NULL, 289 DVF_DETACH_SHUTDOWN); 290 291 #if 0 292 devclass_t ix_devclass; 293 DRIVER_MODULE(ix, pci, ix_driver, ix_devclass, 0, 0); 294 295 MODULE_DEPEND(ix, pci, 1, 1, 1); 296 MODULE_DEPEND(ix, ether, 1, 1, 1); 297 #ifdef DEV_NETMAP 298 MODULE_DEPEND(ix, netmap, 1, 1, 1); 299 #endif 300 #endif 301 302 /* 303 * TUNEABLE PARAMETERS: 304 */ 305 306 /* 307 * AIM: Adaptive Interrupt Moderation 308 * which means that the interrupt rate 309 * is varied over time based on the 310 * traffic for that interrupt vector 311 */ 312 static bool ixgbe_enable_aim = true; 313 #define SYSCTL_INT(_a1, _a2, _a3, _a4, _a5, _a6, _a7) 314 SYSCTL_INT(_hw_ix, OID_AUTO, enable_aim, CTLFLAG_RDTUN, &ixgbe_enable_aim, 0, 315 "Enable adaptive interrupt moderation"); 316 317 static int ixgbe_max_interrupt_rate = (4000000 / IXGBE_LOW_LATENCY); 318 SYSCTL_INT(_hw_ix, OID_AUTO, max_interrupt_rate, CTLFLAG_RDTUN, 319 &ixgbe_max_interrupt_rate, 0, "Maximum interrupts per second"); 320 321 /* How many packets rxeof tries to clean at a time */ 322 static int ixgbe_rx_process_limit = 256; 323 SYSCTL_INT(_hw_ix, OID_AUTO, rx_process_limit, CTLFLAG_RDTUN, 324 &ixgbe_rx_process_limit, 0, "Maximum number of received packets to process at a time, -1 means unlimited"); 325 326 /* How many packets txeof tries to clean at a time */ 327 static int ixgbe_tx_process_limit = 256; 328 SYSCTL_INT(_hw_ix, OID_AUTO, tx_process_limit, CTLFLAG_RDTUN, 329 &ixgbe_tx_process_limit, 0, 330 "Maximum number of sent packets to process at a time, -1 means unlimited"); 331 332 /* Flow control setting, default to full */ 333 static int ixgbe_flow_control = ixgbe_fc_full; 334 SYSCTL_INT(_hw_ix, OID_AUTO, flow_control, CTLFLAG_RDTUN, 335 &ixgbe_flow_control, 0, "Default flow control used for all adapters"); 336 337 /* Which packet processing uses workqueue or softint */ 338 static bool ixgbe_txrx_workqueue = false; 339 340 /* 341 * Smart speed setting, default to on 342 * this only works as a compile option 343 * right now as its during attach, set 344 * this to 'ixgbe_smart_speed_off' to 345 * disable. 346 */ 347 static int ixgbe_smart_speed = ixgbe_smart_speed_on; 348 349 /* 350 * MSI-X should be the default for best performance, 351 * but this allows it to be forced off for testing. 352 */ 353 static int ixgbe_enable_msix = 1; 354 SYSCTL_INT(_hw_ix, OID_AUTO, enable_msix, CTLFLAG_RDTUN, &ixgbe_enable_msix, 0, 355 "Enable MSI-X interrupts"); 356 357 /* 358 * Number of Queues, can be set to 0, 359 * it then autoconfigures based on the 360 * number of cpus with a max of 8. This 361 * can be overridden manually here. 362 */ 363 static int ixgbe_num_queues = 0; 364 SYSCTL_INT(_hw_ix, OID_AUTO, num_queues, CTLFLAG_RDTUN, &ixgbe_num_queues, 0, 365 "Number of queues to configure, 0 indicates autoconfigure"); 366 367 /* 368 * Number of TX descriptors per ring, 369 * setting higher than RX as this seems 370 * the better performing choice. 371 */ 372 static int ixgbe_txd = PERFORM_TXD; 373 SYSCTL_INT(_hw_ix, OID_AUTO, txd, CTLFLAG_RDTUN, &ixgbe_txd, 0, 374 "Number of transmit descriptors per queue"); 375 376 /* Number of RX descriptors per ring */ 377 static int ixgbe_rxd = PERFORM_RXD; 378 SYSCTL_INT(_hw_ix, OID_AUTO, rxd, CTLFLAG_RDTUN, &ixgbe_rxd, 0, 379 "Number of receive descriptors per queue"); 380 381 /* 382 * Defining this on will allow the use 383 * of unsupported SFP+ modules, note that 384 * doing so you are on your own :) 385 */ 386 static int allow_unsupported_sfp = false; 387 #define TUNABLE_INT(__x, __y) 388 TUNABLE_INT("hw.ix.unsupported_sfp", &allow_unsupported_sfp); 389 390 /* 391 * Not sure if Flow Director is fully baked, 392 * so we'll default to turning it off. 393 */ 394 static int ixgbe_enable_fdir = 0; 395 SYSCTL_INT(_hw_ix, OID_AUTO, enable_fdir, CTLFLAG_RDTUN, &ixgbe_enable_fdir, 0, 396 "Enable Flow Director"); 397 398 /* Legacy Transmit (single queue) */ 399 static int ixgbe_enable_legacy_tx = 0; 400 SYSCTL_INT(_hw_ix, OID_AUTO, enable_legacy_tx, CTLFLAG_RDTUN, 401 &ixgbe_enable_legacy_tx, 0, "Enable Legacy TX flow"); 402 403 /* Receive-Side Scaling */ 404 static int ixgbe_enable_rss = 1; 405 SYSCTL_INT(_hw_ix, OID_AUTO, enable_rss, CTLFLAG_RDTUN, &ixgbe_enable_rss, 0, 406 "Enable Receive-Side Scaling (RSS)"); 407 408 #if 0 409 static int (*ixgbe_start_locked)(struct ifnet *, struct tx_ring *); 410 static int (*ixgbe_ring_empty)(struct ifnet *, pcq_t *); 411 #endif 412 413 #ifdef NET_MPSAFE 414 #define IXGBE_MPSAFE 1 415 #define IXGBE_CALLOUT_FLAGS CALLOUT_MPSAFE 416 #define IXGBE_SOFTINT_FLAGS SOFTINT_MPSAFE 417 #define IXGBE_WORKQUEUE_FLAGS WQ_PERCPU | WQ_MPSAFE 418 #define IXGBE_TASKLET_WQ_FLAGS WQ_MPSAFE 419 #else 420 #define IXGBE_CALLOUT_FLAGS 0 421 #define IXGBE_SOFTINT_FLAGS 0 422 #define IXGBE_WORKQUEUE_FLAGS WQ_PERCPU 423 #define IXGBE_TASKLET_WQ_FLAGS 0 424 #endif 425 #define IXGBE_WORKQUEUE_PRI PRI_SOFTNET 426 427 /* Interval between reports of errors */ 428 static const struct timeval ixgbe_errlog_intrvl = { 60, 0 }; /* 60s */ 429 430 /************************************************************************ 431 * ixgbe_initialize_rss_mapping 432 ************************************************************************/ 433 static void 434 ixgbe_initialize_rss_mapping(struct adapter *adapter) 435 { 436 struct ixgbe_hw *hw = &adapter->hw; 437 u32 reta = 0, mrqc, rss_key[10]; 438 int queue_id, table_size, index_mult; 439 int i, j; 440 u32 rss_hash_config; 441 442 /* force use default RSS key. */ 443 #ifdef __NetBSD__ 444 rss_getkey((uint8_t *) &rss_key); 445 #else 446 if (adapter->feat_en & IXGBE_FEATURE_RSS) { 447 /* Fetch the configured RSS key */ 448 rss_getkey((uint8_t *) &rss_key); 449 } else { 450 /* set up random bits */ 451 cprng_fast(&rss_key, sizeof(rss_key)); 452 } 453 #endif 454 455 /* Set multiplier for RETA setup and table size based on MAC */ 456 index_mult = 0x1; 457 table_size = 128; 458 switch (adapter->hw.mac.type) { 459 case ixgbe_mac_82598EB: 460 index_mult = 0x11; 461 break; 462 case ixgbe_mac_X550: 463 case ixgbe_mac_X550EM_x: 464 case ixgbe_mac_X550EM_a: 465 table_size = 512; 466 break; 467 default: 468 break; 469 } 470 471 /* Set up the redirection table */ 472 for (i = 0, j = 0; i < table_size; i++, j++) { 473 if (j == adapter->num_queues) 474 j = 0; 475 476 if (adapter->feat_en & IXGBE_FEATURE_RSS) { 477 /* 478 * Fetch the RSS bucket id for the given indirection 479 * entry. Cap it at the number of configured buckets 480 * (which is num_queues.) 481 */ 482 queue_id = rss_get_indirection_to_bucket(i); 483 queue_id = queue_id % adapter->num_queues; 484 } else 485 queue_id = (j * index_mult); 486 487 /* 488 * The low 8 bits are for hash value (n+0); 489 * The next 8 bits are for hash value (n+1), etc. 490 */ 491 reta = reta >> 8; 492 reta = reta | (((uint32_t) queue_id) << 24); 493 if ((i & 3) == 3) { 494 if (i < 128) 495 IXGBE_WRITE_REG(hw, IXGBE_RETA(i >> 2), reta); 496 else 497 IXGBE_WRITE_REG(hw, IXGBE_ERETA((i >> 2) - 32), 498 reta); 499 reta = 0; 500 } 501 } 502 503 /* Now fill our hash function seeds */ 504 for (i = 0; i < 10; i++) 505 IXGBE_WRITE_REG(hw, IXGBE_RSSRK(i), rss_key[i]); 506 507 /* Perform hash on these packet types */ 508 if (adapter->feat_en & IXGBE_FEATURE_RSS) 509 rss_hash_config = rss_gethashconfig(); 510 else { 511 /* 512 * Disable UDP - IP fragments aren't currently being handled 513 * and so we end up with a mix of 2-tuple and 4-tuple 514 * traffic. 515 */ 516 rss_hash_config = RSS_HASHTYPE_RSS_IPV4 517 | RSS_HASHTYPE_RSS_TCP_IPV4 518 | RSS_HASHTYPE_RSS_IPV6 519 | RSS_HASHTYPE_RSS_TCP_IPV6 520 | RSS_HASHTYPE_RSS_IPV6_EX 521 | RSS_HASHTYPE_RSS_TCP_IPV6_EX; 522 } 523 524 mrqc = IXGBE_MRQC_RSSEN; 525 if (rss_hash_config & RSS_HASHTYPE_RSS_IPV4) 526 mrqc |= IXGBE_MRQC_RSS_FIELD_IPV4; 527 if (rss_hash_config & RSS_HASHTYPE_RSS_TCP_IPV4) 528 mrqc |= IXGBE_MRQC_RSS_FIELD_IPV4_TCP; 529 if (rss_hash_config & RSS_HASHTYPE_RSS_IPV6) 530 mrqc |= IXGBE_MRQC_RSS_FIELD_IPV6; 531 if (rss_hash_config & RSS_HASHTYPE_RSS_TCP_IPV6) 532 mrqc |= IXGBE_MRQC_RSS_FIELD_IPV6_TCP; 533 if (rss_hash_config & RSS_HASHTYPE_RSS_IPV6_EX) 534 mrqc |= IXGBE_MRQC_RSS_FIELD_IPV6_EX; 535 if (rss_hash_config & RSS_HASHTYPE_RSS_TCP_IPV6_EX) 536 mrqc |= IXGBE_MRQC_RSS_FIELD_IPV6_EX_TCP; 537 if (rss_hash_config & RSS_HASHTYPE_RSS_UDP_IPV4) 538 mrqc |= IXGBE_MRQC_RSS_FIELD_IPV4_UDP; 539 if (rss_hash_config & RSS_HASHTYPE_RSS_UDP_IPV6) 540 mrqc |= IXGBE_MRQC_RSS_FIELD_IPV6_UDP; 541 if (rss_hash_config & RSS_HASHTYPE_RSS_UDP_IPV6_EX) 542 mrqc |= IXGBE_MRQC_RSS_FIELD_IPV6_EX_UDP; 543 mrqc |= ixgbe_get_mrqc(adapter->iov_mode); 544 IXGBE_WRITE_REG(hw, IXGBE_MRQC, mrqc); 545 } /* ixgbe_initialize_rss_mapping */ 546 547 /************************************************************************ 548 * ixgbe_initialize_receive_units - Setup receive registers and features. 549 ************************************************************************/ 550 #define BSIZEPKT_ROUNDUP ((1<<IXGBE_SRRCTL_BSIZEPKT_SHIFT)-1) 551 552 static void 553 ixgbe_initialize_receive_units(struct adapter *adapter) 554 { 555 struct rx_ring *rxr = adapter->rx_rings; 556 struct ixgbe_hw *hw = &adapter->hw; 557 struct ifnet *ifp = adapter->ifp; 558 int i, j; 559 u32 bufsz, fctrl, srrctl, rxcsum; 560 u32 hlreg; 561 562 /* 563 * Make sure receives are disabled while 564 * setting up the descriptor ring 565 */ 566 ixgbe_disable_rx(hw); 567 568 /* Enable broadcasts */ 569 fctrl = IXGBE_READ_REG(hw, IXGBE_FCTRL); 570 fctrl |= IXGBE_FCTRL_BAM; 571 if (adapter->hw.mac.type == ixgbe_mac_82598EB) { 572 fctrl |= IXGBE_FCTRL_DPF; 573 fctrl |= IXGBE_FCTRL_PMCF; 574 } 575 IXGBE_WRITE_REG(hw, IXGBE_FCTRL, fctrl); 576 577 /* Set for Jumbo Frames? */ 578 hlreg = IXGBE_READ_REG(hw, IXGBE_HLREG0); 579 if (ifp->if_mtu > ETHERMTU) 580 hlreg |= IXGBE_HLREG0_JUMBOEN; 581 else 582 hlreg &= ~IXGBE_HLREG0_JUMBOEN; 583 584 #ifdef DEV_NETMAP 585 /* CRC stripping is conditional in Netmap */ 586 if ((adapter->feat_en & IXGBE_FEATURE_NETMAP) && 587 (ifp->if_capenable & IFCAP_NETMAP) && 588 !ix_crcstrip) 589 hlreg &= ~IXGBE_HLREG0_RXCRCSTRP; 590 else 591 #endif /* DEV_NETMAP */ 592 hlreg |= IXGBE_HLREG0_RXCRCSTRP; 593 594 IXGBE_WRITE_REG(hw, IXGBE_HLREG0, hlreg); 595 596 bufsz = (adapter->rx_mbuf_sz + BSIZEPKT_ROUNDUP) >> 597 IXGBE_SRRCTL_BSIZEPKT_SHIFT; 598 599 for (i = 0; i < adapter->num_queues; i++, rxr++) { 600 u64 rdba = rxr->rxdma.dma_paddr; 601 u32 reg; 602 int regnum = i / 4; /* 1 register per 4 queues */ 603 int regshift = i % 4; /* 4 bits per 1 queue */ 604 j = rxr->me; 605 606 /* Setup the Base and Length of the Rx Descriptor Ring */ 607 IXGBE_WRITE_REG(hw, IXGBE_RDBAL(j), 608 (rdba & 0x00000000ffffffffULL)); 609 IXGBE_WRITE_REG(hw, IXGBE_RDBAH(j), (rdba >> 32)); 610 IXGBE_WRITE_REG(hw, IXGBE_RDLEN(j), 611 adapter->num_rx_desc * sizeof(union ixgbe_adv_rx_desc)); 612 613 /* Set up the SRRCTL register */ 614 srrctl = IXGBE_READ_REG(hw, IXGBE_SRRCTL(j)); 615 srrctl &= ~IXGBE_SRRCTL_BSIZEHDR_MASK; 616 srrctl &= ~IXGBE_SRRCTL_BSIZEPKT_MASK; 617 srrctl |= bufsz; 618 srrctl |= IXGBE_SRRCTL_DESCTYPE_ADV_ONEBUF; 619 620 /* Set RQSMR (Receive Queue Statistic Mapping) register */ 621 reg = IXGBE_READ_REG(hw, IXGBE_RQSMR(regnum)); 622 reg &= ~(0x000000ffUL << (regshift * 8)); 623 reg |= i << (regshift * 8); 624 IXGBE_WRITE_REG(hw, IXGBE_RQSMR(regnum), reg); 625 626 /* 627 * Set DROP_EN iff we have no flow control and >1 queue. 628 * Note that srrctl was cleared shortly before during reset, 629 * so we do not need to clear the bit, but do it just in case 630 * this code is moved elsewhere. 631 */ 632 if ((adapter->num_queues > 1) && 633 (adapter->hw.fc.requested_mode == ixgbe_fc_none)) 634 srrctl |= IXGBE_SRRCTL_DROP_EN; 635 else 636 srrctl &= ~IXGBE_SRRCTL_DROP_EN; 637 638 IXGBE_WRITE_REG(hw, IXGBE_SRRCTL(j), srrctl); 639 640 /* Setup the HW Rx Head and Tail Descriptor Pointers */ 641 IXGBE_WRITE_REG(hw, IXGBE_RDH(j), 0); 642 IXGBE_WRITE_REG(hw, IXGBE_RDT(j), 0); 643 644 /* Set the driver rx tail address */ 645 rxr->tail = IXGBE_RDT(rxr->me); 646 } 647 648 if (adapter->hw.mac.type != ixgbe_mac_82598EB) { 649 u32 psrtype = IXGBE_PSRTYPE_TCPHDR 650 | IXGBE_PSRTYPE_UDPHDR 651 | IXGBE_PSRTYPE_IPV4HDR 652 | IXGBE_PSRTYPE_IPV6HDR; 653 IXGBE_WRITE_REG(hw, IXGBE_PSRTYPE(0), psrtype); 654 } 655 656 rxcsum = IXGBE_READ_REG(hw, IXGBE_RXCSUM); 657 658 ixgbe_initialize_rss_mapping(adapter); 659 660 if (adapter->num_queues > 1) { 661 /* RSS and RX IPP Checksum are mutually exclusive */ 662 rxcsum |= IXGBE_RXCSUM_PCSD; 663 } 664 665 if (ifp->if_capenable & IFCAP_RXCSUM) 666 rxcsum |= IXGBE_RXCSUM_PCSD; 667 668 /* This is useful for calculating UDP/IP fragment checksums */ 669 if (!(rxcsum & IXGBE_RXCSUM_PCSD)) 670 rxcsum |= IXGBE_RXCSUM_IPPCSE; 671 672 IXGBE_WRITE_REG(hw, IXGBE_RXCSUM, rxcsum); 673 674 } /* ixgbe_initialize_receive_units */ 675 676 /************************************************************************ 677 * ixgbe_initialize_transmit_units - Enable transmit units. 678 ************************************************************************/ 679 static void 680 ixgbe_initialize_transmit_units(struct adapter *adapter) 681 { 682 struct tx_ring *txr = adapter->tx_rings; 683 struct ixgbe_hw *hw = &adapter->hw; 684 int i; 685 686 INIT_DEBUGOUT("ixgbe_initialize_transmit_units"); 687 688 /* Setup the Base and Length of the Tx Descriptor Ring */ 689 for (i = 0; i < adapter->num_queues; i++, txr++) { 690 u64 tdba = txr->txdma.dma_paddr; 691 u32 txctrl = 0; 692 u32 tqsmreg, reg; 693 int regnum = i / 4; /* 1 register per 4 queues */ 694 int regshift = i % 4; /* 4 bits per 1 queue */ 695 int j = txr->me; 696 697 IXGBE_WRITE_REG(hw, IXGBE_TDBAL(j), 698 (tdba & 0x00000000ffffffffULL)); 699 IXGBE_WRITE_REG(hw, IXGBE_TDBAH(j), (tdba >> 32)); 700 IXGBE_WRITE_REG(hw, IXGBE_TDLEN(j), 701 adapter->num_tx_desc * sizeof(union ixgbe_adv_tx_desc)); 702 703 /* 704 * Set TQSMR (Transmit Queue Statistic Mapping) register. 705 * Register location is different between 82598 and others. 706 */ 707 if (adapter->hw.mac.type == ixgbe_mac_82598EB) 708 tqsmreg = IXGBE_TQSMR(regnum); 709 else 710 tqsmreg = IXGBE_TQSM(regnum); 711 reg = IXGBE_READ_REG(hw, tqsmreg); 712 reg &= ~(0x000000ffUL << (regshift * 8)); 713 reg |= i << (regshift * 8); 714 IXGBE_WRITE_REG(hw, tqsmreg, reg); 715 716 /* Setup the HW Tx Head and Tail descriptor pointers */ 717 IXGBE_WRITE_REG(hw, IXGBE_TDH(j), 0); 718 IXGBE_WRITE_REG(hw, IXGBE_TDT(j), 0); 719 720 /* Cache the tail address */ 721 txr->tail = IXGBE_TDT(j); 722 723 txr->txr_no_space = false; 724 725 /* Disable Head Writeback */ 726 /* 727 * Note: for X550 series devices, these registers are actually 728 * prefixed with TPH_ instead of DCA_, but the addresses and 729 * fields remain the same. 730 */ 731 switch (hw->mac.type) { 732 case ixgbe_mac_82598EB: 733 txctrl = IXGBE_READ_REG(hw, IXGBE_DCA_TXCTRL(j)); 734 break; 735 default: 736 txctrl = IXGBE_READ_REG(hw, IXGBE_DCA_TXCTRL_82599(j)); 737 break; 738 } 739 txctrl &= ~IXGBE_DCA_TXCTRL_DESC_WRO_EN; 740 switch (hw->mac.type) { 741 case ixgbe_mac_82598EB: 742 IXGBE_WRITE_REG(hw, IXGBE_DCA_TXCTRL(j), txctrl); 743 break; 744 default: 745 IXGBE_WRITE_REG(hw, IXGBE_DCA_TXCTRL_82599(j), txctrl); 746 break; 747 } 748 749 } 750 751 if (hw->mac.type != ixgbe_mac_82598EB) { 752 u32 dmatxctl, rttdcs; 753 754 dmatxctl = IXGBE_READ_REG(hw, IXGBE_DMATXCTL); 755 dmatxctl |= IXGBE_DMATXCTL_TE; 756 IXGBE_WRITE_REG(hw, IXGBE_DMATXCTL, dmatxctl); 757 /* Disable arbiter to set MTQC */ 758 rttdcs = IXGBE_READ_REG(hw, IXGBE_RTTDCS); 759 rttdcs |= IXGBE_RTTDCS_ARBDIS; 760 IXGBE_WRITE_REG(hw, IXGBE_RTTDCS, rttdcs); 761 IXGBE_WRITE_REG(hw, IXGBE_MTQC, 762 ixgbe_get_mtqc(adapter->iov_mode)); 763 rttdcs &= ~IXGBE_RTTDCS_ARBDIS; 764 IXGBE_WRITE_REG(hw, IXGBE_RTTDCS, rttdcs); 765 } 766 767 return; 768 } /* ixgbe_initialize_transmit_units */ 769 770 static void 771 ixgbe_quirks(struct adapter *adapter) 772 { 773 device_t dev = adapter->dev; 774 struct ixgbe_hw *hw = &adapter->hw; 775 const char *vendor, *product; 776 777 if (hw->device_id == IXGBE_DEV_ID_X550EM_A_SFP_N) { 778 /* 779 * Quirk for inverted logic of SFP+'s MOD_ABS on GIGABYTE 780 * MA10-ST0. 781 */ 782 vendor = pmf_get_platform("system-vendor"); 783 product = pmf_get_platform("system-product"); 784 785 if ((vendor == NULL) || (product == NULL)) 786 return; 787 788 if ((strcmp(vendor, "GIGABYTE") == 0) && 789 (strcmp(product, "MA10-ST0") == 0)) { 790 aprint_verbose_dev(dev, 791 "Enable SFP+ MOD_ABS inverse quirk\n"); 792 adapter->hw.quirks |= IXGBE_QUIRK_MOD_ABS_INVERT; 793 } 794 } 795 } 796 797 /************************************************************************ 798 * ixgbe_attach - Device initialization routine 799 * 800 * Called when the driver is being loaded. 801 * Identifies the type of hardware, allocates all resources 802 * and initializes the hardware. 803 * 804 * return 0 on success, positive on failure 805 ************************************************************************/ 806 static void 807 ixgbe_attach(device_t parent, device_t dev, void *aux) 808 { 809 struct adapter *adapter; 810 struct ixgbe_hw *hw; 811 int error = -1; 812 u32 ctrl_ext; 813 u16 high, low, nvmreg; 814 pcireg_t id, subid; 815 const ixgbe_vendor_info_t *ent; 816 struct pci_attach_args *pa = aux; 817 bool unsupported_sfp = false; 818 const char *str; 819 char wqname[MAXCOMLEN]; 820 char buf[256]; 821 822 INIT_DEBUGOUT("ixgbe_attach: begin"); 823 824 /* Allocate, clear, and link in our adapter structure */ 825 adapter = device_private(dev); 826 adapter->hw.back = adapter; 827 adapter->dev = dev; 828 hw = &adapter->hw; 829 adapter->osdep.pc = pa->pa_pc; 830 adapter->osdep.tag = pa->pa_tag; 831 if (pci_dma64_available(pa)) 832 adapter->osdep.dmat = pa->pa_dmat64; 833 else 834 adapter->osdep.dmat = pa->pa_dmat; 835 adapter->osdep.attached = false; 836 adapter->osdep.detaching = false; 837 838 ent = ixgbe_lookup(pa); 839 840 KASSERT(ent != NULL); 841 842 aprint_normal(": %s, Version - %s\n", 843 ixgbe_strings[ent->index], ixgbe_driver_version); 844 845 /* Core Lock Init */ 846 IXGBE_CORE_LOCK_INIT(adapter, device_xname(dev)); 847 848 /* Set up the timer callout and workqueue */ 849 callout_init(&adapter->timer, IXGBE_CALLOUT_FLAGS); 850 snprintf(wqname, sizeof(wqname), "%s-timer", device_xname(dev)); 851 error = workqueue_create(&adapter->timer_wq, wqname, 852 ixgbe_handle_timer, adapter, IXGBE_WORKQUEUE_PRI, IPL_NET, 853 IXGBE_TASKLET_WQ_FLAGS); 854 if (error) { 855 aprint_error_dev(dev, 856 "could not create timer workqueue (%d)\n", error); 857 goto err_out; 858 } 859 860 /* Determine hardware revision */ 861 id = pci_conf_read(pa->pa_pc, pa->pa_tag, PCI_ID_REG); 862 subid = pci_conf_read(pa->pa_pc, pa->pa_tag, PCI_SUBSYS_ID_REG); 863 864 hw->vendor_id = PCI_VENDOR(id); 865 hw->device_id = PCI_PRODUCT(id); 866 hw->revision_id = 867 PCI_REVISION(pci_conf_read(pa->pa_pc, pa->pa_tag, PCI_CLASS_REG)); 868 hw->subsystem_vendor_id = PCI_SUBSYS_VENDOR(subid); 869 hw->subsystem_device_id = PCI_SUBSYS_ID(subid); 870 871 /* Set quirk flags */ 872 ixgbe_quirks(adapter); 873 874 /* 875 * Make sure BUSMASTER is set 876 */ 877 ixgbe_pci_enable_busmaster(pa->pa_pc, pa->pa_tag); 878 879 /* Do base PCI setup - map BAR0 */ 880 if (ixgbe_allocate_pci_resources(adapter, pa)) { 881 aprint_error_dev(dev, "Allocation of PCI resources failed\n"); 882 error = ENXIO; 883 goto err_out; 884 } 885 886 /* let hardware know driver is loaded */ 887 ctrl_ext = IXGBE_READ_REG(hw, IXGBE_CTRL_EXT); 888 ctrl_ext |= IXGBE_CTRL_EXT_DRV_LOAD; 889 IXGBE_WRITE_REG(hw, IXGBE_CTRL_EXT, ctrl_ext); 890 891 /* 892 * Initialize the shared code 893 */ 894 if (ixgbe_init_shared_code(hw) != 0) { 895 aprint_error_dev(dev, 896 "Unable to initialize the shared code\n"); 897 error = ENXIO; 898 goto err_out; 899 } 900 901 switch (hw->mac.type) { 902 case ixgbe_mac_82598EB: 903 str = "82598EB"; 904 break; 905 case ixgbe_mac_82599EB: 906 str = "82599EB"; 907 break; 908 case ixgbe_mac_X540: 909 str = "X540"; 910 break; 911 case ixgbe_mac_X550: 912 str = "X550"; 913 break; 914 case ixgbe_mac_X550EM_x: 915 str = "X550EM X"; 916 break; 917 case ixgbe_mac_X550EM_a: 918 str = "X550EM A"; 919 break; 920 default: 921 str = "Unknown"; 922 break; 923 } 924 aprint_normal_dev(dev, "device %s\n", str); 925 926 hw->allow_unsupported_sfp = allow_unsupported_sfp; 927 928 /* Pick up the 82599 settings */ 929 if (hw->mac.type != ixgbe_mac_82598EB) 930 hw->phy.smart_speed = ixgbe_smart_speed; 931 932 /* Set the right number of segments */ 933 KASSERT(IXGBE_82599_SCATTER_MAX >= IXGBE_SCATTER_DEFAULT); 934 adapter->num_segs = IXGBE_SCATTER_DEFAULT; 935 936 /* Ensure SW/FW semaphore is free */ 937 ixgbe_init_swfw_semaphore(hw); 938 939 hw->mac.ops.set_lan_id(hw); 940 ixgbe_init_device_features(adapter); 941 942 if (ixgbe_configure_interrupts(adapter)) { 943 error = ENXIO; 944 goto err_out; 945 } 946 947 /* Allocate multicast array memory. */ 948 adapter->mta = malloc(sizeof(*adapter->mta) * 949 MAX_NUM_MULTICAST_ADDRESSES, M_DEVBUF, M_WAITOK); 950 951 /* Enable WoL (if supported) */ 952 ixgbe_check_wol_support(adapter); 953 954 /* Register for VLAN events */ 955 ether_set_vlan_cb(&adapter->osdep.ec, ixgbe_vlan_cb); 956 957 /* Verify adapter fan is still functional (if applicable) */ 958 if (adapter->feat_en & IXGBE_FEATURE_FAN_FAIL) { 959 u32 esdp = IXGBE_READ_REG(hw, IXGBE_ESDP); 960 ixgbe_check_fan_failure(adapter, esdp, FALSE); 961 } 962 963 /* Set an initial default flow control value */ 964 hw->fc.requested_mode = ixgbe_flow_control; 965 966 /* Do descriptor calc and sanity checks */ 967 if (((ixgbe_txd * sizeof(union ixgbe_adv_tx_desc)) % DBA_ALIGN) != 0 || 968 ixgbe_txd < MIN_TXD || ixgbe_txd > MAX_TXD) { 969 aprint_error_dev(dev, "TXD config issue, using default!\n"); 970 adapter->num_tx_desc = DEFAULT_TXD; 971 } else 972 adapter->num_tx_desc = ixgbe_txd; 973 974 if (((ixgbe_rxd * sizeof(union ixgbe_adv_rx_desc)) % DBA_ALIGN) != 0 || 975 ixgbe_rxd < MIN_RXD || ixgbe_rxd > MAX_RXD) { 976 aprint_error_dev(dev, "RXD config issue, using default!\n"); 977 adapter->num_rx_desc = DEFAULT_RXD; 978 } else 979 adapter->num_rx_desc = ixgbe_rxd; 980 981 /* Sysctls for limiting the amount of work done in the taskqueues */ 982 adapter->rx_process_limit 983 = (ixgbe_rx_process_limit <= adapter->num_rx_desc) 984 ? ixgbe_rx_process_limit : adapter->num_rx_desc; 985 adapter->tx_process_limit 986 = (ixgbe_tx_process_limit <= adapter->num_tx_desc) 987 ? ixgbe_tx_process_limit : adapter->num_tx_desc; 988 989 /* Set default high limit of copying mbuf in rxeof */ 990 adapter->rx_copy_len = IXGBE_RX_COPY_LEN_MAX; 991 992 /* Allocate our TX/RX Queues */ 993 if (ixgbe_allocate_queues(adapter)) { 994 error = ENOMEM; 995 goto err_out; 996 } 997 998 hw->phy.reset_if_overtemp = TRUE; 999 error = ixgbe_reset_hw(hw); 1000 hw->phy.reset_if_overtemp = FALSE; 1001 if (error == IXGBE_ERR_SFP_NOT_PRESENT) 1002 error = IXGBE_SUCCESS; 1003 else if (error == IXGBE_ERR_SFP_NOT_SUPPORTED) { 1004 aprint_error_dev(dev, 1005 "Unsupported SFP+ module type was detected.\n"); 1006 unsupported_sfp = true; 1007 error = IXGBE_SUCCESS; 1008 } else if (error) { 1009 aprint_error_dev(dev, 1010 "Hardware initialization failed(error = %d)\n", error); 1011 error = EIO; 1012 goto err_late; 1013 } 1014 1015 /* Make sure we have a good EEPROM before we read from it */ 1016 if (ixgbe_validate_eeprom_checksum(&adapter->hw, NULL) < 0) { 1017 aprint_error_dev(dev, "The EEPROM Checksum Is Not Valid\n"); 1018 error = EIO; 1019 goto err_late; 1020 } 1021 1022 aprint_normal("%s:", device_xname(dev)); 1023 /* NVM Image Version */ 1024 high = low = 0; 1025 switch (hw->mac.type) { 1026 case ixgbe_mac_82598EB: 1027 /* 1028 * Print version from the dev starter version (0x29). The 1029 * location is the same as newer device's IXGBE_NVM_MAP_VER. 1030 */ 1031 hw->eeprom.ops.read(hw, IXGBE_NVM_MAP_VER, &nvmreg); 1032 if (nvmreg == 0xffff) 1033 break; 1034 high = (nvmreg >> 12) & 0x0f; 1035 low = (nvmreg >> 4) & 0xff; 1036 id = nvmreg & 0x0f; 1037 /* 1038 * The following output might not be correct. Some 82598 cards 1039 * have 0x1070 or 0x2090. 82598 spec update notes about 2.9.0. 1040 */ 1041 aprint_normal(" NVM Image Version %u.%u.%u,", high, low, id); 1042 break; 1043 case ixgbe_mac_X540: 1044 case ixgbe_mac_X550EM_a: 1045 hw->eeprom.ops.read(hw, IXGBE_NVM_IMAGE_VER, &nvmreg); 1046 if (nvmreg == 0xffff) 1047 break; 1048 high = (nvmreg >> 12) & 0x0f; 1049 low = (nvmreg >> 4) & 0xff; 1050 id = nvmreg & 0x0f; 1051 aprint_normal(" NVM Image Version %u.", high); 1052 if (hw->mac.type == ixgbe_mac_X540) 1053 str = "%x"; 1054 else 1055 str = "%02x"; 1056 aprint_normal(str, low); 1057 aprint_normal(" ID 0x%x,", id); 1058 break; 1059 case ixgbe_mac_X550EM_x: 1060 case ixgbe_mac_X550: 1061 hw->eeprom.ops.read(hw, IXGBE_NVM_IMAGE_VER, &nvmreg); 1062 if (nvmreg == 0xffff) 1063 break; 1064 high = (nvmreg >> 12) & 0x0f; 1065 low = nvmreg & 0xff; 1066 aprint_normal(" NVM Image Version %u.%02x,", high, low); 1067 break; 1068 default: 1069 break; 1070 } 1071 hw->eeprom.nvm_image_ver_high = high; 1072 hw->eeprom.nvm_image_ver_low = low; 1073 1074 /* PHY firmware revision */ 1075 switch (hw->mac.type) { 1076 case ixgbe_mac_X540: 1077 case ixgbe_mac_X550: 1078 hw->eeprom.ops.read(hw, IXGBE_PHYFW_REV, &nvmreg); 1079 if (nvmreg == 0xffff) 1080 break; 1081 high = (nvmreg >> 12) & 0x0f; 1082 low = (nvmreg >> 4) & 0xff; 1083 id = nvmreg & 0x000f; 1084 aprint_normal(" PHY FW Revision %u.", high); 1085 if (hw->mac.type == ixgbe_mac_X540) 1086 str = "%x"; 1087 else 1088 str = "%02x"; 1089 aprint_normal(str, low); 1090 aprint_normal(" ID 0x%x,", id); 1091 break; 1092 default: 1093 break; 1094 } 1095 1096 /* NVM Map version & OEM NVM Image version */ 1097 switch (hw->mac.type) { 1098 case ixgbe_mac_X550: 1099 case ixgbe_mac_X550EM_x: 1100 case ixgbe_mac_X550EM_a: 1101 hw->eeprom.ops.read(hw, IXGBE_NVM_MAP_VER, &nvmreg); 1102 if (nvmreg != 0xffff) { 1103 high = (nvmreg >> 12) & 0x0f; 1104 low = nvmreg & 0x00ff; 1105 aprint_normal(" NVM Map version %u.%02x,", high, low); 1106 } 1107 hw->eeprom.ops.read(hw, IXGBE_OEM_NVM_IMAGE_VER, &nvmreg); 1108 if (nvmreg != 0xffff) { 1109 high = (nvmreg >> 12) & 0x0f; 1110 low = nvmreg & 0x00ff; 1111 aprint_verbose(" OEM NVM Image version %u.%02x,", high, 1112 low); 1113 } 1114 break; 1115 default: 1116 break; 1117 } 1118 1119 /* Print the ETrackID */ 1120 hw->eeprom.ops.read(hw, IXGBE_ETRACKID_H, &high); 1121 hw->eeprom.ops.read(hw, IXGBE_ETRACKID_L, &low); 1122 aprint_normal(" ETrackID %08x\n", ((uint32_t)high << 16) | low); 1123 1124 /* Printed Board Assembly number */ 1125 error = ixgbe_read_pba_string(hw, buf, IXGBE_PBANUM_LENGTH); 1126 aprint_normal_dev(dev, "PBA number %s\n", error ? "unknown" : buf); 1127 1128 if (adapter->feat_en & IXGBE_FEATURE_MSIX) { 1129 error = ixgbe_allocate_msix(adapter, pa); 1130 if (error) { 1131 /* Free allocated queue structures first */ 1132 ixgbe_free_queues(adapter); 1133 1134 /* Fallback to legacy interrupt */ 1135 if (adapter->feat_cap & IXGBE_FEATURE_MSI) 1136 adapter->feat_en |= IXGBE_FEATURE_MSI; 1137 adapter->num_queues = 1; 1138 1139 /* Allocate our TX/RX Queues again */ 1140 if (ixgbe_allocate_queues(adapter)) { 1141 error = ENOMEM; 1142 goto err_out; 1143 } 1144 } 1145 } 1146 1147 /* Recovery mode */ 1148 switch (adapter->hw.mac.type) { 1149 case ixgbe_mac_X550: 1150 case ixgbe_mac_X550EM_x: 1151 case ixgbe_mac_X550EM_a: 1152 /* >= 2.00 */ 1153 if (hw->eeprom.nvm_image_ver_high >= 2) { 1154 adapter->feat_cap |= IXGBE_FEATURE_RECOVERY_MODE; 1155 adapter->feat_en |= IXGBE_FEATURE_RECOVERY_MODE; 1156 } 1157 break; 1158 default: 1159 break; 1160 } 1161 1162 if ((adapter->feat_en & IXGBE_FEATURE_MSIX) == 0) 1163 error = ixgbe_allocate_legacy(adapter, pa); 1164 if (error) 1165 goto err_late; 1166 1167 /* Tasklets for Link, SFP, Multispeed Fiber and Flow Director */ 1168 mutex_init(&(adapter)->admin_mtx, MUTEX_DEFAULT, IPL_NET); 1169 snprintf(wqname, sizeof(wqname), "%s-admin", device_xname(dev)); 1170 error = workqueue_create(&adapter->admin_wq, wqname, 1171 ixgbe_handle_admin, adapter, IXGBE_WORKQUEUE_PRI, IPL_NET, 1172 IXGBE_TASKLET_WQ_FLAGS); 1173 if (error) { 1174 aprint_error_dev(dev, 1175 "could not create admin workqueue (%d)\n", error); 1176 goto err_out; 1177 } 1178 1179 error = ixgbe_start_hw(hw); 1180 switch (error) { 1181 case IXGBE_ERR_EEPROM_VERSION: 1182 aprint_error_dev(dev, 1183 "This device is a pre-production adapter/" 1184 "LOM. Please be aware there may be issues associated " 1185 "with your hardware.\nIf you are experiencing problems " 1186 "please contact your Intel or hardware representative " 1187 "who provided you with this hardware.\n"); 1188 break; 1189 default: 1190 break; 1191 } 1192 1193 /* Setup OS specific network interface */ 1194 if (ixgbe_setup_interface(dev, adapter) != 0) 1195 goto err_late; 1196 1197 /* 1198 * Print PHY ID only for copper PHY. On device which has SFP(+) cage 1199 * and a module is inserted, phy.id is not MII PHY id but SFF 8024 ID. 1200 */ 1201 if (hw->phy.media_type == ixgbe_media_type_copper) { 1202 uint16_t id1, id2; 1203 int oui, model, rev; 1204 char descr[MII_MAX_DESCR_LEN]; 1205 1206 id1 = hw->phy.id >> 16; 1207 id2 = hw->phy.id & 0xffff; 1208 oui = MII_OUI(id1, id2); 1209 model = MII_MODEL(id2); 1210 rev = MII_REV(id2); 1211 mii_get_descr(descr, sizeof(descr), oui, model); 1212 if (descr[0]) 1213 aprint_normal_dev(dev, "PHY: %s, rev. %d\n", 1214 descr, rev); 1215 else 1216 aprint_normal_dev(dev, 1217 "PHY OUI 0x%06x, model 0x%04x, rev. %d\n", 1218 oui, model, rev); 1219 } 1220 1221 /* Enable EEE power saving */ 1222 if (adapter->feat_cap & IXGBE_FEATURE_EEE) 1223 hw->mac.ops.setup_eee(hw, 1224 adapter->feat_en & IXGBE_FEATURE_EEE); 1225 1226 /* Enable power to the phy. */ 1227 if (!unsupported_sfp) { 1228 /* Enable the optics for 82599 SFP+ fiber */ 1229 ixgbe_enable_tx_laser(hw); 1230 1231 /* 1232 * XXX Currently, ixgbe_set_phy_power() supports only copper 1233 * PHY, so it's not required to test with !unsupported_sfp. 1234 */ 1235 ixgbe_set_phy_power(hw, TRUE); 1236 } 1237 1238 /* Initialize statistics */ 1239 ixgbe_update_stats_counters(adapter); 1240 1241 /* Check PCIE slot type/speed/width */ 1242 ixgbe_get_slot_info(adapter); 1243 1244 /* 1245 * Do time init and sysctl init here, but 1246 * only on the first port of a bypass adapter. 1247 */ 1248 ixgbe_bypass_init(adapter); 1249 1250 /* Set an initial dmac value */ 1251 adapter->dmac = 0; 1252 /* Set initial advertised speeds (if applicable) */ 1253 adapter->advertise = ixgbe_get_default_advertise(adapter); 1254 1255 if (adapter->feat_cap & IXGBE_FEATURE_SRIOV) 1256 ixgbe_define_iov_schemas(dev, &error); 1257 1258 /* Add sysctls */ 1259 ixgbe_add_device_sysctls(adapter); 1260 ixgbe_add_hw_stats(adapter); 1261 1262 /* For Netmap */ 1263 adapter->init_locked = ixgbe_init_locked; 1264 adapter->stop_locked = ixgbe_stop_locked; 1265 1266 if (adapter->feat_en & IXGBE_FEATURE_NETMAP) 1267 ixgbe_netmap_attach(adapter); 1268 1269 snprintb(buf, sizeof(buf), IXGBE_FEATURE_FLAGS, adapter->feat_cap); 1270 aprint_verbose_dev(dev, "feature cap %s\n", buf); 1271 snprintb(buf, sizeof(buf), IXGBE_FEATURE_FLAGS, adapter->feat_en); 1272 aprint_verbose_dev(dev, "feature ena %s\n", buf); 1273 1274 if (pmf_device_register(dev, ixgbe_suspend, ixgbe_resume)) 1275 pmf_class_network_register(dev, adapter->ifp); 1276 else 1277 aprint_error_dev(dev, "couldn't establish power handler\n"); 1278 1279 /* Init recovery mode timer and state variable */ 1280 if (adapter->feat_en & IXGBE_FEATURE_RECOVERY_MODE) { 1281 adapter->recovery_mode = 0; 1282 1283 /* Set up the timer callout */ 1284 callout_init(&adapter->recovery_mode_timer, 1285 IXGBE_CALLOUT_FLAGS); 1286 snprintf(wqname, sizeof(wqname), "%s-recovery", 1287 device_xname(dev)); 1288 error = workqueue_create(&adapter->recovery_mode_timer_wq, 1289 wqname, ixgbe_handle_recovery_mode_timer, adapter, 1290 IXGBE_WORKQUEUE_PRI, IPL_NET, IXGBE_TASKLET_WQ_FLAGS); 1291 if (error) { 1292 aprint_error_dev(dev, "could not create " 1293 "recovery_mode_timer workqueue (%d)\n", error); 1294 goto err_out; 1295 } 1296 1297 /* Start the task */ 1298 callout_reset(&adapter->recovery_mode_timer, hz, 1299 ixgbe_recovery_mode_timer, adapter); 1300 } 1301 1302 INIT_DEBUGOUT("ixgbe_attach: end"); 1303 adapter->osdep.attached = true; 1304 1305 return; 1306 1307 err_late: 1308 ixgbe_free_queues(adapter); 1309 err_out: 1310 ctrl_ext = IXGBE_READ_REG(&adapter->hw, IXGBE_CTRL_EXT); 1311 ctrl_ext &= ~IXGBE_CTRL_EXT_DRV_LOAD; 1312 IXGBE_WRITE_REG(&adapter->hw, IXGBE_CTRL_EXT, ctrl_ext); 1313 ixgbe_free_deferred_handlers(adapter); 1314 ixgbe_free_pci_resources(adapter); 1315 if (adapter->mta != NULL) 1316 free(adapter->mta, M_DEVBUF); 1317 mutex_destroy(&(adapter)->admin_mtx); /* XXX appropriate order? */ 1318 IXGBE_CORE_LOCK_DESTROY(adapter); 1319 1320 return; 1321 } /* ixgbe_attach */ 1322 1323 /************************************************************************ 1324 * ixgbe_check_wol_support 1325 * 1326 * Checks whether the adapter's ports are capable of 1327 * Wake On LAN by reading the adapter's NVM. 1328 * 1329 * Sets each port's hw->wol_enabled value depending 1330 * on the value read here. 1331 ************************************************************************/ 1332 static void 1333 ixgbe_check_wol_support(struct adapter *adapter) 1334 { 1335 struct ixgbe_hw *hw = &adapter->hw; 1336 u16 dev_caps = 0; 1337 1338 /* Find out WoL support for port */ 1339 adapter->wol_support = hw->wol_enabled = 0; 1340 ixgbe_get_device_caps(hw, &dev_caps); 1341 if ((dev_caps & IXGBE_DEVICE_CAPS_WOL_PORT0_1) || 1342 ((dev_caps & IXGBE_DEVICE_CAPS_WOL_PORT0) && 1343 hw->bus.func == 0)) 1344 adapter->wol_support = hw->wol_enabled = 1; 1345 1346 /* Save initial wake up filter configuration */ 1347 adapter->wufc = IXGBE_READ_REG(hw, IXGBE_WUFC); 1348 1349 return; 1350 } /* ixgbe_check_wol_support */ 1351 1352 /************************************************************************ 1353 * ixgbe_setup_interface 1354 * 1355 * Setup networking device structure and register an interface. 1356 ************************************************************************/ 1357 static int 1358 ixgbe_setup_interface(device_t dev, struct adapter *adapter) 1359 { 1360 struct ethercom *ec = &adapter->osdep.ec; 1361 struct ifnet *ifp; 1362 1363 INIT_DEBUGOUT("ixgbe_setup_interface: begin"); 1364 1365 ifp = adapter->ifp = &ec->ec_if; 1366 strlcpy(ifp->if_xname, device_xname(dev), IFNAMSIZ); 1367 ifp->if_baudrate = IF_Gbps(10); 1368 ifp->if_init = ixgbe_init; 1369 ifp->if_stop = ixgbe_ifstop; 1370 ifp->if_softc = adapter; 1371 ifp->if_flags = IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST; 1372 #ifdef IXGBE_MPSAFE 1373 ifp->if_extflags = IFEF_MPSAFE; 1374 #endif 1375 ifp->if_ioctl = ixgbe_ioctl; 1376 #if __FreeBSD_version >= 1100045 1377 /* TSO parameters */ 1378 ifp->if_hw_tsomax = 65518; 1379 ifp->if_hw_tsomaxsegcount = IXGBE_82599_SCATTER; 1380 ifp->if_hw_tsomaxsegsize = 2048; 1381 #endif 1382 if (adapter->feat_en & IXGBE_FEATURE_LEGACY_TX) { 1383 #if 0 1384 ixgbe_start_locked = ixgbe_legacy_start_locked; 1385 #endif 1386 } else { 1387 ifp->if_transmit = ixgbe_mq_start; 1388 #if 0 1389 ixgbe_start_locked = ixgbe_mq_start_locked; 1390 #endif 1391 } 1392 ifp->if_start = ixgbe_legacy_start; 1393 IFQ_SET_MAXLEN(&ifp->if_snd, adapter->num_tx_desc - 2); 1394 IFQ_SET_READY(&ifp->if_snd); 1395 1396 if_initialize(ifp); 1397 adapter->ipq = if_percpuq_create(&adapter->osdep.ec.ec_if); 1398 ether_ifattach(ifp, adapter->hw.mac.addr); 1399 aprint_normal_dev(dev, "Ethernet address %s\n", 1400 ether_sprintf(adapter->hw.mac.addr)); 1401 /* 1402 * We use per TX queue softint, so if_deferred_start_init() isn't 1403 * used. 1404 */ 1405 ether_set_ifflags_cb(ec, ixgbe_ifflags_cb); 1406 1407 adapter->max_frame_size = ifp->if_mtu + ETHER_HDR_LEN + ETHER_CRC_LEN; 1408 1409 /* 1410 * Tell the upper layer(s) we support long frames. 1411 */ 1412 ifp->if_hdrlen = sizeof(struct ether_vlan_header); 1413 1414 /* Set capability flags */ 1415 ifp->if_capabilities |= IFCAP_RXCSUM 1416 | IFCAP_TXCSUM 1417 | IFCAP_TSOv4 1418 | IFCAP_TSOv6; 1419 ifp->if_capenable = 0; 1420 1421 ec->ec_capabilities |= ETHERCAP_VLAN_HWTAGGING 1422 | ETHERCAP_VLAN_HWCSUM 1423 | ETHERCAP_JUMBO_MTU 1424 | ETHERCAP_VLAN_MTU; 1425 1426 /* Enable the above capabilities by default */ 1427 ec->ec_capenable = ec->ec_capabilities; 1428 1429 /* 1430 * Don't turn this on by default, if vlans are 1431 * created on another pseudo device (eg. lagg) 1432 * then vlan events are not passed thru, breaking 1433 * operation, but with HW FILTER off it works. If 1434 * using vlans directly on the ixgbe driver you can 1435 * enable this and get full hardware tag filtering. 1436 */ 1437 ec->ec_capabilities |= ETHERCAP_VLAN_HWFILTER; 1438 1439 /* 1440 * Specify the media types supported by this adapter and register 1441 * callbacks to update media and link information 1442 */ 1443 ec->ec_ifmedia = &adapter->media; 1444 ifmedia_init_with_lock(&adapter->media, IFM_IMASK, ixgbe_media_change, 1445 ixgbe_media_status, &adapter->core_mtx); 1446 1447 adapter->phy_layer = ixgbe_get_supported_physical_layer(&adapter->hw); 1448 ixgbe_add_media_types(adapter); 1449 1450 /* Set autoselect media by default */ 1451 ifmedia_set(&adapter->media, IFM_ETHER | IFM_AUTO); 1452 1453 if_register(ifp); 1454 1455 return (0); 1456 } /* ixgbe_setup_interface */ 1457 1458 /************************************************************************ 1459 * ixgbe_add_media_types 1460 ************************************************************************/ 1461 static void 1462 ixgbe_add_media_types(struct adapter *adapter) 1463 { 1464 struct ixgbe_hw *hw = &adapter->hw; 1465 u64 layer; 1466 1467 layer = adapter->phy_layer; 1468 1469 #define ADD(mm, dd) \ 1470 ifmedia_add(&adapter->media, IFM_ETHER | (mm), (dd), NULL); 1471 1472 ADD(IFM_NONE, 0); 1473 1474 /* Media types with matching NetBSD media defines */ 1475 if (layer & IXGBE_PHYSICAL_LAYER_10GBASE_T) 1476 ADD(IFM_10G_T | IFM_FDX, 0); 1477 if (layer & IXGBE_PHYSICAL_LAYER_1000BASE_T) 1478 ADD(IFM_1000_T | IFM_FDX, 0); 1479 if (layer & IXGBE_PHYSICAL_LAYER_100BASE_TX) 1480 ADD(IFM_100_TX | IFM_FDX, 0); 1481 if (layer & IXGBE_PHYSICAL_LAYER_10BASE_T) 1482 ADD(IFM_10_T | IFM_FDX, 0); 1483 1484 if (layer & IXGBE_PHYSICAL_LAYER_SFP_PLUS_CU || 1485 layer & IXGBE_PHYSICAL_LAYER_SFP_ACTIVE_DA) 1486 ADD(IFM_10G_TWINAX | IFM_FDX, 0); 1487 1488 if (layer & IXGBE_PHYSICAL_LAYER_10GBASE_LR) { 1489 ADD(IFM_10G_LR | IFM_FDX, 0); 1490 if (hw->phy.multispeed_fiber) 1491 ADD(IFM_1000_LX | IFM_FDX, 0); 1492 } 1493 if (layer & IXGBE_PHYSICAL_LAYER_10GBASE_SR) { 1494 ADD(IFM_10G_SR | IFM_FDX, 0); 1495 if (hw->phy.multispeed_fiber) 1496 ADD(IFM_1000_SX | IFM_FDX, 0); 1497 } else if (layer & IXGBE_PHYSICAL_LAYER_1000BASE_SX) 1498 ADD(IFM_1000_SX | IFM_FDX, 0); 1499 if (layer & IXGBE_PHYSICAL_LAYER_10GBASE_CX4) 1500 ADD(IFM_10G_CX4 | IFM_FDX, 0); 1501 1502 if (layer & IXGBE_PHYSICAL_LAYER_10GBASE_KR) 1503 ADD(IFM_10G_KR | IFM_FDX, 0); 1504 if (layer & IXGBE_PHYSICAL_LAYER_10GBASE_KX4) 1505 ADD(IFM_10G_KX4 | IFM_FDX, 0); 1506 if (layer & IXGBE_PHYSICAL_LAYER_1000BASE_KX) 1507 ADD(IFM_1000_KX | IFM_FDX, 0); 1508 if (layer & IXGBE_PHYSICAL_LAYER_2500BASE_KX) 1509 ADD(IFM_2500_KX | IFM_FDX, 0); 1510 if (layer & IXGBE_PHYSICAL_LAYER_2500BASE_T) 1511 ADD(IFM_2500_T | IFM_FDX, 0); 1512 if (layer & IXGBE_PHYSICAL_LAYER_5GBASE_T) 1513 ADD(IFM_5000_T | IFM_FDX, 0); 1514 if (layer & IXGBE_PHYSICAL_LAYER_1000BASE_BX) 1515 ADD(IFM_1000_BX10 | IFM_FDX, 0); 1516 /* XXX no ifmedia_set? */ 1517 1518 ADD(IFM_AUTO, 0); 1519 1520 #undef ADD 1521 } /* ixgbe_add_media_types */ 1522 1523 /************************************************************************ 1524 * ixgbe_is_sfp 1525 ************************************************************************/ 1526 static inline bool 1527 ixgbe_is_sfp(struct ixgbe_hw *hw) 1528 { 1529 switch (hw->mac.type) { 1530 case ixgbe_mac_82598EB: 1531 if (hw->phy.type == ixgbe_phy_nl) 1532 return (TRUE); 1533 return (FALSE); 1534 case ixgbe_mac_82599EB: 1535 case ixgbe_mac_X550EM_x: 1536 case ixgbe_mac_X550EM_a: 1537 switch (hw->mac.ops.get_media_type(hw)) { 1538 case ixgbe_media_type_fiber: 1539 case ixgbe_media_type_fiber_qsfp: 1540 return (TRUE); 1541 default: 1542 return (FALSE); 1543 } 1544 default: 1545 return (FALSE); 1546 } 1547 } /* ixgbe_is_sfp */ 1548 1549 static void 1550 ixgbe_schedule_admin_tasklet(struct adapter *adapter) 1551 { 1552 1553 KASSERT(mutex_owned(&adapter->admin_mtx)); 1554 1555 if (__predict_true(adapter->osdep.detaching == false)) { 1556 if (adapter->admin_pending == 0) 1557 workqueue_enqueue(adapter->admin_wq, 1558 &adapter->admin_wc, NULL); 1559 adapter->admin_pending = 1; 1560 } 1561 } 1562 1563 /************************************************************************ 1564 * ixgbe_config_link 1565 ************************************************************************/ 1566 static void 1567 ixgbe_config_link(struct adapter *adapter) 1568 { 1569 struct ixgbe_hw *hw = &adapter->hw; 1570 u32 autoneg, err = 0; 1571 u32 task_requests = 0; 1572 bool sfp, negotiate = false; 1573 1574 sfp = ixgbe_is_sfp(hw); 1575 1576 if (sfp) { 1577 if (hw->phy.multispeed_fiber) { 1578 ixgbe_enable_tx_laser(hw); 1579 task_requests |= IXGBE_REQUEST_TASK_MSF_WOI; 1580 } 1581 task_requests |= IXGBE_REQUEST_TASK_MOD_WOI; 1582 1583 mutex_enter(&adapter->admin_mtx); 1584 adapter->task_requests |= task_requests; 1585 ixgbe_schedule_admin_tasklet(adapter); 1586 mutex_exit(&adapter->admin_mtx); 1587 } else { 1588 struct ifmedia *ifm = &adapter->media; 1589 1590 if (hw->mac.ops.check_link) 1591 err = ixgbe_check_link(hw, &adapter->link_speed, 1592 &adapter->link_up, FALSE); 1593 if (err) 1594 return; 1595 1596 /* 1597 * Check if it's the first call. If it's the first call, 1598 * get value for auto negotiation. 1599 */ 1600 autoneg = hw->phy.autoneg_advertised; 1601 if ((IFM_SUBTYPE(ifm->ifm_cur->ifm_media) != IFM_NONE) 1602 && ((!autoneg) && (hw->mac.ops.get_link_capabilities))) 1603 err = hw->mac.ops.get_link_capabilities(hw, &autoneg, 1604 &negotiate); 1605 if (err) 1606 return; 1607 if (hw->mac.ops.setup_link) 1608 err = hw->mac.ops.setup_link(hw, autoneg, 1609 adapter->link_up); 1610 } 1611 } /* ixgbe_config_link */ 1612 1613 /************************************************************************ 1614 * ixgbe_update_stats_counters - Update board statistics counters. 1615 ************************************************************************/ 1616 static void 1617 ixgbe_update_stats_counters(struct adapter *adapter) 1618 { 1619 struct ifnet *ifp = adapter->ifp; 1620 struct ixgbe_hw *hw = &adapter->hw; 1621 struct ixgbe_hw_stats *stats = &adapter->stats.pf; 1622 u32 missed_rx = 0, bprc, lxontxc, lxofftxc; 1623 u64 total, total_missed_rx = 0; 1624 uint64_t crcerrs, illerrc, rlec, ruc, rfc, roc, rjc; 1625 unsigned int queue_counters; 1626 int i; 1627 1628 IXGBE_EVC_REGADD2(hw, stats, IXGBE_CRCERRS, crcerrs); 1629 IXGBE_EVC_REGADD2(hw, stats, IXGBE_ILLERRC, illerrc); 1630 1631 IXGBE_EVC_REGADD(hw, stats, IXGBE_ERRBC, errbc); 1632 IXGBE_EVC_REGADD(hw, stats, IXGBE_MSPDC, mspdc); 1633 if (hw->mac.type >= ixgbe_mac_X550) 1634 IXGBE_EVC_REGADD(hw, stats, IXGBE_MBSDC, mbsdc); 1635 1636 /* 16 registers exist */ 1637 queue_counters = uimin(__arraycount(stats->qprc), adapter->num_queues); 1638 for (i = 0; i < queue_counters; i++) { 1639 IXGBE_EVC_REGADD(hw, stats, IXGBE_QPRC(i), qprc[i]); 1640 IXGBE_EVC_REGADD(hw, stats, IXGBE_QPTC(i), qptc[i]); 1641 if (hw->mac.type >= ixgbe_mac_82599EB) { 1642 IXGBE_EVC_ADD(&stats->qbrc[i], 1643 IXGBE_READ_REG(hw, IXGBE_QBRC_L(i)) + 1644 ((u64)IXGBE_READ_REG(hw, IXGBE_QBRC_H(i)) << 32)); 1645 IXGBE_EVC_ADD(&stats->qbtc[i], 1646 IXGBE_READ_REG(hw, IXGBE_QBTC_L(i)) + 1647 ((u64)IXGBE_READ_REG(hw, IXGBE_QBTC_H(i)) << 32)); 1648 IXGBE_EVC_REGADD(hw, stats, IXGBE_QPRDC(i), qprdc[i]); 1649 } else { 1650 /* 82598 */ 1651 IXGBE_EVC_REGADD(hw, stats, IXGBE_QBRC(i), qbrc[i]); 1652 IXGBE_EVC_REGADD(hw, stats, IXGBE_QBTC(i), qbtc[i]); 1653 } 1654 } 1655 1656 /* 8 registers exist */ 1657 for (i = 0; i < IXGBE_TC_COUNTER_NUM; i++) { 1658 uint32_t mp; 1659 1660 /* MPC */ 1661 mp = IXGBE_READ_REG(hw, IXGBE_MPC(i)); 1662 /* global total per queue */ 1663 IXGBE_EVC_ADD(&stats->mpc[i], mp); 1664 /* running comprehensive total for stats display */ 1665 total_missed_rx += mp; 1666 1667 if (hw->mac.type == ixgbe_mac_82598EB) 1668 IXGBE_EVC_REGADD(hw, stats, IXGBE_RNBC(i), rnbc[i]); 1669 1670 IXGBE_EVC_REGADD(hw, stats, IXGBE_PXONTXC(i), pxontxc[i]); 1671 IXGBE_EVC_REGADD(hw, stats, IXGBE_PXOFFTXC(i), pxofftxc[i]); 1672 if (hw->mac.type >= ixgbe_mac_82599EB) { 1673 IXGBE_EVC_REGADD(hw, stats, 1674 IXGBE_PXONRXCNT(i), pxonrxc[i]); 1675 IXGBE_EVC_REGADD(hw, stats, 1676 IXGBE_PXOFFRXCNT(i), pxoffrxc[i]); 1677 IXGBE_EVC_REGADD(hw, stats, 1678 IXGBE_PXON2OFFCNT(i), pxon2offc[i]); 1679 } else { 1680 IXGBE_EVC_REGADD(hw, stats, 1681 IXGBE_PXONRXC(i), pxonrxc[i]); 1682 IXGBE_EVC_REGADD(hw, stats, 1683 IXGBE_PXOFFRXC(i), pxoffrxc[i]); 1684 } 1685 } 1686 IXGBE_EVC_ADD(&stats->mpctotal, total_missed_rx); 1687 1688 /* Document says M[LR]FC are valid when link is up and 10Gbps */ 1689 if ((adapter->link_active == LINK_STATE_UP) 1690 && (adapter->link_speed == IXGBE_LINK_SPEED_10GB_FULL)) { 1691 IXGBE_EVC_REGADD(hw, stats, IXGBE_MLFC, mlfc); 1692 IXGBE_EVC_REGADD(hw, stats, IXGBE_MRFC, mrfc); 1693 } 1694 if (hw->mac.type == ixgbe_mac_X550EM_a) 1695 IXGBE_EVC_REGADD(hw, stats, IXGBE_LINK_DN_CNT, link_dn_cnt); 1696 IXGBE_EVC_REGADD2(hw, stats, IXGBE_RLEC, rlec); 1697 1698 /* Hardware workaround, gprc counts missed packets */ 1699 IXGBE_EVC_ADD(&stats->gprc, 1700 IXGBE_READ_REG(hw, IXGBE_GPRC) - missed_rx); 1701 1702 IXGBE_EVC_REGADD2(hw, stats, IXGBE_LXONTXC, lxontxc); 1703 IXGBE_EVC_REGADD2(hw, stats, IXGBE_LXOFFTXC, lxofftxc); 1704 total = lxontxc + lxofftxc; 1705 1706 if (hw->mac.type != ixgbe_mac_82598EB) { 1707 IXGBE_EVC_ADD(&stats->gorc, IXGBE_READ_REG(hw, IXGBE_GORCL) + 1708 ((u64)IXGBE_READ_REG(hw, IXGBE_GORCH) << 32)); 1709 IXGBE_EVC_ADD(&stats->gotc, IXGBE_READ_REG(hw, IXGBE_GOTCL) + 1710 ((u64)IXGBE_READ_REG(hw, IXGBE_GOTCH) << 32) 1711 - total * ETHER_MIN_LEN); 1712 IXGBE_EVC_ADD(&stats->tor, IXGBE_READ_REG(hw, IXGBE_TORL) + 1713 ((u64)IXGBE_READ_REG(hw, IXGBE_TORH) << 32)); 1714 IXGBE_EVC_REGADD(hw, stats, IXGBE_LXONRXCNT, lxonrxc); 1715 IXGBE_EVC_REGADD(hw, stats, IXGBE_LXOFFRXCNT, lxoffrxc); 1716 } else { 1717 IXGBE_EVC_REGADD(hw, stats, IXGBE_LXONRXC, lxonrxc); 1718 IXGBE_EVC_REGADD(hw, stats, IXGBE_LXOFFRXC, lxoffrxc); 1719 /* 82598 only has a counter in the high register */ 1720 IXGBE_EVC_REGADD(hw, stats, IXGBE_GORCH, gorc); 1721 IXGBE_EVC_ADD(&stats->gotc, IXGBE_READ_REG(hw, IXGBE_GOTCH) 1722 - total * ETHER_MIN_LEN); 1723 IXGBE_EVC_REGADD(hw, stats, IXGBE_TORH, tor); 1724 } 1725 1726 /* 1727 * Workaround: mprc hardware is incorrectly counting 1728 * broadcasts, so for now we subtract those. 1729 */ 1730 IXGBE_EVC_REGADD2(hw, stats, IXGBE_BPRC, bprc); 1731 IXGBE_EVC_ADD(&stats->mprc, IXGBE_READ_REG(hw, IXGBE_MPRC) 1732 - ((hw->mac.type == ixgbe_mac_82598EB) ? bprc : 0)); 1733 1734 IXGBE_EVC_REGADD(hw, stats, IXGBE_PRC64, prc64); 1735 IXGBE_EVC_REGADD(hw, stats, IXGBE_PRC127, prc127); 1736 IXGBE_EVC_REGADD(hw, stats, IXGBE_PRC255, prc255); 1737 IXGBE_EVC_REGADD(hw, stats, IXGBE_PRC511, prc511); 1738 IXGBE_EVC_REGADD(hw, stats, IXGBE_PRC1023, prc1023); 1739 IXGBE_EVC_REGADD(hw, stats, IXGBE_PRC1522, prc1522); 1740 1741 IXGBE_EVC_ADD(&stats->gptc, IXGBE_READ_REG(hw, IXGBE_GPTC) - total); 1742 IXGBE_EVC_ADD(&stats->mptc, IXGBE_READ_REG(hw, IXGBE_MPTC) - total); 1743 IXGBE_EVC_ADD(&stats->ptc64, IXGBE_READ_REG(hw, IXGBE_PTC64) - total); 1744 1745 IXGBE_EVC_REGADD2(hw, stats, IXGBE_RUC, ruc); 1746 IXGBE_EVC_REGADD2(hw, stats, IXGBE_RFC, rfc); 1747 IXGBE_EVC_REGADD2(hw, stats, IXGBE_ROC, roc); 1748 IXGBE_EVC_REGADD2(hw, stats, IXGBE_RJC, rjc); 1749 1750 IXGBE_EVC_REGADD(hw, stats, IXGBE_MNGPRC, mngprc); 1751 IXGBE_EVC_REGADD(hw, stats, IXGBE_MNGPDC, mngpdc); 1752 IXGBE_EVC_REGADD(hw, stats, IXGBE_MNGPTC, mngptc); 1753 IXGBE_EVC_REGADD(hw, stats, IXGBE_TPR, tpr); 1754 IXGBE_EVC_REGADD(hw, stats, IXGBE_TPT, tpt); 1755 IXGBE_EVC_REGADD(hw, stats, IXGBE_PTC127, ptc127); 1756 IXGBE_EVC_REGADD(hw, stats, IXGBE_PTC255, ptc255); 1757 IXGBE_EVC_REGADD(hw, stats, IXGBE_PTC511, ptc511); 1758 IXGBE_EVC_REGADD(hw, stats, IXGBE_PTC1023, ptc1023); 1759 IXGBE_EVC_REGADD(hw, stats, IXGBE_PTC1522, ptc1522); 1760 IXGBE_EVC_REGADD(hw, stats, IXGBE_BPTC, bptc); 1761 IXGBE_EVC_REGADD(hw, stats, IXGBE_XEC, xec); 1762 IXGBE_EVC_REGADD(hw, stats, IXGBE_FCCRC, fccrc); 1763 IXGBE_EVC_REGADD(hw, stats, IXGBE_FCLAST, fclast); 1764 /* Only read FCOE on 82599 */ 1765 if (hw->mac.type != ixgbe_mac_82598EB) { 1766 IXGBE_EVC_REGADD(hw, stats, IXGBE_FCOERPDC, fcoerpdc); 1767 IXGBE_EVC_REGADD(hw, stats, IXGBE_FCOEPRC, fcoeprc); 1768 IXGBE_EVC_REGADD(hw, stats, IXGBE_FCOEPTC, fcoeptc); 1769 IXGBE_EVC_REGADD(hw, stats, IXGBE_FCOEDWRC, fcoedwrc); 1770 IXGBE_EVC_REGADD(hw, stats, IXGBE_FCOEDWTC, fcoedwtc); 1771 } 1772 1773 /* 1774 * Fill out the OS statistics structure. Only RX errors are required 1775 * here because all TX counters are incremented in the TX path and 1776 * normal RX counters are prepared in ether_input(). 1777 */ 1778 net_stat_ref_t nsr = IF_STAT_GETREF(ifp); 1779 if_statadd_ref(nsr, if_iqdrops, total_missed_rx); 1780 1781 /* 1782 * Aggregate following types of errors as RX errors: 1783 * - CRC error count, 1784 * - illegal byte error count, 1785 * - length error count, 1786 * - undersized packets count, 1787 * - fragmented packets count, 1788 * - oversized packets count, 1789 * - jabber count. 1790 */ 1791 if_statadd_ref(nsr, if_ierrors, 1792 crcerrs + illerrc + rlec + ruc + rfc + roc + rjc); 1793 1794 IF_STAT_PUTREF(ifp); 1795 } /* ixgbe_update_stats_counters */ 1796 1797 /************************************************************************ 1798 * ixgbe_add_hw_stats 1799 * 1800 * Add sysctl variables, one per statistic, to the system. 1801 ************************************************************************/ 1802 static void 1803 ixgbe_add_hw_stats(struct adapter *adapter) 1804 { 1805 device_t dev = adapter->dev; 1806 const struct sysctlnode *rnode, *cnode; 1807 struct sysctllog **log = &adapter->sysctllog; 1808 struct tx_ring *txr = adapter->tx_rings; 1809 struct rx_ring *rxr = adapter->rx_rings; 1810 struct ixgbe_hw *hw = &adapter->hw; 1811 struct ixgbe_hw_stats *stats = &adapter->stats.pf; 1812 const char *xname = device_xname(dev); 1813 int i; 1814 1815 /* Driver Statistics */ 1816 evcnt_attach_dynamic(&adapter->efbig_tx_dma_setup, EVCNT_TYPE_MISC, 1817 NULL, xname, "Driver tx dma soft fail EFBIG"); 1818 evcnt_attach_dynamic(&adapter->mbuf_defrag_failed, EVCNT_TYPE_MISC, 1819 NULL, xname, "m_defrag() failed"); 1820 evcnt_attach_dynamic(&adapter->efbig2_tx_dma_setup, EVCNT_TYPE_MISC, 1821 NULL, xname, "Driver tx dma hard fail EFBIG"); 1822 evcnt_attach_dynamic(&adapter->einval_tx_dma_setup, EVCNT_TYPE_MISC, 1823 NULL, xname, "Driver tx dma hard fail EINVAL"); 1824 evcnt_attach_dynamic(&adapter->other_tx_dma_setup, EVCNT_TYPE_MISC, 1825 NULL, xname, "Driver tx dma hard fail other"); 1826 evcnt_attach_dynamic(&adapter->eagain_tx_dma_setup, EVCNT_TYPE_MISC, 1827 NULL, xname, "Driver tx dma soft fail EAGAIN"); 1828 evcnt_attach_dynamic(&adapter->enomem_tx_dma_setup, EVCNT_TYPE_MISC, 1829 NULL, xname, "Driver tx dma soft fail ENOMEM"); 1830 evcnt_attach_dynamic(&adapter->watchdog_events, EVCNT_TYPE_MISC, 1831 NULL, xname, "Watchdog timeouts"); 1832 evcnt_attach_dynamic(&adapter->tso_err, EVCNT_TYPE_MISC, 1833 NULL, xname, "TSO errors"); 1834 evcnt_attach_dynamic(&adapter->admin_irqev, EVCNT_TYPE_INTR, 1835 NULL, xname, "Admin MSI-X IRQ Handled"); 1836 evcnt_attach_dynamic(&adapter->link_workev, EVCNT_TYPE_INTR, 1837 NULL, xname, "Link event"); 1838 evcnt_attach_dynamic(&adapter->mod_workev, EVCNT_TYPE_INTR, 1839 NULL, xname, "SFP+ module event"); 1840 evcnt_attach_dynamic(&adapter->msf_workev, EVCNT_TYPE_INTR, 1841 NULL, xname, "Multispeed event"); 1842 evcnt_attach_dynamic(&adapter->phy_workev, EVCNT_TYPE_INTR, 1843 NULL, xname, "External PHY event"); 1844 1845 /* Max number of traffic class is 8 */ 1846 KASSERT(IXGBE_DCB_MAX_TRAFFIC_CLASS == 8); 1847 for (i = 0; i < IXGBE_TC_COUNTER_NUM; i++) { 1848 snprintf(adapter->tcs[i].evnamebuf, 1849 sizeof(adapter->tcs[i].evnamebuf), "%s tc%d", xname, i); 1850 if (i < __arraycount(stats->mpc)) { 1851 evcnt_attach_dynamic(&stats->mpc[i], 1852 EVCNT_TYPE_MISC, NULL, adapter->tcs[i].evnamebuf, 1853 "RX Missed Packet Count"); 1854 if (hw->mac.type == ixgbe_mac_82598EB) 1855 evcnt_attach_dynamic(&stats->rnbc[i], 1856 EVCNT_TYPE_MISC, NULL, 1857 adapter->tcs[i].evnamebuf, 1858 "Receive No Buffers"); 1859 } 1860 if (i < __arraycount(stats->pxontxc)) { 1861 evcnt_attach_dynamic(&stats->pxontxc[i], 1862 EVCNT_TYPE_MISC, NULL, adapter->tcs[i].evnamebuf, 1863 "pxontxc"); 1864 evcnt_attach_dynamic(&stats->pxonrxc[i], 1865 EVCNT_TYPE_MISC, NULL, adapter->tcs[i].evnamebuf, 1866 "pxonrxc"); 1867 evcnt_attach_dynamic(&stats->pxofftxc[i], 1868 EVCNT_TYPE_MISC, NULL, adapter->tcs[i].evnamebuf, 1869 "pxofftxc"); 1870 evcnt_attach_dynamic(&stats->pxoffrxc[i], 1871 EVCNT_TYPE_MISC, NULL, adapter->tcs[i].evnamebuf, 1872 "pxoffrxc"); 1873 if (hw->mac.type >= ixgbe_mac_82599EB) 1874 evcnt_attach_dynamic(&stats->pxon2offc[i], 1875 EVCNT_TYPE_MISC, NULL, 1876 adapter->tcs[i].evnamebuf, 1877 "pxon2offc"); 1878 } 1879 } 1880 1881 for (i = 0; i < adapter->num_queues; i++, rxr++, txr++) { 1882 #ifdef LRO 1883 struct lro_ctrl *lro = &rxr->lro; 1884 #endif 1885 1886 snprintf(adapter->queues[i].evnamebuf, 1887 sizeof(adapter->queues[i].evnamebuf), "%s q%d", xname, i); 1888 snprintf(adapter->queues[i].namebuf, 1889 sizeof(adapter->queues[i].namebuf), "q%d", i); 1890 1891 if ((rnode = ixgbe_sysctl_instance(adapter)) == NULL) { 1892 aprint_error_dev(dev, 1893 "could not create sysctl root\n"); 1894 break; 1895 } 1896 1897 if (sysctl_createv(log, 0, &rnode, &rnode, 1898 0, CTLTYPE_NODE, 1899 adapter->queues[i].namebuf, SYSCTL_DESCR("Queue Name"), 1900 NULL, 0, NULL, 0, CTL_CREATE, CTL_EOL) != 0) 1901 break; 1902 1903 if (sysctl_createv(log, 0, &rnode, &cnode, 1904 CTLFLAG_READWRITE, CTLTYPE_INT, 1905 "interrupt_rate", SYSCTL_DESCR("Interrupt Rate"), 1906 ixgbe_sysctl_interrupt_rate_handler, 0, 1907 (void *)&adapter->queues[i], 0, CTL_CREATE, CTL_EOL) != 0) 1908 break; 1909 1910 if (sysctl_createv(log, 0, &rnode, &cnode, 1911 CTLFLAG_READONLY, CTLTYPE_INT, 1912 "txd_head", SYSCTL_DESCR("Transmit Descriptor Head"), 1913 ixgbe_sysctl_tdh_handler, 0, (void *)txr, 1914 0, CTL_CREATE, CTL_EOL) != 0) 1915 break; 1916 1917 if (sysctl_createv(log, 0, &rnode, &cnode, 1918 CTLFLAG_READONLY, CTLTYPE_INT, 1919 "txd_tail", SYSCTL_DESCR("Transmit Descriptor Tail"), 1920 ixgbe_sysctl_tdt_handler, 0, (void *)txr, 1921 0, CTL_CREATE, CTL_EOL) != 0) 1922 break; 1923 1924 if (sysctl_createv(log, 0, &rnode, &cnode, 1925 CTLFLAG_READONLY, CTLTYPE_INT, "rxd_nxck", 1926 SYSCTL_DESCR("Receive Descriptor next to check"), 1927 ixgbe_sysctl_next_to_check_handler, 0, (void *)rxr, 0, 1928 CTL_CREATE, CTL_EOL) != 0) 1929 break; 1930 1931 if (sysctl_createv(log, 0, &rnode, &cnode, 1932 CTLFLAG_READONLY, CTLTYPE_INT, "rxd_nxrf", 1933 SYSCTL_DESCR("Receive Descriptor next to refresh"), 1934 ixgbe_sysctl_next_to_refresh_handler, 0, (void *)rxr, 0, 1935 CTL_CREATE, CTL_EOL) != 0) 1936 break; 1937 1938 if (sysctl_createv(log, 0, &rnode, &cnode, 1939 CTLFLAG_READONLY, CTLTYPE_INT, "rxd_head", 1940 SYSCTL_DESCR("Receive Descriptor Head"), 1941 ixgbe_sysctl_rdh_handler, 0, (void *)rxr, 0, 1942 CTL_CREATE, CTL_EOL) != 0) 1943 break; 1944 1945 if (sysctl_createv(log, 0, &rnode, &cnode, 1946 CTLFLAG_READONLY, CTLTYPE_INT, "rxd_tail", 1947 SYSCTL_DESCR("Receive Descriptor Tail"), 1948 ixgbe_sysctl_rdt_handler, 0, (void *)rxr, 0, 1949 CTL_CREATE, CTL_EOL) != 0) 1950 break; 1951 1952 evcnt_attach_dynamic(&adapter->queues[i].irqs, EVCNT_TYPE_INTR, 1953 NULL, adapter->queues[i].evnamebuf, "IRQs on queue"); 1954 evcnt_attach_dynamic(&adapter->queues[i].handleq, 1955 EVCNT_TYPE_MISC, NULL, adapter->queues[i].evnamebuf, 1956 "Handled queue in softint"); 1957 evcnt_attach_dynamic(&adapter->queues[i].req, EVCNT_TYPE_MISC, 1958 NULL, adapter->queues[i].evnamebuf, "Requeued in softint"); 1959 if (i < __arraycount(stats->qbtc)) 1960 evcnt_attach_dynamic(&stats->qbtc[i], EVCNT_TYPE_MISC, 1961 NULL, adapter->queues[i].evnamebuf, 1962 "Queue Bytes Transmitted (reg)"); 1963 evcnt_attach_dynamic(&txr->total_packets, EVCNT_TYPE_MISC, 1964 NULL, adapter->queues[i].evnamebuf, 1965 "Queue Packets Transmitted (soft)"); 1966 if (i < __arraycount(stats->qptc)) 1967 evcnt_attach_dynamic(&stats->qptc[i], EVCNT_TYPE_MISC, 1968 NULL, adapter->queues[i].evnamebuf, 1969 "Queue Packets Transmitted (reg)"); 1970 #ifndef IXGBE_LEGACY_TX 1971 evcnt_attach_dynamic(&txr->pcq_drops, EVCNT_TYPE_MISC, 1972 NULL, adapter->queues[i].evnamebuf, 1973 "Packets dropped in pcq"); 1974 #endif 1975 evcnt_attach_dynamic(&txr->no_desc_avail, EVCNT_TYPE_MISC, 1976 NULL, adapter->queues[i].evnamebuf, 1977 "TX Queue No Descriptor Available"); 1978 evcnt_attach_dynamic(&txr->tso_tx, EVCNT_TYPE_MISC, 1979 NULL, adapter->queues[i].evnamebuf, "TSO"); 1980 1981 evcnt_attach_dynamic(&rxr->rx_bytes, EVCNT_TYPE_MISC, 1982 NULL, adapter->queues[i].evnamebuf, 1983 "Queue Bytes Received (soft)"); 1984 if (i < __arraycount(stats->qbrc)) 1985 evcnt_attach_dynamic(&stats->qbrc[i], EVCNT_TYPE_MISC, 1986 NULL, adapter->queues[i].evnamebuf, 1987 "Queue Bytes Received (reg)"); 1988 evcnt_attach_dynamic(&rxr->rx_packets, EVCNT_TYPE_MISC, 1989 NULL, adapter->queues[i].evnamebuf, 1990 "Queue Packets Received (soft)"); 1991 if (i < __arraycount(stats->qprc)) 1992 evcnt_attach_dynamic(&stats->qprc[i], EVCNT_TYPE_MISC, 1993 NULL, adapter->queues[i].evnamebuf, 1994 "Queue Packets Received (reg)"); 1995 if ((i < __arraycount(stats->qprdc)) && 1996 (hw->mac.type >= ixgbe_mac_82599EB)) 1997 evcnt_attach_dynamic(&stats->qprdc[i], 1998 EVCNT_TYPE_MISC, NULL, 1999 adapter->queues[i].evnamebuf, 2000 "Queue Packets Received Drop"); 2001 2002 evcnt_attach_dynamic(&rxr->no_mbuf, EVCNT_TYPE_MISC, 2003 NULL, adapter->queues[i].evnamebuf, "Rx no mbuf"); 2004 evcnt_attach_dynamic(&rxr->rx_discarded, EVCNT_TYPE_MISC, 2005 NULL, adapter->queues[i].evnamebuf, "Rx discarded"); 2006 evcnt_attach_dynamic(&rxr->rx_copies, EVCNT_TYPE_MISC, 2007 NULL, adapter->queues[i].evnamebuf, "Copied RX Frames"); 2008 #ifdef LRO 2009 SYSCTL_ADD_INT(ctx, queue_list, OID_AUTO, "lro_queued", 2010 CTLFLAG_RD, &lro->lro_queued, 0, 2011 "LRO Queued"); 2012 SYSCTL_ADD_INT(ctx, queue_list, OID_AUTO, "lro_flushed", 2013 CTLFLAG_RD, &lro->lro_flushed, 0, 2014 "LRO Flushed"); 2015 #endif /* LRO */ 2016 } 2017 2018 /* MAC stats get their own sub node */ 2019 2020 snprintf(stats->namebuf, 2021 sizeof(stats->namebuf), "%s MAC Statistics", xname); 2022 2023 evcnt_attach_dynamic(&stats->ipcs, EVCNT_TYPE_MISC, NULL, 2024 stats->namebuf, "rx csum offload - IP"); 2025 evcnt_attach_dynamic(&stats->l4cs, EVCNT_TYPE_MISC, NULL, 2026 stats->namebuf, "rx csum offload - L4"); 2027 evcnt_attach_dynamic(&stats->ipcs_bad, EVCNT_TYPE_MISC, NULL, 2028 stats->namebuf, "rx csum offload - IP bad"); 2029 evcnt_attach_dynamic(&stats->l4cs_bad, EVCNT_TYPE_MISC, NULL, 2030 stats->namebuf, "rx csum offload - L4 bad"); 2031 evcnt_attach_dynamic(&stats->intzero, EVCNT_TYPE_MISC, NULL, 2032 stats->namebuf, "Interrupt conditions zero"); 2033 evcnt_attach_dynamic(&stats->legint, EVCNT_TYPE_MISC, NULL, 2034 stats->namebuf, "Legacy interrupts"); 2035 2036 evcnt_attach_dynamic(&stats->crcerrs, EVCNT_TYPE_MISC, NULL, 2037 stats->namebuf, "CRC Errors"); 2038 evcnt_attach_dynamic(&stats->illerrc, EVCNT_TYPE_MISC, NULL, 2039 stats->namebuf, "Illegal Byte Errors"); 2040 evcnt_attach_dynamic(&stats->errbc, EVCNT_TYPE_MISC, NULL, 2041 stats->namebuf, "Byte Errors"); 2042 evcnt_attach_dynamic(&stats->mspdc, EVCNT_TYPE_MISC, NULL, 2043 stats->namebuf, "MAC Short Packets Discarded"); 2044 if (hw->mac.type >= ixgbe_mac_X550) 2045 evcnt_attach_dynamic(&stats->mbsdc, EVCNT_TYPE_MISC, NULL, 2046 stats->namebuf, "Bad SFD"); 2047 evcnt_attach_dynamic(&stats->mpctotal, EVCNT_TYPE_MISC, NULL, 2048 stats->namebuf, "Total Packets Missed"); 2049 evcnt_attach_dynamic(&stats->mlfc, EVCNT_TYPE_MISC, NULL, 2050 stats->namebuf, "MAC Local Faults"); 2051 evcnt_attach_dynamic(&stats->mrfc, EVCNT_TYPE_MISC, NULL, 2052 stats->namebuf, "MAC Remote Faults"); 2053 if (hw->mac.type == ixgbe_mac_X550EM_a) 2054 evcnt_attach_dynamic(&stats->link_dn_cnt, EVCNT_TYPE_MISC, 2055 NULL, stats->namebuf, "Link down event in the MAC"); 2056 evcnt_attach_dynamic(&stats->rlec, EVCNT_TYPE_MISC, NULL, 2057 stats->namebuf, "Receive Length Errors"); 2058 evcnt_attach_dynamic(&stats->lxontxc, EVCNT_TYPE_MISC, NULL, 2059 stats->namebuf, "Link XON Transmitted"); 2060 evcnt_attach_dynamic(&stats->lxonrxc, EVCNT_TYPE_MISC, NULL, 2061 stats->namebuf, "Link XON Received"); 2062 evcnt_attach_dynamic(&stats->lxofftxc, EVCNT_TYPE_MISC, NULL, 2063 stats->namebuf, "Link XOFF Transmitted"); 2064 evcnt_attach_dynamic(&stats->lxoffrxc, EVCNT_TYPE_MISC, NULL, 2065 stats->namebuf, "Link XOFF Received"); 2066 2067 /* Packet Reception Stats */ 2068 evcnt_attach_dynamic(&stats->tor, EVCNT_TYPE_MISC, NULL, 2069 stats->namebuf, "Total Octets Received"); 2070 evcnt_attach_dynamic(&stats->gorc, EVCNT_TYPE_MISC, NULL, 2071 stats->namebuf, "Good Octets Received"); 2072 evcnt_attach_dynamic(&stats->tpr, EVCNT_TYPE_MISC, NULL, 2073 stats->namebuf, "Total Packets Received"); 2074 evcnt_attach_dynamic(&stats->gprc, EVCNT_TYPE_MISC, NULL, 2075 stats->namebuf, "Good Packets Received"); 2076 evcnt_attach_dynamic(&stats->mprc, EVCNT_TYPE_MISC, NULL, 2077 stats->namebuf, "Multicast Packets Received"); 2078 evcnt_attach_dynamic(&stats->bprc, EVCNT_TYPE_MISC, NULL, 2079 stats->namebuf, "Broadcast Packets Received"); 2080 evcnt_attach_dynamic(&stats->prc64, EVCNT_TYPE_MISC, NULL, 2081 stats->namebuf, "64 byte frames received "); 2082 evcnt_attach_dynamic(&stats->prc127, EVCNT_TYPE_MISC, NULL, 2083 stats->namebuf, "65-127 byte frames received"); 2084 evcnt_attach_dynamic(&stats->prc255, EVCNT_TYPE_MISC, NULL, 2085 stats->namebuf, "128-255 byte frames received"); 2086 evcnt_attach_dynamic(&stats->prc511, EVCNT_TYPE_MISC, NULL, 2087 stats->namebuf, "256-511 byte frames received"); 2088 evcnt_attach_dynamic(&stats->prc1023, EVCNT_TYPE_MISC, NULL, 2089 stats->namebuf, "512-1023 byte frames received"); 2090 evcnt_attach_dynamic(&stats->prc1522, EVCNT_TYPE_MISC, NULL, 2091 stats->namebuf, "1023-1522 byte frames received"); 2092 evcnt_attach_dynamic(&stats->ruc, EVCNT_TYPE_MISC, NULL, 2093 stats->namebuf, "Receive Undersized"); 2094 evcnt_attach_dynamic(&stats->rfc, EVCNT_TYPE_MISC, NULL, 2095 stats->namebuf, "Fragmented Packets Received "); 2096 evcnt_attach_dynamic(&stats->roc, EVCNT_TYPE_MISC, NULL, 2097 stats->namebuf, "Oversized Packets Received"); 2098 evcnt_attach_dynamic(&stats->rjc, EVCNT_TYPE_MISC, NULL, 2099 stats->namebuf, "Received Jabber"); 2100 evcnt_attach_dynamic(&stats->mngprc, EVCNT_TYPE_MISC, NULL, 2101 stats->namebuf, "Management Packets Received"); 2102 evcnt_attach_dynamic(&stats->mngpdc, EVCNT_TYPE_MISC, NULL, 2103 stats->namebuf, "Management Packets Dropped"); 2104 evcnt_attach_dynamic(&stats->xec, EVCNT_TYPE_MISC, NULL, 2105 stats->namebuf, "Checksum Errors"); 2106 2107 /* Packet Transmission Stats */ 2108 evcnt_attach_dynamic(&stats->gotc, EVCNT_TYPE_MISC, NULL, 2109 stats->namebuf, "Good Octets Transmitted"); 2110 evcnt_attach_dynamic(&stats->tpt, EVCNT_TYPE_MISC, NULL, 2111 stats->namebuf, "Total Packets Transmitted"); 2112 evcnt_attach_dynamic(&stats->gptc, EVCNT_TYPE_MISC, NULL, 2113 stats->namebuf, "Good Packets Transmitted"); 2114 evcnt_attach_dynamic(&stats->bptc, EVCNT_TYPE_MISC, NULL, 2115 stats->namebuf, "Broadcast Packets Transmitted"); 2116 evcnt_attach_dynamic(&stats->mptc, EVCNT_TYPE_MISC, NULL, 2117 stats->namebuf, "Multicast Packets Transmitted"); 2118 evcnt_attach_dynamic(&stats->mngptc, EVCNT_TYPE_MISC, NULL, 2119 stats->namebuf, "Management Packets Transmitted"); 2120 evcnt_attach_dynamic(&stats->ptc64, EVCNT_TYPE_MISC, NULL, 2121 stats->namebuf, "64 byte frames transmitted "); 2122 evcnt_attach_dynamic(&stats->ptc127, EVCNT_TYPE_MISC, NULL, 2123 stats->namebuf, "65-127 byte frames transmitted"); 2124 evcnt_attach_dynamic(&stats->ptc255, EVCNT_TYPE_MISC, NULL, 2125 stats->namebuf, "128-255 byte frames transmitted"); 2126 evcnt_attach_dynamic(&stats->ptc511, EVCNT_TYPE_MISC, NULL, 2127 stats->namebuf, "256-511 byte frames transmitted"); 2128 evcnt_attach_dynamic(&stats->ptc1023, EVCNT_TYPE_MISC, NULL, 2129 stats->namebuf, "512-1023 byte frames transmitted"); 2130 evcnt_attach_dynamic(&stats->ptc1522, EVCNT_TYPE_MISC, NULL, 2131 stats->namebuf, "1024-1522 byte frames transmitted"); 2132 } /* ixgbe_add_hw_stats */ 2133 2134 static void 2135 ixgbe_clear_evcnt(struct adapter *adapter) 2136 { 2137 struct tx_ring *txr = adapter->tx_rings; 2138 struct rx_ring *rxr = adapter->rx_rings; 2139 struct ixgbe_hw *hw = &adapter->hw; 2140 struct ixgbe_hw_stats *stats = &adapter->stats.pf; 2141 int i; 2142 2143 IXGBE_EVC_STORE(&adapter->efbig_tx_dma_setup, 0); 2144 IXGBE_EVC_STORE(&adapter->mbuf_defrag_failed, 0); 2145 IXGBE_EVC_STORE(&adapter->efbig2_tx_dma_setup, 0); 2146 IXGBE_EVC_STORE(&adapter->einval_tx_dma_setup, 0); 2147 IXGBE_EVC_STORE(&adapter->other_tx_dma_setup, 0); 2148 IXGBE_EVC_STORE(&adapter->eagain_tx_dma_setup, 0); 2149 IXGBE_EVC_STORE(&adapter->enomem_tx_dma_setup, 0); 2150 IXGBE_EVC_STORE(&adapter->tso_err, 0); 2151 IXGBE_EVC_STORE(&adapter->watchdog_events, 0); 2152 IXGBE_EVC_STORE(&adapter->admin_irqev, 0); 2153 IXGBE_EVC_STORE(&adapter->link_workev, 0); 2154 IXGBE_EVC_STORE(&adapter->mod_workev, 0); 2155 IXGBE_EVC_STORE(&adapter->msf_workev, 0); 2156 IXGBE_EVC_STORE(&adapter->phy_workev, 0); 2157 2158 for (i = 0; i < IXGBE_TC_COUNTER_NUM; i++) { 2159 if (i < __arraycount(stats->mpc)) { 2160 IXGBE_EVC_STORE(&stats->mpc[i], 0); 2161 if (hw->mac.type == ixgbe_mac_82598EB) 2162 IXGBE_EVC_STORE(&stats->rnbc[i], 0); 2163 } 2164 if (i < __arraycount(stats->pxontxc)) { 2165 IXGBE_EVC_STORE(&stats->pxontxc[i], 0); 2166 IXGBE_EVC_STORE(&stats->pxonrxc[i], 0); 2167 IXGBE_EVC_STORE(&stats->pxofftxc[i], 0); 2168 IXGBE_EVC_STORE(&stats->pxoffrxc[i], 0); 2169 if (hw->mac.type >= ixgbe_mac_82599EB) 2170 IXGBE_EVC_STORE(&stats->pxon2offc[i], 0); 2171 } 2172 } 2173 2174 txr = adapter->tx_rings; 2175 for (i = 0; i < adapter->num_queues; i++, rxr++, txr++) { 2176 IXGBE_EVC_STORE(&adapter->queues[i].irqs, 0); 2177 IXGBE_EVC_STORE(&adapter->queues[i].handleq, 0); 2178 IXGBE_EVC_STORE(&adapter->queues[i].req, 0); 2179 IXGBE_EVC_STORE(&txr->total_packets, 0); 2180 #ifndef IXGBE_LEGACY_TX 2181 IXGBE_EVC_STORE(&txr->pcq_drops, 0); 2182 #endif 2183 IXGBE_EVC_STORE(&txr->no_desc_avail, 0); 2184 IXGBE_EVC_STORE(&txr->tso_tx, 0); 2185 txr->q_efbig_tx_dma_setup = 0; 2186 txr->q_mbuf_defrag_failed = 0; 2187 txr->q_efbig2_tx_dma_setup = 0; 2188 txr->q_einval_tx_dma_setup = 0; 2189 txr->q_other_tx_dma_setup = 0; 2190 txr->q_eagain_tx_dma_setup = 0; 2191 txr->q_enomem_tx_dma_setup = 0; 2192 txr->q_tso_err = 0; 2193 2194 if (i < __arraycount(stats->qprc)) { 2195 IXGBE_EVC_STORE(&stats->qprc[i], 0); 2196 IXGBE_EVC_STORE(&stats->qptc[i], 0); 2197 IXGBE_EVC_STORE(&stats->qbrc[i], 0); 2198 IXGBE_EVC_STORE(&stats->qbtc[i], 0); 2199 if (hw->mac.type >= ixgbe_mac_82599EB) 2200 IXGBE_EVC_STORE(&stats->qprdc[i], 0); 2201 } 2202 2203 IXGBE_EVC_STORE(&rxr->rx_packets, 0); 2204 IXGBE_EVC_STORE(&rxr->rx_bytes, 0); 2205 IXGBE_EVC_STORE(&rxr->rx_copies, 0); 2206 IXGBE_EVC_STORE(&rxr->no_mbuf, 0); 2207 IXGBE_EVC_STORE(&rxr->rx_discarded, 0); 2208 } 2209 IXGBE_EVC_STORE(&stats->ipcs, 0); 2210 IXGBE_EVC_STORE(&stats->l4cs, 0); 2211 IXGBE_EVC_STORE(&stats->ipcs_bad, 0); 2212 IXGBE_EVC_STORE(&stats->l4cs_bad, 0); 2213 IXGBE_EVC_STORE(&stats->intzero, 0); 2214 IXGBE_EVC_STORE(&stats->legint, 0); 2215 IXGBE_EVC_STORE(&stats->crcerrs, 0); 2216 IXGBE_EVC_STORE(&stats->illerrc, 0); 2217 IXGBE_EVC_STORE(&stats->errbc, 0); 2218 IXGBE_EVC_STORE(&stats->mspdc, 0); 2219 if (hw->mac.type >= ixgbe_mac_X550) 2220 IXGBE_EVC_STORE(&stats->mbsdc, 0); 2221 IXGBE_EVC_STORE(&stats->mpctotal, 0); 2222 IXGBE_EVC_STORE(&stats->mlfc, 0); 2223 IXGBE_EVC_STORE(&stats->mrfc, 0); 2224 if (hw->mac.type == ixgbe_mac_X550EM_a) 2225 IXGBE_EVC_STORE(&stats->link_dn_cnt, 0); 2226 IXGBE_EVC_STORE(&stats->rlec, 0); 2227 IXGBE_EVC_STORE(&stats->lxontxc, 0); 2228 IXGBE_EVC_STORE(&stats->lxonrxc, 0); 2229 IXGBE_EVC_STORE(&stats->lxofftxc, 0); 2230 IXGBE_EVC_STORE(&stats->lxoffrxc, 0); 2231 2232 /* Packet Reception Stats */ 2233 IXGBE_EVC_STORE(&stats->tor, 0); 2234 IXGBE_EVC_STORE(&stats->gorc, 0); 2235 IXGBE_EVC_STORE(&stats->tpr, 0); 2236 IXGBE_EVC_STORE(&stats->gprc, 0); 2237 IXGBE_EVC_STORE(&stats->mprc, 0); 2238 IXGBE_EVC_STORE(&stats->bprc, 0); 2239 IXGBE_EVC_STORE(&stats->prc64, 0); 2240 IXGBE_EVC_STORE(&stats->prc127, 0); 2241 IXGBE_EVC_STORE(&stats->prc255, 0); 2242 IXGBE_EVC_STORE(&stats->prc511, 0); 2243 IXGBE_EVC_STORE(&stats->prc1023, 0); 2244 IXGBE_EVC_STORE(&stats->prc1522, 0); 2245 IXGBE_EVC_STORE(&stats->ruc, 0); 2246 IXGBE_EVC_STORE(&stats->rfc, 0); 2247 IXGBE_EVC_STORE(&stats->roc, 0); 2248 IXGBE_EVC_STORE(&stats->rjc, 0); 2249 IXGBE_EVC_STORE(&stats->mngprc, 0); 2250 IXGBE_EVC_STORE(&stats->mngpdc, 0); 2251 IXGBE_EVC_STORE(&stats->xec, 0); 2252 2253 /* Packet Transmission Stats */ 2254 IXGBE_EVC_STORE(&stats->gotc, 0); 2255 IXGBE_EVC_STORE(&stats->tpt, 0); 2256 IXGBE_EVC_STORE(&stats->gptc, 0); 2257 IXGBE_EVC_STORE(&stats->bptc, 0); 2258 IXGBE_EVC_STORE(&stats->mptc, 0); 2259 IXGBE_EVC_STORE(&stats->mngptc, 0); 2260 IXGBE_EVC_STORE(&stats->ptc64, 0); 2261 IXGBE_EVC_STORE(&stats->ptc127, 0); 2262 IXGBE_EVC_STORE(&stats->ptc255, 0); 2263 IXGBE_EVC_STORE(&stats->ptc511, 0); 2264 IXGBE_EVC_STORE(&stats->ptc1023, 0); 2265 IXGBE_EVC_STORE(&stats->ptc1522, 0); 2266 } 2267 2268 /************************************************************************ 2269 * ixgbe_sysctl_tdh_handler - Transmit Descriptor Head handler function 2270 * 2271 * Retrieves the TDH value from the hardware 2272 ************************************************************************/ 2273 static int 2274 ixgbe_sysctl_tdh_handler(SYSCTLFN_ARGS) 2275 { 2276 struct sysctlnode node = *rnode; 2277 struct tx_ring *txr = (struct tx_ring *)node.sysctl_data; 2278 struct adapter *adapter; 2279 uint32_t val; 2280 2281 if (!txr) 2282 return (0); 2283 2284 adapter = txr->adapter; 2285 if (ixgbe_fw_recovery_mode_swflag(adapter)) 2286 return (EPERM); 2287 2288 val = IXGBE_READ_REG(&adapter->hw, IXGBE_TDH(txr->me)); 2289 node.sysctl_data = &val; 2290 return sysctl_lookup(SYSCTLFN_CALL(&node)); 2291 } /* ixgbe_sysctl_tdh_handler */ 2292 2293 /************************************************************************ 2294 * ixgbe_sysctl_tdt_handler - Transmit Descriptor Tail handler function 2295 * 2296 * Retrieves the TDT value from the hardware 2297 ************************************************************************/ 2298 static int 2299 ixgbe_sysctl_tdt_handler(SYSCTLFN_ARGS) 2300 { 2301 struct sysctlnode node = *rnode; 2302 struct tx_ring *txr = (struct tx_ring *)node.sysctl_data; 2303 struct adapter *adapter; 2304 uint32_t val; 2305 2306 if (!txr) 2307 return (0); 2308 2309 adapter = txr->adapter; 2310 if (ixgbe_fw_recovery_mode_swflag(adapter)) 2311 return (EPERM); 2312 2313 val = IXGBE_READ_REG(&adapter->hw, IXGBE_TDT(txr->me)); 2314 node.sysctl_data = &val; 2315 return sysctl_lookup(SYSCTLFN_CALL(&node)); 2316 } /* ixgbe_sysctl_tdt_handler */ 2317 2318 /************************************************************************ 2319 * ixgbe_sysctl_next_to_check_handler - Receive Descriptor next to check 2320 * handler function 2321 * 2322 * Retrieves the next_to_check value 2323 ************************************************************************/ 2324 static int 2325 ixgbe_sysctl_next_to_check_handler(SYSCTLFN_ARGS) 2326 { 2327 struct sysctlnode node = *rnode; 2328 struct rx_ring *rxr = (struct rx_ring *)node.sysctl_data; 2329 struct adapter *adapter; 2330 uint32_t val; 2331 2332 if (!rxr) 2333 return (0); 2334 2335 adapter = rxr->adapter; 2336 if (ixgbe_fw_recovery_mode_swflag(adapter)) 2337 return (EPERM); 2338 2339 val = rxr->next_to_check; 2340 node.sysctl_data = &val; 2341 return sysctl_lookup(SYSCTLFN_CALL(&node)); 2342 } /* ixgbe_sysctl_next_to_check_handler */ 2343 2344 /************************************************************************ 2345 * ixgbe_sysctl_next_to_refresh_handler - Receive Descriptor next to check 2346 * handler function 2347 * 2348 * Retrieves the next_to_refresh value 2349 ************************************************************************/ 2350 static int 2351 ixgbe_sysctl_next_to_refresh_handler(SYSCTLFN_ARGS) 2352 { 2353 struct sysctlnode node = *rnode; 2354 struct rx_ring *rxr = (struct rx_ring *)node.sysctl_data; 2355 struct adapter *adapter; 2356 uint32_t val; 2357 2358 if (!rxr) 2359 return (0); 2360 2361 adapter = rxr->adapter; 2362 if (ixgbe_fw_recovery_mode_swflag(adapter)) 2363 return (EPERM); 2364 2365 val = rxr->next_to_refresh; 2366 node.sysctl_data = &val; 2367 return sysctl_lookup(SYSCTLFN_CALL(&node)); 2368 } /* ixgbe_sysctl_next_to_refresh_handler */ 2369 2370 /************************************************************************ 2371 * ixgbe_sysctl_rdh_handler - Receive Descriptor Head handler function 2372 * 2373 * Retrieves the RDH value from the hardware 2374 ************************************************************************/ 2375 static int 2376 ixgbe_sysctl_rdh_handler(SYSCTLFN_ARGS) 2377 { 2378 struct sysctlnode node = *rnode; 2379 struct rx_ring *rxr = (struct rx_ring *)node.sysctl_data; 2380 struct adapter *adapter; 2381 uint32_t val; 2382 2383 if (!rxr) 2384 return (0); 2385 2386 adapter = rxr->adapter; 2387 if (ixgbe_fw_recovery_mode_swflag(adapter)) 2388 return (EPERM); 2389 2390 val = IXGBE_READ_REG(&adapter->hw, IXGBE_RDH(rxr->me)); 2391 node.sysctl_data = &val; 2392 return sysctl_lookup(SYSCTLFN_CALL(&node)); 2393 } /* ixgbe_sysctl_rdh_handler */ 2394 2395 /************************************************************************ 2396 * ixgbe_sysctl_rdt_handler - Receive Descriptor Tail handler function 2397 * 2398 * Retrieves the RDT value from the hardware 2399 ************************************************************************/ 2400 static int 2401 ixgbe_sysctl_rdt_handler(SYSCTLFN_ARGS) 2402 { 2403 struct sysctlnode node = *rnode; 2404 struct rx_ring *rxr = (struct rx_ring *)node.sysctl_data; 2405 struct adapter *adapter; 2406 uint32_t val; 2407 2408 if (!rxr) 2409 return (0); 2410 2411 adapter = rxr->adapter; 2412 if (ixgbe_fw_recovery_mode_swflag(adapter)) 2413 return (EPERM); 2414 2415 val = IXGBE_READ_REG(&adapter->hw, IXGBE_RDT(rxr->me)); 2416 node.sysctl_data = &val; 2417 return sysctl_lookup(SYSCTLFN_CALL(&node)); 2418 } /* ixgbe_sysctl_rdt_handler */ 2419 2420 static int 2421 ixgbe_vlan_cb(struct ethercom *ec, uint16_t vid, bool set) 2422 { 2423 struct ifnet *ifp = &ec->ec_if; 2424 struct adapter *adapter = ifp->if_softc; 2425 int rv; 2426 2427 if (set) 2428 rv = ixgbe_register_vlan(adapter, vid); 2429 else 2430 rv = ixgbe_unregister_vlan(adapter, vid); 2431 2432 if (rv != 0) 2433 return rv; 2434 2435 /* 2436 * Control VLAN HW tagging when ec_nvlan is changed from 1 to 0 2437 * or 0 to 1. 2438 */ 2439 if ((set && (ec->ec_nvlans == 1)) || (!set && (ec->ec_nvlans == 0))) 2440 ixgbe_setup_vlan_hw_tagging(adapter); 2441 2442 return rv; 2443 } 2444 2445 /************************************************************************ 2446 * ixgbe_register_vlan 2447 * 2448 * Run via vlan config EVENT, it enables us to use the 2449 * HW Filter table since we can get the vlan id. This 2450 * just creates the entry in the soft version of the 2451 * VFTA, init will repopulate the real table. 2452 ************************************************************************/ 2453 static int 2454 ixgbe_register_vlan(struct adapter *adapter, u16 vtag) 2455 { 2456 u16 index, bit; 2457 int error; 2458 2459 if ((vtag == 0) || (vtag > 4095)) /* Invalid */ 2460 return EINVAL; 2461 2462 IXGBE_CORE_LOCK(adapter); 2463 index = (vtag >> 5) & 0x7F; 2464 bit = vtag & 0x1F; 2465 adapter->shadow_vfta[index] |= ((u32)1 << bit); 2466 error = adapter->hw.mac.ops.set_vfta(&adapter->hw, vtag, 0, true, 2467 true); 2468 IXGBE_CORE_UNLOCK(adapter); 2469 if (error != 0) 2470 error = EACCES; 2471 2472 return error; 2473 } /* ixgbe_register_vlan */ 2474 2475 /************************************************************************ 2476 * ixgbe_unregister_vlan 2477 * 2478 * Run via vlan unconfig EVENT, remove our entry in the soft vfta. 2479 ************************************************************************/ 2480 static int 2481 ixgbe_unregister_vlan(struct adapter *adapter, u16 vtag) 2482 { 2483 u16 index, bit; 2484 int error; 2485 2486 if ((vtag == 0) || (vtag > 4095)) /* Invalid */ 2487 return EINVAL; 2488 2489 IXGBE_CORE_LOCK(adapter); 2490 index = (vtag >> 5) & 0x7F; 2491 bit = vtag & 0x1F; 2492 adapter->shadow_vfta[index] &= ~((u32)1 << bit); 2493 error = adapter->hw.mac.ops.set_vfta(&adapter->hw, vtag, 0, false, 2494 true); 2495 IXGBE_CORE_UNLOCK(adapter); 2496 if (error != 0) 2497 error = EACCES; 2498 2499 return error; 2500 } /* ixgbe_unregister_vlan */ 2501 2502 static void 2503 ixgbe_setup_vlan_hw_tagging(struct adapter *adapter) 2504 { 2505 struct ethercom *ec = &adapter->osdep.ec; 2506 struct ixgbe_hw *hw = &adapter->hw; 2507 struct rx_ring *rxr; 2508 u32 ctrl; 2509 int i; 2510 bool hwtagging; 2511 2512 /* Enable HW tagging only if any vlan is attached */ 2513 hwtagging = (ec->ec_capenable & ETHERCAP_VLAN_HWTAGGING) 2514 && VLAN_ATTACHED(ec); 2515 2516 /* Setup the queues for vlans */ 2517 for (i = 0; i < adapter->num_queues; i++) { 2518 rxr = &adapter->rx_rings[i]; 2519 /* 2520 * On 82599 and later, the VLAN enable is per/queue in RXDCTL. 2521 */ 2522 if (hw->mac.type != ixgbe_mac_82598EB) { 2523 ctrl = IXGBE_READ_REG(hw, IXGBE_RXDCTL(rxr->me)); 2524 if (hwtagging) 2525 ctrl |= IXGBE_RXDCTL_VME; 2526 else 2527 ctrl &= ~IXGBE_RXDCTL_VME; 2528 IXGBE_WRITE_REG(hw, IXGBE_RXDCTL(rxr->me), ctrl); 2529 } 2530 rxr->vtag_strip = hwtagging ? TRUE : FALSE; 2531 } 2532 2533 /* VLAN hw tagging for 82598 */ 2534 if (hw->mac.type == ixgbe_mac_82598EB) { 2535 ctrl = IXGBE_READ_REG(hw, IXGBE_VLNCTRL); 2536 if (hwtagging) 2537 ctrl |= IXGBE_VLNCTRL_VME; 2538 else 2539 ctrl &= ~IXGBE_VLNCTRL_VME; 2540 IXGBE_WRITE_REG(hw, IXGBE_VLNCTRL, ctrl); 2541 } 2542 } /* ixgbe_setup_vlan_hw_tagging */ 2543 2544 static void 2545 ixgbe_setup_vlan_hw_support(struct adapter *adapter) 2546 { 2547 struct ethercom *ec = &adapter->osdep.ec; 2548 struct ixgbe_hw *hw = &adapter->hw; 2549 int i; 2550 u32 ctrl; 2551 struct vlanid_list *vlanidp; 2552 2553 /* 2554 * This function is called from both if_init and ifflags_cb() 2555 * on NetBSD. 2556 */ 2557 2558 /* 2559 * Part 1: 2560 * Setup VLAN HW tagging 2561 */ 2562 ixgbe_setup_vlan_hw_tagging(adapter); 2563 2564 /* 2565 * Part 2: 2566 * Setup VLAN HW filter 2567 */ 2568 /* Cleanup shadow_vfta */ 2569 for (i = 0; i < IXGBE_VFTA_SIZE; i++) 2570 adapter->shadow_vfta[i] = 0; 2571 /* Generate shadow_vfta from ec_vids */ 2572 ETHER_LOCK(ec); 2573 SIMPLEQ_FOREACH(vlanidp, &ec->ec_vids, vid_list) { 2574 uint32_t idx; 2575 2576 idx = vlanidp->vid / 32; 2577 KASSERT(idx < IXGBE_VFTA_SIZE); 2578 adapter->shadow_vfta[idx] |= (u32)1 << (vlanidp->vid % 32); 2579 } 2580 ETHER_UNLOCK(ec); 2581 for (i = 0; i < IXGBE_VFTA_SIZE; i++) 2582 IXGBE_WRITE_REG(hw, IXGBE_VFTA(i), adapter->shadow_vfta[i]); 2583 2584 ctrl = IXGBE_READ_REG(hw, IXGBE_VLNCTRL); 2585 /* Enable the Filter Table if enabled */ 2586 if (ec->ec_capenable & ETHERCAP_VLAN_HWFILTER) 2587 ctrl |= IXGBE_VLNCTRL_VFE; 2588 else 2589 ctrl &= ~IXGBE_VLNCTRL_VFE; 2590 IXGBE_WRITE_REG(hw, IXGBE_VLNCTRL, ctrl); 2591 } /* ixgbe_setup_vlan_hw_support */ 2592 2593 /************************************************************************ 2594 * ixgbe_get_slot_info 2595 * 2596 * Get the width and transaction speed of 2597 * the slot this adapter is plugged into. 2598 ************************************************************************/ 2599 static void 2600 ixgbe_get_slot_info(struct adapter *adapter) 2601 { 2602 device_t dev = adapter->dev; 2603 struct ixgbe_hw *hw = &adapter->hw; 2604 u32 offset; 2605 u16 link; 2606 int bus_info_valid = TRUE; 2607 2608 /* Some devices are behind an internal bridge */ 2609 switch (hw->device_id) { 2610 case IXGBE_DEV_ID_82599_SFP_SF_QP: 2611 case IXGBE_DEV_ID_82599_QSFP_SF_QP: 2612 goto get_parent_info; 2613 default: 2614 break; 2615 } 2616 2617 ixgbe_get_bus_info(hw); 2618 2619 /* 2620 * Some devices don't use PCI-E, but there is no need 2621 * to display "Unknown" for bus speed and width. 2622 */ 2623 switch (hw->mac.type) { 2624 case ixgbe_mac_X550EM_x: 2625 case ixgbe_mac_X550EM_a: 2626 return; 2627 default: 2628 goto display; 2629 } 2630 2631 get_parent_info: 2632 /* 2633 * For the Quad port adapter we need to parse back 2634 * up the PCI tree to find the speed of the expansion 2635 * slot into which this adapter is plugged. A bit more work. 2636 */ 2637 dev = device_parent(device_parent(dev)); 2638 #if 0 2639 #ifdef IXGBE_DEBUG 2640 device_printf(dev, "parent pcib = %x,%x,%x\n", pci_get_bus(dev), 2641 pci_get_slot(dev), pci_get_function(dev)); 2642 #endif 2643 dev = device_parent(device_parent(dev)); 2644 #ifdef IXGBE_DEBUG 2645 device_printf(dev, "slot pcib = %x,%x,%x\n", pci_get_bus(dev), 2646 pci_get_slot(dev), pci_get_function(dev)); 2647 #endif 2648 #endif 2649 /* Now get the PCI Express Capabilities offset */ 2650 if (pci_get_capability(adapter->osdep.pc, adapter->osdep.tag, 2651 PCI_CAP_PCIEXPRESS, &offset, NULL)) { 2652 /* 2653 * Hmm...can't get PCI-Express capabilities. 2654 * Falling back to default method. 2655 */ 2656 bus_info_valid = FALSE; 2657 ixgbe_get_bus_info(hw); 2658 goto display; 2659 } 2660 /* ...and read the Link Status Register */ 2661 link = pci_conf_read(adapter->osdep.pc, adapter->osdep.tag, 2662 offset + PCIE_LCSR) >> 16; 2663 ixgbe_set_pci_config_data_generic(hw, link); 2664 2665 display: 2666 device_printf(dev, "PCI Express Bus: Speed %s Width %s\n", 2667 ((hw->bus.speed == ixgbe_bus_speed_8000) ? "8.0GT/s" : 2668 (hw->bus.speed == ixgbe_bus_speed_5000) ? "5.0GT/s" : 2669 (hw->bus.speed == ixgbe_bus_speed_2500) ? "2.5GT/s" : 2670 "Unknown"), 2671 ((hw->bus.width == ixgbe_bus_width_pcie_x8) ? "x8" : 2672 (hw->bus.width == ixgbe_bus_width_pcie_x4) ? "x4" : 2673 (hw->bus.width == ixgbe_bus_width_pcie_x1) ? "x1" : 2674 "Unknown")); 2675 2676 if (bus_info_valid) { 2677 if ((hw->device_id != IXGBE_DEV_ID_82599_SFP_SF_QP) && 2678 ((hw->bus.width <= ixgbe_bus_width_pcie_x4) && 2679 (hw->bus.speed == ixgbe_bus_speed_2500))) { 2680 device_printf(dev, "PCI-Express bandwidth available" 2681 " for this card\n is not sufficient for" 2682 " optimal performance.\n"); 2683 device_printf(dev, "For optimal performance a x8 " 2684 "PCIE, or x4 PCIE Gen2 slot is required.\n"); 2685 } 2686 if ((hw->device_id == IXGBE_DEV_ID_82599_SFP_SF_QP) && 2687 ((hw->bus.width <= ixgbe_bus_width_pcie_x8) && 2688 (hw->bus.speed < ixgbe_bus_speed_8000))) { 2689 device_printf(dev, "PCI-Express bandwidth available" 2690 " for this card\n is not sufficient for" 2691 " optimal performance.\n"); 2692 device_printf(dev, "For optimal performance a x8 " 2693 "PCIE Gen3 slot is required.\n"); 2694 } 2695 } else 2696 device_printf(dev, 2697 "Unable to determine slot speed/width. The speed/width " 2698 "reported are that of the internal switch.\n"); 2699 2700 return; 2701 } /* ixgbe_get_slot_info */ 2702 2703 /************************************************************************ 2704 * ixgbe_enable_queue - Queue Interrupt Enabler 2705 ************************************************************************/ 2706 static inline void 2707 ixgbe_enable_queue(struct adapter *adapter, u32 vector) 2708 { 2709 struct ixgbe_hw *hw = &adapter->hw; 2710 struct ix_queue *que = &adapter->queues[vector]; 2711 u64 queue = 1ULL << vector; 2712 u32 mask; 2713 2714 mutex_enter(&que->dc_mtx); 2715 if (que->disabled_count > 0 && --que->disabled_count > 0) 2716 goto out; 2717 2718 if (hw->mac.type == ixgbe_mac_82598EB) { 2719 mask = (IXGBE_EIMS_RTX_QUEUE & queue); 2720 IXGBE_WRITE_REG(hw, IXGBE_EIMS, mask); 2721 } else { 2722 mask = (queue & 0xFFFFFFFF); 2723 if (mask) 2724 IXGBE_WRITE_REG(hw, IXGBE_EIMS_EX(0), mask); 2725 mask = (queue >> 32); 2726 if (mask) 2727 IXGBE_WRITE_REG(hw, IXGBE_EIMS_EX(1), mask); 2728 } 2729 out: 2730 mutex_exit(&que->dc_mtx); 2731 } /* ixgbe_enable_queue */ 2732 2733 /************************************************************************ 2734 * ixgbe_disable_queue_internal 2735 ************************************************************************/ 2736 static inline void 2737 ixgbe_disable_queue_internal(struct adapter *adapter, u32 vector, bool nestok) 2738 { 2739 struct ixgbe_hw *hw = &adapter->hw; 2740 struct ix_queue *que = &adapter->queues[vector]; 2741 u64 queue = 1ULL << vector; 2742 u32 mask; 2743 2744 mutex_enter(&que->dc_mtx); 2745 2746 if (que->disabled_count > 0) { 2747 if (nestok) 2748 que->disabled_count++; 2749 goto out; 2750 } 2751 que->disabled_count++; 2752 2753 if (hw->mac.type == ixgbe_mac_82598EB) { 2754 mask = (IXGBE_EIMS_RTX_QUEUE & queue); 2755 IXGBE_WRITE_REG(hw, IXGBE_EIMC, mask); 2756 } else { 2757 mask = (queue & 0xFFFFFFFF); 2758 if (mask) 2759 IXGBE_WRITE_REG(hw, IXGBE_EIMC_EX(0), mask); 2760 mask = (queue >> 32); 2761 if (mask) 2762 IXGBE_WRITE_REG(hw, IXGBE_EIMC_EX(1), mask); 2763 } 2764 out: 2765 mutex_exit(&que->dc_mtx); 2766 } /* ixgbe_disable_queue_internal */ 2767 2768 /************************************************************************ 2769 * ixgbe_disable_queue 2770 ************************************************************************/ 2771 static inline void 2772 ixgbe_disable_queue(struct adapter *adapter, u32 vector) 2773 { 2774 2775 ixgbe_disable_queue_internal(adapter, vector, true); 2776 } /* ixgbe_disable_queue */ 2777 2778 /************************************************************************ 2779 * ixgbe_sched_handle_que - schedule deferred packet processing 2780 ************************************************************************/ 2781 static inline void 2782 ixgbe_sched_handle_que(struct adapter *adapter, struct ix_queue *que) 2783 { 2784 2785 if (que->txrx_use_workqueue) { 2786 /* 2787 * adapter->que_wq is bound to each CPU instead of 2788 * each NIC queue to reduce workqueue kthread. As we 2789 * should consider about interrupt affinity in this 2790 * function, the workqueue kthread must be WQ_PERCPU. 2791 * If create WQ_PERCPU workqueue kthread for each NIC 2792 * queue, that number of created workqueue kthread is 2793 * (number of used NIC queue) * (number of CPUs) = 2794 * (number of CPUs) ^ 2 most often. 2795 * 2796 * The same NIC queue's interrupts are avoided by 2797 * masking the queue's interrupt. And different 2798 * NIC queue's interrupts use different struct work 2799 * (que->wq_cookie). So, "enqueued flag" to avoid 2800 * twice workqueue_enqueue() is not required . 2801 */ 2802 workqueue_enqueue(adapter->que_wq, &que->wq_cookie, curcpu()); 2803 } else 2804 softint_schedule(que->que_si); 2805 } 2806 2807 /************************************************************************ 2808 * ixgbe_msix_que - MSI-X Queue Interrupt Service routine 2809 ************************************************************************/ 2810 static int 2811 ixgbe_msix_que(void *arg) 2812 { 2813 struct ix_queue *que = arg; 2814 struct adapter *adapter = que->adapter; 2815 struct ifnet *ifp = adapter->ifp; 2816 struct tx_ring *txr = que->txr; 2817 struct rx_ring *rxr = que->rxr; 2818 u32 newitr = 0; 2819 2820 /* Protect against spurious interrupts */ 2821 if ((ifp->if_flags & IFF_RUNNING) == 0) 2822 return 0; 2823 2824 ixgbe_disable_queue(adapter, que->msix); 2825 IXGBE_EVC_ADD(&que->irqs, 1); 2826 2827 /* 2828 * Don't change "que->txrx_use_workqueue" from this point to avoid 2829 * flip-flopping softint/workqueue mode in one deferred processing. 2830 */ 2831 que->txrx_use_workqueue = adapter->txrx_use_workqueue; 2832 2833 IXGBE_TX_LOCK(txr); 2834 ixgbe_txeof(txr); 2835 IXGBE_TX_UNLOCK(txr); 2836 2837 /* Do AIM now? */ 2838 2839 if (adapter->enable_aim == false) 2840 goto no_calc; 2841 /* 2842 * Do Adaptive Interrupt Moderation: 2843 * - Write out last calculated setting 2844 * - Calculate based on average size over 2845 * the last interval. 2846 */ 2847 if (que->eitr_setting) 2848 ixgbe_eitr_write(adapter, que->msix, que->eitr_setting); 2849 2850 que->eitr_setting = 0; 2851 2852 /* Idle, do nothing */ 2853 if ((txr->bytes == 0) && (rxr->bytes == 0)) 2854 goto no_calc; 2855 2856 if ((txr->bytes) && (txr->packets)) 2857 newitr = txr->bytes/txr->packets; 2858 if ((rxr->bytes) && (rxr->packets)) 2859 newitr = uimax(newitr, (rxr->bytes / rxr->packets)); 2860 newitr += 24; /* account for hardware frame, crc */ 2861 2862 /* set an upper boundary */ 2863 newitr = uimin(newitr, 3000); 2864 2865 /* Be nice to the mid range */ 2866 if ((newitr > 300) && (newitr < 1200)) 2867 newitr = (newitr / 3); 2868 else 2869 newitr = (newitr / 2); 2870 2871 /* 2872 * When RSC is used, ITR interval must be larger than RSC_DELAY. 2873 * Currently, we use 2us for RSC_DELAY. The minimum value is always 2874 * greater than 2us on 100M (and 10M?(not documented)), but it's not 2875 * on 1G and higher. 2876 */ 2877 if ((adapter->link_speed != IXGBE_LINK_SPEED_100_FULL) 2878 && (adapter->link_speed != IXGBE_LINK_SPEED_10_FULL)) 2879 if (newitr < IXGBE_MIN_RSC_EITR_10G1G) 2880 newitr = IXGBE_MIN_RSC_EITR_10G1G; 2881 2882 /* save for next interrupt */ 2883 que->eitr_setting = newitr; 2884 2885 /* Reset state */ 2886 txr->bytes = 0; 2887 txr->packets = 0; 2888 rxr->bytes = 0; 2889 rxr->packets = 0; 2890 2891 no_calc: 2892 ixgbe_sched_handle_que(adapter, que); 2893 2894 return 1; 2895 } /* ixgbe_msix_que */ 2896 2897 /************************************************************************ 2898 * ixgbe_media_status - Media Ioctl callback 2899 * 2900 * Called whenever the user queries the status of 2901 * the interface using ifconfig. 2902 ************************************************************************/ 2903 static void 2904 ixgbe_media_status(struct ifnet *ifp, struct ifmediareq *ifmr) 2905 { 2906 struct adapter *adapter = ifp->if_softc; 2907 struct ixgbe_hw *hw = &adapter->hw; 2908 int layer; 2909 2910 INIT_DEBUGOUT("ixgbe_media_status: begin"); 2911 ixgbe_update_link_status(adapter); 2912 2913 ifmr->ifm_status = IFM_AVALID; 2914 ifmr->ifm_active = IFM_ETHER; 2915 2916 if (adapter->link_active != LINK_STATE_UP) { 2917 ifmr->ifm_active |= IFM_NONE; 2918 return; 2919 } 2920 2921 ifmr->ifm_status |= IFM_ACTIVE; 2922 layer = adapter->phy_layer; 2923 2924 if (layer & IXGBE_PHYSICAL_LAYER_10GBASE_T || 2925 layer & IXGBE_PHYSICAL_LAYER_5GBASE_T || 2926 layer & IXGBE_PHYSICAL_LAYER_2500BASE_T || 2927 layer & IXGBE_PHYSICAL_LAYER_1000BASE_T || 2928 layer & IXGBE_PHYSICAL_LAYER_100BASE_TX || 2929 layer & IXGBE_PHYSICAL_LAYER_10BASE_T) 2930 switch (adapter->link_speed) { 2931 case IXGBE_LINK_SPEED_10GB_FULL: 2932 ifmr->ifm_active |= IFM_10G_T | IFM_FDX; 2933 break; 2934 case IXGBE_LINK_SPEED_5GB_FULL: 2935 ifmr->ifm_active |= IFM_5000_T | IFM_FDX; 2936 break; 2937 case IXGBE_LINK_SPEED_2_5GB_FULL: 2938 ifmr->ifm_active |= IFM_2500_T | IFM_FDX; 2939 break; 2940 case IXGBE_LINK_SPEED_1GB_FULL: 2941 ifmr->ifm_active |= IFM_1000_T | IFM_FDX; 2942 break; 2943 case IXGBE_LINK_SPEED_100_FULL: 2944 ifmr->ifm_active |= IFM_100_TX | IFM_FDX; 2945 break; 2946 case IXGBE_LINK_SPEED_10_FULL: 2947 ifmr->ifm_active |= IFM_10_T | IFM_FDX; 2948 break; 2949 } 2950 if (layer & IXGBE_PHYSICAL_LAYER_SFP_PLUS_CU || 2951 layer & IXGBE_PHYSICAL_LAYER_SFP_ACTIVE_DA) 2952 switch (adapter->link_speed) { 2953 case IXGBE_LINK_SPEED_10GB_FULL: 2954 ifmr->ifm_active |= IFM_10G_TWINAX | IFM_FDX; 2955 break; 2956 } 2957 if (layer & IXGBE_PHYSICAL_LAYER_10GBASE_LR) 2958 switch (adapter->link_speed) { 2959 case IXGBE_LINK_SPEED_10GB_FULL: 2960 ifmr->ifm_active |= IFM_10G_LR | IFM_FDX; 2961 break; 2962 case IXGBE_LINK_SPEED_1GB_FULL: 2963 ifmr->ifm_active |= IFM_1000_LX | IFM_FDX; 2964 break; 2965 } 2966 if (layer & IXGBE_PHYSICAL_LAYER_10GBASE_LRM) 2967 switch (adapter->link_speed) { 2968 case IXGBE_LINK_SPEED_10GB_FULL: 2969 ifmr->ifm_active |= IFM_10G_LRM | IFM_FDX; 2970 break; 2971 case IXGBE_LINK_SPEED_1GB_FULL: 2972 ifmr->ifm_active |= IFM_1000_LX | IFM_FDX; 2973 break; 2974 } 2975 if (layer & IXGBE_PHYSICAL_LAYER_10GBASE_SR || 2976 layer & IXGBE_PHYSICAL_LAYER_1000BASE_SX) 2977 switch (adapter->link_speed) { 2978 case IXGBE_LINK_SPEED_10GB_FULL: 2979 ifmr->ifm_active |= IFM_10G_SR | IFM_FDX; 2980 break; 2981 case IXGBE_LINK_SPEED_1GB_FULL: 2982 ifmr->ifm_active |= IFM_1000_SX | IFM_FDX; 2983 break; 2984 } 2985 if (layer & IXGBE_PHYSICAL_LAYER_10GBASE_CX4) 2986 switch (adapter->link_speed) { 2987 case IXGBE_LINK_SPEED_10GB_FULL: 2988 ifmr->ifm_active |= IFM_10G_CX4 | IFM_FDX; 2989 break; 2990 } 2991 /* 2992 * XXX: These need to use the proper media types once 2993 * they're added. 2994 */ 2995 if (layer & IXGBE_PHYSICAL_LAYER_10GBASE_KR) 2996 switch (adapter->link_speed) { 2997 case IXGBE_LINK_SPEED_10GB_FULL: 2998 ifmr->ifm_active |= IFM_10G_KR | IFM_FDX; 2999 break; 3000 case IXGBE_LINK_SPEED_2_5GB_FULL: 3001 ifmr->ifm_active |= IFM_2500_KX | IFM_FDX; 3002 break; 3003 case IXGBE_LINK_SPEED_1GB_FULL: 3004 ifmr->ifm_active |= IFM_1000_KX | IFM_FDX; 3005 break; 3006 } 3007 else if (layer & IXGBE_PHYSICAL_LAYER_10GBASE_KX4 || 3008 layer & IXGBE_PHYSICAL_LAYER_2500BASE_KX || 3009 layer & IXGBE_PHYSICAL_LAYER_1000BASE_KX) 3010 switch (adapter->link_speed) { 3011 case IXGBE_LINK_SPEED_10GB_FULL: 3012 ifmr->ifm_active |= IFM_10G_KX4 | IFM_FDX; 3013 break; 3014 case IXGBE_LINK_SPEED_2_5GB_FULL: 3015 ifmr->ifm_active |= IFM_2500_KX | IFM_FDX; 3016 break; 3017 case IXGBE_LINK_SPEED_1GB_FULL: 3018 ifmr->ifm_active |= IFM_1000_KX | IFM_FDX; 3019 break; 3020 } 3021 3022 /* If nothing is recognized... */ 3023 #if 0 3024 if (IFM_SUBTYPE(ifmr->ifm_active) == 0) 3025 ifmr->ifm_active |= IFM_UNKNOWN; 3026 #endif 3027 3028 ifp->if_baudrate = ifmedia_baudrate(ifmr->ifm_active); 3029 3030 /* Display current flow control setting used on link */ 3031 if (hw->fc.current_mode == ixgbe_fc_rx_pause || 3032 hw->fc.current_mode == ixgbe_fc_full) 3033 ifmr->ifm_active |= IFM_ETH_RXPAUSE; 3034 if (hw->fc.current_mode == ixgbe_fc_tx_pause || 3035 hw->fc.current_mode == ixgbe_fc_full) 3036 ifmr->ifm_active |= IFM_ETH_TXPAUSE; 3037 3038 return; 3039 } /* ixgbe_media_status */ 3040 3041 /************************************************************************ 3042 * ixgbe_media_change - Media Ioctl callback 3043 * 3044 * Called when the user changes speed/duplex using 3045 * media/mediopt option with ifconfig. 3046 ************************************************************************/ 3047 static int 3048 ixgbe_media_change(struct ifnet *ifp) 3049 { 3050 struct adapter *adapter = ifp->if_softc; 3051 struct ifmedia *ifm = &adapter->media; 3052 struct ixgbe_hw *hw = &adapter->hw; 3053 ixgbe_link_speed speed = 0; 3054 ixgbe_link_speed link_caps = 0; 3055 bool negotiate = false; 3056 s32 err = IXGBE_NOT_IMPLEMENTED; 3057 3058 INIT_DEBUGOUT("ixgbe_media_change: begin"); 3059 3060 if (IFM_TYPE(ifm->ifm_media) != IFM_ETHER) 3061 return (EINVAL); 3062 3063 if (hw->phy.media_type == ixgbe_media_type_backplane) 3064 return (EPERM); 3065 3066 /* 3067 * We don't actually need to check against the supported 3068 * media types of the adapter; ifmedia will take care of 3069 * that for us. 3070 */ 3071 switch (IFM_SUBTYPE(ifm->ifm_media)) { 3072 case IFM_AUTO: 3073 err = hw->mac.ops.get_link_capabilities(hw, &link_caps, 3074 &negotiate); 3075 if (err != IXGBE_SUCCESS) { 3076 device_printf(adapter->dev, "Unable to determine " 3077 "supported advertise speeds\n"); 3078 return (ENODEV); 3079 } 3080 speed |= link_caps; 3081 break; 3082 case IFM_10G_T: 3083 case IFM_10G_LRM: 3084 case IFM_10G_LR: 3085 case IFM_10G_TWINAX: 3086 case IFM_10G_SR: 3087 case IFM_10G_CX4: 3088 case IFM_10G_KR: 3089 case IFM_10G_KX4: 3090 speed |= IXGBE_LINK_SPEED_10GB_FULL; 3091 break; 3092 case IFM_5000_T: 3093 speed |= IXGBE_LINK_SPEED_5GB_FULL; 3094 break; 3095 case IFM_2500_T: 3096 case IFM_2500_KX: 3097 speed |= IXGBE_LINK_SPEED_2_5GB_FULL; 3098 break; 3099 case IFM_1000_T: 3100 case IFM_1000_LX: 3101 case IFM_1000_SX: 3102 case IFM_1000_KX: 3103 speed |= IXGBE_LINK_SPEED_1GB_FULL; 3104 break; 3105 case IFM_100_TX: 3106 speed |= IXGBE_LINK_SPEED_100_FULL; 3107 break; 3108 case IFM_10_T: 3109 speed |= IXGBE_LINK_SPEED_10_FULL; 3110 break; 3111 case IFM_NONE: 3112 break; 3113 default: 3114 goto invalid; 3115 } 3116 3117 hw->mac.autotry_restart = TRUE; 3118 hw->mac.ops.setup_link(hw, speed, TRUE); 3119 adapter->advertise = 0; 3120 if (IFM_SUBTYPE(ifm->ifm_media) != IFM_AUTO) { 3121 if ((speed & IXGBE_LINK_SPEED_10GB_FULL) != 0) 3122 adapter->advertise |= 1 << 2; 3123 if ((speed & IXGBE_LINK_SPEED_1GB_FULL) != 0) 3124 adapter->advertise |= 1 << 1; 3125 if ((speed & IXGBE_LINK_SPEED_100_FULL) != 0) 3126 adapter->advertise |= 1 << 0; 3127 if ((speed & IXGBE_LINK_SPEED_10_FULL) != 0) 3128 adapter->advertise |= 1 << 3; 3129 if ((speed & IXGBE_LINK_SPEED_2_5GB_FULL) != 0) 3130 adapter->advertise |= 1 << 4; 3131 if ((speed & IXGBE_LINK_SPEED_5GB_FULL) != 0) 3132 adapter->advertise |= 1 << 5; 3133 } 3134 3135 return (0); 3136 3137 invalid: 3138 device_printf(adapter->dev, "Invalid media type!\n"); 3139 3140 return (EINVAL); 3141 } /* ixgbe_media_change */ 3142 3143 /************************************************************************ 3144 * ixgbe_msix_admin - Link status change ISR (MSI-X) 3145 ************************************************************************/ 3146 static int 3147 ixgbe_msix_admin(void *arg) 3148 { 3149 struct adapter *adapter = arg; 3150 struct ixgbe_hw *hw = &adapter->hw; 3151 u32 eicr; 3152 u32 eims_orig; 3153 u32 eims_disable = 0; 3154 3155 IXGBE_EVC_ADD(&adapter->admin_irqev, 1); 3156 3157 eims_orig = IXGBE_READ_REG(hw, IXGBE_EIMS); 3158 /* Pause other interrupts */ 3159 IXGBE_WRITE_REG(hw, IXGBE_EIMC, IXGBE_MSIX_OTHER_CLEAR_MASK); 3160 3161 /* 3162 * First get the cause. 3163 * 3164 * The specifications of 82598, 82599, X540 and X550 say EICS register 3165 * is write only. However, Linux says it is a workaround for silicon 3166 * errata to read EICS instead of EICR to get interrupt cause. 3167 * At least, reading EICR clears lower 16bits of EIMS on 82598. 3168 */ 3169 eicr = IXGBE_READ_REG(hw, IXGBE_EICS); 3170 /* Be sure the queue bits are not cleared */ 3171 eicr &= ~IXGBE_EICR_RTX_QUEUE; 3172 /* Clear all OTHER interrupts with write */ 3173 IXGBE_WRITE_REG(hw, IXGBE_EICR, eicr); 3174 3175 ixgbe_intr_admin_common(adapter, eicr, &eims_disable); 3176 3177 /* Re-enable some OTHER interrupts */ 3178 IXGBE_WRITE_REG(hw, IXGBE_EIMS, eims_orig & ~eims_disable); 3179 3180 return 1; 3181 } /* ixgbe_msix_admin */ 3182 3183 static void 3184 ixgbe_intr_admin_common(struct adapter *adapter, u32 eicr, u32 *eims_disable) 3185 { 3186 struct ixgbe_hw *hw = &adapter->hw; 3187 u32 task_requests = 0; 3188 s32 retval; 3189 3190 /* Link status change */ 3191 if (eicr & IXGBE_EICR_LSC) { 3192 task_requests |= IXGBE_REQUEST_TASK_LSC; 3193 *eims_disable |= IXGBE_EIMS_LSC; 3194 } 3195 3196 if (ixgbe_is_sfp(hw)) { 3197 u32 eicr_mask; 3198 3199 /* Pluggable optics-related interrupt */ 3200 if (hw->mac.type >= ixgbe_mac_X540) 3201 eicr_mask = IXGBE_EICR_GPI_SDP0_X540; 3202 else 3203 eicr_mask = IXGBE_EICR_GPI_SDP2_BY_MAC(hw); 3204 3205 /* 3206 * An interrupt might not arrive when a module is inserted. 3207 * When an link status change interrupt occurred and the driver 3208 * still regard SFP as unplugged, issue the module softint 3209 * and then issue LSC interrupt. 3210 */ 3211 if ((eicr & eicr_mask) 3212 || ((hw->phy.sfp_type == ixgbe_sfp_type_not_present) 3213 && (eicr & IXGBE_EICR_LSC))) { 3214 task_requests |= IXGBE_REQUEST_TASK_MOD; 3215 *eims_disable |= IXGBE_EIMS_LSC; 3216 } 3217 3218 if ((hw->mac.type == ixgbe_mac_82599EB) && 3219 (eicr & IXGBE_EICR_GPI_SDP1_BY_MAC(hw))) { 3220 task_requests |= IXGBE_REQUEST_TASK_MSF; 3221 *eims_disable |= IXGBE_EIMS_GPI_SDP1_BY_MAC(hw); 3222 } 3223 } 3224 3225 if (adapter->hw.mac.type != ixgbe_mac_82598EB) { 3226 #ifdef IXGBE_FDIR 3227 if ((adapter->feat_en & IXGBE_FEATURE_FDIR) && 3228 (eicr & IXGBE_EICR_FLOW_DIR)) { 3229 if (!atomic_cas_uint(&adapter->fdir_reinit, 0, 1)) { 3230 task_requests |= IXGBE_REQUEST_TASK_FDIR; 3231 /* Disable the interrupt */ 3232 *eims_disable |= IXGBE_EIMS_FLOW_DIR; 3233 } 3234 } 3235 #endif 3236 3237 if (eicr & IXGBE_EICR_ECC) { 3238 if (ratecheck(&adapter->lasterr_time, 3239 &ixgbe_errlog_intrvl)) 3240 device_printf(adapter->dev, 3241 "CRITICAL: ECC ERROR!! Please Reboot!!\n"); 3242 } 3243 3244 /* Check for over temp condition */ 3245 if (adapter->feat_en & IXGBE_FEATURE_TEMP_SENSOR) { 3246 switch (adapter->hw.mac.type) { 3247 case ixgbe_mac_X550EM_a: 3248 if (!(eicr & IXGBE_EICR_GPI_SDP0_X550EM_a)) 3249 break; 3250 retval = hw->phy.ops.check_overtemp(hw); 3251 if (retval != IXGBE_ERR_OVERTEMP) 3252 break; 3253 if (ratecheck(&adapter->lasterr_time, 3254 &ixgbe_errlog_intrvl)) { 3255 device_printf(adapter->dev, 3256 "CRITICAL: OVER TEMP!! " 3257 "PHY IS SHUT DOWN!!\n"); 3258 device_printf(adapter->dev, 3259 "System shutdown required!\n"); 3260 } 3261 break; 3262 default: 3263 if (!(eicr & IXGBE_EICR_TS)) 3264 break; 3265 retval = hw->phy.ops.check_overtemp(hw); 3266 if (retval != IXGBE_ERR_OVERTEMP) 3267 break; 3268 if (ratecheck(&adapter->lasterr_time, 3269 &ixgbe_errlog_intrvl)) { 3270 device_printf(adapter->dev, 3271 "CRITICAL: OVER TEMP!! " 3272 "PHY IS SHUT DOWN!!\n"); 3273 device_printf(adapter->dev, 3274 "System shutdown required!\n"); 3275 } 3276 break; 3277 } 3278 } 3279 3280 /* Check for VF message */ 3281 if ((adapter->feat_en & IXGBE_FEATURE_SRIOV) && 3282 (eicr & IXGBE_EICR_MAILBOX)) { 3283 task_requests |= IXGBE_REQUEST_TASK_MBX; 3284 *eims_disable |= IXGBE_EIMS_MAILBOX; 3285 } 3286 } 3287 3288 /* Check for fan failure */ 3289 if (adapter->feat_en & IXGBE_FEATURE_FAN_FAIL) 3290 ixgbe_check_fan_failure(adapter, eicr, true); 3291 3292 /* External PHY interrupt */ 3293 if ((hw->phy.type == ixgbe_phy_x550em_ext_t) && 3294 (eicr & IXGBE_EICR_GPI_SDP0_X540)) { 3295 task_requests |= IXGBE_REQUEST_TASK_PHY; 3296 *eims_disable |= IXGBE_EICR_GPI_SDP0_X540; 3297 } 3298 3299 if (task_requests != 0) { 3300 mutex_enter(&adapter->admin_mtx); 3301 adapter->task_requests |= task_requests; 3302 ixgbe_schedule_admin_tasklet(adapter); 3303 mutex_exit(&adapter->admin_mtx); 3304 } 3305 } 3306 3307 static void 3308 ixgbe_eitr_write(struct adapter *adapter, uint32_t index, uint32_t itr) 3309 { 3310 3311 if (adapter->hw.mac.type == ixgbe_mac_82598EB) 3312 itr |= itr << 16; 3313 else 3314 itr |= IXGBE_EITR_CNT_WDIS; 3315 3316 IXGBE_WRITE_REG(&adapter->hw, IXGBE_EITR(index), itr); 3317 } 3318 3319 3320 /************************************************************************ 3321 * ixgbe_sysctl_interrupt_rate_handler 3322 ************************************************************************/ 3323 static int 3324 ixgbe_sysctl_interrupt_rate_handler(SYSCTLFN_ARGS) 3325 { 3326 struct sysctlnode node = *rnode; 3327 struct ix_queue *que = (struct ix_queue *)node.sysctl_data; 3328 struct adapter *adapter; 3329 uint32_t reg, usec, rate; 3330 int error; 3331 3332 if (que == NULL) 3333 return 0; 3334 3335 adapter = que->adapter; 3336 if (ixgbe_fw_recovery_mode_swflag(adapter)) 3337 return (EPERM); 3338 3339 reg = IXGBE_READ_REG(&adapter->hw, IXGBE_EITR(que->msix)); 3340 usec = ((reg & 0x0FF8) >> 3); 3341 if (usec > 0) 3342 rate = 500000 / usec; 3343 else 3344 rate = 0; 3345 node.sysctl_data = &rate; 3346 error = sysctl_lookup(SYSCTLFN_CALL(&node)); 3347 if (error || newp == NULL) 3348 return error; 3349 reg &= ~0xfff; /* default, no limitation */ 3350 if (rate > 0 && rate < 500000) { 3351 if (rate < 1000) 3352 rate = 1000; 3353 reg |= ((4000000 / rate) & 0xff8); 3354 /* 3355 * When RSC is used, ITR interval must be larger than 3356 * RSC_DELAY. Currently, we use 2us for RSC_DELAY. 3357 * The minimum value is always greater than 2us on 100M 3358 * (and 10M?(not documented)), but it's not on 1G and higher. 3359 */ 3360 if ((adapter->link_speed != IXGBE_LINK_SPEED_100_FULL) 3361 && (adapter->link_speed != IXGBE_LINK_SPEED_10_FULL)) { 3362 if ((adapter->num_queues > 1) 3363 && (reg < IXGBE_MIN_RSC_EITR_10G1G)) 3364 return EINVAL; 3365 } 3366 ixgbe_max_interrupt_rate = rate; 3367 } else 3368 ixgbe_max_interrupt_rate = 0; 3369 ixgbe_eitr_write(adapter, que->msix, reg); 3370 3371 return (0); 3372 } /* ixgbe_sysctl_interrupt_rate_handler */ 3373 3374 const struct sysctlnode * 3375 ixgbe_sysctl_instance(struct adapter *adapter) 3376 { 3377 const char *dvname; 3378 struct sysctllog **log; 3379 int rc; 3380 const struct sysctlnode *rnode; 3381 3382 if (adapter->sysctltop != NULL) 3383 return adapter->sysctltop; 3384 3385 log = &adapter->sysctllog; 3386 dvname = device_xname(adapter->dev); 3387 3388 if ((rc = sysctl_createv(log, 0, NULL, &rnode, 3389 0, CTLTYPE_NODE, dvname, 3390 SYSCTL_DESCR("ixgbe information and settings"), 3391 NULL, 0, NULL, 0, CTL_HW, CTL_CREATE, CTL_EOL)) != 0) 3392 goto err; 3393 3394 return rnode; 3395 err: 3396 device_printf(adapter->dev, 3397 "%s: sysctl_createv failed, rc = %d\n", __func__, rc); 3398 return NULL; 3399 } 3400 3401 /************************************************************************ 3402 * ixgbe_add_device_sysctls 3403 ************************************************************************/ 3404 static void 3405 ixgbe_add_device_sysctls(struct adapter *adapter) 3406 { 3407 device_t dev = adapter->dev; 3408 struct ixgbe_hw *hw = &adapter->hw; 3409 struct sysctllog **log; 3410 const struct sysctlnode *rnode, *cnode; 3411 3412 log = &adapter->sysctllog; 3413 3414 if ((rnode = ixgbe_sysctl_instance(adapter)) == NULL) { 3415 aprint_error_dev(dev, "could not create sysctl root\n"); 3416 return; 3417 } 3418 3419 if (sysctl_createv(log, 0, &rnode, &cnode, 3420 CTLFLAG_READWRITE, CTLTYPE_INT, 3421 "debug", SYSCTL_DESCR("Debug Info"), 3422 ixgbe_sysctl_debug, 0, (void *)adapter, 0, CTL_CREATE, CTL_EOL) 3423 != 0) 3424 aprint_error_dev(dev, "could not create sysctl\n"); 3425 3426 if (sysctl_createv(log, 0, &rnode, &cnode, 3427 CTLFLAG_READWRITE, CTLTYPE_INT, 3428 "rx_copy_len", SYSCTL_DESCR("RX Copy Length"), 3429 ixgbe_sysctl_rx_copy_len, 0, 3430 (void *)adapter, 0, CTL_CREATE, CTL_EOL) != 0) 3431 aprint_error_dev(dev, "could not create sysctl\n"); 3432 3433 if (sysctl_createv(log, 0, &rnode, &cnode, 3434 CTLFLAG_READONLY, CTLTYPE_INT, 3435 "num_tx_desc", SYSCTL_DESCR("Number of TX descriptors"), 3436 NULL, 0, &adapter->num_tx_desc, 0, CTL_CREATE, CTL_EOL) != 0) 3437 aprint_error_dev(dev, "could not create sysctl\n"); 3438 3439 if (sysctl_createv(log, 0, &rnode, &cnode, 3440 CTLFLAG_READONLY, CTLTYPE_INT, 3441 "num_rx_desc", SYSCTL_DESCR("Number of RX descriptors"), 3442 NULL, 0, &adapter->num_rx_desc, 0, CTL_CREATE, CTL_EOL) != 0) 3443 aprint_error_dev(dev, "could not create sysctl\n"); 3444 3445 if (sysctl_createv(log, 0, &rnode, &cnode, 3446 CTLFLAG_READWRITE, CTLTYPE_INT, "rx_process_limit", 3447 SYSCTL_DESCR("max number of RX packets to process"), 3448 ixgbe_sysctl_rx_process_limit, 0, (void *)adapter, 0, CTL_CREATE, 3449 CTL_EOL) != 0) 3450 aprint_error_dev(dev, "could not create sysctl\n"); 3451 3452 if (sysctl_createv(log, 0, &rnode, &cnode, 3453 CTLFLAG_READWRITE, CTLTYPE_INT, "tx_process_limit", 3454 SYSCTL_DESCR("max number of TX packets to process"), 3455 ixgbe_sysctl_tx_process_limit, 0, (void *)adapter, 0, CTL_CREATE, 3456 CTL_EOL) != 0) 3457 aprint_error_dev(dev, "could not create sysctl\n"); 3458 3459 if (sysctl_createv(log, 0, &rnode, &cnode, 3460 CTLFLAG_READONLY, CTLTYPE_INT, 3461 "num_queues", SYSCTL_DESCR("Number of queues"), 3462 NULL, 0, &adapter->num_queues, 0, CTL_CREATE, CTL_EOL) != 0) 3463 aprint_error_dev(dev, "could not create sysctl\n"); 3464 3465 /* Sysctls for all devices */ 3466 if (sysctl_createv(log, 0, &rnode, &cnode, CTLFLAG_READWRITE, 3467 CTLTYPE_INT, "fc", SYSCTL_DESCR(IXGBE_SYSCTL_DESC_SET_FC), 3468 ixgbe_sysctl_flowcntl, 0, (void *)adapter, 0, CTL_CREATE, 3469 CTL_EOL) != 0) 3470 aprint_error_dev(dev, "could not create sysctl\n"); 3471 3472 adapter->enable_aim = ixgbe_enable_aim; 3473 if (sysctl_createv(log, 0, &rnode, &cnode, CTLFLAG_READWRITE, 3474 CTLTYPE_BOOL, "enable_aim", SYSCTL_DESCR("Interrupt Moderation"), 3475 NULL, 0, &adapter->enable_aim, 0, CTL_CREATE, CTL_EOL) != 0) 3476 aprint_error_dev(dev, "could not create sysctl\n"); 3477 3478 if (sysctl_createv(log, 0, &rnode, &cnode, 3479 CTLFLAG_READWRITE, CTLTYPE_INT, 3480 "advertise_speed", SYSCTL_DESCR(IXGBE_SYSCTL_DESC_ADV_SPEED), 3481 ixgbe_sysctl_advertise, 0, (void *)adapter, 0, CTL_CREATE, 3482 CTL_EOL) != 0) 3483 aprint_error_dev(dev, "could not create sysctl\n"); 3484 3485 /* 3486 * If each "que->txrx_use_workqueue" is changed in sysctl handler, 3487 * it causesflip-flopping softint/workqueue mode in one deferred 3488 * processing. Therefore, preempt_disable()/preempt_enable() are 3489 * required in ixgbe_sched_handle_que() to avoid 3490 * KASSERT(ixgbe_sched_handle_que()) in softint_schedule(). 3491 * I think changing "que->txrx_use_workqueue" in interrupt handler 3492 * is lighter than doing preempt_disable()/preempt_enable() in every 3493 * ixgbe_sched_handle_que(). 3494 */ 3495 adapter->txrx_use_workqueue = ixgbe_txrx_workqueue; 3496 if (sysctl_createv(log, 0, &rnode, &cnode, CTLFLAG_READWRITE, 3497 CTLTYPE_BOOL, "txrx_workqueue", 3498 SYSCTL_DESCR("Use workqueue for packet processing"), 3499 NULL, 0, &adapter->txrx_use_workqueue, 0, CTL_CREATE, 3500 CTL_EOL) != 0) 3501 aprint_error_dev(dev, "could not create sysctl\n"); 3502 3503 #ifdef IXGBE_DEBUG 3504 /* testing sysctls (for all devices) */ 3505 if (sysctl_createv(log, 0, &rnode, &cnode, CTLFLAG_READWRITE, 3506 CTLTYPE_INT, "power_state", SYSCTL_DESCR("PCI Power State"), 3507 ixgbe_sysctl_power_state, 0, (void *)adapter, 0, CTL_CREATE, 3508 CTL_EOL) != 0) 3509 aprint_error_dev(dev, "could not create sysctl\n"); 3510 3511 if (sysctl_createv(log, 0, &rnode, &cnode, CTLFLAG_READONLY, 3512 CTLTYPE_STRING, "print_rss_config", 3513 SYSCTL_DESCR("Prints RSS Configuration"), 3514 ixgbe_sysctl_print_rss_config, 0, (void *)adapter, 0, CTL_CREATE, 3515 CTL_EOL) != 0) 3516 aprint_error_dev(dev, "could not create sysctl\n"); 3517 #endif 3518 /* for X550 series devices */ 3519 if (hw->mac.type >= ixgbe_mac_X550) 3520 if (sysctl_createv(log, 0, &rnode, &cnode, CTLFLAG_READWRITE, 3521 CTLTYPE_INT, "dmac", SYSCTL_DESCR("DMA Coalesce"), 3522 ixgbe_sysctl_dmac, 0, (void *)adapter, 0, CTL_CREATE, 3523 CTL_EOL) != 0) 3524 aprint_error_dev(dev, "could not create sysctl\n"); 3525 3526 /* for WoL-capable devices */ 3527 if (adapter->wol_support) { 3528 if (sysctl_createv(log, 0, &rnode, &cnode, CTLFLAG_READWRITE, 3529 CTLTYPE_BOOL, "wol_enable", 3530 SYSCTL_DESCR("Enable/Disable Wake on LAN"), 3531 ixgbe_sysctl_wol_enable, 0, (void *)adapter, 0, CTL_CREATE, 3532 CTL_EOL) != 0) 3533 aprint_error_dev(dev, "could not create sysctl\n"); 3534 3535 if (sysctl_createv(log, 0, &rnode, &cnode, CTLFLAG_READWRITE, 3536 CTLTYPE_INT, "wufc", 3537 SYSCTL_DESCR("Enable/Disable Wake Up Filters"), 3538 ixgbe_sysctl_wufc, 0, (void *)adapter, 0, CTL_CREATE, 3539 CTL_EOL) != 0) 3540 aprint_error_dev(dev, "could not create sysctl\n"); 3541 } 3542 3543 /* for X552/X557-AT devices */ 3544 if ((hw->device_id == IXGBE_DEV_ID_X550EM_X_10G_T) || 3545 (hw->device_id == IXGBE_DEV_ID_X550EM_A_10G_T)) { 3546 const struct sysctlnode *phy_node; 3547 3548 if (sysctl_createv(log, 0, &rnode, &phy_node, 0, CTLTYPE_NODE, 3549 "phy", SYSCTL_DESCR("External PHY sysctls"), 3550 NULL, 0, NULL, 0, CTL_CREATE, CTL_EOL) != 0) { 3551 aprint_error_dev(dev, "could not create sysctl\n"); 3552 return; 3553 } 3554 3555 if (sysctl_createv(log, 0, &phy_node, &cnode, CTLFLAG_READONLY, 3556 CTLTYPE_INT, "temp", 3557 SYSCTL_DESCR("Current External PHY Temperature (Celsius)"), 3558 ixgbe_sysctl_phy_temp, 0, (void *)adapter, 0, CTL_CREATE, 3559 CTL_EOL) != 0) 3560 aprint_error_dev(dev, "could not create sysctl\n"); 3561 3562 if (sysctl_createv(log, 0, &phy_node, &cnode, CTLFLAG_READONLY, 3563 CTLTYPE_INT, "overtemp_occurred", 3564 SYSCTL_DESCR( 3565 "External PHY High Temperature Event Occurred"), 3566 ixgbe_sysctl_phy_overtemp_occurred, 0, (void *)adapter, 0, 3567 CTL_CREATE, CTL_EOL) != 0) 3568 aprint_error_dev(dev, "could not create sysctl\n"); 3569 } 3570 3571 if ((hw->mac.type == ixgbe_mac_X550EM_a) 3572 && (hw->phy.type == ixgbe_phy_fw)) 3573 if (sysctl_createv(log, 0, &rnode, &cnode, CTLFLAG_READWRITE, 3574 CTLTYPE_BOOL, "force_10_100_autonego", 3575 SYSCTL_DESCR("Force autonego on 10M and 100M"), 3576 NULL, 0, &hw->phy.force_10_100_autonego, 0, 3577 CTL_CREATE, CTL_EOL) != 0) 3578 aprint_error_dev(dev, "could not create sysctl\n"); 3579 3580 if (adapter->feat_cap & IXGBE_FEATURE_EEE) { 3581 if (sysctl_createv(log, 0, &rnode, &cnode, CTLFLAG_READWRITE, 3582 CTLTYPE_INT, "eee_state", 3583 SYSCTL_DESCR("EEE Power Save State"), 3584 ixgbe_sysctl_eee_state, 0, (void *)adapter, 0, CTL_CREATE, 3585 CTL_EOL) != 0) 3586 aprint_error_dev(dev, "could not create sysctl\n"); 3587 } 3588 } /* ixgbe_add_device_sysctls */ 3589 3590 /************************************************************************ 3591 * ixgbe_allocate_pci_resources 3592 ************************************************************************/ 3593 static int 3594 ixgbe_allocate_pci_resources(struct adapter *adapter, 3595 const struct pci_attach_args *pa) 3596 { 3597 pcireg_t memtype, csr; 3598 device_t dev = adapter->dev; 3599 bus_addr_t addr; 3600 int flags; 3601 3602 memtype = pci_mapreg_type(pa->pa_pc, pa->pa_tag, PCI_BAR(0)); 3603 switch (memtype) { 3604 case PCI_MAPREG_TYPE_MEM | PCI_MAPREG_MEM_TYPE_32BIT: 3605 case PCI_MAPREG_TYPE_MEM | PCI_MAPREG_MEM_TYPE_64BIT: 3606 adapter->osdep.mem_bus_space_tag = pa->pa_memt; 3607 if (pci_mapreg_info(pa->pa_pc, pa->pa_tag, PCI_BAR(0), 3608 memtype, &addr, &adapter->osdep.mem_size, &flags) != 0) 3609 goto map_err; 3610 if ((flags & BUS_SPACE_MAP_PREFETCHABLE) != 0) { 3611 aprint_normal_dev(dev, "clearing prefetchable bit\n"); 3612 flags &= ~BUS_SPACE_MAP_PREFETCHABLE; 3613 } 3614 if (bus_space_map(adapter->osdep.mem_bus_space_tag, addr, 3615 adapter->osdep.mem_size, flags, 3616 &adapter->osdep.mem_bus_space_handle) != 0) { 3617 map_err: 3618 adapter->osdep.mem_size = 0; 3619 aprint_error_dev(dev, "unable to map BAR0\n"); 3620 return ENXIO; 3621 } 3622 /* 3623 * Enable address decoding for memory range in case BIOS or 3624 * UEFI don't set it. 3625 */ 3626 csr = pci_conf_read(pa->pa_pc, pa->pa_tag, 3627 PCI_COMMAND_STATUS_REG); 3628 csr |= PCI_COMMAND_MEM_ENABLE; 3629 pci_conf_write(pa->pa_pc, pa->pa_tag, PCI_COMMAND_STATUS_REG, 3630 csr); 3631 break; 3632 default: 3633 aprint_error_dev(dev, "unexpected type on BAR0\n"); 3634 return ENXIO; 3635 } 3636 3637 return (0); 3638 } /* ixgbe_allocate_pci_resources */ 3639 3640 static void 3641 ixgbe_free_deferred_handlers(struct adapter *adapter) 3642 { 3643 struct ix_queue *que = adapter->queues; 3644 struct tx_ring *txr = adapter->tx_rings; 3645 int i; 3646 3647 for (i = 0; i < adapter->num_queues; i++, que++, txr++) { 3648 if (!(adapter->feat_en & IXGBE_FEATURE_LEGACY_TX)) { 3649 if (txr->txr_si != NULL) 3650 softint_disestablish(txr->txr_si); 3651 } 3652 if (que->que_si != NULL) 3653 softint_disestablish(que->que_si); 3654 } 3655 if (adapter->txr_wq != NULL) 3656 workqueue_destroy(adapter->txr_wq); 3657 if (adapter->txr_wq_enqueued != NULL) 3658 percpu_free(adapter->txr_wq_enqueued, sizeof(u_int)); 3659 if (adapter->que_wq != NULL) 3660 workqueue_destroy(adapter->que_wq); 3661 3662 if (adapter->admin_wq != NULL) { 3663 workqueue_destroy(adapter->admin_wq); 3664 adapter->admin_wq = NULL; 3665 } 3666 if (adapter->timer_wq != NULL) { 3667 workqueue_destroy(adapter->timer_wq); 3668 adapter->timer_wq = NULL; 3669 } 3670 if (adapter->recovery_mode_timer_wq != NULL) { 3671 /* 3672 * ixgbe_ifstop() doesn't call the workqueue_wait() for 3673 * the recovery_mode_timer workqueue, so call it here. 3674 */ 3675 workqueue_wait(adapter->recovery_mode_timer_wq, 3676 &adapter->recovery_mode_timer_wc); 3677 atomic_store_relaxed(&adapter->recovery_mode_timer_pending, 0); 3678 workqueue_destroy(adapter->recovery_mode_timer_wq); 3679 adapter->recovery_mode_timer_wq = NULL; 3680 } 3681 } /* ixgbe_free_deferred_handlers */ 3682 3683 /************************************************************************ 3684 * ixgbe_detach - Device removal routine 3685 * 3686 * Called when the driver is being removed. 3687 * Stops the adapter and deallocates all the resources 3688 * that were allocated for driver operation. 3689 * 3690 * return 0 on success, positive on failure 3691 ************************************************************************/ 3692 static int 3693 ixgbe_detach(device_t dev, int flags) 3694 { 3695 struct adapter *adapter = device_private(dev); 3696 struct rx_ring *rxr = adapter->rx_rings; 3697 struct tx_ring *txr = adapter->tx_rings; 3698 struct ixgbe_hw *hw = &adapter->hw; 3699 struct ixgbe_hw_stats *stats = &adapter->stats.pf; 3700 u32 ctrl_ext; 3701 int i; 3702 3703 INIT_DEBUGOUT("ixgbe_detach: begin"); 3704 if (adapter->osdep.attached == false) 3705 return 0; 3706 3707 if (ixgbe_pci_iov_detach(dev) != 0) { 3708 device_printf(dev, "SR-IOV in use; detach first.\n"); 3709 return (EBUSY); 3710 } 3711 3712 if (VLAN_ATTACHED(&adapter->osdep.ec) && 3713 (flags & (DETACH_SHUTDOWN | DETACH_FORCE)) == 0) { 3714 aprint_error_dev(dev, "VLANs in use, detach first\n"); 3715 return (EBUSY); 3716 } 3717 3718 ether_ifdetach(adapter->ifp); 3719 3720 adapter->osdep.detaching = true; 3721 /* 3722 * Stop the interface. ixgbe_setup_low_power_mode() calls 3723 * ixgbe_ifstop(), so it's not required to call ixgbe_ifstop() 3724 * directly. 3725 */ 3726 ixgbe_setup_low_power_mode(adapter); 3727 3728 callout_halt(&adapter->timer, NULL); 3729 if (adapter->feat_en & IXGBE_FEATURE_RECOVERY_MODE) 3730 callout_halt(&adapter->recovery_mode_timer, NULL); 3731 3732 workqueue_wait(adapter->admin_wq, &adapter->admin_wc); 3733 atomic_store_relaxed(&adapter->admin_pending, 0); 3734 workqueue_wait(adapter->timer_wq, &adapter->timer_wc); 3735 atomic_store_relaxed(&adapter->timer_pending, 0); 3736 3737 pmf_device_deregister(dev); 3738 3739 ixgbe_free_deferred_handlers(adapter); 3740 3741 /* let hardware know driver is unloading */ 3742 ctrl_ext = IXGBE_READ_REG(&adapter->hw, IXGBE_CTRL_EXT); 3743 ctrl_ext &= ~IXGBE_CTRL_EXT_DRV_LOAD; 3744 IXGBE_WRITE_REG(&adapter->hw, IXGBE_CTRL_EXT, ctrl_ext); 3745 3746 if (adapter->feat_en & IXGBE_FEATURE_NETMAP) 3747 netmap_detach(adapter->ifp); 3748 3749 ixgbe_free_pci_resources(adapter); 3750 #if 0 /* XXX the NetBSD port is probably missing something here */ 3751 bus_generic_detach(dev); 3752 #endif 3753 if_detach(adapter->ifp); 3754 ifmedia_fini(&adapter->media); 3755 if_percpuq_destroy(adapter->ipq); 3756 3757 sysctl_teardown(&adapter->sysctllog); 3758 evcnt_detach(&adapter->efbig_tx_dma_setup); 3759 evcnt_detach(&adapter->mbuf_defrag_failed); 3760 evcnt_detach(&adapter->efbig2_tx_dma_setup); 3761 evcnt_detach(&adapter->einval_tx_dma_setup); 3762 evcnt_detach(&adapter->other_tx_dma_setup); 3763 evcnt_detach(&adapter->eagain_tx_dma_setup); 3764 evcnt_detach(&adapter->enomem_tx_dma_setup); 3765 evcnt_detach(&adapter->watchdog_events); 3766 evcnt_detach(&adapter->tso_err); 3767 evcnt_detach(&adapter->admin_irqev); 3768 evcnt_detach(&adapter->link_workev); 3769 evcnt_detach(&adapter->mod_workev); 3770 evcnt_detach(&adapter->msf_workev); 3771 evcnt_detach(&adapter->phy_workev); 3772 3773 for (i = 0; i < IXGBE_TC_COUNTER_NUM; i++) { 3774 if (i < __arraycount(stats->mpc)) { 3775 evcnt_detach(&stats->mpc[i]); 3776 if (hw->mac.type == ixgbe_mac_82598EB) 3777 evcnt_detach(&stats->rnbc[i]); 3778 } 3779 if (i < __arraycount(stats->pxontxc)) { 3780 evcnt_detach(&stats->pxontxc[i]); 3781 evcnt_detach(&stats->pxonrxc[i]); 3782 evcnt_detach(&stats->pxofftxc[i]); 3783 evcnt_detach(&stats->pxoffrxc[i]); 3784 if (hw->mac.type >= ixgbe_mac_82599EB) 3785 evcnt_detach(&stats->pxon2offc[i]); 3786 } 3787 } 3788 3789 txr = adapter->tx_rings; 3790 for (i = 0; i < adapter->num_queues; i++, rxr++, txr++) { 3791 evcnt_detach(&adapter->queues[i].irqs); 3792 evcnt_detach(&adapter->queues[i].handleq); 3793 evcnt_detach(&adapter->queues[i].req); 3794 evcnt_detach(&txr->total_packets); 3795 #ifndef IXGBE_LEGACY_TX 3796 evcnt_detach(&txr->pcq_drops); 3797 #endif 3798 evcnt_detach(&txr->no_desc_avail); 3799 evcnt_detach(&txr->tso_tx); 3800 3801 if (i < __arraycount(stats->qprc)) { 3802 evcnt_detach(&stats->qprc[i]); 3803 evcnt_detach(&stats->qptc[i]); 3804 evcnt_detach(&stats->qbrc[i]); 3805 evcnt_detach(&stats->qbtc[i]); 3806 if (hw->mac.type >= ixgbe_mac_82599EB) 3807 evcnt_detach(&stats->qprdc[i]); 3808 } 3809 3810 evcnt_detach(&rxr->rx_packets); 3811 evcnt_detach(&rxr->rx_bytes); 3812 evcnt_detach(&rxr->rx_copies); 3813 evcnt_detach(&rxr->no_mbuf); 3814 evcnt_detach(&rxr->rx_discarded); 3815 } 3816 evcnt_detach(&stats->ipcs); 3817 evcnt_detach(&stats->l4cs); 3818 evcnt_detach(&stats->ipcs_bad); 3819 evcnt_detach(&stats->l4cs_bad); 3820 evcnt_detach(&stats->intzero); 3821 evcnt_detach(&stats->legint); 3822 evcnt_detach(&stats->crcerrs); 3823 evcnt_detach(&stats->illerrc); 3824 evcnt_detach(&stats->errbc); 3825 evcnt_detach(&stats->mspdc); 3826 if (hw->mac.type >= ixgbe_mac_X550) 3827 evcnt_detach(&stats->mbsdc); 3828 evcnt_detach(&stats->mpctotal); 3829 evcnt_detach(&stats->mlfc); 3830 evcnt_detach(&stats->mrfc); 3831 if (hw->mac.type == ixgbe_mac_X550EM_a) 3832 evcnt_detach(&stats->link_dn_cnt); 3833 evcnt_detach(&stats->rlec); 3834 evcnt_detach(&stats->lxontxc); 3835 evcnt_detach(&stats->lxonrxc); 3836 evcnt_detach(&stats->lxofftxc); 3837 evcnt_detach(&stats->lxoffrxc); 3838 3839 /* Packet Reception Stats */ 3840 evcnt_detach(&stats->tor); 3841 evcnt_detach(&stats->gorc); 3842 evcnt_detach(&stats->tpr); 3843 evcnt_detach(&stats->gprc); 3844 evcnt_detach(&stats->mprc); 3845 evcnt_detach(&stats->bprc); 3846 evcnt_detach(&stats->prc64); 3847 evcnt_detach(&stats->prc127); 3848 evcnt_detach(&stats->prc255); 3849 evcnt_detach(&stats->prc511); 3850 evcnt_detach(&stats->prc1023); 3851 evcnt_detach(&stats->prc1522); 3852 evcnt_detach(&stats->ruc); 3853 evcnt_detach(&stats->rfc); 3854 evcnt_detach(&stats->roc); 3855 evcnt_detach(&stats->rjc); 3856 evcnt_detach(&stats->mngprc); 3857 evcnt_detach(&stats->mngpdc); 3858 evcnt_detach(&stats->xec); 3859 3860 /* Packet Transmission Stats */ 3861 evcnt_detach(&stats->gotc); 3862 evcnt_detach(&stats->tpt); 3863 evcnt_detach(&stats->gptc); 3864 evcnt_detach(&stats->bptc); 3865 evcnt_detach(&stats->mptc); 3866 evcnt_detach(&stats->mngptc); 3867 evcnt_detach(&stats->ptc64); 3868 evcnt_detach(&stats->ptc127); 3869 evcnt_detach(&stats->ptc255); 3870 evcnt_detach(&stats->ptc511); 3871 evcnt_detach(&stats->ptc1023); 3872 evcnt_detach(&stats->ptc1522); 3873 3874 ixgbe_free_queues(adapter); 3875 free(adapter->mta, M_DEVBUF); 3876 3877 mutex_destroy(&adapter->admin_mtx); /* XXX appropriate order? */ 3878 IXGBE_CORE_LOCK_DESTROY(adapter); 3879 3880 return (0); 3881 } /* ixgbe_detach */ 3882 3883 /************************************************************************ 3884 * ixgbe_setup_low_power_mode - LPLU/WoL preparation 3885 * 3886 * Prepare the adapter/port for LPLU and/or WoL 3887 ************************************************************************/ 3888 static int 3889 ixgbe_setup_low_power_mode(struct adapter *adapter) 3890 { 3891 struct ixgbe_hw *hw = &adapter->hw; 3892 device_t dev = adapter->dev; 3893 struct ifnet *ifp = adapter->ifp; 3894 s32 error = 0; 3895 3896 /* Limit power management flow to X550EM baseT */ 3897 if (hw->device_id == IXGBE_DEV_ID_X550EM_X_10G_T && 3898 hw->phy.ops.enter_lplu) { 3899 /* X550EM baseT adapters need a special LPLU flow */ 3900 hw->phy.reset_disable = true; 3901 ixgbe_ifstop(ifp, 1); 3902 error = hw->phy.ops.enter_lplu(hw); 3903 if (error) 3904 device_printf(dev, 3905 "Error entering LPLU: %d\n", error); 3906 hw->phy.reset_disable = false; 3907 } else { 3908 /* Just stop for other adapters */ 3909 ixgbe_ifstop(ifp, 1); 3910 } 3911 3912 IXGBE_CORE_LOCK(adapter); 3913 3914 if (!hw->wol_enabled) { 3915 ixgbe_set_phy_power(hw, FALSE); 3916 IXGBE_WRITE_REG(hw, IXGBE_WUFC, 0); 3917 IXGBE_WRITE_REG(hw, IXGBE_WUC, 0); 3918 } else { 3919 /* Turn off support for APM wakeup. (Using ACPI instead) */ 3920 IXGBE_WRITE_REG(hw, IXGBE_GRC_BY_MAC(hw), 3921 IXGBE_READ_REG(hw, IXGBE_GRC_BY_MAC(hw)) & ~(u32)2); 3922 3923 /* 3924 * Clear Wake Up Status register to prevent any previous wakeup 3925 * events from waking us up immediately after we suspend. 3926 */ 3927 IXGBE_WRITE_REG(hw, IXGBE_WUS, 0xffffffff); 3928 3929 /* 3930 * Program the Wakeup Filter Control register with user filter 3931 * settings 3932 */ 3933 IXGBE_WRITE_REG(hw, IXGBE_WUFC, adapter->wufc); 3934 3935 /* Enable wakeups and power management in Wakeup Control */ 3936 IXGBE_WRITE_REG(hw, IXGBE_WUC, 3937 IXGBE_WUC_WKEN | IXGBE_WUC_PME_EN); 3938 } 3939 3940 IXGBE_CORE_UNLOCK(adapter); 3941 3942 return error; 3943 } /* ixgbe_setup_low_power_mode */ 3944 3945 /************************************************************************ 3946 * ixgbe_shutdown - Shutdown entry point 3947 ************************************************************************/ 3948 #if 0 /* XXX NetBSD ought to register something like this through pmf(9) */ 3949 static int 3950 ixgbe_shutdown(device_t dev) 3951 { 3952 struct adapter *adapter = device_private(dev); 3953 int error = 0; 3954 3955 INIT_DEBUGOUT("ixgbe_shutdown: begin"); 3956 3957 error = ixgbe_setup_low_power_mode(adapter); 3958 3959 return (error); 3960 } /* ixgbe_shutdown */ 3961 #endif 3962 3963 /************************************************************************ 3964 * ixgbe_suspend 3965 * 3966 * From D0 to D3 3967 ************************************************************************/ 3968 static bool 3969 ixgbe_suspend(device_t dev, const pmf_qual_t *qual) 3970 { 3971 struct adapter *adapter = device_private(dev); 3972 int error = 0; 3973 3974 INIT_DEBUGOUT("ixgbe_suspend: begin"); 3975 3976 error = ixgbe_setup_low_power_mode(adapter); 3977 3978 return (error); 3979 } /* ixgbe_suspend */ 3980 3981 /************************************************************************ 3982 * ixgbe_resume 3983 * 3984 * From D3 to D0 3985 ************************************************************************/ 3986 static bool 3987 ixgbe_resume(device_t dev, const pmf_qual_t *qual) 3988 { 3989 struct adapter *adapter = device_private(dev); 3990 struct ifnet *ifp = adapter->ifp; 3991 struct ixgbe_hw *hw = &adapter->hw; 3992 u32 wus; 3993 3994 INIT_DEBUGOUT("ixgbe_resume: begin"); 3995 3996 IXGBE_CORE_LOCK(adapter); 3997 3998 /* Read & clear WUS register */ 3999 wus = IXGBE_READ_REG(hw, IXGBE_WUS); 4000 if (wus) 4001 device_printf(dev, "Woken up by (WUS): %#010x\n", 4002 IXGBE_READ_REG(hw, IXGBE_WUS)); 4003 IXGBE_WRITE_REG(hw, IXGBE_WUS, 0xffffffff); 4004 /* And clear WUFC until next low-power transition */ 4005 IXGBE_WRITE_REG(hw, IXGBE_WUFC, 0); 4006 4007 /* 4008 * Required after D3->D0 transition; 4009 * will re-advertise all previous advertised speeds 4010 */ 4011 if (ifp->if_flags & IFF_UP) 4012 ixgbe_init_locked(adapter); 4013 4014 IXGBE_CORE_UNLOCK(adapter); 4015 4016 return true; 4017 } /* ixgbe_resume */ 4018 4019 /* 4020 * Set the various hardware offload abilities. 4021 * 4022 * This takes the ifnet's if_capenable flags (e.g. set by the user using 4023 * ifconfig) and indicates to the OS via the ifnet's if_hwassist field what 4024 * mbuf offload flags the driver will understand. 4025 */ 4026 static void 4027 ixgbe_set_if_hwassist(struct adapter *adapter) 4028 { 4029 /* XXX */ 4030 } 4031 4032 /************************************************************************ 4033 * ixgbe_init_locked - Init entry point 4034 * 4035 * Used in two ways: It is used by the stack as an init 4036 * entry point in network interface structure. It is also 4037 * used by the driver as a hw/sw initialization routine to 4038 * get to a consistent state. 4039 * 4040 * return 0 on success, positive on failure 4041 ************************************************************************/ 4042 static void 4043 ixgbe_init_locked(struct adapter *adapter) 4044 { 4045 struct ifnet *ifp = adapter->ifp; 4046 device_t dev = adapter->dev; 4047 struct ixgbe_hw *hw = &adapter->hw; 4048 struct ix_queue *que; 4049 struct tx_ring *txr; 4050 struct rx_ring *rxr; 4051 u32 txdctl, mhadd; 4052 u32 rxdctl, rxctrl; 4053 u32 ctrl_ext; 4054 bool unsupported_sfp = false; 4055 int i, j, error; 4056 4057 /* XXX check IFF_UP and IFF_RUNNING, power-saving state! */ 4058 4059 KASSERT(mutex_owned(&adapter->core_mtx)); 4060 INIT_DEBUGOUT("ixgbe_init_locked: begin"); 4061 4062 hw->need_unsupported_sfp_recovery = false; 4063 hw->adapter_stopped = FALSE; 4064 ixgbe_stop_adapter(hw); 4065 callout_stop(&adapter->timer); 4066 if (adapter->feat_en & IXGBE_FEATURE_RECOVERY_MODE) 4067 callout_stop(&adapter->recovery_mode_timer); 4068 for (i = 0, que = adapter->queues; i < adapter->num_queues; i++, que++) 4069 que->disabled_count = 0; 4070 4071 /* XXX I moved this here from the SIOCSIFMTU case in ixgbe_ioctl(). */ 4072 adapter->max_frame_size = 4073 ifp->if_mtu + ETHER_HDR_LEN + ETHER_CRC_LEN; 4074 4075 /* Queue indices may change with IOV mode */ 4076 ixgbe_align_all_queue_indices(adapter); 4077 4078 /* reprogram the RAR[0] in case user changed it. */ 4079 ixgbe_set_rar(hw, 0, hw->mac.addr, adapter->pool, IXGBE_RAH_AV); 4080 4081 /* Get the latest mac address, User can use a LAA */ 4082 memcpy(hw->mac.addr, CLLADDR(ifp->if_sadl), 4083 IXGBE_ETH_LENGTH_OF_ADDRESS); 4084 ixgbe_set_rar(hw, 0, hw->mac.addr, adapter->pool, 1); 4085 hw->addr_ctrl.rar_used_count = 1; 4086 4087 /* Set hardware offload abilities from ifnet flags */ 4088 ixgbe_set_if_hwassist(adapter); 4089 4090 /* Prepare transmit descriptors and buffers */ 4091 if (ixgbe_setup_transmit_structures(adapter)) { 4092 device_printf(dev, "Could not setup transmit structures\n"); 4093 ixgbe_stop_locked(adapter); 4094 return; 4095 } 4096 4097 ixgbe_init_hw(hw); 4098 4099 ixgbe_initialize_iov(adapter); 4100 4101 ixgbe_initialize_transmit_units(adapter); 4102 4103 /* Setup Multicast table */ 4104 ixgbe_set_rxfilter(adapter); 4105 4106 /* Use fixed buffer size, even for jumbo frames */ 4107 adapter->rx_mbuf_sz = MCLBYTES; 4108 4109 /* Prepare receive descriptors and buffers */ 4110 error = ixgbe_setup_receive_structures(adapter); 4111 if (error) { 4112 device_printf(dev, 4113 "Could not setup receive structures (err = %d)\n", error); 4114 ixgbe_stop_locked(adapter); 4115 return; 4116 } 4117 4118 /* Configure RX settings */ 4119 ixgbe_initialize_receive_units(adapter); 4120 4121 /* Initialize variable holding task enqueue requests interrupts */ 4122 adapter->task_requests = 0; 4123 4124 /* Enable SDP & MSI-X interrupts based on adapter */ 4125 ixgbe_config_gpie(adapter); 4126 4127 /* Set MTU size */ 4128 if (ifp->if_mtu > ETHERMTU) { 4129 /* aka IXGBE_MAXFRS on 82599 and newer */ 4130 mhadd = IXGBE_READ_REG(hw, IXGBE_MHADD); 4131 mhadd &= ~IXGBE_MHADD_MFS_MASK; 4132 mhadd |= adapter->max_frame_size << IXGBE_MHADD_MFS_SHIFT; 4133 IXGBE_WRITE_REG(hw, IXGBE_MHADD, mhadd); 4134 } 4135 4136 /* Now enable all the queues */ 4137 for (i = 0; i < adapter->num_queues; i++) { 4138 txr = &adapter->tx_rings[i]; 4139 txdctl = IXGBE_READ_REG(hw, IXGBE_TXDCTL(txr->me)); 4140 txdctl |= IXGBE_TXDCTL_ENABLE; 4141 /* Set WTHRESH to 8, burst writeback */ 4142 txdctl |= IXGBE_TX_WTHRESH << IXGBE_TXDCTL_WTHRESH_SHIFT; 4143 /* 4144 * When the internal queue falls below PTHRESH (32), 4145 * start prefetching as long as there are at least 4146 * HTHRESH (1) buffers ready. The values are taken 4147 * from the Intel linux driver 3.8.21. 4148 * Prefetching enables tx line rate even with 1 queue. 4149 */ 4150 txdctl |= (32 << 0) | (1 << 8); 4151 IXGBE_WRITE_REG(hw, IXGBE_TXDCTL(txr->me), txdctl); 4152 } 4153 4154 for (i = 0; i < adapter->num_queues; i++) { 4155 rxr = &adapter->rx_rings[i]; 4156 rxdctl = IXGBE_READ_REG(hw, IXGBE_RXDCTL(rxr->me)); 4157 if (hw->mac.type == ixgbe_mac_82598EB) { 4158 /* 4159 * PTHRESH = 21 4160 * HTHRESH = 4 4161 * WTHRESH = 8 4162 */ 4163 rxdctl &= ~0x3FFFFF; 4164 rxdctl |= 0x080420; 4165 } 4166 rxdctl |= IXGBE_RXDCTL_ENABLE; 4167 IXGBE_WRITE_REG(hw, IXGBE_RXDCTL(rxr->me), rxdctl); 4168 for (j = 0; j < 10; j++) { 4169 if (IXGBE_READ_REG(hw, IXGBE_RXDCTL(rxr->me)) & 4170 IXGBE_RXDCTL_ENABLE) 4171 break; 4172 else 4173 msec_delay(1); 4174 } 4175 IXGBE_WRITE_BARRIER(hw); 4176 4177 /* 4178 * In netmap mode, we must preserve the buffers made 4179 * available to userspace before the if_init() 4180 * (this is true by default on the TX side, because 4181 * init makes all buffers available to userspace). 4182 * 4183 * netmap_reset() and the device specific routines 4184 * (e.g. ixgbe_setup_receive_rings()) map these 4185 * buffers at the end of the NIC ring, so here we 4186 * must set the RDT (tail) register to make sure 4187 * they are not overwritten. 4188 * 4189 * In this driver the NIC ring starts at RDH = 0, 4190 * RDT points to the last slot available for reception (?), 4191 * so RDT = num_rx_desc - 1 means the whole ring is available. 4192 */ 4193 #ifdef DEV_NETMAP 4194 if ((adapter->feat_en & IXGBE_FEATURE_NETMAP) && 4195 (ifp->if_capenable & IFCAP_NETMAP)) { 4196 struct netmap_adapter *na = NA(adapter->ifp); 4197 struct netmap_kring *kring = na->rx_rings[i]; 4198 int t = na->num_rx_desc - 1 - nm_kr_rxspace(kring); 4199 4200 IXGBE_WRITE_REG(hw, IXGBE_RDT(rxr->me), t); 4201 } else 4202 #endif /* DEV_NETMAP */ 4203 IXGBE_WRITE_REG(hw, IXGBE_RDT(rxr->me), 4204 adapter->num_rx_desc - 1); 4205 } 4206 4207 /* Enable Receive engine */ 4208 rxctrl = IXGBE_READ_REG(hw, IXGBE_RXCTRL); 4209 if (hw->mac.type == ixgbe_mac_82598EB) 4210 rxctrl |= IXGBE_RXCTRL_DMBYPS; 4211 rxctrl |= IXGBE_RXCTRL_RXEN; 4212 ixgbe_enable_rx_dma(hw, rxctrl); 4213 4214 callout_reset(&adapter->timer, hz, ixgbe_local_timer, adapter); 4215 atomic_store_relaxed(&adapter->timer_pending, 0); 4216 if (adapter->feat_en & IXGBE_FEATURE_RECOVERY_MODE) 4217 callout_reset(&adapter->recovery_mode_timer, hz, 4218 ixgbe_recovery_mode_timer, adapter); 4219 4220 /* Set up MSI/MSI-X routing */ 4221 if (adapter->feat_en & IXGBE_FEATURE_MSIX) { 4222 ixgbe_configure_ivars(adapter); 4223 /* Set up auto-mask */ 4224 if (hw->mac.type == ixgbe_mac_82598EB) 4225 IXGBE_WRITE_REG(hw, IXGBE_EIAM, IXGBE_EICS_RTX_QUEUE); 4226 else { 4227 IXGBE_WRITE_REG(hw, IXGBE_EIAM_EX(0), 0xFFFFFFFF); 4228 IXGBE_WRITE_REG(hw, IXGBE_EIAM_EX(1), 0xFFFFFFFF); 4229 } 4230 } else { /* Simple settings for Legacy/MSI */ 4231 ixgbe_set_ivar(adapter, 0, 0, 0); 4232 ixgbe_set_ivar(adapter, 0, 0, 1); 4233 IXGBE_WRITE_REG(hw, IXGBE_EIAM, IXGBE_EICS_RTX_QUEUE); 4234 } 4235 4236 ixgbe_init_fdir(adapter); 4237 4238 /* 4239 * Check on any SFP devices that 4240 * need to be kick-started 4241 */ 4242 if (hw->phy.type == ixgbe_phy_none) { 4243 error = hw->phy.ops.identify(hw); 4244 if (error == IXGBE_ERR_SFP_NOT_SUPPORTED) 4245 unsupported_sfp = true; 4246 } else if (hw->phy.type == ixgbe_phy_sfp_unsupported) 4247 unsupported_sfp = true; 4248 4249 if (unsupported_sfp) 4250 device_printf(dev, 4251 "Unsupported SFP+ module type was detected.\n"); 4252 4253 /* Set moderation on the Link interrupt */ 4254 ixgbe_eitr_write(adapter, adapter->vector, IXGBE_LINK_ITR); 4255 4256 /* Enable EEE power saving */ 4257 if (adapter->feat_cap & IXGBE_FEATURE_EEE) 4258 hw->mac.ops.setup_eee(hw, 4259 adapter->feat_en & IXGBE_FEATURE_EEE); 4260 4261 /* Enable power to the phy. */ 4262 if (!unsupported_sfp) { 4263 ixgbe_set_phy_power(hw, TRUE); 4264 4265 /* Config/Enable Link */ 4266 ixgbe_config_link(adapter); 4267 } 4268 4269 /* Hardware Packet Buffer & Flow Control setup */ 4270 ixgbe_config_delay_values(adapter); 4271 4272 /* Initialize the FC settings */ 4273 ixgbe_start_hw(hw); 4274 4275 /* Set up VLAN support and filter */ 4276 ixgbe_setup_vlan_hw_support(adapter); 4277 4278 /* Setup DMA Coalescing */ 4279 ixgbe_config_dmac(adapter); 4280 4281 /* OK to schedule workqueues. */ 4282 adapter->schedule_wqs_ok = true; 4283 4284 /* And now turn on interrupts */ 4285 ixgbe_enable_intr(adapter); 4286 4287 /* Enable the use of the MBX by the VF's */ 4288 if (adapter->feat_en & IXGBE_FEATURE_SRIOV) { 4289 ctrl_ext = IXGBE_READ_REG(hw, IXGBE_CTRL_EXT); 4290 ctrl_ext |= IXGBE_CTRL_EXT_PFRSTD; 4291 IXGBE_WRITE_REG(hw, IXGBE_CTRL_EXT, ctrl_ext); 4292 } 4293 4294 /* Update saved flags. See ixgbe_ifflags_cb() */ 4295 adapter->if_flags = ifp->if_flags; 4296 adapter->ec_capenable = adapter->osdep.ec.ec_capenable; 4297 4298 /* Now inform the stack we're ready */ 4299 ifp->if_flags |= IFF_RUNNING; 4300 4301 return; 4302 } /* ixgbe_init_locked */ 4303 4304 /************************************************************************ 4305 * ixgbe_init 4306 ************************************************************************/ 4307 static int 4308 ixgbe_init(struct ifnet *ifp) 4309 { 4310 struct adapter *adapter = ifp->if_softc; 4311 4312 IXGBE_CORE_LOCK(adapter); 4313 ixgbe_init_locked(adapter); 4314 IXGBE_CORE_UNLOCK(adapter); 4315 4316 return 0; /* XXX ixgbe_init_locked cannot fail? really? */ 4317 } /* ixgbe_init */ 4318 4319 /************************************************************************ 4320 * ixgbe_set_ivar 4321 * 4322 * Setup the correct IVAR register for a particular MSI-X interrupt 4323 * (yes this is all very magic and confusing :) 4324 * - entry is the register array entry 4325 * - vector is the MSI-X vector for this queue 4326 * - type is RX/TX/MISC 4327 ************************************************************************/ 4328 static void 4329 ixgbe_set_ivar(struct adapter *adapter, u8 entry, u8 vector, s8 type) 4330 { 4331 struct ixgbe_hw *hw = &adapter->hw; 4332 u32 ivar, index; 4333 4334 vector |= IXGBE_IVAR_ALLOC_VAL; 4335 4336 switch (hw->mac.type) { 4337 case ixgbe_mac_82598EB: 4338 if (type == -1) 4339 entry = IXGBE_IVAR_OTHER_CAUSES_INDEX; 4340 else 4341 entry += (type * 64); 4342 index = (entry >> 2) & 0x1F; 4343 ivar = IXGBE_READ_REG(hw, IXGBE_IVAR(index)); 4344 ivar &= ~(0xffUL << (8 * (entry & 0x3))); 4345 ivar |= ((u32)vector << (8 * (entry & 0x3))); 4346 IXGBE_WRITE_REG(&adapter->hw, IXGBE_IVAR(index), ivar); 4347 break; 4348 case ixgbe_mac_82599EB: 4349 case ixgbe_mac_X540: 4350 case ixgbe_mac_X550: 4351 case ixgbe_mac_X550EM_x: 4352 case ixgbe_mac_X550EM_a: 4353 if (type == -1) { /* MISC IVAR */ 4354 index = (entry & 1) * 8; 4355 ivar = IXGBE_READ_REG(hw, IXGBE_IVAR_MISC); 4356 ivar &= ~(0xffUL << index); 4357 ivar |= ((u32)vector << index); 4358 IXGBE_WRITE_REG(hw, IXGBE_IVAR_MISC, ivar); 4359 } else { /* RX/TX IVARS */ 4360 index = (16 * (entry & 1)) + (8 * type); 4361 ivar = IXGBE_READ_REG(hw, IXGBE_IVAR(entry >> 1)); 4362 ivar &= ~(0xffUL << index); 4363 ivar |= ((u32)vector << index); 4364 IXGBE_WRITE_REG(hw, IXGBE_IVAR(entry >> 1), ivar); 4365 } 4366 break; 4367 default: 4368 break; 4369 } 4370 } /* ixgbe_set_ivar */ 4371 4372 /************************************************************************ 4373 * ixgbe_configure_ivars 4374 ************************************************************************/ 4375 static void 4376 ixgbe_configure_ivars(struct adapter *adapter) 4377 { 4378 struct ix_queue *que = adapter->queues; 4379 u32 newitr; 4380 4381 if (ixgbe_max_interrupt_rate > 0) 4382 newitr = (4000000 / ixgbe_max_interrupt_rate) & 0x0FF8; 4383 else { 4384 /* 4385 * Disable DMA coalescing if interrupt moderation is 4386 * disabled. 4387 */ 4388 adapter->dmac = 0; 4389 newitr = 0; 4390 } 4391 4392 for (int i = 0; i < adapter->num_queues; i++, que++) { 4393 struct rx_ring *rxr = &adapter->rx_rings[i]; 4394 struct tx_ring *txr = &adapter->tx_rings[i]; 4395 /* First the RX queue entry */ 4396 ixgbe_set_ivar(adapter, rxr->me, que->msix, 0); 4397 /* ... and the TX */ 4398 ixgbe_set_ivar(adapter, txr->me, que->msix, 1); 4399 /* Set an Initial EITR value */ 4400 ixgbe_eitr_write(adapter, que->msix, newitr); 4401 /* 4402 * To eliminate influence of the previous state. 4403 * At this point, Tx/Rx interrupt handler 4404 * (ixgbe_msix_que()) cannot be called, so both 4405 * IXGBE_TX_LOCK and IXGBE_RX_LOCK are not required. 4406 */ 4407 que->eitr_setting = 0; 4408 } 4409 4410 /* For the Link interrupt */ 4411 ixgbe_set_ivar(adapter, 1, adapter->vector, -1); 4412 } /* ixgbe_configure_ivars */ 4413 4414 /************************************************************************ 4415 * ixgbe_config_gpie 4416 ************************************************************************/ 4417 static void 4418 ixgbe_config_gpie(struct adapter *adapter) 4419 { 4420 struct ixgbe_hw *hw = &adapter->hw; 4421 u32 gpie; 4422 4423 gpie = IXGBE_READ_REG(hw, IXGBE_GPIE); 4424 4425 if (adapter->feat_en & IXGBE_FEATURE_MSIX) { 4426 /* Enable Enhanced MSI-X mode */ 4427 gpie |= IXGBE_GPIE_MSIX_MODE 4428 | IXGBE_GPIE_EIAME 4429 | IXGBE_GPIE_PBA_SUPPORT 4430 | IXGBE_GPIE_OCD; 4431 } 4432 4433 /* Fan Failure Interrupt */ 4434 if (adapter->feat_en & IXGBE_FEATURE_FAN_FAIL) 4435 gpie |= IXGBE_SDP1_GPIEN; 4436 4437 /* Thermal Sensor Interrupt */ 4438 if (adapter->feat_en & IXGBE_FEATURE_TEMP_SENSOR) 4439 gpie |= IXGBE_SDP0_GPIEN_X540; 4440 4441 /* Link detection */ 4442 switch (hw->mac.type) { 4443 case ixgbe_mac_82599EB: 4444 gpie |= IXGBE_SDP1_GPIEN | IXGBE_SDP2_GPIEN; 4445 break; 4446 case ixgbe_mac_X550EM_x: 4447 case ixgbe_mac_X550EM_a: 4448 gpie |= IXGBE_SDP0_GPIEN_X540; 4449 break; 4450 default: 4451 break; 4452 } 4453 4454 IXGBE_WRITE_REG(hw, IXGBE_GPIE, gpie); 4455 4456 } /* ixgbe_config_gpie */ 4457 4458 /************************************************************************ 4459 * ixgbe_config_delay_values 4460 * 4461 * Requires adapter->max_frame_size to be set. 4462 ************************************************************************/ 4463 static void 4464 ixgbe_config_delay_values(struct adapter *adapter) 4465 { 4466 struct ixgbe_hw *hw = &adapter->hw; 4467 u32 rxpb, frame, size, tmp; 4468 4469 frame = adapter->max_frame_size; 4470 4471 /* Calculate High Water */ 4472 switch (hw->mac.type) { 4473 case ixgbe_mac_X540: 4474 case ixgbe_mac_X550: 4475 case ixgbe_mac_X550EM_x: 4476 case ixgbe_mac_X550EM_a: 4477 tmp = IXGBE_DV_X540(frame, frame); 4478 break; 4479 default: 4480 tmp = IXGBE_DV(frame, frame); 4481 break; 4482 } 4483 size = IXGBE_BT2KB(tmp); 4484 rxpb = IXGBE_READ_REG(hw, IXGBE_RXPBSIZE(0)) >> 10; 4485 hw->fc.high_water[0] = rxpb - size; 4486 4487 /* Now calculate Low Water */ 4488 switch (hw->mac.type) { 4489 case ixgbe_mac_X540: 4490 case ixgbe_mac_X550: 4491 case ixgbe_mac_X550EM_x: 4492 case ixgbe_mac_X550EM_a: 4493 tmp = IXGBE_LOW_DV_X540(frame); 4494 break; 4495 default: 4496 tmp = IXGBE_LOW_DV(frame); 4497 break; 4498 } 4499 hw->fc.low_water[0] = IXGBE_BT2KB(tmp); 4500 4501 hw->fc.pause_time = IXGBE_FC_PAUSE; 4502 hw->fc.send_xon = TRUE; 4503 } /* ixgbe_config_delay_values */ 4504 4505 /************************************************************************ 4506 * ixgbe_set_rxfilter - Multicast Update 4507 * 4508 * Called whenever multicast address list is updated. 4509 ************************************************************************/ 4510 static void 4511 ixgbe_set_rxfilter(struct adapter *adapter) 4512 { 4513 struct ixgbe_mc_addr *mta; 4514 struct ifnet *ifp = adapter->ifp; 4515 u8 *update_ptr; 4516 int mcnt = 0; 4517 u32 fctrl; 4518 struct ethercom *ec = &adapter->osdep.ec; 4519 struct ether_multi *enm; 4520 struct ether_multistep step; 4521 4522 KASSERT(mutex_owned(&adapter->core_mtx)); 4523 IOCTL_DEBUGOUT("ixgbe_set_rxfilter: begin"); 4524 4525 mta = adapter->mta; 4526 bzero(mta, sizeof(*mta) * MAX_NUM_MULTICAST_ADDRESSES); 4527 4528 ETHER_LOCK(ec); 4529 ec->ec_flags &= ~ETHER_F_ALLMULTI; 4530 ETHER_FIRST_MULTI(step, ec, enm); 4531 while (enm != NULL) { 4532 if ((mcnt == MAX_NUM_MULTICAST_ADDRESSES) || 4533 (memcmp(enm->enm_addrlo, enm->enm_addrhi, 4534 ETHER_ADDR_LEN) != 0)) { 4535 ec->ec_flags |= ETHER_F_ALLMULTI; 4536 break; 4537 } 4538 bcopy(enm->enm_addrlo, 4539 mta[mcnt].addr, IXGBE_ETH_LENGTH_OF_ADDRESS); 4540 mta[mcnt].vmdq = adapter->pool; 4541 mcnt++; 4542 ETHER_NEXT_MULTI(step, enm); 4543 } 4544 4545 fctrl = IXGBE_READ_REG(&adapter->hw, IXGBE_FCTRL); 4546 if (ifp->if_flags & IFF_PROMISC) 4547 fctrl |= (IXGBE_FCTRL_UPE | IXGBE_FCTRL_MPE); 4548 else if (ec->ec_flags & ETHER_F_ALLMULTI) { 4549 fctrl |= IXGBE_FCTRL_MPE; 4550 fctrl &= ~IXGBE_FCTRL_UPE; 4551 } else 4552 fctrl &= ~(IXGBE_FCTRL_UPE | IXGBE_FCTRL_MPE); 4553 4554 IXGBE_WRITE_REG(&adapter->hw, IXGBE_FCTRL, fctrl); 4555 4556 /* Update multicast filter entries only when it's not ALLMULTI */ 4557 if ((ec->ec_flags & ETHER_F_ALLMULTI) == 0) { 4558 ETHER_UNLOCK(ec); 4559 update_ptr = (u8 *)mta; 4560 ixgbe_update_mc_addr_list(&adapter->hw, update_ptr, mcnt, 4561 ixgbe_mc_array_itr, TRUE); 4562 } else 4563 ETHER_UNLOCK(ec); 4564 } /* ixgbe_set_rxfilter */ 4565 4566 /************************************************************************ 4567 * ixgbe_mc_array_itr 4568 * 4569 * An iterator function needed by the multicast shared code. 4570 * It feeds the shared code routine the addresses in the 4571 * array of ixgbe_set_rxfilter() one by one. 4572 ************************************************************************/ 4573 static u8 * 4574 ixgbe_mc_array_itr(struct ixgbe_hw *hw, u8 **update_ptr, u32 *vmdq) 4575 { 4576 struct ixgbe_mc_addr *mta; 4577 4578 mta = (struct ixgbe_mc_addr *)*update_ptr; 4579 *vmdq = mta->vmdq; 4580 4581 *update_ptr = (u8*)(mta + 1); 4582 4583 return (mta->addr); 4584 } /* ixgbe_mc_array_itr */ 4585 4586 /************************************************************************ 4587 * ixgbe_local_timer - Timer routine 4588 * 4589 * Checks for link status, updates statistics, 4590 * and runs the watchdog check. 4591 ************************************************************************/ 4592 static void 4593 ixgbe_local_timer(void *arg) 4594 { 4595 struct adapter *adapter = arg; 4596 4597 if (adapter->schedule_wqs_ok) { 4598 if (atomic_cas_uint(&adapter->timer_pending, 0, 1) == 0) 4599 workqueue_enqueue(adapter->timer_wq, 4600 &adapter->timer_wc, NULL); 4601 } 4602 } 4603 4604 static void 4605 ixgbe_handle_timer(struct work *wk, void *context) 4606 { 4607 struct adapter *adapter = context; 4608 struct ixgbe_hw *hw = &adapter->hw; 4609 device_t dev = adapter->dev; 4610 struct ix_queue *que = adapter->queues; 4611 u64 queues = 0; 4612 u64 v0, v1, v2, v3, v4, v5, v6, v7; 4613 int hung = 0; 4614 int i; 4615 4616 IXGBE_CORE_LOCK(adapter); 4617 4618 /* Check for pluggable optics */ 4619 if (ixgbe_is_sfp(hw)) { 4620 bool sched_mod_task = false; 4621 4622 if (hw->mac.type == ixgbe_mac_82598EB) { 4623 /* 4624 * On 82598EB, SFP+'s MOD_ABS pin is not connected to 4625 * any GPIO(SDP). So just schedule TASK_MOD. 4626 */ 4627 sched_mod_task = true; 4628 } else { 4629 bool was_full, is_full; 4630 4631 was_full = 4632 hw->phy.sfp_type != ixgbe_sfp_type_not_present; 4633 is_full = ixgbe_sfp_cage_full(hw); 4634 4635 /* Do probe if cage state changed */ 4636 if (was_full ^ is_full) 4637 sched_mod_task = true; 4638 } 4639 if (sched_mod_task) { 4640 mutex_enter(&adapter->admin_mtx); 4641 adapter->task_requests |= IXGBE_REQUEST_TASK_MOD_WOI; 4642 ixgbe_schedule_admin_tasklet(adapter); 4643 mutex_exit(&adapter->admin_mtx); 4644 } 4645 } 4646 4647 ixgbe_update_link_status(adapter); 4648 ixgbe_update_stats_counters(adapter); 4649 4650 /* Update some event counters */ 4651 v0 = v1 = v2 = v3 = v4 = v5 = v6 = v7 = 0; 4652 que = adapter->queues; 4653 for (i = 0; i < adapter->num_queues; i++, que++) { 4654 struct tx_ring *txr = que->txr; 4655 4656 v0 += txr->q_efbig_tx_dma_setup; 4657 v1 += txr->q_mbuf_defrag_failed; 4658 v2 += txr->q_efbig2_tx_dma_setup; 4659 v3 += txr->q_einval_tx_dma_setup; 4660 v4 += txr->q_other_tx_dma_setup; 4661 v5 += txr->q_eagain_tx_dma_setup; 4662 v6 += txr->q_enomem_tx_dma_setup; 4663 v7 += txr->q_tso_err; 4664 } 4665 IXGBE_EVC_STORE(&adapter->efbig_tx_dma_setup, v0); 4666 IXGBE_EVC_STORE(&adapter->mbuf_defrag_failed, v1); 4667 IXGBE_EVC_STORE(&adapter->efbig2_tx_dma_setup, v2); 4668 IXGBE_EVC_STORE(&adapter->einval_tx_dma_setup, v3); 4669 IXGBE_EVC_STORE(&adapter->other_tx_dma_setup, v4); 4670 IXGBE_EVC_STORE(&adapter->eagain_tx_dma_setup, v5); 4671 IXGBE_EVC_STORE(&adapter->enomem_tx_dma_setup, v6); 4672 IXGBE_EVC_STORE(&adapter->tso_err, v7); 4673 4674 /* 4675 * Check the TX queues status 4676 * - mark hung queues so we don't schedule on them 4677 * - watchdog only if all queues show hung 4678 */ 4679 que = adapter->queues; 4680 for (i = 0; i < adapter->num_queues; i++, que++) { 4681 /* Keep track of queues with work for soft irq */ 4682 if (que->txr->busy) 4683 queues |= 1ULL << que->me; 4684 /* 4685 * Each time txeof runs without cleaning, but there 4686 * are uncleaned descriptors it increments busy. If 4687 * we get to the MAX we declare it hung. 4688 */ 4689 if (que->busy == IXGBE_QUEUE_HUNG) { 4690 ++hung; 4691 /* Mark the queue as inactive */ 4692 adapter->active_queues &= ~(1ULL << que->me); 4693 continue; 4694 } else { 4695 /* Check if we've come back from hung */ 4696 if ((adapter->active_queues & (1ULL << que->me)) == 0) 4697 adapter->active_queues |= 1ULL << que->me; 4698 } 4699 if (que->busy >= IXGBE_MAX_TX_BUSY) { 4700 device_printf(dev, 4701 "Warning queue %d appears to be hung!\n", i); 4702 que->txr->busy = IXGBE_QUEUE_HUNG; 4703 ++hung; 4704 } 4705 } 4706 4707 /* Only truly watchdog if all queues show hung */ 4708 if (hung == adapter->num_queues) 4709 goto watchdog; 4710 #if 0 /* XXX Avoid unexpectedly disabling interrupt forever (PR#53294) */ 4711 else if (queues != 0) { /* Force an IRQ on queues with work */ 4712 que = adapter->queues; 4713 for (i = 0; i < adapter->num_queues; i++, que++) { 4714 mutex_enter(&que->dc_mtx); 4715 if (que->disabled_count == 0) 4716 ixgbe_rearm_queues(adapter, 4717 queues & ((u64)1 << i)); 4718 mutex_exit(&que->dc_mtx); 4719 } 4720 } 4721 #endif 4722 4723 atomic_store_relaxed(&adapter->timer_pending, 0); 4724 IXGBE_CORE_UNLOCK(adapter); 4725 callout_reset(&adapter->timer, hz, ixgbe_local_timer, adapter); 4726 return; 4727 4728 watchdog: 4729 device_printf(adapter->dev, "Watchdog timeout -- resetting\n"); 4730 adapter->ifp->if_flags &= ~IFF_RUNNING; 4731 IXGBE_EVC_ADD(&adapter->watchdog_events, 1); 4732 ixgbe_init_locked(adapter); 4733 IXGBE_CORE_UNLOCK(adapter); 4734 } /* ixgbe_handle_timer */ 4735 4736 /************************************************************************ 4737 * ixgbe_recovery_mode_timer - Recovery mode timer routine 4738 ************************************************************************/ 4739 static void 4740 ixgbe_recovery_mode_timer(void *arg) 4741 { 4742 struct adapter *adapter = arg; 4743 4744 if (__predict_true(adapter->osdep.detaching == false)) { 4745 if (atomic_cas_uint(&adapter->recovery_mode_timer_pending, 4746 0, 1) == 0) { 4747 workqueue_enqueue(adapter->recovery_mode_timer_wq, 4748 &adapter->recovery_mode_timer_wc, NULL); 4749 } 4750 } 4751 } 4752 4753 static void 4754 ixgbe_handle_recovery_mode_timer(struct work *wk, void *context) 4755 { 4756 struct adapter *adapter = context; 4757 struct ixgbe_hw *hw = &adapter->hw; 4758 4759 IXGBE_CORE_LOCK(adapter); 4760 if (ixgbe_fw_recovery_mode(hw)) { 4761 if (atomic_cas_uint(&adapter->recovery_mode, 0, 1) == 0) { 4762 /* Firmware error detected, entering recovery mode */ 4763 device_printf(adapter->dev, 4764 "Firmware recovery mode detected. Limiting " 4765 "functionality. Refer to the Intel(R) Ethernet " 4766 "Adapters and Devices User Guide for details on " 4767 "firmware recovery mode.\n"); 4768 4769 if (hw->adapter_stopped == FALSE) 4770 ixgbe_stop_locked(adapter); 4771 } 4772 } else 4773 atomic_cas_uint(&adapter->recovery_mode, 1, 0); 4774 4775 atomic_store_relaxed(&adapter->recovery_mode_timer_pending, 0); 4776 callout_reset(&adapter->recovery_mode_timer, hz, 4777 ixgbe_recovery_mode_timer, adapter); 4778 IXGBE_CORE_UNLOCK(adapter); 4779 } /* ixgbe_handle_recovery_mode_timer */ 4780 4781 /************************************************************************ 4782 * ixgbe_handle_mod - Tasklet for SFP module interrupts 4783 * bool int_en: true if it's called when the interrupt is enabled. 4784 ************************************************************************/ 4785 static void 4786 ixgbe_handle_mod(void *context, bool int_en) 4787 { 4788 struct adapter *adapter = context; 4789 struct ixgbe_hw *hw = &adapter->hw; 4790 device_t dev = adapter->dev; 4791 enum ixgbe_sfp_type last_sfp_type; 4792 u32 err; 4793 bool last_unsupported_sfp_recovery; 4794 4795 KASSERT(mutex_owned(&adapter->core_mtx)); 4796 4797 last_sfp_type = hw->phy.sfp_type; 4798 last_unsupported_sfp_recovery = hw->need_unsupported_sfp_recovery; 4799 IXGBE_EVC_ADD(&adapter->mod_workev, 1); 4800 if (adapter->hw.need_crosstalk_fix) { 4801 if ((hw->mac.type != ixgbe_mac_82598EB) && 4802 !ixgbe_sfp_cage_full(hw)) 4803 goto out; 4804 } 4805 4806 err = hw->phy.ops.identify_sfp(hw); 4807 if (err == IXGBE_ERR_SFP_NOT_SUPPORTED) { 4808 if (last_unsupported_sfp_recovery == false) 4809 device_printf(dev, 4810 "Unsupported SFP+ module type was detected.\n"); 4811 goto out; 4812 } 4813 4814 if (hw->need_unsupported_sfp_recovery) { 4815 device_printf(dev, "Recovering from unsupported SFP\n"); 4816 /* 4817 * We could recover the status by calling setup_sfp(), 4818 * setup_link() and some others. It's complex and might not 4819 * work correctly on some unknown cases. To avoid such type of 4820 * problem, call ixgbe_init_locked(). It's simple and safe 4821 * approach. 4822 */ 4823 ixgbe_init_locked(adapter); 4824 } else if ((hw->phy.sfp_type != ixgbe_sfp_type_not_present) && 4825 (hw->phy.sfp_type != last_sfp_type)) { 4826 /* A module is inserted and changed. */ 4827 4828 if (hw->mac.type == ixgbe_mac_82598EB) 4829 err = hw->phy.ops.reset(hw); 4830 else { 4831 err = hw->mac.ops.setup_sfp(hw); 4832 hw->phy.sfp_setup_needed = FALSE; 4833 } 4834 if (err == IXGBE_ERR_SFP_NOT_SUPPORTED) { 4835 device_printf(dev, 4836 "Setup failure - unsupported SFP+ module type.\n"); 4837 goto out; 4838 } 4839 } 4840 4841 out: 4842 /* get_supported_phy_layer will call hw->phy.ops.identify_sfp() */ 4843 adapter->phy_layer = ixgbe_get_supported_physical_layer(hw); 4844 4845 /* Adjust media types shown in ifconfig */ 4846 IXGBE_CORE_UNLOCK(adapter); 4847 ifmedia_removeall(&adapter->media); 4848 ixgbe_add_media_types(adapter); 4849 ifmedia_set(&adapter->media, IFM_ETHER | IFM_AUTO); 4850 IXGBE_CORE_LOCK(adapter); 4851 4852 /* 4853 * Don't schedule MSF event if the chip is 82598. 82598 doesn't support 4854 * MSF. At least, calling ixgbe_handle_msf on 82598 DA makes the link 4855 * flap because the function calls setup_link(). 4856 */ 4857 if (hw->mac.type != ixgbe_mac_82598EB) { 4858 mutex_enter(&adapter->admin_mtx); 4859 if (int_en) 4860 adapter->task_requests |= IXGBE_REQUEST_TASK_MSF; 4861 else 4862 adapter->task_requests |= IXGBE_REQUEST_TASK_MSF_WOI; 4863 mutex_exit(&adapter->admin_mtx); 4864 } 4865 4866 /* 4867 * Don't call ixgbe_schedule_admin_tasklet() because we are on 4868 * the workqueue now. 4869 */ 4870 } /* ixgbe_handle_mod */ 4871 4872 4873 /************************************************************************ 4874 * ixgbe_handle_msf - Tasklet for MSF (multispeed fiber) interrupts 4875 ************************************************************************/ 4876 static void 4877 ixgbe_handle_msf(void *context) 4878 { 4879 struct adapter *adapter = context; 4880 struct ixgbe_hw *hw = &adapter->hw; 4881 u32 autoneg; 4882 bool negotiate; 4883 4884 KASSERT(mutex_owned(&adapter->core_mtx)); 4885 4886 IXGBE_EVC_ADD(&adapter->msf_workev, 1); 4887 4888 autoneg = hw->phy.autoneg_advertised; 4889 if ((!autoneg) && (hw->mac.ops.get_link_capabilities)) 4890 hw->mac.ops.get_link_capabilities(hw, &autoneg, &negotiate); 4891 if (hw->mac.ops.setup_link) 4892 hw->mac.ops.setup_link(hw, autoneg, TRUE); 4893 } /* ixgbe_handle_msf */ 4894 4895 /************************************************************************ 4896 * ixgbe_handle_phy - Tasklet for external PHY interrupts 4897 ************************************************************************/ 4898 static void 4899 ixgbe_handle_phy(void *context) 4900 { 4901 struct adapter *adapter = context; 4902 struct ixgbe_hw *hw = &adapter->hw; 4903 int error; 4904 4905 KASSERT(mutex_owned(&adapter->core_mtx)); 4906 4907 IXGBE_EVC_ADD(&adapter->phy_workev, 1); 4908 error = hw->phy.ops.handle_lasi(hw); 4909 if (error == IXGBE_ERR_OVERTEMP) 4910 device_printf(adapter->dev, 4911 "CRITICAL: EXTERNAL PHY OVER TEMP!! " 4912 " PHY will downshift to lower power state!\n"); 4913 else if (error) 4914 device_printf(adapter->dev, 4915 "Error handling LASI interrupt: %d\n", error); 4916 } /* ixgbe_handle_phy */ 4917 4918 static void 4919 ixgbe_handle_admin(struct work *wk, void *context) 4920 { 4921 struct adapter *adapter = context; 4922 struct ifnet *ifp = adapter->ifp; 4923 struct ixgbe_hw *hw = &adapter->hw; 4924 u32 task_requests; 4925 u32 eims_enable = 0; 4926 4927 mutex_enter(&adapter->admin_mtx); 4928 adapter->admin_pending = 0; 4929 task_requests = adapter->task_requests; 4930 adapter->task_requests = 0; 4931 mutex_exit(&adapter->admin_mtx); 4932 4933 /* 4934 * Hold the IFNET_LOCK across this entire call. This will 4935 * prevent additional changes to adapter->phy_layer 4936 * and serialize calls to this tasklet. We cannot hold the 4937 * CORE_LOCK while calling into the ifmedia functions as 4938 * they call ifmedia_lock() and the lock is CORE_LOCK. 4939 */ 4940 IFNET_LOCK(ifp); 4941 IXGBE_CORE_LOCK(adapter); 4942 if ((task_requests & IXGBE_REQUEST_TASK_LSC) != 0) { 4943 ixgbe_handle_link(adapter); 4944 eims_enable |= IXGBE_EIMS_LSC; 4945 } 4946 if ((task_requests & IXGBE_REQUEST_TASK_MOD_WOI) != 0) 4947 ixgbe_handle_mod(adapter, false); 4948 if ((task_requests & IXGBE_REQUEST_TASK_MOD) != 0) { 4949 ixgbe_handle_mod(adapter, true); 4950 if (hw->mac.type >= ixgbe_mac_X540) 4951 eims_enable |= IXGBE_EICR_GPI_SDP0_X540; 4952 else 4953 eims_enable |= IXGBE_EICR_GPI_SDP2_BY_MAC(hw); 4954 } 4955 if ((task_requests 4956 & (IXGBE_REQUEST_TASK_MSF_WOI | IXGBE_REQUEST_TASK_MSF)) != 0) { 4957 ixgbe_handle_msf(adapter); 4958 if (((task_requests & IXGBE_REQUEST_TASK_MSF) != 0) && 4959 (hw->mac.type == ixgbe_mac_82599EB)) 4960 eims_enable |= IXGBE_EIMS_GPI_SDP1_BY_MAC(hw); 4961 } 4962 if ((task_requests & IXGBE_REQUEST_TASK_PHY) != 0) { 4963 ixgbe_handle_phy(adapter); 4964 eims_enable |= IXGBE_EICR_GPI_SDP0_X540; 4965 } 4966 if ((task_requests & IXGBE_REQUEST_TASK_FDIR) != 0) { 4967 ixgbe_reinit_fdir(adapter); 4968 eims_enable |= IXGBE_EIMS_FLOW_DIR; 4969 } 4970 #if 0 /* notyet */ 4971 if ((task_requests & IXGBE_REQUEST_TASK_MBX) != 0) { 4972 ixgbe_handle_mbx(adapter); 4973 eims_enable |= IXGBE_EIMS_MAILBOX; 4974 } 4975 #endif 4976 IXGBE_WRITE_REG(hw, IXGBE_EIMS, eims_enable); 4977 4978 IXGBE_CORE_UNLOCK(adapter); 4979 IFNET_UNLOCK(ifp); 4980 } /* ixgbe_handle_admin */ 4981 4982 static void 4983 ixgbe_ifstop(struct ifnet *ifp, int disable) 4984 { 4985 struct adapter *adapter = ifp->if_softc; 4986 4987 IXGBE_CORE_LOCK(adapter); 4988 ixgbe_stop_locked(adapter); 4989 IXGBE_CORE_UNLOCK(adapter); 4990 4991 workqueue_wait(adapter->timer_wq, &adapter->timer_wc); 4992 atomic_store_relaxed(&adapter->timer_pending, 0); 4993 } 4994 4995 /************************************************************************ 4996 * ixgbe_stop_locked - Stop the hardware 4997 * 4998 * Disables all traffic on the adapter by issuing a 4999 * global reset on the MAC and deallocates TX/RX buffers. 5000 ************************************************************************/ 5001 static void 5002 ixgbe_stop_locked(void *arg) 5003 { 5004 struct ifnet *ifp; 5005 struct adapter *adapter = arg; 5006 struct ixgbe_hw *hw = &adapter->hw; 5007 5008 ifp = adapter->ifp; 5009 5010 KASSERT(mutex_owned(&adapter->core_mtx)); 5011 5012 INIT_DEBUGOUT("ixgbe_stop_locked: begin\n"); 5013 ixgbe_disable_intr(adapter); 5014 callout_stop(&adapter->timer); 5015 5016 /* Don't schedule workqueues. */ 5017 adapter->schedule_wqs_ok = false; 5018 5019 /* Let the stack know...*/ 5020 ifp->if_flags &= ~IFF_RUNNING; 5021 5022 ixgbe_reset_hw(hw); 5023 hw->adapter_stopped = FALSE; 5024 ixgbe_stop_adapter(hw); 5025 if (hw->mac.type == ixgbe_mac_82599EB) 5026 ixgbe_stop_mac_link_on_d3_82599(hw); 5027 /* Turn off the laser - noop with no optics */ 5028 ixgbe_disable_tx_laser(hw); 5029 5030 /* Update the stack */ 5031 adapter->link_up = FALSE; 5032 ixgbe_update_link_status(adapter); 5033 5034 /* reprogram the RAR[0] in case user changed it. */ 5035 ixgbe_set_rar(&adapter->hw, 0, adapter->hw.mac.addr, 0, IXGBE_RAH_AV); 5036 5037 return; 5038 } /* ixgbe_stop_locked */ 5039 5040 /************************************************************************ 5041 * ixgbe_update_link_status - Update OS on link state 5042 * 5043 * Note: Only updates the OS on the cached link state. 5044 * The real check of the hardware only happens with 5045 * a link interrupt. 5046 ************************************************************************/ 5047 static void 5048 ixgbe_update_link_status(struct adapter *adapter) 5049 { 5050 struct ifnet *ifp = adapter->ifp; 5051 device_t dev = adapter->dev; 5052 struct ixgbe_hw *hw = &adapter->hw; 5053 5054 KASSERT(mutex_owned(&adapter->core_mtx)); 5055 5056 if (adapter->link_up) { 5057 if (adapter->link_active != LINK_STATE_UP) { 5058 /* 5059 * To eliminate influence of the previous state 5060 * in the same way as ixgbe_init_locked(). 5061 */ 5062 struct ix_queue *que = adapter->queues; 5063 for (int i = 0; i < adapter->num_queues; i++, que++) 5064 que->eitr_setting = 0; 5065 5066 if (adapter->link_speed == IXGBE_LINK_SPEED_10GB_FULL){ 5067 /* 5068 * Discard count for both MAC Local Fault and 5069 * Remote Fault because those registers are 5070 * valid only when the link speed is up and 5071 * 10Gbps. 5072 */ 5073 IXGBE_READ_REG(hw, IXGBE_MLFC); 5074 IXGBE_READ_REG(hw, IXGBE_MRFC); 5075 } 5076 5077 if (bootverbose) { 5078 const char *bpsmsg; 5079 5080 switch (adapter->link_speed) { 5081 case IXGBE_LINK_SPEED_10GB_FULL: 5082 bpsmsg = "10 Gbps"; 5083 break; 5084 case IXGBE_LINK_SPEED_5GB_FULL: 5085 bpsmsg = "5 Gbps"; 5086 break; 5087 case IXGBE_LINK_SPEED_2_5GB_FULL: 5088 bpsmsg = "2.5 Gbps"; 5089 break; 5090 case IXGBE_LINK_SPEED_1GB_FULL: 5091 bpsmsg = "1 Gbps"; 5092 break; 5093 case IXGBE_LINK_SPEED_100_FULL: 5094 bpsmsg = "100 Mbps"; 5095 break; 5096 case IXGBE_LINK_SPEED_10_FULL: 5097 bpsmsg = "10 Mbps"; 5098 break; 5099 default: 5100 bpsmsg = "unknown speed"; 5101 break; 5102 } 5103 device_printf(dev, "Link is up %s %s \n", 5104 bpsmsg, "Full Duplex"); 5105 } 5106 adapter->link_active = LINK_STATE_UP; 5107 /* Update any Flow Control changes */ 5108 ixgbe_fc_enable(&adapter->hw); 5109 /* Update DMA coalescing config */ 5110 ixgbe_config_dmac(adapter); 5111 if_link_state_change(ifp, LINK_STATE_UP); 5112 5113 if (adapter->feat_en & IXGBE_FEATURE_SRIOV) 5114 ixgbe_ping_all_vfs(adapter); 5115 } 5116 } else { 5117 /* 5118 * Do it when link active changes to DOWN. i.e. 5119 * a) LINK_STATE_UNKNOWN -> LINK_STATE_DOWN 5120 * b) LINK_STATE_UP -> LINK_STATE_DOWN 5121 */ 5122 if (adapter->link_active != LINK_STATE_DOWN) { 5123 if (bootverbose) 5124 device_printf(dev, "Link is Down\n"); 5125 if_link_state_change(ifp, LINK_STATE_DOWN); 5126 adapter->link_active = LINK_STATE_DOWN; 5127 if (adapter->feat_en & IXGBE_FEATURE_SRIOV) 5128 ixgbe_ping_all_vfs(adapter); 5129 ixgbe_drain_all(adapter); 5130 } 5131 } 5132 } /* ixgbe_update_link_status */ 5133 5134 /************************************************************************ 5135 * ixgbe_config_dmac - Configure DMA Coalescing 5136 ************************************************************************/ 5137 static void 5138 ixgbe_config_dmac(struct adapter *adapter) 5139 { 5140 struct ixgbe_hw *hw = &adapter->hw; 5141 struct ixgbe_dmac_config *dcfg = &hw->mac.dmac_config; 5142 5143 if (hw->mac.type < ixgbe_mac_X550 || !hw->mac.ops.dmac_config) 5144 return; 5145 5146 if (dcfg->watchdog_timer ^ adapter->dmac || 5147 dcfg->link_speed ^ adapter->link_speed) { 5148 dcfg->watchdog_timer = adapter->dmac; 5149 dcfg->fcoe_en = false; 5150 dcfg->link_speed = adapter->link_speed; 5151 dcfg->num_tcs = 1; 5152 5153 INIT_DEBUGOUT2("dmac settings: watchdog %d, link speed %d\n", 5154 dcfg->watchdog_timer, dcfg->link_speed); 5155 5156 hw->mac.ops.dmac_config(hw); 5157 } 5158 } /* ixgbe_config_dmac */ 5159 5160 /************************************************************************ 5161 * ixgbe_enable_intr 5162 ************************************************************************/ 5163 static void 5164 ixgbe_enable_intr(struct adapter *adapter) 5165 { 5166 struct ixgbe_hw *hw = &adapter->hw; 5167 struct ix_queue *que = adapter->queues; 5168 u32 mask, fwsm; 5169 5170 mask = (IXGBE_EIMS_ENABLE_MASK & ~IXGBE_EIMS_RTX_QUEUE); 5171 5172 switch (adapter->hw.mac.type) { 5173 case ixgbe_mac_82599EB: 5174 mask |= IXGBE_EIMS_ECC; 5175 /* Temperature sensor on some adapters */ 5176 mask |= IXGBE_EIMS_GPI_SDP0; 5177 /* SFP+ (RX_LOS_N & MOD_ABS_N) */ 5178 mask |= IXGBE_EIMS_GPI_SDP1; 5179 mask |= IXGBE_EIMS_GPI_SDP2; 5180 break; 5181 case ixgbe_mac_X540: 5182 /* Detect if Thermal Sensor is enabled */ 5183 fwsm = IXGBE_READ_REG(hw, IXGBE_FWSM); 5184 if (fwsm & IXGBE_FWSM_TS_ENABLED) 5185 mask |= IXGBE_EIMS_TS; 5186 mask |= IXGBE_EIMS_ECC; 5187 break; 5188 case ixgbe_mac_X550: 5189 /* MAC thermal sensor is automatically enabled */ 5190 mask |= IXGBE_EIMS_TS; 5191 mask |= IXGBE_EIMS_ECC; 5192 break; 5193 case ixgbe_mac_X550EM_x: 5194 case ixgbe_mac_X550EM_a: 5195 /* Some devices use SDP0 for important information */ 5196 if (hw->device_id == IXGBE_DEV_ID_X550EM_X_SFP || 5197 hw->device_id == IXGBE_DEV_ID_X550EM_A_SFP || 5198 hw->device_id == IXGBE_DEV_ID_X550EM_A_SFP_N || 5199 hw->device_id == IXGBE_DEV_ID_X550EM_X_10G_T) 5200 mask |= IXGBE_EIMS_GPI_SDP0_BY_MAC(hw); 5201 if (hw->phy.type == ixgbe_phy_x550em_ext_t) 5202 mask |= IXGBE_EICR_GPI_SDP0_X540; 5203 mask |= IXGBE_EIMS_ECC; 5204 break; 5205 default: 5206 break; 5207 } 5208 5209 /* Enable Fan Failure detection */ 5210 if (adapter->feat_en & IXGBE_FEATURE_FAN_FAIL) 5211 mask |= IXGBE_EIMS_GPI_SDP1; 5212 /* Enable SR-IOV */ 5213 if (adapter->feat_en & IXGBE_FEATURE_SRIOV) 5214 mask |= IXGBE_EIMS_MAILBOX; 5215 /* Enable Flow Director */ 5216 if (adapter->feat_en & IXGBE_FEATURE_FDIR) 5217 mask |= IXGBE_EIMS_FLOW_DIR; 5218 5219 IXGBE_WRITE_REG(hw, IXGBE_EIMS, mask); 5220 5221 /* With MSI-X we use auto clear */ 5222 if ((adapter->feat_en & IXGBE_FEATURE_MSIX) != 0) { 5223 /* 5224 * We use auto clear for RTX_QUEUE only. Don't use other 5225 * interrupts (e.g. link interrupt). BTW, we don't use 5226 * TCP_TIMER interrupt itself. 5227 */ 5228 IXGBE_WRITE_REG(hw, IXGBE_EIAC, IXGBE_EIMS_RTX_QUEUE); 5229 } 5230 5231 /* 5232 * Now enable all queues, this is done separately to 5233 * allow for handling the extended (beyond 32) MSI-X 5234 * vectors that can be used by 82599 5235 */ 5236 for (int i = 0; i < adapter->num_queues; i++, que++) 5237 ixgbe_enable_queue(adapter, que->msix); 5238 5239 IXGBE_WRITE_FLUSH(hw); 5240 5241 } /* ixgbe_enable_intr */ 5242 5243 /************************************************************************ 5244 * ixgbe_disable_intr_internal 5245 ************************************************************************/ 5246 static void 5247 ixgbe_disable_intr_internal(struct adapter *adapter, bool nestok) 5248 { 5249 struct ix_queue *que = adapter->queues; 5250 5251 /* disable interrupts other than queues */ 5252 IXGBE_WRITE_REG(&adapter->hw, IXGBE_EIMC, ~IXGBE_EIMC_RTX_QUEUE); 5253 5254 if ((adapter->feat_en & IXGBE_FEATURE_MSIX) != 0) 5255 IXGBE_WRITE_REG(&adapter->hw, IXGBE_EIAC, 0); 5256 5257 for (int i = 0; i < adapter->num_queues; i++, que++) 5258 ixgbe_disable_queue_internal(adapter, que->msix, nestok); 5259 5260 IXGBE_WRITE_FLUSH(&adapter->hw); 5261 5262 } /* ixgbe_do_disable_intr_internal */ 5263 5264 /************************************************************************ 5265 * ixgbe_disable_intr 5266 ************************************************************************/ 5267 static void 5268 ixgbe_disable_intr(struct adapter *adapter) 5269 { 5270 5271 ixgbe_disable_intr_internal(adapter, true); 5272 } /* ixgbe_disable_intr */ 5273 5274 /************************************************************************ 5275 * ixgbe_ensure_disabled_intr 5276 ************************************************************************/ 5277 void 5278 ixgbe_ensure_disabled_intr(struct adapter *adapter) 5279 { 5280 5281 ixgbe_disable_intr_internal(adapter, false); 5282 } /* ixgbe_ensure_disabled_intr */ 5283 5284 /************************************************************************ 5285 * ixgbe_legacy_irq - Legacy Interrupt Service routine 5286 ************************************************************************/ 5287 static int 5288 ixgbe_legacy_irq(void *arg) 5289 { 5290 struct ix_queue *que = arg; 5291 struct adapter *adapter = que->adapter; 5292 struct ixgbe_hw *hw = &adapter->hw; 5293 struct ifnet *ifp = adapter->ifp; 5294 struct tx_ring *txr = adapter->tx_rings; 5295 u32 eicr; 5296 u32 eims_orig; 5297 u32 eims_enable = 0; 5298 u32 eims_disable = 0; 5299 5300 eims_orig = IXGBE_READ_REG(hw, IXGBE_EIMS); 5301 /* 5302 * Silicon errata #26 on 82598. Disable all interrupts before reading 5303 * EICR. 5304 */ 5305 IXGBE_WRITE_REG(hw, IXGBE_EIMC, IXGBE_IRQ_CLEAR_MASK); 5306 5307 /* Read and clear EICR */ 5308 eicr = IXGBE_READ_REG(hw, IXGBE_EICR); 5309 5310 if (eicr == 0) { 5311 IXGBE_EVC_ADD(&adapter->stats.pf.intzero, 1); 5312 IXGBE_WRITE_REG(hw, IXGBE_EIMS, eims_orig); 5313 return 0; 5314 } 5315 IXGBE_EVC_ADD(&adapter->stats.pf.legint, 1); 5316 5317 /* Queue (0) intr */ 5318 if (((ifp->if_flags & IFF_RUNNING) != 0) && 5319 (eicr & IXGBE_EIMC_RTX_QUEUE) != 0) { 5320 IXGBE_EVC_ADD(&que->irqs, 1); 5321 5322 /* 5323 * The same as ixgbe_msix_que() about 5324 * "que->txrx_use_workqueue". 5325 */ 5326 que->txrx_use_workqueue = adapter->txrx_use_workqueue; 5327 5328 IXGBE_TX_LOCK(txr); 5329 ixgbe_txeof(txr); 5330 #ifdef notyet 5331 if (!ixgbe_ring_empty(ifp, txr->br)) 5332 ixgbe_start_locked(ifp, txr); 5333 #endif 5334 IXGBE_TX_UNLOCK(txr); 5335 5336 IXGBE_EVC_ADD(&que->req, 1); 5337 ixgbe_sched_handle_que(adapter, que); 5338 /* Disable queue 0 interrupt */ 5339 eims_disable |= 1UL << 0; 5340 } else 5341 eims_enable |= eims_orig & IXGBE_EIMC_RTX_QUEUE; 5342 5343 ixgbe_intr_admin_common(adapter, eicr, &eims_disable); 5344 5345 /* Re-enable some interrupts */ 5346 IXGBE_WRITE_REG(hw, IXGBE_EIMS, 5347 (eims_orig & ~eims_disable) | eims_enable); 5348 5349 return 1; 5350 } /* ixgbe_legacy_irq */ 5351 5352 /************************************************************************ 5353 * ixgbe_free_pciintr_resources 5354 ************************************************************************/ 5355 static void 5356 ixgbe_free_pciintr_resources(struct adapter *adapter) 5357 { 5358 struct ix_queue *que = adapter->queues; 5359 int rid; 5360 5361 /* 5362 * Release all msix queue resources: 5363 */ 5364 for (int i = 0; i < adapter->num_queues; i++, que++) { 5365 if (que->res != NULL) { 5366 pci_intr_disestablish(adapter->osdep.pc, 5367 adapter->osdep.ihs[i]); 5368 adapter->osdep.ihs[i] = NULL; 5369 } 5370 } 5371 5372 /* Clean the Legacy or Link interrupt last */ 5373 if (adapter->vector) /* we are doing MSIX */ 5374 rid = adapter->vector; 5375 else 5376 rid = 0; 5377 5378 if (adapter->osdep.ihs[rid] != NULL) { 5379 pci_intr_disestablish(adapter->osdep.pc, 5380 adapter->osdep.ihs[rid]); 5381 adapter->osdep.ihs[rid] = NULL; 5382 } 5383 5384 if (adapter->osdep.intrs != NULL) { 5385 pci_intr_release(adapter->osdep.pc, adapter->osdep.intrs, 5386 adapter->osdep.nintrs); 5387 adapter->osdep.intrs = NULL; 5388 } 5389 } /* ixgbe_free_pciintr_resources */ 5390 5391 /************************************************************************ 5392 * ixgbe_free_pci_resources 5393 ************************************************************************/ 5394 static void 5395 ixgbe_free_pci_resources(struct adapter *adapter) 5396 { 5397 5398 ixgbe_free_pciintr_resources(adapter); 5399 5400 if (adapter->osdep.mem_size != 0) { 5401 bus_space_unmap(adapter->osdep.mem_bus_space_tag, 5402 adapter->osdep.mem_bus_space_handle, 5403 adapter->osdep.mem_size); 5404 } 5405 } /* ixgbe_free_pci_resources */ 5406 5407 /************************************************************************ 5408 * ixgbe_sysctl_flowcntl 5409 * 5410 * SYSCTL wrapper around setting Flow Control 5411 ************************************************************************/ 5412 static int 5413 ixgbe_sysctl_flowcntl(SYSCTLFN_ARGS) 5414 { 5415 struct sysctlnode node = *rnode; 5416 struct adapter *adapter = (struct adapter *)node.sysctl_data; 5417 int error, fc; 5418 5419 if (ixgbe_fw_recovery_mode_swflag(adapter)) 5420 return (EPERM); 5421 5422 fc = adapter->hw.fc.current_mode; 5423 node.sysctl_data = &fc; 5424 error = sysctl_lookup(SYSCTLFN_CALL(&node)); 5425 if (error != 0 || newp == NULL) 5426 return error; 5427 5428 /* Don't bother if it's not changed */ 5429 if (fc == adapter->hw.fc.current_mode) 5430 return (0); 5431 5432 return ixgbe_set_flowcntl(adapter, fc); 5433 } /* ixgbe_sysctl_flowcntl */ 5434 5435 /************************************************************************ 5436 * ixgbe_set_flowcntl - Set flow control 5437 * 5438 * Flow control values: 5439 * 0 - off 5440 * 1 - rx pause 5441 * 2 - tx pause 5442 * 3 - full 5443 ************************************************************************/ 5444 static int 5445 ixgbe_set_flowcntl(struct adapter *adapter, int fc) 5446 { 5447 switch (fc) { 5448 case ixgbe_fc_rx_pause: 5449 case ixgbe_fc_tx_pause: 5450 case ixgbe_fc_full: 5451 adapter->hw.fc.requested_mode = fc; 5452 if (adapter->num_queues > 1) 5453 ixgbe_disable_rx_drop(adapter); 5454 break; 5455 case ixgbe_fc_none: 5456 adapter->hw.fc.requested_mode = ixgbe_fc_none; 5457 if (adapter->num_queues > 1) 5458 ixgbe_enable_rx_drop(adapter); 5459 break; 5460 default: 5461 return (EINVAL); 5462 } 5463 5464 #if 0 /* XXX NetBSD */ 5465 /* Don't autoneg if forcing a value */ 5466 adapter->hw.fc.disable_fc_autoneg = TRUE; 5467 #endif 5468 ixgbe_fc_enable(&adapter->hw); 5469 5470 return (0); 5471 } /* ixgbe_set_flowcntl */ 5472 5473 /************************************************************************ 5474 * ixgbe_enable_rx_drop 5475 * 5476 * Enable the hardware to drop packets when the buffer is 5477 * full. This is useful with multiqueue, so that no single 5478 * queue being full stalls the entire RX engine. We only 5479 * enable this when Multiqueue is enabled AND Flow Control 5480 * is disabled. 5481 ************************************************************************/ 5482 static void 5483 ixgbe_enable_rx_drop(struct adapter *adapter) 5484 { 5485 struct ixgbe_hw *hw = &adapter->hw; 5486 struct rx_ring *rxr; 5487 u32 srrctl; 5488 5489 for (int i = 0; i < adapter->num_queues; i++) { 5490 rxr = &adapter->rx_rings[i]; 5491 srrctl = IXGBE_READ_REG(hw, IXGBE_SRRCTL(rxr->me)); 5492 srrctl |= IXGBE_SRRCTL_DROP_EN; 5493 IXGBE_WRITE_REG(hw, IXGBE_SRRCTL(rxr->me), srrctl); 5494 } 5495 5496 /* enable drop for each vf */ 5497 for (int i = 0; i < adapter->num_vfs; i++) { 5498 IXGBE_WRITE_REG(hw, IXGBE_QDE, 5499 (IXGBE_QDE_WRITE | (i << IXGBE_QDE_IDX_SHIFT) | 5500 IXGBE_QDE_ENABLE)); 5501 } 5502 } /* ixgbe_enable_rx_drop */ 5503 5504 /************************************************************************ 5505 * ixgbe_disable_rx_drop 5506 ************************************************************************/ 5507 static void 5508 ixgbe_disable_rx_drop(struct adapter *adapter) 5509 { 5510 struct ixgbe_hw *hw = &adapter->hw; 5511 struct rx_ring *rxr; 5512 u32 srrctl; 5513 5514 for (int i = 0; i < adapter->num_queues; i++) { 5515 rxr = &adapter->rx_rings[i]; 5516 srrctl = IXGBE_READ_REG(hw, IXGBE_SRRCTL(rxr->me)); 5517 srrctl &= ~IXGBE_SRRCTL_DROP_EN; 5518 IXGBE_WRITE_REG(hw, IXGBE_SRRCTL(rxr->me), srrctl); 5519 } 5520 5521 /* disable drop for each vf */ 5522 for (int i = 0; i < adapter->num_vfs; i++) { 5523 IXGBE_WRITE_REG(hw, IXGBE_QDE, 5524 (IXGBE_QDE_WRITE | (i << IXGBE_QDE_IDX_SHIFT))); 5525 } 5526 } /* ixgbe_disable_rx_drop */ 5527 5528 /************************************************************************ 5529 * ixgbe_sysctl_advertise 5530 * 5531 * SYSCTL wrapper around setting advertised speed 5532 ************************************************************************/ 5533 static int 5534 ixgbe_sysctl_advertise(SYSCTLFN_ARGS) 5535 { 5536 struct sysctlnode node = *rnode; 5537 struct adapter *adapter = (struct adapter *)node.sysctl_data; 5538 int error = 0, advertise; 5539 5540 if (ixgbe_fw_recovery_mode_swflag(adapter)) 5541 return (EPERM); 5542 5543 advertise = adapter->advertise; 5544 node.sysctl_data = &advertise; 5545 error = sysctl_lookup(SYSCTLFN_CALL(&node)); 5546 if (error != 0 || newp == NULL) 5547 return error; 5548 5549 return ixgbe_set_advertise(adapter, advertise); 5550 } /* ixgbe_sysctl_advertise */ 5551 5552 /************************************************************************ 5553 * ixgbe_set_advertise - Control advertised link speed 5554 * 5555 * Flags: 5556 * 0x00 - Default (all capable link speed) 5557 * 0x1 - advertise 100 Mb 5558 * 0x2 - advertise 1G 5559 * 0x4 - advertise 10G 5560 * 0x8 - advertise 10 Mb (yes, Mb) 5561 * 0x10 - advertise 2.5G 5562 * 0x20 - advertise 5G 5563 ************************************************************************/ 5564 static int 5565 ixgbe_set_advertise(struct adapter *adapter, int advertise) 5566 { 5567 device_t dev; 5568 struct ixgbe_hw *hw; 5569 ixgbe_link_speed speed = 0; 5570 ixgbe_link_speed link_caps = 0; 5571 s32 err = IXGBE_NOT_IMPLEMENTED; 5572 bool negotiate = FALSE; 5573 5574 /* Checks to validate new value */ 5575 if (adapter->advertise == advertise) /* no change */ 5576 return (0); 5577 5578 dev = adapter->dev; 5579 hw = &adapter->hw; 5580 5581 /* No speed changes for backplane media */ 5582 if (hw->phy.media_type == ixgbe_media_type_backplane) 5583 return (ENODEV); 5584 5585 if (!((hw->phy.media_type == ixgbe_media_type_copper) || 5586 (hw->phy.multispeed_fiber))) { 5587 device_printf(dev, 5588 "Advertised speed can only be set on copper or " 5589 "multispeed fiber media types.\n"); 5590 return (EINVAL); 5591 } 5592 5593 if (advertise < 0x0 || advertise > 0x3f) { 5594 device_printf(dev, "Invalid advertised speed; " 5595 "valid modes are 0x0 through 0x3f\n"); 5596 return (EINVAL); 5597 } 5598 5599 if (hw->mac.ops.get_link_capabilities) { 5600 err = hw->mac.ops.get_link_capabilities(hw, &link_caps, 5601 &negotiate); 5602 if (err != IXGBE_SUCCESS) { 5603 device_printf(dev, "Unable to determine supported " 5604 "advertise speeds\n"); 5605 return (ENODEV); 5606 } 5607 } 5608 5609 /* Set new value and report new advertised mode */ 5610 if (advertise & 0x1) { 5611 if (!(link_caps & IXGBE_LINK_SPEED_100_FULL)) { 5612 device_printf(dev, "Interface does not support 100Mb " 5613 "advertised speed\n"); 5614 return (EINVAL); 5615 } 5616 speed |= IXGBE_LINK_SPEED_100_FULL; 5617 } 5618 if (advertise & 0x2) { 5619 if (!(link_caps & IXGBE_LINK_SPEED_1GB_FULL)) { 5620 device_printf(dev, "Interface does not support 1Gb " 5621 "advertised speed\n"); 5622 return (EINVAL); 5623 } 5624 speed |= IXGBE_LINK_SPEED_1GB_FULL; 5625 } 5626 if (advertise & 0x4) { 5627 if (!(link_caps & IXGBE_LINK_SPEED_10GB_FULL)) { 5628 device_printf(dev, "Interface does not support 10Gb " 5629 "advertised speed\n"); 5630 return (EINVAL); 5631 } 5632 speed |= IXGBE_LINK_SPEED_10GB_FULL; 5633 } 5634 if (advertise & 0x8) { 5635 if (!(link_caps & IXGBE_LINK_SPEED_10_FULL)) { 5636 device_printf(dev, "Interface does not support 10Mb " 5637 "advertised speed\n"); 5638 return (EINVAL); 5639 } 5640 speed |= IXGBE_LINK_SPEED_10_FULL; 5641 } 5642 if (advertise & 0x10) { 5643 if (!(link_caps & IXGBE_LINK_SPEED_2_5GB_FULL)) { 5644 device_printf(dev, "Interface does not support 2.5Gb " 5645 "advertised speed\n"); 5646 return (EINVAL); 5647 } 5648 speed |= IXGBE_LINK_SPEED_2_5GB_FULL; 5649 } 5650 if (advertise & 0x20) { 5651 if (!(link_caps & IXGBE_LINK_SPEED_5GB_FULL)) { 5652 device_printf(dev, "Interface does not support 5Gb " 5653 "advertised speed\n"); 5654 return (EINVAL); 5655 } 5656 speed |= IXGBE_LINK_SPEED_5GB_FULL; 5657 } 5658 if (advertise == 0) 5659 speed = link_caps; /* All capable link speed */ 5660 5661 hw->mac.autotry_restart = TRUE; 5662 hw->mac.ops.setup_link(hw, speed, TRUE); 5663 adapter->advertise = advertise; 5664 5665 return (0); 5666 } /* ixgbe_set_advertise */ 5667 5668 /************************************************************************ 5669 * ixgbe_get_default_advertise - Get default advertised speed settings 5670 * 5671 * Formatted for sysctl usage. 5672 * Flags: 5673 * 0x1 - advertise 100 Mb 5674 * 0x2 - advertise 1G 5675 * 0x4 - advertise 10G 5676 * 0x8 - advertise 10 Mb (yes, Mb) 5677 * 0x10 - advertise 2.5G 5678 * 0x20 - advertise 5G 5679 ************************************************************************/ 5680 static int 5681 ixgbe_get_default_advertise(struct adapter *adapter) 5682 { 5683 struct ixgbe_hw *hw = &adapter->hw; 5684 int speed; 5685 ixgbe_link_speed link_caps = 0; 5686 s32 err; 5687 bool negotiate = FALSE; 5688 5689 /* 5690 * Advertised speed means nothing unless it's copper or 5691 * multi-speed fiber 5692 */ 5693 if (!(hw->phy.media_type == ixgbe_media_type_copper) && 5694 !(hw->phy.multispeed_fiber)) 5695 return (0); 5696 5697 err = hw->mac.ops.get_link_capabilities(hw, &link_caps, &negotiate); 5698 if (err != IXGBE_SUCCESS) 5699 return (0); 5700 5701 speed = 5702 ((link_caps & IXGBE_LINK_SPEED_10GB_FULL) ? 0x4 : 0) | 5703 ((link_caps & IXGBE_LINK_SPEED_5GB_FULL) ? 0x20 : 0) | 5704 ((link_caps & IXGBE_LINK_SPEED_2_5GB_FULL) ? 0x10 : 0) | 5705 ((link_caps & IXGBE_LINK_SPEED_1GB_FULL) ? 0x2 : 0) | 5706 ((link_caps & IXGBE_LINK_SPEED_100_FULL) ? 0x1 : 0) | 5707 ((link_caps & IXGBE_LINK_SPEED_10_FULL) ? 0x8 : 0); 5708 5709 return speed; 5710 } /* ixgbe_get_default_advertise */ 5711 5712 /************************************************************************ 5713 * ixgbe_sysctl_dmac - Manage DMA Coalescing 5714 * 5715 * Control values: 5716 * 0/1 - off / on (use default value of 1000) 5717 * 5718 * Legal timer values are: 5719 * 50,100,250,500,1000,2000,5000,10000 5720 * 5721 * Turning off interrupt moderation will also turn this off. 5722 ************************************************************************/ 5723 static int 5724 ixgbe_sysctl_dmac(SYSCTLFN_ARGS) 5725 { 5726 struct sysctlnode node = *rnode; 5727 struct adapter *adapter = (struct adapter *)node.sysctl_data; 5728 struct ifnet *ifp = adapter->ifp; 5729 int error; 5730 int newval; 5731 5732 if (ixgbe_fw_recovery_mode_swflag(adapter)) 5733 return (EPERM); 5734 5735 newval = adapter->dmac; 5736 node.sysctl_data = &newval; 5737 error = sysctl_lookup(SYSCTLFN_CALL(&node)); 5738 if ((error) || (newp == NULL)) 5739 return (error); 5740 5741 switch (newval) { 5742 case 0: 5743 /* Disabled */ 5744 adapter->dmac = 0; 5745 break; 5746 case 1: 5747 /* Enable and use default */ 5748 adapter->dmac = 1000; 5749 break; 5750 case 50: 5751 case 100: 5752 case 250: 5753 case 500: 5754 case 1000: 5755 case 2000: 5756 case 5000: 5757 case 10000: 5758 /* Legal values - allow */ 5759 adapter->dmac = newval; 5760 break; 5761 default: 5762 /* Do nothing, illegal value */ 5763 return (EINVAL); 5764 } 5765 5766 /* Re-initialize hardware if it's already running */ 5767 if (ifp->if_flags & IFF_RUNNING) 5768 if_init(ifp); 5769 5770 return (0); 5771 } 5772 5773 #ifdef IXGBE_DEBUG 5774 /************************************************************************ 5775 * ixgbe_sysctl_power_state 5776 * 5777 * Sysctl to test power states 5778 * Values: 5779 * 0 - set device to D0 5780 * 3 - set device to D3 5781 * (none) - get current device power state 5782 ************************************************************************/ 5783 static int 5784 ixgbe_sysctl_power_state(SYSCTLFN_ARGS) 5785 { 5786 #ifdef notyet 5787 struct sysctlnode node = *rnode; 5788 struct adapter *adapter = (struct adapter *)node.sysctl_data; 5789 device_t dev = adapter->dev; 5790 int curr_ps, new_ps, error = 0; 5791 5792 if (ixgbe_fw_recovery_mode_swflag(adapter)) 5793 return (EPERM); 5794 5795 curr_ps = new_ps = pci_get_powerstate(dev); 5796 5797 error = sysctl_lookup(SYSCTLFN_CALL(&node)); 5798 if ((error) || (req->newp == NULL)) 5799 return (error); 5800 5801 if (new_ps == curr_ps) 5802 return (0); 5803 5804 if (new_ps == 3 && curr_ps == 0) 5805 error = DEVICE_SUSPEND(dev); 5806 else if (new_ps == 0 && curr_ps == 3) 5807 error = DEVICE_RESUME(dev); 5808 else 5809 return (EINVAL); 5810 5811 device_printf(dev, "New state: %d\n", pci_get_powerstate(dev)); 5812 5813 return (error); 5814 #else 5815 return 0; 5816 #endif 5817 } /* ixgbe_sysctl_power_state */ 5818 #endif 5819 5820 /************************************************************************ 5821 * ixgbe_sysctl_wol_enable 5822 * 5823 * Sysctl to enable/disable the WoL capability, 5824 * if supported by the adapter. 5825 * 5826 * Values: 5827 * 0 - disabled 5828 * 1 - enabled 5829 ************************************************************************/ 5830 static int 5831 ixgbe_sysctl_wol_enable(SYSCTLFN_ARGS) 5832 { 5833 struct sysctlnode node = *rnode; 5834 struct adapter *adapter = (struct adapter *)node.sysctl_data; 5835 struct ixgbe_hw *hw = &adapter->hw; 5836 bool new_wol_enabled; 5837 int error = 0; 5838 5839 /* 5840 * It's not required to check recovery mode because this function never 5841 * touches hardware. 5842 */ 5843 new_wol_enabled = hw->wol_enabled; 5844 node.sysctl_data = &new_wol_enabled; 5845 error = sysctl_lookup(SYSCTLFN_CALL(&node)); 5846 if ((error) || (newp == NULL)) 5847 return (error); 5848 if (new_wol_enabled == hw->wol_enabled) 5849 return (0); 5850 5851 if (new_wol_enabled && !adapter->wol_support) 5852 return (ENODEV); 5853 else 5854 hw->wol_enabled = new_wol_enabled; 5855 5856 return (0); 5857 } /* ixgbe_sysctl_wol_enable */ 5858 5859 /************************************************************************ 5860 * ixgbe_sysctl_wufc - Wake Up Filter Control 5861 * 5862 * Sysctl to enable/disable the types of packets that the 5863 * adapter will wake up on upon receipt. 5864 * Flags: 5865 * 0x1 - Link Status Change 5866 * 0x2 - Magic Packet 5867 * 0x4 - Direct Exact 5868 * 0x8 - Directed Multicast 5869 * 0x10 - Broadcast 5870 * 0x20 - ARP/IPv4 Request Packet 5871 * 0x40 - Direct IPv4 Packet 5872 * 0x80 - Direct IPv6 Packet 5873 * 5874 * Settings not listed above will cause the sysctl to return an error. 5875 ************************************************************************/ 5876 static int 5877 ixgbe_sysctl_wufc(SYSCTLFN_ARGS) 5878 { 5879 struct sysctlnode node = *rnode; 5880 struct adapter *adapter = (struct adapter *)node.sysctl_data; 5881 int error = 0; 5882 u32 new_wufc; 5883 5884 /* 5885 * It's not required to check recovery mode because this function never 5886 * touches hardware. 5887 */ 5888 new_wufc = adapter->wufc; 5889 node.sysctl_data = &new_wufc; 5890 error = sysctl_lookup(SYSCTLFN_CALL(&node)); 5891 if ((error) || (newp == NULL)) 5892 return (error); 5893 if (new_wufc == adapter->wufc) 5894 return (0); 5895 5896 if (new_wufc & 0xffffff00) 5897 return (EINVAL); 5898 5899 new_wufc &= 0xff; 5900 new_wufc |= (0xffffff & adapter->wufc); 5901 adapter->wufc = new_wufc; 5902 5903 return (0); 5904 } /* ixgbe_sysctl_wufc */ 5905 5906 #ifdef IXGBE_DEBUG 5907 /************************************************************************ 5908 * ixgbe_sysctl_print_rss_config 5909 ************************************************************************/ 5910 static int 5911 ixgbe_sysctl_print_rss_config(SYSCTLFN_ARGS) 5912 { 5913 #ifdef notyet 5914 struct sysctlnode node = *rnode; 5915 struct adapter *adapter = (struct adapter *)node.sysctl_data; 5916 struct ixgbe_hw *hw = &adapter->hw; 5917 device_t dev = adapter->dev; 5918 struct sbuf *buf; 5919 int error = 0, reta_size; 5920 u32 reg; 5921 5922 if (ixgbe_fw_recovery_mode_swflag(adapter)) 5923 return (EPERM); 5924 5925 buf = sbuf_new_for_sysctl(NULL, NULL, 128, req); 5926 if (!buf) { 5927 device_printf(dev, "Could not allocate sbuf for output.\n"); 5928 return (ENOMEM); 5929 } 5930 5931 // TODO: use sbufs to make a string to print out 5932 /* Set multiplier for RETA setup and table size based on MAC */ 5933 switch (adapter->hw.mac.type) { 5934 case ixgbe_mac_X550: 5935 case ixgbe_mac_X550EM_x: 5936 case ixgbe_mac_X550EM_a: 5937 reta_size = 128; 5938 break; 5939 default: 5940 reta_size = 32; 5941 break; 5942 } 5943 5944 /* Print out the redirection table */ 5945 sbuf_cat(buf, "\n"); 5946 for (int i = 0; i < reta_size; i++) { 5947 if (i < 32) { 5948 reg = IXGBE_READ_REG(hw, IXGBE_RETA(i)); 5949 sbuf_printf(buf, "RETA(%2d): 0x%08x\n", i, reg); 5950 } else { 5951 reg = IXGBE_READ_REG(hw, IXGBE_ERETA(i - 32)); 5952 sbuf_printf(buf, "ERETA(%2d): 0x%08x\n", i - 32, reg); 5953 } 5954 } 5955 5956 // TODO: print more config 5957 5958 error = sbuf_finish(buf); 5959 if (error) 5960 device_printf(dev, "Error finishing sbuf: %d\n", error); 5961 5962 sbuf_delete(buf); 5963 #endif 5964 return (0); 5965 } /* ixgbe_sysctl_print_rss_config */ 5966 #endif /* IXGBE_DEBUG */ 5967 5968 /************************************************************************ 5969 * ixgbe_sysctl_phy_temp - Retrieve temperature of PHY 5970 * 5971 * For X552/X557-AT devices using an external PHY 5972 ************************************************************************/ 5973 static int 5974 ixgbe_sysctl_phy_temp(SYSCTLFN_ARGS) 5975 { 5976 struct sysctlnode node = *rnode; 5977 struct adapter *adapter = (struct adapter *)node.sysctl_data; 5978 struct ixgbe_hw *hw = &adapter->hw; 5979 int val; 5980 u16 reg; 5981 int error; 5982 5983 if (ixgbe_fw_recovery_mode_swflag(adapter)) 5984 return (EPERM); 5985 5986 if ((hw->device_id != IXGBE_DEV_ID_X550EM_X_10G_T) && 5987 (hw->device_id != IXGBE_DEV_ID_X550EM_A_10G_T)) { 5988 device_printf(adapter->dev, 5989 "Device has no supported external thermal sensor.\n"); 5990 return (ENODEV); 5991 } 5992 5993 if (hw->phy.ops.read_reg(hw, IXGBE_PHY_CURRENT_TEMP, 5994 IXGBE_MDIO_VENDOR_SPECIFIC_1_DEV_TYPE, ®)) { 5995 device_printf(adapter->dev, 5996 "Error reading from PHY's current temperature register\n"); 5997 return (EAGAIN); 5998 } 5999 6000 node.sysctl_data = &val; 6001 6002 /* Shift temp for output */ 6003 val = reg >> 8; 6004 6005 error = sysctl_lookup(SYSCTLFN_CALL(&node)); 6006 if ((error) || (newp == NULL)) 6007 return (error); 6008 6009 return (0); 6010 } /* ixgbe_sysctl_phy_temp */ 6011 6012 /************************************************************************ 6013 * ixgbe_sysctl_phy_overtemp_occurred 6014 * 6015 * Reports (directly from the PHY) whether the current PHY 6016 * temperature is over the overtemp threshold. 6017 ************************************************************************/ 6018 static int 6019 ixgbe_sysctl_phy_overtemp_occurred(SYSCTLFN_ARGS) 6020 { 6021 struct sysctlnode node = *rnode; 6022 struct adapter *adapter = (struct adapter *)node.sysctl_data; 6023 struct ixgbe_hw *hw = &adapter->hw; 6024 int val, error; 6025 u16 reg; 6026 6027 if (ixgbe_fw_recovery_mode_swflag(adapter)) 6028 return (EPERM); 6029 6030 if ((hw->device_id != IXGBE_DEV_ID_X550EM_X_10G_T) && 6031 (hw->device_id != IXGBE_DEV_ID_X550EM_A_10G_T)){ 6032 device_printf(adapter->dev, 6033 "Device has no supported external thermal sensor.\n"); 6034 return (ENODEV); 6035 } 6036 6037 if (hw->phy.ops.read_reg(hw, IXGBE_PHY_OVERTEMP_STATUS, 6038 IXGBE_MDIO_VENDOR_SPECIFIC_1_DEV_TYPE, ®)) { 6039 device_printf(adapter->dev, 6040 "Error reading from PHY's temperature status register\n"); 6041 return (EAGAIN); 6042 } 6043 6044 node.sysctl_data = &val; 6045 6046 /* Get occurrence bit */ 6047 val = !!(reg & 0x4000); 6048 6049 error = sysctl_lookup(SYSCTLFN_CALL(&node)); 6050 if ((error) || (newp == NULL)) 6051 return (error); 6052 6053 return (0); 6054 } /* ixgbe_sysctl_phy_overtemp_occurred */ 6055 6056 /************************************************************************ 6057 * ixgbe_sysctl_eee_state 6058 * 6059 * Sysctl to set EEE power saving feature 6060 * Values: 6061 * 0 - disable EEE 6062 * 1 - enable EEE 6063 * (none) - get current device EEE state 6064 ************************************************************************/ 6065 static int 6066 ixgbe_sysctl_eee_state(SYSCTLFN_ARGS) 6067 { 6068 struct sysctlnode node = *rnode; 6069 struct adapter *adapter = (struct adapter *)node.sysctl_data; 6070 struct ifnet *ifp = adapter->ifp; 6071 device_t dev = adapter->dev; 6072 int curr_eee, new_eee, error = 0; 6073 s32 retval; 6074 6075 if (ixgbe_fw_recovery_mode_swflag(adapter)) 6076 return (EPERM); 6077 6078 curr_eee = new_eee = !!(adapter->feat_en & IXGBE_FEATURE_EEE); 6079 node.sysctl_data = &new_eee; 6080 error = sysctl_lookup(SYSCTLFN_CALL(&node)); 6081 if ((error) || (newp == NULL)) 6082 return (error); 6083 6084 /* Nothing to do */ 6085 if (new_eee == curr_eee) 6086 return (0); 6087 6088 /* Not supported */ 6089 if (!(adapter->feat_cap & IXGBE_FEATURE_EEE)) 6090 return (EINVAL); 6091 6092 /* Bounds checking */ 6093 if ((new_eee < 0) || (new_eee > 1)) 6094 return (EINVAL); 6095 6096 retval = ixgbe_setup_eee(&adapter->hw, new_eee); 6097 if (retval) { 6098 device_printf(dev, "Error in EEE setup: 0x%08X\n", retval); 6099 return (EINVAL); 6100 } 6101 6102 /* Restart auto-neg */ 6103 if_init(ifp); 6104 6105 device_printf(dev, "New EEE state: %d\n", new_eee); 6106 6107 /* Cache new value */ 6108 if (new_eee) 6109 adapter->feat_en |= IXGBE_FEATURE_EEE; 6110 else 6111 adapter->feat_en &= ~IXGBE_FEATURE_EEE; 6112 6113 return (error); 6114 } /* ixgbe_sysctl_eee_state */ 6115 6116 #define PRINTQS(adapter, regname) \ 6117 do { \ 6118 struct ixgbe_hw *_hw = &(adapter)->hw; \ 6119 int _i; \ 6120 \ 6121 printf("%s: %s", device_xname((adapter)->dev), #regname); \ 6122 for (_i = 0; _i < (adapter)->num_queues; _i++) { \ 6123 printf((_i == 0) ? "\t" : " "); \ 6124 printf("%08x", IXGBE_READ_REG(_hw, \ 6125 IXGBE_##regname(_i))); \ 6126 } \ 6127 printf("\n"); \ 6128 } while (0) 6129 6130 /************************************************************************ 6131 * ixgbe_print_debug_info 6132 * 6133 * Called only when em_display_debug_stats is enabled. 6134 * Provides a way to take a look at important statistics 6135 * maintained by the driver and hardware. 6136 ************************************************************************/ 6137 static void 6138 ixgbe_print_debug_info(struct adapter *adapter) 6139 { 6140 device_t dev = adapter->dev; 6141 struct ixgbe_hw *hw = &adapter->hw; 6142 int table_size; 6143 int i; 6144 6145 switch (adapter->hw.mac.type) { 6146 case ixgbe_mac_X550: 6147 case ixgbe_mac_X550EM_x: 6148 case ixgbe_mac_X550EM_a: 6149 table_size = 128; 6150 break; 6151 default: 6152 table_size = 32; 6153 break; 6154 } 6155 6156 device_printf(dev, "[E]RETA:\n"); 6157 for (i = 0; i < table_size; i++) { 6158 if (i < 32) 6159 printf("%02x: %08x\n", i, IXGBE_READ_REG(hw, 6160 IXGBE_RETA(i))); 6161 else 6162 printf("%02x: %08x\n", i, IXGBE_READ_REG(hw, 6163 IXGBE_ERETA(i - 32))); 6164 } 6165 6166 device_printf(dev, "queue:"); 6167 for (i = 0; i < adapter->num_queues; i++) { 6168 printf((i == 0) ? "\t" : " "); 6169 printf("%8d", i); 6170 } 6171 printf("\n"); 6172 PRINTQS(adapter, RDBAL); 6173 PRINTQS(adapter, RDBAH); 6174 PRINTQS(adapter, RDLEN); 6175 PRINTQS(adapter, SRRCTL); 6176 PRINTQS(adapter, RDH); 6177 PRINTQS(adapter, RDT); 6178 PRINTQS(adapter, RXDCTL); 6179 6180 device_printf(dev, "RQSMR:"); 6181 for (i = 0; i < adapter->num_queues / 4; i++) { 6182 printf((i == 0) ? "\t" : " "); 6183 printf("%08x", IXGBE_READ_REG(hw, IXGBE_RQSMR(i))); 6184 } 6185 printf("\n"); 6186 6187 device_printf(dev, "disabled_count:"); 6188 for (i = 0; i < adapter->num_queues; i++) { 6189 printf((i == 0) ? "\t" : " "); 6190 printf("%8d", adapter->queues[i].disabled_count); 6191 } 6192 printf("\n"); 6193 6194 device_printf(dev, "EIMS:\t%08x\n", IXGBE_READ_REG(hw, IXGBE_EIMS)); 6195 if (hw->mac.type != ixgbe_mac_82598EB) { 6196 device_printf(dev, "EIMS_EX(0):\t%08x\n", 6197 IXGBE_READ_REG(hw, IXGBE_EIMS_EX(0))); 6198 device_printf(dev, "EIMS_EX(1):\t%08x\n", 6199 IXGBE_READ_REG(hw, IXGBE_EIMS_EX(1))); 6200 } 6201 device_printf(dev, "EIAM:\t%08x\n", IXGBE_READ_REG(hw, IXGBE_EIAM)); 6202 device_printf(dev, "EIAC:\t%08x\n", IXGBE_READ_REG(hw, IXGBE_EIAC)); 6203 } /* ixgbe_print_debug_info */ 6204 6205 /************************************************************************ 6206 * ixgbe_sysctl_debug 6207 ************************************************************************/ 6208 static int 6209 ixgbe_sysctl_debug(SYSCTLFN_ARGS) 6210 { 6211 struct sysctlnode node = *rnode; 6212 struct adapter *adapter = (struct adapter *)node.sysctl_data; 6213 int error, result = 0; 6214 6215 if (ixgbe_fw_recovery_mode_swflag(adapter)) 6216 return (EPERM); 6217 6218 node.sysctl_data = &result; 6219 error = sysctl_lookup(SYSCTLFN_CALL(&node)); 6220 6221 if (error || newp == NULL) 6222 return error; 6223 6224 if (result == 1) 6225 ixgbe_print_debug_info(adapter); 6226 6227 return 0; 6228 } /* ixgbe_sysctl_debug */ 6229 6230 /************************************************************************ 6231 * ixgbe_sysctl_rx_copy_len 6232 ************************************************************************/ 6233 static int 6234 ixgbe_sysctl_rx_copy_len(SYSCTLFN_ARGS) 6235 { 6236 struct sysctlnode node = *rnode; 6237 struct adapter *adapter = (struct adapter *)node.sysctl_data; 6238 int error; 6239 int result = adapter->rx_copy_len; 6240 6241 node.sysctl_data = &result; 6242 error = sysctl_lookup(SYSCTLFN_CALL(&node)); 6243 6244 if (error || newp == NULL) 6245 return error; 6246 6247 if ((result < 0) || (result > IXGBE_RX_COPY_LEN_MAX)) 6248 return EINVAL; 6249 6250 adapter->rx_copy_len = result; 6251 6252 return 0; 6253 } /* ixgbe_sysctl_rx_copy_len */ 6254 6255 /************************************************************************ 6256 * ixgbe_sysctl_tx_process_limit 6257 ************************************************************************/ 6258 static int 6259 ixgbe_sysctl_tx_process_limit(SYSCTLFN_ARGS) 6260 { 6261 struct sysctlnode node = *rnode; 6262 struct adapter *adapter = (struct adapter *)node.sysctl_data; 6263 int error; 6264 int result = adapter->tx_process_limit; 6265 6266 node.sysctl_data = &result; 6267 error = sysctl_lookup(SYSCTLFN_CALL(&node)); 6268 6269 if (error || newp == NULL) 6270 return error; 6271 6272 if ((result <= 0) || (result > adapter->num_tx_desc)) 6273 return EINVAL; 6274 6275 adapter->tx_process_limit = result; 6276 6277 return 0; 6278 } /* ixgbe_sysctl_tx_process_limit */ 6279 6280 /************************************************************************ 6281 * ixgbe_sysctl_rx_process_limit 6282 ************************************************************************/ 6283 static int 6284 ixgbe_sysctl_rx_process_limit(SYSCTLFN_ARGS) 6285 { 6286 struct sysctlnode node = *rnode; 6287 struct adapter *adapter = (struct adapter *)node.sysctl_data; 6288 int error; 6289 int result = adapter->rx_process_limit; 6290 6291 node.sysctl_data = &result; 6292 error = sysctl_lookup(SYSCTLFN_CALL(&node)); 6293 6294 if (error || newp == NULL) 6295 return error; 6296 6297 if ((result <= 0) || (result > adapter->num_rx_desc)) 6298 return EINVAL; 6299 6300 adapter->rx_process_limit = result; 6301 6302 return 0; 6303 } /* ixgbe_sysctl_rx_process_limit */ 6304 6305 /************************************************************************ 6306 * ixgbe_init_device_features 6307 ************************************************************************/ 6308 static void 6309 ixgbe_init_device_features(struct adapter *adapter) 6310 { 6311 adapter->feat_cap = IXGBE_FEATURE_NETMAP 6312 | IXGBE_FEATURE_RSS 6313 | IXGBE_FEATURE_MSI 6314 | IXGBE_FEATURE_MSIX 6315 | IXGBE_FEATURE_LEGACY_IRQ 6316 | IXGBE_FEATURE_LEGACY_TX; 6317 6318 /* Set capabilities first... */ 6319 switch (adapter->hw.mac.type) { 6320 case ixgbe_mac_82598EB: 6321 if (adapter->hw.device_id == IXGBE_DEV_ID_82598AT) 6322 adapter->feat_cap |= IXGBE_FEATURE_FAN_FAIL; 6323 break; 6324 case ixgbe_mac_X540: 6325 adapter->feat_cap |= IXGBE_FEATURE_SRIOV; 6326 adapter->feat_cap |= IXGBE_FEATURE_FDIR; 6327 if ((adapter->hw.device_id == IXGBE_DEV_ID_X540_BYPASS) && 6328 (adapter->hw.bus.func == 0)) 6329 adapter->feat_cap |= IXGBE_FEATURE_BYPASS; 6330 break; 6331 case ixgbe_mac_X550: 6332 /* 6333 * IXGBE_FEATURE_RECOVERY_MODE will be set after reading 6334 * NVM Image version. 6335 */ 6336 adapter->feat_cap |= IXGBE_FEATURE_TEMP_SENSOR; 6337 adapter->feat_cap |= IXGBE_FEATURE_SRIOV; 6338 adapter->feat_cap |= IXGBE_FEATURE_FDIR; 6339 break; 6340 case ixgbe_mac_X550EM_x: 6341 /* 6342 * IXGBE_FEATURE_RECOVERY_MODE will be set after reading 6343 * NVM Image version. 6344 */ 6345 adapter->feat_cap |= IXGBE_FEATURE_SRIOV; 6346 adapter->feat_cap |= IXGBE_FEATURE_FDIR; 6347 break; 6348 case ixgbe_mac_X550EM_a: 6349 /* 6350 * IXGBE_FEATURE_RECOVERY_MODE will be set after reading 6351 * NVM Image version. 6352 */ 6353 adapter->feat_cap |= IXGBE_FEATURE_SRIOV; 6354 adapter->feat_cap |= IXGBE_FEATURE_FDIR; 6355 adapter->feat_cap &= ~IXGBE_FEATURE_LEGACY_IRQ; 6356 if ((adapter->hw.device_id == IXGBE_DEV_ID_X550EM_A_1G_T) || 6357 (adapter->hw.device_id == IXGBE_DEV_ID_X550EM_A_1G_T_L)) { 6358 adapter->feat_cap |= IXGBE_FEATURE_TEMP_SENSOR; 6359 adapter->feat_cap |= IXGBE_FEATURE_EEE; 6360 } 6361 break; 6362 case ixgbe_mac_82599EB: 6363 adapter->feat_cap |= IXGBE_FEATURE_SRIOV; 6364 adapter->feat_cap |= IXGBE_FEATURE_FDIR; 6365 if ((adapter->hw.device_id == IXGBE_DEV_ID_82599_BYPASS) && 6366 (adapter->hw.bus.func == 0)) 6367 adapter->feat_cap |= IXGBE_FEATURE_BYPASS; 6368 if (adapter->hw.device_id == IXGBE_DEV_ID_82599_QSFP_SF_QP) 6369 adapter->feat_cap &= ~IXGBE_FEATURE_LEGACY_IRQ; 6370 break; 6371 default: 6372 break; 6373 } 6374 6375 /* Enabled by default... */ 6376 /* Fan failure detection */ 6377 if (adapter->feat_cap & IXGBE_FEATURE_FAN_FAIL) 6378 adapter->feat_en |= IXGBE_FEATURE_FAN_FAIL; 6379 /* Netmap */ 6380 if (adapter->feat_cap & IXGBE_FEATURE_NETMAP) 6381 adapter->feat_en |= IXGBE_FEATURE_NETMAP; 6382 /* EEE */ 6383 if (adapter->feat_cap & IXGBE_FEATURE_EEE) 6384 adapter->feat_en |= IXGBE_FEATURE_EEE; 6385 /* Thermal Sensor */ 6386 if (adapter->feat_cap & IXGBE_FEATURE_TEMP_SENSOR) 6387 adapter->feat_en |= IXGBE_FEATURE_TEMP_SENSOR; 6388 /* 6389 * Recovery mode: 6390 * NetBSD: IXGBE_FEATURE_RECOVERY_MODE will be controlled after reading 6391 * NVM Image version. 6392 */ 6393 6394 /* Enabled via global sysctl... */ 6395 /* Flow Director */ 6396 if (ixgbe_enable_fdir) { 6397 if (adapter->feat_cap & IXGBE_FEATURE_FDIR) 6398 adapter->feat_en |= IXGBE_FEATURE_FDIR; 6399 else 6400 device_printf(adapter->dev, "Device does not support " 6401 "Flow Director. Leaving disabled."); 6402 } 6403 /* Legacy (single queue) transmit */ 6404 if ((adapter->feat_cap & IXGBE_FEATURE_LEGACY_TX) && 6405 ixgbe_enable_legacy_tx) 6406 adapter->feat_en |= IXGBE_FEATURE_LEGACY_TX; 6407 /* 6408 * Message Signal Interrupts - Extended (MSI-X) 6409 * Normal MSI is only enabled if MSI-X calls fail. 6410 */ 6411 if (!ixgbe_enable_msix) 6412 adapter->feat_cap &= ~IXGBE_FEATURE_MSIX; 6413 /* Receive-Side Scaling (RSS) */ 6414 if ((adapter->feat_cap & IXGBE_FEATURE_RSS) && ixgbe_enable_rss) 6415 adapter->feat_en |= IXGBE_FEATURE_RSS; 6416 6417 /* Disable features with unmet dependencies... */ 6418 /* No MSI-X */ 6419 if (!(adapter->feat_cap & IXGBE_FEATURE_MSIX)) { 6420 adapter->feat_cap &= ~IXGBE_FEATURE_RSS; 6421 adapter->feat_cap &= ~IXGBE_FEATURE_SRIOV; 6422 adapter->feat_en &= ~IXGBE_FEATURE_RSS; 6423 adapter->feat_en &= ~IXGBE_FEATURE_SRIOV; 6424 } 6425 } /* ixgbe_init_device_features */ 6426 6427 /************************************************************************ 6428 * ixgbe_probe - Device identification routine 6429 * 6430 * Determines if the driver should be loaded on 6431 * adapter based on its PCI vendor/device ID. 6432 * 6433 * return BUS_PROBE_DEFAULT on success, positive on failure 6434 ************************************************************************/ 6435 static int 6436 ixgbe_probe(device_t dev, cfdata_t cf, void *aux) 6437 { 6438 const struct pci_attach_args *pa = aux; 6439 6440 return (ixgbe_lookup(pa) != NULL) ? 1 : 0; 6441 } 6442 6443 static const ixgbe_vendor_info_t * 6444 ixgbe_lookup(const struct pci_attach_args *pa) 6445 { 6446 const ixgbe_vendor_info_t *ent; 6447 pcireg_t subid; 6448 6449 INIT_DEBUGOUT("ixgbe_lookup: begin"); 6450 6451 if (PCI_VENDOR(pa->pa_id) != IXGBE_INTEL_VENDOR_ID) 6452 return NULL; 6453 6454 subid = pci_conf_read(pa->pa_pc, pa->pa_tag, PCI_SUBSYS_ID_REG); 6455 6456 for (ent = ixgbe_vendor_info_array; ent->vendor_id != 0; ent++) { 6457 if ((PCI_VENDOR(pa->pa_id) == ent->vendor_id) && 6458 (PCI_PRODUCT(pa->pa_id) == ent->device_id) && 6459 ((PCI_SUBSYS_VENDOR(subid) == ent->subvendor_id) || 6460 (ent->subvendor_id == 0)) && 6461 ((PCI_SUBSYS_ID(subid) == ent->subdevice_id) || 6462 (ent->subdevice_id == 0))) { 6463 return ent; 6464 } 6465 } 6466 return NULL; 6467 } 6468 6469 static int 6470 ixgbe_ifflags_cb(struct ethercom *ec) 6471 { 6472 struct ifnet *ifp = &ec->ec_if; 6473 struct adapter *adapter = ifp->if_softc; 6474 u_short change; 6475 int rv = 0; 6476 6477 IXGBE_CORE_LOCK(adapter); 6478 6479 change = ifp->if_flags ^ adapter->if_flags; 6480 if (change != 0) 6481 adapter->if_flags = ifp->if_flags; 6482 6483 if ((change & ~(IFF_CANTCHANGE | IFF_DEBUG)) != 0) { 6484 rv = ENETRESET; 6485 goto out; 6486 } else if ((change & IFF_PROMISC) != 0) 6487 ixgbe_set_rxfilter(adapter); 6488 6489 /* Check for ec_capenable. */ 6490 change = ec->ec_capenable ^ adapter->ec_capenable; 6491 adapter->ec_capenable = ec->ec_capenable; 6492 if ((change & ~(ETHERCAP_VLAN_MTU | ETHERCAP_VLAN_HWTAGGING 6493 | ETHERCAP_VLAN_HWFILTER)) != 0) { 6494 rv = ENETRESET; 6495 goto out; 6496 } 6497 6498 /* 6499 * Special handling is not required for ETHERCAP_VLAN_MTU. 6500 * MAXFRS(MHADD) does not include the 4bytes of the VLAN header. 6501 */ 6502 6503 /* Set up VLAN support and filter */ 6504 if ((change & (ETHERCAP_VLAN_HWTAGGING | ETHERCAP_VLAN_HWFILTER)) != 0) 6505 ixgbe_setup_vlan_hw_support(adapter); 6506 6507 out: 6508 IXGBE_CORE_UNLOCK(adapter); 6509 6510 return rv; 6511 } 6512 6513 /************************************************************************ 6514 * ixgbe_ioctl - Ioctl entry point 6515 * 6516 * Called when the user wants to configure the interface. 6517 * 6518 * return 0 on success, positive on failure 6519 ************************************************************************/ 6520 static int 6521 ixgbe_ioctl(struct ifnet *ifp, u_long command, void *data) 6522 { 6523 struct adapter *adapter = ifp->if_softc; 6524 struct ixgbe_hw *hw = &adapter->hw; 6525 struct ifcapreq *ifcr = data; 6526 struct ifreq *ifr = data; 6527 int error = 0; 6528 int l4csum_en; 6529 const int l4csum = IFCAP_CSUM_TCPv4_Rx | IFCAP_CSUM_UDPv4_Rx | 6530 IFCAP_CSUM_TCPv6_Rx | IFCAP_CSUM_UDPv6_Rx; 6531 6532 if (ixgbe_fw_recovery_mode_swflag(adapter)) 6533 return (EPERM); 6534 6535 switch (command) { 6536 case SIOCSIFFLAGS: 6537 IOCTL_DEBUGOUT("ioctl: SIOCSIFFLAGS (Set Interface Flags)"); 6538 break; 6539 case SIOCADDMULTI: 6540 case SIOCDELMULTI: 6541 IOCTL_DEBUGOUT("ioctl: SIOC(ADD|DEL)MULTI"); 6542 break; 6543 case SIOCSIFMEDIA: 6544 case SIOCGIFMEDIA: 6545 IOCTL_DEBUGOUT("ioctl: SIOCxIFMEDIA (Get/Set Interface Media)"); 6546 break; 6547 case SIOCSIFCAP: 6548 IOCTL_DEBUGOUT("ioctl: SIOCSIFCAP (Set Capabilities)"); 6549 break; 6550 case SIOCSIFMTU: 6551 IOCTL_DEBUGOUT("ioctl: SIOCSIFMTU (Set Interface MTU)"); 6552 break; 6553 #ifdef __NetBSD__ 6554 case SIOCINITIFADDR: 6555 IOCTL_DEBUGOUT("ioctl: SIOCINITIFADDR"); 6556 break; 6557 case SIOCGIFFLAGS: 6558 IOCTL_DEBUGOUT("ioctl: SIOCGIFFLAGS"); 6559 break; 6560 case SIOCGIFAFLAG_IN: 6561 IOCTL_DEBUGOUT("ioctl: SIOCGIFAFLAG_IN"); 6562 break; 6563 case SIOCGIFADDR: 6564 IOCTL_DEBUGOUT("ioctl: SIOCGIFADDR"); 6565 break; 6566 case SIOCGIFMTU: 6567 IOCTL_DEBUGOUT("ioctl: SIOCGIFMTU (Get Interface MTU)"); 6568 break; 6569 case SIOCGIFCAP: 6570 IOCTL_DEBUGOUT("ioctl: SIOCGIFCAP (Get IF cap)"); 6571 break; 6572 case SIOCGETHERCAP: 6573 IOCTL_DEBUGOUT("ioctl: SIOCGETHERCAP (Get ethercap)"); 6574 break; 6575 case SIOCGLIFADDR: 6576 IOCTL_DEBUGOUT("ioctl: SIOCGLIFADDR (Get Interface addr)"); 6577 break; 6578 case SIOCZIFDATA: 6579 IOCTL_DEBUGOUT("ioctl: SIOCZIFDATA (Zero counter)"); 6580 hw->mac.ops.clear_hw_cntrs(hw); 6581 ixgbe_clear_evcnt(adapter); 6582 break; 6583 case SIOCAIFADDR: 6584 IOCTL_DEBUGOUT("ioctl: SIOCAIFADDR (add/chg IF alias)"); 6585 break; 6586 #endif 6587 default: 6588 IOCTL_DEBUGOUT1("ioctl: UNKNOWN (0x%X)", (int)command); 6589 break; 6590 } 6591 6592 switch (command) { 6593 case SIOCGI2C: 6594 { 6595 struct ixgbe_i2c_req i2c; 6596 6597 IOCTL_DEBUGOUT("ioctl: SIOCGI2C (Get I2C Data)"); 6598 error = copyin(ifr->ifr_data, &i2c, sizeof(i2c)); 6599 if (error != 0) 6600 break; 6601 if (i2c.dev_addr != 0xA0 && i2c.dev_addr != 0xA2) { 6602 error = EINVAL; 6603 break; 6604 } 6605 if (i2c.len > sizeof(i2c.data)) { 6606 error = EINVAL; 6607 break; 6608 } 6609 6610 hw->phy.ops.read_i2c_byte(hw, i2c.offset, 6611 i2c.dev_addr, i2c.data); 6612 error = copyout(&i2c, ifr->ifr_data, sizeof(i2c)); 6613 break; 6614 } 6615 case SIOCSIFCAP: 6616 /* Layer-4 Rx checksum offload has to be turned on and 6617 * off as a unit. 6618 */ 6619 l4csum_en = ifcr->ifcr_capenable & l4csum; 6620 if (l4csum_en != l4csum && l4csum_en != 0) 6621 return EINVAL; 6622 /*FALLTHROUGH*/ 6623 case SIOCADDMULTI: 6624 case SIOCDELMULTI: 6625 case SIOCSIFFLAGS: 6626 case SIOCSIFMTU: 6627 default: 6628 if ((error = ether_ioctl(ifp, command, data)) != ENETRESET) 6629 return error; 6630 if ((ifp->if_flags & IFF_RUNNING) == 0) 6631 ; 6632 else if (command == SIOCSIFCAP || command == SIOCSIFMTU) { 6633 IXGBE_CORE_LOCK(adapter); 6634 if ((ifp->if_flags & IFF_RUNNING) != 0) 6635 ixgbe_init_locked(adapter); 6636 ixgbe_recalculate_max_frame(adapter); 6637 IXGBE_CORE_UNLOCK(adapter); 6638 } else if (command == SIOCADDMULTI || command == SIOCDELMULTI) { 6639 /* 6640 * Multicast list has changed; set the hardware filter 6641 * accordingly. 6642 */ 6643 IXGBE_CORE_LOCK(adapter); 6644 ixgbe_disable_intr(adapter); 6645 ixgbe_set_rxfilter(adapter); 6646 ixgbe_enable_intr(adapter); 6647 IXGBE_CORE_UNLOCK(adapter); 6648 } 6649 return 0; 6650 } 6651 6652 return error; 6653 } /* ixgbe_ioctl */ 6654 6655 /************************************************************************ 6656 * ixgbe_check_fan_failure 6657 ************************************************************************/ 6658 static int 6659 ixgbe_check_fan_failure(struct adapter *adapter, u32 reg, bool in_interrupt) 6660 { 6661 u32 mask; 6662 6663 mask = (in_interrupt) ? IXGBE_EICR_GPI_SDP1_BY_MAC(&adapter->hw) : 6664 IXGBE_ESDP_SDP1; 6665 6666 if ((reg & mask) == 0) 6667 return IXGBE_SUCCESS; 6668 6669 /* 6670 * Use ratecheck() just in case interrupt occur frequently. 6671 * When EXPX9501AT's fan stopped, interrupt occurred only once, 6672 * an red LED on the board turned on and link never up until 6673 * power off. 6674 */ 6675 if (ratecheck(&adapter->lasterr_time, &ixgbe_errlog_intrvl)) 6676 device_printf(adapter->dev, 6677 "\nCRITICAL: FAN FAILURE!! REPLACE IMMEDIATELY!!\n"); 6678 6679 return IXGBE_ERR_FAN_FAILURE; 6680 } /* ixgbe_check_fan_failure */ 6681 6682 /************************************************************************ 6683 * ixgbe_handle_que 6684 ************************************************************************/ 6685 static void 6686 ixgbe_handle_que(void *context) 6687 { 6688 struct ix_queue *que = context; 6689 struct adapter *adapter = que->adapter; 6690 struct tx_ring *txr = que->txr; 6691 struct ifnet *ifp = adapter->ifp; 6692 bool more = false; 6693 6694 IXGBE_EVC_ADD(&que->handleq, 1); 6695 6696 if (ifp->if_flags & IFF_RUNNING) { 6697 IXGBE_TX_LOCK(txr); 6698 more = ixgbe_txeof(txr); 6699 if (!(adapter->feat_en & IXGBE_FEATURE_LEGACY_TX)) 6700 if (!ixgbe_mq_ring_empty(ifp, txr->txr_interq)) 6701 ixgbe_mq_start_locked(ifp, txr); 6702 /* Only for queue 0 */ 6703 /* NetBSD still needs this for CBQ */ 6704 if ((&adapter->queues[0] == que) 6705 && (!ixgbe_legacy_ring_empty(ifp, NULL))) 6706 ixgbe_legacy_start_locked(ifp, txr); 6707 IXGBE_TX_UNLOCK(txr); 6708 more |= ixgbe_rxeof(que); 6709 } 6710 6711 if (more) { 6712 IXGBE_EVC_ADD(&que->req, 1); 6713 ixgbe_sched_handle_que(adapter, que); 6714 } else if (que->res != NULL) { 6715 /* MSIX: Re-enable this interrupt */ 6716 ixgbe_enable_queue(adapter, que->msix); 6717 } else { 6718 /* INTx or MSI */ 6719 ixgbe_enable_queue(adapter, 0); 6720 } 6721 6722 return; 6723 } /* ixgbe_handle_que */ 6724 6725 /************************************************************************ 6726 * ixgbe_handle_que_work 6727 ************************************************************************/ 6728 static void 6729 ixgbe_handle_que_work(struct work *wk, void *context) 6730 { 6731 struct ix_queue *que = container_of(wk, struct ix_queue, wq_cookie); 6732 6733 /* 6734 * "enqueued flag" is not required here. 6735 * See ixgbe_msix_que(). 6736 */ 6737 ixgbe_handle_que(que); 6738 } 6739 6740 /************************************************************************ 6741 * ixgbe_allocate_legacy - Setup the Legacy or MSI Interrupt handler 6742 ************************************************************************/ 6743 static int 6744 ixgbe_allocate_legacy(struct adapter *adapter, 6745 const struct pci_attach_args *pa) 6746 { 6747 device_t dev = adapter->dev; 6748 struct ix_queue *que = adapter->queues; 6749 struct tx_ring *txr = adapter->tx_rings; 6750 int counts[PCI_INTR_TYPE_SIZE]; 6751 pci_intr_type_t intr_type, max_type; 6752 char intrbuf[PCI_INTRSTR_LEN]; 6753 char wqname[MAXCOMLEN]; 6754 const char *intrstr = NULL; 6755 int defertx_error = 0, error; 6756 6757 /* We allocate a single interrupt resource */ 6758 max_type = PCI_INTR_TYPE_MSI; 6759 counts[PCI_INTR_TYPE_MSIX] = 0; 6760 counts[PCI_INTR_TYPE_MSI] = 6761 (adapter->feat_en & IXGBE_FEATURE_MSI) ? 1 : 0; 6762 /* Check not feat_en but feat_cap to fallback to INTx */ 6763 counts[PCI_INTR_TYPE_INTX] = 6764 (adapter->feat_cap & IXGBE_FEATURE_LEGACY_IRQ) ? 1 : 0; 6765 6766 alloc_retry: 6767 if (pci_intr_alloc(pa, &adapter->osdep.intrs, counts, max_type) != 0) { 6768 aprint_error_dev(dev, "couldn't alloc interrupt\n"); 6769 return ENXIO; 6770 } 6771 adapter->osdep.nintrs = 1; 6772 intrstr = pci_intr_string(adapter->osdep.pc, adapter->osdep.intrs[0], 6773 intrbuf, sizeof(intrbuf)); 6774 adapter->osdep.ihs[0] = pci_intr_establish_xname(adapter->osdep.pc, 6775 adapter->osdep.intrs[0], IPL_NET, ixgbe_legacy_irq, que, 6776 device_xname(dev)); 6777 intr_type = pci_intr_type(adapter->osdep.pc, adapter->osdep.intrs[0]); 6778 if (adapter->osdep.ihs[0] == NULL) { 6779 aprint_error_dev(dev,"unable to establish %s\n", 6780 (intr_type == PCI_INTR_TYPE_MSI) ? "MSI" : "INTx"); 6781 pci_intr_release(adapter->osdep.pc, adapter->osdep.intrs, 1); 6782 adapter->osdep.intrs = NULL; 6783 switch (intr_type) { 6784 case PCI_INTR_TYPE_MSI: 6785 /* The next try is for INTx: Disable MSI */ 6786 max_type = PCI_INTR_TYPE_INTX; 6787 counts[PCI_INTR_TYPE_INTX] = 1; 6788 adapter->feat_en &= ~IXGBE_FEATURE_MSI; 6789 if (adapter->feat_cap & IXGBE_FEATURE_LEGACY_IRQ) { 6790 adapter->feat_en |= IXGBE_FEATURE_LEGACY_IRQ; 6791 goto alloc_retry; 6792 } else 6793 break; 6794 case PCI_INTR_TYPE_INTX: 6795 default: 6796 /* See below */ 6797 break; 6798 } 6799 } 6800 if (intr_type == PCI_INTR_TYPE_INTX) { 6801 adapter->feat_en &= ~IXGBE_FEATURE_MSI; 6802 adapter->feat_en |= IXGBE_FEATURE_LEGACY_IRQ; 6803 } 6804 if (adapter->osdep.ihs[0] == NULL) { 6805 aprint_error_dev(dev, 6806 "couldn't establish interrupt%s%s\n", 6807 intrstr ? " at " : "", intrstr ? intrstr : ""); 6808 pci_intr_release(adapter->osdep.pc, adapter->osdep.intrs, 1); 6809 adapter->osdep.intrs = NULL; 6810 return ENXIO; 6811 } 6812 aprint_normal_dev(dev, "interrupting at %s\n", intrstr); 6813 /* 6814 * Try allocating a fast interrupt and the associated deferred 6815 * processing contexts. 6816 */ 6817 if (!(adapter->feat_en & IXGBE_FEATURE_LEGACY_TX)) { 6818 txr->txr_si = 6819 softint_establish(SOFTINT_NET | IXGBE_SOFTINT_FLAGS, 6820 ixgbe_deferred_mq_start, txr); 6821 6822 snprintf(wqname, sizeof(wqname), "%sdeferTx", 6823 device_xname(dev)); 6824 defertx_error = workqueue_create(&adapter->txr_wq, wqname, 6825 ixgbe_deferred_mq_start_work, adapter, IXGBE_WORKQUEUE_PRI, 6826 IPL_NET, IXGBE_WORKQUEUE_FLAGS); 6827 adapter->txr_wq_enqueued = percpu_alloc(sizeof(u_int)); 6828 } 6829 que->que_si = softint_establish(SOFTINT_NET | IXGBE_SOFTINT_FLAGS, 6830 ixgbe_handle_que, que); 6831 snprintf(wqname, sizeof(wqname), "%sTxRx", device_xname(dev)); 6832 error = workqueue_create(&adapter->que_wq, wqname, 6833 ixgbe_handle_que_work, adapter, IXGBE_WORKQUEUE_PRI, IPL_NET, 6834 IXGBE_WORKQUEUE_FLAGS); 6835 6836 if ((!(adapter->feat_en & IXGBE_FEATURE_LEGACY_TX) 6837 && ((txr->txr_si == NULL) || defertx_error != 0)) 6838 || (que->que_si == NULL) || error != 0) { 6839 aprint_error_dev(dev, 6840 "could not establish software interrupts\n"); 6841 6842 return ENXIO; 6843 } 6844 /* For simplicity in the handlers */ 6845 adapter->active_queues = IXGBE_EIMS_ENABLE_MASK; 6846 6847 return (0); 6848 } /* ixgbe_allocate_legacy */ 6849 6850 /************************************************************************ 6851 * ixgbe_allocate_msix - Setup MSI-X Interrupt resources and handlers 6852 ************************************************************************/ 6853 static int 6854 ixgbe_allocate_msix(struct adapter *adapter, const struct pci_attach_args *pa) 6855 { 6856 device_t dev = adapter->dev; 6857 struct ix_queue *que = adapter->queues; 6858 struct tx_ring *txr = adapter->tx_rings; 6859 pci_chipset_tag_t pc; 6860 char intrbuf[PCI_INTRSTR_LEN]; 6861 char intr_xname[32]; 6862 char wqname[MAXCOMLEN]; 6863 const char *intrstr = NULL; 6864 int error, vector = 0; 6865 int cpu_id = 0; 6866 kcpuset_t *affinity; 6867 #ifdef RSS 6868 unsigned int rss_buckets = 0; 6869 kcpuset_t cpu_mask; 6870 #endif 6871 6872 pc = adapter->osdep.pc; 6873 #ifdef RSS 6874 /* 6875 * If we're doing RSS, the number of queues needs to 6876 * match the number of RSS buckets that are configured. 6877 * 6878 * + If there's more queues than RSS buckets, we'll end 6879 * up with queues that get no traffic. 6880 * 6881 * + If there's more RSS buckets than queues, we'll end 6882 * up having multiple RSS buckets map to the same queue, 6883 * so there'll be some contention. 6884 */ 6885 rss_buckets = rss_getnumbuckets(); 6886 if ((adapter->feat_en & IXGBE_FEATURE_RSS) && 6887 (adapter->num_queues != rss_buckets)) { 6888 device_printf(dev, 6889 "%s: number of queues (%d) != number of RSS buckets (%d)" 6890 "; performance will be impacted.\n", 6891 __func__, adapter->num_queues, rss_buckets); 6892 } 6893 #endif 6894 6895 adapter->osdep.nintrs = adapter->num_queues + 1; 6896 if (pci_msix_alloc_exact(pa, &adapter->osdep.intrs, 6897 adapter->osdep.nintrs) != 0) { 6898 aprint_error_dev(dev, 6899 "failed to allocate MSI-X interrupt\n"); 6900 adapter->feat_en &= ~IXGBE_FEATURE_MSIX; 6901 return (ENXIO); 6902 } 6903 6904 kcpuset_create(&affinity, false); 6905 for (int i = 0; i < adapter->num_queues; i++, vector++, que++, txr++) { 6906 snprintf(intr_xname, sizeof(intr_xname), "%s TXRX%d", 6907 device_xname(dev), i); 6908 intrstr = pci_intr_string(pc, adapter->osdep.intrs[i], intrbuf, 6909 sizeof(intrbuf)); 6910 #ifdef IXGBE_MPSAFE 6911 pci_intr_setattr(pc, &adapter->osdep.intrs[i], PCI_INTR_MPSAFE, 6912 true); 6913 #endif 6914 /* Set the handler function */ 6915 que->res = adapter->osdep.ihs[i] = pci_intr_establish_xname(pc, 6916 adapter->osdep.intrs[i], IPL_NET, ixgbe_msix_que, que, 6917 intr_xname); 6918 if (que->res == NULL) { 6919 aprint_error_dev(dev, 6920 "Failed to register QUE handler\n"); 6921 error = ENXIO; 6922 goto err_out; 6923 } 6924 que->msix = vector; 6925 adapter->active_queues |= 1ULL << que->msix; 6926 6927 if (adapter->feat_en & IXGBE_FEATURE_RSS) { 6928 #ifdef RSS 6929 /* 6930 * The queue ID is used as the RSS layer bucket ID. 6931 * We look up the queue ID -> RSS CPU ID and select 6932 * that. 6933 */ 6934 cpu_id = rss_getcpu(i % rss_getnumbuckets()); 6935 CPU_SETOF(cpu_id, &cpu_mask); 6936 #endif 6937 } else { 6938 /* 6939 * Bind the MSI-X vector, and thus the 6940 * rings to the corresponding CPU. 6941 * 6942 * This just happens to match the default RSS 6943 * round-robin bucket -> queue -> CPU allocation. 6944 */ 6945 if (adapter->num_queues > 1) 6946 cpu_id = i; 6947 } 6948 /* Round-robin affinity */ 6949 kcpuset_zero(affinity); 6950 kcpuset_set(affinity, cpu_id % ncpu); 6951 error = interrupt_distribute(adapter->osdep.ihs[i], affinity, 6952 NULL); 6953 aprint_normal_dev(dev, "for TX/RX, interrupting at %s", 6954 intrstr); 6955 if (error == 0) { 6956 #if 1 /* def IXGBE_DEBUG */ 6957 #ifdef RSS 6958 aprint_normal(", bound RSS bucket %d to CPU %d", i, 6959 cpu_id % ncpu); 6960 #else 6961 aprint_normal(", bound queue %d to cpu %d", i, 6962 cpu_id % ncpu); 6963 #endif 6964 #endif /* IXGBE_DEBUG */ 6965 } 6966 aprint_normal("\n"); 6967 6968 if (!(adapter->feat_en & IXGBE_FEATURE_LEGACY_TX)) { 6969 txr->txr_si = softint_establish( 6970 SOFTINT_NET | IXGBE_SOFTINT_FLAGS, 6971 ixgbe_deferred_mq_start, txr); 6972 if (txr->txr_si == NULL) { 6973 aprint_error_dev(dev, 6974 "couldn't establish software interrupt\n"); 6975 error = ENXIO; 6976 goto err_out; 6977 } 6978 } 6979 que->que_si 6980 = softint_establish(SOFTINT_NET | IXGBE_SOFTINT_FLAGS, 6981 ixgbe_handle_que, que); 6982 if (que->que_si == NULL) { 6983 aprint_error_dev(dev, 6984 "couldn't establish software interrupt\n"); 6985 error = ENXIO; 6986 goto err_out; 6987 } 6988 } 6989 snprintf(wqname, sizeof(wqname), "%sdeferTx", device_xname(dev)); 6990 error = workqueue_create(&adapter->txr_wq, wqname, 6991 ixgbe_deferred_mq_start_work, adapter, IXGBE_WORKQUEUE_PRI, IPL_NET, 6992 IXGBE_WORKQUEUE_FLAGS); 6993 if (error) { 6994 aprint_error_dev(dev, 6995 "couldn't create workqueue for deferred Tx\n"); 6996 goto err_out; 6997 } 6998 adapter->txr_wq_enqueued = percpu_alloc(sizeof(u_int)); 6999 7000 snprintf(wqname, sizeof(wqname), "%sTxRx", device_xname(dev)); 7001 error = workqueue_create(&adapter->que_wq, wqname, 7002 ixgbe_handle_que_work, adapter, IXGBE_WORKQUEUE_PRI, IPL_NET, 7003 IXGBE_WORKQUEUE_FLAGS); 7004 if (error) { 7005 aprint_error_dev(dev, "couldn't create workqueue for Tx/Rx\n"); 7006 goto err_out; 7007 } 7008 7009 /* and Link */ 7010 cpu_id++; 7011 snprintf(intr_xname, sizeof(intr_xname), "%s link", device_xname(dev)); 7012 adapter->vector = vector; 7013 intrstr = pci_intr_string(pc, adapter->osdep.intrs[vector], intrbuf, 7014 sizeof(intrbuf)); 7015 #ifdef IXGBE_MPSAFE 7016 pci_intr_setattr(pc, &adapter->osdep.intrs[vector], PCI_INTR_MPSAFE, 7017 true); 7018 #endif 7019 /* Set the link handler function */ 7020 adapter->osdep.ihs[vector] = pci_intr_establish_xname(pc, 7021 adapter->osdep.intrs[vector], IPL_NET, ixgbe_msix_admin, adapter, 7022 intr_xname); 7023 if (adapter->osdep.ihs[vector] == NULL) { 7024 aprint_error_dev(dev, "Failed to register LINK handler\n"); 7025 error = ENXIO; 7026 goto err_out; 7027 } 7028 /* Round-robin affinity */ 7029 kcpuset_zero(affinity); 7030 kcpuset_set(affinity, cpu_id % ncpu); 7031 error = interrupt_distribute(adapter->osdep.ihs[vector], affinity, 7032 NULL); 7033 7034 aprint_normal_dev(dev, 7035 "for link, interrupting at %s", intrstr); 7036 if (error == 0) 7037 aprint_normal(", affinity to cpu %d\n", cpu_id % ncpu); 7038 else 7039 aprint_normal("\n"); 7040 7041 kcpuset_destroy(affinity); 7042 aprint_normal_dev(dev, 7043 "Using MSI-X interrupts with %d vectors\n", vector + 1); 7044 7045 return (0); 7046 7047 err_out: 7048 kcpuset_destroy(affinity); 7049 ixgbe_free_deferred_handlers(adapter); 7050 ixgbe_free_pciintr_resources(adapter); 7051 return (error); 7052 } /* ixgbe_allocate_msix */ 7053 7054 /************************************************************************ 7055 * ixgbe_configure_interrupts 7056 * 7057 * Setup MSI-X, MSI, or legacy interrupts (in that order). 7058 * This will also depend on user settings. 7059 ************************************************************************/ 7060 static int 7061 ixgbe_configure_interrupts(struct adapter *adapter) 7062 { 7063 device_t dev = adapter->dev; 7064 struct ixgbe_mac_info *mac = &adapter->hw.mac; 7065 int want, queues, msgs; 7066 7067 /* Default to 1 queue if MSI-X setup fails */ 7068 adapter->num_queues = 1; 7069 7070 /* Override by tuneable */ 7071 if (!(adapter->feat_cap & IXGBE_FEATURE_MSIX)) 7072 goto msi; 7073 7074 /* 7075 * NetBSD only: Use single vector MSI when number of CPU is 1 to save 7076 * interrupt slot. 7077 */ 7078 if (ncpu == 1) 7079 goto msi; 7080 7081 /* First try MSI-X */ 7082 msgs = pci_msix_count(adapter->osdep.pc, adapter->osdep.tag); 7083 msgs = MIN(msgs, IXG_MAX_NINTR); 7084 if (msgs < 2) 7085 goto msi; 7086 7087 adapter->feat_en |= IXGBE_FEATURE_MSIX; 7088 7089 /* Figure out a reasonable auto config value */ 7090 queues = (ncpu > (msgs - 1)) ? (msgs - 1) : ncpu; 7091 7092 #ifdef RSS 7093 /* If we're doing RSS, clamp at the number of RSS buckets */ 7094 if (adapter->feat_en & IXGBE_FEATURE_RSS) 7095 queues = uimin(queues, rss_getnumbuckets()); 7096 #endif 7097 if (ixgbe_num_queues > queues) { 7098 aprint_error_dev(adapter->dev, 7099 "ixgbe_num_queues (%d) is too large, " 7100 "using reduced amount (%d).\n", ixgbe_num_queues, queues); 7101 ixgbe_num_queues = queues; 7102 } 7103 7104 if (ixgbe_num_queues != 0) 7105 queues = ixgbe_num_queues; 7106 else 7107 queues = uimin(queues, 7108 uimin(mac->max_tx_queues, mac->max_rx_queues)); 7109 7110 /* reflect correct sysctl value */ 7111 ixgbe_num_queues = queues; 7112 7113 /* 7114 * Want one vector (RX/TX pair) per queue 7115 * plus an additional for Link. 7116 */ 7117 want = queues + 1; 7118 if (msgs >= want) 7119 msgs = want; 7120 else { 7121 aprint_error_dev(dev, "MSI-X Configuration Problem, " 7122 "%d vectors but %d queues wanted!\n", msgs, want); 7123 goto msi; 7124 } 7125 adapter->num_queues = queues; 7126 adapter->feat_en |= IXGBE_FEATURE_MSIX; 7127 return (0); 7128 7129 /* 7130 * MSI-X allocation failed or provided us with 7131 * less vectors than needed. Free MSI-X resources 7132 * and we'll try enabling MSI. 7133 */ 7134 msi: 7135 /* Without MSI-X, some features are no longer supported */ 7136 adapter->feat_cap &= ~IXGBE_FEATURE_RSS; 7137 adapter->feat_en &= ~IXGBE_FEATURE_RSS; 7138 adapter->feat_cap &= ~IXGBE_FEATURE_SRIOV; 7139 adapter->feat_en &= ~IXGBE_FEATURE_SRIOV; 7140 7141 msgs = pci_msi_count(adapter->osdep.pc, adapter->osdep.tag); 7142 adapter->feat_en &= ~IXGBE_FEATURE_MSIX; 7143 if (msgs > 1) 7144 msgs = 1; 7145 if (msgs != 0) { 7146 msgs = 1; 7147 adapter->feat_en |= IXGBE_FEATURE_MSI; 7148 return (0); 7149 } 7150 7151 if (!(adapter->feat_cap & IXGBE_FEATURE_LEGACY_IRQ)) { 7152 aprint_error_dev(dev, 7153 "Device does not support legacy interrupts.\n"); 7154 return 1; 7155 } 7156 7157 adapter->feat_en |= IXGBE_FEATURE_LEGACY_IRQ; 7158 7159 return (0); 7160 } /* ixgbe_configure_interrupts */ 7161 7162 7163 /************************************************************************ 7164 * ixgbe_handle_link - Tasklet for MSI-X Link interrupts 7165 * 7166 * Done outside of interrupt context since the driver might sleep 7167 ************************************************************************/ 7168 static void 7169 ixgbe_handle_link(void *context) 7170 { 7171 struct adapter *adapter = context; 7172 struct ixgbe_hw *hw = &adapter->hw; 7173 7174 KASSERT(mutex_owned(&adapter->core_mtx)); 7175 7176 IXGBE_EVC_ADD(&adapter->link_workev, 1); 7177 ixgbe_check_link(hw, &adapter->link_speed, &adapter->link_up, 0); 7178 ixgbe_update_link_status(adapter); 7179 7180 /* Re-enable link interrupts */ 7181 IXGBE_WRITE_REG(hw, IXGBE_EIMS, IXGBE_EIMS_LSC); 7182 } /* ixgbe_handle_link */ 7183 7184 #if 0 7185 /************************************************************************ 7186 * ixgbe_rearm_queues 7187 ************************************************************************/ 7188 static __inline void 7189 ixgbe_rearm_queues(struct adapter *adapter, u64 queues) 7190 { 7191 u32 mask; 7192 7193 switch (adapter->hw.mac.type) { 7194 case ixgbe_mac_82598EB: 7195 mask = (IXGBE_EIMS_RTX_QUEUE & queues); 7196 IXGBE_WRITE_REG(&adapter->hw, IXGBE_EICS, mask); 7197 break; 7198 case ixgbe_mac_82599EB: 7199 case ixgbe_mac_X540: 7200 case ixgbe_mac_X550: 7201 case ixgbe_mac_X550EM_x: 7202 case ixgbe_mac_X550EM_a: 7203 mask = (queues & 0xFFFFFFFF); 7204 IXGBE_WRITE_REG(&adapter->hw, IXGBE_EICS_EX(0), mask); 7205 mask = (queues >> 32); 7206 IXGBE_WRITE_REG(&adapter->hw, IXGBE_EICS_EX(1), mask); 7207 break; 7208 default: 7209 break; 7210 } 7211 } /* ixgbe_rearm_queues */ 7212 #endif 7213
Indexes created Mon Sep 29 21:09:56 GMT 2025