Home | History | Annotate | Line # | Download | only in ixgbe
      1 /* $NetBSD: ixgbe.c,v 1.354 2024/07/10 03:26:30 msaitoh Exp $ */
      2 
      3 /******************************************************************************
      4 
      5   Copyright (c) 2001-2017, Intel Corporation
      6   All rights reserved.
      7 
      8   Redistribution and use in source and binary forms, with or without
      9   modification, are permitted provided that the following conditions are met:
     10 
     11    1. Redistributions of source code must retain the above copyright notice,
     12       this list of conditions and the following disclaimer.
     13 
     14    2. Redistributions in binary form must reproduce the above copyright
     15       notice, this list of conditions and the following disclaimer in the
     16       documentation and/or other materials provided with the distribution.
     17 
     18    3. Neither the name of the Intel Corporation nor the names of its
     19       contributors may be used to endorse or promote products derived from
     20       this software without specific prior written permission.
     21 
     22   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
     23   AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
     24   IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
     25   ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
     26   LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
     27   CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
     28   SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
     29   INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
     30   CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
     31   ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
     32   POSSIBILITY OF SUCH DAMAGE.
     33 
     34 ******************************************************************************/
     35 /*$FreeBSD: head/sys/dev/ixgbe/if_ix.c 331224 2018-03-19 20:55:05Z erj $*/
     36 
     37 /*
     38  * Copyright (c) 2011 The NetBSD Foundation, Inc.
     39  * All rights reserved.
     40  *
     41  * This code is derived from software contributed to The NetBSD Foundation
     42  * by Coyote Point Systems, Inc.
     43  *
     44  * Redistribution and use in source and binary forms, with or without
     45  * modification, are permitted provided that the following conditions
     46  * are met:
     47  * 1. Redistributions of source code must retain the above copyright
     48  *    notice, this list of conditions and the following disclaimer.
     49  * 2. Redistributions in binary form must reproduce the above copyright
     50  *    notice, this list of conditions and the following disclaimer in the
     51  *    documentation and/or other materials provided with the distribution.
     52  *
     53  * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
     54  * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
     55  * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
     56  * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
     57  * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
     58  * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
     59  * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
     60  * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
     61  * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
     62  * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
     63  * POSSIBILITY OF SUCH DAMAGE.
     64  */
     65 
     66 #include <sys/cdefs.h>
     67 __KERNEL_RCSID(0, "$NetBSD: ixgbe.c,v 1.354 2024/07/10 03:26:30 msaitoh Exp $");
     68 
     69 #ifdef _KERNEL_OPT
     70 #include "opt_inet.h"
     71 #include "opt_inet6.h"
     72 #endif
     73 
     74 #include "ixgbe.h"
     75 #include "ixgbe_phy.h"
     76 #include "ixgbe_sriov.h"
     77 
     78 #include <sys/cprng.h>
     79 #include <dev/mii/mii.h>
     80 #include <dev/mii/miivar.h>
     81 
     82 /************************************************************************
     83  * Driver version
     84  ************************************************************************/
     85 static const char ixgbe_driver_version[] = "4.0.1-k";
     86 /* XXX NetBSD: + 3.3.24 */
     87 
     88 /************************************************************************
     89  * PCI Device ID Table
     90  *
     91  *   Used by probe to select devices to load on
     92  *   Last field stores an index into ixgbe_strings
     93  *   Last entry must be all 0s
     94  *
     95  *   { Vendor ID, Device ID, SubVendor ID, SubDevice ID, String Index }
     96  ************************************************************************/
     97 static const ixgbe_vendor_info_t ixgbe_vendor_info_array[] =
     98 {
     99 	{IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598AF_DUAL_PORT, 0, 0, 0},
    100 	{IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598AF_SINGLE_PORT, 0, 0, 0},
    101 	{IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598EB_CX4, 0, 0, 0},
    102 	{IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598AT, 0, 0, 0},
    103 	{IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598AT2, 0, 0, 0},
    104 	{IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598, 0, 0, 0},
    105 	{IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598_BX, 0, 0, 0},
    106 	{IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598_DA_DUAL_PORT, 0, 0, 0},
    107 	{IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598_CX4_DUAL_PORT, 0, 0, 0},
    108 	{IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598EB_XF_LR, 0, 0, 0},
    109 	{IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598_SR_DUAL_PORT_EM, 0, 0, 0},
    110 	{IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598EB_SFP_LOM, 0, 0, 0},
    111 	{IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_KR, 0, 0, 0},
    112 	{IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_KX4, 0, 0, 0},
    113 	{IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_KX4_MEZZ, 0, 0, 0},
    114 	{IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_SFP, 0, 0, 0},
    115 	{IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_SFP_EM, 0, 0, 0},
    116 	{IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_XAUI_LOM, 0, 0, 0},
    117 	{IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_CX4, 0, 0, 0},
    118 	{IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_T3_LOM, 0, 0, 0},
    119 	{IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_LS, 0, 0, 0},
    120 	{IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_COMBO_BACKPLANE, 0, 0, 0},
    121 	{IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_BACKPLANE_FCOE, 0, 0, 0},
    122 	{IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_SFP_SF2, 0, 0, 0},
    123 	{IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_SFP_FCOE, 0, 0, 0},
    124 	{IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599EN_SFP, 0, 0, 0},
    125 	{IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_SFP_SF_QP, 0, 0, 0},
    126 	{IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_QSFP_SF_QP, 0, 0, 0},
    127 	{IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_X540T, 0, 0, 0},
    128 	{IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_X540T1, 0, 0, 0},
    129 	{IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_X550T, 0, 0, 0},
    130 	{IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_X550T1, 0, 0, 0},
    131 	{IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_X550EM_X_KR, 0, 0, 0},
    132 	{IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_X550EM_X_KX4, 0, 0, 0},
    133 	{IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_X550EM_X_10G_T, 0, 0, 0},
    134 	{IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_X550EM_X_1G_T, 0, 0, 0},
    135 	{IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_X550EM_X_SFP, 0, 0, 0},
    136 	{IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_X550EM_X_XFI, 0, 0, 0},
    137 	{IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_X550EM_A_KR, 0, 0, 0},
    138 	{IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_X550EM_A_KR_L, 0, 0, 0},
    139 	{IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_X550EM_A_QSFP, 0, 0, 0},
    140 	{IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_X550EM_A_QSFP_N, 0, 0, 0},
    141 	{IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_X550EM_A_SFP, 0, 0, 0},
    142 	{IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_X550EM_A_SFP_N, 0, 0, 0},
    143 	{IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_X550EM_A_SGMII, 0, 0, 0},
    144 	{IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_X550EM_A_SGMII_L, 0, 0, 0},
    145 	{IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_X550EM_A_10G_T, 0, 0, 0},
    146 	{IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_X550EM_A_1G_T, 0, 0, 0},
    147 	{IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_X550EM_A_1G_T_L, 0, 0, 0},
    148 	{IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_X540_BYPASS, 0, 0, 0},
    149 	{IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_BYPASS, 0, 0, 0},
    150 	/* required last entry */
    151 	{0, 0, 0, 0, 0}
    152 };
    153 
    154 /************************************************************************
    155  * Table of branding strings
    156  ************************************************************************/
    157 static const char    *ixgbe_strings[] = {
    158 	"Intel(R) PRO/10GbE PCI-Express Network Driver"
    159 };
    160 
    161 /************************************************************************
    162  * Function prototypes
    163  ************************************************************************/
    164 static int	ixgbe_probe(device_t, cfdata_t, void *);
    165 static void	ixgbe_quirks(struct ixgbe_softc *);
    166 static void	ixgbe_attach(device_t, device_t, void *);
    167 static int	ixgbe_detach(device_t, int);
    168 #if 0
    169 static int	ixgbe_shutdown(device_t);
    170 #endif
    171 static bool	ixgbe_suspend(device_t, const pmf_qual_t *);
    172 static bool	ixgbe_resume(device_t, const pmf_qual_t *);
    173 static int	ixgbe_ifflags_cb(struct ethercom *);
    174 static int	ixgbe_ioctl(struct ifnet *, u_long, void *);
    175 static int	ixgbe_init(struct ifnet *);
    176 static void	ixgbe_init_locked(struct ixgbe_softc *);
    177 static void	ixgbe_ifstop(struct ifnet *, int);
    178 static void	ixgbe_stop_locked(void *);
    179 static void	ixgbe_init_device_features(struct ixgbe_softc *);
    180 static int	ixgbe_check_fan_failure(struct ixgbe_softc *, u32, bool);
    181 static void	ixgbe_add_media_types(struct ixgbe_softc *);
    182 static void	ixgbe_media_status(struct ifnet *, struct ifmediareq *);
    183 static int	ixgbe_media_change(struct ifnet *);
    184 static int	ixgbe_allocate_pci_resources(struct ixgbe_softc *,
    185 		    const struct pci_attach_args *);
    186 static void	ixgbe_free_deferred_handlers(struct ixgbe_softc *);
    187 static void	ixgbe_get_slot_info(struct ixgbe_softc *);
    188 static int	ixgbe_allocate_msix(struct ixgbe_softc *,
    189 		    const struct pci_attach_args *);
    190 static int	ixgbe_allocate_legacy(struct ixgbe_softc *,
    191 		    const struct pci_attach_args *);
    192 static int	ixgbe_configure_interrupts(struct ixgbe_softc *);
    193 static void	ixgbe_free_pciintr_resources(struct ixgbe_softc *);
    194 static void	ixgbe_free_pci_resources(struct ixgbe_softc *);
    195 static void	ixgbe_local_timer(void *);
    196 static void	ixgbe_handle_timer(struct work *, void *);
    197 static void	ixgbe_recovery_mode_timer(void *);
    198 static void	ixgbe_handle_recovery_mode_timer(struct work *, void *);
    199 static int	ixgbe_setup_interface(device_t, struct ixgbe_softc *);
    200 static void	ixgbe_config_gpie(struct ixgbe_softc *);
    201 static void	ixgbe_config_dmac(struct ixgbe_softc *);
    202 static void	ixgbe_config_delay_values(struct ixgbe_softc *);
    203 static void	ixgbe_schedule_admin_tasklet(struct ixgbe_softc *);
    204 static void	ixgbe_config_link(struct ixgbe_softc *);
    205 static void	ixgbe_check_wol_support(struct ixgbe_softc *);
    206 static int	ixgbe_setup_low_power_mode(struct ixgbe_softc *);
    207 #if 0
    208 static void	ixgbe_rearm_queues(struct ixgbe_softc *, u64);
    209 #endif
    210 
    211 static void	ixgbe_initialize_transmit_units(struct ixgbe_softc *);
    212 static void	ixgbe_initialize_receive_units(struct ixgbe_softc *);
    213 static void	ixgbe_enable_rx_drop(struct ixgbe_softc *);
    214 static void	ixgbe_disable_rx_drop(struct ixgbe_softc *);
    215 static void	ixgbe_initialize_rss_mapping(struct ixgbe_softc *);
    216 
    217 static void	ixgbe_enable_intr(struct ixgbe_softc *);
    218 static void	ixgbe_disable_intr(struct ixgbe_softc *);
    219 static void	ixgbe_update_stats_counters(struct ixgbe_softc *);
    220 static void	ixgbe_set_rxfilter(struct ixgbe_softc *);
    221 static void	ixgbe_update_link_status(struct ixgbe_softc *);
    222 static void	ixgbe_set_ivar(struct ixgbe_softc *, u8, u8, s8);
    223 static void	ixgbe_configure_ivars(struct ixgbe_softc *);
    224 static u8 *	ixgbe_mc_array_itr(struct ixgbe_hw *, u8 **, u32 *);
    225 static void	ixgbe_eitr_write(struct ixgbe_softc *, uint32_t, uint32_t);
    226 
    227 static void	ixgbe_setup_vlan_hw_tagging(struct ixgbe_softc *);
    228 static void	ixgbe_setup_vlan_hw_support(struct ixgbe_softc *);
    229 static int	ixgbe_vlan_cb(struct ethercom *, uint16_t, bool);
    230 static int	ixgbe_register_vlan(struct ixgbe_softc *, u16);
    231 static int	ixgbe_unregister_vlan(struct ixgbe_softc *, u16);
    232 
    233 static void	ixgbe_add_device_sysctls(struct ixgbe_softc *);
    234 static void	ixgbe_add_hw_stats(struct ixgbe_softc *);
    235 static void	ixgbe_clear_evcnt(struct ixgbe_softc *);
    236 static int	ixgbe_set_flowcntl(struct ixgbe_softc *, int);
    237 static int	ixgbe_set_advertise(struct ixgbe_softc *, int);
    238 static int	ixgbe_get_default_advertise(struct ixgbe_softc *);
    239 
    240 /* Sysctl handlers */
    241 static int	ixgbe_sysctl_flowcntl(SYSCTLFN_PROTO);
    242 static int	ixgbe_sysctl_advertise(SYSCTLFN_PROTO);
    243 static int	ixgbe_sysctl_interrupt_rate_handler(SYSCTLFN_PROTO);
    244 static int	ixgbe_sysctl_dmac(SYSCTLFN_PROTO);
    245 static int	ixgbe_sysctl_phy_temp(SYSCTLFN_PROTO);
    246 static int	ixgbe_sysctl_phy_overtemp_occurred(SYSCTLFN_PROTO);
    247 #ifdef IXGBE_DEBUG
    248 static int	ixgbe_sysctl_power_state(SYSCTLFN_PROTO);
    249 static int	ixgbe_sysctl_print_rss_config(SYSCTLFN_PROTO);
    250 #endif
    251 static int	ixgbe_sysctl_next_to_check_handler(SYSCTLFN_PROTO);
    252 static int	ixgbe_sysctl_next_to_refresh_handler(SYSCTLFN_PROTO);
    253 static int	ixgbe_sysctl_rdh_handler(SYSCTLFN_PROTO);
    254 static int	ixgbe_sysctl_rdt_handler(SYSCTLFN_PROTO);
    255 static int	ixgbe_sysctl_tdt_handler(SYSCTLFN_PROTO);
    256 static int	ixgbe_sysctl_tdh_handler(SYSCTLFN_PROTO);
    257 static int	ixgbe_sysctl_eee_state(SYSCTLFN_PROTO);
    258 static int	ixgbe_sysctl_debug(SYSCTLFN_PROTO);
    259 static int	ixgbe_sysctl_rx_copy_len(SYSCTLFN_PROTO);
    260 static int	ixgbe_sysctl_tx_process_limit(SYSCTLFN_PROTO);
    261 static int	ixgbe_sysctl_rx_process_limit(SYSCTLFN_PROTO);
    262 static int	ixgbe_sysctl_wol_enable(SYSCTLFN_PROTO);
    263 static int	ixgbe_sysctl_wufc(SYSCTLFN_PROTO);
    264 
    265 /* Interrupt functions */
    266 static int	ixgbe_msix_que(void *);
    267 static int	ixgbe_msix_admin(void *);
    268 static void	ixgbe_intr_admin_common(struct ixgbe_softc *, u32, u32 *);
    269 static int	ixgbe_legacy_irq(void *);
    270 
    271 /* Event handlers running on workqueue */
    272 static void	ixgbe_handle_que(void *);
    273 static void	ixgbe_handle_link(void *);
    274 static void	ixgbe_handle_msf(void *);
    275 static void	ixgbe_handle_mod(void *, bool);
    276 static void	ixgbe_handle_phy(void *);
    277 
    278 /* Deferred workqueue handlers */
    279 static void	ixgbe_handle_admin(struct work *, void *);
    280 static void	ixgbe_handle_que_work(struct work *, void *);
    281 
    282 static const ixgbe_vendor_info_t *ixgbe_lookup(const struct pci_attach_args *);
    283 
    284 /************************************************************************
    285  *  NetBSD Device Interface Entry Points
    286  ************************************************************************/
    287 CFATTACH_DECL3_NEW(ixg, sizeof(struct ixgbe_softc),
    288     ixgbe_probe, ixgbe_attach, ixgbe_detach, NULL, NULL, NULL,
    289     DVF_DETACH_SHUTDOWN);
    290 
    291 #if 0
    292 devclass_t ix_devclass;
    293 DRIVER_MODULE(ix, pci, ix_driver, ix_devclass, 0, 0);
    294 
    295 MODULE_DEPEND(ix, pci, 1, 1, 1);
    296 MODULE_DEPEND(ix, ether, 1, 1, 1);
    297 #ifdef DEV_NETMAP
    298 MODULE_DEPEND(ix, netmap, 1, 1, 1);
    299 #endif
    300 #endif
    301 
    302 /*
    303  * TUNEABLE PARAMETERS:
    304  */
    305 
    306 /*
    307  * AIM: Adaptive Interrupt Moderation
    308  * which means that the interrupt rate
    309  * is varied over time based on the
    310  * traffic for that interrupt vector
    311  */
    312 static bool ixgbe_enable_aim = true;
    313 #define SYSCTL_INT(_a1, _a2, _a3, _a4, _a5, _a6, _a7)
    314 SYSCTL_INT(_hw_ix, OID_AUTO, enable_aim, CTLFLAG_RDTUN, &ixgbe_enable_aim, 0,
    315     "Enable adaptive interrupt moderation");
    316 
    317 static int ixgbe_max_interrupt_rate = (4000000 / IXGBE_LOW_LATENCY);
    318 SYSCTL_INT(_hw_ix, OID_AUTO, max_interrupt_rate, CTLFLAG_RDTUN,
    319     &ixgbe_max_interrupt_rate, 0, "Maximum interrupts per second");
    320 
    321 /* How many packets rxeof tries to clean at a time */
    322 static int ixgbe_rx_process_limit = 256;
    323 SYSCTL_INT(_hw_ix, OID_AUTO, rx_process_limit, CTLFLAG_RDTUN,
    324     &ixgbe_rx_process_limit, 0, "Maximum number of received packets to process at a time, -1 means unlimited");
    325 
    326 /* How many packets txeof tries to clean at a time */
    327 static int ixgbe_tx_process_limit = 256;
    328 SYSCTL_INT(_hw_ix, OID_AUTO, tx_process_limit, CTLFLAG_RDTUN,
    329     &ixgbe_tx_process_limit, 0,
    330     "Maximum number of sent packets to process at a time, -1 means unlimited");
    331 
    332 /* Flow control setting, default to full */
    333 static int ixgbe_flow_control = ixgbe_fc_full;
    334 SYSCTL_INT(_hw_ix, OID_AUTO, flow_control, CTLFLAG_RDTUN,
    335     &ixgbe_flow_control, 0, "Default flow control used for all adapters");
    336 
    337 /* Which packet processing uses workqueue or softint */
    338 static bool ixgbe_txrx_workqueue = false;
    339 
    340 /*
    341  * Smart speed setting, default to on
    342  * this only works as a compile option
    343  * right now as its during attach, set
    344  * this to 'ixgbe_smart_speed_off' to
    345  * disable.
    346  */
    347 static int ixgbe_smart_speed = ixgbe_smart_speed_on;
    348 
    349 /*
    350  * MSI-X should be the default for best performance,
    351  * but this allows it to be forced off for testing.
    352  */
    353 static int ixgbe_enable_msix = 1;
    354 SYSCTL_INT(_hw_ix, OID_AUTO, enable_msix, CTLFLAG_RDTUN, &ixgbe_enable_msix, 0,
    355     "Enable MSI-X interrupts");
    356 
    357 /*
    358  * Number of Queues, can be set to 0,
    359  * it then autoconfigures based on the
    360  * number of cpus and number of MSI-X vectors.
    361  * This can be overridden manually here.
    362  */
    363 static int ixgbe_num_queues = 0;
    364 SYSCTL_INT(_hw_ix, OID_AUTO, num_queues, CTLFLAG_RDTUN, &ixgbe_num_queues, 0,
    365     "Number of queues to configure, 0 indicates autoconfigure");
    366 
    367 /*
    368  * Number of TX descriptors per ring,
    369  * setting higher than RX as this seems
    370  * the better performing choice.
    371  */
    372 static int ixgbe_txd = DEFAULT_TXD;
    373 SYSCTL_INT(_hw_ix, OID_AUTO, txd, CTLFLAG_RDTUN, &ixgbe_txd, 0,
    374     "Number of transmit descriptors per queue");
    375 
    376 /* Number of RX descriptors per ring */
    377 static int ixgbe_rxd = DEFAULT_RXD;
    378 SYSCTL_INT(_hw_ix, OID_AUTO, rxd, CTLFLAG_RDTUN, &ixgbe_rxd, 0,
    379     "Number of receive descriptors per queue");
    380 
    381 /*
    382  * Defining this on will allow the use
    383  * of unsupported SFP+ modules, note that
    384  * doing so you are on your own :)
    385  */
    386 static int allow_unsupported_sfp = false;
    387 #define TUNABLE_INT(__x, __y)
    388 TUNABLE_INT("hw.ix.unsupported_sfp", &allow_unsupported_sfp);
    389 
    390 /*
    391  * Not sure if Flow Director is fully baked,
    392  * so we'll default to turning it off.
    393  */
    394 static int ixgbe_enable_fdir = 0;
    395 SYSCTL_INT(_hw_ix, OID_AUTO, enable_fdir, CTLFLAG_RDTUN, &ixgbe_enable_fdir, 0,
    396     "Enable Flow Director");
    397 
    398 /* Legacy Transmit (single queue) */
    399 static int ixgbe_enable_legacy_tx = 0;
    400 SYSCTL_INT(_hw_ix, OID_AUTO, enable_legacy_tx, CTLFLAG_RDTUN,
    401     &ixgbe_enable_legacy_tx, 0, "Enable Legacy TX flow");
    402 
    403 /* Receive-Side Scaling */
    404 static int ixgbe_enable_rss = 1;
    405 SYSCTL_INT(_hw_ix, OID_AUTO, enable_rss, CTLFLAG_RDTUN, &ixgbe_enable_rss, 0,
    406     "Enable Receive-Side Scaling (RSS)");
    407 
    408 #if 0
    409 static int (*ixgbe_start_locked)(struct ifnet *, struct tx_ring *);
    410 static int (*ixgbe_ring_empty)(struct ifnet *, pcq_t *);
    411 #endif
    412 
    413 #define IXGBE_WORKQUEUE_PRI PRI_SOFTNET
    414 
    415 /* Interval between reports of errors */
    416 static const struct timeval ixgbe_errlog_intrvl = { 60, 0 };	/* 60s */
    417 
    418 /************************************************************************
    419  * ixgbe_initialize_rss_mapping
    420  ************************************************************************/
    421 static void
    422 ixgbe_initialize_rss_mapping(struct ixgbe_softc *sc)
    423 {
    424 	struct ixgbe_hw	*hw = &sc->hw;
    425 	u32		reta = 0, mrqc, rss_key[10];
    426 	int		queue_id, table_size, index_mult;
    427 	int		i, j;
    428 	u32		rss_hash_config;
    429 
    430 	/* force use default RSS key. */
    431 #ifdef __NetBSD__
    432 	rss_getkey((uint8_t *) &rss_key);
    433 #else
    434 	if (sc->feat_en & IXGBE_FEATURE_RSS) {
    435 		/* Fetch the configured RSS key */
    436 		rss_getkey((uint8_t *) &rss_key);
    437 	} else {
    438 		/* set up random bits */
    439 		cprng_fast(&rss_key, sizeof(rss_key));
    440 	}
    441 #endif
    442 
    443 	/* Set multiplier for RETA setup and table size based on MAC */
    444 	index_mult = 0x1;
    445 	table_size = 128;
    446 	switch (sc->hw.mac.type) {
    447 	case ixgbe_mac_82598EB:
    448 		index_mult = 0x11;
    449 		break;
    450 	case ixgbe_mac_X550:
    451 	case ixgbe_mac_X550EM_x:
    452 	case ixgbe_mac_X550EM_a:
    453 		table_size = 512;
    454 		break;
    455 	default:
    456 		break;
    457 	}
    458 
    459 	/* Set up the redirection table */
    460 	for (i = 0, j = 0; i < table_size; i++, j++) {
    461 		if (j == sc->num_queues)
    462 			j = 0;
    463 
    464 		if (sc->feat_en & IXGBE_FEATURE_RSS) {
    465 			/*
    466 			 * Fetch the RSS bucket id for the given indirection
    467 			 * entry. Cap it at the number of configured buckets
    468 			 * (which is num_queues.)
    469 			 */
    470 			queue_id = rss_get_indirection_to_bucket(i);
    471 			queue_id = queue_id % sc->num_queues;
    472 		} else
    473 			queue_id = (j * index_mult);
    474 
    475 		/*
    476 		 * The low 8 bits are for hash value (n+0);
    477 		 * The next 8 bits are for hash value (n+1), etc.
    478 		 */
    479 		reta = reta >> 8;
    480 		reta = reta | (((uint32_t) queue_id) << 24);
    481 		if ((i & 3) == 3) {
    482 			if (i < 128)
    483 				IXGBE_WRITE_REG(hw, IXGBE_RETA(i >> 2), reta);
    484 			else
    485 				IXGBE_WRITE_REG(hw, IXGBE_ERETA((i >> 2) - 32),
    486 				    reta);
    487 			reta = 0;
    488 		}
    489 	}
    490 
    491 	/* Now fill our hash function seeds */
    492 	for (i = 0; i < 10; i++)
    493 		IXGBE_WRITE_REG(hw, IXGBE_RSSRK(i), rss_key[i]);
    494 
    495 	/* Perform hash on these packet types */
    496 	if (sc->feat_en & IXGBE_FEATURE_RSS)
    497 		rss_hash_config = rss_gethashconfig();
    498 	else {
    499 		/*
    500 		 * Disable UDP - IP fragments aren't currently being handled
    501 		 * and so we end up with a mix of 2-tuple and 4-tuple
    502 		 * traffic.
    503 		 */
    504 		rss_hash_config = RSS_HASHTYPE_RSS_IPV4
    505 				| RSS_HASHTYPE_RSS_TCP_IPV4
    506 				| RSS_HASHTYPE_RSS_IPV6
    507 				| RSS_HASHTYPE_RSS_TCP_IPV6
    508 				| RSS_HASHTYPE_RSS_IPV6_EX
    509 				| RSS_HASHTYPE_RSS_TCP_IPV6_EX;
    510 	}
    511 
    512 	mrqc = IXGBE_MRQC_RSSEN;
    513 	if (rss_hash_config & RSS_HASHTYPE_RSS_IPV4)
    514 		mrqc |= IXGBE_MRQC_RSS_FIELD_IPV4;
    515 	if (rss_hash_config & RSS_HASHTYPE_RSS_TCP_IPV4)
    516 		mrqc |= IXGBE_MRQC_RSS_FIELD_IPV4_TCP;
    517 	if (rss_hash_config & RSS_HASHTYPE_RSS_IPV6)
    518 		mrqc |= IXGBE_MRQC_RSS_FIELD_IPV6;
    519 	if (rss_hash_config & RSS_HASHTYPE_RSS_TCP_IPV6)
    520 		mrqc |= IXGBE_MRQC_RSS_FIELD_IPV6_TCP;
    521 	if (rss_hash_config & RSS_HASHTYPE_RSS_IPV6_EX)
    522 		mrqc |= IXGBE_MRQC_RSS_FIELD_IPV6_EX;
    523 	if (rss_hash_config & RSS_HASHTYPE_RSS_TCP_IPV6_EX)
    524 		mrqc |= IXGBE_MRQC_RSS_FIELD_IPV6_EX_TCP;
    525 	if (rss_hash_config & RSS_HASHTYPE_RSS_UDP_IPV4)
    526 		mrqc |= IXGBE_MRQC_RSS_FIELD_IPV4_UDP;
    527 	if (rss_hash_config & RSS_HASHTYPE_RSS_UDP_IPV6)
    528 		mrqc |= IXGBE_MRQC_RSS_FIELD_IPV6_UDP;
    529 	if (rss_hash_config & RSS_HASHTYPE_RSS_UDP_IPV6_EX)
    530 		mrqc |= IXGBE_MRQC_RSS_FIELD_IPV6_EX_UDP;
    531 	mrqc |= ixgbe_get_mrqc(sc->iov_mode);
    532 	IXGBE_WRITE_REG(hw, IXGBE_MRQC, mrqc);
    533 } /* ixgbe_initialize_rss_mapping */
    534 
    535 /************************************************************************
    536  * ixgbe_initialize_receive_units - Setup receive registers and features.
    537  ************************************************************************/
    538 #define BSIZEPKT_ROUNDUP ((1<<IXGBE_SRRCTL_BSIZEPKT_SHIFT)-1)
    539 
    540 static void
    541 ixgbe_initialize_receive_units(struct ixgbe_softc *sc)
    542 {
    543 	struct rx_ring	*rxr = sc->rx_rings;
    544 	struct ixgbe_hw	*hw = &sc->hw;
    545 	struct ifnet	*ifp = sc->ifp;
    546 	int		i, j;
    547 	u32		bufsz, fctrl, srrctl, rxcsum;
    548 	u32		hlreg;
    549 
    550 	/*
    551 	 * Make sure receives are disabled while
    552 	 * setting up the descriptor ring
    553 	 */
    554 	ixgbe_disable_rx(hw);
    555 
    556 	/* Enable broadcasts */
    557 	fctrl = IXGBE_READ_REG(hw, IXGBE_FCTRL);
    558 	fctrl |= IXGBE_FCTRL_BAM;
    559 	if (sc->hw.mac.type == ixgbe_mac_82598EB) {
    560 		fctrl |= IXGBE_FCTRL_DPF;
    561 		fctrl |= IXGBE_FCTRL_PMCF;
    562 	}
    563 	IXGBE_WRITE_REG(hw, IXGBE_FCTRL, fctrl);
    564 
    565 	/* Set for Jumbo Frames? */
    566 	hlreg = IXGBE_READ_REG(hw, IXGBE_HLREG0);
    567 	if (ifp->if_mtu > ETHERMTU)
    568 		hlreg |= IXGBE_HLREG0_JUMBOEN;
    569 	else
    570 		hlreg &= ~IXGBE_HLREG0_JUMBOEN;
    571 
    572 #ifdef DEV_NETMAP
    573 	/* CRC stripping is conditional in Netmap */
    574 	if ((sc->feat_en & IXGBE_FEATURE_NETMAP) &&
    575 	    (ifp->if_capenable & IFCAP_NETMAP) &&
    576 	    !ix_crcstrip)
    577 		hlreg &= ~IXGBE_HLREG0_RXCRCSTRP;
    578 	else
    579 #endif /* DEV_NETMAP */
    580 		hlreg |= IXGBE_HLREG0_RXCRCSTRP;
    581 
    582 	IXGBE_WRITE_REG(hw, IXGBE_HLREG0, hlreg);
    583 
    584 	bufsz = (sc->rx_mbuf_sz + BSIZEPKT_ROUNDUP) >>
    585 	    IXGBE_SRRCTL_BSIZEPKT_SHIFT;
    586 
    587 	for (i = 0; i < sc->num_queues; i++, rxr++) {
    588 		u64 rdba = rxr->rxdma.dma_paddr;
    589 		u32 reg;
    590 		int regnum = i / 4;	/* 1 register per 4 queues */
    591 		int regshift = i % 4;	/* 4 bits per 1 queue */
    592 		j = rxr->me;
    593 
    594 		/* Setup the Base and Length of the Rx Descriptor Ring */
    595 		IXGBE_WRITE_REG(hw, IXGBE_RDBAL(j),
    596 		    (rdba & 0x00000000ffffffffULL));
    597 		IXGBE_WRITE_REG(hw, IXGBE_RDBAH(j), (rdba >> 32));
    598 		IXGBE_WRITE_REG(hw, IXGBE_RDLEN(j),
    599 		    sc->num_rx_desc * sizeof(union ixgbe_adv_rx_desc));
    600 
    601 		/* Set up the SRRCTL register */
    602 		srrctl = IXGBE_READ_REG(hw, IXGBE_SRRCTL(j));
    603 		srrctl &= ~IXGBE_SRRCTL_BSIZEHDR_MASK;
    604 		srrctl &= ~IXGBE_SRRCTL_BSIZEPKT_MASK;
    605 		srrctl |= bufsz;
    606 		srrctl |= IXGBE_SRRCTL_DESCTYPE_ADV_ONEBUF;
    607 
    608 		/* Set RQSMR (Receive Queue Statistic Mapping) register */
    609 		reg = IXGBE_READ_REG(hw, IXGBE_RQSMR(regnum));
    610 		reg &= ~(0x000000ffUL << (regshift * 8));
    611 		reg |= i << (regshift * 8);
    612 		IXGBE_WRITE_REG(hw, IXGBE_RQSMR(regnum), reg);
    613 
    614 		/*
    615 		 * Set DROP_EN iff we have no flow control and >1 queue.
    616 		 * Note that srrctl was cleared shortly before during reset,
    617 		 * so we do not need to clear the bit, but do it just in case
    618 		 * this code is moved elsewhere.
    619 		 */
    620 		if ((sc->num_queues > 1) &&
    621 		    (sc->hw.fc.requested_mode == ixgbe_fc_none))
    622 			srrctl |= IXGBE_SRRCTL_DROP_EN;
    623 		else
    624 			srrctl &= ~IXGBE_SRRCTL_DROP_EN;
    625 
    626 		IXGBE_WRITE_REG(hw, IXGBE_SRRCTL(j), srrctl);
    627 
    628 		/* Setup the HW Rx Head and Tail Descriptor Pointers */
    629 		IXGBE_WRITE_REG(hw, IXGBE_RDH(j), 0);
    630 		IXGBE_WRITE_REG(hw, IXGBE_RDT(j), 0);
    631 
    632 		/* Set the driver rx tail address */
    633 		rxr->tail =  IXGBE_RDT(rxr->me);
    634 	}
    635 
    636 	if (sc->hw.mac.type != ixgbe_mac_82598EB) {
    637 		u32 psrtype = IXGBE_PSRTYPE_TCPHDR
    638 			    | IXGBE_PSRTYPE_UDPHDR
    639 			    | IXGBE_PSRTYPE_IPV4HDR
    640 			    | IXGBE_PSRTYPE_IPV6HDR;
    641 		IXGBE_WRITE_REG(hw, IXGBE_PSRTYPE(0), psrtype);
    642 	}
    643 
    644 	rxcsum = IXGBE_READ_REG(hw, IXGBE_RXCSUM);
    645 
    646 	ixgbe_initialize_rss_mapping(sc);
    647 
    648 	if (sc->num_queues > 1) {
    649 		/* RSS and RX IPP Checksum are mutually exclusive */
    650 		rxcsum |= IXGBE_RXCSUM_PCSD;
    651 	}
    652 
    653 	if (ifp->if_capenable & IFCAP_RXCSUM)
    654 		rxcsum |= IXGBE_RXCSUM_PCSD;
    655 
    656 	/* This is useful for calculating UDP/IP fragment checksums */
    657 	if (!(rxcsum & IXGBE_RXCSUM_PCSD))
    658 		rxcsum |= IXGBE_RXCSUM_IPPCSE;
    659 
    660 	IXGBE_WRITE_REG(hw, IXGBE_RXCSUM, rxcsum);
    661 
    662 } /* ixgbe_initialize_receive_units */
    663 
    664 /************************************************************************
    665  * ixgbe_initialize_transmit_units - Enable transmit units.
    666  ************************************************************************/
    667 static void
    668 ixgbe_initialize_transmit_units(struct ixgbe_softc *sc)
    669 {
    670 	struct tx_ring	*txr = sc->tx_rings;
    671 	struct ixgbe_hw	*hw = &sc->hw;
    672 	int i;
    673 
    674 	INIT_DEBUGOUT("ixgbe_initialize_transmit_units");
    675 
    676 	/* Setup the Base and Length of the Tx Descriptor Ring */
    677 	for (i = 0; i < sc->num_queues; i++, txr++) {
    678 		u64 tdba = txr->txdma.dma_paddr;
    679 		u32 txctrl = 0;
    680 		u32 tqsmreg, reg;
    681 		int regnum = i / 4;	/* 1 register per 4 queues */
    682 		int regshift = i % 4;	/* 4 bits per 1 queue */
    683 		int j = txr->me;
    684 
    685 		IXGBE_WRITE_REG(hw, IXGBE_TDBAL(j),
    686 		    (tdba & 0x00000000ffffffffULL));
    687 		IXGBE_WRITE_REG(hw, IXGBE_TDBAH(j), (tdba >> 32));
    688 		IXGBE_WRITE_REG(hw, IXGBE_TDLEN(j),
    689 		    sc->num_tx_desc * sizeof(union ixgbe_adv_tx_desc));
    690 
    691 		/*
    692 		 * Set TQSMR (Transmit Queue Statistic Mapping) register.
    693 		 * Register location is different between 82598 and others.
    694 		 */
    695 		if (sc->hw.mac.type == ixgbe_mac_82598EB)
    696 			tqsmreg = IXGBE_TQSMR(regnum);
    697 		else
    698 			tqsmreg = IXGBE_TQSM(regnum);
    699 		reg = IXGBE_READ_REG(hw, tqsmreg);
    700 		reg &= ~(0x000000ffUL << (regshift * 8));
    701 		reg |= i << (regshift * 8);
    702 		IXGBE_WRITE_REG(hw, tqsmreg, reg);
    703 
    704 		/* Setup the HW Tx Head and Tail descriptor pointers */
    705 		IXGBE_WRITE_REG(hw, IXGBE_TDH(j), 0);
    706 		IXGBE_WRITE_REG(hw, IXGBE_TDT(j), 0);
    707 
    708 		/* Cache the tail address */
    709 		txr->tail = IXGBE_TDT(j);
    710 
    711 		txr->txr_no_space = false;
    712 
    713 		/* Disable relax ordering */
    714 		/*
    715 		 * Note: for X550 series devices, these registers are actually
    716 		 * prefixed with TPH_ instead of DCA_, but the addresses and
    717 		 * fields remain the same.
    718 		 */
    719 		switch (hw->mac.type) {
    720 		case ixgbe_mac_82598EB:
    721 			txctrl = IXGBE_READ_REG(hw, IXGBE_DCA_TXCTRL(j));
    722 			break;
    723 		default:
    724 			txctrl = IXGBE_READ_REG(hw, IXGBE_DCA_TXCTRL_82599(j));
    725 			break;
    726 		}
    727 		txctrl &= ~IXGBE_DCA_TXCTRL_DESC_WRO_EN;
    728 		switch (hw->mac.type) {
    729 		case ixgbe_mac_82598EB:
    730 			IXGBE_WRITE_REG(hw, IXGBE_DCA_TXCTRL(j), txctrl);
    731 			break;
    732 		default:
    733 			IXGBE_WRITE_REG(hw, IXGBE_DCA_TXCTRL_82599(j), txctrl);
    734 			break;
    735 		}
    736 
    737 	}
    738 
    739 	if (hw->mac.type != ixgbe_mac_82598EB) {
    740 		u32 dmatxctl, rttdcs;
    741 
    742 		dmatxctl = IXGBE_READ_REG(hw, IXGBE_DMATXCTL);
    743 		dmatxctl |= IXGBE_DMATXCTL_TE;
    744 		IXGBE_WRITE_REG(hw, IXGBE_DMATXCTL, dmatxctl);
    745 		/* Disable arbiter to set MTQC */
    746 		rttdcs = IXGBE_READ_REG(hw, IXGBE_RTTDCS);
    747 		rttdcs |= IXGBE_RTTDCS_ARBDIS;
    748 		IXGBE_WRITE_REG(hw, IXGBE_RTTDCS, rttdcs);
    749 		IXGBE_WRITE_REG(hw, IXGBE_MTQC,
    750 		    ixgbe_get_mtqc(sc->iov_mode));
    751 		rttdcs &= ~IXGBE_RTTDCS_ARBDIS;
    752 		IXGBE_WRITE_REG(hw, IXGBE_RTTDCS, rttdcs);
    753 	}
    754 } /* ixgbe_initialize_transmit_units */
    755 
    756 static void
    757 ixgbe_quirks(struct ixgbe_softc *sc)
    758 {
    759 	device_t dev = sc->dev;
    760 	struct ixgbe_hw *hw = &sc->hw;
    761 	const char *vendor, *product;
    762 
    763 	if (hw->device_id == IXGBE_DEV_ID_X550EM_A_SFP_N) {
    764 		/*
    765 		 * Quirk for inverted logic of SFP+'s MOD_ABS on GIGABYTE
    766 		 * MA10-ST0.
    767 		 */
    768 		vendor = pmf_get_platform("system-vendor");
    769 		product = pmf_get_platform("system-product");
    770 
    771 		if ((vendor == NULL) || (product == NULL))
    772 			return;
    773 
    774 		if ((strcmp(vendor, "GIGABYTE") == 0) &&
    775 		    (strcmp(product, "MA10-ST0") == 0)) {
    776 			aprint_verbose_dev(dev,
    777 			    "Enable SFP+ MOD_ABS inverse quirk\n");
    778 			sc->hw.quirks |= IXGBE_QUIRK_MOD_ABS_INVERT;
    779 		}
    780 	}
    781 }
    782 
    783 /************************************************************************
    784  * ixgbe_attach - Device initialization routine
    785  *
    786  *   Called when the driver is being loaded.
    787  *   Identifies the type of hardware, allocates all resources
    788  *   and initializes the hardware.
    789  *
    790  *   return 0 on success, positive on failure
    791  ************************************************************************/
    792 static void
    793 ixgbe_attach(device_t parent, device_t dev, void *aux)
    794 {
    795 	struct ixgbe_softc *sc;
    796 	struct ixgbe_hw *hw;
    797 	int		error = -1;
    798 	u32		ctrl_ext;
    799 	u16		high, low, nvmreg, dev_caps;
    800 	pcireg_t	id, subid;
    801 	const ixgbe_vendor_info_t *ent;
    802 	struct pci_attach_args *pa = aux;
    803 	bool unsupported_sfp = false;
    804 	const char *str;
    805 	char wqname[MAXCOMLEN];
    806 	char buf[256];
    807 
    808 	INIT_DEBUGOUT("ixgbe_attach: begin");
    809 
    810 	/* Allocate, clear, and link in our adapter structure */
    811 	sc = device_private(dev);
    812 	sc->hw.back = sc;
    813 	sc->dev = dev;
    814 	hw = &sc->hw;
    815 	sc->osdep.pc = pa->pa_pc;
    816 	sc->osdep.tag = pa->pa_tag;
    817 	if (pci_dma64_available(pa))
    818 		sc->osdep.dmat = pa->pa_dmat64;
    819 	else
    820 		sc->osdep.dmat = pa->pa_dmat;
    821 	sc->osdep.attached = false;
    822 	sc->osdep.detaching = false;
    823 
    824 	ent = ixgbe_lookup(pa);
    825 
    826 	KASSERT(ent != NULL);
    827 
    828 	aprint_normal(": %s, Version - %s\n",
    829 	    ixgbe_strings[ent->index], ixgbe_driver_version);
    830 
    831 	/* Core Lock Init */
    832 	IXGBE_CORE_LOCK_INIT(sc, device_xname(dev));
    833 
    834 	/* Set up the timer callout and workqueue */
    835 	callout_init(&sc->timer, CALLOUT_MPSAFE);
    836 	snprintf(wqname, sizeof(wqname), "%s-timer", device_xname(dev));
    837 	error = workqueue_create(&sc->timer_wq, wqname,
    838 	    ixgbe_handle_timer, sc, IXGBE_WORKQUEUE_PRI, IPL_NET, WQ_MPSAFE);
    839 	if (error) {
    840 		aprint_error_dev(dev,
    841 		    "could not create timer workqueue (%d)\n", error);
    842 		goto err_out;
    843 	}
    844 
    845 	/* Determine hardware revision */
    846 	id = pci_conf_read(pa->pa_pc, pa->pa_tag, PCI_ID_REG);
    847 	subid = pci_conf_read(pa->pa_pc, pa->pa_tag, PCI_SUBSYS_ID_REG);
    848 
    849 	hw->vendor_id = PCI_VENDOR(id);
    850 	hw->device_id = PCI_PRODUCT(id);
    851 	hw->revision_id =
    852 	    PCI_REVISION(pci_conf_read(pa->pa_pc, pa->pa_tag, PCI_CLASS_REG));
    853 	hw->subsystem_vendor_id = PCI_SUBSYS_VENDOR(subid);
    854 	hw->subsystem_device_id = PCI_SUBSYS_ID(subid);
    855 
    856 	/* Set quirk flags */
    857 	ixgbe_quirks(sc);
    858 
    859 	/*
    860 	 * Make sure BUSMASTER is set
    861 	 */
    862 	ixgbe_pci_enable_busmaster(pa->pa_pc, pa->pa_tag);
    863 
    864 	/* Do base PCI setup - map BAR0 */
    865 	if (ixgbe_allocate_pci_resources(sc, pa)) {
    866 		aprint_error_dev(dev, "Allocation of PCI resources failed\n");
    867 		error = ENXIO;
    868 		goto err_out;
    869 	}
    870 
    871 	/* let hardware know driver is loaded */
    872 	ctrl_ext = IXGBE_READ_REG(hw, IXGBE_CTRL_EXT);
    873 	ctrl_ext |= IXGBE_CTRL_EXT_DRV_LOAD;
    874 	IXGBE_WRITE_REG(hw, IXGBE_CTRL_EXT, ctrl_ext);
    875 
    876 	/*
    877 	 * Initialize the shared code
    878 	 */
    879 	if (ixgbe_init_shared_code(hw) != 0) {
    880 		aprint_error_dev(dev,
    881 		    "Unable to initialize the shared code\n");
    882 		error = ENXIO;
    883 		goto err_out;
    884 	}
    885 
    886 	switch (hw->mac.type) {
    887 	case ixgbe_mac_82598EB:
    888 		str = "82598EB";
    889 		break;
    890 	case ixgbe_mac_82599EB:
    891 		str = "82599EB";
    892 		break;
    893 	case ixgbe_mac_X540:
    894 		str = "X540";
    895 		break;
    896 	case ixgbe_mac_X550:
    897 		str = "X550";
    898 		break;
    899 	case ixgbe_mac_X550EM_x:
    900 		str = "X550EM X";
    901 		break;
    902 	case ixgbe_mac_X550EM_a:
    903 		str = "X550EM A";
    904 		break;
    905 	default:
    906 		str = "Unknown";
    907 		break;
    908 	}
    909 	aprint_normal_dev(dev, "device %s\n", str);
    910 
    911 	hw->allow_unsupported_sfp = allow_unsupported_sfp;
    912 
    913 	/* Pick up the 82599 settings */
    914 	if (hw->mac.type != ixgbe_mac_82598EB)
    915 		hw->phy.smart_speed = ixgbe_smart_speed;
    916 
    917 	/* Set the right number of segments */
    918 	KASSERT(IXGBE_82599_SCATTER_MAX >= IXGBE_SCATTER_DEFAULT);
    919 	sc->num_segs = IXGBE_SCATTER_DEFAULT;
    920 
    921 	/* Ensure SW/FW semaphore is free */
    922 	ixgbe_init_swfw_semaphore(hw);
    923 
    924 	hw->mac.ops.set_lan_id(hw);
    925 	ixgbe_init_device_features(sc);
    926 
    927 	if (ixgbe_configure_interrupts(sc)) {
    928 		error = ENXIO;
    929 		goto err_out;
    930 	}
    931 
    932 	/* Allocate multicast array memory. */
    933 	sc->mta = malloc(sizeof(*sc->mta) *
    934 	    MAX_NUM_MULTICAST_ADDRESSES, M_DEVBUF, M_WAITOK);
    935 
    936 	/* Enable WoL (if supported) */
    937 	ixgbe_check_wol_support(sc);
    938 
    939 	/* Register for VLAN events */
    940 	ether_set_vlan_cb(&sc->osdep.ec, ixgbe_vlan_cb);
    941 
    942 	/* Verify adapter fan is still functional (if applicable) */
    943 	if (sc->feat_en & IXGBE_FEATURE_FAN_FAIL) {
    944 		u32 esdp = IXGBE_READ_REG(hw, IXGBE_ESDP);
    945 		ixgbe_check_fan_failure(sc, esdp, FALSE);
    946 	}
    947 
    948 	/* Set an initial default flow control value */
    949 	hw->fc.requested_mode = ixgbe_flow_control;
    950 
    951 	/* Do descriptor calc and sanity checks */
    952 	if (((ixgbe_txd * sizeof(union ixgbe_adv_tx_desc)) % DBA_ALIGN) != 0 ||
    953 	    ixgbe_txd < MIN_TXD || ixgbe_txd > MAX_TXD) {
    954 		aprint_error_dev(dev, "Invalid TX ring size (%d). "
    955 		    "It must be between %d and %d, "
    956 		    "inclusive, and must be a multiple of %zu. "
    957 		    "Using default value of %d instead.\n",
    958 		    ixgbe_txd, MIN_TXD, MAX_TXD,
    959 		    DBA_ALIGN / sizeof(union ixgbe_adv_tx_desc),
    960 		    DEFAULT_TXD);
    961 		sc->num_tx_desc = DEFAULT_TXD;
    962 	} else
    963 		sc->num_tx_desc = ixgbe_txd;
    964 
    965 	if (((ixgbe_rxd * sizeof(union ixgbe_adv_rx_desc)) % DBA_ALIGN) != 0 ||
    966 	    ixgbe_rxd < MIN_RXD || ixgbe_rxd > MAX_RXD) {
    967 		aprint_error_dev(dev, "Invalid RX ring size (%d). "
    968 		    "It must be between %d and %d, "
    969 		    "inclusive, and must be a multiple of %zu. "
    970 		    "Using default value of %d instead.\n",
    971 		    ixgbe_rxd, MIN_RXD, MAX_RXD,
    972 		    DBA_ALIGN / sizeof(union ixgbe_adv_rx_desc),
    973 		    DEFAULT_RXD);
    974 		sc->num_rx_desc = DEFAULT_RXD;
    975 	} else
    976 		sc->num_rx_desc = ixgbe_rxd;
    977 
    978 	/* Sysctls for limiting the amount of work done in the taskqueues */
    979 	sc->rx_process_limit
    980 	    = (ixgbe_rx_process_limit <= sc->num_rx_desc)
    981 	    ? ixgbe_rx_process_limit : sc->num_rx_desc;
    982 	sc->tx_process_limit
    983 	    = (ixgbe_tx_process_limit <= sc->num_tx_desc)
    984 	    ? ixgbe_tx_process_limit : sc->num_tx_desc;
    985 
    986 	/* Set default high limit of copying mbuf in rxeof */
    987 	sc->rx_copy_len = IXGBE_RX_COPY_LEN_MAX;
    988 
    989 	/* Allocate our TX/RX Queues */
    990 	if (ixgbe_allocate_queues(sc)) {
    991 		error = ENOMEM;
    992 		goto err_out;
    993 	}
    994 
    995 	hw->phy.reset_if_overtemp = TRUE;
    996 	error = ixgbe_reset_hw(hw);
    997 	hw->phy.reset_if_overtemp = FALSE;
    998 	if (error == IXGBE_ERR_SFP_NOT_PRESENT)
    999 		error = IXGBE_SUCCESS;
   1000 	else if (error == IXGBE_ERR_SFP_NOT_SUPPORTED) {
   1001 		aprint_error_dev(dev,
   1002 		    "Unsupported SFP+ module type was detected.\n");
   1003 		unsupported_sfp = true;
   1004 		error = IXGBE_SUCCESS;
   1005 	} else if (error) {
   1006 		aprint_error_dev(dev,
   1007 		    "Hardware initialization failed(error = %d)\n", error);
   1008 		error = EIO;
   1009 		goto err_late;
   1010 	}
   1011 
   1012 	/* Make sure we have a good EEPROM before we read from it */
   1013 	if (ixgbe_validate_eeprom_checksum(&sc->hw, NULL) < 0) {
   1014 		aprint_error_dev(dev, "The EEPROM Checksum Is Not Valid\n");
   1015 		error = EIO;
   1016 		goto err_late;
   1017 	}
   1018 
   1019 	aprint_normal("%s:", device_xname(dev));
   1020 	/* NVM Image Version */
   1021 	high = low = 0;
   1022 	switch (hw->mac.type) {
   1023 	case ixgbe_mac_82598EB:
   1024 		/*
   1025 		 * Print version from the dev starter version (0x29). The
   1026 		 * location is the same as newer device's IXGBE_NVM_MAP_VER.
   1027 		 */
   1028 		hw->eeprom.ops.read(hw, IXGBE_NVM_MAP_VER, &nvmreg);
   1029 		if (nvmreg == 0xffff)
   1030 			break;
   1031 		high = (nvmreg >> 12) & 0x0f;
   1032 		low = (nvmreg >> 4) & 0xff;
   1033 		id = nvmreg & 0x0f;
   1034 		/*
   1035 		 * The following output might not be correct. Some 82598 cards
   1036 		 * have 0x1070 or 0x2090. 82598 spec update notes about 2.9.0.
   1037 		 */
   1038 		aprint_normal(" NVM Image Version %u.%u.%u,", high, low, id);
   1039 		break;
   1040 	case ixgbe_mac_X540:
   1041 	case ixgbe_mac_X550EM_a:
   1042 		hw->eeprom.ops.read(hw, IXGBE_NVM_IMAGE_VER, &nvmreg);
   1043 		if (nvmreg == 0xffff)
   1044 			break;
   1045 		high = (nvmreg >> 12) & 0x0f;
   1046 		low = (nvmreg >> 4) & 0xff;
   1047 		id = nvmreg & 0x0f;
   1048 		aprint_normal(" NVM Image Version %u.", high);
   1049 		if (hw->mac.type == ixgbe_mac_X540)
   1050 			str = "%x";
   1051 		else
   1052 			str = "%02x";
   1053 		aprint_normal(str, low);
   1054 		aprint_normal(" ID 0x%x,", id);
   1055 		break;
   1056 	case ixgbe_mac_X550EM_x:
   1057 	case ixgbe_mac_X550:
   1058 		hw->eeprom.ops.read(hw, IXGBE_NVM_IMAGE_VER, &nvmreg);
   1059 		if (nvmreg == 0xffff)
   1060 			break;
   1061 		high = (nvmreg >> 12) & 0x0f;
   1062 		low = nvmreg & 0xff;
   1063 		aprint_normal(" NVM Image Version %u.%02x,", high, low);
   1064 		break;
   1065 	default:
   1066 		break;
   1067 	}
   1068 	hw->eeprom.nvm_image_ver_high = high;
   1069 	hw->eeprom.nvm_image_ver_low = low;
   1070 
   1071 	/* PHY firmware revision */
   1072 	switch (hw->mac.type) {
   1073 	case ixgbe_mac_X540:
   1074 	case ixgbe_mac_X550:
   1075 		hw->eeprom.ops.read(hw, IXGBE_PHYFW_REV, &nvmreg);
   1076 		if (nvmreg == 0xffff)
   1077 			break;
   1078 		high = (nvmreg >> 12) & 0x0f;
   1079 		low = (nvmreg >> 4) & 0xff;
   1080 		id = nvmreg & 0x000f;
   1081 		aprint_normal(" PHY FW Revision %u.", high);
   1082 		if (hw->mac.type == ixgbe_mac_X540)
   1083 			str = "%x";
   1084 		else
   1085 			str = "%02x";
   1086 		aprint_normal(str, low);
   1087 		aprint_normal(" ID 0x%x,", id);
   1088 		break;
   1089 	default:
   1090 		break;
   1091 	}
   1092 
   1093 	/* NVM Map version & OEM NVM Image version */
   1094 	switch (hw->mac.type) {
   1095 	case ixgbe_mac_X550:
   1096 	case ixgbe_mac_X550EM_x:
   1097 	case ixgbe_mac_X550EM_a:
   1098 		hw->eeprom.ops.read(hw, IXGBE_NVM_MAP_VER, &nvmreg);
   1099 		if (nvmreg != 0xffff) {
   1100 			high = (nvmreg >> 12) & 0x0f;
   1101 			low = nvmreg & 0x00ff;
   1102 			aprint_normal(" NVM Map version %u.%02x,", high, low);
   1103 		}
   1104 		hw->eeprom.ops.read(hw, IXGBE_OEM_NVM_IMAGE_VER, &nvmreg);
   1105 		if (nvmreg != 0xffff) {
   1106 			high = (nvmreg >> 12) & 0x0f;
   1107 			low = nvmreg & 0x00ff;
   1108 			aprint_verbose(" OEM NVM Image version %u.%02x,", high,
   1109 			    low);
   1110 		}
   1111 		break;
   1112 	default:
   1113 		break;
   1114 	}
   1115 
   1116 	/* Print the ETrackID */
   1117 	hw->eeprom.ops.read(hw, IXGBE_ETRACKID_H, &high);
   1118 	hw->eeprom.ops.read(hw, IXGBE_ETRACKID_L, &low);
   1119 	aprint_normal(" ETrackID %08x\n", ((uint32_t)high << 16) | low);
   1120 
   1121 	/* Printed Board Assembly number */
   1122 	error = ixgbe_read_pba_string(hw, buf, IXGBE_PBANUM_LENGTH);
   1123 	aprint_normal_dev(dev, "PBA number %s\n", error ? "unknown" : buf);
   1124 
   1125 	/* Recovery mode */
   1126 	switch (sc->hw.mac.type) {
   1127 	case ixgbe_mac_X550:
   1128 	case ixgbe_mac_X550EM_x:
   1129 	case ixgbe_mac_X550EM_a:
   1130 		/* >= 2.00 */
   1131 		if (hw->eeprom.nvm_image_ver_high >= 2) {
   1132 			sc->feat_cap |= IXGBE_FEATURE_RECOVERY_MODE;
   1133 			sc->feat_en |= IXGBE_FEATURE_RECOVERY_MODE;
   1134 		}
   1135 		break;
   1136 	default:
   1137 		break;
   1138 	}
   1139 
   1140 	if (sc->feat_en & IXGBE_FEATURE_MSIX) {
   1141 		error = ixgbe_allocate_msix(sc, pa);
   1142 		if (error) {
   1143 			/* Free allocated queue structures first */
   1144 			ixgbe_free_queues(sc);
   1145 
   1146 			/* Fallback to legacy interrupt */
   1147 			if (sc->feat_cap & IXGBE_FEATURE_MSI)
   1148 				sc->feat_en |= IXGBE_FEATURE_MSI;
   1149 			sc->num_queues = 1;
   1150 
   1151 			/* Allocate our TX/RX Queues again */
   1152 			if (ixgbe_allocate_queues(sc)) {
   1153 				error = ENOMEM;
   1154 				goto err_out;
   1155 			}
   1156 		}
   1157 	}
   1158 
   1159 	if ((sc->feat_en & IXGBE_FEATURE_MSIX) == 0)
   1160 		error = ixgbe_allocate_legacy(sc, pa);
   1161 	if (error)
   1162 		goto err_late;
   1163 
   1164 	/* Tasklets for Link, SFP, Multispeed Fiber and Flow Director */
   1165 	mutex_init(&(sc)->admin_mtx, MUTEX_DEFAULT, IPL_NET);
   1166 	snprintf(wqname, sizeof(wqname), "%s-admin", device_xname(dev));
   1167 	error = workqueue_create(&sc->admin_wq, wqname,
   1168 	    ixgbe_handle_admin, sc, IXGBE_WORKQUEUE_PRI, IPL_NET, WQ_MPSAFE);
   1169 	if (error) {
   1170 		aprint_error_dev(dev,
   1171 		    "could not create admin workqueue (%d)\n", error);
   1172 		goto err_out;
   1173 	}
   1174 
   1175 	error = ixgbe_start_hw(hw);
   1176 	switch (error) {
   1177 	case IXGBE_ERR_EEPROM_VERSION:
   1178 		aprint_error_dev(dev,
   1179 		    "This device is a pre-production adapter/"
   1180 		    "LOM.  Please be aware there may be issues associated "
   1181 		    "with your hardware.\nIf you are experiencing problems "
   1182 		    "please contact your Intel or hardware representative "
   1183 		    "who provided you with this hardware.\n");
   1184 		break;
   1185 	default:
   1186 		break;
   1187 	}
   1188 
   1189 	/* Setup OS specific network interface */
   1190 	if (ixgbe_setup_interface(dev, sc) != 0)
   1191 		goto err_late;
   1192 
   1193 	/*
   1194 	 *  Print PHY ID only for copper PHY. On device which has SFP(+) cage
   1195 	 * and a module is inserted, phy.id is not MII PHY id but SFF 8024 ID.
   1196 	 */
   1197 	if (hw->phy.media_type == ixgbe_media_type_copper) {
   1198 		uint16_t id1, id2;
   1199 		int oui, model, rev;
   1200 		char descr[MII_MAX_DESCR_LEN];
   1201 
   1202 		id1 = hw->phy.id >> 16;
   1203 		id2 = hw->phy.id & 0xffff;
   1204 		oui = MII_OUI(id1, id2);
   1205 		model = MII_MODEL(id2);
   1206 		rev = MII_REV(id2);
   1207 		mii_get_descr(descr, sizeof(descr), oui, model);
   1208 		if (descr[0])
   1209 			aprint_normal_dev(dev, "PHY: %s, rev. %d\n",
   1210 			    descr, rev);
   1211 		else
   1212 			aprint_normal_dev(dev,
   1213 			    "PHY OUI 0x%06x, model 0x%04x, rev. %d\n",
   1214 			    oui, model, rev);
   1215 	}
   1216 
   1217 	/* Enable EEE power saving */
   1218 	if (sc->feat_cap & IXGBE_FEATURE_EEE)
   1219 		hw->mac.ops.setup_eee(hw,
   1220 		    sc->feat_en & IXGBE_FEATURE_EEE);
   1221 
   1222 	/* Enable power to the phy. */
   1223 	if (!unsupported_sfp) {
   1224 		/* Enable the optics for 82599 SFP+ fiber */
   1225 		ixgbe_enable_tx_laser(hw);
   1226 
   1227 		/*
   1228 		 * XXX Currently, ixgbe_set_phy_power() supports only copper
   1229 		 * PHY, so it's not required to test with !unsupported_sfp.
   1230 		 */
   1231 		ixgbe_set_phy_power(hw, TRUE);
   1232 	}
   1233 
   1234 	/* Initialize statistics */
   1235 	ixgbe_update_stats_counters(sc);
   1236 
   1237 	/* Check PCIE slot type/speed/width */
   1238 	ixgbe_get_slot_info(sc);
   1239 
   1240 	/*
   1241 	 * Do time init and sysctl init here, but
   1242 	 * only on the first port of a bypass adapter.
   1243 	 */
   1244 	ixgbe_bypass_init(sc);
   1245 
   1246 	/* Set an initial dmac value */
   1247 	sc->dmac = 0;
   1248 	/* Set initial advertised speeds (if applicable) */
   1249 	sc->advertise = ixgbe_get_default_advertise(sc);
   1250 
   1251 	if (sc->feat_cap & IXGBE_FEATURE_SRIOV)
   1252 		ixgbe_define_iov_schemas(dev, &error);
   1253 
   1254 	/* Add sysctls */
   1255 	ixgbe_add_device_sysctls(sc);
   1256 	ixgbe_add_hw_stats(sc);
   1257 
   1258 	/* For Netmap */
   1259 	sc->init_locked = ixgbe_init_locked;
   1260 	sc->stop_locked = ixgbe_stop_locked;
   1261 
   1262 	if (sc->feat_en & IXGBE_FEATURE_NETMAP)
   1263 		ixgbe_netmap_attach(sc);
   1264 
   1265 	/* Print some flags */
   1266 	snprintb(buf, sizeof(buf), IXGBE_FEATURE_FLAGS, sc->feat_cap);
   1267 	aprint_verbose_dev(dev, "feature cap %s\n", buf);
   1268 	snprintb(buf, sizeof(buf), IXGBE_FEATURE_FLAGS, sc->feat_en);
   1269 	aprint_verbose_dev(dev, "feature ena %s\n", buf);
   1270 	if (ixgbe_get_device_caps(hw, &dev_caps) == 0) {
   1271 		snprintb(buf, sizeof(buf), IXGBE_DEVICE_CAPS_FLAGS, dev_caps);
   1272 		aprint_verbose_dev(dev, "device cap %s\n", buf);
   1273 	}
   1274 
   1275 	if (pmf_device_register(dev, ixgbe_suspend, ixgbe_resume))
   1276 		pmf_class_network_register(dev, sc->ifp);
   1277 	else
   1278 		aprint_error_dev(dev, "couldn't establish power handler\n");
   1279 
   1280 	/* Init recovery mode timer and state variable */
   1281 	if (sc->feat_en & IXGBE_FEATURE_RECOVERY_MODE) {
   1282 		sc->recovery_mode = 0;
   1283 
   1284 		/* Set up the timer callout */
   1285 		callout_init(&sc->recovery_mode_timer, CALLOUT_MPSAFE);
   1286 		snprintf(wqname, sizeof(wqname), "%s-recovery",
   1287 		    device_xname(dev));
   1288 		error = workqueue_create(&sc->recovery_mode_timer_wq,
   1289 		    wqname, ixgbe_handle_recovery_mode_timer, sc,
   1290 		    IXGBE_WORKQUEUE_PRI, IPL_NET, WQ_MPSAFE);
   1291 		if (error) {
   1292 			aprint_error_dev(dev, "could not create "
   1293 			    "recovery_mode_timer workqueue (%d)\n", error);
   1294 			goto err_out;
   1295 		}
   1296 
   1297 		/* Start the task */
   1298 		callout_reset(&sc->recovery_mode_timer, hz,
   1299 		    ixgbe_recovery_mode_timer, sc);
   1300 	}
   1301 
   1302 	INIT_DEBUGOUT("ixgbe_attach: end");
   1303 	sc->osdep.attached = true;
   1304 
   1305 	return;
   1306 
   1307 err_late:
   1308 	ixgbe_free_queues(sc);
   1309 err_out:
   1310 	ctrl_ext = IXGBE_READ_REG(&sc->hw, IXGBE_CTRL_EXT);
   1311 	ctrl_ext &= ~IXGBE_CTRL_EXT_DRV_LOAD;
   1312 	IXGBE_WRITE_REG(&sc->hw, IXGBE_CTRL_EXT, ctrl_ext);
   1313 	ixgbe_free_deferred_handlers(sc);
   1314 	ixgbe_free_pci_resources(sc);
   1315 	if (sc->mta != NULL)
   1316 		free(sc->mta, M_DEVBUF);
   1317 	mutex_destroy(&(sc)->admin_mtx); /* XXX appropriate order? */
   1318 	IXGBE_CORE_LOCK_DESTROY(sc);
   1319 
   1320 	return;
   1321 } /* ixgbe_attach */
   1322 
   1323 /************************************************************************
   1324  * ixgbe_check_wol_support
   1325  *
   1326  *   Checks whether the adapter's ports are capable of
   1327  *   Wake On LAN by reading the adapter's NVM.
   1328  *
   1329  *   Sets each port's hw->wol_enabled value depending
   1330  *   on the value read here.
   1331  ************************************************************************/
   1332 static void
   1333 ixgbe_check_wol_support(struct ixgbe_softc *sc)
   1334 {
   1335 	struct ixgbe_hw *hw = &sc->hw;
   1336 	u16		dev_caps = 0;
   1337 
   1338 	/* Find out WoL support for port */
   1339 	sc->wol_support = hw->wol_enabled = 0;
   1340 	ixgbe_get_device_caps(hw, &dev_caps);
   1341 	if ((dev_caps & IXGBE_DEVICE_CAPS_WOL_PORT0_1) ||
   1342 	    ((dev_caps & IXGBE_DEVICE_CAPS_WOL_PORT0) &&
   1343 	     hw->bus.func == 0))
   1344 		sc->wol_support = hw->wol_enabled = 1;
   1345 
   1346 	/* Save initial wake up filter configuration */
   1347 	sc->wufc = IXGBE_READ_REG(hw, IXGBE_WUFC);
   1348 
   1349 	return;
   1350 } /* ixgbe_check_wol_support */
   1351 
   1352 /************************************************************************
   1353  * ixgbe_setup_interface
   1354  *
   1355  *   Setup networking device structure and register an interface.
   1356  ************************************************************************/
   1357 static int
   1358 ixgbe_setup_interface(device_t dev, struct ixgbe_softc *sc)
   1359 {
   1360 	struct ethercom *ec = &sc->osdep.ec;
   1361 	struct ifnet   *ifp;
   1362 
   1363 	INIT_DEBUGOUT("ixgbe_setup_interface: begin");
   1364 
   1365 	ifp = sc->ifp = &ec->ec_if;
   1366 	strlcpy(ifp->if_xname, device_xname(dev), IFNAMSIZ);
   1367 	ifp->if_baudrate = IF_Gbps(10);
   1368 	ifp->if_init = ixgbe_init;
   1369 	ifp->if_stop = ixgbe_ifstop;
   1370 	ifp->if_softc = sc;
   1371 	ifp->if_flags = IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST;
   1372 	ifp->if_extflags = IFEF_MPSAFE;
   1373 	ifp->if_ioctl = ixgbe_ioctl;
   1374 #if __FreeBSD_version >= 1100045
   1375 	/* TSO parameters */
   1376 	ifp->if_hw_tsomax = 65518;
   1377 	ifp->if_hw_tsomaxsegcount = IXGBE_82599_SCATTER;
   1378 	ifp->if_hw_tsomaxsegsize = 2048;
   1379 #endif
   1380 	if (sc->feat_en & IXGBE_FEATURE_LEGACY_TX) {
   1381 #if 0
   1382 		ixgbe_start_locked = ixgbe_legacy_start_locked;
   1383 #endif
   1384 	} else {
   1385 		ifp->if_transmit = ixgbe_mq_start;
   1386 #if 0
   1387 		ixgbe_start_locked = ixgbe_mq_start_locked;
   1388 #endif
   1389 	}
   1390 	ifp->if_start = ixgbe_legacy_start;
   1391 	IFQ_SET_MAXLEN(&ifp->if_snd, sc->num_tx_desc - 2);
   1392 	IFQ_SET_READY(&ifp->if_snd);
   1393 
   1394 	if_initialize(ifp);
   1395 	sc->ipq = if_percpuq_create(&sc->osdep.ec.ec_if);
   1396 	/*
   1397 	 * We use per TX queue softint, so if_deferred_start_init() isn't
   1398 	 * used.
   1399 	 */
   1400 
   1401 	sc->max_frame_size = ifp->if_mtu + ETHER_HDR_LEN + ETHER_CRC_LEN;
   1402 
   1403 	/*
   1404 	 * Tell the upper layer(s) we support long frames.
   1405 	 */
   1406 	ifp->if_hdrlen = sizeof(struct ether_vlan_header);
   1407 
   1408 	/* Set capability flags */
   1409 	ifp->if_capabilities |= IFCAP_RXCSUM
   1410 			     |	IFCAP_TXCSUM
   1411 			     |	IFCAP_TSOv4
   1412 			     |	IFCAP_TSOv6;
   1413 	ifp->if_capenable = 0;
   1414 
   1415 	ec->ec_capabilities |= ETHERCAP_VLAN_HWTAGGING
   1416 			    |  ETHERCAP_VLAN_HWCSUM
   1417 			    |  ETHERCAP_JUMBO_MTU
   1418 			    |  ETHERCAP_VLAN_MTU;
   1419 
   1420 	/* Enable the above capabilities by default */
   1421 	ec->ec_capenable = ec->ec_capabilities;
   1422 
   1423 	ether_ifattach(ifp, sc->hw.mac.addr);
   1424 	aprint_normal_dev(dev, "Ethernet address %s\n",
   1425 	    ether_sprintf(sc->hw.mac.addr));
   1426 	ether_set_ifflags_cb(ec, ixgbe_ifflags_cb);
   1427 
   1428 	/*
   1429 	 * Don't turn this on by default, if vlans are
   1430 	 * created on another pseudo device (eg. lagg)
   1431 	 * then vlan events are not passed thru, breaking
   1432 	 * operation, but with HW FILTER off it works. If
   1433 	 * using vlans directly on the ixgbe driver you can
   1434 	 * enable this and get full hardware tag filtering.
   1435 	 */
   1436 	ec->ec_capabilities |= ETHERCAP_VLAN_HWFILTER;
   1437 
   1438 	/*
   1439 	 * Specify the media types supported by this adapter and register
   1440 	 * callbacks to update media and link information
   1441 	 */
   1442 	ec->ec_ifmedia = &sc->media;
   1443 	ifmedia_init_with_lock(&sc->media, IFM_IMASK, ixgbe_media_change,
   1444 	    ixgbe_media_status, &sc->core_mtx);
   1445 
   1446 	sc->phy_layer = ixgbe_get_supported_physical_layer(&sc->hw);
   1447 	ixgbe_add_media_types(sc);
   1448 
   1449 	/* Set autoselect media by default */
   1450 	ifmedia_set(&sc->media, IFM_ETHER | IFM_AUTO);
   1451 
   1452 	if_register(ifp);
   1453 
   1454 	return (0);
   1455 } /* ixgbe_setup_interface */
   1456 
   1457 /************************************************************************
   1458  * ixgbe_add_media_types
   1459  ************************************************************************/
   1460 static void
   1461 ixgbe_add_media_types(struct ixgbe_softc *sc)
   1462 {
   1463 	struct ixgbe_hw *hw = &sc->hw;
   1464 	u64		layer;
   1465 
   1466 	layer = sc->phy_layer;
   1467 
   1468 #define	ADD(mm, dd)							\
   1469 	ifmedia_add(&sc->media, IFM_ETHER | (mm), (dd), NULL);
   1470 
   1471 	ADD(IFM_NONE, 0);
   1472 
   1473 	/* Media types with matching NetBSD media defines */
   1474 	if (layer & IXGBE_PHYSICAL_LAYER_10GBASE_T)
   1475 		ADD(IFM_10G_T | IFM_FDX, 0);
   1476 	if (layer & IXGBE_PHYSICAL_LAYER_1000BASE_T)
   1477 		ADD(IFM_1000_T | IFM_FDX, 0);
   1478 	if (layer & IXGBE_PHYSICAL_LAYER_100BASE_TX)
   1479 		ADD(IFM_100_TX | IFM_FDX, 0);
   1480 	if (layer & IXGBE_PHYSICAL_LAYER_10BASE_T)
   1481 		ADD(IFM_10_T | IFM_FDX, 0);
   1482 
   1483 	if (layer & IXGBE_PHYSICAL_LAYER_SFP_PLUS_CU ||
   1484 	    layer & IXGBE_PHYSICAL_LAYER_SFP_ACTIVE_DA)
   1485 		ADD(IFM_10G_TWINAX | IFM_FDX, 0);
   1486 
   1487 	if (layer & IXGBE_PHYSICAL_LAYER_10GBASE_LR) {
   1488 		ADD(IFM_10G_LR | IFM_FDX, 0);
   1489 		if (hw->phy.multispeed_fiber)
   1490 			ADD(IFM_1000_LX | IFM_FDX, 0);
   1491 	}
   1492 	if (layer & IXGBE_PHYSICAL_LAYER_10GBASE_SR) {
   1493 		ADD(IFM_10G_SR | IFM_FDX, 0);
   1494 		if (hw->phy.multispeed_fiber)
   1495 			ADD(IFM_1000_SX | IFM_FDX, 0);
   1496 	} else if (layer & IXGBE_PHYSICAL_LAYER_1000BASE_SX)
   1497 		ADD(IFM_1000_SX | IFM_FDX, 0);
   1498 	if (layer & IXGBE_PHYSICAL_LAYER_10GBASE_CX4)
   1499 		ADD(IFM_10G_CX4 | IFM_FDX, 0);
   1500 
   1501 	if (layer & IXGBE_PHYSICAL_LAYER_10GBASE_KR)
   1502 		ADD(IFM_10G_KR | IFM_FDX, 0);
   1503 	if (layer & IXGBE_PHYSICAL_LAYER_10GBASE_KX4)
   1504 		ADD(IFM_10G_KX4 | IFM_FDX, 0);
   1505 	if (layer & IXGBE_PHYSICAL_LAYER_1000BASE_KX)
   1506 		ADD(IFM_1000_KX | IFM_FDX, 0);
   1507 	if (layer & IXGBE_PHYSICAL_LAYER_2500BASE_KX)
   1508 		ADD(IFM_2500_KX | IFM_FDX, 0);
   1509 	if (layer & IXGBE_PHYSICAL_LAYER_2500BASE_T)
   1510 		ADD(IFM_2500_T | IFM_FDX, 0);
   1511 	if (layer & IXGBE_PHYSICAL_LAYER_5GBASE_T)
   1512 		ADD(IFM_5000_T | IFM_FDX, 0);
   1513 	if (layer & IXGBE_PHYSICAL_LAYER_1000BASE_BX)
   1514 		ADD(IFM_1000_BX10 | IFM_FDX, 0);
   1515 	/* XXX no ifmedia_set? */
   1516 
   1517 	ADD(IFM_AUTO, 0);
   1518 
   1519 #undef ADD
   1520 } /* ixgbe_add_media_types */
   1521 
   1522 /************************************************************************
   1523  * ixgbe_is_sfp
   1524  ************************************************************************/
   1525 static inline bool
   1526 ixgbe_is_sfp(struct ixgbe_hw *hw)
   1527 {
   1528 	switch (hw->mac.type) {
   1529 	case ixgbe_mac_82598EB:
   1530 		if (hw->phy.type == ixgbe_phy_nl)
   1531 			return (TRUE);
   1532 		return (FALSE);
   1533 	case ixgbe_mac_82599EB:
   1534 	case ixgbe_mac_X550EM_x:
   1535 	case ixgbe_mac_X550EM_a:
   1536 		switch (hw->mac.ops.get_media_type(hw)) {
   1537 		case ixgbe_media_type_fiber:
   1538 		case ixgbe_media_type_fiber_qsfp:
   1539 			return (TRUE);
   1540 		default:
   1541 			return (FALSE);
   1542 		}
   1543 	default:
   1544 		return (FALSE);
   1545 	}
   1546 } /* ixgbe_is_sfp */
   1547 
   1548 static void
   1549 ixgbe_schedule_admin_tasklet(struct ixgbe_softc *sc)
   1550 {
   1551 
   1552 	KASSERT(mutex_owned(&sc->admin_mtx));
   1553 
   1554 	if (__predict_true(sc->osdep.detaching == false)) {
   1555 		if (sc->admin_pending == 0)
   1556 			workqueue_enqueue(sc->admin_wq,
   1557 			    &sc->admin_wc, NULL);
   1558 		sc->admin_pending = 1;
   1559 	}
   1560 }
   1561 
   1562 /************************************************************************
   1563  * ixgbe_config_link
   1564  ************************************************************************/
   1565 static void
   1566 ixgbe_config_link(struct ixgbe_softc *sc)
   1567 {
   1568 	struct ixgbe_hw *hw = &sc->hw;
   1569 	u32		autoneg, err = 0;
   1570 	u32		task_requests = 0;
   1571 	bool		sfp, negotiate = false;
   1572 
   1573 	sfp = ixgbe_is_sfp(hw);
   1574 
   1575 	if (sfp) {
   1576 		if (hw->phy.multispeed_fiber) {
   1577 			ixgbe_enable_tx_laser(hw);
   1578 			task_requests |= IXGBE_REQUEST_TASK_MSF_WOI;
   1579 		}
   1580 		task_requests |= IXGBE_REQUEST_TASK_MOD_WOI;
   1581 
   1582 		mutex_enter(&sc->admin_mtx);
   1583 		sc->task_requests |= task_requests;
   1584 		ixgbe_schedule_admin_tasklet(sc);
   1585 		mutex_exit(&sc->admin_mtx);
   1586 	} else {
   1587 		struct ifmedia	*ifm = &sc->media;
   1588 
   1589 		if (hw->mac.ops.check_link)
   1590 			err = ixgbe_check_link(hw, &sc->link_speed,
   1591 			    &sc->link_up, FALSE);
   1592 		if (err)
   1593 			return;
   1594 
   1595 		/*
   1596 		 * Check if it's the first call. If it's the first call,
   1597 		 * get value for auto negotiation.
   1598 		 */
   1599 		autoneg = hw->phy.autoneg_advertised;
   1600 		if ((IFM_SUBTYPE(ifm->ifm_cur->ifm_media) != IFM_NONE)
   1601 		    && ((!autoneg) && (hw->mac.ops.get_link_capabilities)))
   1602 			err = hw->mac.ops.get_link_capabilities(hw, &autoneg,
   1603 			    &negotiate);
   1604 		if (err)
   1605 			return;
   1606 		if (hw->mac.ops.setup_link)
   1607 			err = hw->mac.ops.setup_link(hw, autoneg,
   1608 			    sc->link_up);
   1609 	}
   1610 } /* ixgbe_config_link */
   1611 
   1612 /************************************************************************
   1613  * ixgbe_update_stats_counters - Update board statistics counters.
   1614  ************************************************************************/
   1615 static void
   1616 ixgbe_update_stats_counters(struct ixgbe_softc *sc)
   1617 {
   1618 	struct ifnet	      *ifp = sc->ifp;
   1619 	struct ixgbe_hw	      *hw = &sc->hw;
   1620 	struct ixgbe_hw_stats *stats = &sc->stats.pf;
   1621 	u32		      missed_rx = 0, bprc, lxontxc, lxofftxc;
   1622 	u64		      total, total_missed_rx = 0, total_qprdc = 0;
   1623 	uint64_t	      crcerrs, illerrc, rlec, ruc, rfc, roc, rjc;
   1624 	unsigned int	      queue_counters;
   1625 	int		      i;
   1626 
   1627 	IXGBE_EVC_REGADD2(hw, stats, IXGBE_CRCERRS, crcerrs);
   1628 	IXGBE_EVC_REGADD2(hw, stats, IXGBE_ILLERRC, illerrc);
   1629 
   1630 	IXGBE_EVC_REGADD(hw, stats, IXGBE_ERRBC, errbc);
   1631 	IXGBE_EVC_REGADD(hw, stats, IXGBE_MSPDC, mspdc);
   1632 	if (hw->mac.type >= ixgbe_mac_X550)
   1633 		IXGBE_EVC_REGADD(hw, stats, IXGBE_MBSDC, mbsdc);
   1634 
   1635 	/* 16 registers exist */
   1636 	queue_counters = uimin(__arraycount(stats->qprc), sc->num_queues);
   1637 	for (i = 0; i < queue_counters; i++) {
   1638 		IXGBE_EVC_REGADD(hw, stats, IXGBE_QPRC(i), qprc[i]);
   1639 		IXGBE_EVC_REGADD(hw, stats, IXGBE_QPTC(i), qptc[i]);
   1640 		if (hw->mac.type >= ixgbe_mac_82599EB) {
   1641 			uint32_t qprdc;
   1642 
   1643 			IXGBE_EVC_ADD(&stats->qbrc[i],
   1644 			    IXGBE_READ_REG(hw, IXGBE_QBRC_L(i)) +
   1645 			    ((u64)IXGBE_READ_REG(hw, IXGBE_QBRC_H(i)) << 32));
   1646 			IXGBE_EVC_ADD(&stats->qbtc[i],
   1647 			    IXGBE_READ_REG(hw, IXGBE_QBTC_L(i)) +
   1648 			    ((u64)IXGBE_READ_REG(hw, IXGBE_QBTC_H(i)) << 32));
   1649 			/* QPRDC will be added to iqdrops. */
   1650 			qprdc = IXGBE_READ_REG(hw, IXGBE_QPRDC(i));
   1651 			IXGBE_EVC_ADD(&stats->qprdc[i], qprdc);
   1652 			total_qprdc += qprdc;
   1653 		} else {
   1654 			/* 82598 */
   1655 			IXGBE_EVC_REGADD(hw, stats, IXGBE_QBRC(i), qbrc[i]);
   1656 			IXGBE_EVC_REGADD(hw, stats, IXGBE_QBTC(i), qbtc[i]);
   1657 		}
   1658 	}
   1659 
   1660 	/* 8 registers exist */
   1661 	for (i = 0; i < IXGBE_TC_COUNTER_NUM; i++) {
   1662 		uint32_t mp;
   1663 
   1664 		/* MPC */
   1665 		mp = IXGBE_READ_REG(hw, IXGBE_MPC(i));
   1666 		/* global total per queue */
   1667 		IXGBE_EVC_ADD(&stats->mpc[i], mp);
   1668 		/* running comprehensive total for stats display */
   1669 		total_missed_rx += mp;
   1670 
   1671 		if (hw->mac.type == ixgbe_mac_82598EB)
   1672 			IXGBE_EVC_REGADD(hw, stats, IXGBE_RNBC(i), rnbc[i]);
   1673 
   1674 		IXGBE_EVC_REGADD(hw, stats, IXGBE_PXONTXC(i), pxontxc[i]);
   1675 		IXGBE_EVC_REGADD(hw, stats, IXGBE_PXOFFTXC(i), pxofftxc[i]);
   1676 		if (hw->mac.type >= ixgbe_mac_82599EB) {
   1677 			IXGBE_EVC_REGADD(hw, stats,
   1678 			    IXGBE_PXONRXCNT(i), pxonrxc[i]);
   1679 			IXGBE_EVC_REGADD(hw, stats,
   1680 			    IXGBE_PXOFFRXCNT(i), pxoffrxc[i]);
   1681 			IXGBE_EVC_REGADD(hw, stats,
   1682 			    IXGBE_PXON2OFFCNT(i), pxon2offc[i]);
   1683 		} else {
   1684 			IXGBE_EVC_REGADD(hw, stats,
   1685 			    IXGBE_PXONRXC(i), pxonrxc[i]);
   1686 			IXGBE_EVC_REGADD(hw, stats,
   1687 			    IXGBE_PXOFFRXC(i), pxoffrxc[i]);
   1688 		}
   1689 	}
   1690 	IXGBE_EVC_ADD(&stats->mpctotal, total_missed_rx);
   1691 
   1692 	/* Document says M[LR]FC are valid when link is up and 10Gbps */
   1693 	if ((sc->link_active == LINK_STATE_UP)
   1694 	    && (sc->link_speed == IXGBE_LINK_SPEED_10GB_FULL)) {
   1695 		IXGBE_EVC_REGADD(hw, stats, IXGBE_MLFC, mlfc);
   1696 		IXGBE_EVC_REGADD(hw, stats, IXGBE_MRFC, mrfc);
   1697 	}
   1698 	if (hw->mac.type == ixgbe_mac_X550EM_a)
   1699 		IXGBE_EVC_REGADD(hw, stats, IXGBE_LINK_DN_CNT, link_dn_cnt);
   1700 	IXGBE_EVC_REGADD2(hw, stats, IXGBE_RLEC, rlec);
   1701 
   1702 	/* Hardware workaround, gprc counts missed packets */
   1703 	IXGBE_EVC_ADD(&stats->gprc,
   1704 	    IXGBE_READ_REG(hw, IXGBE_GPRC) - missed_rx);
   1705 
   1706 	IXGBE_EVC_REGADD2(hw, stats, IXGBE_LXONTXC, lxontxc);
   1707 	IXGBE_EVC_REGADD2(hw, stats, IXGBE_LXOFFTXC, lxofftxc);
   1708 	total = lxontxc + lxofftxc;
   1709 
   1710 	if (hw->mac.type != ixgbe_mac_82598EB) {
   1711 		IXGBE_EVC_ADD(&stats->gorc, IXGBE_READ_REG(hw, IXGBE_GORCL) +
   1712 		    ((u64)IXGBE_READ_REG(hw, IXGBE_GORCH) << 32));
   1713 		IXGBE_EVC_ADD(&stats->gotc, IXGBE_READ_REG(hw, IXGBE_GOTCL) +
   1714 		    ((u64)IXGBE_READ_REG(hw, IXGBE_GOTCH) << 32)
   1715 		    - total * ETHER_MIN_LEN);
   1716 		IXGBE_EVC_ADD(&stats->tor, IXGBE_READ_REG(hw, IXGBE_TORL) +
   1717 		    ((u64)IXGBE_READ_REG(hw, IXGBE_TORH) << 32));
   1718 		IXGBE_EVC_REGADD(hw, stats, IXGBE_LXONRXCNT, lxonrxc);
   1719 		IXGBE_EVC_REGADD(hw, stats, IXGBE_LXOFFRXCNT, lxoffrxc);
   1720 	} else {
   1721 		IXGBE_EVC_REGADD(hw, stats, IXGBE_LXONRXC, lxonrxc);
   1722 		IXGBE_EVC_REGADD(hw, stats, IXGBE_LXOFFRXC, lxoffrxc);
   1723 		/* 82598 only has a counter in the high register */
   1724 		IXGBE_EVC_REGADD(hw, stats, IXGBE_GORCH, gorc);
   1725 		IXGBE_EVC_ADD(&stats->gotc, IXGBE_READ_REG(hw, IXGBE_GOTCH)
   1726 		    - total * ETHER_MIN_LEN);
   1727 		IXGBE_EVC_REGADD(hw, stats, IXGBE_TORH, tor);
   1728 	}
   1729 
   1730 	/*
   1731 	 * Workaround: mprc hardware is incorrectly counting
   1732 	 * broadcasts, so for now we subtract those.
   1733 	 */
   1734 	IXGBE_EVC_REGADD2(hw, stats, IXGBE_BPRC, bprc);
   1735 	IXGBE_EVC_ADD(&stats->mprc, IXGBE_READ_REG(hw, IXGBE_MPRC)
   1736 	    - ((hw->mac.type == ixgbe_mac_82598EB) ? bprc : 0));
   1737 
   1738 	IXGBE_EVC_REGADD(hw, stats, IXGBE_PRC64, prc64);
   1739 	IXGBE_EVC_REGADD(hw, stats, IXGBE_PRC127, prc127);
   1740 	IXGBE_EVC_REGADD(hw, stats, IXGBE_PRC255, prc255);
   1741 	IXGBE_EVC_REGADD(hw, stats, IXGBE_PRC511, prc511);
   1742 	IXGBE_EVC_REGADD(hw, stats, IXGBE_PRC1023, prc1023);
   1743 	IXGBE_EVC_REGADD(hw, stats, IXGBE_PRC1522, prc1522);
   1744 
   1745 	IXGBE_EVC_ADD(&stats->gptc, IXGBE_READ_REG(hw, IXGBE_GPTC) - total);
   1746 	IXGBE_EVC_ADD(&stats->mptc, IXGBE_READ_REG(hw, IXGBE_MPTC) - total);
   1747 	IXGBE_EVC_ADD(&stats->ptc64, IXGBE_READ_REG(hw, IXGBE_PTC64) - total);
   1748 
   1749 	IXGBE_EVC_REGADD2(hw, stats, IXGBE_RUC, ruc);
   1750 	IXGBE_EVC_REGADD2(hw, stats, IXGBE_RFC, rfc);
   1751 	IXGBE_EVC_REGADD2(hw, stats, IXGBE_ROC, roc);
   1752 	IXGBE_EVC_REGADD2(hw, stats, IXGBE_RJC, rjc);
   1753 
   1754 	IXGBE_EVC_REGADD(hw, stats, IXGBE_MNGPRC, mngprc);
   1755 	IXGBE_EVC_REGADD(hw, stats, IXGBE_MNGPDC, mngpdc);
   1756 	IXGBE_EVC_REGADD(hw, stats, IXGBE_MNGPTC, mngptc);
   1757 	IXGBE_EVC_REGADD(hw, stats, IXGBE_TPR, tpr);
   1758 	IXGBE_EVC_REGADD(hw, stats, IXGBE_TPT, tpt);
   1759 	IXGBE_EVC_REGADD(hw, stats, IXGBE_PTC127, ptc127);
   1760 	IXGBE_EVC_REGADD(hw, stats, IXGBE_PTC255, ptc255);
   1761 	IXGBE_EVC_REGADD(hw, stats, IXGBE_PTC511, ptc511);
   1762 	IXGBE_EVC_REGADD(hw, stats, IXGBE_PTC1023, ptc1023);
   1763 	IXGBE_EVC_REGADD(hw, stats, IXGBE_PTC1522, ptc1522);
   1764 	IXGBE_EVC_REGADD(hw, stats, IXGBE_BPTC, bptc);
   1765 	IXGBE_EVC_REGADD(hw, stats, IXGBE_XEC, xec);
   1766 	IXGBE_EVC_REGADD(hw, stats, IXGBE_FCCRC, fccrc);
   1767 	IXGBE_EVC_REGADD(hw, stats, IXGBE_FCLAST, fclast);
   1768 	/* Only read FCOE on 82599 */
   1769 	if (hw->mac.type != ixgbe_mac_82598EB) {
   1770 		IXGBE_EVC_REGADD(hw, stats, IXGBE_FCOERPDC, fcoerpdc);
   1771 		IXGBE_EVC_REGADD(hw, stats, IXGBE_FCOEPRC, fcoeprc);
   1772 		IXGBE_EVC_REGADD(hw, stats, IXGBE_FCOEPTC, fcoeptc);
   1773 		IXGBE_EVC_REGADD(hw, stats, IXGBE_FCOEDWRC, fcoedwrc);
   1774 		IXGBE_EVC_REGADD(hw, stats, IXGBE_FCOEDWTC, fcoedwtc);
   1775 	}
   1776 
   1777 	/*
   1778 	 * Fill out the OS statistics structure. Only RX errors are required
   1779 	 * here because all TX counters are incremented in the TX path and
   1780 	 * normal RX counters are prepared in ether_input().
   1781 	 */
   1782 	net_stat_ref_t nsr = IF_STAT_GETREF(ifp);
   1783 	if_statadd_ref(ifp, nsr, if_iqdrops, total_missed_rx + total_qprdc);
   1784 
   1785 	/*
   1786 	 * Aggregate following types of errors as RX errors:
   1787 	 * - CRC error count,
   1788 	 * - illegal byte error count,
   1789 	 * - length error count,
   1790 	 * - undersized packets count,
   1791 	 * - fragmented packets count,
   1792 	 * - oversized packets count,
   1793 	 * - jabber count.
   1794 	 */
   1795 	if_statadd_ref(ifp, nsr, if_ierrors,
   1796 	    crcerrs + illerrc + rlec + ruc + rfc + roc + rjc);
   1797 
   1798 	IF_STAT_PUTREF(ifp);
   1799 } /* ixgbe_update_stats_counters */
   1800 
   1801 /************************************************************************
   1802  * ixgbe_add_hw_stats
   1803  *
   1804  *   Add sysctl variables, one per statistic, to the system.
   1805  ************************************************************************/
   1806 static void
   1807 ixgbe_add_hw_stats(struct ixgbe_softc *sc)
   1808 {
   1809 	device_t dev = sc->dev;
   1810 	const struct sysctlnode *rnode, *cnode;
   1811 	struct sysctllog **log = &sc->sysctllog;
   1812 	struct tx_ring *txr = sc->tx_rings;
   1813 	struct rx_ring *rxr = sc->rx_rings;
   1814 	struct ixgbe_hw *hw = &sc->hw;
   1815 	struct ixgbe_hw_stats *stats = &sc->stats.pf;
   1816 	const char *xname = device_xname(dev);
   1817 	int i;
   1818 
   1819 	/* Driver Statistics */
   1820 	evcnt_attach_dynamic(&sc->efbig_tx_dma_setup, EVCNT_TYPE_MISC,
   1821 	    NULL, xname, "Driver tx dma soft fail EFBIG");
   1822 	evcnt_attach_dynamic(&sc->mbuf_defrag_failed, EVCNT_TYPE_MISC,
   1823 	    NULL, xname, "m_defrag() failed");
   1824 	evcnt_attach_dynamic(&sc->efbig2_tx_dma_setup, EVCNT_TYPE_MISC,
   1825 	    NULL, xname, "Driver tx dma hard fail EFBIG");
   1826 	evcnt_attach_dynamic(&sc->einval_tx_dma_setup, EVCNT_TYPE_MISC,
   1827 	    NULL, xname, "Driver tx dma hard fail EINVAL");
   1828 	evcnt_attach_dynamic(&sc->other_tx_dma_setup, EVCNT_TYPE_MISC,
   1829 	    NULL, xname, "Driver tx dma hard fail other");
   1830 	evcnt_attach_dynamic(&sc->eagain_tx_dma_setup, EVCNT_TYPE_MISC,
   1831 	    NULL, xname, "Driver tx dma soft fail EAGAIN");
   1832 	evcnt_attach_dynamic(&sc->enomem_tx_dma_setup, EVCNT_TYPE_MISC,
   1833 	    NULL, xname, "Driver tx dma soft fail ENOMEM");
   1834 	evcnt_attach_dynamic(&sc->watchdog_events, EVCNT_TYPE_MISC,
   1835 	    NULL, xname, "Watchdog timeouts");
   1836 	evcnt_attach_dynamic(&sc->tso_err, EVCNT_TYPE_MISC,
   1837 	    NULL, xname, "TSO errors");
   1838 	evcnt_attach_dynamic(&sc->admin_irqev, EVCNT_TYPE_INTR,
   1839 	    NULL, xname, "Admin MSI-X IRQ Handled");
   1840 	evcnt_attach_dynamic(&sc->link_workev, EVCNT_TYPE_INTR,
   1841 	    NULL, xname, "Link event");
   1842 	evcnt_attach_dynamic(&sc->mod_workev, EVCNT_TYPE_INTR,
   1843 	    NULL, xname, "SFP+ module event");
   1844 	evcnt_attach_dynamic(&sc->msf_workev, EVCNT_TYPE_INTR,
   1845 	    NULL, xname, "Multispeed event");
   1846 	evcnt_attach_dynamic(&sc->phy_workev, EVCNT_TYPE_INTR,
   1847 	    NULL, xname, "External PHY event");
   1848 
   1849 	/* Max number of traffic class is 8 */
   1850 	KASSERT(IXGBE_DCB_MAX_TRAFFIC_CLASS == 8);
   1851 	for (i = 0; i < IXGBE_TC_COUNTER_NUM; i++) {
   1852 		snprintf(sc->tcs[i].evnamebuf,
   1853 		    sizeof(sc->tcs[i].evnamebuf), "%s tc%d", xname, i);
   1854 		if (i < __arraycount(stats->mpc)) {
   1855 			evcnt_attach_dynamic(&stats->mpc[i],
   1856 			    EVCNT_TYPE_MISC, NULL, sc->tcs[i].evnamebuf,
   1857 			    "RX Missed Packet Count");
   1858 			if (hw->mac.type == ixgbe_mac_82598EB)
   1859 				evcnt_attach_dynamic(&stats->rnbc[i],
   1860 				    EVCNT_TYPE_MISC, NULL,
   1861 				    sc->tcs[i].evnamebuf,
   1862 				    "Receive No Buffers");
   1863 		}
   1864 		if (i < __arraycount(stats->pxontxc)) {
   1865 			evcnt_attach_dynamic(&stats->pxontxc[i],
   1866 			    EVCNT_TYPE_MISC, NULL, sc->tcs[i].evnamebuf,
   1867 			    "Priority XON Transmitted");
   1868 			evcnt_attach_dynamic(&stats->pxofftxc[i],
   1869 			    EVCNT_TYPE_MISC, NULL, sc->tcs[i].evnamebuf,
   1870 			    "Priority XOFF Transmitted");
   1871 			if (hw->mac.type >= ixgbe_mac_82599EB)
   1872 				evcnt_attach_dynamic(&stats->pxon2offc[i],
   1873 				    EVCNT_TYPE_MISC, NULL,
   1874 				    sc->tcs[i].evnamebuf,
   1875 				    "Priority XON to XOFF");
   1876 			evcnt_attach_dynamic(&stats->pxonrxc[i],
   1877 			    EVCNT_TYPE_MISC, NULL, sc->tcs[i].evnamebuf,
   1878 			    "Priority XON Received");
   1879 			evcnt_attach_dynamic(&stats->pxoffrxc[i],
   1880 			    EVCNT_TYPE_MISC, NULL, sc->tcs[i].evnamebuf,
   1881 			    "Priority XOFF Received");
   1882 		}
   1883 	}
   1884 
   1885 	for (i = 0; i < sc->num_queues; i++, rxr++, txr++) {
   1886 #ifdef LRO
   1887 		struct lro_ctrl *lro = &rxr->lro;
   1888 #endif
   1889 
   1890 		snprintf(sc->queues[i].evnamebuf,
   1891 		    sizeof(sc->queues[i].evnamebuf), "%s q%d", xname, i);
   1892 		snprintf(sc->queues[i].namebuf,
   1893 		    sizeof(sc->queues[i].namebuf), "q%d", i);
   1894 
   1895 		if ((rnode = ixgbe_sysctl_instance(sc)) == NULL) {
   1896 			aprint_error_dev(dev,
   1897 			    "could not create sysctl root\n");
   1898 			break;
   1899 		}
   1900 
   1901 		if (sysctl_createv(log, 0, &rnode, &rnode,
   1902 		    0, CTLTYPE_NODE,
   1903 		    sc->queues[i].namebuf, SYSCTL_DESCR("Queue Name"),
   1904 		    NULL, 0, NULL, 0, CTL_CREATE, CTL_EOL) != 0)
   1905 			break;
   1906 
   1907 		if (sysctl_createv(log, 0, &rnode, &cnode,
   1908 		    CTLFLAG_READWRITE, CTLTYPE_INT,
   1909 		    "interrupt_rate", SYSCTL_DESCR("Interrupt Rate"),
   1910 		    ixgbe_sysctl_interrupt_rate_handler, 0,
   1911 		    (void *)&sc->queues[i], 0, CTL_CREATE, CTL_EOL) != 0)
   1912 			break;
   1913 
   1914 		if (sysctl_createv(log, 0, &rnode, &cnode,
   1915 		    CTLFLAG_READONLY, CTLTYPE_INT,
   1916 		    "txd_head", SYSCTL_DESCR("Transmit Descriptor Head"),
   1917 		    ixgbe_sysctl_tdh_handler, 0, (void *)txr,
   1918 		    0, CTL_CREATE, CTL_EOL) != 0)
   1919 			break;
   1920 
   1921 		if (sysctl_createv(log, 0, &rnode, &cnode,
   1922 		    CTLFLAG_READONLY, CTLTYPE_INT,
   1923 		    "txd_tail", SYSCTL_DESCR("Transmit Descriptor Tail"),
   1924 		    ixgbe_sysctl_tdt_handler, 0, (void *)txr,
   1925 		    0, CTL_CREATE, CTL_EOL) != 0)
   1926 			break;
   1927 
   1928 		if (sysctl_createv(log, 0, &rnode, &cnode,
   1929 		    CTLFLAG_READONLY, CTLTYPE_INT, "rxd_nxck",
   1930 		    SYSCTL_DESCR("Receive Descriptor next to check"),
   1931 		    ixgbe_sysctl_next_to_check_handler, 0, (void *)rxr, 0,
   1932 		    CTL_CREATE, CTL_EOL) != 0)
   1933 			break;
   1934 
   1935 		if (sysctl_createv(log, 0, &rnode, &cnode,
   1936 		    CTLFLAG_READONLY, CTLTYPE_INT, "rxd_nxrf",
   1937 		    SYSCTL_DESCR("Receive Descriptor next to refresh"),
   1938 		    ixgbe_sysctl_next_to_refresh_handler, 0, (void *)rxr, 0,
   1939 		    CTL_CREATE, CTL_EOL) != 0)
   1940 			break;
   1941 
   1942 		if (sysctl_createv(log, 0, &rnode, &cnode,
   1943 		    CTLFLAG_READONLY, CTLTYPE_INT, "rxd_head",
   1944 		    SYSCTL_DESCR("Receive Descriptor Head"),
   1945 		    ixgbe_sysctl_rdh_handler, 0, (void *)rxr, 0,
   1946 		    CTL_CREATE, CTL_EOL) != 0)
   1947 			break;
   1948 
   1949 		if (sysctl_createv(log, 0, &rnode, &cnode,
   1950 		    CTLFLAG_READONLY, CTLTYPE_INT, "rxd_tail",
   1951 		    SYSCTL_DESCR("Receive Descriptor Tail"),
   1952 		    ixgbe_sysctl_rdt_handler, 0, (void *)rxr, 0,
   1953 		    CTL_CREATE, CTL_EOL) != 0)
   1954 			break;
   1955 
   1956 		evcnt_attach_dynamic(&sc->queues[i].irqs, EVCNT_TYPE_INTR,
   1957 		    NULL, sc->queues[i].evnamebuf, "IRQs on queue");
   1958 		evcnt_attach_dynamic(&sc->queues[i].handleq,
   1959 		    EVCNT_TYPE_MISC, NULL, sc->queues[i].evnamebuf,
   1960 		    "Handled queue in softint");
   1961 		evcnt_attach_dynamic(&sc->queues[i].req, EVCNT_TYPE_MISC,
   1962 		    NULL, sc->queues[i].evnamebuf, "Requeued in softint");
   1963 		if (i < __arraycount(stats->qbtc))
   1964 			evcnt_attach_dynamic(&stats->qbtc[i], EVCNT_TYPE_MISC,
   1965 			    NULL, sc->queues[i].evnamebuf,
   1966 			    "Queue Bytes Transmitted (reg)");
   1967 		evcnt_attach_dynamic(&txr->total_packets, EVCNT_TYPE_MISC,
   1968 		    NULL, sc->queues[i].evnamebuf,
   1969 		    "Queue Packets Transmitted (soft)");
   1970 		if (i < __arraycount(stats->qptc))
   1971 			evcnt_attach_dynamic(&stats->qptc[i], EVCNT_TYPE_MISC,
   1972 			    NULL, sc->queues[i].evnamebuf,
   1973 			    "Queue Packets Transmitted (reg)");
   1974 #ifndef IXGBE_LEGACY_TX
   1975 		evcnt_attach_dynamic(&txr->pcq_drops, EVCNT_TYPE_MISC,
   1976 		    NULL, sc->queues[i].evnamebuf,
   1977 		    "Packets dropped in pcq");
   1978 #endif
   1979 		evcnt_attach_dynamic(&txr->no_desc_avail, EVCNT_TYPE_MISC,
   1980 		    NULL, sc->queues[i].evnamebuf,
   1981 		    "TX Queue No Descriptor Available");
   1982 		evcnt_attach_dynamic(&txr->tso_tx, EVCNT_TYPE_MISC,
   1983 		    NULL, sc->queues[i].evnamebuf, "TSO");
   1984 
   1985 		evcnt_attach_dynamic(&rxr->rx_bytes, EVCNT_TYPE_MISC,
   1986 		    NULL, sc->queues[i].evnamebuf,
   1987 		    "Queue Bytes Received (soft)");
   1988 		if (i < __arraycount(stats->qbrc))
   1989 			evcnt_attach_dynamic(&stats->qbrc[i], EVCNT_TYPE_MISC,
   1990 			    NULL, sc->queues[i].evnamebuf,
   1991 			    "Queue Bytes Received (reg)");
   1992 		evcnt_attach_dynamic(&rxr->rx_packets, EVCNT_TYPE_MISC,
   1993 		    NULL, sc->queues[i].evnamebuf,
   1994 		    "Queue Packets Received (soft)");
   1995 		if (i < __arraycount(stats->qprc))
   1996 			evcnt_attach_dynamic(&stats->qprc[i], EVCNT_TYPE_MISC,
   1997 			    NULL, sc->queues[i].evnamebuf,
   1998 			    "Queue Packets Received (reg)");
   1999 		if ((i < __arraycount(stats->qprdc)) &&
   2000 		    (hw->mac.type >= ixgbe_mac_82599EB))
   2001 				evcnt_attach_dynamic(&stats->qprdc[i],
   2002 				    EVCNT_TYPE_MISC, NULL,
   2003 				    sc->queues[i].evnamebuf,
   2004 				    "Queue Packets Received Drop");
   2005 
   2006 		evcnt_attach_dynamic(&rxr->no_mbuf, EVCNT_TYPE_MISC,
   2007 		    NULL, sc->queues[i].evnamebuf, "Rx no mbuf");
   2008 		evcnt_attach_dynamic(&rxr->rx_discarded, EVCNT_TYPE_MISC,
   2009 		    NULL, sc->queues[i].evnamebuf, "Rx discarded");
   2010 		evcnt_attach_dynamic(&rxr->rx_copies, EVCNT_TYPE_MISC,
   2011 		    NULL, sc->queues[i].evnamebuf, "Copied RX Frames");
   2012 #ifdef LRO
   2013 		SYSCTL_ADD_INT(ctx, queue_list, OID_AUTO, "lro_queued",
   2014 				CTLFLAG_RD, &lro->lro_queued, 0,
   2015 				"LRO Queued");
   2016 		SYSCTL_ADD_INT(ctx, queue_list, OID_AUTO, "lro_flushed",
   2017 				CTLFLAG_RD, &lro->lro_flushed, 0,
   2018 				"LRO Flushed");
   2019 #endif /* LRO */
   2020 	}
   2021 
   2022 	/* MAC stats get their own sub node */
   2023 
   2024 	snprintf(stats->namebuf,
   2025 	    sizeof(stats->namebuf), "%s MAC Statistics", xname);
   2026 
   2027 	evcnt_attach_dynamic(&stats->ipcs, EVCNT_TYPE_MISC, NULL,
   2028 	    stats->namebuf, "rx csum offload - IP");
   2029 	evcnt_attach_dynamic(&stats->l4cs, EVCNT_TYPE_MISC, NULL,
   2030 	    stats->namebuf, "rx csum offload - L4");
   2031 	evcnt_attach_dynamic(&stats->ipcs_bad, EVCNT_TYPE_MISC, NULL,
   2032 	    stats->namebuf, "rx csum offload - IP bad");
   2033 	evcnt_attach_dynamic(&stats->l4cs_bad, EVCNT_TYPE_MISC, NULL,
   2034 	    stats->namebuf, "rx csum offload - L4 bad");
   2035 	evcnt_attach_dynamic(&stats->intzero, EVCNT_TYPE_MISC, NULL,
   2036 	    stats->namebuf, "Interrupt conditions zero");
   2037 	evcnt_attach_dynamic(&stats->legint, EVCNT_TYPE_MISC, NULL,
   2038 	    stats->namebuf, "Legacy interrupts");
   2039 
   2040 	evcnt_attach_dynamic(&stats->crcerrs, EVCNT_TYPE_MISC, NULL,
   2041 	    stats->namebuf, "CRC Errors");
   2042 	evcnt_attach_dynamic(&stats->illerrc, EVCNT_TYPE_MISC, NULL,
   2043 	    stats->namebuf, "Illegal Byte Errors");
   2044 	evcnt_attach_dynamic(&stats->errbc, EVCNT_TYPE_MISC, NULL,
   2045 	    stats->namebuf, "Byte Errors");
   2046 	evcnt_attach_dynamic(&stats->mspdc, EVCNT_TYPE_MISC, NULL,
   2047 	    stats->namebuf, "MAC Short Packets Discarded");
   2048 	if (hw->mac.type >= ixgbe_mac_X550)
   2049 		evcnt_attach_dynamic(&stats->mbsdc, EVCNT_TYPE_MISC, NULL,
   2050 		    stats->namebuf, "Bad SFD");
   2051 	evcnt_attach_dynamic(&stats->mpctotal, EVCNT_TYPE_MISC, NULL,
   2052 	    stats->namebuf, "Total Packets Missed");
   2053 	evcnt_attach_dynamic(&stats->mlfc, EVCNT_TYPE_MISC, NULL,
   2054 	    stats->namebuf, "MAC Local Faults");
   2055 	evcnt_attach_dynamic(&stats->mrfc, EVCNT_TYPE_MISC, NULL,
   2056 	    stats->namebuf, "MAC Remote Faults");
   2057 	if (hw->mac.type == ixgbe_mac_X550EM_a)
   2058 		evcnt_attach_dynamic(&stats->link_dn_cnt, EVCNT_TYPE_MISC,
   2059 		    NULL, stats->namebuf, "Link down event in the MAC");
   2060 	evcnt_attach_dynamic(&stats->rlec, EVCNT_TYPE_MISC, NULL,
   2061 	    stats->namebuf, "Receive Length Errors");
   2062 	evcnt_attach_dynamic(&stats->lxontxc, EVCNT_TYPE_MISC, NULL,
   2063 	    stats->namebuf, "Link XON Transmitted");
   2064 	evcnt_attach_dynamic(&stats->lxofftxc, EVCNT_TYPE_MISC, NULL,
   2065 	    stats->namebuf, "Link XOFF Transmitted");
   2066 	evcnt_attach_dynamic(&stats->lxonrxc, EVCNT_TYPE_MISC, NULL,
   2067 	    stats->namebuf, "Link XON Received");
   2068 	evcnt_attach_dynamic(&stats->lxoffrxc, EVCNT_TYPE_MISC, NULL,
   2069 	    stats->namebuf, "Link XOFF Received");
   2070 
   2071 	/* Packet Reception Stats */
   2072 	evcnt_attach_dynamic(&stats->tor, EVCNT_TYPE_MISC, NULL,
   2073 	    stats->namebuf, "Total Octets Received");
   2074 	evcnt_attach_dynamic(&stats->gorc, EVCNT_TYPE_MISC, NULL,
   2075 	    stats->namebuf, "Good Octets Received");
   2076 	evcnt_attach_dynamic(&stats->tpr, EVCNT_TYPE_MISC, NULL,
   2077 	    stats->namebuf, "Total Packets Received");
   2078 	evcnt_attach_dynamic(&stats->gprc, EVCNT_TYPE_MISC, NULL,
   2079 	    stats->namebuf, "Good Packets Received");
   2080 	evcnt_attach_dynamic(&stats->mprc, EVCNT_TYPE_MISC, NULL,
   2081 	    stats->namebuf, "Multicast Packets Received");
   2082 	evcnt_attach_dynamic(&stats->bprc, EVCNT_TYPE_MISC, NULL,
   2083 	    stats->namebuf, "Broadcast Packets Received");
   2084 	evcnt_attach_dynamic(&stats->prc64, EVCNT_TYPE_MISC, NULL,
   2085 	    stats->namebuf, "64 byte frames received ");
   2086 	evcnt_attach_dynamic(&stats->prc127, EVCNT_TYPE_MISC, NULL,
   2087 	    stats->namebuf, "65-127 byte frames received");
   2088 	evcnt_attach_dynamic(&stats->prc255, EVCNT_TYPE_MISC, NULL,
   2089 	    stats->namebuf, "128-255 byte frames received");
   2090 	evcnt_attach_dynamic(&stats->prc511, EVCNT_TYPE_MISC, NULL,
   2091 	    stats->namebuf, "256-511 byte frames received");
   2092 	evcnt_attach_dynamic(&stats->prc1023, EVCNT_TYPE_MISC, NULL,
   2093 	    stats->namebuf, "512-1023 byte frames received");
   2094 	evcnt_attach_dynamic(&stats->prc1522, EVCNT_TYPE_MISC, NULL,
   2095 	    stats->namebuf, "1023-1522 byte frames received");
   2096 	evcnt_attach_dynamic(&stats->ruc, EVCNT_TYPE_MISC, NULL,
   2097 	    stats->namebuf, "Receive Undersized");
   2098 	evcnt_attach_dynamic(&stats->rfc, EVCNT_TYPE_MISC, NULL,
   2099 	    stats->namebuf, "Fragmented Packets Received ");
   2100 	evcnt_attach_dynamic(&stats->roc, EVCNT_TYPE_MISC, NULL,
   2101 	    stats->namebuf, "Oversized Packets Received");
   2102 	evcnt_attach_dynamic(&stats->rjc, EVCNT_TYPE_MISC, NULL,
   2103 	    stats->namebuf, "Received Jabber");
   2104 	evcnt_attach_dynamic(&stats->mngprc, EVCNT_TYPE_MISC, NULL,
   2105 	    stats->namebuf, "Management Packets Received");
   2106 	evcnt_attach_dynamic(&stats->mngpdc, EVCNT_TYPE_MISC, NULL,
   2107 	    stats->namebuf, "Management Packets Dropped");
   2108 	evcnt_attach_dynamic(&stats->xec, EVCNT_TYPE_MISC, NULL,
   2109 	    stats->namebuf, "Checksum Errors");
   2110 
   2111 	/* Packet Transmission Stats */
   2112 	evcnt_attach_dynamic(&stats->gotc, EVCNT_TYPE_MISC, NULL,
   2113 	    stats->namebuf, "Good Octets Transmitted");
   2114 	evcnt_attach_dynamic(&stats->tpt, EVCNT_TYPE_MISC, NULL,
   2115 	    stats->namebuf, "Total Packets Transmitted");
   2116 	evcnt_attach_dynamic(&stats->gptc, EVCNT_TYPE_MISC, NULL,
   2117 	    stats->namebuf, "Good Packets Transmitted");
   2118 	evcnt_attach_dynamic(&stats->bptc, EVCNT_TYPE_MISC, NULL,
   2119 	    stats->namebuf, "Broadcast Packets Transmitted");
   2120 	evcnt_attach_dynamic(&stats->mptc, EVCNT_TYPE_MISC, NULL,
   2121 	    stats->namebuf, "Multicast Packets Transmitted");
   2122 	evcnt_attach_dynamic(&stats->mngptc, EVCNT_TYPE_MISC, NULL,
   2123 	    stats->namebuf, "Management Packets Transmitted");
   2124 	evcnt_attach_dynamic(&stats->ptc64, EVCNT_TYPE_MISC, NULL,
   2125 	    stats->namebuf, "64 byte frames transmitted ");
   2126 	evcnt_attach_dynamic(&stats->ptc127, EVCNT_TYPE_MISC, NULL,
   2127 	    stats->namebuf, "65-127 byte frames transmitted");
   2128 	evcnt_attach_dynamic(&stats->ptc255, EVCNT_TYPE_MISC, NULL,
   2129 	    stats->namebuf, "128-255 byte frames transmitted");
   2130 	evcnt_attach_dynamic(&stats->ptc511, EVCNT_TYPE_MISC, NULL,
   2131 	    stats->namebuf, "256-511 byte frames transmitted");
   2132 	evcnt_attach_dynamic(&stats->ptc1023, EVCNT_TYPE_MISC, NULL,
   2133 	    stats->namebuf, "512-1023 byte frames transmitted");
   2134 	evcnt_attach_dynamic(&stats->ptc1522, EVCNT_TYPE_MISC, NULL,
   2135 	    stats->namebuf, "1024-1522 byte frames transmitted");
   2136 } /* ixgbe_add_hw_stats */
   2137 
   2138 static void
   2139 ixgbe_clear_evcnt(struct ixgbe_softc *sc)
   2140 {
   2141 	struct tx_ring *txr = sc->tx_rings;
   2142 	struct rx_ring *rxr = sc->rx_rings;
   2143 	struct ixgbe_hw *hw = &sc->hw;
   2144 	struct ixgbe_hw_stats *stats = &sc->stats.pf;
   2145 	int i;
   2146 
   2147 	IXGBE_EVC_STORE(&sc->efbig_tx_dma_setup, 0);
   2148 	IXGBE_EVC_STORE(&sc->mbuf_defrag_failed, 0);
   2149 	IXGBE_EVC_STORE(&sc->efbig2_tx_dma_setup, 0);
   2150 	IXGBE_EVC_STORE(&sc->einval_tx_dma_setup, 0);
   2151 	IXGBE_EVC_STORE(&sc->other_tx_dma_setup, 0);
   2152 	IXGBE_EVC_STORE(&sc->eagain_tx_dma_setup, 0);
   2153 	IXGBE_EVC_STORE(&sc->enomem_tx_dma_setup, 0);
   2154 	IXGBE_EVC_STORE(&sc->tso_err, 0);
   2155 	IXGBE_EVC_STORE(&sc->watchdog_events, 0);
   2156 	IXGBE_EVC_STORE(&sc->admin_irqev, 0);
   2157 	IXGBE_EVC_STORE(&sc->link_workev, 0);
   2158 	IXGBE_EVC_STORE(&sc->mod_workev, 0);
   2159 	IXGBE_EVC_STORE(&sc->msf_workev, 0);
   2160 	IXGBE_EVC_STORE(&sc->phy_workev, 0);
   2161 
   2162 	for (i = 0; i < IXGBE_TC_COUNTER_NUM; i++) {
   2163 		if (i < __arraycount(stats->mpc)) {
   2164 			IXGBE_EVC_STORE(&stats->mpc[i], 0);
   2165 			if (hw->mac.type == ixgbe_mac_82598EB)
   2166 				IXGBE_EVC_STORE(&stats->rnbc[i], 0);
   2167 		}
   2168 		if (i < __arraycount(stats->pxontxc)) {
   2169 			IXGBE_EVC_STORE(&stats->pxontxc[i], 0);
   2170 			IXGBE_EVC_STORE(&stats->pxonrxc[i], 0);
   2171 			IXGBE_EVC_STORE(&stats->pxofftxc[i], 0);
   2172 			IXGBE_EVC_STORE(&stats->pxoffrxc[i], 0);
   2173 			if (hw->mac.type >= ixgbe_mac_82599EB)
   2174 				IXGBE_EVC_STORE(&stats->pxon2offc[i], 0);
   2175 		}
   2176 	}
   2177 
   2178 	txr = sc->tx_rings;
   2179 	for (i = 0; i < sc->num_queues; i++, rxr++, txr++) {
   2180 		IXGBE_EVC_STORE(&sc->queues[i].irqs, 0);
   2181 		IXGBE_EVC_STORE(&sc->queues[i].handleq, 0);
   2182 		IXGBE_EVC_STORE(&sc->queues[i].req, 0);
   2183 		IXGBE_EVC_STORE(&txr->total_packets, 0);
   2184 #ifndef IXGBE_LEGACY_TX
   2185 		IXGBE_EVC_STORE(&txr->pcq_drops, 0);
   2186 #endif
   2187 		IXGBE_EVC_STORE(&txr->no_desc_avail, 0);
   2188 		IXGBE_EVC_STORE(&txr->tso_tx, 0);
   2189 		txr->q_efbig_tx_dma_setup = 0;
   2190 		txr->q_mbuf_defrag_failed = 0;
   2191 		txr->q_efbig2_tx_dma_setup = 0;
   2192 		txr->q_einval_tx_dma_setup = 0;
   2193 		txr->q_other_tx_dma_setup = 0;
   2194 		txr->q_eagain_tx_dma_setup = 0;
   2195 		txr->q_enomem_tx_dma_setup = 0;
   2196 		txr->q_tso_err = 0;
   2197 
   2198 		if (i < __arraycount(stats->qprc)) {
   2199 			IXGBE_EVC_STORE(&stats->qprc[i], 0);
   2200 			IXGBE_EVC_STORE(&stats->qptc[i], 0);
   2201 			IXGBE_EVC_STORE(&stats->qbrc[i], 0);
   2202 			IXGBE_EVC_STORE(&stats->qbtc[i], 0);
   2203 			if (hw->mac.type >= ixgbe_mac_82599EB)
   2204 				IXGBE_EVC_STORE(&stats->qprdc[i], 0);
   2205 		}
   2206 
   2207 		IXGBE_EVC_STORE(&rxr->rx_packets, 0);
   2208 		IXGBE_EVC_STORE(&rxr->rx_bytes, 0);
   2209 		IXGBE_EVC_STORE(&rxr->rx_copies, 0);
   2210 		IXGBE_EVC_STORE(&rxr->no_mbuf, 0);
   2211 		IXGBE_EVC_STORE(&rxr->rx_discarded, 0);
   2212 	}
   2213 	IXGBE_EVC_STORE(&stats->ipcs, 0);
   2214 	IXGBE_EVC_STORE(&stats->l4cs, 0);
   2215 	IXGBE_EVC_STORE(&stats->ipcs_bad, 0);
   2216 	IXGBE_EVC_STORE(&stats->l4cs_bad, 0);
   2217 	IXGBE_EVC_STORE(&stats->intzero, 0);
   2218 	IXGBE_EVC_STORE(&stats->legint, 0);
   2219 	IXGBE_EVC_STORE(&stats->crcerrs, 0);
   2220 	IXGBE_EVC_STORE(&stats->illerrc, 0);
   2221 	IXGBE_EVC_STORE(&stats->errbc, 0);
   2222 	IXGBE_EVC_STORE(&stats->mspdc, 0);
   2223 	if (hw->mac.type >= ixgbe_mac_X550)
   2224 		IXGBE_EVC_STORE(&stats->mbsdc, 0);
   2225 	IXGBE_EVC_STORE(&stats->mpctotal, 0);
   2226 	IXGBE_EVC_STORE(&stats->mlfc, 0);
   2227 	IXGBE_EVC_STORE(&stats->mrfc, 0);
   2228 	if (hw->mac.type == ixgbe_mac_X550EM_a)
   2229 		IXGBE_EVC_STORE(&stats->link_dn_cnt, 0);
   2230 	IXGBE_EVC_STORE(&stats->rlec, 0);
   2231 	IXGBE_EVC_STORE(&stats->lxontxc, 0);
   2232 	IXGBE_EVC_STORE(&stats->lxonrxc, 0);
   2233 	IXGBE_EVC_STORE(&stats->lxofftxc, 0);
   2234 	IXGBE_EVC_STORE(&stats->lxoffrxc, 0);
   2235 
   2236 	/* Packet Reception Stats */
   2237 	IXGBE_EVC_STORE(&stats->tor, 0);
   2238 	IXGBE_EVC_STORE(&stats->gorc, 0);
   2239 	IXGBE_EVC_STORE(&stats->tpr, 0);
   2240 	IXGBE_EVC_STORE(&stats->gprc, 0);
   2241 	IXGBE_EVC_STORE(&stats->mprc, 0);
   2242 	IXGBE_EVC_STORE(&stats->bprc, 0);
   2243 	IXGBE_EVC_STORE(&stats->prc64, 0);
   2244 	IXGBE_EVC_STORE(&stats->prc127, 0);
   2245 	IXGBE_EVC_STORE(&stats->prc255, 0);
   2246 	IXGBE_EVC_STORE(&stats->prc511, 0);
   2247 	IXGBE_EVC_STORE(&stats->prc1023, 0);
   2248 	IXGBE_EVC_STORE(&stats->prc1522, 0);
   2249 	IXGBE_EVC_STORE(&stats->ruc, 0);
   2250 	IXGBE_EVC_STORE(&stats->rfc, 0);
   2251 	IXGBE_EVC_STORE(&stats->roc, 0);
   2252 	IXGBE_EVC_STORE(&stats->rjc, 0);
   2253 	IXGBE_EVC_STORE(&stats->mngprc, 0);
   2254 	IXGBE_EVC_STORE(&stats->mngpdc, 0);
   2255 	IXGBE_EVC_STORE(&stats->xec, 0);
   2256 
   2257 	/* Packet Transmission Stats */
   2258 	IXGBE_EVC_STORE(&stats->gotc, 0);
   2259 	IXGBE_EVC_STORE(&stats->tpt, 0);
   2260 	IXGBE_EVC_STORE(&stats->gptc, 0);
   2261 	IXGBE_EVC_STORE(&stats->bptc, 0);
   2262 	IXGBE_EVC_STORE(&stats->mptc, 0);
   2263 	IXGBE_EVC_STORE(&stats->mngptc, 0);
   2264 	IXGBE_EVC_STORE(&stats->ptc64, 0);
   2265 	IXGBE_EVC_STORE(&stats->ptc127, 0);
   2266 	IXGBE_EVC_STORE(&stats->ptc255, 0);
   2267 	IXGBE_EVC_STORE(&stats->ptc511, 0);
   2268 	IXGBE_EVC_STORE(&stats->ptc1023, 0);
   2269 	IXGBE_EVC_STORE(&stats->ptc1522, 0);
   2270 }
   2271 
   2272 /************************************************************************
   2273  * ixgbe_sysctl_tdh_handler - Transmit Descriptor Head handler function
   2274  *
   2275  *   Retrieves the TDH value from the hardware
   2276  ************************************************************************/
   2277 static int
   2278 ixgbe_sysctl_tdh_handler(SYSCTLFN_ARGS)
   2279 {
   2280 	struct sysctlnode node = *rnode;
   2281 	struct tx_ring *txr = (struct tx_ring *)node.sysctl_data;
   2282 	struct ixgbe_softc *sc;
   2283 	uint32_t val;
   2284 
   2285 	if (!txr)
   2286 		return (0);
   2287 
   2288 	sc = txr->sc;
   2289 	if (ixgbe_fw_recovery_mode_swflag(sc))
   2290 		return (EPERM);
   2291 
   2292 	val = IXGBE_READ_REG(&sc->hw, IXGBE_TDH(txr->me));
   2293 	node.sysctl_data = &val;
   2294 	return sysctl_lookup(SYSCTLFN_CALL(&node));
   2295 } /* ixgbe_sysctl_tdh_handler */
   2296 
   2297 /************************************************************************
   2298  * ixgbe_sysctl_tdt_handler - Transmit Descriptor Tail handler function
   2299  *
   2300  *   Retrieves the TDT value from the hardware
   2301  ************************************************************************/
   2302 static int
   2303 ixgbe_sysctl_tdt_handler(SYSCTLFN_ARGS)
   2304 {
   2305 	struct sysctlnode node = *rnode;
   2306 	struct tx_ring *txr = (struct tx_ring *)node.sysctl_data;
   2307 	struct ixgbe_softc *sc;
   2308 	uint32_t val;
   2309 
   2310 	if (!txr)
   2311 		return (0);
   2312 
   2313 	sc = txr->sc;
   2314 	if (ixgbe_fw_recovery_mode_swflag(sc))
   2315 		return (EPERM);
   2316 
   2317 	val = IXGBE_READ_REG(&sc->hw, IXGBE_TDT(txr->me));
   2318 	node.sysctl_data = &val;
   2319 	return sysctl_lookup(SYSCTLFN_CALL(&node));
   2320 } /* ixgbe_sysctl_tdt_handler */
   2321 
   2322 /************************************************************************
   2323  * ixgbe_sysctl_next_to_check_handler - Receive Descriptor next to check
   2324  * handler function
   2325  *
   2326  *   Retrieves the next_to_check value
   2327  ************************************************************************/
   2328 static int
   2329 ixgbe_sysctl_next_to_check_handler(SYSCTLFN_ARGS)
   2330 {
   2331 	struct sysctlnode node = *rnode;
   2332 	struct rx_ring *rxr = (struct rx_ring *)node.sysctl_data;
   2333 	struct ixgbe_softc *sc;
   2334 	uint32_t val;
   2335 
   2336 	if (!rxr)
   2337 		return (0);
   2338 
   2339 	sc = rxr->sc;
   2340 	if (ixgbe_fw_recovery_mode_swflag(sc))
   2341 		return (EPERM);
   2342 
   2343 	val = rxr->next_to_check;
   2344 	node.sysctl_data = &val;
   2345 	return sysctl_lookup(SYSCTLFN_CALL(&node));
   2346 } /* ixgbe_sysctl_next_to_check_handler */
   2347 
   2348 /************************************************************************
   2349  * ixgbe_sysctl_next_to_refresh_handler - Receive Descriptor next to check
   2350  * handler function
   2351  *
   2352  *   Retrieves the next_to_refresh value
   2353  ************************************************************************/
   2354 static int
   2355 ixgbe_sysctl_next_to_refresh_handler(SYSCTLFN_ARGS)
   2356 {
   2357 	struct sysctlnode node = *rnode;
   2358 	struct rx_ring *rxr = (struct rx_ring *)node.sysctl_data;
   2359 	struct ixgbe_softc *sc;
   2360 	uint32_t val;
   2361 
   2362 	if (!rxr)
   2363 		return (0);
   2364 
   2365 	sc = rxr->sc;
   2366 	if (ixgbe_fw_recovery_mode_swflag(sc))
   2367 		return (EPERM);
   2368 
   2369 	val = rxr->next_to_refresh;
   2370 	node.sysctl_data = &val;
   2371 	return sysctl_lookup(SYSCTLFN_CALL(&node));
   2372 } /* ixgbe_sysctl_next_to_refresh_handler */
   2373 
   2374 /************************************************************************
   2375  * ixgbe_sysctl_rdh_handler - Receive Descriptor Head handler function
   2376  *
   2377  *   Retrieves the RDH value from the hardware
   2378  ************************************************************************/
   2379 static int
   2380 ixgbe_sysctl_rdh_handler(SYSCTLFN_ARGS)
   2381 {
   2382 	struct sysctlnode node = *rnode;
   2383 	struct rx_ring *rxr = (struct rx_ring *)node.sysctl_data;
   2384 	struct ixgbe_softc *sc;
   2385 	uint32_t val;
   2386 
   2387 	if (!rxr)
   2388 		return (0);
   2389 
   2390 	sc = rxr->sc;
   2391 	if (ixgbe_fw_recovery_mode_swflag(sc))
   2392 		return (EPERM);
   2393 
   2394 	val = IXGBE_READ_REG(&sc->hw, IXGBE_RDH(rxr->me));
   2395 	node.sysctl_data = &val;
   2396 	return sysctl_lookup(SYSCTLFN_CALL(&node));
   2397 } /* ixgbe_sysctl_rdh_handler */
   2398 
   2399 /************************************************************************
   2400  * ixgbe_sysctl_rdt_handler - Receive Descriptor Tail handler function
   2401  *
   2402  *   Retrieves the RDT value from the hardware
   2403  ************************************************************************/
   2404 static int
   2405 ixgbe_sysctl_rdt_handler(SYSCTLFN_ARGS)
   2406 {
   2407 	struct sysctlnode node = *rnode;
   2408 	struct rx_ring *rxr = (struct rx_ring *)node.sysctl_data;
   2409 	struct ixgbe_softc *sc;
   2410 	uint32_t val;
   2411 
   2412 	if (!rxr)
   2413 		return (0);
   2414 
   2415 	sc = rxr->sc;
   2416 	if (ixgbe_fw_recovery_mode_swflag(sc))
   2417 		return (EPERM);
   2418 
   2419 	val = IXGBE_READ_REG(&sc->hw, IXGBE_RDT(rxr->me));
   2420 	node.sysctl_data = &val;
   2421 	return sysctl_lookup(SYSCTLFN_CALL(&node));
   2422 } /* ixgbe_sysctl_rdt_handler */
   2423 
   2424 static int
   2425 ixgbe_vlan_cb(struct ethercom *ec, uint16_t vid, bool set)
   2426 {
   2427 	struct ifnet *ifp = &ec->ec_if;
   2428 	struct ixgbe_softc *sc = ifp->if_softc;
   2429 	int rv;
   2430 
   2431 	if (set)
   2432 		rv = ixgbe_register_vlan(sc, vid);
   2433 	else
   2434 		rv = ixgbe_unregister_vlan(sc, vid);
   2435 
   2436 	if (rv != 0)
   2437 		return rv;
   2438 
   2439 	/*
   2440 	 * Control VLAN HW tagging when ec_nvlan is changed from 1 to 0
   2441 	 * or 0 to 1.
   2442 	 */
   2443 	if ((set && (ec->ec_nvlans == 1)) || (!set && (ec->ec_nvlans == 0)))
   2444 		ixgbe_setup_vlan_hw_tagging(sc);
   2445 
   2446 	return rv;
   2447 }
   2448 
   2449 /************************************************************************
   2450  * ixgbe_register_vlan
   2451  *
   2452  *   Run via vlan config EVENT, it enables us to use the
   2453  *   HW Filter table since we can get the vlan id. This
   2454  *   just creates the entry in the soft version of the
   2455  *   VFTA, init will repopulate the real table.
   2456  ************************************************************************/
   2457 static int
   2458 ixgbe_register_vlan(struct ixgbe_softc *sc, u16 vtag)
   2459 {
   2460 	u16		index, bit;
   2461 	int		error;
   2462 
   2463 	if ((vtag == 0) || (vtag > 4095))	/* Invalid */
   2464 		return EINVAL;
   2465 
   2466 	IXGBE_CORE_LOCK(sc);
   2467 	index = (vtag >> 5) & 0x7F;
   2468 	bit = vtag & 0x1F;
   2469 	sc->shadow_vfta[index] |= ((u32)1 << bit);
   2470 	error = sc->hw.mac.ops.set_vfta(&sc->hw, vtag, 0, true,
   2471 	    true);
   2472 	IXGBE_CORE_UNLOCK(sc);
   2473 	if (error != 0)
   2474 		error = EACCES;
   2475 
   2476 	return error;
   2477 } /* ixgbe_register_vlan */
   2478 
   2479 /************************************************************************
   2480  * ixgbe_unregister_vlan
   2481  *
   2482  *   Run via vlan unconfig EVENT, remove our entry in the soft vfta.
   2483  ************************************************************************/
   2484 static int
   2485 ixgbe_unregister_vlan(struct ixgbe_softc *sc, u16 vtag)
   2486 {
   2487 	u16		index, bit;
   2488 	int		error;
   2489 
   2490 	if ((vtag == 0) || (vtag > 4095))	/* Invalid */
   2491 		return EINVAL;
   2492 
   2493 	IXGBE_CORE_LOCK(sc);
   2494 	index = (vtag >> 5) & 0x7F;
   2495 	bit = vtag & 0x1F;
   2496 	sc->shadow_vfta[index] &= ~((u32)1 << bit);
   2497 	error = sc->hw.mac.ops.set_vfta(&sc->hw, vtag, 0, false,
   2498 	    true);
   2499 	IXGBE_CORE_UNLOCK(sc);
   2500 	if (error != 0)
   2501 		error = EACCES;
   2502 
   2503 	return error;
   2504 } /* ixgbe_unregister_vlan */
   2505 
   2506 static void
   2507 ixgbe_setup_vlan_hw_tagging(struct ixgbe_softc *sc)
   2508 {
   2509 	struct ethercom *ec = &sc->osdep.ec;
   2510 	struct ixgbe_hw *hw = &sc->hw;
   2511 	struct rx_ring	*rxr;
   2512 	u32		ctrl;
   2513 	int		i;
   2514 	bool		hwtagging;
   2515 
   2516 	/* Enable HW tagging only if any vlan is attached */
   2517 	hwtagging = (ec->ec_capenable & ETHERCAP_VLAN_HWTAGGING)
   2518 	    && VLAN_ATTACHED(ec);
   2519 
   2520 	/* Setup the queues for vlans */
   2521 	for (i = 0; i < sc->num_queues; i++) {
   2522 		rxr = &sc->rx_rings[i];
   2523 		/*
   2524 		 * On 82599 and later, the VLAN enable is per/queue in RXDCTL.
   2525 		 */
   2526 		if (hw->mac.type != ixgbe_mac_82598EB) {
   2527 			ctrl = IXGBE_READ_REG(hw, IXGBE_RXDCTL(rxr->me));
   2528 			if (hwtagging)
   2529 				ctrl |= IXGBE_RXDCTL_VME;
   2530 			else
   2531 				ctrl &= ~IXGBE_RXDCTL_VME;
   2532 			IXGBE_WRITE_REG(hw, IXGBE_RXDCTL(rxr->me), ctrl);
   2533 		}
   2534 		rxr->vtag_strip = hwtagging ? TRUE : FALSE;
   2535 	}
   2536 
   2537 	/* VLAN hw tagging for 82598 */
   2538 	if (hw->mac.type == ixgbe_mac_82598EB) {
   2539 		ctrl = IXGBE_READ_REG(hw, IXGBE_VLNCTRL);
   2540 		if (hwtagging)
   2541 			ctrl |= IXGBE_VLNCTRL_VME;
   2542 		else
   2543 			ctrl &= ~IXGBE_VLNCTRL_VME;
   2544 		IXGBE_WRITE_REG(hw, IXGBE_VLNCTRL, ctrl);
   2545 	}
   2546 } /* ixgbe_setup_vlan_hw_tagging */
   2547 
   2548 static void
   2549 ixgbe_setup_vlan_hw_support(struct ixgbe_softc *sc)
   2550 {
   2551 	struct ethercom *ec = &sc->osdep.ec;
   2552 	struct ixgbe_hw *hw = &sc->hw;
   2553 	int		i;
   2554 	u32		ctrl;
   2555 	struct vlanid_list *vlanidp;
   2556 
   2557 	/*
   2558 	 * This function is called from both if_init and ifflags_cb()
   2559 	 * on NetBSD.
   2560 	 */
   2561 
   2562 	/*
   2563 	 * Part 1:
   2564 	 * Setup VLAN HW tagging
   2565 	 */
   2566 	ixgbe_setup_vlan_hw_tagging(sc);
   2567 
   2568 	/*
   2569 	 * Part 2:
   2570 	 * Setup VLAN HW filter
   2571 	 */
   2572 	/* Cleanup shadow_vfta */
   2573 	for (i = 0; i < IXGBE_VFTA_SIZE; i++)
   2574 		sc->shadow_vfta[i] = 0;
   2575 	/* Generate shadow_vfta from ec_vids */
   2576 	ETHER_LOCK(ec);
   2577 	SIMPLEQ_FOREACH(vlanidp, &ec->ec_vids, vid_list) {
   2578 		uint32_t idx;
   2579 
   2580 		idx = vlanidp->vid / 32;
   2581 		KASSERT(idx < IXGBE_VFTA_SIZE);
   2582 		sc->shadow_vfta[idx] |= (u32)1 << (vlanidp->vid % 32);
   2583 	}
   2584 	ETHER_UNLOCK(ec);
   2585 	for (i = 0; i < IXGBE_VFTA_SIZE; i++)
   2586 		IXGBE_WRITE_REG(hw, IXGBE_VFTA(i), sc->shadow_vfta[i]);
   2587 
   2588 	ctrl = IXGBE_READ_REG(hw, IXGBE_VLNCTRL);
   2589 	/* Enable the Filter Table if enabled */
   2590 	if (ec->ec_capenable & ETHERCAP_VLAN_HWFILTER)
   2591 		ctrl |= IXGBE_VLNCTRL_VFE;
   2592 	else
   2593 		ctrl &= ~IXGBE_VLNCTRL_VFE;
   2594 	IXGBE_WRITE_REG(hw, IXGBE_VLNCTRL, ctrl);
   2595 } /* ixgbe_setup_vlan_hw_support */
   2596 
   2597 /************************************************************************
   2598  * ixgbe_get_slot_info
   2599  *
   2600  *   Get the width and transaction speed of
   2601  *   the slot this adapter is plugged into.
   2602  ************************************************************************/
   2603 static void
   2604 ixgbe_get_slot_info(struct ixgbe_softc *sc)
   2605 {
   2606 	device_t		dev = sc->dev;
   2607 	struct ixgbe_hw		*hw = &sc->hw;
   2608 	u32		      offset;
   2609 	u16			link;
   2610 	int		      bus_info_valid = TRUE;
   2611 
   2612 	/* Some devices are behind an internal bridge */
   2613 	switch (hw->device_id) {
   2614 	case IXGBE_DEV_ID_82599_SFP_SF_QP:
   2615 	case IXGBE_DEV_ID_82599_QSFP_SF_QP:
   2616 		goto get_parent_info;
   2617 	default:
   2618 		break;
   2619 	}
   2620 
   2621 	ixgbe_get_bus_info(hw);
   2622 
   2623 	/*
   2624 	 * Some devices don't use PCI-E, but there is no need
   2625 	 * to display "Unknown" for bus speed and width.
   2626 	 */
   2627 	switch (hw->mac.type) {
   2628 	case ixgbe_mac_X550EM_x:
   2629 	case ixgbe_mac_X550EM_a:
   2630 		return;
   2631 	default:
   2632 		goto display;
   2633 	}
   2634 
   2635 get_parent_info:
   2636 	/*
   2637 	 * For the Quad port adapter we need to parse back
   2638 	 * up the PCI tree to find the speed of the expansion
   2639 	 * slot into which this adapter is plugged. A bit more work.
   2640 	 */
   2641 	dev = device_parent(device_parent(dev));
   2642 #if 0
   2643 #ifdef IXGBE_DEBUG
   2644 	device_printf(dev, "parent pcib = %x,%x,%x\n", pci_get_bus(dev),
   2645 	    pci_get_slot(dev), pci_get_function(dev));
   2646 #endif
   2647 	dev = device_parent(device_parent(dev));
   2648 #ifdef IXGBE_DEBUG
   2649 	device_printf(dev, "slot pcib = %x,%x,%x\n", pci_get_bus(dev),
   2650 	    pci_get_slot(dev), pci_get_function(dev));
   2651 #endif
   2652 #endif
   2653 	/* Now get the PCI Express Capabilities offset */
   2654 	if (pci_get_capability(sc->osdep.pc, sc->osdep.tag,
   2655 	    PCI_CAP_PCIEXPRESS, &offset, NULL)) {
   2656 		/*
   2657 		 * Hmm...can't get PCI-Express capabilities.
   2658 		 * Falling back to default method.
   2659 		 */
   2660 		bus_info_valid = FALSE;
   2661 		ixgbe_get_bus_info(hw);
   2662 		goto display;
   2663 	}
   2664 	/* ...and read the Link Status Register */
   2665 	link = pci_conf_read(sc->osdep.pc, sc->osdep.tag,
   2666 	    offset + PCIE_LCSR) >> 16;
   2667 	ixgbe_set_pci_config_data_generic(hw, link);
   2668 
   2669 display:
   2670 	device_printf(dev, "PCI Express Bus: Speed %s Width %s\n",
   2671 	    ((hw->bus.speed == ixgbe_bus_speed_8000)	? "8.0GT/s" :
   2672 	     (hw->bus.speed == ixgbe_bus_speed_5000)	? "5.0GT/s" :
   2673 	     (hw->bus.speed == ixgbe_bus_speed_2500)	? "2.5GT/s" :
   2674 	     "Unknown"),
   2675 	    ((hw->bus.width == ixgbe_bus_width_pcie_x8) ? "x8" :
   2676 	     (hw->bus.width == ixgbe_bus_width_pcie_x4) ? "x4" :
   2677 	     (hw->bus.width == ixgbe_bus_width_pcie_x1) ? "x1" :
   2678 	     "Unknown"));
   2679 
   2680 	if (bus_info_valid) {
   2681 		if ((hw->device_id != IXGBE_DEV_ID_82599_SFP_SF_QP) &&
   2682 		    ((hw->bus.width <= ixgbe_bus_width_pcie_x4) &&
   2683 			(hw->bus.speed == ixgbe_bus_speed_2500))) {
   2684 			device_printf(dev, "PCI-Express bandwidth available"
   2685 			    " for this card\n     is not sufficient for"
   2686 			    " optimal performance.\n");
   2687 			device_printf(dev, "For optimal performance a x8 "
   2688 			    "PCIE, or x4 PCIE Gen2 slot is required.\n");
   2689 		}
   2690 		if ((hw->device_id == IXGBE_DEV_ID_82599_SFP_SF_QP) &&
   2691 		    ((hw->bus.width <= ixgbe_bus_width_pcie_x8) &&
   2692 			(hw->bus.speed < ixgbe_bus_speed_8000))) {
   2693 			device_printf(dev, "PCI-Express bandwidth available"
   2694 			    " for this card\n     is not sufficient for"
   2695 			    " optimal performance.\n");
   2696 			device_printf(dev, "For optimal performance a x8 "
   2697 			    "PCIE Gen3 slot is required.\n");
   2698 		}
   2699 	} else
   2700 		device_printf(dev,
   2701 		    "Unable to determine slot speed/width. The speed/width "
   2702 		    "reported are that of the internal switch.\n");
   2703 
   2704 	return;
   2705 } /* ixgbe_get_slot_info */
   2706 
   2707 /************************************************************************
   2708  * ixgbe_enable_queue - Queue Interrupt Enabler
   2709  ************************************************************************/
   2710 static inline void
   2711 ixgbe_enable_queue(struct ixgbe_softc *sc, u32 vector)
   2712 {
   2713 	struct ixgbe_hw *hw = &sc->hw;
   2714 	struct ix_queue *que = &sc->queues[vector];
   2715 	u64		queue = 1ULL << vector;
   2716 	u32		mask;
   2717 
   2718 	mutex_enter(&que->dc_mtx);
   2719 	if (que->disabled_count > 0 && --que->disabled_count > 0)
   2720 		goto out;
   2721 
   2722 	if (hw->mac.type == ixgbe_mac_82598EB) {
   2723 		mask = (IXGBE_EIMS_RTX_QUEUE & queue);
   2724 		IXGBE_WRITE_REG(hw, IXGBE_EIMS, mask);
   2725 	} else {
   2726 		mask = (queue & 0xFFFFFFFF);
   2727 		if (mask)
   2728 			IXGBE_WRITE_REG(hw, IXGBE_EIMS_EX(0), mask);
   2729 		mask = (queue >> 32);
   2730 		if (mask)
   2731 			IXGBE_WRITE_REG(hw, IXGBE_EIMS_EX(1), mask);
   2732 	}
   2733 out:
   2734 	mutex_exit(&que->dc_mtx);
   2735 } /* ixgbe_enable_queue */
   2736 
   2737 /************************************************************************
   2738  * ixgbe_disable_queue_internal
   2739  ************************************************************************/
   2740 static inline void
   2741 ixgbe_disable_queue_internal(struct ixgbe_softc *sc, u32 vector, bool nestok)
   2742 {
   2743 	struct ixgbe_hw *hw = &sc->hw;
   2744 	struct ix_queue *que = &sc->queues[vector];
   2745 	u64		queue = 1ULL << vector;
   2746 	u32		mask;
   2747 
   2748 	mutex_enter(&que->dc_mtx);
   2749 
   2750 	if (que->disabled_count > 0) {
   2751 		if (nestok)
   2752 			que->disabled_count++;
   2753 		goto out;
   2754 	}
   2755 	que->disabled_count++;
   2756 
   2757 	if (hw->mac.type == ixgbe_mac_82598EB) {
   2758 		mask = (IXGBE_EIMS_RTX_QUEUE & queue);
   2759 		IXGBE_WRITE_REG(hw, IXGBE_EIMC, mask);
   2760 	} else {
   2761 		mask = (queue & 0xFFFFFFFF);
   2762 		if (mask)
   2763 			IXGBE_WRITE_REG(hw, IXGBE_EIMC_EX(0), mask);
   2764 		mask = (queue >> 32);
   2765 		if (mask)
   2766 			IXGBE_WRITE_REG(hw, IXGBE_EIMC_EX(1), mask);
   2767 	}
   2768 out:
   2769 	mutex_exit(&que->dc_mtx);
   2770 } /* ixgbe_disable_queue_internal */
   2771 
   2772 /************************************************************************
   2773  * ixgbe_disable_queue
   2774  ************************************************************************/
   2775 static inline void
   2776 ixgbe_disable_queue(struct ixgbe_softc *sc, u32 vector)
   2777 {
   2778 
   2779 	ixgbe_disable_queue_internal(sc, vector, true);
   2780 } /* ixgbe_disable_queue */
   2781 
   2782 /************************************************************************
   2783  * ixgbe_sched_handle_que - schedule deferred packet processing
   2784  ************************************************************************/
   2785 static inline void
   2786 ixgbe_sched_handle_que(struct ixgbe_softc *sc, struct ix_queue *que)
   2787 {
   2788 
   2789 	if (que->txrx_use_workqueue) {
   2790 		/*
   2791 		 * sc->que_wq is bound to each CPU instead of
   2792 		 * each NIC queue to reduce workqueue kthread. As we
   2793 		 * should consider about interrupt affinity in this
   2794 		 * function, the workqueue kthread must be WQ_PERCPU.
   2795 		 * If create WQ_PERCPU workqueue kthread for each NIC
   2796 		 * queue, that number of created workqueue kthread is
   2797 		 * (number of used NIC queue) * (number of CPUs) =
   2798 		 * (number of CPUs) ^ 2 most often.
   2799 		 *
   2800 		 * The same NIC queue's interrupts are avoided by
   2801 		 * masking the queue's interrupt. And different
   2802 		 * NIC queue's interrupts use different struct work
   2803 		 * (que->wq_cookie). So, "enqueued flag" to avoid
   2804 		 * twice workqueue_enqueue() is not required .
   2805 		 */
   2806 		workqueue_enqueue(sc->que_wq, &que->wq_cookie, curcpu());
   2807 	} else
   2808 		softint_schedule(que->que_si);
   2809 }
   2810 
   2811 /************************************************************************
   2812  * ixgbe_msix_que - MSI-X Queue Interrupt Service routine
   2813  ************************************************************************/
   2814 static int
   2815 ixgbe_msix_que(void *arg)
   2816 {
   2817 	struct ix_queue	*que = arg;
   2818 	struct ixgbe_softc *sc = que->sc;
   2819 	struct ifnet	*ifp = sc->ifp;
   2820 	struct tx_ring	*txr = que->txr;
   2821 	struct rx_ring	*rxr = que->rxr;
   2822 	u32		newitr = 0;
   2823 
   2824 	/* Protect against spurious interrupts */
   2825 	if ((ifp->if_flags & IFF_RUNNING) == 0)
   2826 		return 0;
   2827 
   2828 	ixgbe_disable_queue(sc, que->msix);
   2829 	IXGBE_EVC_ADD(&que->irqs, 1);
   2830 
   2831 	/*
   2832 	 * Don't change "que->txrx_use_workqueue" from this point to avoid
   2833 	 * flip-flopping softint/workqueue mode in one deferred processing.
   2834 	 */
   2835 	que->txrx_use_workqueue = sc->txrx_use_workqueue;
   2836 
   2837 	IXGBE_TX_LOCK(txr);
   2838 	ixgbe_txeof(txr);
   2839 	IXGBE_TX_UNLOCK(txr);
   2840 
   2841 	/* Do AIM now? */
   2842 
   2843 	if (sc->enable_aim == false)
   2844 		goto no_calc;
   2845 	/*
   2846 	 * Do Adaptive Interrupt Moderation:
   2847 	 *  - Write out last calculated setting
   2848 	 *  - Calculate based on average size over
   2849 	 *    the last interval.
   2850 	 */
   2851 	if (que->eitr_setting)
   2852 		ixgbe_eitr_write(sc, que->msix, que->eitr_setting);
   2853 
   2854 	que->eitr_setting = 0;
   2855 
   2856 	/* Idle, do nothing */
   2857 	if ((txr->bytes == 0) && (rxr->bytes == 0))
   2858 		goto no_calc;
   2859 
   2860 	if ((txr->bytes) && (txr->packets))
   2861 		newitr = txr->bytes/txr->packets;
   2862 	if ((rxr->bytes) && (rxr->packets))
   2863 		newitr = uimax(newitr, (rxr->bytes / rxr->packets));
   2864 	newitr += 24; /* account for hardware frame, crc */
   2865 
   2866 	/* set an upper boundary */
   2867 	newitr = uimin(newitr, 3000);
   2868 
   2869 	/* Be nice to the mid range */
   2870 	if ((newitr > 300) && (newitr < 1200))
   2871 		newitr = (newitr / 3);
   2872 	else
   2873 		newitr = (newitr / 2);
   2874 
   2875 	/*
   2876 	 * When RSC is used, ITR interval must be larger than RSC_DELAY.
   2877 	 * Currently, we use 2us for RSC_DELAY. The minimum value is always
   2878 	 * greater than 2us on 100M (and 10M?(not documented)), but it's not
   2879 	 * on 1G and higher.
   2880 	 */
   2881 	if ((sc->link_speed != IXGBE_LINK_SPEED_100_FULL)
   2882 	    && (sc->link_speed != IXGBE_LINK_SPEED_10_FULL))
   2883 		if (newitr < IXGBE_MIN_RSC_EITR_10G1G)
   2884 			newitr = IXGBE_MIN_RSC_EITR_10G1G;
   2885 
   2886 	/* save for next interrupt */
   2887 	que->eitr_setting = newitr;
   2888 
   2889 	/* Reset state */
   2890 	txr->bytes = 0;
   2891 	txr->packets = 0;
   2892 	rxr->bytes = 0;
   2893 	rxr->packets = 0;
   2894 
   2895 no_calc:
   2896 	ixgbe_sched_handle_que(sc, que);
   2897 
   2898 	return 1;
   2899 } /* ixgbe_msix_que */
   2900 
   2901 /************************************************************************
   2902  * ixgbe_media_status - Media Ioctl callback
   2903  *
   2904  *   Called whenever the user queries the status of
   2905  *   the interface using ifconfig.
   2906  ************************************************************************/
   2907 static void
   2908 ixgbe_media_status(struct ifnet *ifp, struct ifmediareq *ifmr)
   2909 {
   2910 	struct ixgbe_softc *sc = ifp->if_softc;
   2911 	struct ixgbe_hw *hw = &sc->hw;
   2912 	int layer;
   2913 
   2914 	INIT_DEBUGOUT("ixgbe_media_status: begin");
   2915 	ixgbe_update_link_status(sc);
   2916 
   2917 	ifmr->ifm_status = IFM_AVALID;
   2918 	ifmr->ifm_active = IFM_ETHER;
   2919 
   2920 	if (sc->link_active != LINK_STATE_UP) {
   2921 		ifmr->ifm_active |= IFM_NONE;
   2922 		return;
   2923 	}
   2924 
   2925 	ifmr->ifm_status |= IFM_ACTIVE;
   2926 	layer = sc->phy_layer;
   2927 
   2928 	if (layer & IXGBE_PHYSICAL_LAYER_10GBASE_T ||
   2929 	    layer & IXGBE_PHYSICAL_LAYER_5GBASE_T ||
   2930 	    layer & IXGBE_PHYSICAL_LAYER_2500BASE_T ||
   2931 	    layer & IXGBE_PHYSICAL_LAYER_1000BASE_T ||
   2932 	    layer & IXGBE_PHYSICAL_LAYER_100BASE_TX ||
   2933 	    layer & IXGBE_PHYSICAL_LAYER_10BASE_T)
   2934 		switch (sc->link_speed) {
   2935 		case IXGBE_LINK_SPEED_10GB_FULL:
   2936 			ifmr->ifm_active |= IFM_10G_T | IFM_FDX;
   2937 			break;
   2938 		case IXGBE_LINK_SPEED_5GB_FULL:
   2939 			ifmr->ifm_active |= IFM_5000_T | IFM_FDX;
   2940 			break;
   2941 		case IXGBE_LINK_SPEED_2_5GB_FULL:
   2942 			ifmr->ifm_active |= IFM_2500_T | IFM_FDX;
   2943 			break;
   2944 		case IXGBE_LINK_SPEED_1GB_FULL:
   2945 			ifmr->ifm_active |= IFM_1000_T | IFM_FDX;
   2946 			break;
   2947 		case IXGBE_LINK_SPEED_100_FULL:
   2948 			ifmr->ifm_active |= IFM_100_TX | IFM_FDX;
   2949 			break;
   2950 		case IXGBE_LINK_SPEED_10_FULL:
   2951 			ifmr->ifm_active |= IFM_10_T | IFM_FDX;
   2952 			break;
   2953 		}
   2954 	if (layer & IXGBE_PHYSICAL_LAYER_SFP_PLUS_CU ||
   2955 	    layer & IXGBE_PHYSICAL_LAYER_SFP_ACTIVE_DA)
   2956 		switch (sc->link_speed) {
   2957 		case IXGBE_LINK_SPEED_10GB_FULL:
   2958 			ifmr->ifm_active |= IFM_10G_TWINAX | IFM_FDX;
   2959 			break;
   2960 		}
   2961 	if (layer & IXGBE_PHYSICAL_LAYER_10GBASE_LR)
   2962 		switch (sc->link_speed) {
   2963 		case IXGBE_LINK_SPEED_10GB_FULL:
   2964 			ifmr->ifm_active |= IFM_10G_LR | IFM_FDX;
   2965 			break;
   2966 		case IXGBE_LINK_SPEED_1GB_FULL:
   2967 			ifmr->ifm_active |= IFM_1000_LX | IFM_FDX;
   2968 			break;
   2969 		}
   2970 	if (layer & IXGBE_PHYSICAL_LAYER_10GBASE_LRM)
   2971 		switch (sc->link_speed) {
   2972 		case IXGBE_LINK_SPEED_10GB_FULL:
   2973 			ifmr->ifm_active |= IFM_10G_LRM | IFM_FDX;
   2974 			break;
   2975 		case IXGBE_LINK_SPEED_1GB_FULL:
   2976 			ifmr->ifm_active |= IFM_1000_LX | IFM_FDX;
   2977 			break;
   2978 		}
   2979 	if (layer & IXGBE_PHYSICAL_LAYER_10GBASE_SR ||
   2980 	    layer & IXGBE_PHYSICAL_LAYER_1000BASE_SX)
   2981 		switch (sc->link_speed) {
   2982 		case IXGBE_LINK_SPEED_10GB_FULL:
   2983 			ifmr->ifm_active |= IFM_10G_SR | IFM_FDX;
   2984 			break;
   2985 		case IXGBE_LINK_SPEED_1GB_FULL:
   2986 			ifmr->ifm_active |= IFM_1000_SX | IFM_FDX;
   2987 			break;
   2988 		}
   2989 	if (layer & IXGBE_PHYSICAL_LAYER_10GBASE_CX4)
   2990 		switch (sc->link_speed) {
   2991 		case IXGBE_LINK_SPEED_10GB_FULL:
   2992 			ifmr->ifm_active |= IFM_10G_CX4 | IFM_FDX;
   2993 			break;
   2994 		}
   2995 	/*
   2996 	 * XXX: These need to use the proper media types once
   2997 	 * they're added.
   2998 	 */
   2999 	if (layer & IXGBE_PHYSICAL_LAYER_10GBASE_KR)
   3000 		switch (sc->link_speed) {
   3001 		case IXGBE_LINK_SPEED_10GB_FULL:
   3002 			ifmr->ifm_active |= IFM_10G_KR | IFM_FDX;
   3003 			break;
   3004 		case IXGBE_LINK_SPEED_2_5GB_FULL:
   3005 			ifmr->ifm_active |= IFM_2500_KX | IFM_FDX;
   3006 			break;
   3007 		case IXGBE_LINK_SPEED_1GB_FULL:
   3008 			ifmr->ifm_active |= IFM_1000_KX | IFM_FDX;
   3009 			break;
   3010 		}
   3011 	else if (layer & IXGBE_PHYSICAL_LAYER_10GBASE_KX4 ||
   3012 	    layer & IXGBE_PHYSICAL_LAYER_2500BASE_KX ||
   3013 	    layer & IXGBE_PHYSICAL_LAYER_1000BASE_KX)
   3014 		switch (sc->link_speed) {
   3015 		case IXGBE_LINK_SPEED_10GB_FULL:
   3016 			ifmr->ifm_active |= IFM_10G_KX4 | IFM_FDX;
   3017 			break;
   3018 		case IXGBE_LINK_SPEED_2_5GB_FULL:
   3019 			ifmr->ifm_active |= IFM_2500_KX | IFM_FDX;
   3020 			break;
   3021 		case IXGBE_LINK_SPEED_1GB_FULL:
   3022 			ifmr->ifm_active |= IFM_1000_KX | IFM_FDX;
   3023 			break;
   3024 		}
   3025 
   3026 	/* If nothing is recognized... */
   3027 #if 0
   3028 	if (IFM_SUBTYPE(ifmr->ifm_active) == 0)
   3029 		ifmr->ifm_active |= IFM_UNKNOWN;
   3030 #endif
   3031 
   3032 	ifp->if_baudrate = ifmedia_baudrate(ifmr->ifm_active);
   3033 
   3034 	/* Display current flow control setting used on link */
   3035 	if (hw->fc.current_mode == ixgbe_fc_rx_pause ||
   3036 	    hw->fc.current_mode == ixgbe_fc_full)
   3037 		ifmr->ifm_active |= IFM_ETH_RXPAUSE;
   3038 	if (hw->fc.current_mode == ixgbe_fc_tx_pause ||
   3039 	    hw->fc.current_mode == ixgbe_fc_full)
   3040 		ifmr->ifm_active |= IFM_ETH_TXPAUSE;
   3041 
   3042 	return;
   3043 } /* ixgbe_media_status */
   3044 
   3045 /************************************************************************
   3046  * ixgbe_media_change - Media Ioctl callback
   3047  *
   3048  *   Called when the user changes speed/duplex using
   3049  *   media/mediopt option with ifconfig.
   3050  ************************************************************************/
   3051 static int
   3052 ixgbe_media_change(struct ifnet *ifp)
   3053 {
   3054 	struct ixgbe_softc *sc = ifp->if_softc;
   3055 	struct ifmedia	 *ifm = &sc->media;
   3056 	struct ixgbe_hw	 *hw = &sc->hw;
   3057 	ixgbe_link_speed speed = 0;
   3058 	ixgbe_link_speed link_caps = 0;
   3059 	bool negotiate = false;
   3060 	s32 err = IXGBE_NOT_IMPLEMENTED;
   3061 
   3062 	INIT_DEBUGOUT("ixgbe_media_change: begin");
   3063 
   3064 	if (IFM_TYPE(ifm->ifm_media) != IFM_ETHER)
   3065 		return (EINVAL);
   3066 
   3067 	if (hw->phy.media_type == ixgbe_media_type_backplane)
   3068 		return (EPERM);
   3069 
   3070 	/*
   3071 	 * We don't actually need to check against the supported
   3072 	 * media types of the adapter; ifmedia will take care of
   3073 	 * that for us.
   3074 	 */
   3075 	switch (IFM_SUBTYPE(ifm->ifm_media)) {
   3076 	case IFM_AUTO:
   3077 		err = hw->mac.ops.get_link_capabilities(hw, &link_caps,
   3078 		    &negotiate);
   3079 		if (err != IXGBE_SUCCESS) {
   3080 			device_printf(sc->dev, "Unable to determine "
   3081 			    "supported advertise speeds\n");
   3082 			return (ENODEV);
   3083 		}
   3084 		speed |= link_caps;
   3085 		break;
   3086 	case IFM_10G_T:
   3087 	case IFM_10G_LRM:
   3088 	case IFM_10G_LR:
   3089 	case IFM_10G_TWINAX:
   3090 	case IFM_10G_SR:
   3091 	case IFM_10G_CX4:
   3092 	case IFM_10G_KR:
   3093 	case IFM_10G_KX4:
   3094 		speed |= IXGBE_LINK_SPEED_10GB_FULL;
   3095 		break;
   3096 	case IFM_5000_T:
   3097 		speed |= IXGBE_LINK_SPEED_5GB_FULL;
   3098 		break;
   3099 	case IFM_2500_T:
   3100 	case IFM_2500_KX:
   3101 		speed |= IXGBE_LINK_SPEED_2_5GB_FULL;
   3102 		break;
   3103 	case IFM_1000_T:
   3104 	case IFM_1000_LX:
   3105 	case IFM_1000_SX:
   3106 	case IFM_1000_KX:
   3107 		speed |= IXGBE_LINK_SPEED_1GB_FULL;
   3108 		break;
   3109 	case IFM_100_TX:
   3110 		speed |= IXGBE_LINK_SPEED_100_FULL;
   3111 		break;
   3112 	case IFM_10_T:
   3113 		speed |= IXGBE_LINK_SPEED_10_FULL;
   3114 		break;
   3115 	case IFM_NONE:
   3116 		break;
   3117 	default:
   3118 		goto invalid;
   3119 	}
   3120 
   3121 	hw->mac.autotry_restart = TRUE;
   3122 	hw->mac.ops.setup_link(hw, speed, TRUE);
   3123 	sc->advertise = 0;
   3124 	if (IFM_SUBTYPE(ifm->ifm_media) != IFM_AUTO) {
   3125 		if ((speed & IXGBE_LINK_SPEED_10GB_FULL) != 0)
   3126 			sc->advertise |= 1 << 2;
   3127 		if ((speed & IXGBE_LINK_SPEED_1GB_FULL) != 0)
   3128 			sc->advertise |= 1 << 1;
   3129 		if ((speed & IXGBE_LINK_SPEED_100_FULL) != 0)
   3130 			sc->advertise |= 1 << 0;
   3131 		if ((speed & IXGBE_LINK_SPEED_10_FULL) != 0)
   3132 			sc->advertise |= 1 << 3;
   3133 		if ((speed & IXGBE_LINK_SPEED_2_5GB_FULL) != 0)
   3134 			sc->advertise |= 1 << 4;
   3135 		if ((speed & IXGBE_LINK_SPEED_5GB_FULL) != 0)
   3136 			sc->advertise |= 1 << 5;
   3137 	}
   3138 
   3139 	return (0);
   3140 
   3141 invalid:
   3142 	device_printf(sc->dev, "Invalid media type!\n");
   3143 
   3144 	return (EINVAL);
   3145 } /* ixgbe_media_change */
   3146 
   3147 /************************************************************************
   3148  * ixgbe_msix_admin - Link status change ISR (MSI-X)
   3149  ************************************************************************/
   3150 static int
   3151 ixgbe_msix_admin(void *arg)
   3152 {
   3153 	struct ixgbe_softc *sc = arg;
   3154 	struct ixgbe_hw *hw = &sc->hw;
   3155 	u32		eicr;
   3156 	u32		eims_orig;
   3157 	u32		eims_disable = 0;
   3158 
   3159 	IXGBE_EVC_ADD(&sc->admin_irqev, 1);
   3160 
   3161 	eims_orig = IXGBE_READ_REG(hw, IXGBE_EIMS);
   3162 	/* Pause other interrupts */
   3163 	IXGBE_WRITE_REG(hw, IXGBE_EIMC, IXGBE_MSIX_OTHER_CLEAR_MASK);
   3164 
   3165 	/*
   3166 	 * First get the cause.
   3167 	 *
   3168 	 * The specifications of 82598, 82599, X540 and X550 say EICS register
   3169 	 * is write only. However, Linux says it is a workaround for silicon
   3170 	 * errata to read EICS instead of EICR to get interrupt cause.
   3171 	 * At least, reading EICR clears lower 16bits of EIMS on 82598.
   3172 	 */
   3173 	eicr = IXGBE_READ_REG(hw, IXGBE_EICS);
   3174 	/* Be sure the queue bits are not cleared */
   3175 	eicr &= ~IXGBE_EICR_RTX_QUEUE;
   3176 	/* Clear all OTHER interrupts with write */
   3177 	IXGBE_WRITE_REG(hw, IXGBE_EICR, eicr);
   3178 
   3179 	ixgbe_intr_admin_common(sc, eicr, &eims_disable);
   3180 
   3181 	/* Re-enable some OTHER interrupts */
   3182 	IXGBE_WRITE_REG(hw, IXGBE_EIMS, eims_orig & ~eims_disable);
   3183 
   3184 	return 1;
   3185 } /* ixgbe_msix_admin */
   3186 
   3187 static void
   3188 ixgbe_intr_admin_common(struct ixgbe_softc *sc, u32 eicr, u32 *eims_disable)
   3189 {
   3190 	struct ixgbe_hw *hw = &sc->hw;
   3191 	u32		task_requests = 0;
   3192 	s32		retval;
   3193 
   3194 	/* Link status change */
   3195 	if (eicr & IXGBE_EICR_LSC) {
   3196 		task_requests |= IXGBE_REQUEST_TASK_LSC;
   3197 		*eims_disable |= IXGBE_EIMS_LSC;
   3198 	}
   3199 
   3200 	if (ixgbe_is_sfp(hw)) {
   3201 		u32 eicr_mask;
   3202 
   3203 		/* Pluggable optics-related interrupt */
   3204 		if (hw->mac.type >= ixgbe_mac_X540)
   3205 			eicr_mask = IXGBE_EICR_GPI_SDP0_X540;
   3206 		else
   3207 			eicr_mask = IXGBE_EICR_GPI_SDP2_BY_MAC(hw);
   3208 
   3209 		/*
   3210 		 *  An interrupt might not arrive when a module is inserted.
   3211 		 * When an link status change interrupt occurred and the driver
   3212 		 * still regard SFP as unplugged, issue the module softint
   3213 		 * and then issue LSC interrupt.
   3214 		 */
   3215 		if ((eicr & eicr_mask)
   3216 		    || ((hw->phy.sfp_type == ixgbe_sfp_type_not_present)
   3217 			&& (eicr & IXGBE_EICR_LSC))) {
   3218 			task_requests |= IXGBE_REQUEST_TASK_MOD;
   3219 			*eims_disable |= IXGBE_EIMS_LSC;
   3220 		}
   3221 
   3222 		if ((hw->mac.type == ixgbe_mac_82599EB) &&
   3223 		    (eicr & IXGBE_EICR_GPI_SDP1_BY_MAC(hw))) {
   3224 			task_requests |= IXGBE_REQUEST_TASK_MSF;
   3225 			*eims_disable |= IXGBE_EIMS_GPI_SDP1_BY_MAC(hw);
   3226 		}
   3227 	}
   3228 
   3229 	if (sc->hw.mac.type != ixgbe_mac_82598EB) {
   3230 #ifdef IXGBE_FDIR
   3231 		if ((sc->feat_en & IXGBE_FEATURE_FDIR) &&
   3232 		    (eicr & IXGBE_EICR_FLOW_DIR)) {
   3233 			if (!atomic_cas_uint(&sc->fdir_reinit, 0, 1)) {
   3234 				task_requests |= IXGBE_REQUEST_TASK_FDIR;
   3235 				/* Disable the interrupt */
   3236 				*eims_disable |= IXGBE_EIMS_FLOW_DIR;
   3237 			}
   3238 		}
   3239 #endif
   3240 
   3241 		if (eicr & IXGBE_EICR_ECC) {
   3242 			if (ratecheck(&sc->lasterr_time,
   3243 			    &ixgbe_errlog_intrvl))
   3244 				device_printf(sc->dev,
   3245 				    "CRITICAL: ECC ERROR!! Please Reboot!!\n");
   3246 		}
   3247 
   3248 		/* Check for over temp condition */
   3249 		if (sc->feat_en & IXGBE_FEATURE_TEMP_SENSOR) {
   3250 			switch (sc->hw.mac.type) {
   3251 			case ixgbe_mac_X550EM_a:
   3252 				if (!(eicr & IXGBE_EICR_GPI_SDP0_X550EM_a))
   3253 					break;
   3254 				retval = hw->phy.ops.check_overtemp(hw);
   3255 				if (retval != IXGBE_ERR_OVERTEMP)
   3256 					break;
   3257 				if (ratecheck(&sc->lasterr_time,
   3258 				    &ixgbe_errlog_intrvl)) {
   3259 					device_printf(sc->dev,
   3260 					    "CRITICAL: OVER TEMP!! "
   3261 					    "PHY IS SHUT DOWN!!\n");
   3262 					device_printf(sc->dev,
   3263 					    "System shutdown required!\n");
   3264 				}
   3265 				break;
   3266 			default:
   3267 				if (!(eicr & IXGBE_EICR_TS))
   3268 					break;
   3269 				retval = hw->phy.ops.check_overtemp(hw);
   3270 				if (retval != IXGBE_ERR_OVERTEMP)
   3271 					break;
   3272 				if (ratecheck(&sc->lasterr_time,
   3273 				    &ixgbe_errlog_intrvl)) {
   3274 					device_printf(sc->dev,
   3275 					    "CRITICAL: OVER TEMP!! "
   3276 					    "PHY IS SHUT DOWN!!\n");
   3277 					device_printf(sc->dev,
   3278 					    "System shutdown required!\n");
   3279 				}
   3280 				break;
   3281 			}
   3282 		}
   3283 
   3284 		/* Check for VF message */
   3285 		if ((sc->feat_en & IXGBE_FEATURE_SRIOV) &&
   3286 		    (eicr & IXGBE_EICR_MAILBOX)) {
   3287 			task_requests |= IXGBE_REQUEST_TASK_MBX;
   3288 			*eims_disable |= IXGBE_EIMS_MAILBOX;
   3289 		}
   3290 	}
   3291 
   3292 	/* Check for fan failure */
   3293 	if (sc->feat_en & IXGBE_FEATURE_FAN_FAIL)
   3294 		ixgbe_check_fan_failure(sc, eicr, true);
   3295 
   3296 	/* External PHY interrupt */
   3297 	if ((hw->phy.type == ixgbe_phy_x550em_ext_t) &&
   3298 	    (eicr & IXGBE_EICR_GPI_SDP0_X540)) {
   3299 		task_requests |= IXGBE_REQUEST_TASK_PHY;
   3300 		*eims_disable |= IXGBE_EICR_GPI_SDP0_X540;
   3301 	}
   3302 
   3303 	if (task_requests != 0) {
   3304 		mutex_enter(&sc->admin_mtx);
   3305 		sc->task_requests |= task_requests;
   3306 		ixgbe_schedule_admin_tasklet(sc);
   3307 		mutex_exit(&sc->admin_mtx);
   3308 	}
   3309 }
   3310 
   3311 static void
   3312 ixgbe_eitr_write(struct ixgbe_softc *sc, uint32_t index, uint32_t itr)
   3313 {
   3314 
   3315 	if (sc->hw.mac.type == ixgbe_mac_82598EB)
   3316 		itr |= itr << 16;
   3317 	else
   3318 		itr |= IXGBE_EITR_CNT_WDIS;
   3319 
   3320 	IXGBE_WRITE_REG(&sc->hw, IXGBE_EITR(index), itr);
   3321 }
   3322 
   3323 
   3324 /************************************************************************
   3325  * ixgbe_sysctl_interrupt_rate_handler
   3326  ************************************************************************/
   3327 static int
   3328 ixgbe_sysctl_interrupt_rate_handler(SYSCTLFN_ARGS)
   3329 {
   3330 	struct sysctlnode node = *rnode;
   3331 	struct ix_queue *que = (struct ix_queue *)node.sysctl_data;
   3332 	struct ixgbe_softc *sc;
   3333 	uint32_t reg, usec, rate;
   3334 	int error;
   3335 
   3336 	if (que == NULL)
   3337 		return 0;
   3338 
   3339 	sc = que->sc;
   3340 	if (ixgbe_fw_recovery_mode_swflag(sc))
   3341 		return (EPERM);
   3342 
   3343 	reg = IXGBE_READ_REG(&sc->hw, IXGBE_EITR(que->msix));
   3344 	usec = ((reg & 0x0FF8) >> 3);
   3345 	if (usec > 0)
   3346 		rate = 500000 / usec;
   3347 	else
   3348 		rate = 0;
   3349 	node.sysctl_data = &rate;
   3350 	error = sysctl_lookup(SYSCTLFN_CALL(&node));
   3351 	if (error || newp == NULL)
   3352 		return error;
   3353 	reg &= ~0xfff; /* default, no limitation */
   3354 	if (rate > 0 && rate < 500000) {
   3355 		if (rate < 1000)
   3356 			rate = 1000;
   3357 		reg |= ((4000000 / rate) & 0xff8);
   3358 		/*
   3359 		 * When RSC is used, ITR interval must be larger than
   3360 		 * RSC_DELAY. Currently, we use 2us for RSC_DELAY.
   3361 		 * The minimum value is always greater than 2us on 100M
   3362 		 * (and 10M?(not documented)), but it's not on 1G and higher.
   3363 		 */
   3364 		if ((sc->link_speed != IXGBE_LINK_SPEED_100_FULL)
   3365 		    && (sc->link_speed != IXGBE_LINK_SPEED_10_FULL)) {
   3366 			if ((sc->num_queues > 1)
   3367 			    && (reg < IXGBE_MIN_RSC_EITR_10G1G))
   3368 				return EINVAL;
   3369 		}
   3370 		sc->max_interrupt_rate = rate;
   3371 	} else
   3372 		sc->max_interrupt_rate = 0;
   3373 	ixgbe_eitr_write(sc, que->msix, reg);
   3374 
   3375 	return (0);
   3376 } /* ixgbe_sysctl_interrupt_rate_handler */
   3377 
   3378 const struct sysctlnode *
   3379 ixgbe_sysctl_instance(struct ixgbe_softc *sc)
   3380 {
   3381 	const char *dvname;
   3382 	struct sysctllog **log;
   3383 	int rc;
   3384 	const struct sysctlnode *rnode;
   3385 
   3386 	if (sc->sysctltop != NULL)
   3387 		return sc->sysctltop;
   3388 
   3389 	log = &sc->sysctllog;
   3390 	dvname = device_xname(sc->dev);
   3391 
   3392 	if ((rc = sysctl_createv(log, 0, NULL, &rnode,
   3393 	    0, CTLTYPE_NODE, dvname,
   3394 	    SYSCTL_DESCR("ixgbe information and settings"),
   3395 	    NULL, 0, NULL, 0, CTL_HW, CTL_CREATE, CTL_EOL)) != 0)
   3396 		goto err;
   3397 
   3398 	return rnode;
   3399 err:
   3400 	device_printf(sc->dev,
   3401 	    "%s: sysctl_createv failed, rc = %d\n", __func__, rc);
   3402 	return NULL;
   3403 }
   3404 
   3405 /************************************************************************
   3406  * ixgbe_add_device_sysctls
   3407  ************************************************************************/
   3408 static void
   3409 ixgbe_add_device_sysctls(struct ixgbe_softc *sc)
   3410 {
   3411 	device_t	       dev = sc->dev;
   3412 	struct ixgbe_hw	       *hw = &sc->hw;
   3413 	struct sysctllog **log;
   3414 	const struct sysctlnode *rnode, *cnode;
   3415 
   3416 	log = &sc->sysctllog;
   3417 
   3418 	if ((rnode = ixgbe_sysctl_instance(sc)) == NULL) {
   3419 		aprint_error_dev(dev, "could not create sysctl root\n");
   3420 		return;
   3421 	}
   3422 
   3423 	if (sysctl_createv(log, 0, &rnode, &cnode,
   3424 	    CTLFLAG_READWRITE, CTLTYPE_INT,
   3425 	    "debug", SYSCTL_DESCR("Debug Info"),
   3426 	    ixgbe_sysctl_debug, 0, (void *)sc, 0, CTL_CREATE, CTL_EOL)
   3427 	    != 0)
   3428 		aprint_error_dev(dev, "could not create sysctl\n");
   3429 
   3430 	if (sysctl_createv(log, 0, &rnode, &cnode,
   3431 	    CTLFLAG_READWRITE, CTLTYPE_INT,
   3432 	    "rx_copy_len", SYSCTL_DESCR("RX Copy Length"),
   3433 	    ixgbe_sysctl_rx_copy_len, 0,
   3434 	    (void *)sc, 0, CTL_CREATE, CTL_EOL) != 0)
   3435 		aprint_error_dev(dev, "could not create sysctl\n");
   3436 
   3437 	if (sysctl_createv(log, 0, &rnode, &cnode,
   3438 	    CTLFLAG_READONLY, CTLTYPE_INT,
   3439 	    "num_tx_desc", SYSCTL_DESCR("Number of TX descriptors"),
   3440 	    NULL, 0, &sc->num_tx_desc, 0, CTL_CREATE, CTL_EOL) != 0)
   3441 		aprint_error_dev(dev, "could not create sysctl\n");
   3442 
   3443 	if (sysctl_createv(log, 0, &rnode, &cnode,
   3444 	    CTLFLAG_READONLY, CTLTYPE_INT,
   3445 	    "num_rx_desc", SYSCTL_DESCR("Number of RX descriptors"),
   3446 	    NULL, 0, &sc->num_rx_desc, 0, CTL_CREATE, CTL_EOL) != 0)
   3447 		aprint_error_dev(dev, "could not create sysctl\n");
   3448 
   3449 	if (sysctl_createv(log, 0, &rnode, &cnode,
   3450 	    CTLFLAG_READWRITE, CTLTYPE_INT, "rx_process_limit",
   3451 	    SYSCTL_DESCR("max number of RX packets to process"),
   3452 	    ixgbe_sysctl_rx_process_limit, 0, (void *)sc, 0, CTL_CREATE,
   3453 	    CTL_EOL) != 0)
   3454 		aprint_error_dev(dev, "could not create sysctl\n");
   3455 
   3456 	if (sysctl_createv(log, 0, &rnode, &cnode,
   3457 	    CTLFLAG_READWRITE, CTLTYPE_INT, "tx_process_limit",
   3458 	    SYSCTL_DESCR("max number of TX packets to process"),
   3459 	    ixgbe_sysctl_tx_process_limit, 0, (void *)sc, 0, CTL_CREATE,
   3460 	    CTL_EOL) != 0)
   3461 		aprint_error_dev(dev, "could not create sysctl\n");
   3462 
   3463 	if (sysctl_createv(log, 0, &rnode, &cnode,
   3464 	    CTLFLAG_READONLY, CTLTYPE_INT,
   3465 	    "num_queues", SYSCTL_DESCR("Number of queues"),
   3466 	    NULL, 0, &sc->num_queues, 0, CTL_CREATE, CTL_EOL) != 0)
   3467 		aprint_error_dev(dev, "could not create sysctl\n");
   3468 
   3469 	/* Sysctls for all devices */
   3470 	if (sysctl_createv(log, 0, &rnode, &cnode, CTLFLAG_READWRITE,
   3471 	    CTLTYPE_INT, "fc", SYSCTL_DESCR(IXGBE_SYSCTL_DESC_SET_FC),
   3472 	    ixgbe_sysctl_flowcntl, 0, (void *)sc, 0, CTL_CREATE,
   3473 	    CTL_EOL) != 0)
   3474 		aprint_error_dev(dev, "could not create sysctl\n");
   3475 
   3476 	sc->enable_aim = ixgbe_enable_aim;
   3477 	sc->max_interrupt_rate = ixgbe_max_interrupt_rate;
   3478 	if (sysctl_createv(log, 0, &rnode, &cnode, CTLFLAG_READWRITE,
   3479 	    CTLTYPE_BOOL, "enable_aim", SYSCTL_DESCR("Interrupt Moderation"),
   3480 	    NULL, 0, &sc->enable_aim, 0, CTL_CREATE, CTL_EOL) != 0)
   3481 		aprint_error_dev(dev, "could not create sysctl\n");
   3482 
   3483 	if (sysctl_createv(log, 0, &rnode, &cnode,
   3484 	    CTLFLAG_READWRITE, CTLTYPE_INT,
   3485 	    "advertise_speed", SYSCTL_DESCR(IXGBE_SYSCTL_DESC_ADV_SPEED),
   3486 	    ixgbe_sysctl_advertise, 0, (void *)sc, 0, CTL_CREATE,
   3487 	    CTL_EOL) != 0)
   3488 		aprint_error_dev(dev, "could not create sysctl\n");
   3489 
   3490 	/*
   3491 	 * If each "que->txrx_use_workqueue" is changed in sysctl handler,
   3492 	 * it causesflip-flopping softint/workqueue mode in one deferred
   3493 	 * processing. Therefore, preempt_disable()/preempt_enable() are
   3494 	 * required in ixgbe_sched_handle_que() to avoid
   3495 	 * KASSERT(ixgbe_sched_handle_que()) in softint_schedule().
   3496 	 * I think changing "que->txrx_use_workqueue" in interrupt handler
   3497 	 * is lighter than doing preempt_disable()/preempt_enable() in every
   3498 	 * ixgbe_sched_handle_que().
   3499 	 */
   3500 	sc->txrx_use_workqueue = ixgbe_txrx_workqueue;
   3501 	if (sysctl_createv(log, 0, &rnode, &cnode, CTLFLAG_READWRITE,
   3502 	    CTLTYPE_BOOL, "txrx_workqueue",
   3503 	    SYSCTL_DESCR("Use workqueue for packet processing"),
   3504 	    NULL, 0, &sc->txrx_use_workqueue, 0, CTL_CREATE,
   3505 	    CTL_EOL) != 0)
   3506 		aprint_error_dev(dev, "could not create sysctl\n");
   3507 
   3508 #ifdef IXGBE_DEBUG
   3509 	/* testing sysctls (for all devices) */
   3510 	if (sysctl_createv(log, 0, &rnode, &cnode, CTLFLAG_READWRITE,
   3511 	    CTLTYPE_INT, "power_state", SYSCTL_DESCR("PCI Power State"),
   3512 	    ixgbe_sysctl_power_state, 0, (void *)sc, 0, CTL_CREATE,
   3513 	    CTL_EOL) != 0)
   3514 		aprint_error_dev(dev, "could not create sysctl\n");
   3515 
   3516 	if (sysctl_createv(log, 0, &rnode, &cnode, CTLFLAG_READONLY,
   3517 	    CTLTYPE_STRING, "print_rss_config",
   3518 	    SYSCTL_DESCR("Prints RSS Configuration"),
   3519 	    ixgbe_sysctl_print_rss_config, 0, (void *)sc, 0, CTL_CREATE,
   3520 	    CTL_EOL) != 0)
   3521 		aprint_error_dev(dev, "could not create sysctl\n");
   3522 #endif
   3523 	/* for X550 series devices */
   3524 	if (hw->mac.type >= ixgbe_mac_X550)
   3525 		if (sysctl_createv(log, 0, &rnode, &cnode, CTLFLAG_READWRITE,
   3526 		    CTLTYPE_INT, "dmac", SYSCTL_DESCR("DMA Coalesce"),
   3527 		    ixgbe_sysctl_dmac, 0, (void *)sc, 0, CTL_CREATE,
   3528 		    CTL_EOL) != 0)
   3529 			aprint_error_dev(dev, "could not create sysctl\n");
   3530 
   3531 	/* for WoL-capable devices */
   3532 	if (sc->wol_support) {
   3533 		if (sysctl_createv(log, 0, &rnode, &cnode, CTLFLAG_READWRITE,
   3534 		    CTLTYPE_BOOL, "wol_enable",
   3535 		    SYSCTL_DESCR("Enable/Disable Wake on LAN"),
   3536 		    ixgbe_sysctl_wol_enable, 0, (void *)sc, 0, CTL_CREATE,
   3537 		    CTL_EOL) != 0)
   3538 			aprint_error_dev(dev, "could not create sysctl\n");
   3539 
   3540 		if (sysctl_createv(log, 0, &rnode, &cnode, CTLFLAG_READWRITE,
   3541 		    CTLTYPE_INT, "wufc",
   3542 		    SYSCTL_DESCR("Enable/Disable Wake Up Filters"),
   3543 		    ixgbe_sysctl_wufc, 0, (void *)sc, 0, CTL_CREATE,
   3544 		    CTL_EOL) != 0)
   3545 			aprint_error_dev(dev, "could not create sysctl\n");
   3546 	}
   3547 
   3548 	/* for X552/X557-AT devices */
   3549 	if ((hw->device_id == IXGBE_DEV_ID_X550EM_X_10G_T) ||
   3550 	    (hw->device_id == IXGBE_DEV_ID_X550EM_A_10G_T)) {
   3551 		const struct sysctlnode *phy_node;
   3552 
   3553 		if (sysctl_createv(log, 0, &rnode, &phy_node, 0, CTLTYPE_NODE,
   3554 		    "phy", SYSCTL_DESCR("External PHY sysctls"),
   3555 		    NULL, 0, NULL, 0, CTL_CREATE, CTL_EOL) != 0) {
   3556 			aprint_error_dev(dev, "could not create sysctl\n");
   3557 			return;
   3558 		}
   3559 
   3560 		if (sysctl_createv(log, 0, &phy_node, &cnode, CTLFLAG_READONLY,
   3561 		    CTLTYPE_INT, "temp",
   3562 		    SYSCTL_DESCR("Current External PHY Temperature (Celsius)"),
   3563 		    ixgbe_sysctl_phy_temp, 0, (void *)sc, 0, CTL_CREATE,
   3564 		    CTL_EOL) != 0)
   3565 			aprint_error_dev(dev, "could not create sysctl\n");
   3566 
   3567 		if (sysctl_createv(log, 0, &phy_node, &cnode, CTLFLAG_READONLY,
   3568 		    CTLTYPE_INT, "overtemp_occurred",
   3569 		    SYSCTL_DESCR(
   3570 			    "External PHY High Temperature Event Occurred"),
   3571 		    ixgbe_sysctl_phy_overtemp_occurred, 0, (void *)sc, 0,
   3572 		    CTL_CREATE, CTL_EOL) != 0)
   3573 			aprint_error_dev(dev, "could not create sysctl\n");
   3574 	}
   3575 
   3576 	if ((hw->mac.type == ixgbe_mac_X550EM_a)
   3577 	    && (hw->phy.type == ixgbe_phy_fw))
   3578 		if (sysctl_createv(log, 0, &rnode, &cnode, CTLFLAG_READWRITE,
   3579 		    CTLTYPE_BOOL, "force_10_100_autonego",
   3580 		    SYSCTL_DESCR("Force autonego on 10M and 100M"),
   3581 		    NULL, 0, &hw->phy.force_10_100_autonego, 0,
   3582 		    CTL_CREATE, CTL_EOL) != 0)
   3583 			aprint_error_dev(dev, "could not create sysctl\n");
   3584 
   3585 	if (sc->feat_cap & IXGBE_FEATURE_EEE) {
   3586 		if (sysctl_createv(log, 0, &rnode, &cnode, CTLFLAG_READWRITE,
   3587 		    CTLTYPE_INT, "eee_state",
   3588 		    SYSCTL_DESCR("EEE Power Save State"),
   3589 		    ixgbe_sysctl_eee_state, 0, (void *)sc, 0, CTL_CREATE,
   3590 		    CTL_EOL) != 0)
   3591 			aprint_error_dev(dev, "could not create sysctl\n");
   3592 	}
   3593 } /* ixgbe_add_device_sysctls */
   3594 
   3595 /************************************************************************
   3596  * ixgbe_allocate_pci_resources
   3597  ************************************************************************/
   3598 static int
   3599 ixgbe_allocate_pci_resources(struct ixgbe_softc *sc,
   3600     const struct pci_attach_args *pa)
   3601 {
   3602 	pcireg_t memtype, csr;
   3603 	device_t dev = sc->dev;
   3604 	bus_addr_t addr;
   3605 	int flags;
   3606 
   3607 	memtype = pci_mapreg_type(pa->pa_pc, pa->pa_tag, PCI_BAR(0));
   3608 	switch (memtype) {
   3609 	case PCI_MAPREG_TYPE_MEM | PCI_MAPREG_MEM_TYPE_32BIT:
   3610 	case PCI_MAPREG_TYPE_MEM | PCI_MAPREG_MEM_TYPE_64BIT:
   3611 		sc->osdep.mem_bus_space_tag = pa->pa_memt;
   3612 		if (pci_mapreg_info(pa->pa_pc, pa->pa_tag, PCI_BAR(0),
   3613 		      memtype, &addr, &sc->osdep.mem_size, &flags) != 0)
   3614 			goto map_err;
   3615 		if ((flags & BUS_SPACE_MAP_PREFETCHABLE) != 0) {
   3616 			aprint_normal_dev(dev, "clearing prefetchable bit\n");
   3617 			flags &= ~BUS_SPACE_MAP_PREFETCHABLE;
   3618 		}
   3619 		if (bus_space_map(sc->osdep.mem_bus_space_tag, addr,
   3620 		     sc->osdep.mem_size, flags,
   3621 		     &sc->osdep.mem_bus_space_handle) != 0) {
   3622 map_err:
   3623 			sc->osdep.mem_size = 0;
   3624 			aprint_error_dev(dev, "unable to map BAR0\n");
   3625 			return ENXIO;
   3626 		}
   3627 		/*
   3628 		 * Enable address decoding for memory range in case BIOS or
   3629 		 * UEFI don't set it.
   3630 		 */
   3631 		csr = pci_conf_read(pa->pa_pc, pa->pa_tag,
   3632 		    PCI_COMMAND_STATUS_REG);
   3633 		csr |= PCI_COMMAND_MEM_ENABLE;
   3634 		pci_conf_write(pa->pa_pc, pa->pa_tag, PCI_COMMAND_STATUS_REG,
   3635 		    csr);
   3636 		break;
   3637 	default:
   3638 		aprint_error_dev(dev, "unexpected type on BAR0\n");
   3639 		return ENXIO;
   3640 	}
   3641 
   3642 	return (0);
   3643 } /* ixgbe_allocate_pci_resources */
   3644 
   3645 static void
   3646 ixgbe_free_deferred_handlers(struct ixgbe_softc *sc)
   3647 {
   3648 	struct ix_queue *que = sc->queues;
   3649 	struct tx_ring *txr = sc->tx_rings;
   3650 	int i;
   3651 
   3652 	for (i = 0; i < sc->num_queues; i++, que++, txr++) {
   3653 		if (!(sc->feat_en & IXGBE_FEATURE_LEGACY_TX)) {
   3654 			if (txr->txr_si != NULL)
   3655 				softint_disestablish(txr->txr_si);
   3656 		}
   3657 		if (que->que_si != NULL)
   3658 			softint_disestablish(que->que_si);
   3659 	}
   3660 	if (sc->txr_wq != NULL)
   3661 		workqueue_destroy(sc->txr_wq);
   3662 	if (sc->txr_wq_enqueued != NULL)
   3663 		percpu_free(sc->txr_wq_enqueued, sizeof(u_int));
   3664 	if (sc->que_wq != NULL)
   3665 		workqueue_destroy(sc->que_wq);
   3666 
   3667 	if (sc->admin_wq != NULL) {
   3668 		workqueue_destroy(sc->admin_wq);
   3669 		sc->admin_wq = NULL;
   3670 	}
   3671 	if (sc->timer_wq != NULL) {
   3672 		workqueue_destroy(sc->timer_wq);
   3673 		sc->timer_wq = NULL;
   3674 	}
   3675 	if (sc->recovery_mode_timer_wq != NULL) {
   3676 		/*
   3677 		 * ixgbe_ifstop() doesn't call the workqueue_wait() for
   3678 		 * the recovery_mode_timer workqueue, so call it here.
   3679 		 */
   3680 		workqueue_wait(sc->recovery_mode_timer_wq,
   3681 		    &sc->recovery_mode_timer_wc);
   3682 		atomic_store_relaxed(&sc->recovery_mode_timer_pending, 0);
   3683 		workqueue_destroy(sc->recovery_mode_timer_wq);
   3684 		sc->recovery_mode_timer_wq = NULL;
   3685 	}
   3686 } /* ixgbe_free_deferred_handlers */
   3687 
   3688 /************************************************************************
   3689  * ixgbe_detach - Device removal routine
   3690  *
   3691  *   Called when the driver is being removed.
   3692  *   Stops the adapter and deallocates all the resources
   3693  *   that were allocated for driver operation.
   3694  *
   3695  *   return 0 on success, positive on failure
   3696  ************************************************************************/
   3697 static int
   3698 ixgbe_detach(device_t dev, int flags)
   3699 {
   3700 	struct ixgbe_softc *sc = device_private(dev);
   3701 	struct rx_ring *rxr = sc->rx_rings;
   3702 	struct tx_ring *txr = sc->tx_rings;
   3703 	struct ixgbe_hw *hw = &sc->hw;
   3704 	struct ixgbe_hw_stats *stats = &sc->stats.pf;
   3705 	u32	ctrl_ext;
   3706 	int i;
   3707 
   3708 	INIT_DEBUGOUT("ixgbe_detach: begin");
   3709 	if (sc->osdep.attached == false)
   3710 		return 0;
   3711 
   3712 	if (ixgbe_pci_iov_detach(dev) != 0) {
   3713 		device_printf(dev, "SR-IOV in use; detach first.\n");
   3714 		return (EBUSY);
   3715 	}
   3716 
   3717 	if (VLAN_ATTACHED(&sc->osdep.ec) &&
   3718 	    (flags & (DETACH_SHUTDOWN | DETACH_FORCE)) == 0) {
   3719 		aprint_error_dev(dev, "VLANs in use, detach first\n");
   3720 		return (EBUSY);
   3721 	}
   3722 
   3723 	ether_ifdetach(sc->ifp);
   3724 
   3725 	sc->osdep.detaching = true;
   3726 	/*
   3727 	 * Stop the interface. ixgbe_setup_low_power_mode() calls
   3728 	 * ixgbe_ifstop(), so it's not required to call ixgbe_ifstop()
   3729 	 * directly.
   3730 	 */
   3731 	ixgbe_setup_low_power_mode(sc);
   3732 
   3733 	callout_halt(&sc->timer, NULL);
   3734 	if (sc->feat_en & IXGBE_FEATURE_RECOVERY_MODE)
   3735 		callout_halt(&sc->recovery_mode_timer, NULL);
   3736 
   3737 	workqueue_wait(sc->admin_wq, &sc->admin_wc);
   3738 	atomic_store_relaxed(&sc->admin_pending, 0);
   3739 	workqueue_wait(sc->timer_wq, &sc->timer_wc);
   3740 	atomic_store_relaxed(&sc->timer_pending, 0);
   3741 
   3742 	pmf_device_deregister(dev);
   3743 
   3744 	ixgbe_free_deferred_handlers(sc);
   3745 
   3746 	/* let hardware know driver is unloading */
   3747 	ctrl_ext = IXGBE_READ_REG(&sc->hw, IXGBE_CTRL_EXT);
   3748 	ctrl_ext &= ~IXGBE_CTRL_EXT_DRV_LOAD;
   3749 	IXGBE_WRITE_REG(&sc->hw, IXGBE_CTRL_EXT, ctrl_ext);
   3750 
   3751 	if (sc->feat_en & IXGBE_FEATURE_NETMAP)
   3752 		netmap_detach(sc->ifp);
   3753 
   3754 	ixgbe_free_pci_resources(sc);
   3755 #if 0	/* XXX the NetBSD port is probably missing something here */
   3756 	bus_generic_detach(dev);
   3757 #endif
   3758 	if_detach(sc->ifp);
   3759 	ifmedia_fini(&sc->media);
   3760 	if_percpuq_destroy(sc->ipq);
   3761 
   3762 	sysctl_teardown(&sc->sysctllog);
   3763 	evcnt_detach(&sc->efbig_tx_dma_setup);
   3764 	evcnt_detach(&sc->mbuf_defrag_failed);
   3765 	evcnt_detach(&sc->efbig2_tx_dma_setup);
   3766 	evcnt_detach(&sc->einval_tx_dma_setup);
   3767 	evcnt_detach(&sc->other_tx_dma_setup);
   3768 	evcnt_detach(&sc->eagain_tx_dma_setup);
   3769 	evcnt_detach(&sc->enomem_tx_dma_setup);
   3770 	evcnt_detach(&sc->watchdog_events);
   3771 	evcnt_detach(&sc->tso_err);
   3772 	evcnt_detach(&sc->admin_irqev);
   3773 	evcnt_detach(&sc->link_workev);
   3774 	evcnt_detach(&sc->mod_workev);
   3775 	evcnt_detach(&sc->msf_workev);
   3776 	evcnt_detach(&sc->phy_workev);
   3777 
   3778 	for (i = 0; i < IXGBE_TC_COUNTER_NUM; i++) {
   3779 		if (i < __arraycount(stats->mpc)) {
   3780 			evcnt_detach(&stats->mpc[i]);
   3781 			if (hw->mac.type == ixgbe_mac_82598EB)
   3782 				evcnt_detach(&stats->rnbc[i]);
   3783 		}
   3784 		if (i < __arraycount(stats->pxontxc)) {
   3785 			evcnt_detach(&stats->pxontxc[i]);
   3786 			evcnt_detach(&stats->pxonrxc[i]);
   3787 			evcnt_detach(&stats->pxofftxc[i]);
   3788 			evcnt_detach(&stats->pxoffrxc[i]);
   3789 			if (hw->mac.type >= ixgbe_mac_82599EB)
   3790 				evcnt_detach(&stats->pxon2offc[i]);
   3791 		}
   3792 	}
   3793 
   3794 	txr = sc->tx_rings;
   3795 	for (i = 0; i < sc->num_queues; i++, rxr++, txr++) {
   3796 		evcnt_detach(&sc->queues[i].irqs);
   3797 		evcnt_detach(&sc->queues[i].handleq);
   3798 		evcnt_detach(&sc->queues[i].req);
   3799 		evcnt_detach(&txr->total_packets);
   3800 #ifndef IXGBE_LEGACY_TX
   3801 		evcnt_detach(&txr->pcq_drops);
   3802 #endif
   3803 		evcnt_detach(&txr->no_desc_avail);
   3804 		evcnt_detach(&txr->tso_tx);
   3805 
   3806 		if (i < __arraycount(stats->qprc)) {
   3807 			evcnt_detach(&stats->qprc[i]);
   3808 			evcnt_detach(&stats->qptc[i]);
   3809 			evcnt_detach(&stats->qbrc[i]);
   3810 			evcnt_detach(&stats->qbtc[i]);
   3811 			if (hw->mac.type >= ixgbe_mac_82599EB)
   3812 				evcnt_detach(&stats->qprdc[i]);
   3813 		}
   3814 
   3815 		evcnt_detach(&rxr->rx_packets);
   3816 		evcnt_detach(&rxr->rx_bytes);
   3817 		evcnt_detach(&rxr->rx_copies);
   3818 		evcnt_detach(&rxr->no_mbuf);
   3819 		evcnt_detach(&rxr->rx_discarded);
   3820 	}
   3821 	evcnt_detach(&stats->ipcs);
   3822 	evcnt_detach(&stats->l4cs);
   3823 	evcnt_detach(&stats->ipcs_bad);
   3824 	evcnt_detach(&stats->l4cs_bad);
   3825 	evcnt_detach(&stats->intzero);
   3826 	evcnt_detach(&stats->legint);
   3827 	evcnt_detach(&stats->crcerrs);
   3828 	evcnt_detach(&stats->illerrc);
   3829 	evcnt_detach(&stats->errbc);
   3830 	evcnt_detach(&stats->mspdc);
   3831 	if (hw->mac.type >= ixgbe_mac_X550)
   3832 		evcnt_detach(&stats->mbsdc);
   3833 	evcnt_detach(&stats->mpctotal);
   3834 	evcnt_detach(&stats->mlfc);
   3835 	evcnt_detach(&stats->mrfc);
   3836 	if (hw->mac.type == ixgbe_mac_X550EM_a)
   3837 		evcnt_detach(&stats->link_dn_cnt);
   3838 	evcnt_detach(&stats->rlec);
   3839 	evcnt_detach(&stats->lxontxc);
   3840 	evcnt_detach(&stats->lxonrxc);
   3841 	evcnt_detach(&stats->lxofftxc);
   3842 	evcnt_detach(&stats->lxoffrxc);
   3843 
   3844 	/* Packet Reception Stats */
   3845 	evcnt_detach(&stats->tor);
   3846 	evcnt_detach(&stats->gorc);
   3847 	evcnt_detach(&stats->tpr);
   3848 	evcnt_detach(&stats->gprc);
   3849 	evcnt_detach(&stats->mprc);
   3850 	evcnt_detach(&stats->bprc);
   3851 	evcnt_detach(&stats->prc64);
   3852 	evcnt_detach(&stats->prc127);
   3853 	evcnt_detach(&stats->prc255);
   3854 	evcnt_detach(&stats->prc511);
   3855 	evcnt_detach(&stats->prc1023);
   3856 	evcnt_detach(&stats->prc1522);
   3857 	evcnt_detach(&stats->ruc);
   3858 	evcnt_detach(&stats->rfc);
   3859 	evcnt_detach(&stats->roc);
   3860 	evcnt_detach(&stats->rjc);
   3861 	evcnt_detach(&stats->mngprc);
   3862 	evcnt_detach(&stats->mngpdc);
   3863 	evcnt_detach(&stats->xec);
   3864 
   3865 	/* Packet Transmission Stats */
   3866 	evcnt_detach(&stats->gotc);
   3867 	evcnt_detach(&stats->tpt);
   3868 	evcnt_detach(&stats->gptc);
   3869 	evcnt_detach(&stats->bptc);
   3870 	evcnt_detach(&stats->mptc);
   3871 	evcnt_detach(&stats->mngptc);
   3872 	evcnt_detach(&stats->ptc64);
   3873 	evcnt_detach(&stats->ptc127);
   3874 	evcnt_detach(&stats->ptc255);
   3875 	evcnt_detach(&stats->ptc511);
   3876 	evcnt_detach(&stats->ptc1023);
   3877 	evcnt_detach(&stats->ptc1522);
   3878 
   3879 	ixgbe_free_queues(sc);
   3880 	free(sc->mta, M_DEVBUF);
   3881 
   3882 	mutex_destroy(&sc->admin_mtx); /* XXX appropriate order? */
   3883 	IXGBE_CORE_LOCK_DESTROY(sc);
   3884 
   3885 	return (0);
   3886 } /* ixgbe_detach */
   3887 
   3888 /************************************************************************
   3889  * ixgbe_setup_low_power_mode - LPLU/WoL preparation
   3890  *
   3891  *   Prepare the adapter/port for LPLU and/or WoL
   3892  ************************************************************************/
   3893 static int
   3894 ixgbe_setup_low_power_mode(struct ixgbe_softc *sc)
   3895 {
   3896 	struct ixgbe_hw *hw = &sc->hw;
   3897 	device_t	dev = sc->dev;
   3898 	struct ifnet	*ifp = sc->ifp;
   3899 	s32		error = 0;
   3900 
   3901 	/* Limit power management flow to X550EM baseT */
   3902 	if (hw->device_id == IXGBE_DEV_ID_X550EM_X_10G_T &&
   3903 	    hw->phy.ops.enter_lplu) {
   3904 		/* X550EM baseT adapters need a special LPLU flow */
   3905 		hw->phy.reset_disable = true;
   3906 		ixgbe_ifstop(ifp, 1);
   3907 		error = hw->phy.ops.enter_lplu(hw);
   3908 		if (error)
   3909 			device_printf(dev,
   3910 			    "Error entering LPLU: %d\n", error);
   3911 		hw->phy.reset_disable = false;
   3912 	} else {
   3913 		/* Just stop for other adapters */
   3914 		ixgbe_ifstop(ifp, 1);
   3915 	}
   3916 
   3917 	IXGBE_CORE_LOCK(sc);
   3918 
   3919 	if (!hw->wol_enabled) {
   3920 		ixgbe_set_phy_power(hw, FALSE);
   3921 		IXGBE_WRITE_REG(hw, IXGBE_WUFC, 0);
   3922 		IXGBE_WRITE_REG(hw, IXGBE_WUC, 0);
   3923 	} else {
   3924 		/* Turn off support for APM wakeup. (Using ACPI instead) */
   3925 		IXGBE_WRITE_REG(hw, IXGBE_GRC_BY_MAC(hw),
   3926 		    IXGBE_READ_REG(hw, IXGBE_GRC_BY_MAC(hw)) & ~(u32)2);
   3927 
   3928 		/*
   3929 		 * Clear Wake Up Status register to prevent any previous wakeup
   3930 		 * events from waking us up immediately after we suspend.
   3931 		 */
   3932 		IXGBE_WRITE_REG(hw, IXGBE_WUS, 0xffffffff);
   3933 
   3934 		/*
   3935 		 * Program the Wakeup Filter Control register with user filter
   3936 		 * settings
   3937 		 */
   3938 		IXGBE_WRITE_REG(hw, IXGBE_WUFC, sc->wufc);
   3939 
   3940 		/* Enable wakeups and power management in Wakeup Control */
   3941 		IXGBE_WRITE_REG(hw, IXGBE_WUC,
   3942 		    IXGBE_WUC_WKEN | IXGBE_WUC_PME_EN);
   3943 	}
   3944 
   3945 	IXGBE_CORE_UNLOCK(sc);
   3946 
   3947 	return error;
   3948 } /* ixgbe_setup_low_power_mode */
   3949 
   3950 /************************************************************************
   3951  * ixgbe_shutdown - Shutdown entry point
   3952  ************************************************************************/
   3953 #if 0 /* XXX NetBSD ought to register something like this through pmf(9) */
   3954 static int
   3955 ixgbe_shutdown(device_t dev)
   3956 {
   3957 	struct ixgbe_softc *sc = device_private(dev);
   3958 	int error = 0;
   3959 
   3960 	INIT_DEBUGOUT("ixgbe_shutdown: begin");
   3961 
   3962 	error = ixgbe_setup_low_power_mode(sc);
   3963 
   3964 	return (error);
   3965 } /* ixgbe_shutdown */
   3966 #endif
   3967 
   3968 /************************************************************************
   3969  * ixgbe_suspend
   3970  *
   3971  *   From D0 to D3
   3972  ************************************************************************/
   3973 static bool
   3974 ixgbe_suspend(device_t dev, const pmf_qual_t *qual)
   3975 {
   3976 	struct ixgbe_softc *sc = device_private(dev);
   3977 	int	       error = 0;
   3978 
   3979 	INIT_DEBUGOUT("ixgbe_suspend: begin");
   3980 
   3981 	error = ixgbe_setup_low_power_mode(sc);
   3982 
   3983 	return (error);
   3984 } /* ixgbe_suspend */
   3985 
   3986 /************************************************************************
   3987  * ixgbe_resume
   3988  *
   3989  *   From D3 to D0
   3990  ************************************************************************/
   3991 static bool
   3992 ixgbe_resume(device_t dev, const pmf_qual_t *qual)
   3993 {
   3994 	struct ixgbe_softc *sc = device_private(dev);
   3995 	struct ifnet	*ifp = sc->ifp;
   3996 	struct ixgbe_hw *hw = &sc->hw;
   3997 	u32		wus;
   3998 
   3999 	INIT_DEBUGOUT("ixgbe_resume: begin");
   4000 
   4001 	IXGBE_CORE_LOCK(sc);
   4002 
   4003 	/* Read & clear WUS register */
   4004 	wus = IXGBE_READ_REG(hw, IXGBE_WUS);
   4005 	if (wus)
   4006 		device_printf(dev, "Woken up by (WUS): %#010x\n",
   4007 		    IXGBE_READ_REG(hw, IXGBE_WUS));
   4008 	IXGBE_WRITE_REG(hw, IXGBE_WUS, 0xffffffff);
   4009 	/* And clear WUFC until next low-power transition */
   4010 	IXGBE_WRITE_REG(hw, IXGBE_WUFC, 0);
   4011 
   4012 	/*
   4013 	 * Required after D3->D0 transition;
   4014 	 * will re-advertise all previous advertised speeds
   4015 	 */
   4016 	if (ifp->if_flags & IFF_UP)
   4017 		ixgbe_init_locked(sc);
   4018 
   4019 	IXGBE_CORE_UNLOCK(sc);
   4020 
   4021 	return true;
   4022 } /* ixgbe_resume */
   4023 
   4024 /*
   4025  * Set the various hardware offload abilities.
   4026  *
   4027  * This takes the ifnet's if_capenable flags (e.g. set by the user using
   4028  * ifconfig) and indicates to the OS via the ifnet's if_hwassist field what
   4029  * mbuf offload flags the driver will understand.
   4030  */
   4031 static void
   4032 ixgbe_set_if_hwassist(struct ixgbe_softc *sc)
   4033 {
   4034 	/* XXX */
   4035 }
   4036 
   4037 /************************************************************************
   4038  * ixgbe_init_locked - Init entry point
   4039  *
   4040  *   Used in two ways: It is used by the stack as an init
   4041  *   entry point in network interface structure. It is also
   4042  *   used by the driver as a hw/sw initialization routine to
   4043  *   get to a consistent state.
   4044  *
   4045  *   return 0 on success, positive on failure
   4046  ************************************************************************/
   4047 static void
   4048 ixgbe_init_locked(struct ixgbe_softc *sc)
   4049 {
   4050 	struct ifnet   *ifp = sc->ifp;
   4051 	device_t	dev = sc->dev;
   4052 	struct ixgbe_hw *hw = &sc->hw;
   4053 	struct ix_queue *que;
   4054 	struct tx_ring	*txr;
   4055 	struct rx_ring	*rxr;
   4056 	u32		txdctl, mhadd;
   4057 	u32		rxdctl, rxctrl;
   4058 	u32		ctrl_ext;
   4059 	bool		unsupported_sfp = false;
   4060 	int		i, j, error;
   4061 
   4062 	/* XXX check IFF_UP and IFF_RUNNING, power-saving state! */
   4063 
   4064 	KASSERT(mutex_owned(&sc->core_mtx));
   4065 	INIT_DEBUGOUT("ixgbe_init_locked: begin");
   4066 
   4067 	hw->need_unsupported_sfp_recovery = false;
   4068 	hw->adapter_stopped = FALSE;
   4069 	ixgbe_stop_adapter(hw);
   4070 	callout_stop(&sc->timer);
   4071 	if (sc->feat_en & IXGBE_FEATURE_RECOVERY_MODE)
   4072 		callout_stop(&sc->recovery_mode_timer);
   4073 	for (i = 0, que = sc->queues; i < sc->num_queues; i++, que++)
   4074 		que->disabled_count = 0;
   4075 
   4076 	/* XXX I moved this here from the SIOCSIFMTU case in ixgbe_ioctl(). */
   4077 	sc->max_frame_size =
   4078 		ifp->if_mtu + ETHER_HDR_LEN + ETHER_CRC_LEN;
   4079 
   4080 	/* Queue indices may change with IOV mode */
   4081 	ixgbe_align_all_queue_indices(sc);
   4082 
   4083 	/* reprogram the RAR[0] in case user changed it. */
   4084 	ixgbe_set_rar(hw, 0, hw->mac.addr, sc->pool, IXGBE_RAH_AV);
   4085 
   4086 	/* Get the latest mac address, User can use a LAA */
   4087 	memcpy(hw->mac.addr, CLLADDR(ifp->if_sadl),
   4088 	    IXGBE_ETH_LENGTH_OF_ADDRESS);
   4089 	ixgbe_set_rar(hw, 0, hw->mac.addr, sc->pool, 1);
   4090 	hw->addr_ctrl.rar_used_count = 1;
   4091 
   4092 	/* Set hardware offload abilities from ifnet flags */
   4093 	ixgbe_set_if_hwassist(sc);
   4094 
   4095 	/* Prepare transmit descriptors and buffers */
   4096 	if (ixgbe_setup_transmit_structures(sc)) {
   4097 		device_printf(dev, "Could not setup transmit structures\n");
   4098 		ixgbe_stop_locked(sc);
   4099 		return;
   4100 	}
   4101 
   4102 	ixgbe_init_hw(hw);
   4103 
   4104 	ixgbe_initialize_iov(sc);
   4105 
   4106 	ixgbe_initialize_transmit_units(sc);
   4107 
   4108 	/* Setup Multicast table */
   4109 	ixgbe_set_rxfilter(sc);
   4110 
   4111 	/* Use fixed buffer size, even for jumbo frames */
   4112 	sc->rx_mbuf_sz = MCLBYTES;
   4113 
   4114 	/* Prepare receive descriptors and buffers */
   4115 	error = ixgbe_setup_receive_structures(sc);
   4116 	if (error) {
   4117 		device_printf(dev,
   4118 		    "Could not setup receive structures (err = %d)\n", error);
   4119 		ixgbe_stop_locked(sc);
   4120 		return;
   4121 	}
   4122 
   4123 	/* Configure RX settings */
   4124 	ixgbe_initialize_receive_units(sc);
   4125 
   4126 	/* Initialize variable holding task enqueue requests interrupts */
   4127 	sc->task_requests = 0;
   4128 
   4129 	/* Enable SDP & MSI-X interrupts based on adapter */
   4130 	ixgbe_config_gpie(sc);
   4131 
   4132 	/* Set MTU size */
   4133 	if (ifp->if_mtu > ETHERMTU) {
   4134 		/* aka IXGBE_MAXFRS on 82599 and newer */
   4135 		mhadd = IXGBE_READ_REG(hw, IXGBE_MHADD);
   4136 		mhadd &= ~IXGBE_MHADD_MFS_MASK;
   4137 		mhadd |= sc->max_frame_size << IXGBE_MHADD_MFS_SHIFT;
   4138 		IXGBE_WRITE_REG(hw, IXGBE_MHADD, mhadd);
   4139 	}
   4140 
   4141 	/* Now enable all the queues */
   4142 	for (i = 0; i < sc->num_queues; i++) {
   4143 		txr = &sc->tx_rings[i];
   4144 		txdctl = IXGBE_READ_REG(hw, IXGBE_TXDCTL(txr->me));
   4145 		txdctl |= IXGBE_TXDCTL_ENABLE;
   4146 		/* Set WTHRESH to 8, burst writeback */
   4147 		txdctl &= ~IXGBE_TXDCTL_WTHRESH_MASK;
   4148 		txdctl |= IXGBE_TX_WTHRESH << IXGBE_TXDCTL_WTHRESH_SHIFT;
   4149 		/*
   4150 		 * When the internal queue falls below PTHRESH (32),
   4151 		 * start prefetching as long as there are at least
   4152 		 * HTHRESH (1) buffers ready. The values are taken
   4153 		 * from the Intel linux driver 3.8.21.
   4154 		 * Prefetching enables tx line rate even with 1 queue.
   4155 		 */
   4156 		txdctl |= (32 << 0) | (1 << 8);
   4157 		IXGBE_WRITE_REG(hw, IXGBE_TXDCTL(txr->me), txdctl);
   4158 	}
   4159 
   4160 	for (i = 0; i < sc->num_queues; i++) {
   4161 		rxr = &sc->rx_rings[i];
   4162 		rxdctl = IXGBE_READ_REG(hw, IXGBE_RXDCTL(rxr->me));
   4163 		if (hw->mac.type == ixgbe_mac_82598EB) {
   4164 			/*
   4165 			 * PTHRESH = 21
   4166 			 * HTHRESH = 4
   4167 			 * WTHRESH = 8
   4168 			 */
   4169 			rxdctl &= ~0x3FFFFF;
   4170 			rxdctl |= 0x080420;
   4171 		}
   4172 		rxdctl |= IXGBE_RXDCTL_ENABLE;
   4173 		IXGBE_WRITE_REG(hw, IXGBE_RXDCTL(rxr->me), rxdctl);
   4174 		for (j = 0; j < 10; j++) {
   4175 			if (IXGBE_READ_REG(hw, IXGBE_RXDCTL(rxr->me)) &
   4176 			    IXGBE_RXDCTL_ENABLE)
   4177 				break;
   4178 			else
   4179 				msec_delay(1);
   4180 		}
   4181 		IXGBE_WRITE_BARRIER(hw);
   4182 
   4183 		/*
   4184 		 * In netmap mode, we must preserve the buffers made
   4185 		 * available to userspace before the if_init()
   4186 		 * (this is true by default on the TX side, because
   4187 		 * init makes all buffers available to userspace).
   4188 		 *
   4189 		 * netmap_reset() and the device specific routines
   4190 		 * (e.g. ixgbe_setup_receive_rings()) map these
   4191 		 * buffers at the end of the NIC ring, so here we
   4192 		 * must set the RDT (tail) register to make sure
   4193 		 * they are not overwritten.
   4194 		 *
   4195 		 * In this driver the NIC ring starts at RDH = 0,
   4196 		 * RDT points to the last slot available for reception (?),
   4197 		 * so RDT = num_rx_desc - 1 means the whole ring is available.
   4198 		 */
   4199 #ifdef DEV_NETMAP
   4200 		if ((sc->feat_en & IXGBE_FEATURE_NETMAP) &&
   4201 		    (ifp->if_capenable & IFCAP_NETMAP)) {
   4202 			struct netmap_adapter *na = NA(sc->ifp);
   4203 			struct netmap_kring *kring = na->rx_rings[i];
   4204 			int t = na->num_rx_desc - 1 - nm_kr_rxspace(kring);
   4205 
   4206 			IXGBE_WRITE_REG(hw, IXGBE_RDT(rxr->me), t);
   4207 		} else
   4208 #endif /* DEV_NETMAP */
   4209 			IXGBE_WRITE_REG(hw, IXGBE_RDT(rxr->me),
   4210 			    sc->num_rx_desc - 1);
   4211 	}
   4212 
   4213 	/* Enable Receive engine */
   4214 	rxctrl = IXGBE_READ_REG(hw, IXGBE_RXCTRL);
   4215 	if (hw->mac.type == ixgbe_mac_82598EB)
   4216 		rxctrl |= IXGBE_RXCTRL_DMBYPS;
   4217 	rxctrl |= IXGBE_RXCTRL_RXEN;
   4218 	ixgbe_enable_rx_dma(hw, rxctrl);
   4219 
   4220 	callout_reset(&sc->timer, hz, ixgbe_local_timer, sc);
   4221 	atomic_store_relaxed(&sc->timer_pending, 0);
   4222 	if (sc->feat_en & IXGBE_FEATURE_RECOVERY_MODE)
   4223 		callout_reset(&sc->recovery_mode_timer, hz,
   4224 		    ixgbe_recovery_mode_timer, sc);
   4225 
   4226 	/* Set up MSI/MSI-X routing */
   4227 	if (sc->feat_en & IXGBE_FEATURE_MSIX) {
   4228 		ixgbe_configure_ivars(sc);
   4229 		/* Set up auto-mask */
   4230 		if (hw->mac.type == ixgbe_mac_82598EB)
   4231 			IXGBE_WRITE_REG(hw, IXGBE_EIAM, IXGBE_EICS_RTX_QUEUE);
   4232 		else {
   4233 			IXGBE_WRITE_REG(hw, IXGBE_EIAM_EX(0), 0xFFFFFFFF);
   4234 			IXGBE_WRITE_REG(hw, IXGBE_EIAM_EX(1), 0xFFFFFFFF);
   4235 		}
   4236 	} else {  /* Simple settings for Legacy/MSI */
   4237 		ixgbe_set_ivar(sc, 0, 0, 0);
   4238 		ixgbe_set_ivar(sc, 0, 0, 1);
   4239 		IXGBE_WRITE_REG(hw, IXGBE_EIAM, IXGBE_EICS_RTX_QUEUE);
   4240 	}
   4241 
   4242 	ixgbe_init_fdir(sc);
   4243 
   4244 	/*
   4245 	 * Check on any SFP devices that
   4246 	 * need to be kick-started
   4247 	 */
   4248 	if (hw->phy.type == ixgbe_phy_none) {
   4249 		error = hw->phy.ops.identify(hw);
   4250 		if (error == IXGBE_ERR_SFP_NOT_SUPPORTED)
   4251 			unsupported_sfp = true;
   4252 	} else if (hw->phy.type == ixgbe_phy_sfp_unsupported)
   4253 		unsupported_sfp = true;
   4254 
   4255 	if (unsupported_sfp)
   4256 		device_printf(dev,
   4257 		    "Unsupported SFP+ module type was detected.\n");
   4258 
   4259 	/* Set moderation on the Link interrupt */
   4260 	ixgbe_eitr_write(sc, sc->vector, IXGBE_LINK_ITR);
   4261 
   4262 	/* Enable EEE power saving */
   4263 	if (sc->feat_cap & IXGBE_FEATURE_EEE)
   4264 		hw->mac.ops.setup_eee(hw,
   4265 		    sc->feat_en & IXGBE_FEATURE_EEE);
   4266 
   4267 	/* Enable power to the phy. */
   4268 	if (!unsupported_sfp) {
   4269 		ixgbe_set_phy_power(hw, TRUE);
   4270 
   4271 		/* Config/Enable Link */
   4272 		ixgbe_config_link(sc);
   4273 	}
   4274 
   4275 	/* Hardware Packet Buffer & Flow Control setup */
   4276 	ixgbe_config_delay_values(sc);
   4277 
   4278 	/* Initialize the FC settings */
   4279 	ixgbe_start_hw(hw);
   4280 
   4281 	/* Set up VLAN support and filter */
   4282 	ixgbe_setup_vlan_hw_support(sc);
   4283 
   4284 	/* Setup DMA Coalescing */
   4285 	ixgbe_config_dmac(sc);
   4286 
   4287 	/* OK to schedule workqueues. */
   4288 	sc->schedule_wqs_ok = true;
   4289 
   4290 	/* Enable the use of the MBX by the VF's */
   4291 	if (sc->feat_en & IXGBE_FEATURE_SRIOV) {
   4292 		ctrl_ext = IXGBE_READ_REG(hw, IXGBE_CTRL_EXT);
   4293 		ctrl_ext |= IXGBE_CTRL_EXT_PFRSTD;
   4294 		IXGBE_WRITE_REG(hw, IXGBE_CTRL_EXT, ctrl_ext);
   4295 	}
   4296 
   4297 	/* Update saved flags. See ixgbe_ifflags_cb() */
   4298 	sc->if_flags = ifp->if_flags;
   4299 	sc->ec_capenable = sc->osdep.ec.ec_capenable;
   4300 
   4301 	/* Inform the stack we're ready */
   4302 	ifp->if_flags |= IFF_RUNNING;
   4303 
   4304 	/* And now turn on interrupts */
   4305 	ixgbe_enable_intr(sc);
   4306 
   4307 	return;
   4308 } /* ixgbe_init_locked */
   4309 
   4310 /************************************************************************
   4311  * ixgbe_init
   4312  ************************************************************************/
   4313 static int
   4314 ixgbe_init(struct ifnet *ifp)
   4315 {
   4316 	struct ixgbe_softc *sc = ifp->if_softc;
   4317 
   4318 	IXGBE_CORE_LOCK(sc);
   4319 	ixgbe_init_locked(sc);
   4320 	IXGBE_CORE_UNLOCK(sc);
   4321 
   4322 	return 0;	/* XXX ixgbe_init_locked cannot fail?  really? */
   4323 } /* ixgbe_init */
   4324 
   4325 /************************************************************************
   4326  * ixgbe_set_ivar
   4327  *
   4328  *   Setup the correct IVAR register for a particular MSI-X interrupt
   4329  *     (yes this is all very magic and confusing :)
   4330  *    - entry is the register array entry
   4331  *    - vector is the MSI-X vector for this queue
   4332  *    - type is RX/TX/MISC
   4333  ************************************************************************/
   4334 static void
   4335 ixgbe_set_ivar(struct ixgbe_softc *sc, u8 entry, u8 vector, s8 type)
   4336 {
   4337 	struct ixgbe_hw *hw = &sc->hw;
   4338 	u32 ivar, index;
   4339 
   4340 	vector |= IXGBE_IVAR_ALLOC_VAL;
   4341 
   4342 	switch (hw->mac.type) {
   4343 	case ixgbe_mac_82598EB:
   4344 		if (type == -1)
   4345 			entry = IXGBE_IVAR_OTHER_CAUSES_INDEX;
   4346 		else
   4347 			entry += (type * 64);
   4348 		index = (entry >> 2) & 0x1F;
   4349 		ivar = IXGBE_READ_REG(hw, IXGBE_IVAR(index));
   4350 		ivar &= ~(0xffUL << (8 * (entry & 0x3)));
   4351 		ivar |= ((u32)vector << (8 * (entry & 0x3)));
   4352 		IXGBE_WRITE_REG(&sc->hw, IXGBE_IVAR(index), ivar);
   4353 		break;
   4354 	case ixgbe_mac_82599EB:
   4355 	case ixgbe_mac_X540:
   4356 	case ixgbe_mac_X550:
   4357 	case ixgbe_mac_X550EM_x:
   4358 	case ixgbe_mac_X550EM_a:
   4359 		if (type == -1) { /* MISC IVAR */
   4360 			index = (entry & 1) * 8;
   4361 			ivar = IXGBE_READ_REG(hw, IXGBE_IVAR_MISC);
   4362 			ivar &= ~(0xffUL << index);
   4363 			ivar |= ((u32)vector << index);
   4364 			IXGBE_WRITE_REG(hw, IXGBE_IVAR_MISC, ivar);
   4365 		} else {	/* RX/TX IVARS */
   4366 			index = (16 * (entry & 1)) + (8 * type);
   4367 			ivar = IXGBE_READ_REG(hw, IXGBE_IVAR(entry >> 1));
   4368 			ivar &= ~(0xffUL << index);
   4369 			ivar |= ((u32)vector << index);
   4370 			IXGBE_WRITE_REG(hw, IXGBE_IVAR(entry >> 1), ivar);
   4371 		}
   4372 		break;
   4373 	default:
   4374 		break;
   4375 	}
   4376 } /* ixgbe_set_ivar */
   4377 
   4378 /************************************************************************
   4379  * ixgbe_configure_ivars
   4380  ************************************************************************/
   4381 static void
   4382 ixgbe_configure_ivars(struct ixgbe_softc *sc)
   4383 {
   4384 	struct ix_queue *que = sc->queues;
   4385 	u32		newitr;
   4386 
   4387 	if (sc->max_interrupt_rate > 0)
   4388 		newitr = (4000000 / sc->max_interrupt_rate) & 0x0FF8;
   4389 	else {
   4390 		/*
   4391 		 * Disable DMA coalescing if interrupt moderation is
   4392 		 * disabled.
   4393 		 */
   4394 		sc->dmac = 0;
   4395 		newitr = 0;
   4396 	}
   4397 
   4398 	for (int i = 0; i < sc->num_queues; i++, que++) {
   4399 		struct rx_ring *rxr = &sc->rx_rings[i];
   4400 		struct tx_ring *txr = &sc->tx_rings[i];
   4401 		/* First the RX queue entry */
   4402 		ixgbe_set_ivar(sc, rxr->me, que->msix, 0);
   4403 		/* ... and the TX */
   4404 		ixgbe_set_ivar(sc, txr->me, que->msix, 1);
   4405 		/* Set an Initial EITR value */
   4406 		ixgbe_eitr_write(sc, que->msix, newitr);
   4407 		/*
   4408 		 * To eliminate influence of the previous state.
   4409 		 * At this point, Tx/Rx interrupt handler
   4410 		 * (ixgbe_msix_que()) cannot be called, so  both
   4411 		 * IXGBE_TX_LOCK and IXGBE_RX_LOCK are not required.
   4412 		 */
   4413 		que->eitr_setting = 0;
   4414 	}
   4415 
   4416 	/* For the Link interrupt */
   4417 	ixgbe_set_ivar(sc, 1, sc->vector, -1);
   4418 } /* ixgbe_configure_ivars */
   4419 
   4420 /************************************************************************
   4421  * ixgbe_config_gpie
   4422  ************************************************************************/
   4423 static void
   4424 ixgbe_config_gpie(struct ixgbe_softc *sc)
   4425 {
   4426 	struct ixgbe_hw *hw = &sc->hw;
   4427 	u32		gpie;
   4428 
   4429 	gpie = IXGBE_READ_REG(hw, IXGBE_GPIE);
   4430 
   4431 	if (sc->feat_en & IXGBE_FEATURE_MSIX) {
   4432 		/* Enable Enhanced MSI-X mode */
   4433 		gpie |= IXGBE_GPIE_MSIX_MODE
   4434 		     |	IXGBE_GPIE_EIAME
   4435 		     |	IXGBE_GPIE_PBA_SUPPORT
   4436 		     |	IXGBE_GPIE_OCD;
   4437 	}
   4438 
   4439 	/* Fan Failure Interrupt */
   4440 	if (sc->feat_en & IXGBE_FEATURE_FAN_FAIL)
   4441 		gpie |= IXGBE_SDP1_GPIEN;
   4442 
   4443 	/* Thermal Sensor Interrupt */
   4444 	if (sc->feat_en & IXGBE_FEATURE_TEMP_SENSOR)
   4445 		gpie |= IXGBE_SDP0_GPIEN_X540;
   4446 
   4447 	/* Link detection */
   4448 	switch (hw->mac.type) {
   4449 	case ixgbe_mac_82599EB:
   4450 		gpie |= IXGBE_SDP1_GPIEN | IXGBE_SDP2_GPIEN;
   4451 		break;
   4452 	case ixgbe_mac_X550EM_x:
   4453 	case ixgbe_mac_X550EM_a:
   4454 		gpie |= IXGBE_SDP0_GPIEN_X540;
   4455 		break;
   4456 	default:
   4457 		break;
   4458 	}
   4459 
   4460 	IXGBE_WRITE_REG(hw, IXGBE_GPIE, gpie);
   4461 
   4462 } /* ixgbe_config_gpie */
   4463 
   4464 /************************************************************************
   4465  * ixgbe_config_delay_values
   4466  *
   4467  *   Requires sc->max_frame_size to be set.
   4468  ************************************************************************/
   4469 static void
   4470 ixgbe_config_delay_values(struct ixgbe_softc *sc)
   4471 {
   4472 	struct ixgbe_hw *hw = &sc->hw;
   4473 	u32		rxpb, frame, size, tmp;
   4474 
   4475 	frame = sc->max_frame_size;
   4476 
   4477 	/* Calculate High Water */
   4478 	switch (hw->mac.type) {
   4479 	case ixgbe_mac_X540:
   4480 	case ixgbe_mac_X550:
   4481 	case ixgbe_mac_X550EM_x:
   4482 	case ixgbe_mac_X550EM_a:
   4483 		tmp = IXGBE_DV_X540(frame, frame);
   4484 		break;
   4485 	default:
   4486 		tmp = IXGBE_DV(frame, frame);
   4487 		break;
   4488 	}
   4489 	size = IXGBE_BT2KB(tmp);
   4490 	rxpb = IXGBE_READ_REG(hw, IXGBE_RXPBSIZE(0)) >> 10;
   4491 	hw->fc.high_water[0] = rxpb - size;
   4492 
   4493 	/* Now calculate Low Water */
   4494 	switch (hw->mac.type) {
   4495 	case ixgbe_mac_X540:
   4496 	case ixgbe_mac_X550:
   4497 	case ixgbe_mac_X550EM_x:
   4498 	case ixgbe_mac_X550EM_a:
   4499 		tmp = IXGBE_LOW_DV_X540(frame);
   4500 		break;
   4501 	default:
   4502 		tmp = IXGBE_LOW_DV(frame);
   4503 		break;
   4504 	}
   4505 	hw->fc.low_water[0] = IXGBE_BT2KB(tmp);
   4506 
   4507 	hw->fc.pause_time = IXGBE_FC_PAUSE;
   4508 	hw->fc.send_xon = TRUE;
   4509 } /* ixgbe_config_delay_values */
   4510 
   4511 /************************************************************************
   4512  * ixgbe_set_rxfilter - Multicast Update
   4513  *
   4514  *   Called whenever multicast address list is updated.
   4515  ************************************************************************/
   4516 static void
   4517 ixgbe_set_rxfilter(struct ixgbe_softc *sc)
   4518 {
   4519 	struct ixgbe_mc_addr	*mta;
   4520 	struct ifnet		*ifp = sc->ifp;
   4521 	u8			*update_ptr;
   4522 	int			mcnt = 0;
   4523 	u32			fctrl;
   4524 	struct ethercom		*ec = &sc->osdep.ec;
   4525 	struct ether_multi	*enm;
   4526 	struct ether_multistep	step;
   4527 
   4528 	KASSERT(mutex_owned(&sc->core_mtx));
   4529 	IOCTL_DEBUGOUT("ixgbe_set_rxfilter: begin");
   4530 
   4531 	mta = sc->mta;
   4532 	bzero(mta, sizeof(*mta) * MAX_NUM_MULTICAST_ADDRESSES);
   4533 
   4534 	ETHER_LOCK(ec);
   4535 	ec->ec_flags &= ~ETHER_F_ALLMULTI;
   4536 	ETHER_FIRST_MULTI(step, ec, enm);
   4537 	while (enm != NULL) {
   4538 		if ((mcnt == MAX_NUM_MULTICAST_ADDRESSES) ||
   4539 		    (memcmp(enm->enm_addrlo, enm->enm_addrhi,
   4540 			ETHER_ADDR_LEN) != 0)) {
   4541 			ec->ec_flags |= ETHER_F_ALLMULTI;
   4542 			break;
   4543 		}
   4544 		bcopy(enm->enm_addrlo,
   4545 		    mta[mcnt].addr, IXGBE_ETH_LENGTH_OF_ADDRESS);
   4546 		mta[mcnt].vmdq = sc->pool;
   4547 		mcnt++;
   4548 		ETHER_NEXT_MULTI(step, enm);
   4549 	}
   4550 
   4551 	fctrl = IXGBE_READ_REG(&sc->hw, IXGBE_FCTRL);
   4552 	if (ifp->if_flags & IFF_PROMISC)
   4553 		fctrl |= (IXGBE_FCTRL_UPE | IXGBE_FCTRL_MPE);
   4554 	else if (ec->ec_flags & ETHER_F_ALLMULTI) {
   4555 		fctrl |= IXGBE_FCTRL_MPE;
   4556 		fctrl &= ~IXGBE_FCTRL_UPE;
   4557 	} else
   4558 		fctrl &= ~(IXGBE_FCTRL_UPE | IXGBE_FCTRL_MPE);
   4559 
   4560 	/* Update multicast filter entries only when it's not ALLMULTI */
   4561 	if ((ec->ec_flags & ETHER_F_ALLMULTI) == 0) {
   4562 		ETHER_UNLOCK(ec);
   4563 		update_ptr = (u8 *)mta;
   4564 		ixgbe_update_mc_addr_list(&sc->hw, update_ptr, mcnt,
   4565 		    ixgbe_mc_array_itr, TRUE);
   4566 	} else
   4567 		ETHER_UNLOCK(ec);
   4568 
   4569 	IXGBE_WRITE_REG(&sc->hw, IXGBE_FCTRL, fctrl);
   4570 } /* ixgbe_set_rxfilter */
   4571 
   4572 /************************************************************************
   4573  * ixgbe_mc_array_itr
   4574  *
   4575  *   An iterator function needed by the multicast shared code.
   4576  *   It feeds the shared code routine the addresses in the
   4577  *   array of ixgbe_set_rxfilter() one by one.
   4578  ************************************************************************/
   4579 static u8 *
   4580 ixgbe_mc_array_itr(struct ixgbe_hw *hw, u8 **update_ptr, u32 *vmdq)
   4581 {
   4582 	struct ixgbe_mc_addr *mta;
   4583 
   4584 	mta = (struct ixgbe_mc_addr *)*update_ptr;
   4585 	*vmdq = mta->vmdq;
   4586 
   4587 	*update_ptr = (u8*)(mta + 1);
   4588 
   4589 	return (mta->addr);
   4590 } /* ixgbe_mc_array_itr */
   4591 
   4592 /************************************************************************
   4593  * ixgbe_local_timer - Timer routine
   4594  *
   4595  *   Checks for link status, updates statistics,
   4596  *   and runs the watchdog check.
   4597  ************************************************************************/
   4598 static void
   4599 ixgbe_local_timer(void *arg)
   4600 {
   4601 	struct ixgbe_softc *sc = arg;
   4602 
   4603 	if (sc->schedule_wqs_ok) {
   4604 		if (atomic_cas_uint(&sc->timer_pending, 0, 1) == 0)
   4605 			workqueue_enqueue(sc->timer_wq,
   4606 			    &sc->timer_wc, NULL);
   4607 	}
   4608 }
   4609 
   4610 static void
   4611 ixgbe_handle_timer(struct work *wk, void *context)
   4612 {
   4613 	struct ixgbe_softc *sc = context;
   4614 	struct ixgbe_hw *hw = &sc->hw;
   4615 	device_t	dev = sc->dev;
   4616 	struct ix_queue	*que = sc->queues;
   4617 	u64		queues = 0;
   4618 	u64		v0, v1, v2, v3, v4, v5, v6, v7;
   4619 	int		hung = 0;
   4620 	int		i;
   4621 
   4622 	IXGBE_CORE_LOCK(sc);
   4623 
   4624 	/* Check for pluggable optics */
   4625 	if (ixgbe_is_sfp(hw)) {
   4626 		bool sched_mod_task = false;
   4627 
   4628 		if (hw->mac.type == ixgbe_mac_82598EB) {
   4629 			/*
   4630 			 * On 82598EB, SFP+'s MOD_ABS pin is not connected to
   4631 			 * any GPIO(SDP). So just schedule TASK_MOD.
   4632 			 */
   4633 			sched_mod_task = true;
   4634 		} else {
   4635 			bool was_full, is_full;
   4636 
   4637 			was_full =
   4638 			    hw->phy.sfp_type != ixgbe_sfp_type_not_present;
   4639 			is_full = ixgbe_sfp_cage_full(hw);
   4640 
   4641 			/* Do probe if cage state changed */
   4642 			if (was_full ^ is_full)
   4643 				sched_mod_task = true;
   4644 		}
   4645 		if (sched_mod_task) {
   4646 			mutex_enter(&sc->admin_mtx);
   4647 			sc->task_requests |= IXGBE_REQUEST_TASK_MOD_WOI;
   4648 			ixgbe_schedule_admin_tasklet(sc);
   4649 			mutex_exit(&sc->admin_mtx);
   4650 		}
   4651 	}
   4652 
   4653 	ixgbe_update_link_status(sc);
   4654 	ixgbe_update_stats_counters(sc);
   4655 
   4656 	/* Update some event counters */
   4657 	v0 = v1 = v2 = v3 = v4 = v5 = v6 = v7 = 0;
   4658 	que = sc->queues;
   4659 	for (i = 0; i < sc->num_queues; i++, que++) {
   4660 		struct tx_ring	*txr = que->txr;
   4661 
   4662 		v0 += txr->q_efbig_tx_dma_setup;
   4663 		v1 += txr->q_mbuf_defrag_failed;
   4664 		v2 += txr->q_efbig2_tx_dma_setup;
   4665 		v3 += txr->q_einval_tx_dma_setup;
   4666 		v4 += txr->q_other_tx_dma_setup;
   4667 		v5 += txr->q_eagain_tx_dma_setup;
   4668 		v6 += txr->q_enomem_tx_dma_setup;
   4669 		v7 += txr->q_tso_err;
   4670 	}
   4671 	IXGBE_EVC_STORE(&sc->efbig_tx_dma_setup, v0);
   4672 	IXGBE_EVC_STORE(&sc->mbuf_defrag_failed, v1);
   4673 	IXGBE_EVC_STORE(&sc->efbig2_tx_dma_setup, v2);
   4674 	IXGBE_EVC_STORE(&sc->einval_tx_dma_setup, v3);
   4675 	IXGBE_EVC_STORE(&sc->other_tx_dma_setup, v4);
   4676 	IXGBE_EVC_STORE(&sc->eagain_tx_dma_setup, v5);
   4677 	IXGBE_EVC_STORE(&sc->enomem_tx_dma_setup, v6);
   4678 	IXGBE_EVC_STORE(&sc->tso_err, v7);
   4679 
   4680 	/*
   4681 	 * Check the TX queues status
   4682 	 *	- mark hung queues so we don't schedule on them
   4683 	 *	- watchdog only if all queues show hung
   4684 	 */
   4685 	que = sc->queues;
   4686 	for (i = 0; i < sc->num_queues; i++, que++) {
   4687 		/* Keep track of queues with work for soft irq */
   4688 		if (que->txr->busy)
   4689 			queues |= 1ULL << que->me;
   4690 		/*
   4691 		 * Each time txeof runs without cleaning, but there
   4692 		 * are uncleaned descriptors it increments busy. If
   4693 		 * we get to the MAX we declare it hung.
   4694 		 */
   4695 		if (que->busy == IXGBE_QUEUE_HUNG) {
   4696 			++hung;
   4697 			/* Mark the queue as inactive */
   4698 			sc->active_queues &= ~(1ULL << que->me);
   4699 			continue;
   4700 		} else {
   4701 			/* Check if we've come back from hung */
   4702 			if ((sc->active_queues & (1ULL << que->me)) == 0)
   4703 				sc->active_queues |= 1ULL << que->me;
   4704 		}
   4705 		if (que->busy >= IXGBE_MAX_TX_BUSY) {
   4706 			device_printf(dev,
   4707 			    "Warning queue %d appears to be hung!\n", i);
   4708 			que->txr->busy = IXGBE_QUEUE_HUNG;
   4709 			++hung;
   4710 		}
   4711 	}
   4712 
   4713 	/* Only truly watchdog if all queues show hung */
   4714 	if (hung == sc->num_queues)
   4715 		goto watchdog;
   4716 #if 0 /* XXX Avoid unexpectedly disabling interrupt forever (PR#53294) */
   4717 	else if (queues != 0) { /* Force an IRQ on queues with work */
   4718 		que = sc->queues;
   4719 		for (i = 0; i < sc->num_queues; i++, que++) {
   4720 			mutex_enter(&que->dc_mtx);
   4721 			if (que->disabled_count == 0)
   4722 				ixgbe_rearm_queues(sc,
   4723 				    queues & ((u64)1 << i));
   4724 			mutex_exit(&que->dc_mtx);
   4725 		}
   4726 	}
   4727 #endif
   4728 
   4729 	atomic_store_relaxed(&sc->timer_pending, 0);
   4730 	IXGBE_CORE_UNLOCK(sc);
   4731 	callout_reset(&sc->timer, hz, ixgbe_local_timer, sc);
   4732 	return;
   4733 
   4734 watchdog:
   4735 	device_printf(sc->dev, "Watchdog timeout -- resetting\n");
   4736 	sc->ifp->if_flags &= ~IFF_RUNNING;
   4737 	IXGBE_EVC_ADD(&sc->watchdog_events, 1);
   4738 	ixgbe_init_locked(sc);
   4739 	IXGBE_CORE_UNLOCK(sc);
   4740 } /* ixgbe_handle_timer */
   4741 
   4742 /************************************************************************
   4743  * ixgbe_recovery_mode_timer - Recovery mode timer routine
   4744  ************************************************************************/
   4745 static void
   4746 ixgbe_recovery_mode_timer(void *arg)
   4747 {
   4748 	struct ixgbe_softc *sc = arg;
   4749 
   4750 	if (__predict_true(sc->osdep.detaching == false)) {
   4751 		if (atomic_cas_uint(&sc->recovery_mode_timer_pending,
   4752 			0, 1) == 0) {
   4753 			workqueue_enqueue(sc->recovery_mode_timer_wq,
   4754 			    &sc->recovery_mode_timer_wc, NULL);
   4755 		}
   4756 	}
   4757 }
   4758 
   4759 static void
   4760 ixgbe_handle_recovery_mode_timer(struct work *wk, void *context)
   4761 {
   4762 	struct ixgbe_softc *sc = context;
   4763 	struct ixgbe_hw *hw = &sc->hw;
   4764 
   4765 	IXGBE_CORE_LOCK(sc);
   4766 	if (ixgbe_fw_recovery_mode(hw)) {
   4767 		if (atomic_cas_uint(&sc->recovery_mode, 0, 1) == 0) {
   4768 			/* Firmware error detected, entering recovery mode */
   4769 			device_printf(sc->dev,
   4770 			    "Firmware recovery mode detected. Limiting "
   4771 			    "functionality. Refer to the Intel(R) Ethernet "
   4772 			    "Adapters and Devices User Guide for details on "
   4773 			    "firmware recovery mode.\n");
   4774 
   4775 			if (hw->adapter_stopped == FALSE)
   4776 				ixgbe_stop_locked(sc);
   4777 		}
   4778 	} else
   4779 		atomic_cas_uint(&sc->recovery_mode, 1, 0);
   4780 
   4781 	atomic_store_relaxed(&sc->recovery_mode_timer_pending, 0);
   4782 	callout_reset(&sc->recovery_mode_timer, hz,
   4783 	    ixgbe_recovery_mode_timer, sc);
   4784 	IXGBE_CORE_UNLOCK(sc);
   4785 } /* ixgbe_handle_recovery_mode_timer */
   4786 
   4787 /************************************************************************
   4788  * ixgbe_handle_mod - Tasklet for SFP module interrupts
   4789  * bool int_en: true if it's called when the interrupt is enabled.
   4790  ************************************************************************/
   4791 static void
   4792 ixgbe_handle_mod(void *context, bool int_en)
   4793 {
   4794 	struct ixgbe_softc *sc = context;
   4795 	struct ixgbe_hw *hw = &sc->hw;
   4796 	device_t	dev = sc->dev;
   4797 	enum ixgbe_sfp_type last_sfp_type;
   4798 	u32		err;
   4799 	bool		last_unsupported_sfp_recovery;
   4800 
   4801 	KASSERT(mutex_owned(&sc->core_mtx));
   4802 
   4803 	last_sfp_type = hw->phy.sfp_type;
   4804 	last_unsupported_sfp_recovery = hw->need_unsupported_sfp_recovery;
   4805 	IXGBE_EVC_ADD(&sc->mod_workev, 1);
   4806 	if (sc->hw.need_crosstalk_fix) {
   4807 		if ((hw->mac.type != ixgbe_mac_82598EB) &&
   4808 		    !ixgbe_sfp_cage_full(hw))
   4809 			goto out;
   4810 	}
   4811 
   4812 	err = hw->phy.ops.identify_sfp(hw);
   4813 	if (err == IXGBE_ERR_SFP_NOT_SUPPORTED) {
   4814 		if (last_unsupported_sfp_recovery == false)
   4815 			device_printf(dev,
   4816 			    "Unsupported SFP+ module type was detected.\n");
   4817 		goto out;
   4818 	}
   4819 
   4820 	if (hw->need_unsupported_sfp_recovery) {
   4821 		device_printf(dev, "Recovering from unsupported SFP\n");
   4822 		/*
   4823 		 *  We could recover the status by calling setup_sfp(),
   4824 		 * setup_link() and some others. It's complex and might not
   4825 		 * work correctly on some unknown cases. To avoid such type of
   4826 		 * problem, call ixgbe_init_locked(). It's simple and safe
   4827 		 * approach.
   4828 		 */
   4829 		ixgbe_init_locked(sc);
   4830 	} else if ((hw->phy.sfp_type != ixgbe_sfp_type_not_present) &&
   4831 	    (hw->phy.sfp_type != last_sfp_type)) {
   4832 		/* A module is inserted and changed. */
   4833 
   4834 		if (hw->mac.type == ixgbe_mac_82598EB)
   4835 			err = hw->phy.ops.reset(hw);
   4836 		else {
   4837 			err = hw->mac.ops.setup_sfp(hw);
   4838 			hw->phy.sfp_setup_needed = FALSE;
   4839 		}
   4840 		if (err == IXGBE_ERR_SFP_NOT_SUPPORTED) {
   4841 			device_printf(dev,
   4842 			    "Setup failure - unsupported SFP+ module type.\n");
   4843 			goto out;
   4844 		}
   4845 	}
   4846 
   4847 out:
   4848 	/* get_supported_phy_layer will call hw->phy.ops.identify_sfp() */
   4849 	sc->phy_layer = ixgbe_get_supported_physical_layer(hw);
   4850 
   4851 	/* Adjust media types shown in ifconfig */
   4852 	IXGBE_CORE_UNLOCK(sc);
   4853 	ifmedia_removeall(&sc->media);
   4854 	ixgbe_add_media_types(sc);
   4855 	ifmedia_set(&sc->media, IFM_ETHER | IFM_AUTO);
   4856 	IXGBE_CORE_LOCK(sc);
   4857 
   4858 	/*
   4859 	 * Don't schedule MSF event if the chip is 82598. 82598 doesn't support
   4860 	 * MSF. At least, calling ixgbe_handle_msf on 82598 DA makes the link
   4861 	 * flap because the function calls setup_link().
   4862 	 */
   4863 	if (hw->mac.type != ixgbe_mac_82598EB) {
   4864 		mutex_enter(&sc->admin_mtx);
   4865 		if (int_en)
   4866 			sc->task_requests |= IXGBE_REQUEST_TASK_MSF;
   4867 		else
   4868 			sc->task_requests |= IXGBE_REQUEST_TASK_MSF_WOI;
   4869 		mutex_exit(&sc->admin_mtx);
   4870 	}
   4871 
   4872 	/*
   4873 	 * Don't call ixgbe_schedule_admin_tasklet() because we are on
   4874 	 * the workqueue now.
   4875 	 */
   4876 } /* ixgbe_handle_mod */
   4877 
   4878 
   4879 /************************************************************************
   4880  * ixgbe_handle_msf - Tasklet for MSF (multispeed fiber) interrupts
   4881  ************************************************************************/
   4882 static void
   4883 ixgbe_handle_msf(void *context)
   4884 {
   4885 	struct ixgbe_softc *sc = context;
   4886 	struct ixgbe_hw *hw = &sc->hw;
   4887 	u32		autoneg;
   4888 	bool		negotiate;
   4889 
   4890 	KASSERT(mutex_owned(&sc->core_mtx));
   4891 
   4892 	IXGBE_EVC_ADD(&sc->msf_workev, 1);
   4893 
   4894 	autoneg = hw->phy.autoneg_advertised;
   4895 	if ((!autoneg) && (hw->mac.ops.get_link_capabilities))
   4896 		hw->mac.ops.get_link_capabilities(hw, &autoneg, &negotiate);
   4897 	if (hw->mac.ops.setup_link)
   4898 		hw->mac.ops.setup_link(hw, autoneg, TRUE);
   4899 } /* ixgbe_handle_msf */
   4900 
   4901 /************************************************************************
   4902  * ixgbe_handle_phy - Tasklet for external PHY interrupts
   4903  ************************************************************************/
   4904 static void
   4905 ixgbe_handle_phy(void *context)
   4906 {
   4907 	struct ixgbe_softc *sc = context;
   4908 	struct ixgbe_hw *hw = &sc->hw;
   4909 	int error;
   4910 
   4911 	KASSERT(mutex_owned(&sc->core_mtx));
   4912 
   4913 	IXGBE_EVC_ADD(&sc->phy_workev, 1);
   4914 	error = hw->phy.ops.handle_lasi(hw);
   4915 	if (error == IXGBE_ERR_OVERTEMP)
   4916 		device_printf(sc->dev,
   4917 		    "CRITICAL: EXTERNAL PHY OVER TEMP!! "
   4918 		    " PHY will downshift to lower power state!\n");
   4919 	else if (error)
   4920 		device_printf(sc->dev,
   4921 		    "Error handling LASI interrupt: %d\n", error);
   4922 } /* ixgbe_handle_phy */
   4923 
   4924 static void
   4925 ixgbe_handle_admin(struct work *wk, void *context)
   4926 {
   4927 	struct ixgbe_softc *sc = context;
   4928 	struct ifnet	*ifp = sc->ifp;
   4929 	struct ixgbe_hw *hw = &sc->hw;
   4930 	u32		task_requests;
   4931 	u32		eims_enable = 0;
   4932 
   4933 	mutex_enter(&sc->admin_mtx);
   4934 	sc->admin_pending = 0;
   4935 	task_requests = sc->task_requests;
   4936 	sc->task_requests = 0;
   4937 	mutex_exit(&sc->admin_mtx);
   4938 
   4939 	/*
   4940 	 * Hold the IFNET_LOCK across this entire call.  This will
   4941 	 * prevent additional changes to sc->phy_layer
   4942 	 * and serialize calls to this tasklet.  We cannot hold the
   4943 	 * CORE_LOCK while calling into the ifmedia functions as
   4944 	 * they call ifmedia_lock() and the lock is CORE_LOCK.
   4945 	 */
   4946 	IFNET_LOCK(ifp);
   4947 	IXGBE_CORE_LOCK(sc);
   4948 	if ((task_requests & IXGBE_REQUEST_TASK_LSC) != 0) {
   4949 		ixgbe_handle_link(sc);
   4950 		eims_enable |= IXGBE_EIMS_LSC;
   4951 	}
   4952 	if ((task_requests & IXGBE_REQUEST_TASK_MOD_WOI) != 0)
   4953 		ixgbe_handle_mod(sc, false);
   4954 	if ((task_requests & IXGBE_REQUEST_TASK_MOD) != 0) {
   4955 		ixgbe_handle_mod(sc, true);
   4956 		if (hw->mac.type >= ixgbe_mac_X540)
   4957 			eims_enable |= IXGBE_EICR_GPI_SDP0_X540;
   4958 		else
   4959 			eims_enable |= IXGBE_EICR_GPI_SDP2_BY_MAC(hw);
   4960 	}
   4961 	if ((task_requests
   4962 	    & (IXGBE_REQUEST_TASK_MSF_WOI | IXGBE_REQUEST_TASK_MSF)) != 0) {
   4963 		ixgbe_handle_msf(sc);
   4964 		if (((task_requests & IXGBE_REQUEST_TASK_MSF) != 0) &&
   4965 		    (hw->mac.type == ixgbe_mac_82599EB))
   4966 		    eims_enable |= IXGBE_EIMS_GPI_SDP1_BY_MAC(hw);
   4967 	}
   4968 	if ((task_requests & IXGBE_REQUEST_TASK_PHY) != 0) {
   4969 		ixgbe_handle_phy(sc);
   4970 		eims_enable |= IXGBE_EICR_GPI_SDP0_X540;
   4971 	}
   4972 	if ((task_requests & IXGBE_REQUEST_TASK_FDIR) != 0) {
   4973 		ixgbe_reinit_fdir(sc);
   4974 		eims_enable |= IXGBE_EIMS_FLOW_DIR;
   4975 	}
   4976 #if 0 /* notyet */
   4977 	if ((task_requests & IXGBE_REQUEST_TASK_MBX) != 0) {
   4978 		ixgbe_handle_mbx(sc);
   4979 		eims_enable |= IXGBE_EIMS_MAILBOX;
   4980 	}
   4981 #endif
   4982 	IXGBE_WRITE_REG(hw, IXGBE_EIMS, eims_enable);
   4983 
   4984 	IXGBE_CORE_UNLOCK(sc);
   4985 	IFNET_UNLOCK(ifp);
   4986 } /* ixgbe_handle_admin */
   4987 
   4988 static void
   4989 ixgbe_ifstop(struct ifnet *ifp, int disable)
   4990 {
   4991 	struct ixgbe_softc *sc = ifp->if_softc;
   4992 
   4993 	IXGBE_CORE_LOCK(sc);
   4994 	ixgbe_stop_locked(sc);
   4995 	IXGBE_CORE_UNLOCK(sc);
   4996 
   4997 	workqueue_wait(sc->timer_wq, &sc->timer_wc);
   4998 	atomic_store_relaxed(&sc->timer_pending, 0);
   4999 }
   5000 
   5001 /************************************************************************
   5002  * ixgbe_stop_locked - Stop the hardware
   5003  *
   5004  *   Disables all traffic on the adapter by issuing a
   5005  *   global reset on the MAC and deallocates TX/RX buffers.
   5006  ************************************************************************/
   5007 static void
   5008 ixgbe_stop_locked(void *arg)
   5009 {
   5010 	struct ifnet	*ifp;
   5011 	struct ixgbe_softc *sc = arg;
   5012 	struct ixgbe_hw *hw = &sc->hw;
   5013 
   5014 	ifp = sc->ifp;
   5015 
   5016 	KASSERT(mutex_owned(&sc->core_mtx));
   5017 
   5018 	INIT_DEBUGOUT("ixgbe_stop_locked: begin\n");
   5019 	ixgbe_disable_intr(sc);
   5020 	callout_stop(&sc->timer);
   5021 
   5022 	/* Don't schedule workqueues. */
   5023 	sc->schedule_wqs_ok = false;
   5024 
   5025 	/* Let the stack know...*/
   5026 	ifp->if_flags &= ~IFF_RUNNING;
   5027 
   5028 	ixgbe_reset_hw(hw);
   5029 	hw->adapter_stopped = FALSE;
   5030 	ixgbe_stop_adapter(hw);
   5031 	if (hw->mac.type == ixgbe_mac_82599EB)
   5032 		ixgbe_stop_mac_link_on_d3_82599(hw);
   5033 	/* Turn off the laser - noop with no optics */
   5034 	ixgbe_disable_tx_laser(hw);
   5035 
   5036 	/* Update the stack */
   5037 	sc->link_up = FALSE;
   5038 	ixgbe_update_link_status(sc);
   5039 
   5040 	/* reprogram the RAR[0] in case user changed it. */
   5041 	ixgbe_set_rar(&sc->hw, 0, sc->hw.mac.addr, 0, IXGBE_RAH_AV);
   5042 
   5043 	return;
   5044 } /* ixgbe_stop_locked */
   5045 
   5046 /************************************************************************
   5047  * ixgbe_update_link_status - Update OS on link state
   5048  *
   5049  * Note: Only updates the OS on the cached link state.
   5050  *	 The real check of the hardware only happens with
   5051  *	 a link interrupt.
   5052  ************************************************************************/
   5053 static void
   5054 ixgbe_update_link_status(struct ixgbe_softc *sc)
   5055 {
   5056 	struct ifnet	*ifp = sc->ifp;
   5057 	device_t	dev = sc->dev;
   5058 	struct ixgbe_hw *hw = &sc->hw;
   5059 
   5060 	KASSERT(mutex_owned(&sc->core_mtx));
   5061 
   5062 	if (sc->link_up) {
   5063 		if (sc->link_active != LINK_STATE_UP) {
   5064 			/*
   5065 			 * To eliminate influence of the previous state
   5066 			 * in the same way as ixgbe_init_locked().
   5067 			 */
   5068 			struct ix_queue	*que = sc->queues;
   5069 			for (int i = 0; i < sc->num_queues; i++, que++)
   5070 				que->eitr_setting = 0;
   5071 
   5072 			if (sc->link_speed == IXGBE_LINK_SPEED_10GB_FULL) {
   5073 				/*
   5074 				 *  Discard count for both MAC Local Fault and
   5075 				 * Remote Fault because those registers are
   5076 				 * valid only when the link speed is up and
   5077 				 * 10Gbps.
   5078 				 */
   5079 				IXGBE_READ_REG(hw, IXGBE_MLFC);
   5080 				IXGBE_READ_REG(hw, IXGBE_MRFC);
   5081 			}
   5082 
   5083 			if (bootverbose) {
   5084 				const char *bpsmsg;
   5085 
   5086 				switch (sc->link_speed) {
   5087 				case IXGBE_LINK_SPEED_10GB_FULL:
   5088 					bpsmsg = "10 Gbps";
   5089 					break;
   5090 				case IXGBE_LINK_SPEED_5GB_FULL:
   5091 					bpsmsg = "5 Gbps";
   5092 					break;
   5093 				case IXGBE_LINK_SPEED_2_5GB_FULL:
   5094 					bpsmsg = "2.5 Gbps";
   5095 					break;
   5096 				case IXGBE_LINK_SPEED_1GB_FULL:
   5097 					bpsmsg = "1 Gbps";
   5098 					break;
   5099 				case IXGBE_LINK_SPEED_100_FULL:
   5100 					bpsmsg = "100 Mbps";
   5101 					break;
   5102 				case IXGBE_LINK_SPEED_10_FULL:
   5103 					bpsmsg = "10 Mbps";
   5104 					break;
   5105 				default:
   5106 					bpsmsg = "unknown speed";
   5107 					break;
   5108 				}
   5109 				device_printf(dev, "Link is up %s %s \n",
   5110 				    bpsmsg, "Full Duplex");
   5111 			}
   5112 			sc->link_active = LINK_STATE_UP;
   5113 			/* Update any Flow Control changes */
   5114 			ixgbe_fc_enable(&sc->hw);
   5115 			/* Update DMA coalescing config */
   5116 			ixgbe_config_dmac(sc);
   5117 			if_link_state_change(ifp, LINK_STATE_UP);
   5118 
   5119 			if (sc->feat_en & IXGBE_FEATURE_SRIOV)
   5120 				ixgbe_ping_all_vfs(sc);
   5121 		}
   5122 	} else {
   5123 		/*
   5124 		 * Do it when link active changes to DOWN. i.e.
   5125 		 * a) LINK_STATE_UNKNOWN -> LINK_STATE_DOWN
   5126 		 * b) LINK_STATE_UP	 -> LINK_STATE_DOWN
   5127 		 */
   5128 		if (sc->link_active != LINK_STATE_DOWN) {
   5129 			if (bootverbose)
   5130 				device_printf(dev, "Link is Down\n");
   5131 			if_link_state_change(ifp, LINK_STATE_DOWN);
   5132 			sc->link_active = LINK_STATE_DOWN;
   5133 			if (sc->feat_en & IXGBE_FEATURE_SRIOV)
   5134 				ixgbe_ping_all_vfs(sc);
   5135 			ixgbe_drain_all(sc);
   5136 		}
   5137 	}
   5138 } /* ixgbe_update_link_status */
   5139 
   5140 /************************************************************************
   5141  * ixgbe_config_dmac - Configure DMA Coalescing
   5142  ************************************************************************/
   5143 static void
   5144 ixgbe_config_dmac(struct ixgbe_softc *sc)
   5145 {
   5146 	struct ixgbe_hw *hw = &sc->hw;
   5147 	struct ixgbe_dmac_config *dcfg = &hw->mac.dmac_config;
   5148 
   5149 	if (hw->mac.type < ixgbe_mac_X550 || !hw->mac.ops.dmac_config)
   5150 		return;
   5151 
   5152 	if (dcfg->watchdog_timer ^ sc->dmac ||
   5153 	    dcfg->link_speed ^ sc->link_speed) {
   5154 		dcfg->watchdog_timer = sc->dmac;
   5155 		dcfg->fcoe_en = false;
   5156 		dcfg->link_speed = sc->link_speed;
   5157 		dcfg->num_tcs = 1;
   5158 
   5159 		INIT_DEBUGOUT2("dmac settings: watchdog %d, link speed %d\n",
   5160 		    dcfg->watchdog_timer, dcfg->link_speed);
   5161 
   5162 		hw->mac.ops.dmac_config(hw);
   5163 	}
   5164 } /* ixgbe_config_dmac */
   5165 
   5166 /************************************************************************
   5167  * ixgbe_enable_intr
   5168  ************************************************************************/
   5169 static void
   5170 ixgbe_enable_intr(struct ixgbe_softc *sc)
   5171 {
   5172 	struct ixgbe_hw	*hw = &sc->hw;
   5173 	struct ix_queue	*que = sc->queues;
   5174 	u32		mask, fwsm;
   5175 
   5176 	mask = (IXGBE_EIMS_ENABLE_MASK & ~IXGBE_EIMS_RTX_QUEUE);
   5177 
   5178 	switch (sc->hw.mac.type) {
   5179 	case ixgbe_mac_82599EB:
   5180 		mask |= IXGBE_EIMS_ECC;
   5181 		/* Temperature sensor on some adapters */
   5182 		mask |= IXGBE_EIMS_GPI_SDP0;
   5183 		/* SFP+ (RX_LOS_N & MOD_ABS_N) */
   5184 		mask |= IXGBE_EIMS_GPI_SDP1;
   5185 		mask |= IXGBE_EIMS_GPI_SDP2;
   5186 		break;
   5187 	case ixgbe_mac_X540:
   5188 		/* Detect if Thermal Sensor is enabled */
   5189 		fwsm = IXGBE_READ_REG(hw, IXGBE_FWSM);
   5190 		if (fwsm & IXGBE_FWSM_TS_ENABLED)
   5191 			mask |= IXGBE_EIMS_TS;
   5192 		mask |= IXGBE_EIMS_ECC;
   5193 		break;
   5194 	case ixgbe_mac_X550:
   5195 		/* MAC thermal sensor is automatically enabled */
   5196 		mask |= IXGBE_EIMS_TS;
   5197 		mask |= IXGBE_EIMS_ECC;
   5198 		break;
   5199 	case ixgbe_mac_X550EM_x:
   5200 	case ixgbe_mac_X550EM_a:
   5201 		/* Some devices use SDP0 for important information */
   5202 		if (hw->device_id == IXGBE_DEV_ID_X550EM_X_SFP ||
   5203 		    hw->device_id == IXGBE_DEV_ID_X550EM_A_SFP ||
   5204 		    hw->device_id == IXGBE_DEV_ID_X550EM_A_SFP_N ||
   5205 		    hw->device_id == IXGBE_DEV_ID_X550EM_X_10G_T)
   5206 			mask |= IXGBE_EIMS_GPI_SDP0_BY_MAC(hw);
   5207 		if (hw->phy.type == ixgbe_phy_x550em_ext_t)
   5208 			mask |= IXGBE_EICR_GPI_SDP0_X540;
   5209 		mask |= IXGBE_EIMS_ECC;
   5210 		break;
   5211 	default:
   5212 		break;
   5213 	}
   5214 
   5215 	/* Enable Fan Failure detection */
   5216 	if (sc->feat_en & IXGBE_FEATURE_FAN_FAIL)
   5217 		mask |= IXGBE_EIMS_GPI_SDP1;
   5218 	/* Enable SR-IOV */
   5219 	if (sc->feat_en & IXGBE_FEATURE_SRIOV)
   5220 		mask |= IXGBE_EIMS_MAILBOX;
   5221 	/* Enable Flow Director */
   5222 	if (sc->feat_en & IXGBE_FEATURE_FDIR)
   5223 		mask |= IXGBE_EIMS_FLOW_DIR;
   5224 
   5225 	IXGBE_WRITE_REG(hw, IXGBE_EIMS, mask);
   5226 
   5227 	/* With MSI-X we use auto clear */
   5228 	if ((sc->feat_en & IXGBE_FEATURE_MSIX) != 0) {
   5229 		/*
   5230 		 * We use auto clear for RTX_QUEUE only. Don't use other
   5231 		 * interrupts (e.g. link interrupt). BTW, we don't use
   5232 		 *  TCP_TIMER interrupt itself.
   5233 		 */
   5234 		IXGBE_WRITE_REG(hw, IXGBE_EIAC, IXGBE_EIMS_RTX_QUEUE);
   5235 	}
   5236 
   5237 	/*
   5238 	 * Now enable all queues, this is done separately to
   5239 	 * allow for handling the extended (beyond 32) MSI-X
   5240 	 * vectors that can be used by 82599
   5241 	 */
   5242 	for (int i = 0; i < sc->num_queues; i++, que++)
   5243 		ixgbe_enable_queue(sc, que->msix);
   5244 
   5245 	IXGBE_WRITE_FLUSH(hw);
   5246 
   5247 } /* ixgbe_enable_intr */
   5248 
   5249 /************************************************************************
   5250  * ixgbe_disable_intr_internal
   5251  ************************************************************************/
   5252 static void
   5253 ixgbe_disable_intr_internal(struct ixgbe_softc *sc, bool nestok)
   5254 {
   5255 	struct ix_queue	*que = sc->queues;
   5256 
   5257 	/* disable interrupts other than queues */
   5258 	IXGBE_WRITE_REG(&sc->hw, IXGBE_EIMC, ~IXGBE_EIMC_RTX_QUEUE);
   5259 
   5260 	if ((sc->feat_en & IXGBE_FEATURE_MSIX) != 0)
   5261 		IXGBE_WRITE_REG(&sc->hw, IXGBE_EIAC, 0);
   5262 
   5263 	for (int i = 0; i < sc->num_queues; i++, que++)
   5264 		ixgbe_disable_queue_internal(sc, que->msix, nestok);
   5265 
   5266 	IXGBE_WRITE_FLUSH(&sc->hw);
   5267 
   5268 } /* ixgbe_do_disable_intr_internal */
   5269 
   5270 /************************************************************************
   5271  * ixgbe_disable_intr
   5272  ************************************************************************/
   5273 static void
   5274 ixgbe_disable_intr(struct ixgbe_softc *sc)
   5275 {
   5276 
   5277 	ixgbe_disable_intr_internal(sc, true);
   5278 } /* ixgbe_disable_intr */
   5279 
   5280 /************************************************************************
   5281  * ixgbe_ensure_disabled_intr
   5282  ************************************************************************/
   5283 void
   5284 ixgbe_ensure_disabled_intr(struct ixgbe_softc *sc)
   5285 {
   5286 
   5287 	ixgbe_disable_intr_internal(sc, false);
   5288 } /* ixgbe_ensure_disabled_intr */
   5289 
   5290 /************************************************************************
   5291  * ixgbe_legacy_irq - Legacy Interrupt Service routine
   5292  ************************************************************************/
   5293 static int
   5294 ixgbe_legacy_irq(void *arg)
   5295 {
   5296 	struct ix_queue *que = arg;
   5297 	struct ixgbe_softc *sc = que->sc;
   5298 	struct ixgbe_hw	*hw = &sc->hw;
   5299 	struct ifnet	*ifp = sc->ifp;
   5300 	struct tx_ring	*txr = sc->tx_rings;
   5301 	u32		eicr;
   5302 	u32		eims_orig;
   5303 	u32		eims_enable = 0;
   5304 	u32		eims_disable = 0;
   5305 
   5306 	eims_orig = IXGBE_READ_REG(hw, IXGBE_EIMS);
   5307 	/*
   5308 	 * Silicon errata #26 on 82598. Disable all interrupts before reading
   5309 	 * EICR.
   5310 	 */
   5311 	IXGBE_WRITE_REG(hw, IXGBE_EIMC, IXGBE_IRQ_CLEAR_MASK);
   5312 
   5313 	/* Read and clear EICR */
   5314 	eicr = IXGBE_READ_REG(hw, IXGBE_EICR);
   5315 
   5316 	if (eicr == 0) {
   5317 		IXGBE_EVC_ADD(&sc->stats.pf.intzero, 1);
   5318 		IXGBE_WRITE_REG(hw, IXGBE_EIMS, eims_orig);
   5319 		return 0;
   5320 	}
   5321 	IXGBE_EVC_ADD(&sc->stats.pf.legint, 1);
   5322 
   5323 	/* Queue (0) intr */
   5324 	if (((ifp->if_flags & IFF_RUNNING) != 0) &&
   5325 	    (eicr & IXGBE_EIMC_RTX_QUEUE) != 0) {
   5326 		IXGBE_EVC_ADD(&que->irqs, 1);
   5327 
   5328 		/*
   5329 		 * The same as ixgbe_msix_que() about
   5330 		 * "que->txrx_use_workqueue".
   5331 		 */
   5332 		que->txrx_use_workqueue = sc->txrx_use_workqueue;
   5333 
   5334 		IXGBE_TX_LOCK(txr);
   5335 		ixgbe_txeof(txr);
   5336 #ifdef notyet
   5337 		if (!ixgbe_ring_empty(ifp, txr->br))
   5338 			ixgbe_start_locked(ifp, txr);
   5339 #endif
   5340 		IXGBE_TX_UNLOCK(txr);
   5341 
   5342 		IXGBE_EVC_ADD(&que->req, 1);
   5343 		ixgbe_sched_handle_que(sc, que);
   5344 		/* Disable queue 0 interrupt */
   5345 		eims_disable |= 1UL << 0;
   5346 	} else
   5347 		eims_enable |= eims_orig & IXGBE_EIMC_RTX_QUEUE;
   5348 
   5349 	ixgbe_intr_admin_common(sc, eicr, &eims_disable);
   5350 
   5351 	/* Re-enable some interrupts */
   5352 	IXGBE_WRITE_REG(hw, IXGBE_EIMS,
   5353 	    (eims_orig & ~eims_disable) | eims_enable);
   5354 
   5355 	return 1;
   5356 } /* ixgbe_legacy_irq */
   5357 
   5358 /************************************************************************
   5359  * ixgbe_free_pciintr_resources
   5360  ************************************************************************/
   5361 static void
   5362 ixgbe_free_pciintr_resources(struct ixgbe_softc *sc)
   5363 {
   5364 	struct ix_queue *que = sc->queues;
   5365 	int		rid;
   5366 
   5367 	/*
   5368 	 * Release all msix queue resources:
   5369 	 */
   5370 	for (int i = 0; i < sc->num_queues; i++, que++) {
   5371 		if (que->res != NULL) {
   5372 			pci_intr_disestablish(sc->osdep.pc, sc->osdep.ihs[i]);
   5373 			sc->osdep.ihs[i] = NULL;
   5374 		}
   5375 	}
   5376 
   5377 	/* Clean the Legacy or Link interrupt last */
   5378 	if (sc->vector) /* we are doing MSIX */
   5379 		rid = sc->vector;
   5380 	else
   5381 		rid = 0;
   5382 
   5383 	if (sc->osdep.ihs[rid] != NULL) {
   5384 		pci_intr_disestablish(sc->osdep.pc, sc->osdep.ihs[rid]);
   5385 		sc->osdep.ihs[rid] = NULL;
   5386 	}
   5387 
   5388 	if (sc->osdep.intrs != NULL) {
   5389 		pci_intr_release(sc->osdep.pc, sc->osdep.intrs,
   5390 		    sc->osdep.nintrs);
   5391 		sc->osdep.intrs = NULL;
   5392 	}
   5393 } /* ixgbe_free_pciintr_resources */
   5394 
   5395 /************************************************************************
   5396  * ixgbe_free_pci_resources
   5397  ************************************************************************/
   5398 static void
   5399 ixgbe_free_pci_resources(struct ixgbe_softc *sc)
   5400 {
   5401 
   5402 	ixgbe_free_pciintr_resources(sc);
   5403 
   5404 	if (sc->osdep.mem_size != 0) {
   5405 		bus_space_unmap(sc->osdep.mem_bus_space_tag,
   5406 		    sc->osdep.mem_bus_space_handle,
   5407 		    sc->osdep.mem_size);
   5408 	}
   5409 } /* ixgbe_free_pci_resources */
   5410 
   5411 /************************************************************************
   5412  * ixgbe_sysctl_flowcntl
   5413  *
   5414  *   SYSCTL wrapper around setting Flow Control
   5415  ************************************************************************/
   5416 static int
   5417 ixgbe_sysctl_flowcntl(SYSCTLFN_ARGS)
   5418 {
   5419 	struct sysctlnode node = *rnode;
   5420 	struct ixgbe_softc *sc = (struct ixgbe_softc *)node.sysctl_data;
   5421 	int error, fc;
   5422 
   5423 	if (ixgbe_fw_recovery_mode_swflag(sc))
   5424 		return (EPERM);
   5425 
   5426 	fc = sc->hw.fc.current_mode;
   5427 	node.sysctl_data = &fc;
   5428 	error = sysctl_lookup(SYSCTLFN_CALL(&node));
   5429 	if (error != 0 || newp == NULL)
   5430 		return error;
   5431 
   5432 	/* Don't bother if it's not changed */
   5433 	if (fc == sc->hw.fc.current_mode)
   5434 		return (0);
   5435 
   5436 	return ixgbe_set_flowcntl(sc, fc);
   5437 } /* ixgbe_sysctl_flowcntl */
   5438 
   5439 /************************************************************************
   5440  * ixgbe_set_flowcntl - Set flow control
   5441  *
   5442  *   Flow control values:
   5443  *     0 - off
   5444  *     1 - rx pause
   5445  *     2 - tx pause
   5446  *     3 - full
   5447  ************************************************************************/
   5448 static int
   5449 ixgbe_set_flowcntl(struct ixgbe_softc *sc, int fc)
   5450 {
   5451 	switch (fc) {
   5452 		case ixgbe_fc_rx_pause:
   5453 		case ixgbe_fc_tx_pause:
   5454 		case ixgbe_fc_full:
   5455 			sc->hw.fc.requested_mode = fc;
   5456 			if (sc->num_queues > 1)
   5457 				ixgbe_disable_rx_drop(sc);
   5458 			break;
   5459 		case ixgbe_fc_none:
   5460 			sc->hw.fc.requested_mode = ixgbe_fc_none;
   5461 			if (sc->num_queues > 1)
   5462 				ixgbe_enable_rx_drop(sc);
   5463 			break;
   5464 		default:
   5465 			return (EINVAL);
   5466 	}
   5467 
   5468 #if 0 /* XXX NetBSD */
   5469 	/* Don't autoneg if forcing a value */
   5470 	sc->hw.fc.disable_fc_autoneg = TRUE;
   5471 #endif
   5472 	ixgbe_fc_enable(&sc->hw);
   5473 
   5474 	return (0);
   5475 } /* ixgbe_set_flowcntl */
   5476 
   5477 /************************************************************************
   5478  * ixgbe_enable_rx_drop
   5479  *
   5480  *   Enable the hardware to drop packets when the buffer is
   5481  *   full. This is useful with multiqueue, so that no single
   5482  *   queue being full stalls the entire RX engine. We only
   5483  *   enable this when Multiqueue is enabled AND Flow Control
   5484  *   is disabled.
   5485  ************************************************************************/
   5486 static void
   5487 ixgbe_enable_rx_drop(struct ixgbe_softc *sc)
   5488 {
   5489 	struct ixgbe_hw *hw = &sc->hw;
   5490 	struct rx_ring	*rxr;
   5491 	u32		srrctl;
   5492 
   5493 	for (int i = 0; i < sc->num_queues; i++) {
   5494 		rxr = &sc->rx_rings[i];
   5495 		srrctl = IXGBE_READ_REG(hw, IXGBE_SRRCTL(rxr->me));
   5496 		srrctl |= IXGBE_SRRCTL_DROP_EN;
   5497 		IXGBE_WRITE_REG(hw, IXGBE_SRRCTL(rxr->me), srrctl);
   5498 	}
   5499 
   5500 	/* enable drop for each vf */
   5501 	for (int i = 0; i < sc->num_vfs; i++) {
   5502 		IXGBE_WRITE_REG(hw, IXGBE_QDE,
   5503 		    (IXGBE_QDE_WRITE | (i << IXGBE_QDE_IDX_SHIFT) |
   5504 		    IXGBE_QDE_ENABLE));
   5505 	}
   5506 } /* ixgbe_enable_rx_drop */
   5507 
   5508 /************************************************************************
   5509  * ixgbe_disable_rx_drop
   5510  ************************************************************************/
   5511 static void
   5512 ixgbe_disable_rx_drop(struct ixgbe_softc *sc)
   5513 {
   5514 	struct ixgbe_hw *hw = &sc->hw;
   5515 	struct rx_ring	*rxr;
   5516 	u32		srrctl;
   5517 
   5518 	for (int i = 0; i < sc->num_queues; i++) {
   5519 		rxr = &sc->rx_rings[i];
   5520 		srrctl = IXGBE_READ_REG(hw, IXGBE_SRRCTL(rxr->me));
   5521 		srrctl &= ~IXGBE_SRRCTL_DROP_EN;
   5522 		IXGBE_WRITE_REG(hw, IXGBE_SRRCTL(rxr->me), srrctl);
   5523 	}
   5524 
   5525 	/* disable drop for each vf */
   5526 	for (int i = 0; i < sc->num_vfs; i++) {
   5527 		IXGBE_WRITE_REG(hw, IXGBE_QDE,
   5528 		    (IXGBE_QDE_WRITE | (i << IXGBE_QDE_IDX_SHIFT)));
   5529 	}
   5530 } /* ixgbe_disable_rx_drop */
   5531 
   5532 /************************************************************************
   5533  * ixgbe_sysctl_advertise
   5534  *
   5535  *   SYSCTL wrapper around setting advertised speed
   5536  ************************************************************************/
   5537 static int
   5538 ixgbe_sysctl_advertise(SYSCTLFN_ARGS)
   5539 {
   5540 	struct sysctlnode node = *rnode;
   5541 	struct ixgbe_softc *sc = (struct ixgbe_softc *)node.sysctl_data;
   5542 	int	       error = 0, advertise;
   5543 
   5544 	if (ixgbe_fw_recovery_mode_swflag(sc))
   5545 		return (EPERM);
   5546 
   5547 	advertise = sc->advertise;
   5548 	node.sysctl_data = &advertise;
   5549 	error = sysctl_lookup(SYSCTLFN_CALL(&node));
   5550 	if (error != 0 || newp == NULL)
   5551 		return error;
   5552 
   5553 	return ixgbe_set_advertise(sc, advertise);
   5554 } /* ixgbe_sysctl_advertise */
   5555 
   5556 /************************************************************************
   5557  * ixgbe_set_advertise - Control advertised link speed
   5558  *
   5559  *   Flags:
   5560  *     0x00 - Default (all capable link speed)
   5561  *     0x1  - advertise 100 Mb
   5562  *     0x2  - advertise 1G
   5563  *     0x4  - advertise 10G
   5564  *     0x8  - advertise 10 Mb (yes, Mb)
   5565  *     0x10 - advertise 2.5G
   5566  *     0x20 - advertise 5G
   5567  ************************************************************************/
   5568 static int
   5569 ixgbe_set_advertise(struct ixgbe_softc *sc, int advertise)
   5570 {
   5571 	device_t	 dev;
   5572 	struct ixgbe_hw	 *hw;
   5573 	ixgbe_link_speed speed = 0;
   5574 	ixgbe_link_speed link_caps = 0;
   5575 	s32		 err = IXGBE_NOT_IMPLEMENTED;
   5576 	bool		 negotiate = FALSE;
   5577 
   5578 	/* Checks to validate new value */
   5579 	if (sc->advertise == advertise) /* no change */
   5580 		return (0);
   5581 
   5582 	dev = sc->dev;
   5583 	hw = &sc->hw;
   5584 
   5585 	/* No speed changes for backplane media */
   5586 	if (hw->phy.media_type == ixgbe_media_type_backplane)
   5587 		return (ENODEV);
   5588 
   5589 	if (!((hw->phy.media_type == ixgbe_media_type_copper) ||
   5590 	    (hw->phy.multispeed_fiber))) {
   5591 		device_printf(dev,
   5592 		    "Advertised speed can only be set on copper or "
   5593 		    "multispeed fiber media types.\n");
   5594 		return (EINVAL);
   5595 	}
   5596 
   5597 	if (advertise < 0x0 || advertise > 0x3f) {
   5598 		device_printf(dev, "Invalid advertised speed; "
   5599 		    "valid modes are 0x0 through 0x3f\n");
   5600 		return (EINVAL);
   5601 	}
   5602 
   5603 	if (hw->mac.ops.get_link_capabilities) {
   5604 		err = hw->mac.ops.get_link_capabilities(hw, &link_caps,
   5605 		    &negotiate);
   5606 		if (err != IXGBE_SUCCESS) {
   5607 			device_printf(dev, "Unable to determine supported "
   5608 			    "advertise speeds\n");
   5609 			return (ENODEV);
   5610 		}
   5611 	}
   5612 
   5613 	/* Set new value and report new advertised mode */
   5614 	if (advertise & 0x1) {
   5615 		if (!(link_caps & IXGBE_LINK_SPEED_100_FULL)) {
   5616 			device_printf(dev, "Interface does not support 100Mb "
   5617 			    "advertised speed\n");
   5618 			return (EINVAL);
   5619 		}
   5620 		speed |= IXGBE_LINK_SPEED_100_FULL;
   5621 	}
   5622 	if (advertise & 0x2) {
   5623 		if (!(link_caps & IXGBE_LINK_SPEED_1GB_FULL)) {
   5624 			device_printf(dev, "Interface does not support 1Gb "
   5625 			    "advertised speed\n");
   5626 			return (EINVAL);
   5627 		}
   5628 		speed |= IXGBE_LINK_SPEED_1GB_FULL;
   5629 	}
   5630 	if (advertise & 0x4) {
   5631 		if (!(link_caps & IXGBE_LINK_SPEED_10GB_FULL)) {
   5632 			device_printf(dev, "Interface does not support 10Gb "
   5633 			    "advertised speed\n");
   5634 			return (EINVAL);
   5635 		}
   5636 		speed |= IXGBE_LINK_SPEED_10GB_FULL;
   5637 	}
   5638 	if (advertise & 0x8) {
   5639 		if (!(link_caps & IXGBE_LINK_SPEED_10_FULL)) {
   5640 			device_printf(dev, "Interface does not support 10Mb "
   5641 			    "advertised speed\n");
   5642 			return (EINVAL);
   5643 		}
   5644 		speed |= IXGBE_LINK_SPEED_10_FULL;
   5645 	}
   5646 	if (advertise & 0x10) {
   5647 		if (!(link_caps & IXGBE_LINK_SPEED_2_5GB_FULL)) {
   5648 			device_printf(dev, "Interface does not support 2.5Gb "
   5649 			    "advertised speed\n");
   5650 			return (EINVAL);
   5651 		}
   5652 		speed |= IXGBE_LINK_SPEED_2_5GB_FULL;
   5653 	}
   5654 	if (advertise & 0x20) {
   5655 		if (!(link_caps & IXGBE_LINK_SPEED_5GB_FULL)) {
   5656 			device_printf(dev, "Interface does not support 5Gb "
   5657 			    "advertised speed\n");
   5658 			return (EINVAL);
   5659 		}
   5660 		speed |= IXGBE_LINK_SPEED_5GB_FULL;
   5661 	}
   5662 	if (advertise == 0)
   5663 		speed = link_caps; /* All capable link speed */
   5664 
   5665 	hw->mac.autotry_restart = TRUE;
   5666 	hw->mac.ops.setup_link(hw, speed, TRUE);
   5667 	sc->advertise = advertise;
   5668 
   5669 	return (0);
   5670 } /* ixgbe_set_advertise */
   5671 
   5672 /************************************************************************
   5673  * ixgbe_get_default_advertise - Get default advertised speed settings
   5674  *
   5675  *   Formatted for sysctl usage.
   5676  *   Flags:
   5677  *     0x1  - advertise 100 Mb
   5678  *     0x2  - advertise 1G
   5679  *     0x4  - advertise 10G
   5680  *     0x8  - advertise 10 Mb (yes, Mb)
   5681  *     0x10 - advertise 2.5G
   5682  *     0x20 - advertise 5G
   5683  ************************************************************************/
   5684 static int
   5685 ixgbe_get_default_advertise(struct ixgbe_softc *sc)
   5686 {
   5687 	struct ixgbe_hw	 *hw = &sc->hw;
   5688 	int		 speed;
   5689 	ixgbe_link_speed link_caps = 0;
   5690 	s32		 err;
   5691 	bool		 negotiate = FALSE;
   5692 
   5693 	/*
   5694 	 * Advertised speed means nothing unless it's copper or
   5695 	 * multi-speed fiber
   5696 	 */
   5697 	if (!(hw->phy.media_type == ixgbe_media_type_copper) &&
   5698 	    !(hw->phy.multispeed_fiber))
   5699 		return (0);
   5700 
   5701 	err = hw->mac.ops.get_link_capabilities(hw, &link_caps, &negotiate);
   5702 	if (err != IXGBE_SUCCESS)
   5703 		return (0);
   5704 
   5705 	speed =
   5706 	    ((link_caps & IXGBE_LINK_SPEED_10GB_FULL)  ? 0x4  : 0) |
   5707 	    ((link_caps & IXGBE_LINK_SPEED_5GB_FULL)   ? 0x20 : 0) |
   5708 	    ((link_caps & IXGBE_LINK_SPEED_2_5GB_FULL) ? 0x10 : 0) |
   5709 	    ((link_caps & IXGBE_LINK_SPEED_1GB_FULL)   ? 0x2  : 0) |
   5710 	    ((link_caps & IXGBE_LINK_SPEED_100_FULL)   ? 0x1  : 0) |
   5711 	    ((link_caps & IXGBE_LINK_SPEED_10_FULL)    ? 0x8  : 0);
   5712 
   5713 	return speed;
   5714 } /* ixgbe_get_default_advertise */
   5715 
   5716 /************************************************************************
   5717  * ixgbe_sysctl_dmac - Manage DMA Coalescing
   5718  *
   5719  *   Control values:
   5720  *     0/1 - off / on (use default value of 1000)
   5721  *
   5722  *     Legal timer values are:
   5723  *     50,100,250,500,1000,2000,5000,10000
   5724  *
   5725  *     Turning off interrupt moderation will also turn this off.
   5726  ************************************************************************/
   5727 static int
   5728 ixgbe_sysctl_dmac(SYSCTLFN_ARGS)
   5729 {
   5730 	struct sysctlnode node = *rnode;
   5731 	struct ixgbe_softc *sc = (struct ixgbe_softc *)node.sysctl_data;
   5732 	struct ifnet   *ifp = sc->ifp;
   5733 	int	       error;
   5734 	int	       newval;
   5735 
   5736 	if (ixgbe_fw_recovery_mode_swflag(sc))
   5737 		return (EPERM);
   5738 
   5739 	newval = sc->dmac;
   5740 	node.sysctl_data = &newval;
   5741 	error = sysctl_lookup(SYSCTLFN_CALL(&node));
   5742 	if ((error) || (newp == NULL))
   5743 		return (error);
   5744 
   5745 	switch (newval) {
   5746 	case 0:
   5747 		/* Disabled */
   5748 		sc->dmac = 0;
   5749 		break;
   5750 	case 1:
   5751 		/* Enable and use default */
   5752 		sc->dmac = 1000;
   5753 		break;
   5754 	case 50:
   5755 	case 100:
   5756 	case 250:
   5757 	case 500:
   5758 	case 1000:
   5759 	case 2000:
   5760 	case 5000:
   5761 	case 10000:
   5762 		/* Legal values - allow */
   5763 		sc->dmac = newval;
   5764 		break;
   5765 	default:
   5766 		/* Do nothing, illegal value */
   5767 		return (EINVAL);
   5768 	}
   5769 
   5770 	/* Re-initialize hardware if it's already running */
   5771 	if (ifp->if_flags & IFF_RUNNING)
   5772 		if_init(ifp);
   5773 
   5774 	return (0);
   5775 }
   5776 
   5777 #ifdef IXGBE_DEBUG
   5778 /************************************************************************
   5779  * ixgbe_sysctl_power_state
   5780  *
   5781  *   Sysctl to test power states
   5782  *   Values:
   5783  *     0      - set device to D0
   5784  *     3      - set device to D3
   5785  *     (none) - get current device power state
   5786  ************************************************************************/
   5787 static int
   5788 ixgbe_sysctl_power_state(SYSCTLFN_ARGS)
   5789 {
   5790 #ifdef notyet
   5791 	struct sysctlnode node = *rnode;
   5792 	struct ixgbe_softc *sc = (struct ixgbe_softc *)node.sysctl_data;
   5793 	device_t       dev =  sc->dev;
   5794 	int	       curr_ps, new_ps, error = 0;
   5795 
   5796 	if (ixgbe_fw_recovery_mode_swflag(sc))
   5797 		return (EPERM);
   5798 
   5799 	curr_ps = new_ps = pci_get_powerstate(dev);
   5800 
   5801 	error = sysctl_lookup(SYSCTLFN_CALL(&node));
   5802 	if ((error) || (req->newp == NULL))
   5803 		return (error);
   5804 
   5805 	if (new_ps == curr_ps)
   5806 		return (0);
   5807 
   5808 	if (new_ps == 3 && curr_ps == 0)
   5809 		error = DEVICE_SUSPEND(dev);
   5810 	else if (new_ps == 0 && curr_ps == 3)
   5811 		error = DEVICE_RESUME(dev);
   5812 	else
   5813 		return (EINVAL);
   5814 
   5815 	device_printf(dev, "New state: %d\n", pci_get_powerstate(dev));
   5816 
   5817 	return (error);
   5818 #else
   5819 	return 0;
   5820 #endif
   5821 } /* ixgbe_sysctl_power_state */
   5822 #endif
   5823 
   5824 /************************************************************************
   5825  * ixgbe_sysctl_wol_enable
   5826  *
   5827  *   Sysctl to enable/disable the WoL capability,
   5828  *   if supported by the adapter.
   5829  *
   5830  *   Values:
   5831  *     0 - disabled
   5832  *     1 - enabled
   5833  ************************************************************************/
   5834 static int
   5835 ixgbe_sysctl_wol_enable(SYSCTLFN_ARGS)
   5836 {
   5837 	struct sysctlnode node = *rnode;
   5838 	struct ixgbe_softc *sc = (struct ixgbe_softc *)node.sysctl_data;
   5839 	struct ixgbe_hw *hw = &sc->hw;
   5840 	bool		new_wol_enabled;
   5841 	int		error = 0;
   5842 
   5843 	/*
   5844 	 * It's not required to check recovery mode because this function never
   5845 	 * touches hardware.
   5846 	 */
   5847 	new_wol_enabled = hw->wol_enabled;
   5848 	node.sysctl_data = &new_wol_enabled;
   5849 	error = sysctl_lookup(SYSCTLFN_CALL(&node));
   5850 	if ((error) || (newp == NULL))
   5851 		return (error);
   5852 	if (new_wol_enabled == hw->wol_enabled)
   5853 		return (0);
   5854 
   5855 	if (new_wol_enabled && !sc->wol_support)
   5856 		return (ENODEV);
   5857 	else
   5858 		hw->wol_enabled = new_wol_enabled;
   5859 
   5860 	return (0);
   5861 } /* ixgbe_sysctl_wol_enable */
   5862 
   5863 /************************************************************************
   5864  * ixgbe_sysctl_wufc - Wake Up Filter Control
   5865  *
   5866  *   Sysctl to enable/disable the types of packets that the
   5867  *   adapter will wake up on upon receipt.
   5868  *   Flags:
   5869  *     0x1  - Link Status Change
   5870  *     0x2  - Magic Packet
   5871  *     0x4  - Direct Exact
   5872  *     0x8  - Directed Multicast
   5873  *     0x10 - Broadcast
   5874  *     0x20 - ARP/IPv4 Request Packet
   5875  *     0x40 - Direct IPv4 Packet
   5876  *     0x80 - Direct IPv6 Packet
   5877  *
   5878  *   Settings not listed above will cause the sysctl to return an error.
   5879  ************************************************************************/
   5880 static int
   5881 ixgbe_sysctl_wufc(SYSCTLFN_ARGS)
   5882 {
   5883 	struct sysctlnode node = *rnode;
   5884 	struct ixgbe_softc *sc = (struct ixgbe_softc *)node.sysctl_data;
   5885 	int error = 0;
   5886 	u32 new_wufc;
   5887 
   5888 	/*
   5889 	 * It's not required to check recovery mode because this function never
   5890 	 * touches hardware.
   5891 	 */
   5892 	new_wufc = sc->wufc;
   5893 	node.sysctl_data = &new_wufc;
   5894 	error = sysctl_lookup(SYSCTLFN_CALL(&node));
   5895 	if ((error) || (newp == NULL))
   5896 		return (error);
   5897 	if (new_wufc == sc->wufc)
   5898 		return (0);
   5899 
   5900 	if (new_wufc & 0xffffff00)
   5901 		return (EINVAL);
   5902 
   5903 	new_wufc &= 0xff;
   5904 	new_wufc |= (0xffffff & sc->wufc);
   5905 	sc->wufc = new_wufc;
   5906 
   5907 	return (0);
   5908 } /* ixgbe_sysctl_wufc */
   5909 
   5910 #ifdef IXGBE_DEBUG
   5911 /************************************************************************
   5912  * ixgbe_sysctl_print_rss_config
   5913  ************************************************************************/
   5914 static int
   5915 ixgbe_sysctl_print_rss_config(SYSCTLFN_ARGS)
   5916 {
   5917 #ifdef notyet
   5918 	struct sysctlnode node = *rnode;
   5919 	struct ixgbe_softc *sc = (struct ixgbe_softc *)node.sysctl_data;
   5920 	struct ixgbe_hw *hw = &sc->hw;
   5921 	device_t	dev = sc->dev;
   5922 	struct sbuf	*buf;
   5923 	int		error = 0, reta_size;
   5924 	u32		reg;
   5925 
   5926 	if (ixgbe_fw_recovery_mode_swflag(sc))
   5927 		return (EPERM);
   5928 
   5929 	buf = sbuf_new_for_sysctl(NULL, NULL, 128, req);
   5930 	if (!buf) {
   5931 		device_printf(dev, "Could not allocate sbuf for output.\n");
   5932 		return (ENOMEM);
   5933 	}
   5934 
   5935 	// TODO: use sbufs to make a string to print out
   5936 	/* Set multiplier for RETA setup and table size based on MAC */
   5937 	switch (sc->hw.mac.type) {
   5938 	case ixgbe_mac_X550:
   5939 	case ixgbe_mac_X550EM_x:
   5940 	case ixgbe_mac_X550EM_a:
   5941 		reta_size = 128;
   5942 		break;
   5943 	default:
   5944 		reta_size = 32;
   5945 		break;
   5946 	}
   5947 
   5948 	/* Print out the redirection table */
   5949 	sbuf_cat(buf, "\n");
   5950 	for (int i = 0; i < reta_size; i++) {
   5951 		if (i < 32) {
   5952 			reg = IXGBE_READ_REG(hw, IXGBE_RETA(i));
   5953 			sbuf_printf(buf, "RETA(%2d): 0x%08x\n", i, reg);
   5954 		} else {
   5955 			reg = IXGBE_READ_REG(hw, IXGBE_ERETA(i - 32));
   5956 			sbuf_printf(buf, "ERETA(%2d): 0x%08x\n", i - 32, reg);
   5957 		}
   5958 	}
   5959 
   5960 	// TODO: print more config
   5961 
   5962 	error = sbuf_finish(buf);
   5963 	if (error)
   5964 		device_printf(dev, "Error finishing sbuf: %d\n", error);
   5965 
   5966 	sbuf_delete(buf);
   5967 #endif
   5968 	return (0);
   5969 } /* ixgbe_sysctl_print_rss_config */
   5970 #endif /* IXGBE_DEBUG */
   5971 
   5972 /************************************************************************
   5973  * ixgbe_sysctl_phy_temp - Retrieve temperature of PHY
   5974  *
   5975  *   For X552/X557-AT devices using an external PHY
   5976  ************************************************************************/
   5977 static int
   5978 ixgbe_sysctl_phy_temp(SYSCTLFN_ARGS)
   5979 {
   5980 	struct sysctlnode node = *rnode;
   5981 	struct ixgbe_softc *sc = (struct ixgbe_softc *)node.sysctl_data;
   5982 	struct ixgbe_hw *hw = &sc->hw;
   5983 	int val;
   5984 	u16 reg;
   5985 	int		error;
   5986 
   5987 	if (ixgbe_fw_recovery_mode_swflag(sc))
   5988 		return (EPERM);
   5989 
   5990 	if ((hw->device_id != IXGBE_DEV_ID_X550EM_X_10G_T) &&
   5991 	    (hw->device_id != IXGBE_DEV_ID_X550EM_A_10G_T)) {
   5992 		device_printf(sc->dev,
   5993 		    "Device has no supported external thermal sensor.\n");
   5994 		return (ENODEV);
   5995 	}
   5996 
   5997 	if (hw->phy.ops.read_reg(hw, IXGBE_PHY_CURRENT_TEMP,
   5998 		IXGBE_MDIO_VENDOR_SPECIFIC_1_DEV_TYPE, &reg)) {
   5999 		device_printf(sc->dev,
   6000 		    "Error reading from PHY's current temperature register\n");
   6001 		return (EAGAIN);
   6002 	}
   6003 
   6004 	node.sysctl_data = &val;
   6005 
   6006 	/* Shift temp for output */
   6007 	val = reg >> 8;
   6008 
   6009 	error = sysctl_lookup(SYSCTLFN_CALL(&node));
   6010 	if ((error) || (newp == NULL))
   6011 		return (error);
   6012 
   6013 	return (0);
   6014 } /* ixgbe_sysctl_phy_temp */
   6015 
   6016 /************************************************************************
   6017  * ixgbe_sysctl_phy_overtemp_occurred
   6018  *
   6019  *   Reports (directly from the PHY) whether the current PHY
   6020  *   temperature is over the overtemp threshold.
   6021  ************************************************************************/
   6022 static int
   6023 ixgbe_sysctl_phy_overtemp_occurred(SYSCTLFN_ARGS)
   6024 {
   6025 	struct sysctlnode node = *rnode;
   6026 	struct ixgbe_softc *sc = (struct ixgbe_softc *)node.sysctl_data;
   6027 	struct ixgbe_hw *hw = &sc->hw;
   6028 	int val, error;
   6029 	u16 reg;
   6030 
   6031 	if (ixgbe_fw_recovery_mode_swflag(sc))
   6032 		return (EPERM);
   6033 
   6034 	if ((hw->device_id != IXGBE_DEV_ID_X550EM_X_10G_T) &&
   6035 	    (hw->device_id != IXGBE_DEV_ID_X550EM_A_10G_T)) {
   6036 		device_printf(sc->dev,
   6037 		    "Device has no supported external thermal sensor.\n");
   6038 		return (ENODEV);
   6039 	}
   6040 
   6041 	if (hw->phy.ops.read_reg(hw, IXGBE_PHY_OVERTEMP_STATUS,
   6042 		IXGBE_MDIO_VENDOR_SPECIFIC_1_DEV_TYPE, &reg)) {
   6043 		device_printf(sc->dev,
   6044 		    "Error reading from PHY's temperature status register\n");
   6045 		return (EAGAIN);
   6046 	}
   6047 
   6048 	node.sysctl_data = &val;
   6049 
   6050 	/* Get occurrence bit */
   6051 	val = !!(reg & 0x4000);
   6052 
   6053 	error = sysctl_lookup(SYSCTLFN_CALL(&node));
   6054 	if ((error) || (newp == NULL))
   6055 		return (error);
   6056 
   6057 	return (0);
   6058 } /* ixgbe_sysctl_phy_overtemp_occurred */
   6059 
   6060 /************************************************************************
   6061  * ixgbe_sysctl_eee_state
   6062  *
   6063  *   Sysctl to set EEE power saving feature
   6064  *   Values:
   6065  *     0      - disable EEE
   6066  *     1      - enable EEE
   6067  *     (none) - get current device EEE state
   6068  ************************************************************************/
   6069 static int
   6070 ixgbe_sysctl_eee_state(SYSCTLFN_ARGS)
   6071 {
   6072 	struct sysctlnode node = *rnode;
   6073 	struct ixgbe_softc *sc = (struct ixgbe_softc *)node.sysctl_data;
   6074 	struct ifnet   *ifp = sc->ifp;
   6075 	device_t       dev = sc->dev;
   6076 	int	       curr_eee, new_eee, error = 0;
   6077 	s32	       retval;
   6078 
   6079 	if (ixgbe_fw_recovery_mode_swflag(sc))
   6080 		return (EPERM);
   6081 
   6082 	curr_eee = new_eee = !!(sc->feat_en & IXGBE_FEATURE_EEE);
   6083 	node.sysctl_data = &new_eee;
   6084 	error = sysctl_lookup(SYSCTLFN_CALL(&node));
   6085 	if ((error) || (newp == NULL))
   6086 		return (error);
   6087 
   6088 	/* Nothing to do */
   6089 	if (new_eee == curr_eee)
   6090 		return (0);
   6091 
   6092 	/* Not supported */
   6093 	if (!(sc->feat_cap & IXGBE_FEATURE_EEE))
   6094 		return (EINVAL);
   6095 
   6096 	/* Bounds checking */
   6097 	if ((new_eee < 0) || (new_eee > 1))
   6098 		return (EINVAL);
   6099 
   6100 	retval = ixgbe_setup_eee(&sc->hw, new_eee);
   6101 	if (retval) {
   6102 		device_printf(dev, "Error in EEE setup: 0x%08X\n", retval);
   6103 		return (EINVAL);
   6104 	}
   6105 
   6106 	/* Restart auto-neg */
   6107 	if_init(ifp);
   6108 
   6109 	device_printf(dev, "New EEE state: %d\n", new_eee);
   6110 
   6111 	/* Cache new value */
   6112 	if (new_eee)
   6113 		sc->feat_en |= IXGBE_FEATURE_EEE;
   6114 	else
   6115 		sc->feat_en &= ~IXGBE_FEATURE_EEE;
   6116 
   6117 	return (error);
   6118 } /* ixgbe_sysctl_eee_state */
   6119 
   6120 #define PRINTQS(sc, regname)						\
   6121 	do {								\
   6122 		struct ixgbe_hw	*_hw = &(sc)->hw;			\
   6123 		int _i;							\
   6124 									\
   6125 		printf("%s: %s", device_xname((sc)->dev), #regname);	\
   6126 		for (_i = 0; _i < (sc)->num_queues; _i++) {		\
   6127 			printf((_i == 0) ? "\t" : " ");			\
   6128 			printf("%08x", IXGBE_READ_REG(_hw,		\
   6129 				IXGBE_##regname(_i)));			\
   6130 		}							\
   6131 		printf("\n");						\
   6132 	} while (0)
   6133 
   6134 /************************************************************************
   6135  * ixgbe_print_debug_info
   6136  *
   6137  *   Called only when em_display_debug_stats is enabled.
   6138  *   Provides a way to take a look at important statistics
   6139  *   maintained by the driver and hardware.
   6140  ************************************************************************/
   6141 static void
   6142 ixgbe_print_debug_info(struct ixgbe_softc *sc)
   6143 {
   6144 	device_t	dev = sc->dev;
   6145 	struct ixgbe_hw *hw = &sc->hw;
   6146 	int table_size;
   6147 	int i;
   6148 
   6149 	switch (sc->hw.mac.type) {
   6150 	case ixgbe_mac_X550:
   6151 	case ixgbe_mac_X550EM_x:
   6152 	case ixgbe_mac_X550EM_a:
   6153 		table_size = 128;
   6154 		break;
   6155 	default:
   6156 		table_size = 32;
   6157 		break;
   6158 	}
   6159 
   6160 	device_printf(dev, "[E]RETA:\n");
   6161 	for (i = 0; i < table_size; i++) {
   6162 		if (i < 32)
   6163 			printf("%02x: %08x\n", i, IXGBE_READ_REG(hw,
   6164 				IXGBE_RETA(i)));
   6165 		else
   6166 			printf("%02x: %08x\n", i, IXGBE_READ_REG(hw,
   6167 				IXGBE_ERETA(i - 32)));
   6168 	}
   6169 
   6170 	device_printf(dev, "queue:");
   6171 	for (i = 0; i < sc->num_queues; i++) {
   6172 		printf((i == 0) ? "\t" : " ");
   6173 		printf("%8d", i);
   6174 	}
   6175 	printf("\n");
   6176 	PRINTQS(sc, RDBAL);
   6177 	PRINTQS(sc, RDBAH);
   6178 	PRINTQS(sc, RDLEN);
   6179 	PRINTQS(sc, SRRCTL);
   6180 	PRINTQS(sc, RDH);
   6181 	PRINTQS(sc, RDT);
   6182 	PRINTQS(sc, RXDCTL);
   6183 
   6184 	device_printf(dev, "RQSMR:");
   6185 	for (i = 0; i < sc->num_queues / 4; i++) {
   6186 		printf((i == 0) ? "\t" : " ");
   6187 		printf("%08x", IXGBE_READ_REG(hw, IXGBE_RQSMR(i)));
   6188 	}
   6189 	printf("\n");
   6190 
   6191 	device_printf(dev, "disabled_count:");
   6192 	for (i = 0; i < sc->num_queues; i++) {
   6193 		printf((i == 0) ? "\t" : " ");
   6194 		printf("%8d", sc->queues[i].disabled_count);
   6195 	}
   6196 	printf("\n");
   6197 
   6198 	device_printf(dev, "EIMS:\t%08x\n", IXGBE_READ_REG(hw, IXGBE_EIMS));
   6199 	if (hw->mac.type != ixgbe_mac_82598EB) {
   6200 		device_printf(dev, "EIMS_EX(0):\t%08x\n",
   6201 			      IXGBE_READ_REG(hw, IXGBE_EIMS_EX(0)));
   6202 		device_printf(dev, "EIMS_EX(1):\t%08x\n",
   6203 			      IXGBE_READ_REG(hw, IXGBE_EIMS_EX(1)));
   6204 	}
   6205 	device_printf(dev, "EIAM:\t%08x\n", IXGBE_READ_REG(hw, IXGBE_EIAM));
   6206 	device_printf(dev, "EIAC:\t%08x\n", IXGBE_READ_REG(hw, IXGBE_EIAC));
   6207 } /* ixgbe_print_debug_info */
   6208 
   6209 /************************************************************************
   6210  * ixgbe_sysctl_debug
   6211  ************************************************************************/
   6212 static int
   6213 ixgbe_sysctl_debug(SYSCTLFN_ARGS)
   6214 {
   6215 	struct sysctlnode node = *rnode;
   6216 	struct ixgbe_softc *sc = (struct ixgbe_softc *)node.sysctl_data;
   6217 	int	       error, result = 0;
   6218 
   6219 	if (ixgbe_fw_recovery_mode_swflag(sc))
   6220 		return (EPERM);
   6221 
   6222 	node.sysctl_data = &result;
   6223 	error = sysctl_lookup(SYSCTLFN_CALL(&node));
   6224 
   6225 	if (error || newp == NULL)
   6226 		return error;
   6227 
   6228 	if (result == 1)
   6229 		ixgbe_print_debug_info(sc);
   6230 
   6231 	return 0;
   6232 } /* ixgbe_sysctl_debug */
   6233 
   6234 /************************************************************************
   6235  * ixgbe_sysctl_rx_copy_len
   6236  ************************************************************************/
   6237 static int
   6238 ixgbe_sysctl_rx_copy_len(SYSCTLFN_ARGS)
   6239 {
   6240 	struct sysctlnode node = *rnode;
   6241 	struct ixgbe_softc *sc = (struct ixgbe_softc *)node.sysctl_data;
   6242 	int error;
   6243 	int result = sc->rx_copy_len;
   6244 
   6245 	node.sysctl_data = &result;
   6246 	error = sysctl_lookup(SYSCTLFN_CALL(&node));
   6247 
   6248 	if (error || newp == NULL)
   6249 		return error;
   6250 
   6251 	if ((result < 0) || (result > IXGBE_RX_COPY_LEN_MAX))
   6252 		return EINVAL;
   6253 
   6254 	sc->rx_copy_len = result;
   6255 
   6256 	return 0;
   6257 } /* ixgbe_sysctl_rx_copy_len */
   6258 
   6259 /************************************************************************
   6260  * ixgbe_sysctl_tx_process_limit
   6261  ************************************************************************/
   6262 static int
   6263 ixgbe_sysctl_tx_process_limit(SYSCTLFN_ARGS)
   6264 {
   6265 	struct sysctlnode node = *rnode;
   6266 	struct ixgbe_softc *sc = (struct ixgbe_softc *)node.sysctl_data;
   6267 	int error;
   6268 	int result = sc->tx_process_limit;
   6269 
   6270 	node.sysctl_data = &result;
   6271 	error = sysctl_lookup(SYSCTLFN_CALL(&node));
   6272 
   6273 	if (error || newp == NULL)
   6274 		return error;
   6275 
   6276 	if ((result <= 0) || (result > sc->num_tx_desc))
   6277 		return EINVAL;
   6278 
   6279 	sc->tx_process_limit = result;
   6280 
   6281 	return 0;
   6282 } /* ixgbe_sysctl_tx_process_limit */
   6283 
   6284 /************************************************************************
   6285  * ixgbe_sysctl_rx_process_limit
   6286  ************************************************************************/
   6287 static int
   6288 ixgbe_sysctl_rx_process_limit(SYSCTLFN_ARGS)
   6289 {
   6290 	struct sysctlnode node = *rnode;
   6291 	struct ixgbe_softc *sc = (struct ixgbe_softc *)node.sysctl_data;
   6292 	int error;
   6293 	int result = sc->rx_process_limit;
   6294 
   6295 	node.sysctl_data = &result;
   6296 	error = sysctl_lookup(SYSCTLFN_CALL(&node));
   6297 
   6298 	if (error || newp == NULL)
   6299 		return error;
   6300 
   6301 	if ((result <= 0) || (result > sc->num_rx_desc))
   6302 		return EINVAL;
   6303 
   6304 	sc->rx_process_limit = result;
   6305 
   6306 	return 0;
   6307 } /* ixgbe_sysctl_rx_process_limit */
   6308 
   6309 /************************************************************************
   6310  * ixgbe_init_device_features
   6311  ************************************************************************/
   6312 static void
   6313 ixgbe_init_device_features(struct ixgbe_softc *sc)
   6314 {
   6315 	sc->feat_cap = IXGBE_FEATURE_NETMAP
   6316 			  | IXGBE_FEATURE_RSS
   6317 			  | IXGBE_FEATURE_MSI
   6318 			  | IXGBE_FEATURE_MSIX
   6319 			  | IXGBE_FEATURE_LEGACY_IRQ
   6320 			  | IXGBE_FEATURE_LEGACY_TX;
   6321 
   6322 	/* Set capabilities first... */
   6323 	switch (sc->hw.mac.type) {
   6324 	case ixgbe_mac_82598EB:
   6325 		if (sc->hw.device_id == IXGBE_DEV_ID_82598AT)
   6326 			sc->feat_cap |= IXGBE_FEATURE_FAN_FAIL;
   6327 		break;
   6328 	case ixgbe_mac_X540:
   6329 		sc->feat_cap |= IXGBE_FEATURE_SRIOV;
   6330 		sc->feat_cap |= IXGBE_FEATURE_FDIR;
   6331 		if ((sc->hw.device_id == IXGBE_DEV_ID_X540_BYPASS) &&
   6332 		    (sc->hw.bus.func == 0))
   6333 			sc->feat_cap |= IXGBE_FEATURE_BYPASS;
   6334 		break;
   6335 	case ixgbe_mac_X550:
   6336 		/*
   6337 		 * IXGBE_FEATURE_RECOVERY_MODE will be set after reading
   6338 		 * NVM Image version.
   6339 		 */
   6340 		sc->feat_cap |= IXGBE_FEATURE_TEMP_SENSOR;
   6341 		sc->feat_cap |= IXGBE_FEATURE_SRIOV;
   6342 		sc->feat_cap |= IXGBE_FEATURE_FDIR;
   6343 		break;
   6344 	case ixgbe_mac_X550EM_x:
   6345 		/*
   6346 		 * IXGBE_FEATURE_RECOVERY_MODE will be set after reading
   6347 		 * NVM Image version.
   6348 		 */
   6349 		sc->feat_cap |= IXGBE_FEATURE_SRIOV;
   6350 		sc->feat_cap |= IXGBE_FEATURE_FDIR;
   6351 		break;
   6352 	case ixgbe_mac_X550EM_a:
   6353 		/*
   6354 		 * IXGBE_FEATURE_RECOVERY_MODE will be set after reading
   6355 		 * NVM Image version.
   6356 		 */
   6357 		sc->feat_cap |= IXGBE_FEATURE_SRIOV;
   6358 		sc->feat_cap |= IXGBE_FEATURE_FDIR;
   6359 		sc->feat_cap &= ~IXGBE_FEATURE_LEGACY_IRQ;
   6360 		if ((sc->hw.device_id == IXGBE_DEV_ID_X550EM_A_1G_T) ||
   6361 		    (sc->hw.device_id == IXGBE_DEV_ID_X550EM_A_1G_T_L)) {
   6362 			sc->feat_cap |= IXGBE_FEATURE_TEMP_SENSOR;
   6363 			sc->feat_cap |= IXGBE_FEATURE_EEE;
   6364 		}
   6365 		break;
   6366 	case ixgbe_mac_82599EB:
   6367 		sc->feat_cap |= IXGBE_FEATURE_SRIOV;
   6368 		sc->feat_cap |= IXGBE_FEATURE_FDIR;
   6369 		if ((sc->hw.device_id == IXGBE_DEV_ID_82599_BYPASS) &&
   6370 		    (sc->hw.bus.func == 0))
   6371 			sc->feat_cap |= IXGBE_FEATURE_BYPASS;
   6372 		if (sc->hw.device_id == IXGBE_DEV_ID_82599_QSFP_SF_QP)
   6373 			sc->feat_cap &= ~IXGBE_FEATURE_LEGACY_IRQ;
   6374 		break;
   6375 	default:
   6376 		break;
   6377 	}
   6378 
   6379 	/* Enabled by default... */
   6380 	/* Fan failure detection */
   6381 	if (sc->feat_cap & IXGBE_FEATURE_FAN_FAIL)
   6382 		sc->feat_en |= IXGBE_FEATURE_FAN_FAIL;
   6383 	/* Netmap */
   6384 	if (sc->feat_cap & IXGBE_FEATURE_NETMAP)
   6385 		sc->feat_en |= IXGBE_FEATURE_NETMAP;
   6386 	/* EEE */
   6387 	if (sc->feat_cap & IXGBE_FEATURE_EEE)
   6388 		sc->feat_en |= IXGBE_FEATURE_EEE;
   6389 	/* Thermal Sensor */
   6390 	if (sc->feat_cap & IXGBE_FEATURE_TEMP_SENSOR)
   6391 		sc->feat_en |= IXGBE_FEATURE_TEMP_SENSOR;
   6392 	/*
   6393 	 * Recovery mode:
   6394 	 * NetBSD: IXGBE_FEATURE_RECOVERY_MODE will be controlled after reading
   6395 	 * NVM Image version.
   6396 	 */
   6397 
   6398 	/* Enabled via global sysctl... */
   6399 	/* Flow Director */
   6400 	if (ixgbe_enable_fdir) {
   6401 		if (sc->feat_cap & IXGBE_FEATURE_FDIR)
   6402 			sc->feat_en |= IXGBE_FEATURE_FDIR;
   6403 		else
   6404 			device_printf(sc->dev, "Device does not support "
   6405 			    "Flow Director. Leaving disabled.");
   6406 	}
   6407 	/* Legacy (single queue) transmit */
   6408 	if ((sc->feat_cap & IXGBE_FEATURE_LEGACY_TX) &&
   6409 	    ixgbe_enable_legacy_tx)
   6410 		sc->feat_en |= IXGBE_FEATURE_LEGACY_TX;
   6411 	/*
   6412 	 * Message Signal Interrupts - Extended (MSI-X)
   6413 	 * Normal MSI is only enabled if MSI-X calls fail.
   6414 	 */
   6415 	if (!ixgbe_enable_msix)
   6416 		sc->feat_cap &= ~IXGBE_FEATURE_MSIX;
   6417 	/* Receive-Side Scaling (RSS) */
   6418 	if ((sc->feat_cap & IXGBE_FEATURE_RSS) && ixgbe_enable_rss)
   6419 		sc->feat_en |= IXGBE_FEATURE_RSS;
   6420 
   6421 	/* Disable features with unmet dependencies... */
   6422 	/* No MSI-X */
   6423 	if (!(sc->feat_cap & IXGBE_FEATURE_MSIX)) {
   6424 		sc->feat_cap &= ~IXGBE_FEATURE_RSS;
   6425 		sc->feat_cap &= ~IXGBE_FEATURE_SRIOV;
   6426 		sc->feat_en &= ~IXGBE_FEATURE_RSS;
   6427 		sc->feat_en &= ~IXGBE_FEATURE_SRIOV;
   6428 	}
   6429 } /* ixgbe_init_device_features */
   6430 
   6431 /************************************************************************
   6432  * ixgbe_probe - Device identification routine
   6433  *
   6434  *   Determines if the driver should be loaded on
   6435  *   adapter based on its PCI vendor/device ID.
   6436  *
   6437  *   return BUS_PROBE_DEFAULT on success, positive on failure
   6438  ************************************************************************/
   6439 static int
   6440 ixgbe_probe(device_t dev, cfdata_t cf, void *aux)
   6441 {
   6442 	const struct pci_attach_args *pa = aux;
   6443 
   6444 	return (ixgbe_lookup(pa) != NULL) ? 1 : 0;
   6445 }
   6446 
   6447 static const ixgbe_vendor_info_t *
   6448 ixgbe_lookup(const struct pci_attach_args *pa)
   6449 {
   6450 	const ixgbe_vendor_info_t *ent;
   6451 	pcireg_t subid;
   6452 
   6453 	INIT_DEBUGOUT("ixgbe_lookup: begin");
   6454 
   6455 	if (PCI_VENDOR(pa->pa_id) != IXGBE_INTEL_VENDOR_ID)
   6456 		return NULL;
   6457 
   6458 	subid = pci_conf_read(pa->pa_pc, pa->pa_tag, PCI_SUBSYS_ID_REG);
   6459 
   6460 	for (ent = ixgbe_vendor_info_array; ent->vendor_id != 0; ent++) {
   6461 		if ((PCI_VENDOR(pa->pa_id) == ent->vendor_id) &&
   6462 		    (PCI_PRODUCT(pa->pa_id) == ent->device_id) &&
   6463 		    ((PCI_SUBSYS_VENDOR(subid) == ent->subvendor_id) ||
   6464 			(ent->subvendor_id == 0)) &&
   6465 		    ((PCI_SUBSYS_ID(subid) == ent->subdevice_id) ||
   6466 			(ent->subdevice_id == 0))) {
   6467 			return ent;
   6468 		}
   6469 	}
   6470 	return NULL;
   6471 }
   6472 
   6473 static int
   6474 ixgbe_ifflags_cb(struct ethercom *ec)
   6475 {
   6476 	struct ifnet *ifp = &ec->ec_if;
   6477 	struct ixgbe_softc *sc = ifp->if_softc;
   6478 	u_short change;
   6479 	int rv = 0;
   6480 
   6481 	IXGBE_CORE_LOCK(sc);
   6482 
   6483 	change = ifp->if_flags ^ sc->if_flags;
   6484 	if (change != 0)
   6485 		sc->if_flags = ifp->if_flags;
   6486 
   6487 	if ((change & ~(IFF_CANTCHANGE | IFF_DEBUG)) != 0) {
   6488 		rv = ENETRESET;
   6489 		goto out;
   6490 	} else if ((change & IFF_PROMISC) != 0)
   6491 		ixgbe_set_rxfilter(sc);
   6492 
   6493 	/* Check for ec_capenable. */
   6494 	change = ec->ec_capenable ^ sc->ec_capenable;
   6495 	sc->ec_capenable = ec->ec_capenable;
   6496 	if ((change & ~(ETHERCAP_VLAN_MTU | ETHERCAP_VLAN_HWTAGGING
   6497 	    | ETHERCAP_VLAN_HWFILTER)) != 0) {
   6498 		rv = ENETRESET;
   6499 		goto out;
   6500 	}
   6501 
   6502 	/*
   6503 	 * Special handling is not required for ETHERCAP_VLAN_MTU.
   6504 	 * MAXFRS(MHADD) does not include the 4bytes of the VLAN header.
   6505 	 */
   6506 
   6507 	/* Set up VLAN support and filter */
   6508 	if ((change & (ETHERCAP_VLAN_HWTAGGING | ETHERCAP_VLAN_HWFILTER)) != 0)
   6509 		ixgbe_setup_vlan_hw_support(sc);
   6510 
   6511 out:
   6512 	IXGBE_CORE_UNLOCK(sc);
   6513 
   6514 	return rv;
   6515 }
   6516 
   6517 /************************************************************************
   6518  * ixgbe_ioctl - Ioctl entry point
   6519  *
   6520  *   Called when the user wants to configure the interface.
   6521  *
   6522  *   return 0 on success, positive on failure
   6523  ************************************************************************/
   6524 static int
   6525 ixgbe_ioctl(struct ifnet *ifp, u_long command, void *data)
   6526 {
   6527 	struct ixgbe_softc *sc = ifp->if_softc;
   6528 	struct ixgbe_hw *hw = &sc->hw;
   6529 	struct ifcapreq *ifcr = data;
   6530 	struct ifreq	*ifr = data;
   6531 	int		error = 0;
   6532 	int l4csum_en;
   6533 	const int l4csum = IFCAP_CSUM_TCPv4_Rx | IFCAP_CSUM_UDPv4_Rx |
   6534 	     IFCAP_CSUM_TCPv6_Rx | IFCAP_CSUM_UDPv6_Rx;
   6535 
   6536 	if (ixgbe_fw_recovery_mode_swflag(sc))
   6537 		return (EPERM);
   6538 
   6539 	switch (command) {
   6540 	case SIOCSIFFLAGS:
   6541 		IOCTL_DEBUGOUT("ioctl: SIOCSIFFLAGS (Set Interface Flags)");
   6542 		break;
   6543 	case SIOCADDMULTI:
   6544 	case SIOCDELMULTI:
   6545 		IOCTL_DEBUGOUT("ioctl: SIOC(ADD|DEL)MULTI");
   6546 		break;
   6547 	case SIOCSIFMEDIA:
   6548 	case SIOCGIFMEDIA:
   6549 		IOCTL_DEBUGOUT("ioctl: SIOCxIFMEDIA (Get/Set Interface Media)");
   6550 		break;
   6551 	case SIOCSIFCAP:
   6552 		IOCTL_DEBUGOUT("ioctl: SIOCSIFCAP (Set Capabilities)");
   6553 		break;
   6554 	case SIOCSIFMTU:
   6555 		IOCTL_DEBUGOUT("ioctl: SIOCSIFMTU (Set Interface MTU)");
   6556 		break;
   6557 #ifdef __NetBSD__
   6558 	case SIOCINITIFADDR:
   6559 		IOCTL_DEBUGOUT("ioctl: SIOCINITIFADDR");
   6560 		break;
   6561 	case SIOCGIFFLAGS:
   6562 		IOCTL_DEBUGOUT("ioctl: SIOCGIFFLAGS");
   6563 		break;
   6564 	case SIOCGIFAFLAG_IN:
   6565 		IOCTL_DEBUGOUT("ioctl: SIOCGIFAFLAG_IN");
   6566 		break;
   6567 	case SIOCGIFADDR:
   6568 		IOCTL_DEBUGOUT("ioctl: SIOCGIFADDR");
   6569 		break;
   6570 	case SIOCGIFMTU:
   6571 		IOCTL_DEBUGOUT("ioctl: SIOCGIFMTU (Get Interface MTU)");
   6572 		break;
   6573 	case SIOCGIFCAP:
   6574 		IOCTL_DEBUGOUT("ioctl: SIOCGIFCAP (Get IF cap)");
   6575 		break;
   6576 	case SIOCGETHERCAP:
   6577 		IOCTL_DEBUGOUT("ioctl: SIOCGETHERCAP (Get ethercap)");
   6578 		break;
   6579 	case SIOCGLIFADDR:
   6580 		IOCTL_DEBUGOUT("ioctl: SIOCGLIFADDR (Get Interface addr)");
   6581 		break;
   6582 	case SIOCZIFDATA:
   6583 		IOCTL_DEBUGOUT("ioctl: SIOCZIFDATA (Zero counter)");
   6584 		hw->mac.ops.clear_hw_cntrs(hw);
   6585 		ixgbe_clear_evcnt(sc);
   6586 		break;
   6587 	case SIOCAIFADDR:
   6588 		IOCTL_DEBUGOUT("ioctl: SIOCAIFADDR (add/chg IF alias)");
   6589 		break;
   6590 #endif
   6591 	default:
   6592 		IOCTL_DEBUGOUT1("ioctl: UNKNOWN (0x%X)", (int)command);
   6593 		break;
   6594 	}
   6595 
   6596 	switch (command) {
   6597 	case SIOCGI2C:
   6598 	{
   6599 		struct ixgbe_i2c_req	i2c;
   6600 
   6601 		IOCTL_DEBUGOUT("ioctl: SIOCGI2C (Get I2C Data)");
   6602 		error = copyin(ifr->ifr_data, &i2c, sizeof(i2c));
   6603 		if (error != 0)
   6604 			break;
   6605 		if (i2c.dev_addr != 0xA0 && i2c.dev_addr != 0xA2) {
   6606 			error = EINVAL;
   6607 			break;
   6608 		}
   6609 		if (i2c.len > sizeof(i2c.data)) {
   6610 			error = EINVAL;
   6611 			break;
   6612 		}
   6613 
   6614 		hw->phy.ops.read_i2c_byte(hw, i2c.offset,
   6615 		    i2c.dev_addr, i2c.data);
   6616 		error = copyout(&i2c, ifr->ifr_data, sizeof(i2c));
   6617 		break;
   6618 	}
   6619 	case SIOCSIFCAP:
   6620 		/* Layer-4 Rx checksum offload has to be turned on and
   6621 		 * off as a unit.
   6622 		 */
   6623 		l4csum_en = ifcr->ifcr_capenable & l4csum;
   6624 		if (l4csum_en != l4csum && l4csum_en != 0)
   6625 			return EINVAL;
   6626 		/*FALLTHROUGH*/
   6627 	case SIOCADDMULTI:
   6628 	case SIOCDELMULTI:
   6629 	case SIOCSIFFLAGS:
   6630 	case SIOCSIFMTU:
   6631 	default:
   6632 		if ((error = ether_ioctl(ifp, command, data)) != ENETRESET)
   6633 			return error;
   6634 		if ((ifp->if_flags & IFF_RUNNING) == 0)
   6635 			;
   6636 		else if (command == SIOCSIFCAP || command == SIOCSIFMTU) {
   6637 			IXGBE_CORE_LOCK(sc);
   6638 			if ((ifp->if_flags & IFF_RUNNING) != 0)
   6639 				ixgbe_init_locked(sc);
   6640 			ixgbe_recalculate_max_frame(sc);
   6641 			IXGBE_CORE_UNLOCK(sc);
   6642 		} else if (command == SIOCADDMULTI || command == SIOCDELMULTI) {
   6643 			/*
   6644 			 * Multicast list has changed; set the hardware filter
   6645 			 * accordingly.
   6646 			 */
   6647 			IXGBE_CORE_LOCK(sc);
   6648 			ixgbe_disable_intr(sc);
   6649 			ixgbe_set_rxfilter(sc);
   6650 			ixgbe_enable_intr(sc);
   6651 			IXGBE_CORE_UNLOCK(sc);
   6652 		}
   6653 		return 0;
   6654 	}
   6655 
   6656 	return error;
   6657 } /* ixgbe_ioctl */
   6658 
   6659 /************************************************************************
   6660  * ixgbe_check_fan_failure
   6661  ************************************************************************/
   6662 static int
   6663 ixgbe_check_fan_failure(struct ixgbe_softc *sc, u32 reg, bool in_interrupt)
   6664 {
   6665 	u32 mask;
   6666 
   6667 	mask = (in_interrupt) ? IXGBE_EICR_GPI_SDP1_BY_MAC(&sc->hw) :
   6668 	    IXGBE_ESDP_SDP1;
   6669 
   6670 	if ((reg & mask) == 0)
   6671 		return IXGBE_SUCCESS;
   6672 
   6673 	/*
   6674 	 * Use ratecheck() just in case interrupt occur frequently.
   6675 	 * When EXPX9501AT's fan stopped, interrupt occurred only once,
   6676 	 * an red LED on the board turned on and link never up until
   6677 	 * power off.
   6678 	 */
   6679 	if (ratecheck(&sc->lasterr_time, &ixgbe_errlog_intrvl))
   6680 		device_printf(sc->dev,
   6681 		    "\nCRITICAL: FAN FAILURE!! REPLACE IMMEDIATELY!!\n");
   6682 
   6683 	return IXGBE_ERR_FAN_FAILURE;
   6684 } /* ixgbe_check_fan_failure */
   6685 
   6686 /************************************************************************
   6687  * ixgbe_handle_que
   6688  ************************************************************************/
   6689 static void
   6690 ixgbe_handle_que(void *context)
   6691 {
   6692 	struct ix_queue *que = context;
   6693 	struct ixgbe_softc *sc = que->sc;
   6694 	struct tx_ring	*txr = que->txr;
   6695 	struct ifnet	*ifp = sc->ifp;
   6696 	bool		more = false;
   6697 
   6698 	IXGBE_EVC_ADD(&que->handleq, 1);
   6699 
   6700 	if (ifp->if_flags & IFF_RUNNING) {
   6701 		IXGBE_TX_LOCK(txr);
   6702 		more = ixgbe_txeof(txr);
   6703 		if (!(sc->feat_en & IXGBE_FEATURE_LEGACY_TX))
   6704 			if (!ixgbe_mq_ring_empty(ifp, txr->txr_interq))
   6705 				ixgbe_mq_start_locked(ifp, txr);
   6706 		/* Only for queue 0 */
   6707 		/* NetBSD still needs this for CBQ */
   6708 		if ((&sc->queues[0] == que)
   6709 		    && (!ixgbe_legacy_ring_empty(ifp, NULL)))
   6710 			ixgbe_legacy_start_locked(ifp, txr);
   6711 		IXGBE_TX_UNLOCK(txr);
   6712 		more |= ixgbe_rxeof(que);
   6713 	}
   6714 
   6715 	if (more) {
   6716 		IXGBE_EVC_ADD(&que->req, 1);
   6717 		ixgbe_sched_handle_que(sc, que);
   6718 	} else if (que->res != NULL) {
   6719 		/* MSIX: Re-enable this interrupt */
   6720 		ixgbe_enable_queue(sc, que->msix);
   6721 	} else {
   6722 		/* INTx or MSI */
   6723 		ixgbe_enable_queue(sc, 0);
   6724 	}
   6725 
   6726 	return;
   6727 } /* ixgbe_handle_que */
   6728 
   6729 /************************************************************************
   6730  * ixgbe_handle_que_work
   6731  ************************************************************************/
   6732 static void
   6733 ixgbe_handle_que_work(struct work *wk, void *context)
   6734 {
   6735 	struct ix_queue *que = container_of(wk, struct ix_queue, wq_cookie);
   6736 
   6737 	/*
   6738 	 * "enqueued flag" is not required here.
   6739 	 * See ixgbe_msix_que().
   6740 	 */
   6741 	ixgbe_handle_que(que);
   6742 }
   6743 
   6744 /************************************************************************
   6745  * ixgbe_allocate_legacy - Setup the Legacy or MSI Interrupt handler
   6746  ************************************************************************/
   6747 static int
   6748 ixgbe_allocate_legacy(struct ixgbe_softc *sc,
   6749     const struct pci_attach_args *pa)
   6750 {
   6751 	device_t	dev = sc->dev;
   6752 	struct ix_queue *que = sc->queues;
   6753 	struct tx_ring	*txr = sc->tx_rings;
   6754 	int		counts[PCI_INTR_TYPE_SIZE];
   6755 	pci_intr_type_t intr_type, max_type;
   6756 	char		intrbuf[PCI_INTRSTR_LEN];
   6757 	char		wqname[MAXCOMLEN];
   6758 	const char	*intrstr = NULL;
   6759 	int defertx_error = 0, error;
   6760 
   6761 	/* We allocate a single interrupt resource */
   6762 	max_type = PCI_INTR_TYPE_MSI;
   6763 	counts[PCI_INTR_TYPE_MSIX] = 0;
   6764 	counts[PCI_INTR_TYPE_MSI] =
   6765 	    (sc->feat_en & IXGBE_FEATURE_MSI) ? 1 : 0;
   6766 	/* Check not feat_en but feat_cap to fallback to INTx */
   6767 	counts[PCI_INTR_TYPE_INTX] =
   6768 	    (sc->feat_cap & IXGBE_FEATURE_LEGACY_IRQ) ? 1 : 0;
   6769 
   6770 alloc_retry:
   6771 	if (pci_intr_alloc(pa, &sc->osdep.intrs, counts, max_type) != 0) {
   6772 		aprint_error_dev(dev, "couldn't alloc interrupt\n");
   6773 		return ENXIO;
   6774 	}
   6775 	sc->osdep.nintrs = 1;
   6776 	intrstr = pci_intr_string(sc->osdep.pc, sc->osdep.intrs[0],
   6777 	    intrbuf, sizeof(intrbuf));
   6778 	sc->osdep.ihs[0] = pci_intr_establish_xname(sc->osdep.pc,
   6779 	    sc->osdep.intrs[0], IPL_NET, ixgbe_legacy_irq, que,
   6780 	    device_xname(dev));
   6781 	intr_type = pci_intr_type(sc->osdep.pc, sc->osdep.intrs[0]);
   6782 	if (sc->osdep.ihs[0] == NULL) {
   6783 		aprint_error_dev(dev,"unable to establish %s\n",
   6784 		    (intr_type == PCI_INTR_TYPE_MSI) ? "MSI" : "INTx");
   6785 		pci_intr_release(sc->osdep.pc, sc->osdep.intrs, 1);
   6786 		sc->osdep.intrs = NULL;
   6787 		switch (intr_type) {
   6788 		case PCI_INTR_TYPE_MSI:
   6789 			/* The next try is for INTx: Disable MSI */
   6790 			max_type = PCI_INTR_TYPE_INTX;
   6791 			counts[PCI_INTR_TYPE_INTX] = 1;
   6792 			sc->feat_en &= ~IXGBE_FEATURE_MSI;
   6793 			if (sc->feat_cap & IXGBE_FEATURE_LEGACY_IRQ) {
   6794 				sc->feat_en |= IXGBE_FEATURE_LEGACY_IRQ;
   6795 				goto alloc_retry;
   6796 			} else
   6797 				break;
   6798 		case PCI_INTR_TYPE_INTX:
   6799 		default:
   6800 			/* See below */
   6801 			break;
   6802 		}
   6803 	}
   6804 	if (intr_type == PCI_INTR_TYPE_INTX) {
   6805 		sc->feat_en &= ~IXGBE_FEATURE_MSI;
   6806 		sc->feat_en |= IXGBE_FEATURE_LEGACY_IRQ;
   6807 	}
   6808 	if (sc->osdep.ihs[0] == NULL) {
   6809 		aprint_error_dev(dev,
   6810 		    "couldn't establish interrupt%s%s\n",
   6811 		    intrstr ? " at " : "", intrstr ? intrstr : "");
   6812 		pci_intr_release(sc->osdep.pc, sc->osdep.intrs, 1);
   6813 		sc->osdep.intrs = NULL;
   6814 		return ENXIO;
   6815 	}
   6816 	aprint_normal_dev(dev, "interrupting at %s\n", intrstr);
   6817 	/*
   6818 	 * Try allocating a fast interrupt and the associated deferred
   6819 	 * processing contexts.
   6820 	 */
   6821 	if (!(sc->feat_en & IXGBE_FEATURE_LEGACY_TX)) {
   6822 		txr->txr_si =
   6823 		    softint_establish(SOFTINT_NET | SOFTINT_MPSAFE,
   6824 			ixgbe_deferred_mq_start, txr);
   6825 
   6826 		snprintf(wqname, sizeof(wqname), "%sdeferTx",
   6827 		    device_xname(dev));
   6828 		defertx_error = workqueue_create(&sc->txr_wq, wqname,
   6829 		    ixgbe_deferred_mq_start_work, sc, IXGBE_WORKQUEUE_PRI,
   6830 		    IPL_NET, WQ_PERCPU | WQ_MPSAFE);
   6831 		sc->txr_wq_enqueued = percpu_alloc(sizeof(u_int));
   6832 	}
   6833 	que->que_si = softint_establish(SOFTINT_NET | SOFTINT_MPSAFE,
   6834 	    ixgbe_handle_que, que);
   6835 	snprintf(wqname, sizeof(wqname), "%sTxRx", device_xname(dev));
   6836 	error = workqueue_create(&sc->que_wq, wqname,
   6837 	    ixgbe_handle_que_work, sc, IXGBE_WORKQUEUE_PRI, IPL_NET,
   6838 	    WQ_PERCPU | WQ_MPSAFE);
   6839 
   6840 	if ((!(sc->feat_en & IXGBE_FEATURE_LEGACY_TX)
   6841 		&& ((txr->txr_si == NULL) || defertx_error != 0))
   6842 	    || (que->que_si == NULL) || error != 0) {
   6843 		aprint_error_dev(dev,
   6844 		    "could not establish software interrupts\n");
   6845 
   6846 		return ENXIO;
   6847 	}
   6848 	/* For simplicity in the handlers */
   6849 	sc->active_queues = IXGBE_EIMS_ENABLE_MASK;
   6850 
   6851 	return (0);
   6852 } /* ixgbe_allocate_legacy */
   6853 
   6854 /************************************************************************
   6855  * ixgbe_allocate_msix - Setup MSI-X Interrupt resources and handlers
   6856  ************************************************************************/
   6857 static int
   6858 ixgbe_allocate_msix(struct ixgbe_softc *sc, const struct pci_attach_args *pa)
   6859 {
   6860 	device_t	dev = sc->dev;
   6861 	struct ix_queue	*que = sc->queues;
   6862 	struct tx_ring	*txr = sc->tx_rings;
   6863 	pci_chipset_tag_t pc;
   6864 	char		intrbuf[PCI_INTRSTR_LEN];
   6865 	char		intr_xname[32];
   6866 	char		wqname[MAXCOMLEN];
   6867 	const char	*intrstr = NULL;
   6868 	int		error, vector = 0;
   6869 	int		cpu_id = 0;
   6870 	kcpuset_t	*affinity;
   6871 #ifdef RSS
   6872 	unsigned int	rss_buckets = 0;
   6873 	kcpuset_t	cpu_mask;
   6874 #endif
   6875 
   6876 	pc = sc->osdep.pc;
   6877 #ifdef	RSS
   6878 	/*
   6879 	 * If we're doing RSS, the number of queues needs to
   6880 	 * match the number of RSS buckets that are configured.
   6881 	 *
   6882 	 * + If there's more queues than RSS buckets, we'll end
   6883 	 *   up with queues that get no traffic.
   6884 	 *
   6885 	 * + If there's more RSS buckets than queues, we'll end
   6886 	 *   up having multiple RSS buckets map to the same queue,
   6887 	 *   so there'll be some contention.
   6888 	 */
   6889 	rss_buckets = rss_getnumbuckets();
   6890 	if ((sc->feat_en & IXGBE_FEATURE_RSS) &&
   6891 	    (sc->num_queues != rss_buckets)) {
   6892 		device_printf(dev,
   6893 		    "%s: number of queues (%d) != number of RSS buckets (%d)"
   6894 		    "; performance will be impacted.\n",
   6895 		    __func__, sc->num_queues, rss_buckets);
   6896 	}
   6897 #endif
   6898 
   6899 	sc->osdep.nintrs = sc->num_queues + 1;
   6900 	if (pci_msix_alloc_exact(pa, &sc->osdep.intrs,
   6901 	    sc->osdep.nintrs) != 0) {
   6902 		aprint_error_dev(dev,
   6903 		    "failed to allocate MSI-X interrupt\n");
   6904 		sc->feat_en &= ~IXGBE_FEATURE_MSIX;
   6905 		return (ENXIO);
   6906 	}
   6907 
   6908 	kcpuset_create(&affinity, false);
   6909 	for (int i = 0; i < sc->num_queues; i++, vector++, que++, txr++) {
   6910 		snprintf(intr_xname, sizeof(intr_xname), "%s TXRX%d",
   6911 		    device_xname(dev), i);
   6912 		intrstr = pci_intr_string(pc, sc->osdep.intrs[i], intrbuf,
   6913 		    sizeof(intrbuf));
   6914 		pci_intr_setattr(pc, &sc->osdep.intrs[i], PCI_INTR_MPSAFE,
   6915 		    true);
   6916 
   6917 		/* Set the handler function */
   6918 		que->res = sc->osdep.ihs[i] = pci_intr_establish_xname(pc,
   6919 		    sc->osdep.intrs[i], IPL_NET, ixgbe_msix_que, que,
   6920 		    intr_xname);
   6921 		if (que->res == NULL) {
   6922 			aprint_error_dev(dev,
   6923 			    "Failed to register QUE handler\n");
   6924 			error = ENXIO;
   6925 			goto err_out;
   6926 		}
   6927 		que->msix = vector;
   6928 		sc->active_queues |= 1ULL << que->msix;
   6929 
   6930 		if (sc->feat_en & IXGBE_FEATURE_RSS) {
   6931 #ifdef	RSS
   6932 			/*
   6933 			 * The queue ID is used as the RSS layer bucket ID.
   6934 			 * We look up the queue ID -> RSS CPU ID and select
   6935 			 * that.
   6936 			 */
   6937 			cpu_id = rss_getcpu(i % rss_getnumbuckets());
   6938 			CPU_SETOF(cpu_id, &cpu_mask);
   6939 #endif
   6940 		} else {
   6941 			/*
   6942 			 * Bind the MSI-X vector, and thus the
   6943 			 * rings to the corresponding CPU.
   6944 			 *
   6945 			 * This just happens to match the default RSS
   6946 			 * round-robin bucket -> queue -> CPU allocation.
   6947 			 */
   6948 			if (sc->num_queues > 1)
   6949 				cpu_id = i;
   6950 		}
   6951 		/* Round-robin affinity */
   6952 		kcpuset_zero(affinity);
   6953 		kcpuset_set(affinity, cpu_id % ncpu);
   6954 		error = interrupt_distribute(sc->osdep.ihs[i], affinity,
   6955 		    NULL);
   6956 		aprint_normal_dev(dev, "for TX/RX, interrupting at %s",
   6957 		    intrstr);
   6958 		if (error == 0) {
   6959 #if 1 /* def IXGBE_DEBUG */
   6960 #ifdef	RSS
   6961 			aprint_normal(", bound RSS bucket %d to CPU %d", i,
   6962 			    cpu_id % ncpu);
   6963 #else
   6964 			aprint_normal(", bound queue %d to cpu %d", i,
   6965 			    cpu_id % ncpu);
   6966 #endif
   6967 #endif /* IXGBE_DEBUG */
   6968 		}
   6969 		aprint_normal("\n");
   6970 
   6971 		if (!(sc->feat_en & IXGBE_FEATURE_LEGACY_TX)) {
   6972 			txr->txr_si = softint_establish(
   6973 				SOFTINT_NET | SOFTINT_MPSAFE,
   6974 				ixgbe_deferred_mq_start, txr);
   6975 			if (txr->txr_si == NULL) {
   6976 				aprint_error_dev(dev,
   6977 				    "couldn't establish software interrupt\n");
   6978 				error = ENXIO;
   6979 				goto err_out;
   6980 			}
   6981 		}
   6982 		que->que_si
   6983 		    = softint_establish(SOFTINT_NET | SOFTINT_MPSAFE,
   6984 			ixgbe_handle_que, que);
   6985 		if (que->que_si == NULL) {
   6986 			aprint_error_dev(dev,
   6987 			    "couldn't establish software interrupt\n");
   6988 			error = ENXIO;
   6989 			goto err_out;
   6990 		}
   6991 	}
   6992 	snprintf(wqname, sizeof(wqname), "%sdeferTx", device_xname(dev));
   6993 	error = workqueue_create(&sc->txr_wq, wqname,
   6994 	    ixgbe_deferred_mq_start_work, sc, IXGBE_WORKQUEUE_PRI, IPL_NET,
   6995 	    WQ_PERCPU | WQ_MPSAFE);
   6996 	if (error) {
   6997 		aprint_error_dev(dev,
   6998 		    "couldn't create workqueue for deferred Tx\n");
   6999 		goto err_out;
   7000 	}
   7001 	sc->txr_wq_enqueued = percpu_alloc(sizeof(u_int));
   7002 
   7003 	snprintf(wqname, sizeof(wqname), "%sTxRx", device_xname(dev));
   7004 	error = workqueue_create(&sc->que_wq, wqname,
   7005 	    ixgbe_handle_que_work, sc, IXGBE_WORKQUEUE_PRI, IPL_NET,
   7006 	    WQ_PERCPU | WQ_MPSAFE);
   7007 	if (error) {
   7008 		aprint_error_dev(dev, "couldn't create workqueue for Tx/Rx\n");
   7009 		goto err_out;
   7010 	}
   7011 
   7012 	/* and Link */
   7013 	cpu_id++;
   7014 	snprintf(intr_xname, sizeof(intr_xname), "%s link", device_xname(dev));
   7015 	sc->vector = vector;
   7016 	intrstr = pci_intr_string(pc, sc->osdep.intrs[vector], intrbuf,
   7017 	    sizeof(intrbuf));
   7018 	pci_intr_setattr(pc, &sc->osdep.intrs[vector], PCI_INTR_MPSAFE, true);
   7019 
   7020 	/* Set the link handler function */
   7021 	sc->osdep.ihs[vector] = pci_intr_establish_xname(pc,
   7022 	    sc->osdep.intrs[vector], IPL_NET, ixgbe_msix_admin, sc,
   7023 	    intr_xname);
   7024 	if (sc->osdep.ihs[vector] == NULL) {
   7025 		aprint_error_dev(dev, "Failed to register LINK handler\n");
   7026 		error = ENXIO;
   7027 		goto err_out;
   7028 	}
   7029 	/* Round-robin affinity */
   7030 	kcpuset_zero(affinity);
   7031 	kcpuset_set(affinity, cpu_id % ncpu);
   7032 	error = interrupt_distribute(sc->osdep.ihs[vector], affinity,
   7033 	    NULL);
   7034 
   7035 	aprint_normal_dev(dev,
   7036 	    "for link, interrupting at %s", intrstr);
   7037 	if (error == 0)
   7038 		aprint_normal(", affinity to cpu %d\n", cpu_id % ncpu);
   7039 	else
   7040 		aprint_normal("\n");
   7041 
   7042 	kcpuset_destroy(affinity);
   7043 	aprint_normal_dev(dev,
   7044 	    "Using MSI-X interrupts with %d vectors\n", vector + 1);
   7045 
   7046 	return (0);
   7047 
   7048 err_out:
   7049 	kcpuset_destroy(affinity);
   7050 	ixgbe_free_deferred_handlers(sc);
   7051 	ixgbe_free_pciintr_resources(sc);
   7052 	return (error);
   7053 } /* ixgbe_allocate_msix */
   7054 
   7055 /************************************************************************
   7056  * ixgbe_configure_interrupts
   7057  *
   7058  *   Setup MSI-X, MSI, or legacy interrupts (in that order).
   7059  *   This will also depend on user settings.
   7060  ************************************************************************/
   7061 static int
   7062 ixgbe_configure_interrupts(struct ixgbe_softc *sc)
   7063 {
   7064 	device_t dev = sc->dev;
   7065 	struct ixgbe_mac_info *mac = &sc->hw.mac;
   7066 	int want, queues, msgs;
   7067 
   7068 	/* Default to 1 queue if MSI-X setup fails */
   7069 	sc->num_queues = 1;
   7070 
   7071 	/* Override by tuneable */
   7072 	if (!(sc->feat_cap & IXGBE_FEATURE_MSIX))
   7073 		goto msi;
   7074 
   7075 	/*
   7076 	 *  NetBSD only: Use single vector MSI when number of CPU is 1 to save
   7077 	 * interrupt slot.
   7078 	 */
   7079 	if (ncpu == 1)
   7080 		goto msi;
   7081 
   7082 	/* First try MSI-X */
   7083 	msgs = pci_msix_count(sc->osdep.pc, sc->osdep.tag);
   7084 	msgs = MIN(msgs, IXG_MAX_NINTR);
   7085 	if (msgs < 2)
   7086 		goto msi;
   7087 
   7088 	sc->feat_en |= IXGBE_FEATURE_MSIX;
   7089 
   7090 	/* Figure out a reasonable auto config value */
   7091 	queues = (ncpu > (msgs - 1)) ? (msgs - 1) : ncpu;
   7092 
   7093 #ifdef	RSS
   7094 	/* If we're doing RSS, clamp at the number of RSS buckets */
   7095 	if (sc->feat_en & IXGBE_FEATURE_RSS)
   7096 		queues = uimin(queues, rss_getnumbuckets());
   7097 #endif
   7098 	if (ixgbe_num_queues > queues) {
   7099 		aprint_error_dev(sc->dev,
   7100 		    "ixgbe_num_queues (%d) is too large, "
   7101 		    "using reduced amount (%d).\n", ixgbe_num_queues, queues);
   7102 		ixgbe_num_queues = queues;
   7103 	}
   7104 
   7105 	if (ixgbe_num_queues != 0)
   7106 		queues = ixgbe_num_queues;
   7107 	else
   7108 		queues = uimin(queues,
   7109 		    uimin(mac->max_tx_queues, mac->max_rx_queues));
   7110 
   7111 	/*
   7112 	 * Want one vector (RX/TX pair) per queue
   7113 	 * plus an additional for Link.
   7114 	 */
   7115 	want = queues + 1;
   7116 	if (msgs >= want)
   7117 		msgs = want;
   7118 	else {
   7119 		aprint_error_dev(dev, "MSI-X Configuration Problem, "
   7120 		    "%d vectors but %d queues wanted!\n", msgs, want);
   7121 		goto msi;
   7122 	}
   7123 	sc->num_queues = queues;
   7124 	sc->feat_en |= IXGBE_FEATURE_MSIX;
   7125 	return (0);
   7126 
   7127 	/*
   7128 	 * MSI-X allocation failed or provided us with
   7129 	 * less vectors than needed. Free MSI-X resources
   7130 	 * and we'll try enabling MSI.
   7131 	 */
   7132 msi:
   7133 	/* Without MSI-X, some features are no longer supported */
   7134 	sc->feat_cap &= ~IXGBE_FEATURE_RSS;
   7135 	sc->feat_en  &= ~IXGBE_FEATURE_RSS;
   7136 	sc->feat_cap &= ~IXGBE_FEATURE_SRIOV;
   7137 	sc->feat_en  &= ~IXGBE_FEATURE_SRIOV;
   7138 
   7139 	msgs = pci_msi_count(sc->osdep.pc, sc->osdep.tag);
   7140 	sc->feat_en &= ~IXGBE_FEATURE_MSIX;
   7141 	if (msgs > 1)
   7142 		msgs = 1;
   7143 	if (msgs != 0) {
   7144 		msgs = 1;
   7145 		sc->feat_en |= IXGBE_FEATURE_MSI;
   7146 		return (0);
   7147 	}
   7148 
   7149 	if (!(sc->feat_cap & IXGBE_FEATURE_LEGACY_IRQ)) {
   7150 		aprint_error_dev(dev,
   7151 		    "Device does not support legacy interrupts.\n");
   7152 		return 1;
   7153 	}
   7154 
   7155 	sc->feat_en |= IXGBE_FEATURE_LEGACY_IRQ;
   7156 
   7157 	return (0);
   7158 } /* ixgbe_configure_interrupts */
   7159 
   7160 
   7161 /************************************************************************
   7162  * ixgbe_handle_link - Tasklet for MSI-X Link interrupts
   7163  *
   7164  *   Done outside of interrupt context since the driver might sleep
   7165  ************************************************************************/
   7166 static void
   7167 ixgbe_handle_link(void *context)
   7168 {
   7169 	struct ixgbe_softc *sc = context;
   7170 	struct ixgbe_hw *hw = &sc->hw;
   7171 
   7172 	KASSERT(mutex_owned(&sc->core_mtx));
   7173 
   7174 	IXGBE_EVC_ADD(&sc->link_workev, 1);
   7175 	ixgbe_check_link(hw, &sc->link_speed, &sc->link_up, 0);
   7176 	ixgbe_update_link_status(sc);
   7177 
   7178 	/* Re-enable link interrupts */
   7179 	IXGBE_WRITE_REG(hw, IXGBE_EIMS, IXGBE_EIMS_LSC);
   7180 } /* ixgbe_handle_link */
   7181 
   7182 #if 0
   7183 /************************************************************************
   7184  * ixgbe_rearm_queues
   7185  ************************************************************************/
   7186 static __inline void
   7187 ixgbe_rearm_queues(struct ixgbe_softc *sc, u64 queues)
   7188 {
   7189 	u32 mask;
   7190 
   7191 	switch (sc->hw.mac.type) {
   7192 	case ixgbe_mac_82598EB:
   7193 		mask = (IXGBE_EIMS_RTX_QUEUE & queues);
   7194 		IXGBE_WRITE_REG(&sc->hw, IXGBE_EICS, mask);
   7195 		break;
   7196 	case ixgbe_mac_82599EB:
   7197 	case ixgbe_mac_X540:
   7198 	case ixgbe_mac_X550:
   7199 	case ixgbe_mac_X550EM_x:
   7200 	case ixgbe_mac_X550EM_a:
   7201 		mask = (queues & 0xFFFFFFFF);
   7202 		IXGBE_WRITE_REG(&sc->hw, IXGBE_EICS_EX(0), mask);
   7203 		mask = (queues >> 32);
   7204 		IXGBE_WRITE_REG(&sc->hw, IXGBE_EICS_EX(1), mask);
   7205 		break;
   7206 	default:
   7207 		break;
   7208 	}
   7209 } /* ixgbe_rearm_queues */
   7210 #endif
   7211