ixgbe.c revision 1.25
1 /****************************************************************************** 2 3 Copyright (c) 2001-2012, Intel Corporation 4 All rights reserved. 5 6 Redistribution and use in source and binary forms, with or without 7 modification, are permitted provided that the following conditions are met: 8 9 1. Redistributions of source code must retain the above copyright notice, 10 this list of conditions and the following disclaimer. 11 12 2. Redistributions in binary form must reproduce the above copyright 13 notice, this list of conditions and the following disclaimer in the 14 documentation and/or other materials provided with the distribution. 15 16 3. Neither the name of the Intel Corporation nor the names of its 17 contributors may be used to endorse or promote products derived from 18 this software without specific prior written permission. 19 20 THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 21 AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 22 IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 23 ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE 24 LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 25 CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 26 SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 27 INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 28 CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 29 ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 30 POSSIBILITY OF SUCH DAMAGE. 31 32 ******************************************************************************/ 33 /* 34 * Copyright (c) 2011 The NetBSD Foundation, Inc. 35 * All rights reserved. 36 * 37 * This code is derived from software contributed to The NetBSD Foundation 38 * by Coyote Point Systems, Inc. 39 * 40 * Redistribution and use in source and binary forms, with or without 41 * modification, are permitted provided that the following conditions 42 * are met: 43 * 1. Redistributions of source code must retain the above copyright 44 * notice, this list of conditions and the following disclaimer. 45 * 2. Redistributions in binary form must reproduce the above copyright 46 * notice, this list of conditions and the following disclaimer in the 47 * documentation and/or other materials provided with the distribution. 48 * 49 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS 50 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED 51 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 52 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS 53 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 54 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 55 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 56 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 57 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 58 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 59 * POSSIBILITY OF SUCH DAMAGE. 60 */ 61 /*$FreeBSD: head/sys/dev/ixgbe/ixgbe.c 238149 2012-07-05 20:51:44Z jfv $*/ 62 /*$NetBSD: ixgbe.c,v 1.25 2015/04/02 09:26:55 msaitoh Exp $*/ 63 64 #include "opt_inet.h" 65 #include "opt_inet6.h" 66 67 #include "ixgbe.h" 68 69 /********************************************************************* 70 * Set this to one to display debug statistics 71 *********************************************************************/ 72 int ixgbe_display_debug_stats = 0; 73 74 /********************************************************************* 75 * Driver version 76 *********************************************************************/ 77 char ixgbe_driver_version[] = "2.4.8"; 78 79 /********************************************************************* 80 * PCI Device ID Table 81 * 82 * Used by probe to select devices to load on 83 * Last field stores an index into ixgbe_strings 84 * Last entry must be all 0s 85 * 86 * { Vendor ID, Device ID, SubVendor ID, SubDevice ID, String Index } 87 *********************************************************************/ 88 89 static ixgbe_vendor_info_t ixgbe_vendor_info_array[] = 90 { 91 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598AF_DUAL_PORT, 0, 0, 0}, 92 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598AF_SINGLE_PORT, 0, 0, 0}, 93 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598EB_CX4, 0, 0, 0}, 94 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598AT, 0, 0, 0}, 95 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598AT2, 0, 0, 0}, 96 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598, 0, 0, 0}, 97 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598_DA_DUAL_PORT, 0, 0, 0}, 98 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598_CX4_DUAL_PORT, 0, 0, 0}, 99 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598EB_XF_LR, 0, 0, 0}, 100 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598_SR_DUAL_PORT_EM, 0, 0, 0}, 101 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598EB_SFP_LOM, 0, 0, 0}, 102 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_KX4, 0, 0, 0}, 103 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_KX4_MEZZ, 0, 0, 0}, 104 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_SFP, 0, 0, 0}, 105 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_XAUI_LOM, 0, 0, 0}, 106 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_CX4, 0, 0, 0}, 107 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_T3_LOM, 0, 0, 0}, 108 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_COMBO_BACKPLANE, 0, 0, 0}, 109 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_BACKPLANE_FCOE, 0, 0, 0}, 110 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_SFP_SF2, 0, 0, 0}, 111 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_SFP_FCOE, 0, 0, 0}, 112 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599EN_SFP, 0, 0, 0}, 113 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_SFP_SF_QP, 0, 0, 0}, 114 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_X540T1, 0, 0, 0}, 115 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_X540T, 0, 0, 0}, 116 /* required last entry */ 117 {0, 0, 0, 0, 0} 118 }; 119 120 /********************************************************************* 121 * Table of branding strings 122 *********************************************************************/ 123 124 static const char *ixgbe_strings[] = { 125 "Intel(R) PRO/10GbE PCI-Express Network Driver" 126 }; 127 128 /********************************************************************* 129 * Function prototypes 130 *********************************************************************/ 131 static int ixgbe_probe(device_t, cfdata_t, void *); 132 static void ixgbe_attach(device_t, device_t, void *); 133 static int ixgbe_detach(device_t, int); 134 #if 0 135 static int ixgbe_shutdown(device_t); 136 #endif 137 static void ixgbe_start(struct ifnet *); 138 static void ixgbe_start_locked(struct tx_ring *, struct ifnet *); 139 #if __FreeBSD_version >= 800000 140 static int ixgbe_mq_start(struct ifnet *, struct mbuf *); 141 static int ixgbe_mq_start_locked(struct ifnet *, 142 struct tx_ring *, struct mbuf *); 143 static void ixgbe_qflush(struct ifnet *); 144 #endif 145 static int ixgbe_ioctl(struct ifnet *, u_long, void *); 146 static void ixgbe_ifstop(struct ifnet *, int); 147 static int ixgbe_init(struct ifnet *); 148 static void ixgbe_init_locked(struct adapter *); 149 static void ixgbe_stop(void *); 150 static void ixgbe_media_status(struct ifnet *, struct ifmediareq *); 151 static int ixgbe_media_change(struct ifnet *); 152 static void ixgbe_identify_hardware(struct adapter *); 153 static int ixgbe_allocate_pci_resources(struct adapter *, 154 const struct pci_attach_args *); 155 static int ixgbe_allocate_msix(struct adapter *, 156 const struct pci_attach_args *); 157 static int ixgbe_allocate_legacy(struct adapter *, 158 const struct pci_attach_args *); 159 static int ixgbe_allocate_queues(struct adapter *); 160 static int ixgbe_setup_msix(struct adapter *); 161 static void ixgbe_free_pci_resources(struct adapter *); 162 static void ixgbe_local_timer(void *); 163 static int ixgbe_setup_interface(device_t, struct adapter *); 164 static void ixgbe_config_link(struct adapter *); 165 166 static int ixgbe_allocate_transmit_buffers(struct tx_ring *); 167 static int ixgbe_setup_transmit_structures(struct adapter *); 168 static void ixgbe_setup_transmit_ring(struct tx_ring *); 169 static void ixgbe_initialize_transmit_units(struct adapter *); 170 static void ixgbe_free_transmit_structures(struct adapter *); 171 static void ixgbe_free_transmit_buffers(struct tx_ring *); 172 173 static int ixgbe_allocate_receive_buffers(struct rx_ring *); 174 static int ixgbe_setup_receive_structures(struct adapter *); 175 static int ixgbe_setup_receive_ring(struct rx_ring *); 176 static void ixgbe_initialize_receive_units(struct adapter *); 177 static void ixgbe_free_receive_structures(struct adapter *); 178 static void ixgbe_free_receive_buffers(struct rx_ring *); 179 static void ixgbe_setup_hw_rsc(struct rx_ring *); 180 181 static void ixgbe_enable_intr(struct adapter *); 182 static void ixgbe_disable_intr(struct adapter *); 183 static void ixgbe_update_stats_counters(struct adapter *); 184 static bool ixgbe_txeof(struct tx_ring *); 185 static bool ixgbe_rxeof(struct ix_queue *, int); 186 static void ixgbe_rx_checksum(u32, struct mbuf *, u32, 187 struct ixgbe_hw_stats *); 188 static void ixgbe_set_promisc(struct adapter *); 189 static void ixgbe_set_multi(struct adapter *); 190 static void ixgbe_update_link_status(struct adapter *); 191 static void ixgbe_refresh_mbufs(struct rx_ring *, int); 192 static int ixgbe_xmit(struct tx_ring *, struct mbuf *); 193 static int ixgbe_set_flowcntl(SYSCTLFN_PROTO); 194 static int ixgbe_set_advertise(SYSCTLFN_PROTO); 195 static int ixgbe_set_thermal_test(SYSCTLFN_PROTO); 196 static int ixgbe_dma_malloc(struct adapter *, bus_size_t, 197 struct ixgbe_dma_alloc *, int); 198 static void ixgbe_dma_free(struct adapter *, struct ixgbe_dma_alloc *); 199 static void ixgbe_add_rx_process_limit(struct adapter *, const char *, 200 const char *, int *, int); 201 static u32 ixgbe_tx_ctx_setup(struct tx_ring *, struct mbuf *); 202 static bool ixgbe_tso_setup(struct tx_ring *, struct mbuf *, u32 *, u32 *); 203 static void ixgbe_set_ivar(struct adapter *, u8, u8, s8); 204 static void ixgbe_configure_ivars(struct adapter *); 205 static u8 * ixgbe_mc_array_itr(struct ixgbe_hw *, u8 **, u32 *); 206 207 static void ixgbe_setup_vlan_hw_support(struct adapter *); 208 #if 0 209 static void ixgbe_register_vlan(void *, struct ifnet *, u16); 210 static void ixgbe_unregister_vlan(void *, struct ifnet *, u16); 211 #endif 212 213 static void ixgbe_add_hw_stats(struct adapter *adapter); 214 215 static __inline void ixgbe_rx_discard(struct rx_ring *, int); 216 static __inline void ixgbe_rx_input(struct rx_ring *, struct ifnet *, 217 struct mbuf *, u32); 218 219 /* Support for pluggable optic modules */ 220 static bool ixgbe_sfp_probe(struct adapter *); 221 static void ixgbe_setup_optics(struct adapter *); 222 223 /* Legacy (single vector interrupt handler */ 224 static int ixgbe_legacy_irq(void *); 225 226 #if defined(NETBSD_MSI_OR_MSIX) 227 /* The MSI/X Interrupt handlers */ 228 static void ixgbe_msix_que(void *); 229 static void ixgbe_msix_link(void *); 230 #endif 231 232 /* Software interrupts for deferred work */ 233 static void ixgbe_handle_que(void *); 234 static void ixgbe_handle_link(void *); 235 static void ixgbe_handle_msf(void *); 236 static void ixgbe_handle_mod(void *); 237 238 const struct sysctlnode *ixgbe_sysctl_instance(struct adapter *); 239 static ixgbe_vendor_info_t *ixgbe_lookup(const struct pci_attach_args *); 240 241 #ifdef IXGBE_FDIR 242 static void ixgbe_atr(struct tx_ring *, struct mbuf *); 243 static void ixgbe_reinit_fdir(void *, int); 244 #endif 245 246 /********************************************************************* 247 * FreeBSD Device Interface Entry Points 248 *********************************************************************/ 249 250 CFATTACH_DECL3_NEW(ixg, sizeof(struct adapter), 251 ixgbe_probe, ixgbe_attach, ixgbe_detach, NULL, NULL, NULL, 252 DVF_DETACH_SHUTDOWN); 253 254 #if 0 255 devclass_t ixgbe_devclass; 256 DRIVER_MODULE(ixgbe, pci, ixgbe_driver, ixgbe_devclass, 0, 0); 257 258 MODULE_DEPEND(ixgbe, pci, 1, 1, 1); 259 MODULE_DEPEND(ixgbe, ether, 1, 1, 1); 260 #endif 261 262 /* 263 ** TUNEABLE PARAMETERS: 264 */ 265 266 /* 267 ** AIM: Adaptive Interrupt Moderation 268 ** which means that the interrupt rate 269 ** is varied over time based on the 270 ** traffic for that interrupt vector 271 */ 272 static int ixgbe_enable_aim = TRUE; 273 #define TUNABLE_INT(__x, __y) 274 TUNABLE_INT("hw.ixgbe.enable_aim", &ixgbe_enable_aim); 275 276 static int ixgbe_max_interrupt_rate = (4000000 / IXGBE_LOW_LATENCY); 277 TUNABLE_INT("hw.ixgbe.max_interrupt_rate", &ixgbe_max_interrupt_rate); 278 279 /* How many packets rxeof tries to clean at a time */ 280 static int ixgbe_rx_process_limit = 256; 281 TUNABLE_INT("hw.ixgbe.rx_process_limit", &ixgbe_rx_process_limit); 282 283 /* 284 ** Smart speed setting, default to on 285 ** this only works as a compile option 286 ** right now as its during attach, set 287 ** this to 'ixgbe_smart_speed_off' to 288 ** disable. 289 */ 290 static int ixgbe_smart_speed = ixgbe_smart_speed_on; 291 292 /* 293 * MSIX should be the default for best performance, 294 * but this allows it to be forced off for testing. 295 */ 296 static int ixgbe_enable_msix = 1; 297 TUNABLE_INT("hw.ixgbe.enable_msix", &ixgbe_enable_msix); 298 299 /* 300 * Header split: this causes the hardware to DMA 301 * the header into a separate mbuf from the payload, 302 * it can be a performance win in some workloads, but 303 * in others it actually hurts, its off by default. 304 */ 305 static int ixgbe_header_split = FALSE; 306 TUNABLE_INT("hw.ixgbe.hdr_split", &ixgbe_header_split); 307 308 #if defined(NETBSD_MSI_OR_MSIX) 309 /* 310 * Number of Queues, can be set to 0, 311 * it then autoconfigures based on the 312 * number of cpus with a max of 8. This 313 * can be overriden manually here. 314 */ 315 static int ixgbe_num_queues = 0; 316 TUNABLE_INT("hw.ixgbe.num_queues", &ixgbe_num_queues); 317 #endif 318 319 /* 320 ** Number of TX descriptors per ring, 321 ** setting higher than RX as this seems 322 ** the better performing choice. 323 */ 324 static int ixgbe_txd = PERFORM_TXD; 325 TUNABLE_INT("hw.ixgbe.txd", &ixgbe_txd); 326 327 /* Number of RX descriptors per ring */ 328 static int ixgbe_rxd = PERFORM_RXD; 329 TUNABLE_INT("hw.ixgbe.rxd", &ixgbe_rxd); 330 331 /* Keep running tab on them for sanity check */ 332 static int ixgbe_total_ports; 333 334 #ifdef IXGBE_FDIR 335 /* 336 ** For Flow Director: this is the 337 ** number of TX packets we sample 338 ** for the filter pool, this means 339 ** every 20th packet will be probed. 340 ** 341 ** This feature can be disabled by 342 ** setting this to 0. 343 */ 344 static int atr_sample_rate = 20; 345 /* 346 ** Flow Director actually 'steals' 347 ** part of the packet buffer as its 348 ** filter pool, this variable controls 349 ** how much it uses: 350 ** 0 = 64K, 1 = 128K, 2 = 256K 351 */ 352 static int fdir_pballoc = 1; 353 #endif 354 355 #ifdef DEV_NETMAP 356 /* 357 * The #ifdef DEV_NETMAP / #endif blocks in this file are meant to 358 * be a reference on how to implement netmap support in a driver. 359 * Additional comments are in ixgbe_netmap.h . 360 * 361 * <dev/netmap/ixgbe_netmap.h> contains functions for netmap support 362 * that extend the standard driver. 363 */ 364 #include <dev/netmap/ixgbe_netmap.h> 365 #endif /* DEV_NETMAP */ 366 367 /********************************************************************* 368 * Device identification routine 369 * 370 * ixgbe_probe determines if the driver should be loaded on 371 * adapter based on PCI vendor/device id of the adapter. 372 * 373 * return 1 on success, 0 on failure 374 *********************************************************************/ 375 376 static int 377 ixgbe_probe(device_t dev, cfdata_t cf, void *aux) 378 { 379 const struct pci_attach_args *pa = aux; 380 381 return (ixgbe_lookup(pa) != NULL) ? 1 : 0; 382 } 383 384 static ixgbe_vendor_info_t * 385 ixgbe_lookup(const struct pci_attach_args *pa) 386 { 387 pcireg_t subid; 388 ixgbe_vendor_info_t *ent; 389 390 INIT_DEBUGOUT("ixgbe_probe: begin"); 391 392 if (PCI_VENDOR(pa->pa_id) != IXGBE_INTEL_VENDOR_ID) 393 return NULL; 394 395 subid = pci_conf_read(pa->pa_pc, pa->pa_tag, PCI_SUBSYS_ID_REG); 396 397 for (ent = ixgbe_vendor_info_array; ent->vendor_id != 0; ent++) { 398 if (PCI_VENDOR(pa->pa_id) == ent->vendor_id && 399 PCI_PRODUCT(pa->pa_id) == ent->device_id && 400 401 (PCI_SUBSYS_VENDOR(subid) == ent->subvendor_id || 402 ent->subvendor_id == 0) && 403 404 (PCI_SUBSYS_ID(subid) == ent->subdevice_id || 405 ent->subdevice_id == 0)) { 406 ++ixgbe_total_ports; 407 return ent; 408 } 409 } 410 return NULL; 411 } 412 413 414 static void 415 ixgbe_sysctl_attach(struct adapter *adapter) 416 { 417 struct sysctllog **log; 418 const struct sysctlnode *rnode, *cnode; 419 device_t dev; 420 421 dev = adapter->dev; 422 log = &adapter->sysctllog; 423 424 if ((rnode = ixgbe_sysctl_instance(adapter)) == NULL) { 425 aprint_error_dev(dev, "could not create sysctl root\n"); 426 return; 427 } 428 429 if (sysctl_createv(log, 0, &rnode, &cnode, 430 CTLFLAG_READONLY, CTLTYPE_INT, 431 "num_rx_desc", SYSCTL_DESCR("Number of rx descriptors"), 432 NULL, 0, &adapter->num_rx_desc, 0, CTL_CREATE, CTL_EOL) != 0) 433 aprint_error_dev(dev, "could not create sysctl\n"); 434 435 if (sysctl_createv(log, 0, &rnode, &cnode, 436 CTLFLAG_READONLY, CTLTYPE_INT, 437 "num_queues", SYSCTL_DESCR("Number of queues"), 438 NULL, 0, &adapter->num_queues, 0, CTL_CREATE, CTL_EOL) != 0) 439 aprint_error_dev(dev, "could not create sysctl\n"); 440 441 if (sysctl_createv(log, 0, &rnode, &cnode, 442 CTLFLAG_READWRITE, CTLTYPE_INT, 443 "fc", SYSCTL_DESCR("Flow Control"), 444 ixgbe_set_flowcntl, 0, (void *)adapter, 0, CTL_CREATE, CTL_EOL) != 0) 445 aprint_error_dev(dev, "could not create sysctl\n"); 446 447 /* XXX This is an *instance* sysctl controlling a *global* variable. 448 * XXX It's that way in the FreeBSD driver that this derives from. 449 */ 450 if (sysctl_createv(log, 0, &rnode, &cnode, 451 CTLFLAG_READWRITE, CTLTYPE_INT, 452 "enable_aim", SYSCTL_DESCR("Interrupt Moderation"), 453 NULL, 0, &ixgbe_enable_aim, 0, CTL_CREATE, CTL_EOL) != 0) 454 aprint_error_dev(dev, "could not create sysctl\n"); 455 456 if (sysctl_createv(log, 0, &rnode, &cnode, 457 CTLFLAG_READWRITE, CTLTYPE_INT, 458 "advertise_speed", SYSCTL_DESCR("Link Speed"), 459 ixgbe_set_advertise, 0, (void *)adapter, 0, CTL_CREATE, CTL_EOL) != 0) 460 aprint_error_dev(dev, "could not create sysctl\n"); 461 462 if (sysctl_createv(log, 0, &rnode, &cnode, 463 CTLFLAG_READWRITE, CTLTYPE_INT, 464 "ts", SYSCTL_DESCR("Thermal Test"), 465 ixgbe_set_thermal_test, 0, (void *)adapter, 0, CTL_CREATE, CTL_EOL) != 0) 466 aprint_error_dev(dev, "could not create sysctl\n"); 467 } 468 469 /********************************************************************* 470 * Device initialization routine 471 * 472 * The attach entry point is called when the driver is being loaded. 473 * This routine identifies the type of hardware, allocates all resources 474 * and initializes the hardware. 475 * 476 * return 0 on success, positive on failure 477 *********************************************************************/ 478 479 static void 480 ixgbe_attach(device_t parent, device_t dev, void *aux) 481 { 482 struct adapter *adapter; 483 struct ixgbe_hw *hw; 484 int error = 0; 485 u16 csum; 486 u32 ctrl_ext; 487 ixgbe_vendor_info_t *ent; 488 const struct pci_attach_args *pa = aux; 489 490 INIT_DEBUGOUT("ixgbe_attach: begin"); 491 492 /* Allocate, clear, and link in our adapter structure */ 493 adapter = device_private(dev); 494 adapter->dev = adapter->osdep.dev = dev; 495 hw = &adapter->hw; 496 adapter->osdep.pc = pa->pa_pc; 497 adapter->osdep.tag = pa->pa_tag; 498 adapter->osdep.dmat = pa->pa_dmat; 499 500 ent = ixgbe_lookup(pa); 501 502 KASSERT(ent != NULL); 503 504 aprint_normal(": %s, Version - %s\n", 505 ixgbe_strings[ent->index], ixgbe_driver_version); 506 507 /* Core Lock Init*/ 508 IXGBE_CORE_LOCK_INIT(adapter, device_xname(dev)); 509 510 /* SYSCTL APIs */ 511 512 ixgbe_sysctl_attach(adapter); 513 514 /* Set up the timer callout */ 515 callout_init(&adapter->timer, 0); 516 517 /* Determine hardware revision */ 518 ixgbe_identify_hardware(adapter); 519 520 /* Do base PCI setup - map BAR0 */ 521 if (ixgbe_allocate_pci_resources(adapter, pa)) { 522 aprint_error_dev(dev, "Allocation of PCI resources failed\n"); 523 error = ENXIO; 524 goto err_out; 525 } 526 527 /* Do descriptor calc and sanity checks */ 528 if (((ixgbe_txd * sizeof(union ixgbe_adv_tx_desc)) % DBA_ALIGN) != 0 || 529 ixgbe_txd < MIN_TXD || ixgbe_txd > MAX_TXD) { 530 aprint_error_dev(dev, "TXD config issue, using default!\n"); 531 adapter->num_tx_desc = DEFAULT_TXD; 532 } else 533 adapter->num_tx_desc = ixgbe_txd; 534 535 /* 536 ** With many RX rings it is easy to exceed the 537 ** system mbuf allocation. Tuning nmbclusters 538 ** can alleviate this. 539 */ 540 if (nmbclusters > 0 ) { 541 int s; 542 s = (ixgbe_rxd * adapter->num_queues) * ixgbe_total_ports; 543 if (s > nmbclusters) { 544 aprint_error_dev(dev, "RX Descriptors exceed " 545 "system mbuf max, using default instead!\n"); 546 ixgbe_rxd = DEFAULT_RXD; 547 } 548 } 549 550 if (((ixgbe_rxd * sizeof(union ixgbe_adv_rx_desc)) % DBA_ALIGN) != 0 || 551 ixgbe_rxd < MIN_TXD || ixgbe_rxd > MAX_TXD) { 552 aprint_error_dev(dev, "RXD config issue, using default!\n"); 553 adapter->num_rx_desc = DEFAULT_RXD; 554 } else 555 adapter->num_rx_desc = ixgbe_rxd; 556 557 /* Allocate our TX/RX Queues */ 558 if (ixgbe_allocate_queues(adapter)) { 559 error = ENOMEM; 560 goto err_out; 561 } 562 563 /* Allocate multicast array memory. */ 564 adapter->mta = malloc(sizeof(u8) * IXGBE_ETH_LENGTH_OF_ADDRESS * 565 MAX_NUM_MULTICAST_ADDRESSES, M_DEVBUF, M_NOWAIT); 566 if (adapter->mta == NULL) { 567 aprint_error_dev(dev, "Cannot allocate multicast setup array\n"); 568 error = ENOMEM; 569 goto err_late; 570 } 571 572 /* Initialize the shared code */ 573 error = ixgbe_init_shared_code(hw); 574 if (error == IXGBE_ERR_SFP_NOT_PRESENT) { 575 /* 576 ** No optics in this port, set up 577 ** so the timer routine will probe 578 ** for later insertion. 579 */ 580 adapter->sfp_probe = TRUE; 581 error = 0; 582 } else if (error == IXGBE_ERR_SFP_NOT_SUPPORTED) { 583 aprint_error_dev(dev,"Unsupported SFP+ module detected!\n"); 584 error = EIO; 585 goto err_late; 586 } else if (error) { 587 aprint_error_dev(dev,"Unable to initialize the shared code\n"); 588 error = EIO; 589 goto err_late; 590 } 591 592 /* Make sure we have a good EEPROM before we read from it */ 593 if (ixgbe_validate_eeprom_checksum(&adapter->hw, &csum) < 0) { 594 aprint_error_dev(dev,"The EEPROM Checksum Is Not Valid\n"); 595 error = EIO; 596 goto err_late; 597 } 598 599 error = ixgbe_init_hw(hw); 600 switch (error) { 601 case IXGBE_ERR_EEPROM_VERSION: 602 aprint_error_dev(dev, "This device is a pre-production adapter/" 603 "LOM. Please be aware there may be issues associated " 604 "with your hardware.\n If you are experiencing problems " 605 "please contact your Intel or hardware representative " 606 "who provided you with this hardware.\n"); 607 break; 608 case IXGBE_ERR_SFP_NOT_SUPPORTED: 609 aprint_error_dev(dev,"Unsupported SFP+ Module\n"); 610 error = EIO; 611 aprint_error_dev(dev,"Hardware Initialization Failure\n"); 612 goto err_late; 613 case IXGBE_ERR_SFP_NOT_PRESENT: 614 device_printf(dev,"No SFP+ Module found\n"); 615 /* falls thru */ 616 default: 617 break; 618 } 619 620 /* Detect and set physical type */ 621 ixgbe_setup_optics(adapter); 622 623 if ((adapter->msix > 1) && (ixgbe_enable_msix)) 624 error = ixgbe_allocate_msix(adapter, pa); 625 else 626 error = ixgbe_allocate_legacy(adapter, pa); 627 if (error) 628 goto err_late; 629 630 /* Setup OS specific network interface */ 631 if (ixgbe_setup_interface(dev, adapter) != 0) 632 goto err_late; 633 634 /* Sysctl for limiting the amount of work done in software interrupts */ 635 ixgbe_add_rx_process_limit(adapter, "rx_processing_limit", 636 "max number of rx packets to process", &adapter->rx_process_limit, 637 ixgbe_rx_process_limit); 638 639 /* Initialize statistics */ 640 ixgbe_update_stats_counters(adapter); 641 642 /* Print PCIE bus type/speed/width info */ 643 ixgbe_get_bus_info(hw); 644 aprint_normal_dev(dev,"PCI Express Bus: Speed %s %s\n", 645 ((hw->bus.speed == ixgbe_bus_speed_5000) ? "5.0Gb/s": 646 (hw->bus.speed == ixgbe_bus_speed_2500) ? "2.5Gb/s":"Unknown"), 647 (hw->bus.width == ixgbe_bus_width_pcie_x8) ? "Width x8" : 648 (hw->bus.width == ixgbe_bus_width_pcie_x4) ? "Width x4" : 649 (hw->bus.width == ixgbe_bus_width_pcie_x1) ? "Width x1" : 650 ("Unknown")); 651 652 if ((hw->bus.width <= ixgbe_bus_width_pcie_x4) && 653 (hw->bus.speed == ixgbe_bus_speed_2500)) { 654 aprint_error_dev(dev, "PCI-Express bandwidth available" 655 " for this card\n is not sufficient for" 656 " optimal performance.\n"); 657 aprint_error_dev(dev, "For optimal performance a x8 " 658 "PCIE, or x4 PCIE 2 slot is required.\n"); 659 } 660 661 /* let hardware know driver is loaded */ 662 ctrl_ext = IXGBE_READ_REG(hw, IXGBE_CTRL_EXT); 663 ctrl_ext |= IXGBE_CTRL_EXT_DRV_LOAD; 664 IXGBE_WRITE_REG(hw, IXGBE_CTRL_EXT, ctrl_ext); 665 666 ixgbe_add_hw_stats(adapter); 667 668 #ifdef DEV_NETMAP 669 ixgbe_netmap_attach(adapter); 670 #endif /* DEV_NETMAP */ 671 INIT_DEBUGOUT("ixgbe_attach: end"); 672 return; 673 err_late: 674 ixgbe_free_transmit_structures(adapter); 675 ixgbe_free_receive_structures(adapter); 676 err_out: 677 if (adapter->ifp != NULL) 678 if_free(adapter->ifp); 679 ixgbe_free_pci_resources(adapter); 680 if (adapter->mta != NULL) 681 free(adapter->mta, M_DEVBUF); 682 return; 683 684 } 685 686 /********************************************************************* 687 * Device removal routine 688 * 689 * The detach entry point is called when the driver is being removed. 690 * This routine stops the adapter and deallocates all the resources 691 * that were allocated for driver operation. 692 * 693 * return 0 on success, positive on failure 694 *********************************************************************/ 695 696 static int 697 ixgbe_detach(device_t dev, int flags) 698 { 699 struct adapter *adapter = device_private(dev); 700 struct tx_ring *txr = adapter->tx_rings; 701 struct rx_ring *rxr = adapter->rx_rings; 702 struct ixgbe_hw_stats *stats = &adapter->stats; 703 struct ix_queue *que = adapter->queues; 704 u32 ctrl_ext; 705 706 INIT_DEBUGOUT("ixgbe_detach: begin"); 707 708 /* Make sure VLANs are not using driver */ 709 if (!VLAN_ATTACHED(&adapter->osdep.ec)) 710 ; /* nothing to do: no VLANs */ 711 else if ((flags & (DETACH_SHUTDOWN|DETACH_FORCE)) != 0) 712 vlan_ifdetach(adapter->ifp); 713 else { 714 aprint_error_dev(dev, "VLANs in use\n"); 715 return EBUSY; 716 } 717 718 IXGBE_CORE_LOCK(adapter); 719 ixgbe_stop(adapter); 720 IXGBE_CORE_UNLOCK(adapter); 721 722 for (int i = 0; i < adapter->num_queues; i++, que++) { 723 softint_disestablish(que->que_si); 724 } 725 726 /* Drain the Link queue */ 727 softint_disestablish(adapter->link_si); 728 softint_disestablish(adapter->mod_si); 729 softint_disestablish(adapter->msf_si); 730 #ifdef IXGBE_FDIR 731 softint_disestablish(adapter->fdir_si); 732 #endif 733 734 /* let hardware know driver is unloading */ 735 ctrl_ext = IXGBE_READ_REG(&adapter->hw, IXGBE_CTRL_EXT); 736 ctrl_ext &= ~IXGBE_CTRL_EXT_DRV_LOAD; 737 IXGBE_WRITE_REG(&adapter->hw, IXGBE_CTRL_EXT, ctrl_ext); 738 739 ether_ifdetach(adapter->ifp); 740 callout_halt(&adapter->timer, NULL); 741 #ifdef DEV_NETMAP 742 netmap_detach(adapter->ifp); 743 #endif /* DEV_NETMAP */ 744 ixgbe_free_pci_resources(adapter); 745 #if 0 /* XXX the NetBSD port is probably missing something here */ 746 bus_generic_detach(dev); 747 #endif 748 if_detach(adapter->ifp); 749 750 sysctl_teardown(&adapter->sysctllog); 751 evcnt_detach(&adapter->handleq); 752 evcnt_detach(&adapter->req); 753 evcnt_detach(&adapter->morerx); 754 evcnt_detach(&adapter->moretx); 755 evcnt_detach(&adapter->txloops); 756 evcnt_detach(&adapter->efbig_tx_dma_setup); 757 evcnt_detach(&adapter->m_defrag_failed); 758 evcnt_detach(&adapter->efbig2_tx_dma_setup); 759 evcnt_detach(&adapter->einval_tx_dma_setup); 760 evcnt_detach(&adapter->other_tx_dma_setup); 761 evcnt_detach(&adapter->eagain_tx_dma_setup); 762 evcnt_detach(&adapter->enomem_tx_dma_setup); 763 evcnt_detach(&adapter->watchdog_events); 764 evcnt_detach(&adapter->tso_err); 765 evcnt_detach(&adapter->tso_tx); 766 evcnt_detach(&adapter->link_irq); 767 for (int i = 0; i < adapter->num_queues; i++, rxr++, txr++) { 768 evcnt_detach(&txr->no_desc_avail); 769 evcnt_detach(&txr->total_packets); 770 771 if (i < __arraycount(adapter->stats.mpc)) { 772 evcnt_detach(&adapter->stats.mpc[i]); 773 } 774 if (i < __arraycount(adapter->stats.pxontxc)) { 775 evcnt_detach(&adapter->stats.pxontxc[i]); 776 evcnt_detach(&adapter->stats.pxonrxc[i]); 777 evcnt_detach(&adapter->stats.pxofftxc[i]); 778 evcnt_detach(&adapter->stats.pxoffrxc[i]); 779 evcnt_detach(&adapter->stats.pxon2offc[i]); 780 } 781 if (i < __arraycount(adapter->stats.qprc)) { 782 evcnt_detach(&adapter->stats.qprc[i]); 783 evcnt_detach(&adapter->stats.qptc[i]); 784 evcnt_detach(&adapter->stats.qbrc[i]); 785 evcnt_detach(&adapter->stats.qbtc[i]); 786 evcnt_detach(&adapter->stats.qprdc[i]); 787 } 788 789 evcnt_detach(&rxr->rx_packets); 790 evcnt_detach(&rxr->rx_bytes); 791 evcnt_detach(&rxr->no_jmbuf); 792 evcnt_detach(&rxr->rx_discarded); 793 evcnt_detach(&rxr->rx_split_packets); 794 evcnt_detach(&rxr->rx_irq); 795 } 796 evcnt_detach(&stats->ipcs); 797 evcnt_detach(&stats->l4cs); 798 evcnt_detach(&stats->ipcs_bad); 799 evcnt_detach(&stats->l4cs_bad); 800 evcnt_detach(&stats->intzero); 801 evcnt_detach(&stats->legint); 802 evcnt_detach(&stats->crcerrs); 803 evcnt_detach(&stats->illerrc); 804 evcnt_detach(&stats->errbc); 805 evcnt_detach(&stats->mspdc); 806 evcnt_detach(&stats->mlfc); 807 evcnt_detach(&stats->mrfc); 808 evcnt_detach(&stats->rlec); 809 evcnt_detach(&stats->lxontxc); 810 evcnt_detach(&stats->lxonrxc); 811 evcnt_detach(&stats->lxofftxc); 812 evcnt_detach(&stats->lxoffrxc); 813 814 /* Packet Reception Stats */ 815 evcnt_detach(&stats->tor); 816 evcnt_detach(&stats->gorc); 817 evcnt_detach(&stats->tpr); 818 evcnt_detach(&stats->gprc); 819 evcnt_detach(&stats->mprc); 820 evcnt_detach(&stats->bprc); 821 evcnt_detach(&stats->prc64); 822 evcnt_detach(&stats->prc127); 823 evcnt_detach(&stats->prc255); 824 evcnt_detach(&stats->prc511); 825 evcnt_detach(&stats->prc1023); 826 evcnt_detach(&stats->prc1522); 827 evcnt_detach(&stats->ruc); 828 evcnt_detach(&stats->rfc); 829 evcnt_detach(&stats->roc); 830 evcnt_detach(&stats->rjc); 831 evcnt_detach(&stats->mngprc); 832 evcnt_detach(&stats->xec); 833 834 /* Packet Transmission Stats */ 835 evcnt_detach(&stats->gotc); 836 evcnt_detach(&stats->tpt); 837 evcnt_detach(&stats->gptc); 838 evcnt_detach(&stats->bptc); 839 evcnt_detach(&stats->mptc); 840 evcnt_detach(&stats->mngptc); 841 evcnt_detach(&stats->ptc64); 842 evcnt_detach(&stats->ptc127); 843 evcnt_detach(&stats->ptc255); 844 evcnt_detach(&stats->ptc511); 845 evcnt_detach(&stats->ptc1023); 846 evcnt_detach(&stats->ptc1522); 847 848 /* FC Stats */ 849 evcnt_detach(&stats->fccrc); 850 evcnt_detach(&stats->fclast); 851 if (adapter->hw.mac.type != ixgbe_mac_82598EB) { 852 evcnt_detach(&stats->fcoerpdc); 853 evcnt_detach(&stats->fcoeprc); 854 evcnt_detach(&stats->fcoeptc); 855 evcnt_detach(&stats->fcoedwrc); 856 evcnt_detach(&stats->fcoedwtc); 857 } 858 859 ixgbe_free_transmit_structures(adapter); 860 ixgbe_free_receive_structures(adapter); 861 free(adapter->mta, M_DEVBUF); 862 863 IXGBE_CORE_LOCK_DESTROY(adapter); 864 return (0); 865 } 866 867 /********************************************************************* 868 * 869 * Shutdown entry point 870 * 871 **********************************************************************/ 872 873 #if 0 /* XXX NetBSD ought to register something like this through pmf(9) */ 874 static int 875 ixgbe_shutdown(device_t dev) 876 { 877 struct adapter *adapter = device_private(dev); 878 IXGBE_CORE_LOCK(adapter); 879 ixgbe_stop(adapter); 880 IXGBE_CORE_UNLOCK(adapter); 881 return (0); 882 } 883 #endif 884 885 886 /********************************************************************* 887 * Transmit entry point 888 * 889 * ixgbe_start is called by the stack to initiate a transmit. 890 * The driver will remain in this routine as long as there are 891 * packets to transmit and transmit resources are available. 892 * In case resources are not available stack is notified and 893 * the packet is requeued. 894 **********************************************************************/ 895 896 static void 897 ixgbe_start_locked(struct tx_ring *txr, struct ifnet * ifp) 898 { 899 int rc; 900 struct mbuf *m_head; 901 struct adapter *adapter = txr->adapter; 902 903 IXGBE_TX_LOCK_ASSERT(txr); 904 905 if ((ifp->if_flags & (IFF_RUNNING|IFF_OACTIVE)) != 906 IFF_RUNNING) 907 return; 908 if (!adapter->link_active) 909 return; 910 911 while (!IFQ_IS_EMPTY(&ifp->if_snd)) { 912 if (txr->tx_avail <= IXGBE_QUEUE_MIN_FREE) { 913 txr->queue_status |= IXGBE_QUEUE_DEPLETED; 914 break; 915 } 916 917 IFQ_POLL(&ifp->if_snd, m_head); 918 if (m_head == NULL) 919 break; 920 921 if ((rc = ixgbe_xmit(txr, m_head)) == EAGAIN) { 922 if (txr->tx_avail <= IXGBE_QUEUE_MIN_FREE) 923 txr->queue_status |= IXGBE_QUEUE_DEPLETED; 924 break; 925 } 926 IFQ_DEQUEUE(&ifp->if_snd, m_head); 927 if (rc == EFBIG) { 928 struct mbuf *mtmp; 929 930 if ((mtmp = m_defrag(m_head, M_DONTWAIT)) != NULL) { 931 m_head = mtmp; 932 rc = ixgbe_xmit(txr, m_head); 933 if (rc != 0) 934 adapter->efbig2_tx_dma_setup.ev_count++; 935 } else 936 adapter->m_defrag_failed.ev_count++; 937 } 938 if (rc != 0) { 939 m_freem(m_head); 940 continue; 941 } 942 943 /* Send a copy of the frame to the BPF listener */ 944 bpf_mtap(ifp, m_head); 945 946 /* Set watchdog on */ 947 getmicrotime(&txr->watchdog_time); 948 txr->queue_status = IXGBE_QUEUE_WORKING; 949 950 } 951 return; 952 } 953 954 /* 955 * Legacy TX start - called by the stack, this 956 * always uses the first tx ring, and should 957 * not be used with multiqueue tx enabled. 958 */ 959 static void 960 ixgbe_start(struct ifnet *ifp) 961 { 962 struct adapter *adapter = ifp->if_softc; 963 struct tx_ring *txr = adapter->tx_rings; 964 965 if (ifp->if_flags & IFF_RUNNING) { 966 IXGBE_TX_LOCK(txr); 967 ixgbe_start_locked(txr, ifp); 968 IXGBE_TX_UNLOCK(txr); 969 } 970 return; 971 } 972 973 #if __FreeBSD_version >= 800000 974 /* 975 ** Multiqueue Transmit driver 976 ** 977 */ 978 static int 979 ixgbe_mq_start(struct ifnet *ifp, struct mbuf *m) 980 { 981 struct adapter *adapter = ifp->if_softc; 982 struct ix_queue *que; 983 struct tx_ring *txr; 984 int i = 0, err = 0; 985 986 /* Which queue to use */ 987 if ((m->m_flags & M_FLOWID) != 0) 988 i = m->m_pkthdr.flowid % adapter->num_queues; 989 else 990 i = curcpu % adapter->num_queues; 991 992 txr = &adapter->tx_rings[i]; 993 que = &adapter->queues[i]; 994 995 if (((txr->queue_status & IXGBE_QUEUE_DEPLETED) == 0) && 996 IXGBE_TX_TRYLOCK(txr)) { 997 err = ixgbe_mq_start_locked(ifp, txr, m); 998 IXGBE_TX_UNLOCK(txr); 999 } else { 1000 err = drbr_enqueue(ifp, txr->br, m); 1001 softint_schedule(que->que_si); 1002 } 1003 1004 return (err); 1005 } 1006 1007 static int 1008 ixgbe_mq_start_locked(struct ifnet *ifp, struct tx_ring *txr, struct mbuf *m) 1009 { 1010 struct adapter *adapter = txr->adapter; 1011 struct mbuf *next; 1012 int enqueued, err = 0; 1013 1014 if (((ifp->if_flags & IFF_RUNNING) == 0) || 1015 (txr->queue_status == IXGBE_QUEUE_DEPLETED) || 1016 adapter->link_active == 0) { 1017 if (m != NULL) 1018 err = drbr_enqueue(ifp, txr->br, m); 1019 return (err); 1020 } 1021 1022 enqueued = 0; 1023 if (m == NULL) { 1024 next = drbr_dequeue(ifp, txr->br); 1025 } else if (drbr_needs_enqueue(ifp, txr->br)) { 1026 if ((err = drbr_enqueue(ifp, txr->br, m)) != 0) 1027 return (err); 1028 next = drbr_dequeue(ifp, txr->br); 1029 } else 1030 next = m; 1031 1032 /* Process the queue */ 1033 while (next != NULL) { 1034 if ((err = ixgbe_xmit(txr, &next)) != 0) { 1035 if (next != NULL) 1036 err = drbr_enqueue(ifp, txr->br, next); 1037 break; 1038 } 1039 enqueued++; 1040 drbr_stats_update(ifp, next->m_pkthdr.len, next->m_flags); 1041 /* Send a copy of the frame to the BPF listener */ 1042 bpf_mtap(ifp, next); 1043 if ((ifp->if_flags & IFF_RUNNING) == 0) 1044 break; 1045 if (txr->tx_avail < IXGBE_TX_OP_THRESHOLD) 1046 ixgbe_txeof(txr); 1047 if (txr->tx_avail < IXGBE_TX_OP_THRESHOLD) { 1048 ifp->if_flags |= IFF_OACTIVE; 1049 break; 1050 } 1051 next = drbr_dequeue(ifp, txr->br); 1052 } 1053 1054 if (enqueued > 0) { 1055 /* Set watchdog on */ 1056 txr->queue_status |= IXGBE_QUEUE_WORKING; 1057 getmicrotime(&txr->watchdog_time); 1058 } 1059 1060 if (txr->tx_avail < IXGBE_TX_CLEANUP_THRESHOLD) 1061 ixgbe_txeof(txr); 1062 1063 return (err); 1064 } 1065 1066 /* 1067 ** Flush all ring buffers 1068 */ 1069 static void 1070 ixgbe_qflush(struct ifnet *ifp) 1071 { 1072 struct adapter *adapter = ifp->if_softc; 1073 struct tx_ring *txr = adapter->tx_rings; 1074 struct mbuf *m; 1075 1076 for (int i = 0; i < adapter->num_queues; i++, txr++) { 1077 IXGBE_TX_LOCK(txr); 1078 while ((m = buf_ring_dequeue_sc(txr->br)) != NULL) 1079 m_freem(m); 1080 IXGBE_TX_UNLOCK(txr); 1081 } 1082 if_qflush(ifp); 1083 } 1084 #endif /* __FreeBSD_version >= 800000 */ 1085 1086 static int 1087 ixgbe_ifflags_cb(struct ethercom *ec) 1088 { 1089 struct ifnet *ifp = &ec->ec_if; 1090 struct adapter *adapter = ifp->if_softc; 1091 int change = ifp->if_flags ^ adapter->if_flags, rc = 0; 1092 1093 IXGBE_CORE_LOCK(adapter); 1094 1095 if (change != 0) 1096 adapter->if_flags = ifp->if_flags; 1097 1098 if ((change & ~(IFF_CANTCHANGE|IFF_DEBUG)) != 0) 1099 rc = ENETRESET; 1100 else if ((change & (IFF_PROMISC | IFF_ALLMULTI)) != 0) 1101 ixgbe_set_promisc(adapter); 1102 1103 /* Set up VLAN support and filter */ 1104 ixgbe_setup_vlan_hw_support(adapter); 1105 1106 IXGBE_CORE_UNLOCK(adapter); 1107 1108 return rc; 1109 } 1110 1111 /********************************************************************* 1112 * Ioctl entry point 1113 * 1114 * ixgbe_ioctl is called when the user wants to configure the 1115 * interface. 1116 * 1117 * return 0 on success, positive on failure 1118 **********************************************************************/ 1119 1120 static int 1121 ixgbe_ioctl(struct ifnet * ifp, u_long command, void *data) 1122 { 1123 struct adapter *adapter = ifp->if_softc; 1124 struct ifcapreq *ifcr = data; 1125 struct ifreq *ifr = data; 1126 int error = 0; 1127 int l4csum_en; 1128 const int l4csum = IFCAP_CSUM_TCPv4_Rx|IFCAP_CSUM_UDPv4_Rx| 1129 IFCAP_CSUM_TCPv6_Rx|IFCAP_CSUM_UDPv6_Rx; 1130 1131 switch (command) { 1132 case SIOCSIFFLAGS: 1133 IOCTL_DEBUGOUT("ioctl: SIOCSIFFLAGS (Set Interface Flags)"); 1134 break; 1135 case SIOCADDMULTI: 1136 case SIOCDELMULTI: 1137 IOCTL_DEBUGOUT("ioctl: SIOC(ADD|DEL)MULTI"); 1138 break; 1139 case SIOCSIFMEDIA: 1140 case SIOCGIFMEDIA: 1141 IOCTL_DEBUGOUT("ioctl: SIOCxIFMEDIA (Get/Set Interface Media)"); 1142 break; 1143 case SIOCSIFCAP: 1144 IOCTL_DEBUGOUT("ioctl: SIOCSIFCAP (Set Capabilities)"); 1145 break; 1146 case SIOCSIFMTU: 1147 IOCTL_DEBUGOUT("ioctl: SIOCSIFMTU (Set Interface MTU)"); 1148 break; 1149 default: 1150 IOCTL_DEBUGOUT1("ioctl: UNKNOWN (0x%X)\n", (int)command); 1151 break; 1152 } 1153 1154 switch (command) { 1155 case SIOCSIFMEDIA: 1156 case SIOCGIFMEDIA: 1157 return ifmedia_ioctl(ifp, ifr, &adapter->media, command); 1158 case SIOCSIFCAP: 1159 /* Layer-4 Rx checksum offload has to be turned on and 1160 * off as a unit. 1161 */ 1162 l4csum_en = ifcr->ifcr_capenable & l4csum; 1163 if (l4csum_en != l4csum && l4csum_en != 0) 1164 return EINVAL; 1165 /*FALLTHROUGH*/ 1166 case SIOCADDMULTI: 1167 case SIOCDELMULTI: 1168 case SIOCSIFFLAGS: 1169 case SIOCSIFMTU: 1170 default: 1171 if ((error = ether_ioctl(ifp, command, data)) != ENETRESET) 1172 return error; 1173 if ((ifp->if_flags & IFF_RUNNING) == 0) 1174 ; 1175 else if (command == SIOCSIFCAP || command == SIOCSIFMTU) { 1176 IXGBE_CORE_LOCK(adapter); 1177 ixgbe_init_locked(adapter); 1178 IXGBE_CORE_UNLOCK(adapter); 1179 } else if (command == SIOCADDMULTI || command == SIOCDELMULTI) { 1180 /* 1181 * Multicast list has changed; set the hardware filter 1182 * accordingly. 1183 */ 1184 IXGBE_CORE_LOCK(adapter); 1185 ixgbe_disable_intr(adapter); 1186 ixgbe_set_multi(adapter); 1187 ixgbe_enable_intr(adapter); 1188 IXGBE_CORE_UNLOCK(adapter); 1189 } 1190 return 0; 1191 } 1192 } 1193 1194 /********************************************************************* 1195 * Init entry point 1196 * 1197 * This routine is used in two ways. It is used by the stack as 1198 * init entry point in network interface structure. It is also used 1199 * by the driver as a hw/sw initialization routine to get to a 1200 * consistent state. 1201 * 1202 * return 0 on success, positive on failure 1203 **********************************************************************/ 1204 #define IXGBE_MHADD_MFS_SHIFT 16 1205 1206 static void 1207 ixgbe_init_locked(struct adapter *adapter) 1208 { 1209 struct ifnet *ifp = adapter->ifp; 1210 device_t dev = adapter->dev; 1211 struct ixgbe_hw *hw = &adapter->hw; 1212 u32 k, txdctl, mhadd, gpie; 1213 u32 rxdctl, rxctrl; 1214 1215 /* XXX check IFF_UP and IFF_RUNNING, power-saving state! */ 1216 1217 KASSERT(mutex_owned(&adapter->core_mtx)); 1218 INIT_DEBUGOUT("ixgbe_init: begin"); 1219 hw->adapter_stopped = FALSE; 1220 ixgbe_stop_adapter(hw); 1221 callout_stop(&adapter->timer); 1222 1223 /* XXX I moved this here from the SIOCSIFMTU case in ixgbe_ioctl(). */ 1224 adapter->max_frame_size = 1225 ifp->if_mtu + ETHER_HDR_LEN + ETHER_CRC_LEN; 1226 1227 /* reprogram the RAR[0] in case user changed it. */ 1228 ixgbe_set_rar(hw, 0, adapter->hw.mac.addr, 0, IXGBE_RAH_AV); 1229 1230 /* Get the latest mac address, User can use a LAA */ 1231 memcpy(hw->mac.addr, CLLADDR(adapter->ifp->if_sadl), 1232 IXGBE_ETH_LENGTH_OF_ADDRESS); 1233 ixgbe_set_rar(hw, 0, hw->mac.addr, 0, 1); 1234 hw->addr_ctrl.rar_used_count = 1; 1235 1236 /* Prepare transmit descriptors and buffers */ 1237 if (ixgbe_setup_transmit_structures(adapter)) { 1238 device_printf(dev,"Could not setup transmit structures\n"); 1239 ixgbe_stop(adapter); 1240 return; 1241 } 1242 1243 ixgbe_init_hw(hw); 1244 ixgbe_initialize_transmit_units(adapter); 1245 1246 /* Setup Multicast table */ 1247 ixgbe_set_multi(adapter); 1248 1249 /* 1250 ** Determine the correct mbuf pool 1251 ** for doing jumbo/headersplit 1252 */ 1253 if (adapter->max_frame_size <= 2048) 1254 adapter->rx_mbuf_sz = MCLBYTES; 1255 else if (adapter->max_frame_size <= 4096) 1256 adapter->rx_mbuf_sz = MJUMPAGESIZE; 1257 else if (adapter->max_frame_size <= 9216) 1258 adapter->rx_mbuf_sz = MJUM9BYTES; 1259 else 1260 adapter->rx_mbuf_sz = MJUM16BYTES; 1261 1262 /* Prepare receive descriptors and buffers */ 1263 if (ixgbe_setup_receive_structures(adapter)) { 1264 device_printf(dev,"Could not setup receive structures\n"); 1265 ixgbe_stop(adapter); 1266 return; 1267 } 1268 1269 /* Configure RX settings */ 1270 ixgbe_initialize_receive_units(adapter); 1271 1272 gpie = IXGBE_READ_REG(&adapter->hw, IXGBE_GPIE); 1273 1274 /* Enable Fan Failure Interrupt */ 1275 gpie |= IXGBE_SDP1_GPIEN; 1276 1277 /* Add for Thermal detection */ 1278 if (hw->mac.type == ixgbe_mac_82599EB) 1279 gpie |= IXGBE_SDP2_GPIEN; 1280 1281 /* Thermal Failure Detection */ 1282 if (hw->mac.type == ixgbe_mac_X540) 1283 gpie |= IXGBE_SDP0_GPIEN; 1284 1285 if (adapter->msix > 1) { 1286 /* Enable Enhanced MSIX mode */ 1287 gpie |= IXGBE_GPIE_MSIX_MODE; 1288 gpie |= IXGBE_GPIE_EIAME | IXGBE_GPIE_PBA_SUPPORT | 1289 IXGBE_GPIE_OCD; 1290 } 1291 IXGBE_WRITE_REG(hw, IXGBE_GPIE, gpie); 1292 1293 /* Set MTU size */ 1294 if (ifp->if_mtu > ETHERMTU) { 1295 mhadd = IXGBE_READ_REG(hw, IXGBE_MHADD); 1296 mhadd &= ~IXGBE_MHADD_MFS_MASK; 1297 mhadd |= adapter->max_frame_size << IXGBE_MHADD_MFS_SHIFT; 1298 IXGBE_WRITE_REG(hw, IXGBE_MHADD, mhadd); 1299 } 1300 1301 /* Now enable all the queues */ 1302 1303 for (int i = 0; i < adapter->num_queues; i++) { 1304 txdctl = IXGBE_READ_REG(hw, IXGBE_TXDCTL(i)); 1305 txdctl |= IXGBE_TXDCTL_ENABLE; 1306 /* Set WTHRESH to 8, burst writeback */ 1307 txdctl |= (8 << 16); 1308 /* 1309 * When the internal queue falls below PTHRESH (32), 1310 * start prefetching as long as there are at least 1311 * HTHRESH (1) buffers ready. The values are taken 1312 * from the Intel linux driver 3.8.21. 1313 * Prefetching enables tx line rate even with 1 queue. 1314 */ 1315 txdctl |= (32 << 0) | (1 << 8); 1316 IXGBE_WRITE_REG(hw, IXGBE_TXDCTL(i), txdctl); 1317 } 1318 1319 for (int i = 0; i < adapter->num_queues; i++) { 1320 rxdctl = IXGBE_READ_REG(hw, IXGBE_RXDCTL(i)); 1321 if (hw->mac.type == ixgbe_mac_82598EB) { 1322 /* 1323 ** PTHRESH = 21 1324 ** HTHRESH = 4 1325 ** WTHRESH = 8 1326 */ 1327 rxdctl &= ~0x3FFFFF; 1328 rxdctl |= 0x080420; 1329 } 1330 rxdctl |= IXGBE_RXDCTL_ENABLE; 1331 IXGBE_WRITE_REG(hw, IXGBE_RXDCTL(i), rxdctl); 1332 /* XXX I don't trust this loop, and I don't trust the 1333 * XXX memory barrier. What is this meant to do? --dyoung 1334 */ 1335 for (k = 0; k < 10; k++) { 1336 if (IXGBE_READ_REG(hw, IXGBE_RXDCTL(i)) & 1337 IXGBE_RXDCTL_ENABLE) 1338 break; 1339 else 1340 msec_delay(1); 1341 } 1342 wmb(); 1343 #ifdef DEV_NETMAP 1344 /* 1345 * In netmap mode, we must preserve the buffers made 1346 * available to userspace before the if_init() 1347 * (this is true by default on the TX side, because 1348 * init makes all buffers available to userspace). 1349 * 1350 * netmap_reset() and the device specific routines 1351 * (e.g. ixgbe_setup_receive_rings()) map these 1352 * buffers at the end of the NIC ring, so here we 1353 * must set the RDT (tail) register to make sure 1354 * they are not overwritten. 1355 * 1356 * In this driver the NIC ring starts at RDH = 0, 1357 * RDT points to the last slot available for reception (?), 1358 * so RDT = num_rx_desc - 1 means the whole ring is available. 1359 */ 1360 if (ifp->if_capenable & IFCAP_NETMAP) { 1361 struct netmap_adapter *na = NA(adapter->ifp); 1362 struct netmap_kring *kring = &na->rx_rings[i]; 1363 int t = na->num_rx_desc - 1 - kring->nr_hwavail; 1364 1365 IXGBE_WRITE_REG(hw, IXGBE_RDT(i), t); 1366 } else 1367 #endif /* DEV_NETMAP */ 1368 IXGBE_WRITE_REG(hw, IXGBE_RDT(i), adapter->num_rx_desc - 1); 1369 } 1370 1371 /* Set up VLAN support and filter */ 1372 ixgbe_setup_vlan_hw_support(adapter); 1373 1374 /* Enable Receive engine */ 1375 rxctrl = IXGBE_READ_REG(hw, IXGBE_RXCTRL); 1376 if (hw->mac.type == ixgbe_mac_82598EB) 1377 rxctrl |= IXGBE_RXCTRL_DMBYPS; 1378 rxctrl |= IXGBE_RXCTRL_RXEN; 1379 ixgbe_enable_rx_dma(hw, rxctrl); 1380 1381 callout_reset(&adapter->timer, hz, ixgbe_local_timer, adapter); 1382 1383 /* Set up MSI/X routing */ 1384 if (ixgbe_enable_msix) { 1385 ixgbe_configure_ivars(adapter); 1386 /* Set up auto-mask */ 1387 if (hw->mac.type == ixgbe_mac_82598EB) 1388 IXGBE_WRITE_REG(hw, IXGBE_EIAM, IXGBE_EICS_RTX_QUEUE); 1389 else { 1390 IXGBE_WRITE_REG(hw, IXGBE_EIAM_EX(0), 0xFFFFFFFF); 1391 IXGBE_WRITE_REG(hw, IXGBE_EIAM_EX(1), 0xFFFFFFFF); 1392 } 1393 } else { /* Simple settings for Legacy/MSI */ 1394 ixgbe_set_ivar(adapter, 0, 0, 0); 1395 ixgbe_set_ivar(adapter, 0, 0, 1); 1396 IXGBE_WRITE_REG(hw, IXGBE_EIAM, IXGBE_EICS_RTX_QUEUE); 1397 } 1398 1399 #ifdef IXGBE_FDIR 1400 /* Init Flow director */ 1401 if (hw->mac.type != ixgbe_mac_82598EB) { 1402 u32 hdrm = 32 << fdir_pballoc; 1403 1404 hw->mac.ops.setup_rxpba(hw, 0, hdrm, PBA_STRATEGY_EQUAL); 1405 ixgbe_init_fdir_signature_82599(&adapter->hw, fdir_pballoc); 1406 } 1407 #endif 1408 1409 /* 1410 ** Check on any SFP devices that 1411 ** need to be kick-started 1412 */ 1413 if (hw->phy.type == ixgbe_phy_none) { 1414 int err = hw->phy.ops.identify(hw); 1415 if (err == IXGBE_ERR_SFP_NOT_SUPPORTED) { 1416 device_printf(dev, 1417 "Unsupported SFP+ module type was detected.\n"); 1418 return; 1419 } 1420 } 1421 1422 /* Set moderation on the Link interrupt */ 1423 IXGBE_WRITE_REG(hw, IXGBE_EITR(adapter->linkvec), IXGBE_LINK_ITR); 1424 1425 /* Config/Enable Link */ 1426 ixgbe_config_link(adapter); 1427 1428 /* Hardware Packet Buffer & Flow Control setup */ 1429 { 1430 u32 rxpb, frame, size, tmp; 1431 1432 frame = adapter->max_frame_size; 1433 1434 /* Calculate High Water */ 1435 if (hw->mac.type == ixgbe_mac_X540) 1436 tmp = IXGBE_DV_X540(frame, frame); 1437 else 1438 tmp = IXGBE_DV(frame, frame); 1439 size = IXGBE_BT2KB(tmp); 1440 rxpb = IXGBE_READ_REG(hw, IXGBE_RXPBSIZE(0)) >> 10; 1441 hw->fc.high_water[0] = rxpb - size; 1442 1443 /* Now calculate Low Water */ 1444 if (hw->mac.type == ixgbe_mac_X540) 1445 tmp = IXGBE_LOW_DV_X540(frame); 1446 else 1447 tmp = IXGBE_LOW_DV(frame); 1448 hw->fc.low_water[0] = IXGBE_BT2KB(tmp); 1449 1450 adapter->fc = hw->fc.requested_mode = ixgbe_fc_full; 1451 hw->fc.pause_time = IXGBE_FC_PAUSE; 1452 hw->fc.send_xon = TRUE; 1453 } 1454 /* Initialize the FC settings */ 1455 ixgbe_start_hw(hw); 1456 1457 /* And now turn on interrupts */ 1458 ixgbe_enable_intr(adapter); 1459 1460 /* Now inform the stack we're ready */ 1461 ifp->if_flags |= IFF_RUNNING; 1462 ifp->if_flags &= ~IFF_OACTIVE; 1463 1464 return; 1465 } 1466 1467 static int 1468 ixgbe_init(struct ifnet *ifp) 1469 { 1470 struct adapter *adapter = ifp->if_softc; 1471 1472 IXGBE_CORE_LOCK(adapter); 1473 ixgbe_init_locked(adapter); 1474 IXGBE_CORE_UNLOCK(adapter); 1475 return 0; /* XXX ixgbe_init_locked cannot fail? really? */ 1476 } 1477 1478 1479 /* 1480 ** 1481 ** MSIX Interrupt Handlers and Tasklets 1482 ** 1483 */ 1484 1485 static inline void 1486 ixgbe_enable_queue(struct adapter *adapter, u32 vector) 1487 { 1488 struct ixgbe_hw *hw = &adapter->hw; 1489 u64 queue = (u64)(1ULL << vector); 1490 u32 mask; 1491 1492 if (hw->mac.type == ixgbe_mac_82598EB) { 1493 mask = (IXGBE_EIMS_RTX_QUEUE & queue); 1494 IXGBE_WRITE_REG(hw, IXGBE_EIMS, mask); 1495 } else { 1496 mask = (queue & 0xFFFFFFFF); 1497 if (mask) 1498 IXGBE_WRITE_REG(hw, IXGBE_EIMS_EX(0), mask); 1499 mask = (queue >> 32); 1500 if (mask) 1501 IXGBE_WRITE_REG(hw, IXGBE_EIMS_EX(1), mask); 1502 } 1503 } 1504 1505 __unused static inline void 1506 ixgbe_disable_queue(struct adapter *adapter, u32 vector) 1507 { 1508 struct ixgbe_hw *hw = &adapter->hw; 1509 u64 queue = (u64)(1ULL << vector); 1510 u32 mask; 1511 1512 if (hw->mac.type == ixgbe_mac_82598EB) { 1513 mask = (IXGBE_EIMS_RTX_QUEUE & queue); 1514 IXGBE_WRITE_REG(hw, IXGBE_EIMC, mask); 1515 } else { 1516 mask = (queue & 0xFFFFFFFF); 1517 if (mask) 1518 IXGBE_WRITE_REG(hw, IXGBE_EIMC_EX(0), mask); 1519 mask = (queue >> 32); 1520 if (mask) 1521 IXGBE_WRITE_REG(hw, IXGBE_EIMC_EX(1), mask); 1522 } 1523 } 1524 1525 static inline void 1526 ixgbe_rearm_queues(struct adapter *adapter, u64 queues) 1527 { 1528 u32 mask; 1529 1530 if (adapter->hw.mac.type == ixgbe_mac_82598EB) { 1531 mask = (IXGBE_EIMS_RTX_QUEUE & queues); 1532 IXGBE_WRITE_REG(&adapter->hw, IXGBE_EICS, mask); 1533 } else { 1534 mask = (queues & 0xFFFFFFFF); 1535 IXGBE_WRITE_REG(&adapter->hw, IXGBE_EICS_EX(0), mask); 1536 mask = (queues >> 32); 1537 IXGBE_WRITE_REG(&adapter->hw, IXGBE_EICS_EX(1), mask); 1538 } 1539 } 1540 1541 1542 static void 1543 ixgbe_handle_que(void *context) 1544 { 1545 struct ix_queue *que = context; 1546 struct adapter *adapter = que->adapter; 1547 struct tx_ring *txr = que->txr; 1548 struct ifnet *ifp = adapter->ifp; 1549 bool more; 1550 1551 adapter->handleq.ev_count++; 1552 1553 if (ifp->if_flags & IFF_RUNNING) { 1554 more = ixgbe_rxeof(que, adapter->rx_process_limit); 1555 IXGBE_TX_LOCK(txr); 1556 ixgbe_txeof(txr); 1557 #if __FreeBSD_version >= 800000 1558 if (!drbr_empty(ifp, txr->br)) 1559 ixgbe_mq_start_locked(ifp, txr, NULL); 1560 #else 1561 if (!IFQ_IS_EMPTY(&ifp->if_snd)) 1562 ixgbe_start_locked(txr, ifp); 1563 #endif 1564 IXGBE_TX_UNLOCK(txr); 1565 if (more) { 1566 adapter->req.ev_count++; 1567 softint_schedule(que->que_si); 1568 return; 1569 } 1570 } 1571 1572 /* Reenable this interrupt */ 1573 ixgbe_enable_queue(adapter, que->msix); 1574 1575 return; 1576 } 1577 1578 1579 /********************************************************************* 1580 * 1581 * Legacy Interrupt Service routine 1582 * 1583 **********************************************************************/ 1584 1585 static int 1586 ixgbe_legacy_irq(void *arg) 1587 { 1588 struct ix_queue *que = arg; 1589 struct adapter *adapter = que->adapter; 1590 struct ifnet *ifp = adapter->ifp; 1591 struct ixgbe_hw *hw = &adapter->hw; 1592 struct tx_ring *txr = adapter->tx_rings; 1593 bool more_tx = false, more_rx = false; 1594 u32 reg_eicr, loop = MAX_LOOP; 1595 1596 reg_eicr = IXGBE_READ_REG(hw, IXGBE_EICR); 1597 1598 adapter->stats.legint.ev_count++; 1599 ++que->irqs; 1600 if (reg_eicr == 0) { 1601 adapter->stats.intzero.ev_count++; 1602 if ((ifp->if_flags & IFF_UP) != 0) 1603 ixgbe_enable_intr(adapter); 1604 return 0; 1605 } 1606 1607 if ((ifp->if_flags & IFF_RUNNING) != 0) { 1608 more_rx = ixgbe_rxeof(que, adapter->rx_process_limit); 1609 1610 IXGBE_TX_LOCK(txr); 1611 do { 1612 adapter->txloops.ev_count++; 1613 more_tx = ixgbe_txeof(txr); 1614 } while (loop-- && more_tx); 1615 IXGBE_TX_UNLOCK(txr); 1616 } 1617 1618 if (more_rx || more_tx) { 1619 if (more_rx) 1620 adapter->morerx.ev_count++; 1621 if (more_tx) 1622 adapter->moretx.ev_count++; 1623 softint_schedule(que->que_si); 1624 } 1625 1626 /* Check for fan failure */ 1627 if ((hw->phy.media_type == ixgbe_media_type_copper) && 1628 (reg_eicr & IXGBE_EICR_GPI_SDP1)) { 1629 device_printf(adapter->dev, "\nCRITICAL: FAN FAILURE!! " 1630 "REPLACE IMMEDIATELY!!\n"); 1631 IXGBE_WRITE_REG(hw, IXGBE_EIMS, IXGBE_EICR_GPI_SDP1); 1632 } 1633 1634 /* Link status change */ 1635 if (reg_eicr & IXGBE_EICR_LSC) 1636 softint_schedule(adapter->link_si); 1637 1638 ixgbe_enable_intr(adapter); 1639 return 1; 1640 } 1641 1642 1643 #if defined(NETBSD_MSI_OR_MSIX) 1644 /********************************************************************* 1645 * 1646 * MSIX Queue Interrupt Service routine 1647 * 1648 **********************************************************************/ 1649 void 1650 ixgbe_msix_que(void *arg) 1651 { 1652 struct ix_queue *que = arg; 1653 struct adapter *adapter = que->adapter; 1654 struct tx_ring *txr = que->txr; 1655 struct rx_ring *rxr = que->rxr; 1656 bool more_tx, more_rx; 1657 u32 newitr = 0; 1658 1659 ixgbe_disable_queue(adapter, que->msix); 1660 ++que->irqs; 1661 1662 more_rx = ixgbe_rxeof(que, adapter->rx_process_limit); 1663 1664 IXGBE_TX_LOCK(txr); 1665 more_tx = ixgbe_txeof(txr); 1666 /* 1667 ** Make certain that if the stack 1668 ** has anything queued the task gets 1669 ** scheduled to handle it. 1670 */ 1671 #if __FreeBSD_version < 800000 1672 if (!IFQ_IS_EMPTY(&adapter->ifp->if_snd)) 1673 #else 1674 if (!drbr_empty(adapter->ifp, txr->br)) 1675 #endif 1676 more_tx = 1; 1677 IXGBE_TX_UNLOCK(txr); 1678 1679 /* Do AIM now? */ 1680 1681 if (ixgbe_enable_aim == FALSE) 1682 goto no_calc; 1683 /* 1684 ** Do Adaptive Interrupt Moderation: 1685 ** - Write out last calculated setting 1686 ** - Calculate based on average size over 1687 ** the last interval. 1688 */ 1689 if (que->eitr_setting) 1690 IXGBE_WRITE_REG(&adapter->hw, 1691 IXGBE_EITR(que->msix), que->eitr_setting); 1692 1693 que->eitr_setting = 0; 1694 1695 /* Idle, do nothing */ 1696 if ((txr->bytes == 0) && (rxr->bytes == 0)) 1697 goto no_calc; 1698 1699 if ((txr->bytes) && (txr->packets)) 1700 newitr = txr->bytes/txr->packets; 1701 if ((rxr->bytes) && (rxr->packets)) 1702 newitr = max(newitr, 1703 (rxr->bytes / rxr->packets)); 1704 newitr += 24; /* account for hardware frame, crc */ 1705 1706 /* set an upper boundary */ 1707 newitr = min(newitr, 3000); 1708 1709 /* Be nice to the mid range */ 1710 if ((newitr > 300) && (newitr < 1200)) 1711 newitr = (newitr / 3); 1712 else 1713 newitr = (newitr / 2); 1714 1715 if (adapter->hw.mac.type == ixgbe_mac_82598EB) 1716 newitr |= newitr << 16; 1717 else 1718 newitr |= IXGBE_EITR_CNT_WDIS; 1719 1720 /* save for next interrupt */ 1721 que->eitr_setting = newitr; 1722 1723 /* Reset state */ 1724 txr->bytes = 0; 1725 txr->packets = 0; 1726 rxr->bytes = 0; 1727 rxr->packets = 0; 1728 1729 no_calc: 1730 if (more_tx || more_rx) 1731 softint_schedule(que->que_si); 1732 else /* Reenable this interrupt */ 1733 ixgbe_enable_queue(adapter, que->msix); 1734 return; 1735 } 1736 1737 1738 static void 1739 ixgbe_msix_link(void *arg) 1740 { 1741 struct adapter *adapter = arg; 1742 struct ixgbe_hw *hw = &adapter->hw; 1743 u32 reg_eicr; 1744 1745 ++adapter->link_irq.ev_count; 1746 1747 /* First get the cause */ 1748 reg_eicr = IXGBE_READ_REG(hw, IXGBE_EICS); 1749 /* Clear interrupt with write */ 1750 IXGBE_WRITE_REG(hw, IXGBE_EICR, reg_eicr); 1751 1752 /* Link status change */ 1753 if (reg_eicr & IXGBE_EICR_LSC) 1754 softint_schedule(adapter->link_si); 1755 1756 if (adapter->hw.mac.type != ixgbe_mac_82598EB) { 1757 #ifdef IXGBE_FDIR 1758 if (reg_eicr & IXGBE_EICR_FLOW_DIR) { 1759 /* This is probably overkill :) */ 1760 if (!atomic_cmpset_int(&adapter->fdir_reinit, 0, 1)) 1761 return; 1762 /* Disable the interrupt */ 1763 IXGBE_WRITE_REG(hw, IXGBE_EIMC, IXGBE_EICR_FLOW_DIR); 1764 softint_schedule(adapter->fdir_si); 1765 } else 1766 #endif 1767 if (reg_eicr & IXGBE_EICR_ECC) { 1768 device_printf(adapter->dev, "\nCRITICAL: ECC ERROR!! " 1769 "Please Reboot!!\n"); 1770 IXGBE_WRITE_REG(hw, IXGBE_EICR, IXGBE_EICR_ECC); 1771 } else 1772 1773 if (reg_eicr & IXGBE_EICR_GPI_SDP1) { 1774 /* Clear the interrupt */ 1775 IXGBE_WRITE_REG(hw, IXGBE_EICR, IXGBE_EICR_GPI_SDP1); 1776 softint_schedule(adapter->msf_si); 1777 } else if (reg_eicr & IXGBE_EICR_GPI_SDP2) { 1778 /* Clear the interrupt */ 1779 IXGBE_WRITE_REG(hw, IXGBE_EICR, IXGBE_EICR_GPI_SDP2); 1780 softint_schedule(adapter->mod_si); 1781 } 1782 } 1783 1784 /* Check for fan failure */ 1785 if ((hw->device_id == IXGBE_DEV_ID_82598AT) && 1786 (reg_eicr & IXGBE_EICR_GPI_SDP1)) { 1787 device_printf(adapter->dev, "\nCRITICAL: FAN FAILURE!! " 1788 "REPLACE IMMEDIATELY!!\n"); 1789 IXGBE_WRITE_REG(hw, IXGBE_EICR, IXGBE_EICR_GPI_SDP1); 1790 } 1791 1792 /* Check for over temp condition */ 1793 if ((hw->mac.type == ixgbe_mac_X540) && 1794 (reg_eicr & IXGBE_EICR_GPI_SDP0)) { 1795 device_printf(adapter->dev, "\nCRITICAL: OVER TEMP!! " 1796 "PHY IS SHUT DOWN!!\n"); 1797 device_printf(adapter->dev, "System shutdown required\n"); 1798 IXGBE_WRITE_REG(hw, IXGBE_EICR, IXGBE_EICR_GPI_SDP0); 1799 } 1800 1801 IXGBE_WRITE_REG(&adapter->hw, IXGBE_EIMS, IXGBE_EIMS_OTHER); 1802 return; 1803 } 1804 #endif 1805 1806 /********************************************************************* 1807 * 1808 * Media Ioctl callback 1809 * 1810 * This routine is called whenever the user queries the status of 1811 * the interface using ifconfig. 1812 * 1813 **********************************************************************/ 1814 static void 1815 ixgbe_media_status(struct ifnet * ifp, struct ifmediareq * ifmr) 1816 { 1817 struct adapter *adapter = ifp->if_softc; 1818 1819 INIT_DEBUGOUT("ixgbe_media_status: begin"); 1820 IXGBE_CORE_LOCK(adapter); 1821 ixgbe_update_link_status(adapter); 1822 1823 ifmr->ifm_status = IFM_AVALID; 1824 ifmr->ifm_active = IFM_ETHER; 1825 1826 if (!adapter->link_active) { 1827 IXGBE_CORE_UNLOCK(adapter); 1828 return; 1829 } 1830 1831 ifmr->ifm_status |= IFM_ACTIVE; 1832 1833 switch (adapter->link_speed) { 1834 case IXGBE_LINK_SPEED_100_FULL: 1835 ifmr->ifm_active |= IFM_100_TX | IFM_FDX; 1836 break; 1837 case IXGBE_LINK_SPEED_1GB_FULL: 1838 ifmr->ifm_active |= IFM_1000_T | IFM_FDX; 1839 break; 1840 case IXGBE_LINK_SPEED_10GB_FULL: 1841 ifmr->ifm_active |= adapter->optics | IFM_FDX; 1842 break; 1843 } 1844 1845 IXGBE_CORE_UNLOCK(adapter); 1846 1847 return; 1848 } 1849 1850 /********************************************************************* 1851 * 1852 * Media Ioctl callback 1853 * 1854 * This routine is called when the user changes speed/duplex using 1855 * media/mediopt option with ifconfig. 1856 * 1857 **********************************************************************/ 1858 static int 1859 ixgbe_media_change(struct ifnet * ifp) 1860 { 1861 struct adapter *adapter = ifp->if_softc; 1862 struct ifmedia *ifm = &adapter->media; 1863 1864 INIT_DEBUGOUT("ixgbe_media_change: begin"); 1865 1866 if (IFM_TYPE(ifm->ifm_media) != IFM_ETHER) 1867 return (EINVAL); 1868 1869 switch (IFM_SUBTYPE(ifm->ifm_media)) { 1870 case IFM_AUTO: 1871 adapter->hw.phy.autoneg_advertised = 1872 IXGBE_LINK_SPEED_100_FULL | 1873 IXGBE_LINK_SPEED_1GB_FULL | 1874 IXGBE_LINK_SPEED_10GB_FULL; 1875 break; 1876 default: 1877 device_printf(adapter->dev, "Only auto media type\n"); 1878 return (EINVAL); 1879 } 1880 1881 return (0); 1882 } 1883 1884 /********************************************************************* 1885 * 1886 * This routine maps the mbufs to tx descriptors, allowing the 1887 * TX engine to transmit the packets. 1888 * - return 0 on success, positive on failure 1889 * 1890 **********************************************************************/ 1891 1892 static int 1893 ixgbe_xmit(struct tx_ring *txr, struct mbuf *m_head) 1894 { 1895 struct m_tag *mtag; 1896 struct adapter *adapter = txr->adapter; 1897 struct ethercom *ec = &adapter->osdep.ec; 1898 u32 olinfo_status = 0, cmd_type_len; 1899 u32 paylen = 0; 1900 int i, j, error; 1901 int first, last = 0; 1902 bus_dmamap_t map; 1903 struct ixgbe_tx_buf *txbuf; 1904 union ixgbe_adv_tx_desc *txd = NULL; 1905 1906 /* Basic descriptor defines */ 1907 cmd_type_len = (IXGBE_ADVTXD_DTYP_DATA | 1908 IXGBE_ADVTXD_DCMD_IFCS | IXGBE_ADVTXD_DCMD_DEXT); 1909 1910 if ((mtag = VLAN_OUTPUT_TAG(ec, m_head)) != NULL) 1911 cmd_type_len |= IXGBE_ADVTXD_DCMD_VLE; 1912 1913 /* 1914 * Important to capture the first descriptor 1915 * used because it will contain the index of 1916 * the one we tell the hardware to report back 1917 */ 1918 first = txr->next_avail_desc; 1919 txbuf = &txr->tx_buffers[first]; 1920 map = txbuf->map; 1921 1922 /* 1923 * Map the packet for DMA. 1924 */ 1925 error = bus_dmamap_load_mbuf(txr->txtag->dt_dmat, map, 1926 m_head, BUS_DMA_NOWAIT); 1927 1928 switch (error) { 1929 case EAGAIN: 1930 adapter->eagain_tx_dma_setup.ev_count++; 1931 return EAGAIN; 1932 case ENOMEM: 1933 adapter->enomem_tx_dma_setup.ev_count++; 1934 return EAGAIN; 1935 case EFBIG: 1936 adapter->efbig_tx_dma_setup.ev_count++; 1937 return error; 1938 case EINVAL: 1939 adapter->einval_tx_dma_setup.ev_count++; 1940 return error; 1941 default: 1942 adapter->other_tx_dma_setup.ev_count++; 1943 return error; 1944 case 0: 1945 break; 1946 } 1947 1948 /* Make certain there are enough descriptors */ 1949 if (map->dm_nsegs > txr->tx_avail - 2) { 1950 txr->no_desc_avail.ev_count++; 1951 ixgbe_dmamap_unload(txr->txtag, txbuf->map); 1952 return EAGAIN; 1953 } 1954 1955 /* 1956 ** Set up the appropriate offload context 1957 ** this becomes the first descriptor of 1958 ** a packet. 1959 */ 1960 if (m_head->m_pkthdr.csum_flags & (M_CSUM_TSOv4|M_CSUM_TSOv6)) { 1961 if (ixgbe_tso_setup(txr, m_head, &paylen, &olinfo_status)) { 1962 cmd_type_len |= IXGBE_ADVTXD_DCMD_TSE; 1963 olinfo_status |= IXGBE_TXD_POPTS_TXSM << 8; 1964 olinfo_status |= paylen << IXGBE_ADVTXD_PAYLEN_SHIFT; 1965 ++adapter->tso_tx.ev_count; 1966 } else { 1967 ++adapter->tso_err.ev_count; 1968 /* XXX unload DMA map! --dyoung */ 1969 return ENXIO; 1970 } 1971 } else 1972 olinfo_status |= ixgbe_tx_ctx_setup(txr, m_head); 1973 1974 #ifdef IXGBE_IEEE1588 1975 /* This is changing soon to an mtag detection */ 1976 if (we detect this mbuf has a TSTAMP mtag) 1977 cmd_type_len |= IXGBE_ADVTXD_MAC_TSTAMP; 1978 #endif 1979 1980 #ifdef IXGBE_FDIR 1981 /* Do the flow director magic */ 1982 if ((txr->atr_sample) && (!adapter->fdir_reinit)) { 1983 ++txr->atr_count; 1984 if (txr->atr_count >= atr_sample_rate) { 1985 ixgbe_atr(txr, m_head); 1986 txr->atr_count = 0; 1987 } 1988 } 1989 #endif 1990 /* Record payload length */ 1991 if (paylen == 0) 1992 olinfo_status |= m_head->m_pkthdr.len << 1993 IXGBE_ADVTXD_PAYLEN_SHIFT; 1994 1995 i = txr->next_avail_desc; 1996 for (j = 0; j < map->dm_nsegs; j++) { 1997 bus_size_t seglen; 1998 bus_addr_t segaddr; 1999 2000 txbuf = &txr->tx_buffers[i]; 2001 txd = &txr->tx_base[i]; 2002 seglen = map->dm_segs[j].ds_len; 2003 segaddr = htole64(map->dm_segs[j].ds_addr); 2004 2005 txd->read.buffer_addr = segaddr; 2006 txd->read.cmd_type_len = htole32(txr->txd_cmd | 2007 cmd_type_len |seglen); 2008 txd->read.olinfo_status = htole32(olinfo_status); 2009 last = i; /* descriptor that will get completion IRQ */ 2010 2011 if (++i == adapter->num_tx_desc) 2012 i = 0; 2013 2014 txbuf->m_head = NULL; 2015 txbuf->eop_index = -1; 2016 } 2017 2018 txd->read.cmd_type_len |= 2019 htole32(IXGBE_TXD_CMD_EOP | IXGBE_TXD_CMD_RS); 2020 txr->tx_avail -= map->dm_nsegs; 2021 txr->next_avail_desc = i; 2022 2023 txbuf->m_head = m_head; 2024 /* Swap the dma map between the first and last descriptor */ 2025 txr->tx_buffers[first].map = txbuf->map; 2026 txbuf->map = map; 2027 bus_dmamap_sync(txr->txtag->dt_dmat, map, 0, m_head->m_pkthdr.len, 2028 BUS_DMASYNC_PREWRITE); 2029 2030 /* Set the index of the descriptor that will be marked done */ 2031 txbuf = &txr->tx_buffers[first]; 2032 txbuf->eop_index = last; 2033 2034 ixgbe_dmamap_sync(txr->txdma.dma_tag, txr->txdma.dma_map, 2035 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); 2036 /* 2037 * Advance the Transmit Descriptor Tail (Tdt), this tells the 2038 * hardware that this frame is available to transmit. 2039 */ 2040 ++txr->total_packets.ev_count; 2041 IXGBE_WRITE_REG(&adapter->hw, IXGBE_TDT(txr->me), i); 2042 2043 return 0; 2044 } 2045 2046 static void 2047 ixgbe_set_promisc(struct adapter *adapter) 2048 { 2049 u_int32_t reg_rctl; 2050 struct ifnet *ifp = adapter->ifp; 2051 2052 reg_rctl = IXGBE_READ_REG(&adapter->hw, IXGBE_FCTRL); 2053 reg_rctl &= (~IXGBE_FCTRL_UPE); 2054 reg_rctl &= (~IXGBE_FCTRL_MPE); 2055 IXGBE_WRITE_REG(&adapter->hw, IXGBE_FCTRL, reg_rctl); 2056 2057 if (ifp->if_flags & IFF_PROMISC) { 2058 reg_rctl |= (IXGBE_FCTRL_UPE | IXGBE_FCTRL_MPE); 2059 IXGBE_WRITE_REG(&adapter->hw, IXGBE_FCTRL, reg_rctl); 2060 } else if (ifp->if_flags & IFF_ALLMULTI) { 2061 reg_rctl |= IXGBE_FCTRL_MPE; 2062 reg_rctl &= ~IXGBE_FCTRL_UPE; 2063 IXGBE_WRITE_REG(&adapter->hw, IXGBE_FCTRL, reg_rctl); 2064 } 2065 return; 2066 } 2067 2068 2069 /********************************************************************* 2070 * Multicast Update 2071 * 2072 * This routine is called whenever multicast address list is updated. 2073 * 2074 **********************************************************************/ 2075 #define IXGBE_RAR_ENTRIES 16 2076 2077 static void 2078 ixgbe_set_multi(struct adapter *adapter) 2079 { 2080 struct ether_multi *enm; 2081 struct ether_multistep step; 2082 u32 fctrl; 2083 u8 *mta; 2084 u8 *update_ptr; 2085 int mcnt = 0; 2086 struct ethercom *ec = &adapter->osdep.ec; 2087 struct ifnet *ifp = adapter->ifp; 2088 2089 IOCTL_DEBUGOUT("ixgbe_set_multi: begin"); 2090 2091 mta = adapter->mta; 2092 bzero(mta, sizeof(u8) * IXGBE_ETH_LENGTH_OF_ADDRESS * 2093 MAX_NUM_MULTICAST_ADDRESSES); 2094 2095 fctrl = IXGBE_READ_REG(&adapter->hw, IXGBE_FCTRL); 2096 fctrl |= (IXGBE_FCTRL_UPE | IXGBE_FCTRL_MPE); 2097 if (ifp->if_flags & IFF_PROMISC) 2098 fctrl |= (IXGBE_FCTRL_UPE | IXGBE_FCTRL_MPE); 2099 else if (ifp->if_flags & IFF_ALLMULTI) { 2100 fctrl |= IXGBE_FCTRL_MPE; 2101 fctrl &= ~IXGBE_FCTRL_UPE; 2102 } else 2103 fctrl &= ~(IXGBE_FCTRL_UPE | IXGBE_FCTRL_MPE); 2104 2105 IXGBE_WRITE_REG(&adapter->hw, IXGBE_FCTRL, fctrl); 2106 2107 ETHER_FIRST_MULTI(step, ec, enm); 2108 while (enm != NULL) { 2109 if (memcmp(enm->enm_addrlo, enm->enm_addrhi, 2110 ETHER_ADDR_LEN) != 0) { 2111 fctrl |= IXGBE_FCTRL_MPE; 2112 IXGBE_WRITE_REG(&adapter->hw, IXGBE_FCTRL, fctrl); 2113 break; 2114 } 2115 bcopy(enm->enm_addrlo, 2116 &mta[mcnt * IXGBE_ETH_LENGTH_OF_ADDRESS], 2117 IXGBE_ETH_LENGTH_OF_ADDRESS); 2118 mcnt++; 2119 ETHER_NEXT_MULTI(step, enm); 2120 } 2121 2122 update_ptr = mta; 2123 ixgbe_update_mc_addr_list(&adapter->hw, 2124 update_ptr, mcnt, ixgbe_mc_array_itr, TRUE); 2125 2126 return; 2127 } 2128 2129 /* 2130 * This is an iterator function now needed by the multicast 2131 * shared code. It simply feeds the shared code routine the 2132 * addresses in the array of ixgbe_set_multi() one by one. 2133 */ 2134 static u8 * 2135 ixgbe_mc_array_itr(struct ixgbe_hw *hw, u8 **update_ptr, u32 *vmdq) 2136 { 2137 u8 *addr = *update_ptr; 2138 u8 *newptr; 2139 *vmdq = 0; 2140 2141 newptr = addr + IXGBE_ETH_LENGTH_OF_ADDRESS; 2142 *update_ptr = newptr; 2143 return addr; 2144 } 2145 2146 2147 /********************************************************************* 2148 * Timer routine 2149 * 2150 * This routine checks for link status,updates statistics, 2151 * and runs the watchdog check. 2152 * 2153 **********************************************************************/ 2154 2155 static void 2156 ixgbe_local_timer1(void *arg) 2157 { 2158 struct adapter *adapter = arg; 2159 device_t dev = adapter->dev; 2160 struct ifnet *ifp = adapter->ifp; 2161 struct ix_queue *que = adapter->queues; 2162 struct tx_ring *txr = adapter->tx_rings; 2163 int hung, busy, paused; 2164 2165 KASSERT(mutex_owned(&adapter->core_mtx)); 2166 hung = busy = paused = 0; 2167 2168 /* Check for pluggable optics */ 2169 if (adapter->sfp_probe) 2170 if (!ixgbe_sfp_probe(adapter)) 2171 goto out; /* Nothing to do */ 2172 2173 ixgbe_update_link_status(adapter); 2174 ixgbe_update_stats_counters(adapter); 2175 2176 /* 2177 * If the interface has been paused 2178 * then don't do the watchdog check 2179 */ 2180 if (IXGBE_READ_REG(&adapter->hw, IXGBE_TFCS) & IXGBE_TFCS_TXOFF) 2181 paused = 1; 2182 2183 /* 2184 ** Check the TX queues status 2185 ** - central locked handling of OACTIVE 2186 ** - watchdog only if all queues show hung 2187 */ 2188 for (int i = 0; i < adapter->num_queues; i++, que++, txr++) { 2189 if ((txr->queue_status & IXGBE_QUEUE_HUNG) && 2190 (paused == 0)) 2191 ++hung; 2192 if (txr->queue_status & IXGBE_QUEUE_DEPLETED) 2193 ++busy; 2194 if ((txr->queue_status & IXGBE_QUEUE_IDLE) == 0) 2195 softint_schedule(que->que_si); 2196 } 2197 /* Only truely watchdog if all queues show hung */ 2198 if (hung == adapter->num_queues) 2199 goto watchdog; 2200 /* Only turn off the stack flow when ALL are depleted */ 2201 if (busy == adapter->num_queues) 2202 ifp->if_flags |= IFF_OACTIVE; 2203 else if ((ifp->if_flags & IFF_OACTIVE) && 2204 (busy < adapter->num_queues)) 2205 ifp->if_flags &= ~IFF_OACTIVE; 2206 2207 out: 2208 ixgbe_rearm_queues(adapter, adapter->que_mask); 2209 callout_reset(&adapter->timer, hz, ixgbe_local_timer, adapter); 2210 return; 2211 2212 watchdog: 2213 device_printf(adapter->dev, "Watchdog timeout -- resetting\n"); 2214 device_printf(dev,"Queue(%d) tdh = %d, hw tdt = %d\n", txr->me, 2215 IXGBE_READ_REG(&adapter->hw, IXGBE_TDH(txr->me)), 2216 IXGBE_READ_REG(&adapter->hw, IXGBE_TDT(txr->me))); 2217 device_printf(dev,"TX(%d) desc avail = %d," 2218 "Next TX to Clean = %d\n", 2219 txr->me, txr->tx_avail, txr->next_to_clean); 2220 adapter->ifp->if_flags &= ~IFF_RUNNING; 2221 adapter->watchdog_events.ev_count++; 2222 ixgbe_init_locked(adapter); 2223 } 2224 2225 static void 2226 ixgbe_local_timer(void *arg) 2227 { 2228 struct adapter *adapter = arg; 2229 2230 IXGBE_CORE_LOCK(adapter); 2231 ixgbe_local_timer1(adapter); 2232 IXGBE_CORE_UNLOCK(adapter); 2233 } 2234 2235 /* 2236 ** Note: this routine updates the OS on the link state 2237 ** the real check of the hardware only happens with 2238 ** a link interrupt. 2239 */ 2240 static void 2241 ixgbe_update_link_status(struct adapter *adapter) 2242 { 2243 struct ifnet *ifp = adapter->ifp; 2244 struct tx_ring *txr = adapter->tx_rings; 2245 device_t dev = adapter->dev; 2246 2247 2248 if (adapter->link_up){ 2249 if (adapter->link_active == FALSE) { 2250 if (bootverbose) 2251 device_printf(dev,"Link is up %d Gbps %s \n", 2252 ((adapter->link_speed == 128)? 10:1), 2253 "Full Duplex"); 2254 adapter->link_active = TRUE; 2255 /* Update any Flow Control changes */ 2256 ixgbe_fc_enable(&adapter->hw); 2257 if_link_state_change(ifp, LINK_STATE_UP); 2258 } 2259 } else { /* Link down */ 2260 if (adapter->link_active == TRUE) { 2261 if (bootverbose) 2262 device_printf(dev,"Link is Down\n"); 2263 if_link_state_change(ifp, LINK_STATE_DOWN); 2264 adapter->link_active = FALSE; 2265 for (int i = 0; i < adapter->num_queues; 2266 i++, txr++) 2267 txr->queue_status = IXGBE_QUEUE_IDLE; 2268 } 2269 } 2270 2271 return; 2272 } 2273 2274 2275 static void 2276 ixgbe_ifstop(struct ifnet *ifp, int disable) 2277 { 2278 struct adapter *adapter = ifp->if_softc; 2279 2280 IXGBE_CORE_LOCK(adapter); 2281 ixgbe_stop(adapter); 2282 IXGBE_CORE_UNLOCK(adapter); 2283 } 2284 2285 /********************************************************************* 2286 * 2287 * This routine disables all traffic on the adapter by issuing a 2288 * global reset on the MAC and deallocates TX/RX buffers. 2289 * 2290 **********************************************************************/ 2291 2292 static void 2293 ixgbe_stop(void *arg) 2294 { 2295 struct ifnet *ifp; 2296 struct adapter *adapter = arg; 2297 struct ixgbe_hw *hw = &adapter->hw; 2298 ifp = adapter->ifp; 2299 2300 KASSERT(mutex_owned(&adapter->core_mtx)); 2301 2302 INIT_DEBUGOUT("ixgbe_stop: begin\n"); 2303 ixgbe_disable_intr(adapter); 2304 callout_stop(&adapter->timer); 2305 2306 /* Let the stack know...*/ 2307 ifp->if_flags &= ~IFF_RUNNING; 2308 ifp->if_flags |= IFF_OACTIVE; 2309 2310 ixgbe_reset_hw(hw); 2311 hw->adapter_stopped = FALSE; 2312 ixgbe_stop_adapter(hw); 2313 /* Turn off the laser */ 2314 if (hw->phy.multispeed_fiber) 2315 ixgbe_disable_tx_laser(hw); 2316 2317 /* reprogram the RAR[0] in case user changed it. */ 2318 ixgbe_set_rar(&adapter->hw, 0, adapter->hw.mac.addr, 0, IXGBE_RAH_AV); 2319 2320 return; 2321 } 2322 2323 2324 /********************************************************************* 2325 * 2326 * Determine hardware revision. 2327 * 2328 **********************************************************************/ 2329 static void 2330 ixgbe_identify_hardware(struct adapter *adapter) 2331 { 2332 pcitag_t tag; 2333 pci_chipset_tag_t pc; 2334 pcireg_t subid, id; 2335 struct ixgbe_hw *hw = &adapter->hw; 2336 2337 pc = adapter->osdep.pc; 2338 tag = adapter->osdep.tag; 2339 2340 id = pci_conf_read(pc, tag, PCI_ID_REG); 2341 subid = pci_conf_read(pc, tag, PCI_SUBSYS_ID_REG); 2342 2343 /* Save off the information about this board */ 2344 hw->vendor_id = PCI_VENDOR(id); 2345 hw->device_id = PCI_PRODUCT(id); 2346 hw->revision_id = 2347 PCI_REVISION(pci_conf_read(pc, tag, PCI_CLASS_REG)); 2348 hw->subsystem_vendor_id = PCI_SUBSYS_VENDOR(subid); 2349 hw->subsystem_device_id = PCI_SUBSYS_ID(subid); 2350 2351 /* We need this here to set the num_segs below */ 2352 ixgbe_set_mac_type(hw); 2353 2354 /* Pick up the 82599 and VF settings */ 2355 if (hw->mac.type != ixgbe_mac_82598EB) { 2356 hw->phy.smart_speed = ixgbe_smart_speed; 2357 adapter->num_segs = IXGBE_82599_SCATTER; 2358 } else 2359 adapter->num_segs = IXGBE_82598_SCATTER; 2360 2361 return; 2362 } 2363 2364 /********************************************************************* 2365 * 2366 * Determine optic type 2367 * 2368 **********************************************************************/ 2369 static void 2370 ixgbe_setup_optics(struct adapter *adapter) 2371 { 2372 struct ixgbe_hw *hw = &adapter->hw; 2373 int layer; 2374 2375 layer = ixgbe_get_supported_physical_layer(hw); 2376 if (layer & IXGBE_PHYSICAL_LAYER_10GBASE_T) { 2377 adapter->optics = IFM_10G_T; 2378 return; 2379 } 2380 2381 if (layer & IXGBE_PHYSICAL_LAYER_1000BASE_T) { 2382 adapter->optics = IFM_1000_T; 2383 return; 2384 } 2385 2386 if (layer & (IXGBE_PHYSICAL_LAYER_10GBASE_LR | 2387 IXGBE_PHYSICAL_LAYER_10GBASE_LRM)) { 2388 adapter->optics = IFM_10G_LR; 2389 return; 2390 } 2391 2392 if (layer & IXGBE_PHYSICAL_LAYER_10GBASE_SR) { 2393 adapter->optics = IFM_10G_SR; 2394 return; 2395 } 2396 2397 if (layer & IXGBE_PHYSICAL_LAYER_SFP_PLUS_CU) { 2398 adapter->optics = IFM_10G_TWINAX; 2399 return; 2400 } 2401 2402 if (layer & (IXGBE_PHYSICAL_LAYER_10GBASE_KX4 | 2403 IXGBE_PHYSICAL_LAYER_10GBASE_CX4)) { 2404 adapter->optics = IFM_10G_CX4; 2405 return; 2406 } 2407 2408 /* If we get here just set the default */ 2409 adapter->optics = IFM_ETHER | IFM_AUTO; 2410 return; 2411 } 2412 2413 /********************************************************************* 2414 * 2415 * Setup the Legacy or MSI Interrupt handler 2416 * 2417 **********************************************************************/ 2418 static int 2419 ixgbe_allocate_legacy(struct adapter *adapter, const struct pci_attach_args *pa) 2420 { 2421 device_t dev = adapter->dev; 2422 struct ix_queue *que = adapter->queues; 2423 char intrbuf[PCI_INTRSTR_LEN]; 2424 #if 0 2425 int rid = 0; 2426 2427 /* MSI RID at 1 */ 2428 if (adapter->msix == 1) 2429 rid = 1; 2430 #endif 2431 2432 /* We allocate a single interrupt resource */ 2433 if (pci_intr_map(pa, &adapter->osdep.ih) != 0) { 2434 aprint_error_dev(dev, "unable to map interrupt\n"); 2435 return ENXIO; 2436 } else { 2437 aprint_normal_dev(dev, "interrupting at %s\n", 2438 pci_intr_string(adapter->osdep.pc, adapter->osdep.ih, 2439 intrbuf, sizeof(intrbuf))); 2440 } 2441 2442 /* 2443 * Try allocating a fast interrupt and the associated deferred 2444 * processing contexts. 2445 */ 2446 que->que_si = softint_establish(SOFTINT_NET, ixgbe_handle_que, que); 2447 2448 /* Tasklets for Link, SFP and Multispeed Fiber */ 2449 adapter->link_si = 2450 softint_establish(SOFTINT_NET, ixgbe_handle_link, adapter); 2451 adapter->mod_si = 2452 softint_establish(SOFTINT_NET, ixgbe_handle_mod, adapter); 2453 adapter->msf_si = 2454 softint_establish(SOFTINT_NET, ixgbe_handle_msf, adapter); 2455 2456 #ifdef IXGBE_FDIR 2457 adapter->fdir_si = 2458 softint_establish(SOFTINT_NET, ixgbe_reinit_fdir, adapter); 2459 #endif 2460 if (que->que_si == NULL || 2461 adapter->link_si == NULL || 2462 adapter->mod_si == NULL || 2463 #ifdef IXGBE_FDIR 2464 adapter->fdir_si == NULL || 2465 #endif 2466 adapter->msf_si == NULL) { 2467 aprint_error_dev(dev, 2468 "could not establish software interrupts\n"); 2469 return ENXIO; 2470 } 2471 2472 adapter->osdep.intr = pci_intr_establish(adapter->osdep.pc, 2473 adapter->osdep.ih, IPL_NET, ixgbe_legacy_irq, que); 2474 if (adapter->osdep.intr == NULL) { 2475 aprint_error_dev(dev, "failed to register interrupt handler\n"); 2476 softint_disestablish(que->que_si); 2477 softint_disestablish(adapter->link_si); 2478 softint_disestablish(adapter->mod_si); 2479 softint_disestablish(adapter->msf_si); 2480 #ifdef IXGBE_FDIR 2481 softint_disestablish(adapter->fdir_si); 2482 #endif 2483 return ENXIO; 2484 } 2485 /* For simplicity in the handlers */ 2486 adapter->que_mask = IXGBE_EIMS_ENABLE_MASK; 2487 2488 return (0); 2489 } 2490 2491 2492 /********************************************************************* 2493 * 2494 * Setup MSIX Interrupt resources and handlers 2495 * 2496 **********************************************************************/ 2497 static int 2498 ixgbe_allocate_msix(struct adapter *adapter, const struct pci_attach_args *pa) 2499 { 2500 #if !defined(NETBSD_MSI_OR_MSIX) 2501 return 0; 2502 #else 2503 device_t dev = adapter->dev; 2504 struct ix_queue *que = adapter->queues; 2505 int error, rid, vector = 0; 2506 2507 for (int i = 0; i < adapter->num_queues; i++, vector++, que++) { 2508 rid = vector + 1; 2509 que->res = bus_alloc_resource_any(dev, SYS_RES_IRQ, &rid, 2510 RF_SHAREABLE | RF_ACTIVE); 2511 if (que->res == NULL) { 2512 aprint_error_dev(dev,"Unable to allocate" 2513 " bus resource: que interrupt [%d]\n", vector); 2514 return (ENXIO); 2515 } 2516 /* Set the handler function */ 2517 error = bus_setup_intr(dev, que->res, 2518 INTR_TYPE_NET | INTR_MPSAFE, NULL, 2519 ixgbe_msix_que, que, &que->tag); 2520 if (error) { 2521 que->res = NULL; 2522 aprint_error_dev(dev, 2523 "Failed to register QUE handler\n"); 2524 return error; 2525 } 2526 #if __FreeBSD_version >= 800504 2527 bus_describe_intr(dev, que->res, que->tag, "que %d", i); 2528 #endif 2529 que->msix = vector; 2530 adapter->que_mask |= (u64)(1 << que->msix); 2531 /* 2532 ** Bind the msix vector, and thus the 2533 ** ring to the corresponding cpu. 2534 */ 2535 if (adapter->num_queues > 1) 2536 bus_bind_intr(dev, que->res, i); 2537 2538 que->que_si = softint_establish(ixgbe_handle_que, que); 2539 if (que->que_si == NULL) { 2540 aprint_error_dev(dev, 2541 "could not establish software interrupt\n"); 2542 } 2543 } 2544 2545 /* and Link */ 2546 rid = vector + 1; 2547 adapter->res = bus_alloc_resource_any(dev, 2548 SYS_RES_IRQ, &rid, RF_SHAREABLE | RF_ACTIVE); 2549 if (!adapter->res) { 2550 aprint_error_dev(dev,"Unable to allocate bus resource: " 2551 "Link interrupt [%d]\n", rid); 2552 return (ENXIO); 2553 } 2554 /* Set the link handler function */ 2555 error = bus_setup_intr(dev, adapter->res, 2556 INTR_TYPE_NET | INTR_MPSAFE, NULL, 2557 ixgbe_msix_link, adapter, &adapter->tag); 2558 if (error) { 2559 adapter->res = NULL; 2560 aprint_error_dev(dev, "Failed to register LINK handler\n"); 2561 return (error); 2562 } 2563 #if __FreeBSD_version >= 800504 2564 bus_describe_intr(dev, adapter->res, adapter->tag, "link"); 2565 #endif 2566 adapter->linkvec = vector; 2567 /* Tasklets for Link, SFP and Multispeed Fiber */ 2568 adapter->link_si = 2569 softint_establish(SOFTINT_NET, ixgbe_handle_link, adapter); 2570 adapter->mod_si = 2571 softint_establish(SOFTINT_NET, ixgbe_handle_mod, adapter); 2572 adapter->msf_si = 2573 softint_establish(SOFTINT_NET, ixgbe_handle_msf, adapter); 2574 #ifdef IXGBE_FDIR 2575 adapter->fdir_si = 2576 softint_establish(SOFTINT_NET, ixgbe_reinit_fdir, adapter); 2577 #endif 2578 2579 return (0); 2580 #endif 2581 } 2582 2583 /* 2584 * Setup Either MSI/X or MSI 2585 */ 2586 static int 2587 ixgbe_setup_msix(struct adapter *adapter) 2588 { 2589 #if !defined(NETBSD_MSI_OR_MSIX) 2590 return 0; 2591 #else 2592 device_t dev = adapter->dev; 2593 int rid, want, queues, msgs; 2594 2595 /* Override by tuneable */ 2596 if (ixgbe_enable_msix == 0) 2597 goto msi; 2598 2599 /* First try MSI/X */ 2600 rid = PCI_BAR(MSIX_82598_BAR); 2601 adapter->msix_mem = bus_alloc_resource_any(dev, 2602 SYS_RES_MEMORY, &rid, RF_ACTIVE); 2603 if (!adapter->msix_mem) { 2604 rid += 4; /* 82599 maps in higher BAR */ 2605 adapter->msix_mem = bus_alloc_resource_any(dev, 2606 SYS_RES_MEMORY, &rid, RF_ACTIVE); 2607 } 2608 if (!adapter->msix_mem) { 2609 /* May not be enabled */ 2610 device_printf(adapter->dev, 2611 "Unable to map MSIX table \n"); 2612 goto msi; 2613 } 2614 2615 msgs = pci_msix_count(dev); 2616 if (msgs == 0) { /* system has msix disabled */ 2617 bus_release_resource(dev, SYS_RES_MEMORY, 2618 rid, adapter->msix_mem); 2619 adapter->msix_mem = NULL; 2620 goto msi; 2621 } 2622 2623 /* Figure out a reasonable auto config value */ 2624 queues = (mp_ncpus > (msgs-1)) ? (msgs-1) : mp_ncpus; 2625 2626 if (ixgbe_num_queues != 0) 2627 queues = ixgbe_num_queues; 2628 /* Set max queues to 8 when autoconfiguring */ 2629 else if ((ixgbe_num_queues == 0) && (queues > 8)) 2630 queues = 8; 2631 2632 /* 2633 ** Want one vector (RX/TX pair) per queue 2634 ** plus an additional for Link. 2635 */ 2636 want = queues + 1; 2637 if (msgs >= want) 2638 msgs = want; 2639 else { 2640 device_printf(adapter->dev, 2641 "MSIX Configuration Problem, " 2642 "%d vectors but %d queues wanted!\n", 2643 msgs, want); 2644 return (0); /* Will go to Legacy setup */ 2645 } 2646 if ((msgs) && pci_alloc_msix(dev, &msgs) == 0) { 2647 device_printf(adapter->dev, 2648 "Using MSIX interrupts with %d vectors\n", msgs); 2649 adapter->num_queues = queues; 2650 return (msgs); 2651 } 2652 msi: 2653 msgs = pci_msi_count(dev); 2654 if (msgs == 1 && pci_alloc_msi(dev, &msgs) == 0) 2655 device_printf(adapter->dev,"Using an MSI interrupt\n"); 2656 else 2657 device_printf(adapter->dev,"Using a Legacy interrupt\n"); 2658 return (msgs); 2659 #endif 2660 } 2661 2662 2663 static int 2664 ixgbe_allocate_pci_resources(struct adapter *adapter, const struct pci_attach_args *pa) 2665 { 2666 pcireg_t memtype; 2667 device_t dev = adapter->dev; 2668 bus_addr_t addr; 2669 int flags; 2670 2671 memtype = pci_mapreg_type(pa->pa_pc, pa->pa_tag, PCI_BAR(0)); 2672 switch (memtype) { 2673 case PCI_MAPREG_TYPE_MEM | PCI_MAPREG_MEM_TYPE_32BIT: 2674 case PCI_MAPREG_TYPE_MEM | PCI_MAPREG_MEM_TYPE_64BIT: 2675 adapter->osdep.mem_bus_space_tag = pa->pa_memt; 2676 if (pci_mapreg_info(pa->pa_pc, pa->pa_tag, PCI_BAR(0), 2677 memtype, &addr, &adapter->osdep.mem_size, &flags) != 0) 2678 goto map_err; 2679 if ((flags & BUS_SPACE_MAP_PREFETCHABLE) != 0) { 2680 aprint_normal_dev(dev, "clearing prefetchable bit\n"); 2681 flags &= ~BUS_SPACE_MAP_PREFETCHABLE; 2682 } 2683 if (bus_space_map(adapter->osdep.mem_bus_space_tag, addr, 2684 adapter->osdep.mem_size, flags, 2685 &adapter->osdep.mem_bus_space_handle) != 0) { 2686 map_err: 2687 adapter->osdep.mem_size = 0; 2688 aprint_error_dev(dev, "unable to map BAR0\n"); 2689 return ENXIO; 2690 } 2691 break; 2692 default: 2693 aprint_error_dev(dev, "unexpected type on BAR0\n"); 2694 return ENXIO; 2695 } 2696 2697 /* Legacy defaults */ 2698 adapter->num_queues = 1; 2699 adapter->hw.back = &adapter->osdep; 2700 2701 /* 2702 ** Now setup MSI or MSI/X, should 2703 ** return us the number of supported 2704 ** vectors. (Will be 1 for MSI) 2705 */ 2706 adapter->msix = ixgbe_setup_msix(adapter); 2707 return (0); 2708 } 2709 2710 static void 2711 ixgbe_free_pci_resources(struct adapter * adapter) 2712 { 2713 #if defined(NETBSD_MSI_OR_MSIX) 2714 struct ix_queue *que = adapter->queues; 2715 device_t dev = adapter->dev; 2716 #endif 2717 int rid; 2718 2719 #if defined(NETBSD_MSI_OR_MSIX) 2720 int memrid; 2721 if (adapter->hw.mac.type == ixgbe_mac_82598EB) 2722 memrid = PCI_BAR(MSIX_82598_BAR); 2723 else 2724 memrid = PCI_BAR(MSIX_82599_BAR); 2725 2726 /* 2727 ** There is a slight possibility of a failure mode 2728 ** in attach that will result in entering this function 2729 ** before interrupt resources have been initialized, and 2730 ** in that case we do not want to execute the loops below 2731 ** We can detect this reliably by the state of the adapter 2732 ** res pointer. 2733 */ 2734 if (adapter->res == NULL) 2735 goto mem; 2736 2737 /* 2738 ** Release all msix queue resources: 2739 */ 2740 for (int i = 0; i < adapter->num_queues; i++, que++) { 2741 rid = que->msix + 1; 2742 if (que->tag != NULL) { 2743 bus_teardown_intr(dev, que->res, que->tag); 2744 que->tag = NULL; 2745 } 2746 if (que->res != NULL) 2747 bus_release_resource(dev, SYS_RES_IRQ, rid, que->res); 2748 } 2749 #endif 2750 2751 /* Clean the Legacy or Link interrupt last */ 2752 if (adapter->linkvec) /* we are doing MSIX */ 2753 rid = adapter->linkvec + 1; 2754 else 2755 (adapter->msix != 0) ? (rid = 1):(rid = 0); 2756 2757 pci_intr_disestablish(adapter->osdep.pc, adapter->osdep.intr); 2758 adapter->osdep.intr = NULL; 2759 2760 #if defined(NETBSD_MSI_OR_MSIX) 2761 mem: 2762 if (adapter->msix) 2763 pci_release_msi(dev); 2764 2765 if (adapter->msix_mem != NULL) 2766 bus_release_resource(dev, SYS_RES_MEMORY, 2767 memrid, adapter->msix_mem); 2768 #endif 2769 2770 if (adapter->osdep.mem_size != 0) { 2771 bus_space_unmap(adapter->osdep.mem_bus_space_tag, 2772 adapter->osdep.mem_bus_space_handle, 2773 adapter->osdep.mem_size); 2774 } 2775 2776 return; 2777 } 2778 2779 /********************************************************************* 2780 * 2781 * Setup networking device structure and register an interface. 2782 * 2783 **********************************************************************/ 2784 static int 2785 ixgbe_setup_interface(device_t dev, struct adapter *adapter) 2786 { 2787 struct ethercom *ec = &adapter->osdep.ec; 2788 struct ixgbe_hw *hw = &adapter->hw; 2789 struct ifnet *ifp; 2790 2791 INIT_DEBUGOUT("ixgbe_setup_interface: begin"); 2792 2793 ifp = adapter->ifp = &ec->ec_if; 2794 strlcpy(ifp->if_xname, device_xname(dev), IFNAMSIZ); 2795 ifp->if_baudrate = 1000000000; 2796 ifp->if_init = ixgbe_init; 2797 ifp->if_stop = ixgbe_ifstop; 2798 ifp->if_softc = adapter; 2799 ifp->if_flags = IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST; 2800 ifp->if_ioctl = ixgbe_ioctl; 2801 ifp->if_start = ixgbe_start; 2802 #if __FreeBSD_version >= 800000 2803 ifp->if_transmit = ixgbe_mq_start; 2804 ifp->if_qflush = ixgbe_qflush; 2805 #endif 2806 ifp->if_snd.ifq_maxlen = adapter->num_tx_desc - 2; 2807 2808 if_attach(ifp); 2809 ether_ifattach(ifp, adapter->hw.mac.addr); 2810 ether_set_ifflags_cb(ec, ixgbe_ifflags_cb); 2811 2812 adapter->max_frame_size = 2813 ifp->if_mtu + ETHER_HDR_LEN + ETHER_CRC_LEN; 2814 2815 /* 2816 * Tell the upper layer(s) we support long frames. 2817 */ 2818 ifp->if_hdrlen = sizeof(struct ether_vlan_header); 2819 2820 ifp->if_capabilities |= IFCAP_HWCSUM | IFCAP_TSOv4 | IFCAP_TSOv6; 2821 ifp->if_capenable = 0; 2822 2823 ec->ec_capabilities |= ETHERCAP_VLAN_HWCSUM; 2824 ec->ec_capabilities |= ETHERCAP_JUMBO_MTU; 2825 ec->ec_capabilities |= ETHERCAP_VLAN_HWTAGGING 2826 | ETHERCAP_VLAN_MTU; 2827 ec->ec_capenable = ec->ec_capabilities; 2828 2829 /* Don't enable LRO by default */ 2830 ifp->if_capabilities |= IFCAP_LRO; 2831 2832 /* 2833 ** Don't turn this on by default, if vlans are 2834 ** created on another pseudo device (eg. lagg) 2835 ** then vlan events are not passed thru, breaking 2836 ** operation, but with HW FILTER off it works. If 2837 ** using vlans directly on the ixgbe driver you can 2838 ** enable this and get full hardware tag filtering. 2839 */ 2840 ec->ec_capabilities |= ETHERCAP_VLAN_HWFILTER; 2841 2842 /* 2843 * Specify the media types supported by this adapter and register 2844 * callbacks to update media and link information 2845 */ 2846 ifmedia_init(&adapter->media, IFM_IMASK, ixgbe_media_change, 2847 ixgbe_media_status); 2848 ifmedia_add(&adapter->media, IFM_ETHER | adapter->optics, 0, NULL); 2849 ifmedia_set(&adapter->media, IFM_ETHER | adapter->optics); 2850 if (hw->device_id == IXGBE_DEV_ID_82598AT) { 2851 ifmedia_add(&adapter->media, 2852 IFM_ETHER | IFM_1000_T | IFM_FDX, 0, NULL); 2853 ifmedia_add(&adapter->media, 2854 IFM_ETHER | IFM_1000_T, 0, NULL); 2855 } 2856 ifmedia_add(&adapter->media, IFM_ETHER | IFM_AUTO, 0, NULL); 2857 ifmedia_set(&adapter->media, IFM_ETHER | IFM_AUTO); 2858 2859 return (0); 2860 } 2861 2862 static void 2863 ixgbe_config_link(struct adapter *adapter) 2864 { 2865 struct ixgbe_hw *hw = &adapter->hw; 2866 u32 autoneg, err = 0; 2867 bool sfp, negotiate; 2868 2869 sfp = ixgbe_is_sfp(hw); 2870 2871 if (sfp) { 2872 void *ip; 2873 2874 if (hw->phy.multispeed_fiber) { 2875 hw->mac.ops.setup_sfp(hw); 2876 ixgbe_enable_tx_laser(hw); 2877 ip = adapter->msf_si; 2878 } else { 2879 ip = adapter->mod_si; 2880 } 2881 2882 kpreempt_disable(); 2883 softint_schedule(ip); 2884 kpreempt_enable(); 2885 } else { 2886 if (hw->mac.ops.check_link) 2887 err = ixgbe_check_link(hw, &autoneg, 2888 &adapter->link_up, FALSE); 2889 if (err) 2890 goto out; 2891 autoneg = hw->phy.autoneg_advertised; 2892 if ((!autoneg) && (hw->mac.ops.get_link_capabilities)) 2893 err = hw->mac.ops.get_link_capabilities(hw, 2894 &autoneg, &negotiate); 2895 else 2896 negotiate = 0; 2897 if (err) 2898 goto out; 2899 if (hw->mac.ops.setup_link) 2900 err = hw->mac.ops.setup_link(hw, autoneg, 2901 negotiate, adapter->link_up); 2902 } 2903 out: 2904 return; 2905 } 2906 2907 /******************************************************************** 2908 * Manage DMA'able memory. 2909 *******************************************************************/ 2910 2911 static int 2912 ixgbe_dma_malloc(struct adapter *adapter, const bus_size_t size, 2913 struct ixgbe_dma_alloc *dma, const int mapflags) 2914 { 2915 device_t dev = adapter->dev; 2916 int r, rsegs; 2917 2918 r = ixgbe_dma_tag_create(adapter->osdep.dmat, /* parent */ 2919 DBA_ALIGN, 0, /* alignment, bounds */ 2920 size, /* maxsize */ 2921 1, /* nsegments */ 2922 size, /* maxsegsize */ 2923 BUS_DMA_ALLOCNOW, /* flags */ 2924 &dma->dma_tag); 2925 if (r != 0) { 2926 aprint_error_dev(dev, 2927 "%s: ixgbe_dma_tag_create failed; error %d\n", __func__, r); 2928 goto fail_0; 2929 } 2930 2931 r = bus_dmamem_alloc(dma->dma_tag->dt_dmat, 2932 size, 2933 dma->dma_tag->dt_alignment, 2934 dma->dma_tag->dt_boundary, 2935 &dma->dma_seg, 1, &rsegs, BUS_DMA_NOWAIT); 2936 if (r != 0) { 2937 aprint_error_dev(dev, 2938 "%s: bus_dmamem_alloc failed; error %d\n", __func__, r); 2939 goto fail_1; 2940 } 2941 2942 r = bus_dmamem_map(dma->dma_tag->dt_dmat, &dma->dma_seg, rsegs, 2943 size, &dma->dma_vaddr, BUS_DMA_NOWAIT); 2944 if (r != 0) { 2945 aprint_error_dev(dev, "%s: bus_dmamem_map failed; error %d\n", 2946 __func__, r); 2947 goto fail_2; 2948 } 2949 2950 r = ixgbe_dmamap_create(dma->dma_tag, 0, &dma->dma_map); 2951 if (r != 0) { 2952 aprint_error_dev(dev, "%s: bus_dmamem_map failed; error %d\n", 2953 __func__, r); 2954 goto fail_3; 2955 } 2956 2957 r = bus_dmamap_load(dma->dma_tag->dt_dmat, dma->dma_map, dma->dma_vaddr, 2958 size, 2959 NULL, 2960 mapflags | BUS_DMA_NOWAIT); 2961 if (r != 0) { 2962 aprint_error_dev(dev, "%s: bus_dmamap_load failed; error %d\n", 2963 __func__, r); 2964 goto fail_4; 2965 } 2966 dma->dma_paddr = dma->dma_map->dm_segs[0].ds_addr; 2967 dma->dma_size = size; 2968 return 0; 2969 fail_4: 2970 ixgbe_dmamap_destroy(dma->dma_tag, dma->dma_map); 2971 fail_3: 2972 bus_dmamem_unmap(dma->dma_tag->dt_dmat, dma->dma_vaddr, size); 2973 fail_2: 2974 bus_dmamem_free(dma->dma_tag->dt_dmat, &dma->dma_seg, rsegs); 2975 fail_1: 2976 ixgbe_dma_tag_destroy(dma->dma_tag); 2977 fail_0: 2978 return r; 2979 } 2980 2981 static void 2982 ixgbe_dma_free(struct adapter *adapter, struct ixgbe_dma_alloc *dma) 2983 { 2984 bus_dmamap_sync(dma->dma_tag->dt_dmat, dma->dma_map, 0, dma->dma_size, 2985 BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE); 2986 ixgbe_dmamap_unload(dma->dma_tag, dma->dma_map); 2987 bus_dmamem_free(dma->dma_tag->dt_dmat, &dma->dma_seg, 1); 2988 ixgbe_dma_tag_destroy(dma->dma_tag); 2989 } 2990 2991 2992 /********************************************************************* 2993 * 2994 * Allocate memory for the transmit and receive rings, and then 2995 * the descriptors associated with each, called only once at attach. 2996 * 2997 **********************************************************************/ 2998 static int 2999 ixgbe_allocate_queues(struct adapter *adapter) 3000 { 3001 device_t dev = adapter->dev; 3002 struct ix_queue *que; 3003 struct tx_ring *txr; 3004 struct rx_ring *rxr; 3005 int rsize, tsize, error = IXGBE_SUCCESS; 3006 int txconf = 0, rxconf = 0; 3007 3008 /* First allocate the top level queue structs */ 3009 if (!(adapter->queues = 3010 (struct ix_queue *) malloc(sizeof(struct ix_queue) * 3011 adapter->num_queues, M_DEVBUF, M_NOWAIT | M_ZERO))) { 3012 aprint_error_dev(dev, "Unable to allocate queue memory\n"); 3013 error = ENOMEM; 3014 goto fail; 3015 } 3016 3017 /* First allocate the TX ring struct memory */ 3018 if (!(adapter->tx_rings = 3019 (struct tx_ring *) malloc(sizeof(struct tx_ring) * 3020 adapter->num_queues, M_DEVBUF, M_NOWAIT | M_ZERO))) { 3021 aprint_error_dev(dev, "Unable to allocate TX ring memory\n"); 3022 error = ENOMEM; 3023 goto tx_fail; 3024 } 3025 3026 /* Next allocate the RX */ 3027 if (!(adapter->rx_rings = 3028 (struct rx_ring *) malloc(sizeof(struct rx_ring) * 3029 adapter->num_queues, M_DEVBUF, M_NOWAIT | M_ZERO))) { 3030 aprint_error_dev(dev, "Unable to allocate RX ring memory\n"); 3031 error = ENOMEM; 3032 goto rx_fail; 3033 } 3034 3035 /* For the ring itself */ 3036 tsize = roundup2(adapter->num_tx_desc * 3037 sizeof(union ixgbe_adv_tx_desc), DBA_ALIGN); 3038 3039 /* 3040 * Now set up the TX queues, txconf is needed to handle the 3041 * possibility that things fail midcourse and we need to 3042 * undo memory gracefully 3043 */ 3044 for (int i = 0; i < adapter->num_queues; i++, txconf++) { 3045 /* Set up some basics */ 3046 txr = &adapter->tx_rings[i]; 3047 txr->adapter = adapter; 3048 txr->me = i; 3049 3050 /* Initialize the TX side lock */ 3051 snprintf(txr->mtx_name, sizeof(txr->mtx_name), "%s:tx(%d)", 3052 device_xname(dev), txr->me); 3053 mutex_init(&txr->tx_mtx, MUTEX_DEFAULT, IPL_NET); 3054 3055 if (ixgbe_dma_malloc(adapter, tsize, 3056 &txr->txdma, BUS_DMA_NOWAIT)) { 3057 aprint_error_dev(dev, 3058 "Unable to allocate TX Descriptor memory\n"); 3059 error = ENOMEM; 3060 goto err_tx_desc; 3061 } 3062 txr->tx_base = (union ixgbe_adv_tx_desc *)txr->txdma.dma_vaddr; 3063 bzero((void *)txr->tx_base, tsize); 3064 3065 /* Now allocate transmit buffers for the ring */ 3066 if (ixgbe_allocate_transmit_buffers(txr)) { 3067 aprint_error_dev(dev, 3068 "Critical Failure setting up transmit buffers\n"); 3069 error = ENOMEM; 3070 goto err_tx_desc; 3071 } 3072 #if __FreeBSD_version >= 800000 3073 /* Allocate a buf ring */ 3074 txr->br = buf_ring_alloc(IXGBE_BR_SIZE, M_DEVBUF, 3075 M_WAITOK, &txr->tx_mtx); 3076 if (txr->br == NULL) { 3077 aprint_error_dev(dev, 3078 "Critical Failure setting up buf ring\n"); 3079 error = ENOMEM; 3080 goto err_tx_desc; 3081 } 3082 #endif 3083 } 3084 3085 /* 3086 * Next the RX queues... 3087 */ 3088 rsize = roundup2(adapter->num_rx_desc * 3089 sizeof(union ixgbe_adv_rx_desc), DBA_ALIGN); 3090 for (int i = 0; i < adapter->num_queues; i++, rxconf++) { 3091 rxr = &adapter->rx_rings[i]; 3092 /* Set up some basics */ 3093 rxr->adapter = adapter; 3094 rxr->me = i; 3095 3096 /* Initialize the RX side lock */ 3097 snprintf(rxr->mtx_name, sizeof(rxr->mtx_name), "%s:rx(%d)", 3098 device_xname(dev), rxr->me); 3099 mutex_init(&rxr->rx_mtx, MUTEX_DEFAULT, IPL_NET); 3100 3101 if (ixgbe_dma_malloc(adapter, rsize, 3102 &rxr->rxdma, BUS_DMA_NOWAIT)) { 3103 aprint_error_dev(dev, 3104 "Unable to allocate RxDescriptor memory\n"); 3105 error = ENOMEM; 3106 goto err_rx_desc; 3107 } 3108 rxr->rx_base = (union ixgbe_adv_rx_desc *)rxr->rxdma.dma_vaddr; 3109 bzero((void *)rxr->rx_base, rsize); 3110 3111 /* Allocate receive buffers for the ring*/ 3112 if (ixgbe_allocate_receive_buffers(rxr)) { 3113 aprint_error_dev(dev, 3114 "Critical Failure setting up receive buffers\n"); 3115 error = ENOMEM; 3116 goto err_rx_desc; 3117 } 3118 } 3119 3120 /* 3121 ** Finally set up the queue holding structs 3122 */ 3123 for (int i = 0; i < adapter->num_queues; i++) { 3124 que = &adapter->queues[i]; 3125 que->adapter = adapter; 3126 que->txr = &adapter->tx_rings[i]; 3127 que->rxr = &adapter->rx_rings[i]; 3128 } 3129 3130 return (0); 3131 3132 err_rx_desc: 3133 for (rxr = adapter->rx_rings; rxconf > 0; rxr++, rxconf--) 3134 ixgbe_dma_free(adapter, &rxr->rxdma); 3135 err_tx_desc: 3136 for (txr = adapter->tx_rings; txconf > 0; txr++, txconf--) 3137 ixgbe_dma_free(adapter, &txr->txdma); 3138 free(adapter->rx_rings, M_DEVBUF); 3139 rx_fail: 3140 free(adapter->tx_rings, M_DEVBUF); 3141 tx_fail: 3142 free(adapter->queues, M_DEVBUF); 3143 fail: 3144 return (error); 3145 } 3146 3147 /********************************************************************* 3148 * 3149 * Allocate memory for tx_buffer structures. The tx_buffer stores all 3150 * the information needed to transmit a packet on the wire. This is 3151 * called only once at attach, setup is done every reset. 3152 * 3153 **********************************************************************/ 3154 static int 3155 ixgbe_allocate_transmit_buffers(struct tx_ring *txr) 3156 { 3157 struct adapter *adapter = txr->adapter; 3158 device_t dev = adapter->dev; 3159 struct ixgbe_tx_buf *txbuf; 3160 int error, i; 3161 3162 /* 3163 * Setup DMA descriptor areas. 3164 */ 3165 if ((error = ixgbe_dma_tag_create(adapter->osdep.dmat, /* parent */ 3166 1, 0, /* alignment, bounds */ 3167 IXGBE_TSO_SIZE, /* maxsize */ 3168 adapter->num_segs, /* nsegments */ 3169 PAGE_SIZE, /* maxsegsize */ 3170 0, /* flags */ 3171 &txr->txtag))) { 3172 aprint_error_dev(dev,"Unable to allocate TX DMA tag\n"); 3173 goto fail; 3174 } 3175 3176 if (!(txr->tx_buffers = 3177 (struct ixgbe_tx_buf *) malloc(sizeof(struct ixgbe_tx_buf) * 3178 adapter->num_tx_desc, M_DEVBUF, M_NOWAIT | M_ZERO))) { 3179 aprint_error_dev(dev, "Unable to allocate tx_buffer memory\n"); 3180 error = ENOMEM; 3181 goto fail; 3182 } 3183 3184 /* Create the descriptor buffer dma maps */ 3185 txbuf = txr->tx_buffers; 3186 for (i = 0; i < adapter->num_tx_desc; i++, txbuf++) { 3187 error = ixgbe_dmamap_create(txr->txtag, 0, &txbuf->map); 3188 if (error != 0) { 3189 aprint_error_dev(dev, 3190 "Unable to create TX DMA map (%d)\n", error); 3191 goto fail; 3192 } 3193 } 3194 3195 return 0; 3196 fail: 3197 /* We free all, it handles case where we are in the middle */ 3198 ixgbe_free_transmit_structures(adapter); 3199 return (error); 3200 } 3201 3202 /********************************************************************* 3203 * 3204 * Initialize a transmit ring. 3205 * 3206 **********************************************************************/ 3207 static void 3208 ixgbe_setup_transmit_ring(struct tx_ring *txr) 3209 { 3210 struct adapter *adapter = txr->adapter; 3211 struct ixgbe_tx_buf *txbuf; 3212 int i; 3213 #ifdef DEV_NETMAP 3214 struct netmap_adapter *na = NA(adapter->ifp); 3215 struct netmap_slot *slot; 3216 #endif /* DEV_NETMAP */ 3217 3218 /* Clear the old ring contents */ 3219 IXGBE_TX_LOCK(txr); 3220 #ifdef DEV_NETMAP 3221 /* 3222 * (under lock): if in netmap mode, do some consistency 3223 * checks and set slot to entry 0 of the netmap ring. 3224 */ 3225 slot = netmap_reset(na, NR_TX, txr->me, 0); 3226 #endif /* DEV_NETMAP */ 3227 bzero((void *)txr->tx_base, 3228 (sizeof(union ixgbe_adv_tx_desc)) * adapter->num_tx_desc); 3229 /* Reset indices */ 3230 txr->next_avail_desc = 0; 3231 txr->next_to_clean = 0; 3232 3233 /* Free any existing tx buffers. */ 3234 txbuf = txr->tx_buffers; 3235 for (i = 0; i < adapter->num_tx_desc; i++, txbuf++) { 3236 if (txbuf->m_head != NULL) { 3237 bus_dmamap_sync(txr->txtag->dt_dmat, txbuf->map, 3238 0, txbuf->m_head->m_pkthdr.len, 3239 BUS_DMASYNC_POSTWRITE); 3240 ixgbe_dmamap_unload(txr->txtag, txbuf->map); 3241 m_freem(txbuf->m_head); 3242 txbuf->m_head = NULL; 3243 } 3244 #ifdef DEV_NETMAP 3245 /* 3246 * In netmap mode, set the map for the packet buffer. 3247 * NOTE: Some drivers (not this one) also need to set 3248 * the physical buffer address in the NIC ring. 3249 * Slots in the netmap ring (indexed by "si") are 3250 * kring->nkr_hwofs positions "ahead" wrt the 3251 * corresponding slot in the NIC ring. In some drivers 3252 * (not here) nkr_hwofs can be negative. Function 3253 * netmap_idx_n2k() handles wraparounds properly. 3254 */ 3255 if (slot) { 3256 int si = netmap_idx_n2k(&na->tx_rings[txr->me], i); 3257 netmap_load_map(txr->txtag, txbuf->map, NMB(slot + si)); 3258 } 3259 #endif /* DEV_NETMAP */ 3260 /* Clear the EOP index */ 3261 txbuf->eop_index = -1; 3262 } 3263 3264 #ifdef IXGBE_FDIR 3265 /* Set the rate at which we sample packets */ 3266 if (adapter->hw.mac.type != ixgbe_mac_82598EB) 3267 txr->atr_sample = atr_sample_rate; 3268 #endif 3269 3270 /* Set number of descriptors available */ 3271 txr->tx_avail = adapter->num_tx_desc; 3272 3273 ixgbe_dmamap_sync(txr->txdma.dma_tag, txr->txdma.dma_map, 3274 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); 3275 IXGBE_TX_UNLOCK(txr); 3276 } 3277 3278 /********************************************************************* 3279 * 3280 * Initialize all transmit rings. 3281 * 3282 **********************************************************************/ 3283 static int 3284 ixgbe_setup_transmit_structures(struct adapter *adapter) 3285 { 3286 struct tx_ring *txr = adapter->tx_rings; 3287 3288 for (int i = 0; i < adapter->num_queues; i++, txr++) 3289 ixgbe_setup_transmit_ring(txr); 3290 3291 return (0); 3292 } 3293 3294 /********************************************************************* 3295 * 3296 * Enable transmit unit. 3297 * 3298 **********************************************************************/ 3299 static void 3300 ixgbe_initialize_transmit_units(struct adapter *adapter) 3301 { 3302 struct tx_ring *txr = adapter->tx_rings; 3303 struct ixgbe_hw *hw = &adapter->hw; 3304 3305 /* Setup the Base and Length of the Tx Descriptor Ring */ 3306 3307 for (int i = 0; i < adapter->num_queues; i++, txr++) { 3308 u64 tdba = txr->txdma.dma_paddr; 3309 u32 txctrl; 3310 3311 IXGBE_WRITE_REG(hw, IXGBE_TDBAL(i), 3312 (tdba & 0x00000000ffffffffULL)); 3313 IXGBE_WRITE_REG(hw, IXGBE_TDBAH(i), (tdba >> 32)); 3314 IXGBE_WRITE_REG(hw, IXGBE_TDLEN(i), 3315 adapter->num_tx_desc * sizeof(struct ixgbe_legacy_tx_desc)); 3316 3317 /* Setup the HW Tx Head and Tail descriptor pointers */ 3318 IXGBE_WRITE_REG(hw, IXGBE_TDH(i), 0); 3319 IXGBE_WRITE_REG(hw, IXGBE_TDT(i), 0); 3320 3321 /* Setup Transmit Descriptor Cmd Settings */ 3322 txr->txd_cmd = IXGBE_TXD_CMD_IFCS; 3323 txr->queue_status = IXGBE_QUEUE_IDLE; 3324 3325 /* Disable Head Writeback */ 3326 switch (hw->mac.type) { 3327 case ixgbe_mac_82598EB: 3328 txctrl = IXGBE_READ_REG(hw, IXGBE_DCA_TXCTRL(i)); 3329 break; 3330 case ixgbe_mac_82599EB: 3331 case ixgbe_mac_X540: 3332 default: 3333 txctrl = IXGBE_READ_REG(hw, IXGBE_DCA_TXCTRL_82599(i)); 3334 break; 3335 } 3336 txctrl &= ~IXGBE_DCA_TXCTRL_DESC_WRO_EN; 3337 switch (hw->mac.type) { 3338 case ixgbe_mac_82598EB: 3339 IXGBE_WRITE_REG(hw, IXGBE_DCA_TXCTRL(i), txctrl); 3340 break; 3341 case ixgbe_mac_82599EB: 3342 case ixgbe_mac_X540: 3343 default: 3344 IXGBE_WRITE_REG(hw, IXGBE_DCA_TXCTRL_82599(i), txctrl); 3345 break; 3346 } 3347 3348 } 3349 3350 if (hw->mac.type != ixgbe_mac_82598EB) { 3351 u32 dmatxctl, rttdcs; 3352 dmatxctl = IXGBE_READ_REG(hw, IXGBE_DMATXCTL); 3353 dmatxctl |= IXGBE_DMATXCTL_TE; 3354 IXGBE_WRITE_REG(hw, IXGBE_DMATXCTL, dmatxctl); 3355 /* Disable arbiter to set MTQC */ 3356 rttdcs = IXGBE_READ_REG(hw, IXGBE_RTTDCS); 3357 rttdcs |= IXGBE_RTTDCS_ARBDIS; 3358 IXGBE_WRITE_REG(hw, IXGBE_RTTDCS, rttdcs); 3359 IXGBE_WRITE_REG(hw, IXGBE_MTQC, IXGBE_MTQC_64Q_1PB); 3360 rttdcs &= ~IXGBE_RTTDCS_ARBDIS; 3361 IXGBE_WRITE_REG(hw, IXGBE_RTTDCS, rttdcs); 3362 } 3363 3364 return; 3365 } 3366 3367 /********************************************************************* 3368 * 3369 * Free all transmit rings. 3370 * 3371 **********************************************************************/ 3372 static void 3373 ixgbe_free_transmit_structures(struct adapter *adapter) 3374 { 3375 struct tx_ring *txr = adapter->tx_rings; 3376 3377 for (int i = 0; i < adapter->num_queues; i++, txr++) { 3378 ixgbe_free_transmit_buffers(txr); 3379 ixgbe_dma_free(adapter, &txr->txdma); 3380 IXGBE_TX_LOCK_DESTROY(txr); 3381 } 3382 free(adapter->tx_rings, M_DEVBUF); 3383 } 3384 3385 /********************************************************************* 3386 * 3387 * Free transmit ring related data structures. 3388 * 3389 **********************************************************************/ 3390 static void 3391 ixgbe_free_transmit_buffers(struct tx_ring *txr) 3392 { 3393 struct adapter *adapter = txr->adapter; 3394 struct ixgbe_tx_buf *tx_buffer; 3395 int i; 3396 3397 INIT_DEBUGOUT("free_transmit_ring: begin"); 3398 3399 if (txr->tx_buffers == NULL) 3400 return; 3401 3402 tx_buffer = txr->tx_buffers; 3403 for (i = 0; i < adapter->num_tx_desc; i++, tx_buffer++) { 3404 if (tx_buffer->m_head != NULL) { 3405 bus_dmamap_sync(txr->txtag->dt_dmat, tx_buffer->map, 3406 0, tx_buffer->m_head->m_pkthdr.len, 3407 BUS_DMASYNC_POSTWRITE); 3408 ixgbe_dmamap_unload(txr->txtag, tx_buffer->map); 3409 m_freem(tx_buffer->m_head); 3410 tx_buffer->m_head = NULL; 3411 if (tx_buffer->map != NULL) { 3412 ixgbe_dmamap_destroy(txr->txtag, 3413 tx_buffer->map); 3414 tx_buffer->map = NULL; 3415 } 3416 } else if (tx_buffer->map != NULL) { 3417 ixgbe_dmamap_unload(txr->txtag, tx_buffer->map); 3418 ixgbe_dmamap_destroy(txr->txtag, tx_buffer->map); 3419 tx_buffer->map = NULL; 3420 } 3421 } 3422 #if __FreeBSD_version >= 800000 3423 if (txr->br != NULL) 3424 buf_ring_free(txr->br, M_DEVBUF); 3425 #endif 3426 if (txr->tx_buffers != NULL) { 3427 free(txr->tx_buffers, M_DEVBUF); 3428 txr->tx_buffers = NULL; 3429 } 3430 if (txr->txtag != NULL) { 3431 ixgbe_dma_tag_destroy(txr->txtag); 3432 txr->txtag = NULL; 3433 } 3434 return; 3435 } 3436 3437 /********************************************************************* 3438 * 3439 * Advanced Context Descriptor setup for VLAN or L4 CSUM 3440 * 3441 **********************************************************************/ 3442 3443 static u32 3444 ixgbe_tx_ctx_setup(struct tx_ring *txr, struct mbuf *mp) 3445 { 3446 struct m_tag *mtag; 3447 struct adapter *adapter = txr->adapter; 3448 struct ethercom *ec = &adapter->osdep.ec; 3449 struct ixgbe_adv_tx_context_desc *TXD; 3450 struct ixgbe_tx_buf *tx_buffer; 3451 u32 olinfo = 0, vlan_macip_lens = 0, type_tucmd_mlhl = 0; 3452 struct ether_vlan_header *eh; 3453 struct ip ip; 3454 struct ip6_hdr ip6; 3455 int ehdrlen, ip_hlen = 0; 3456 u16 etype; 3457 u8 ipproto __diagused = 0; 3458 bool offload; 3459 int ctxd = txr->next_avail_desc; 3460 u16 vtag = 0; 3461 3462 offload = ((mp->m_pkthdr.csum_flags & M_CSUM_OFFLOAD) != 0); 3463 3464 tx_buffer = &txr->tx_buffers[ctxd]; 3465 TXD = (struct ixgbe_adv_tx_context_desc *) &txr->tx_base[ctxd]; 3466 3467 /* 3468 ** In advanced descriptors the vlan tag must 3469 ** be placed into the descriptor itself. 3470 */ 3471 if ((mtag = VLAN_OUTPUT_TAG(ec, mp)) != NULL) { 3472 vtag = htole16(VLAN_TAG_VALUE(mtag) & 0xffff); 3473 vlan_macip_lens |= (vtag << IXGBE_ADVTXD_VLAN_SHIFT); 3474 } else if (!offload) 3475 return 0; 3476 3477 /* 3478 * Determine where frame payload starts. 3479 * Jump over vlan headers if already present, 3480 * helpful for QinQ too. 3481 */ 3482 KASSERT(mp->m_len >= offsetof(struct ether_vlan_header, evl_tag)); 3483 eh = mtod(mp, struct ether_vlan_header *); 3484 if (eh->evl_encap_proto == htons(ETHERTYPE_VLAN)) { 3485 KASSERT(mp->m_len >= sizeof(struct ether_vlan_header)); 3486 etype = ntohs(eh->evl_proto); 3487 ehdrlen = ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN; 3488 } else { 3489 etype = ntohs(eh->evl_encap_proto); 3490 ehdrlen = ETHER_HDR_LEN; 3491 } 3492 3493 /* Set the ether header length */ 3494 vlan_macip_lens |= ehdrlen << IXGBE_ADVTXD_MACLEN_SHIFT; 3495 3496 switch (etype) { 3497 case ETHERTYPE_IP: 3498 m_copydata(mp, ehdrlen, sizeof(ip), &ip); 3499 ip_hlen = ip.ip_hl << 2; 3500 ipproto = ip.ip_p; 3501 #if 0 3502 ip.ip_sum = 0; 3503 m_copyback(mp, ehdrlen, sizeof(ip), &ip); 3504 #else 3505 KASSERT((mp->m_pkthdr.csum_flags & M_CSUM_IPv4) == 0 || 3506 ip.ip_sum == 0); 3507 #endif 3508 type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_IPV4; 3509 break; 3510 case ETHERTYPE_IPV6: 3511 m_copydata(mp, ehdrlen, sizeof(ip6), &ip6); 3512 ip_hlen = sizeof(ip6); 3513 /* XXX-BZ this will go badly in case of ext hdrs. */ 3514 ipproto = ip6.ip6_nxt; 3515 type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_IPV6; 3516 break; 3517 default: 3518 break; 3519 } 3520 3521 if ((mp->m_pkthdr.csum_flags & M_CSUM_IPv4) != 0) 3522 olinfo |= IXGBE_TXD_POPTS_IXSM << 8; 3523 3524 vlan_macip_lens |= ip_hlen; 3525 type_tucmd_mlhl |= IXGBE_ADVTXD_DCMD_DEXT | IXGBE_ADVTXD_DTYP_CTXT; 3526 3527 if (mp->m_pkthdr.csum_flags & (M_CSUM_TCPv4|M_CSUM_TCPv6)) { 3528 type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_L4T_TCP; 3529 olinfo |= IXGBE_TXD_POPTS_TXSM << 8; 3530 KASSERT(ipproto == IPPROTO_TCP); 3531 } else if (mp->m_pkthdr.csum_flags & (M_CSUM_UDPv4|M_CSUM_UDPv6)) { 3532 type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_L4T_UDP; 3533 olinfo |= IXGBE_TXD_POPTS_TXSM << 8; 3534 KASSERT(ipproto == IPPROTO_UDP); 3535 } 3536 3537 /* Now copy bits into descriptor */ 3538 TXD->vlan_macip_lens |= htole32(vlan_macip_lens); 3539 TXD->type_tucmd_mlhl |= htole32(type_tucmd_mlhl); 3540 TXD->seqnum_seed = htole32(0); 3541 TXD->mss_l4len_idx = htole32(0); 3542 3543 tx_buffer->m_head = NULL; 3544 tx_buffer->eop_index = -1; 3545 3546 /* We've consumed the first desc, adjust counters */ 3547 if (++ctxd == adapter->num_tx_desc) 3548 ctxd = 0; 3549 txr->next_avail_desc = ctxd; 3550 --txr->tx_avail; 3551 3552 return olinfo; 3553 } 3554 3555 /********************************************************************** 3556 * 3557 * Setup work for hardware segmentation offload (TSO) on 3558 * adapters using advanced tx descriptors 3559 * 3560 **********************************************************************/ 3561 static bool 3562 ixgbe_tso_setup(struct tx_ring *txr, struct mbuf *mp, u32 *paylen, 3563 u32 *olinfo_status) 3564 { 3565 struct m_tag *mtag; 3566 struct adapter *adapter = txr->adapter; 3567 struct ethercom *ec = &adapter->osdep.ec; 3568 struct ixgbe_adv_tx_context_desc *TXD; 3569 struct ixgbe_tx_buf *tx_buffer; 3570 u32 vlan_macip_lens = 0, type_tucmd_mlhl = 0; 3571 u32 mss_l4len_idx = 0, len; 3572 u16 vtag = 0, eh_type; 3573 int ctxd, ehdrlen, ip_hlen, tcp_hlen; 3574 struct ether_vlan_header *eh; 3575 #ifdef INET6 3576 struct ip6_hdr *ip6; 3577 #endif 3578 #ifdef INET 3579 struct ip *ip; 3580 #endif 3581 struct tcphdr *th; 3582 3583 3584 /* 3585 * Determine where frame payload starts. 3586 * Jump over vlan headers if already present 3587 */ 3588 eh = mtod(mp, struct ether_vlan_header *); 3589 if (eh->evl_encap_proto == htons(ETHERTYPE_VLAN)) { 3590 ehdrlen = ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN; 3591 eh_type = eh->evl_proto; 3592 } else { 3593 ehdrlen = ETHER_HDR_LEN; 3594 eh_type = eh->evl_encap_proto; 3595 } 3596 3597 /* Ensure we have at least the IP+TCP header in the first mbuf. */ 3598 len = ehdrlen + sizeof(struct tcphdr); 3599 switch (ntohs(eh_type)) { 3600 #ifdef INET6 3601 case ETHERTYPE_IPV6: 3602 if (mp->m_len < len + sizeof(struct ip6_hdr)) 3603 return FALSE; 3604 ip6 = (struct ip6_hdr *)(mp->m_data + ehdrlen); 3605 /* XXX-BZ For now we do not pretend to support ext. hdrs. */ 3606 if (ip6->ip6_nxt != IPPROTO_TCP) 3607 return FALSE; 3608 ip_hlen = sizeof(struct ip6_hdr); 3609 th = (struct tcphdr *)((char *)ip6 + ip_hlen); 3610 th->th_sum = in6_cksum_phdr(&ip6->ip6_src, 3611 &ip6->ip6_dst, 0, htonl(IPPROTO_TCP)); 3612 type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_IPV6; 3613 break; 3614 #endif 3615 #ifdef INET 3616 case ETHERTYPE_IP: 3617 if (mp->m_len < len + sizeof(struct ip)) 3618 return FALSE; 3619 ip = (struct ip *)(mp->m_data + ehdrlen); 3620 if (ip->ip_p != IPPROTO_TCP) 3621 return FALSE; 3622 ip->ip_sum = 0; 3623 ip_hlen = ip->ip_hl << 2; 3624 th = (struct tcphdr *)((char *)ip + ip_hlen); 3625 th->th_sum = in_cksum_phdr(ip->ip_src.s_addr, 3626 ip->ip_dst.s_addr, htons(IPPROTO_TCP)); 3627 type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_IPV4; 3628 /* Tell transmit desc to also do IPv4 checksum. */ 3629 *olinfo_status |= IXGBE_TXD_POPTS_IXSM << 8; 3630 break; 3631 #endif 3632 default: 3633 panic("%s: CSUM_TSO but no supported IP version (0x%04x)", 3634 __func__, ntohs(eh_type)); 3635 break; 3636 } 3637 3638 ctxd = txr->next_avail_desc; 3639 tx_buffer = &txr->tx_buffers[ctxd]; 3640 TXD = (struct ixgbe_adv_tx_context_desc *) &txr->tx_base[ctxd]; 3641 3642 tcp_hlen = th->th_off << 2; 3643 3644 /* This is used in the transmit desc in encap */ 3645 *paylen = mp->m_pkthdr.len - ehdrlen - ip_hlen - tcp_hlen; 3646 3647 /* VLAN MACLEN IPLEN */ 3648 if ((mtag = VLAN_OUTPUT_TAG(ec, mp)) != NULL) { 3649 vtag = htole16(VLAN_TAG_VALUE(mtag) & 0xffff); 3650 vlan_macip_lens |= (vtag << IXGBE_ADVTXD_VLAN_SHIFT); 3651 } 3652 3653 vlan_macip_lens |= ehdrlen << IXGBE_ADVTXD_MACLEN_SHIFT; 3654 vlan_macip_lens |= ip_hlen; 3655 TXD->vlan_macip_lens |= htole32(vlan_macip_lens); 3656 3657 /* ADV DTYPE TUCMD */ 3658 type_tucmd_mlhl |= IXGBE_ADVTXD_DCMD_DEXT | IXGBE_ADVTXD_DTYP_CTXT; 3659 type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_L4T_TCP; 3660 TXD->type_tucmd_mlhl |= htole32(type_tucmd_mlhl); 3661 3662 /* MSS L4LEN IDX */ 3663 mss_l4len_idx |= (mp->m_pkthdr.segsz << IXGBE_ADVTXD_MSS_SHIFT); 3664 mss_l4len_idx |= (tcp_hlen << IXGBE_ADVTXD_L4LEN_SHIFT); 3665 TXD->mss_l4len_idx = htole32(mss_l4len_idx); 3666 3667 TXD->seqnum_seed = htole32(0); 3668 tx_buffer->m_head = NULL; 3669 tx_buffer->eop_index = -1; 3670 3671 if (++ctxd == adapter->num_tx_desc) 3672 ctxd = 0; 3673 3674 txr->tx_avail--; 3675 txr->next_avail_desc = ctxd; 3676 return TRUE; 3677 } 3678 3679 #ifdef IXGBE_FDIR 3680 /* 3681 ** This routine parses packet headers so that Flow 3682 ** Director can make a hashed filter table entry 3683 ** allowing traffic flows to be identified and kept 3684 ** on the same cpu. This would be a performance 3685 ** hit, but we only do it at IXGBE_FDIR_RATE of 3686 ** packets. 3687 */ 3688 static void 3689 ixgbe_atr(struct tx_ring *txr, struct mbuf *mp) 3690 { 3691 struct adapter *adapter = txr->adapter; 3692 struct ix_queue *que; 3693 struct ip *ip; 3694 struct tcphdr *th; 3695 struct udphdr *uh; 3696 struct ether_vlan_header *eh; 3697 union ixgbe_atr_hash_dword input = {.dword = 0}; 3698 union ixgbe_atr_hash_dword common = {.dword = 0}; 3699 int ehdrlen, ip_hlen; 3700 u16 etype; 3701 3702 eh = mtod(mp, struct ether_vlan_header *); 3703 if (eh->evl_encap_proto == htons(ETHERTYPE_VLAN)) { 3704 ehdrlen = ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN; 3705 etype = eh->evl_proto; 3706 } else { 3707 ehdrlen = ETHER_HDR_LEN; 3708 etype = eh->evl_encap_proto; 3709 } 3710 3711 /* Only handling IPv4 */ 3712 if (etype != htons(ETHERTYPE_IP)) 3713 return; 3714 3715 ip = (struct ip *)(mp->m_data + ehdrlen); 3716 ip_hlen = ip->ip_hl << 2; 3717 3718 /* check if we're UDP or TCP */ 3719 switch (ip->ip_p) { 3720 case IPPROTO_TCP: 3721 th = (struct tcphdr *)((char *)ip + ip_hlen); 3722 /* src and dst are inverted */ 3723 common.port.dst ^= th->th_sport; 3724 common.port.src ^= th->th_dport; 3725 input.formatted.flow_type ^= IXGBE_ATR_FLOW_TYPE_TCPV4; 3726 break; 3727 case IPPROTO_UDP: 3728 uh = (struct udphdr *)((char *)ip + ip_hlen); 3729 /* src and dst are inverted */ 3730 common.port.dst ^= uh->uh_sport; 3731 common.port.src ^= uh->uh_dport; 3732 input.formatted.flow_type ^= IXGBE_ATR_FLOW_TYPE_UDPV4; 3733 break; 3734 default: 3735 return; 3736 } 3737 3738 input.formatted.vlan_id = htobe16(mp->m_pkthdr.ether_vtag); 3739 if (mp->m_pkthdr.ether_vtag) 3740 common.flex_bytes ^= htons(ETHERTYPE_VLAN); 3741 else 3742 common.flex_bytes ^= etype; 3743 common.ip ^= ip->ip_src.s_addr ^ ip->ip_dst.s_addr; 3744 3745 que = &adapter->queues[txr->me]; 3746 /* 3747 ** This assumes the Rx queue and Tx 3748 ** queue are bound to the same CPU 3749 */ 3750 ixgbe_fdir_add_signature_filter_82599(&adapter->hw, 3751 input, common, que->msix); 3752 } 3753 #endif /* IXGBE_FDIR */ 3754 3755 /********************************************************************** 3756 * 3757 * Examine each tx_buffer in the used queue. If the hardware is done 3758 * processing the packet then free associated resources. The 3759 * tx_buffer is put back on the free queue. 3760 * 3761 **********************************************************************/ 3762 static bool 3763 ixgbe_txeof(struct tx_ring *txr) 3764 { 3765 struct adapter *adapter = txr->adapter; 3766 struct ifnet *ifp = adapter->ifp; 3767 u32 first, last, done, processed; 3768 struct ixgbe_tx_buf *tx_buffer; 3769 struct ixgbe_legacy_tx_desc *tx_desc, *eop_desc; 3770 struct timeval now, elapsed; 3771 3772 KASSERT(mutex_owned(&txr->tx_mtx)); 3773 3774 #ifdef DEV_NETMAP 3775 if (ifp->if_capenable & IFCAP_NETMAP) { 3776 struct netmap_adapter *na = NA(ifp); 3777 struct netmap_kring *kring = &na->tx_rings[txr->me]; 3778 3779 tx_desc = (struct ixgbe_legacy_tx_desc *)txr->tx_base; 3780 3781 bus_dmamap_sync(txr->txdma.dma_tag, txr->txdma.dma_map, 3782 BUS_DMASYNC_POSTREAD); 3783 /* 3784 * In netmap mode, all the work is done in the context 3785 * of the client thread. Interrupt handlers only wake up 3786 * clients, which may be sleeping on individual rings 3787 * or on a global resource for all rings. 3788 * To implement tx interrupt mitigation, we wake up the client 3789 * thread roughly every half ring, even if the NIC interrupts 3790 * more frequently. This is implemented as follows: 3791 * - ixgbe_txsync() sets kring->nr_kflags with the index of 3792 * the slot that should wake up the thread (nkr_num_slots 3793 * means the user thread should not be woken up); 3794 * - the driver ignores tx interrupts unless netmap_mitigate=0 3795 * or the slot has the DD bit set. 3796 * 3797 * When the driver has separate locks, we need to 3798 * release and re-acquire txlock to avoid deadlocks. 3799 * XXX see if we can find a better way. 3800 */ 3801 if (!netmap_mitigate || 3802 (kring->nr_kflags < kring->nkr_num_slots && 3803 tx_desc[kring->nr_kflags].upper.fields.status & IXGBE_TXD_STAT_DD)) { 3804 kring->nr_kflags = kring->nkr_num_slots; 3805 selwakeuppri(&na->tx_rings[txr->me].si, PI_NET); 3806 IXGBE_TX_UNLOCK(txr); 3807 IXGBE_CORE_LOCK(adapter); 3808 selwakeuppri(&na->tx_si, PI_NET); 3809 IXGBE_CORE_UNLOCK(adapter); 3810 IXGBE_TX_LOCK(txr); 3811 } 3812 return FALSE; 3813 } 3814 #endif /* DEV_NETMAP */ 3815 3816 if (txr->tx_avail == adapter->num_tx_desc) { 3817 txr->queue_status = IXGBE_QUEUE_IDLE; 3818 return false; 3819 } 3820 3821 processed = 0; 3822 first = txr->next_to_clean; 3823 tx_buffer = &txr->tx_buffers[first]; 3824 /* For cleanup we just use legacy struct */ 3825 tx_desc = (struct ixgbe_legacy_tx_desc *)&txr->tx_base[first]; 3826 last = tx_buffer->eop_index; 3827 if (last == -1) 3828 return false; 3829 eop_desc = (struct ixgbe_legacy_tx_desc *)&txr->tx_base[last]; 3830 3831 /* 3832 ** Get the index of the first descriptor 3833 ** BEYOND the EOP and call that 'done'. 3834 ** I do this so the comparison in the 3835 ** inner while loop below can be simple 3836 */ 3837 if (++last == adapter->num_tx_desc) last = 0; 3838 done = last; 3839 3840 ixgbe_dmamap_sync(txr->txdma.dma_tag, txr->txdma.dma_map, 3841 BUS_DMASYNC_POSTREAD); 3842 /* 3843 ** Only the EOP descriptor of a packet now has the DD 3844 ** bit set, this is what we look for... 3845 */ 3846 while (eop_desc->upper.fields.status & IXGBE_TXD_STAT_DD) { 3847 /* We clean the range of the packet */ 3848 while (first != done) { 3849 tx_desc->upper.data = 0; 3850 tx_desc->lower.data = 0; 3851 tx_desc->buffer_addr = 0; 3852 ++txr->tx_avail; 3853 ++processed; 3854 3855 if (tx_buffer->m_head) { 3856 txr->bytes += 3857 tx_buffer->m_head->m_pkthdr.len; 3858 bus_dmamap_sync(txr->txtag->dt_dmat, 3859 tx_buffer->map, 3860 0, tx_buffer->m_head->m_pkthdr.len, 3861 BUS_DMASYNC_POSTWRITE); 3862 ixgbe_dmamap_unload(txr->txtag, tx_buffer->map); 3863 m_freem(tx_buffer->m_head); 3864 tx_buffer->m_head = NULL; 3865 } 3866 tx_buffer->eop_index = -1; 3867 getmicrotime(&txr->watchdog_time); 3868 3869 if (++first == adapter->num_tx_desc) 3870 first = 0; 3871 3872 tx_buffer = &txr->tx_buffers[first]; 3873 tx_desc = 3874 (struct ixgbe_legacy_tx_desc *)&txr->tx_base[first]; 3875 } 3876 ++txr->packets; 3877 ++ifp->if_opackets; 3878 /* See if there is more work now */ 3879 last = tx_buffer->eop_index; 3880 if (last != -1) { 3881 eop_desc = 3882 (struct ixgbe_legacy_tx_desc *)&txr->tx_base[last]; 3883 /* Get next done point */ 3884 if (++last == adapter->num_tx_desc) last = 0; 3885 done = last; 3886 } else 3887 break; 3888 } 3889 ixgbe_dmamap_sync(txr->txdma.dma_tag, txr->txdma.dma_map, 3890 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); 3891 3892 txr->next_to_clean = first; 3893 3894 /* 3895 ** Watchdog calculation, we know there's 3896 ** work outstanding or the first return 3897 ** would have been taken, so none processed 3898 ** for too long indicates a hang. 3899 */ 3900 getmicrotime(&now); 3901 timersub(&now, &txr->watchdog_time, &elapsed); 3902 if (!processed && tvtohz(&elapsed) > IXGBE_WATCHDOG) 3903 txr->queue_status = IXGBE_QUEUE_HUNG; 3904 3905 /* With a minimum free clear the depleted state bit. */ 3906 if (txr->tx_avail > IXGBE_TX_CLEANUP_THRESHOLD) 3907 txr->queue_status &= ~IXGBE_QUEUE_DEPLETED; 3908 3909 if (txr->tx_avail == adapter->num_tx_desc) { 3910 txr->queue_status = IXGBE_QUEUE_IDLE; 3911 return false; 3912 } 3913 3914 return true; 3915 } 3916 3917 /********************************************************************* 3918 * 3919 * Refresh mbuf buffers for RX descriptor rings 3920 * - now keeps its own state so discards due to resource 3921 * exhaustion are unnecessary, if an mbuf cannot be obtained 3922 * it just returns, keeping its placeholder, thus it can simply 3923 * be recalled to try again. 3924 * 3925 **********************************************************************/ 3926 static void 3927 ixgbe_refresh_mbufs(struct rx_ring *rxr, int limit) 3928 { 3929 struct adapter *adapter = rxr->adapter; 3930 struct ixgbe_rx_buf *rxbuf; 3931 struct mbuf *mh, *mp; 3932 int i, j, error; 3933 bool refreshed = false; 3934 3935 i = j = rxr->next_to_refresh; 3936 /* Control the loop with one beyond */ 3937 if (++j == adapter->num_rx_desc) 3938 j = 0; 3939 3940 while (j != limit) { 3941 rxbuf = &rxr->rx_buffers[i]; 3942 if (rxr->hdr_split == FALSE) 3943 goto no_split; 3944 3945 if (rxbuf->m_head == NULL) { 3946 mh = m_gethdr(M_DONTWAIT, MT_DATA); 3947 if (mh == NULL) 3948 goto update; 3949 } else 3950 mh = rxbuf->m_head; 3951 3952 mh->m_pkthdr.len = mh->m_len = MHLEN; 3953 mh->m_len = MHLEN; 3954 mh->m_flags |= M_PKTHDR; 3955 /* Get the memory mapping */ 3956 error = bus_dmamap_load_mbuf(rxr->htag->dt_dmat, 3957 rxbuf->hmap, mh, BUS_DMA_NOWAIT); 3958 if (error != 0) { 3959 printf("Refresh mbufs: hdr dmamap load" 3960 " failure - %d\n", error); 3961 m_free(mh); 3962 rxbuf->m_head = NULL; 3963 goto update; 3964 } 3965 rxbuf->m_head = mh; 3966 ixgbe_dmamap_sync(rxr->htag, rxbuf->hmap, BUS_DMASYNC_PREREAD); 3967 rxr->rx_base[i].read.hdr_addr = 3968 htole64(rxbuf->hmap->dm_segs[0].ds_addr); 3969 3970 no_split: 3971 if (rxbuf->m_pack == NULL) { 3972 mp = ixgbe_getjcl(&adapter->jcl_head, M_DONTWAIT, 3973 MT_DATA, M_PKTHDR, adapter->rx_mbuf_sz); 3974 if (mp == NULL) { 3975 rxr->no_jmbuf.ev_count++; 3976 goto update; 3977 } 3978 } else 3979 mp = rxbuf->m_pack; 3980 3981 mp->m_pkthdr.len = mp->m_len = adapter->rx_mbuf_sz; 3982 /* Get the memory mapping */ 3983 error = bus_dmamap_load_mbuf(rxr->ptag->dt_dmat, 3984 rxbuf->pmap, mp, BUS_DMA_NOWAIT); 3985 if (error != 0) { 3986 printf("Refresh mbufs: payload dmamap load" 3987 " failure - %d\n", error); 3988 m_free(mp); 3989 rxbuf->m_pack = NULL; 3990 goto update; 3991 } 3992 rxbuf->m_pack = mp; 3993 bus_dmamap_sync(rxr->ptag->dt_dmat, rxbuf->pmap, 3994 0, mp->m_pkthdr.len, BUS_DMASYNC_PREREAD); 3995 rxr->rx_base[i].read.pkt_addr = 3996 htole64(rxbuf->pmap->dm_segs[0].ds_addr); 3997 3998 refreshed = true; 3999 /* Next is precalculated */ 4000 i = j; 4001 rxr->next_to_refresh = i; 4002 if (++j == adapter->num_rx_desc) 4003 j = 0; 4004 } 4005 update: 4006 if (refreshed) /* Update hardware tail index */ 4007 IXGBE_WRITE_REG(&adapter->hw, 4008 IXGBE_RDT(rxr->me), rxr->next_to_refresh); 4009 return; 4010 } 4011 4012 /********************************************************************* 4013 * 4014 * Allocate memory for rx_buffer structures. Since we use one 4015 * rx_buffer per received packet, the maximum number of rx_buffer's 4016 * that we'll need is equal to the number of receive descriptors 4017 * that we've allocated. 4018 * 4019 **********************************************************************/ 4020 static int 4021 ixgbe_allocate_receive_buffers(struct rx_ring *rxr) 4022 { 4023 struct adapter *adapter = rxr->adapter; 4024 device_t dev = adapter->dev; 4025 struct ixgbe_rx_buf *rxbuf; 4026 int i, bsize, error; 4027 4028 bsize = sizeof(struct ixgbe_rx_buf) * adapter->num_rx_desc; 4029 if (!(rxr->rx_buffers = 4030 (struct ixgbe_rx_buf *) malloc(bsize, 4031 M_DEVBUF, M_NOWAIT | M_ZERO))) { 4032 aprint_error_dev(dev, "Unable to allocate rx_buffer memory\n"); 4033 error = ENOMEM; 4034 goto fail; 4035 } 4036 4037 if ((error = ixgbe_dma_tag_create(adapter->osdep.dmat, /* parent */ 4038 1, 0, /* alignment, bounds */ 4039 MSIZE, /* maxsize */ 4040 1, /* nsegments */ 4041 MSIZE, /* maxsegsize */ 4042 0, /* flags */ 4043 &rxr->htag))) { 4044 aprint_error_dev(dev, "Unable to create RX DMA tag\n"); 4045 goto fail; 4046 } 4047 4048 if ((error = ixgbe_dma_tag_create(adapter->osdep.dmat, /* parent */ 4049 1, 0, /* alignment, bounds */ 4050 MJUM16BYTES, /* maxsize */ 4051 1, /* nsegments */ 4052 MJUM16BYTES, /* maxsegsize */ 4053 0, /* flags */ 4054 &rxr->ptag))) { 4055 aprint_error_dev(dev, "Unable to create RX DMA tag\n"); 4056 goto fail; 4057 } 4058 4059 for (i = 0; i < adapter->num_rx_desc; i++, rxbuf++) { 4060 rxbuf = &rxr->rx_buffers[i]; 4061 error = ixgbe_dmamap_create(rxr->htag, 4062 BUS_DMA_NOWAIT, &rxbuf->hmap); 4063 if (error) { 4064 aprint_error_dev(dev, "Unable to create RX head map\n"); 4065 goto fail; 4066 } 4067 error = ixgbe_dmamap_create(rxr->ptag, 4068 BUS_DMA_NOWAIT, &rxbuf->pmap); 4069 if (error) { 4070 aprint_error_dev(dev, "Unable to create RX pkt map\n"); 4071 goto fail; 4072 } 4073 } 4074 4075 return (0); 4076 4077 fail: 4078 /* Frees all, but can handle partial completion */ 4079 ixgbe_free_receive_structures(adapter); 4080 return (error); 4081 } 4082 4083 /* 4084 ** Used to detect a descriptor that has 4085 ** been merged by Hardware RSC. 4086 */ 4087 static inline u32 4088 ixgbe_rsc_count(union ixgbe_adv_rx_desc *rx) 4089 { 4090 return (le32toh(rx->wb.lower.lo_dword.data) & 4091 IXGBE_RXDADV_RSCCNT_MASK) >> IXGBE_RXDADV_RSCCNT_SHIFT; 4092 } 4093 4094 /********************************************************************* 4095 * 4096 * Initialize Hardware RSC (LRO) feature on 82599 4097 * for an RX ring, this is toggled by the LRO capability 4098 * even though it is transparent to the stack. 4099 * 4100 **********************************************************************/ 4101 static void 4102 ixgbe_setup_hw_rsc(struct rx_ring *rxr) 4103 { 4104 struct adapter *adapter = rxr->adapter; 4105 struct ixgbe_hw *hw = &adapter->hw; 4106 u32 rscctrl, rdrxctl; 4107 4108 rdrxctl = IXGBE_READ_REG(hw, IXGBE_RDRXCTL); 4109 rdrxctl &= ~IXGBE_RDRXCTL_RSCFRSTSIZE; 4110 #ifdef DEV_NETMAP /* crcstrip is optional in netmap */ 4111 if (adapter->ifp->if_capenable & IFCAP_NETMAP && !ix_crcstrip) 4112 #endif /* DEV_NETMAP */ 4113 rdrxctl |= IXGBE_RDRXCTL_CRCSTRIP; 4114 rdrxctl |= IXGBE_RDRXCTL_RSCACKC; 4115 IXGBE_WRITE_REG(hw, IXGBE_RDRXCTL, rdrxctl); 4116 4117 rscctrl = IXGBE_READ_REG(hw, IXGBE_RSCCTL(rxr->me)); 4118 rscctrl |= IXGBE_RSCCTL_RSCEN; 4119 /* 4120 ** Limit the total number of descriptors that 4121 ** can be combined, so it does not exceed 64K 4122 */ 4123 if (adapter->rx_mbuf_sz == MCLBYTES) 4124 rscctrl |= IXGBE_RSCCTL_MAXDESC_16; 4125 else if (adapter->rx_mbuf_sz == MJUMPAGESIZE) 4126 rscctrl |= IXGBE_RSCCTL_MAXDESC_8; 4127 else if (adapter->rx_mbuf_sz == MJUM9BYTES) 4128 rscctrl |= IXGBE_RSCCTL_MAXDESC_4; 4129 else /* Using 16K cluster */ 4130 rscctrl |= IXGBE_RSCCTL_MAXDESC_1; 4131 4132 IXGBE_WRITE_REG(hw, IXGBE_RSCCTL(rxr->me), rscctrl); 4133 4134 /* Enable TCP header recognition */ 4135 IXGBE_WRITE_REG(hw, IXGBE_PSRTYPE(0), 4136 (IXGBE_READ_REG(hw, IXGBE_PSRTYPE(0)) | 4137 IXGBE_PSRTYPE_TCPHDR)); 4138 4139 /* Disable RSC for ACK packets */ 4140 IXGBE_WRITE_REG(hw, IXGBE_RSCDBU, 4141 (IXGBE_RSCDBU_RSCACKDIS | IXGBE_READ_REG(hw, IXGBE_RSCDBU))); 4142 4143 rxr->hw_rsc = TRUE; 4144 } 4145 4146 4147 static void 4148 ixgbe_free_receive_ring(struct rx_ring *rxr) 4149 { 4150 struct adapter *adapter; 4151 struct ixgbe_rx_buf *rxbuf; 4152 int i; 4153 4154 adapter = rxr->adapter; 4155 for (i = 0; i < adapter->num_rx_desc; i++) { 4156 rxbuf = &rxr->rx_buffers[i]; 4157 if (rxbuf->m_head != NULL) { 4158 ixgbe_dmamap_sync(rxr->htag, rxbuf->hmap, 4159 BUS_DMASYNC_POSTREAD); 4160 ixgbe_dmamap_unload(rxr->htag, rxbuf->hmap); 4161 rxbuf->m_head->m_flags |= M_PKTHDR; 4162 m_freem(rxbuf->m_head); 4163 } 4164 if (rxbuf->m_pack != NULL) { 4165 bus_dmamap_sync(rxr->ptag->dt_dmat, rxbuf->pmap, 4166 0, rxbuf->m_pack->m_pkthdr.len, 4167 BUS_DMASYNC_POSTREAD); 4168 ixgbe_dmamap_unload(rxr->ptag, rxbuf->pmap); 4169 rxbuf->m_pack->m_flags |= M_PKTHDR; 4170 m_freem(rxbuf->m_pack); 4171 } 4172 rxbuf->m_head = NULL; 4173 rxbuf->m_pack = NULL; 4174 } 4175 } 4176 4177 4178 /********************************************************************* 4179 * 4180 * Initialize a receive ring and its buffers. 4181 * 4182 **********************************************************************/ 4183 static int 4184 ixgbe_setup_receive_ring(struct rx_ring *rxr) 4185 { 4186 struct adapter *adapter; 4187 struct ifnet *ifp; 4188 struct ixgbe_rx_buf *rxbuf; 4189 #ifdef LRO 4190 struct lro_ctrl *lro = &rxr->lro; 4191 #endif /* LRO */ 4192 int rsize, error = 0; 4193 #ifdef DEV_NETMAP 4194 struct netmap_adapter *na = NA(rxr->adapter->ifp); 4195 struct netmap_slot *slot; 4196 #endif /* DEV_NETMAP */ 4197 4198 adapter = rxr->adapter; 4199 ifp = adapter->ifp; 4200 4201 /* Clear the ring contents */ 4202 IXGBE_RX_LOCK(rxr); 4203 #ifdef DEV_NETMAP 4204 /* same as in ixgbe_setup_transmit_ring() */ 4205 slot = netmap_reset(na, NR_RX, rxr->me, 0); 4206 #endif /* DEV_NETMAP */ 4207 rsize = roundup2(adapter->num_rx_desc * 4208 sizeof(union ixgbe_adv_rx_desc), DBA_ALIGN); 4209 bzero((void *)rxr->rx_base, rsize); 4210 4211 /* Free current RX buffer structs and their mbufs */ 4212 ixgbe_free_receive_ring(rxr); 4213 4214 IXGBE_RX_UNLOCK(rxr); 4215 4216 /* Now reinitialize our supply of jumbo mbufs. The number 4217 * or size of jumbo mbufs may have changed. 4218 */ 4219 ixgbe_jcl_reinit(&adapter->jcl_head, rxr->ptag->dt_dmat, 4220 2 * adapter->num_rx_desc, adapter->rx_mbuf_sz); 4221 4222 IXGBE_RX_LOCK(rxr); 4223 4224 /* Configure header split? */ 4225 if (ixgbe_header_split) 4226 rxr->hdr_split = TRUE; 4227 4228 /* Now replenish the mbufs */ 4229 for (int j = 0; j != adapter->num_rx_desc; ++j) { 4230 struct mbuf *mh, *mp; 4231 4232 rxbuf = &rxr->rx_buffers[j]; 4233 #ifdef DEV_NETMAP 4234 /* 4235 * In netmap mode, fill the map and set the buffer 4236 * address in the NIC ring, considering the offset 4237 * between the netmap and NIC rings (see comment in 4238 * ixgbe_setup_transmit_ring() ). No need to allocate 4239 * an mbuf, so end the block with a continue; 4240 */ 4241 if (slot) { 4242 int sj = netmap_idx_n2k(&na->rx_rings[rxr->me], j); 4243 uint64_t paddr; 4244 void *addr; 4245 4246 addr = PNMB(slot + sj, &paddr); 4247 netmap_load_map(rxr->ptag, rxbuf->pmap, addr); 4248 /* Update descriptor */ 4249 rxr->rx_base[j].read.pkt_addr = htole64(paddr); 4250 continue; 4251 } 4252 #endif /* DEV_NETMAP */ 4253 /* 4254 ** Don't allocate mbufs if not 4255 ** doing header split, its wasteful 4256 */ 4257 if (rxr->hdr_split == FALSE) 4258 goto skip_head; 4259 4260 /* First the header */ 4261 rxbuf->m_head = m_gethdr(M_DONTWAIT, MT_DATA); 4262 if (rxbuf->m_head == NULL) { 4263 error = ENOBUFS; 4264 goto fail; 4265 } 4266 m_adj(rxbuf->m_head, ETHER_ALIGN); 4267 mh = rxbuf->m_head; 4268 mh->m_len = mh->m_pkthdr.len = MHLEN; 4269 mh->m_flags |= M_PKTHDR; 4270 /* Get the memory mapping */ 4271 error = bus_dmamap_load_mbuf(rxr->htag->dt_dmat, 4272 rxbuf->hmap, rxbuf->m_head, BUS_DMA_NOWAIT); 4273 if (error != 0) /* Nothing elegant to do here */ 4274 goto fail; 4275 bus_dmamap_sync(rxr->htag->dt_dmat, rxbuf->hmap, 4276 0, mh->m_pkthdr.len, BUS_DMASYNC_PREREAD); 4277 /* Update descriptor */ 4278 rxr->rx_base[j].read.hdr_addr = 4279 htole64(rxbuf->hmap->dm_segs[0].ds_addr); 4280 4281 skip_head: 4282 /* Now the payload cluster */ 4283 rxbuf->m_pack = ixgbe_getjcl(&adapter->jcl_head, M_DONTWAIT, 4284 MT_DATA, M_PKTHDR, adapter->rx_mbuf_sz); 4285 if (rxbuf->m_pack == NULL) { 4286 error = ENOBUFS; 4287 goto fail; 4288 } 4289 mp = rxbuf->m_pack; 4290 mp->m_pkthdr.len = mp->m_len = adapter->rx_mbuf_sz; 4291 /* Get the memory mapping */ 4292 error = bus_dmamap_load_mbuf(rxr->ptag->dt_dmat, 4293 rxbuf->pmap, mp, BUS_DMA_NOWAIT); 4294 if (error != 0) 4295 goto fail; 4296 bus_dmamap_sync(rxr->ptag->dt_dmat, rxbuf->pmap, 4297 0, adapter->rx_mbuf_sz, BUS_DMASYNC_PREREAD); 4298 /* Update descriptor */ 4299 rxr->rx_base[j].read.pkt_addr = 4300 htole64(rxbuf->pmap->dm_segs[0].ds_addr); 4301 } 4302 4303 4304 /* Setup our descriptor indices */ 4305 rxr->next_to_check = 0; 4306 rxr->next_to_refresh = 0; 4307 rxr->lro_enabled = FALSE; 4308 rxr->rx_split_packets.ev_count = 0; 4309 rxr->rx_bytes.ev_count = 0; 4310 rxr->discard = FALSE; 4311 rxr->vtag_strip = FALSE; 4312 4313 ixgbe_dmamap_sync(rxr->rxdma.dma_tag, rxr->rxdma.dma_map, 4314 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); 4315 4316 /* 4317 ** Now set up the LRO interface: 4318 ** 82598 uses software LRO, the 4319 ** 82599 and X540 use a hardware assist. 4320 */ 4321 if ((adapter->hw.mac.type != ixgbe_mac_82598EB) && 4322 (ifp->if_capenable & IFCAP_RXCSUM) && 4323 (ifp->if_capenable & IFCAP_LRO)) 4324 ixgbe_setup_hw_rsc(rxr); 4325 #ifdef LRO 4326 else if (ifp->if_capenable & IFCAP_LRO) { 4327 device_t dev = adapter->dev; 4328 int err = tcp_lro_init(lro); 4329 if (err) { 4330 device_printf(dev, "LRO Initialization failed!\n"); 4331 goto fail; 4332 } 4333 INIT_DEBUGOUT("RX Soft LRO Initialized\n"); 4334 rxr->lro_enabled = TRUE; 4335 lro->ifp = adapter->ifp; 4336 } 4337 #endif /* LRO */ 4338 4339 IXGBE_RX_UNLOCK(rxr); 4340 return (0); 4341 4342 fail: 4343 ixgbe_free_receive_ring(rxr); 4344 IXGBE_RX_UNLOCK(rxr); 4345 return (error); 4346 } 4347 4348 /********************************************************************* 4349 * 4350 * Initialize all receive rings. 4351 * 4352 **********************************************************************/ 4353 static int 4354 ixgbe_setup_receive_structures(struct adapter *adapter) 4355 { 4356 struct rx_ring *rxr = adapter->rx_rings; 4357 int j; 4358 4359 for (j = 0; j < adapter->num_queues; j++, rxr++) 4360 if (ixgbe_setup_receive_ring(rxr)) 4361 goto fail; 4362 4363 return (0); 4364 fail: 4365 /* 4366 * Free RX buffers allocated so far, we will only handle 4367 * the rings that completed, the failing case will have 4368 * cleaned up for itself. 'j' failed, so its the terminus. 4369 */ 4370 for (int i = 0; i < j; ++i) { 4371 rxr = &adapter->rx_rings[i]; 4372 ixgbe_free_receive_ring(rxr); 4373 } 4374 4375 return (ENOBUFS); 4376 } 4377 4378 /********************************************************************* 4379 * 4380 * Setup receive registers and features. 4381 * 4382 **********************************************************************/ 4383 #define IXGBE_SRRCTL_BSIZEHDRSIZE_SHIFT 2 4384 4385 #define BSIZEPKT_ROUNDUP ((1<<IXGBE_SRRCTL_BSIZEPKT_SHIFT)-1) 4386 4387 static void 4388 ixgbe_initialize_receive_units(struct adapter *adapter) 4389 { 4390 int i; 4391 struct rx_ring *rxr = adapter->rx_rings; 4392 struct ixgbe_hw *hw = &adapter->hw; 4393 struct ifnet *ifp = adapter->ifp; 4394 u32 bufsz, rxctrl, fctrl, srrctl, rxcsum; 4395 u32 reta, mrqc = 0, hlreg, r[10]; 4396 4397 4398 /* 4399 * Make sure receives are disabled while 4400 * setting up the descriptor ring 4401 */ 4402 rxctrl = IXGBE_READ_REG(hw, IXGBE_RXCTRL); 4403 IXGBE_WRITE_REG(hw, IXGBE_RXCTRL, 4404 rxctrl & ~IXGBE_RXCTRL_RXEN); 4405 4406 /* Enable broadcasts */ 4407 fctrl = IXGBE_READ_REG(hw, IXGBE_FCTRL); 4408 fctrl |= IXGBE_FCTRL_BAM; 4409 fctrl |= IXGBE_FCTRL_DPF; 4410 fctrl |= IXGBE_FCTRL_PMCF; 4411 IXGBE_WRITE_REG(hw, IXGBE_FCTRL, fctrl); 4412 4413 /* Set for Jumbo Frames? */ 4414 hlreg = IXGBE_READ_REG(hw, IXGBE_HLREG0); 4415 if (ifp->if_mtu > ETHERMTU) 4416 hlreg |= IXGBE_HLREG0_JUMBOEN; 4417 else 4418 hlreg &= ~IXGBE_HLREG0_JUMBOEN; 4419 #ifdef DEV_NETMAP 4420 /* crcstrip is conditional in netmap (in RDRXCTL too ?) */ 4421 if (ifp->if_capenable & IFCAP_NETMAP && !ix_crcstrip) 4422 hlreg &= ~IXGBE_HLREG0_RXCRCSTRP; 4423 else 4424 hlreg |= IXGBE_HLREG0_RXCRCSTRP; 4425 #endif /* DEV_NETMAP */ 4426 IXGBE_WRITE_REG(hw, IXGBE_HLREG0, hlreg); 4427 4428 bufsz = (adapter->rx_mbuf_sz + 4429 BSIZEPKT_ROUNDUP) >> IXGBE_SRRCTL_BSIZEPKT_SHIFT; 4430 4431 for (i = 0; i < adapter->num_queues; i++, rxr++) { 4432 u64 rdba = rxr->rxdma.dma_paddr; 4433 4434 /* Setup the Base and Length of the Rx Descriptor Ring */ 4435 IXGBE_WRITE_REG(hw, IXGBE_RDBAL(i), 4436 (rdba & 0x00000000ffffffffULL)); 4437 IXGBE_WRITE_REG(hw, IXGBE_RDBAH(i), (rdba >> 32)); 4438 IXGBE_WRITE_REG(hw, IXGBE_RDLEN(i), 4439 adapter->num_rx_desc * sizeof(union ixgbe_adv_rx_desc)); 4440 4441 /* Set up the SRRCTL register */ 4442 srrctl = IXGBE_READ_REG(hw, IXGBE_SRRCTL(i)); 4443 srrctl &= ~IXGBE_SRRCTL_BSIZEHDR_MASK; 4444 srrctl &= ~IXGBE_SRRCTL_BSIZEPKT_MASK; 4445 srrctl |= bufsz; 4446 if (rxr->hdr_split) { 4447 /* Use a standard mbuf for the header */ 4448 srrctl |= ((IXGBE_RX_HDR << 4449 IXGBE_SRRCTL_BSIZEHDRSIZE_SHIFT) 4450 & IXGBE_SRRCTL_BSIZEHDR_MASK); 4451 srrctl |= IXGBE_SRRCTL_DESCTYPE_HDR_SPLIT_ALWAYS; 4452 } else 4453 srrctl |= IXGBE_SRRCTL_DESCTYPE_ADV_ONEBUF; 4454 IXGBE_WRITE_REG(hw, IXGBE_SRRCTL(i), srrctl); 4455 4456 /* Setup the HW Rx Head and Tail Descriptor Pointers */ 4457 IXGBE_WRITE_REG(hw, IXGBE_RDH(i), 0); 4458 IXGBE_WRITE_REG(hw, IXGBE_RDT(i), 0); 4459 } 4460 4461 if (adapter->hw.mac.type != ixgbe_mac_82598EB) { 4462 u32 psrtype = IXGBE_PSRTYPE_TCPHDR | 4463 IXGBE_PSRTYPE_UDPHDR | 4464 IXGBE_PSRTYPE_IPV4HDR | 4465 IXGBE_PSRTYPE_IPV6HDR; 4466 IXGBE_WRITE_REG(hw, IXGBE_PSRTYPE(0), psrtype); 4467 } 4468 4469 rxcsum = IXGBE_READ_REG(hw, IXGBE_RXCSUM); 4470 4471 /* Setup RSS */ 4472 if (adapter->num_queues > 1) { 4473 int j; 4474 reta = 0; 4475 4476 /* set up random bits */ 4477 cprng_fast(&r, sizeof(r)); 4478 4479 /* Set up the redirection table */ 4480 for (i = 0, j = 0; i < 128; i++, j++) { 4481 if (j == adapter->num_queues) j = 0; 4482 reta = (reta << 8) | (j * 0x11); 4483 if ((i & 3) == 3) 4484 IXGBE_WRITE_REG(hw, IXGBE_RETA(i >> 2), reta); 4485 } 4486 4487 /* Now fill our hash function seeds */ 4488 for (i = 0; i < 10; i++) 4489 IXGBE_WRITE_REG(hw, IXGBE_RSSRK(i), r[i]); 4490 4491 /* Perform hash on these packet types */ 4492 mrqc = IXGBE_MRQC_RSSEN 4493 | IXGBE_MRQC_RSS_FIELD_IPV4 4494 | IXGBE_MRQC_RSS_FIELD_IPV4_TCP 4495 | IXGBE_MRQC_RSS_FIELD_IPV4_UDP 4496 | IXGBE_MRQC_RSS_FIELD_IPV6_EX_TCP 4497 | IXGBE_MRQC_RSS_FIELD_IPV6_EX 4498 | IXGBE_MRQC_RSS_FIELD_IPV6 4499 | IXGBE_MRQC_RSS_FIELD_IPV6_TCP 4500 | IXGBE_MRQC_RSS_FIELD_IPV6_UDP 4501 | IXGBE_MRQC_RSS_FIELD_IPV6_EX_UDP; 4502 IXGBE_WRITE_REG(hw, IXGBE_MRQC, mrqc); 4503 4504 /* RSS and RX IPP Checksum are mutually exclusive */ 4505 rxcsum |= IXGBE_RXCSUM_PCSD; 4506 } 4507 4508 if (ifp->if_capenable & IFCAP_RXCSUM) 4509 rxcsum |= IXGBE_RXCSUM_PCSD; 4510 4511 if (!(rxcsum & IXGBE_RXCSUM_PCSD)) 4512 rxcsum |= IXGBE_RXCSUM_IPPCSE; 4513 4514 IXGBE_WRITE_REG(hw, IXGBE_RXCSUM, rxcsum); 4515 4516 return; 4517 } 4518 4519 /********************************************************************* 4520 * 4521 * Free all receive rings. 4522 * 4523 **********************************************************************/ 4524 static void 4525 ixgbe_free_receive_structures(struct adapter *adapter) 4526 { 4527 struct rx_ring *rxr = adapter->rx_rings; 4528 4529 for (int i = 0; i < adapter->num_queues; i++, rxr++) { 4530 #ifdef LRO 4531 struct lro_ctrl *lro = &rxr->lro; 4532 #endif /* LRO */ 4533 ixgbe_free_receive_buffers(rxr); 4534 #ifdef LRO 4535 /* Free LRO memory */ 4536 tcp_lro_free(lro); 4537 #endif /* LRO */ 4538 /* Free the ring memory as well */ 4539 ixgbe_dma_free(adapter, &rxr->rxdma); 4540 IXGBE_RX_LOCK_DESTROY(rxr); 4541 } 4542 4543 free(adapter->rx_rings, M_DEVBUF); 4544 } 4545 4546 4547 /********************************************************************* 4548 * 4549 * Free receive ring data structures 4550 * 4551 **********************************************************************/ 4552 static void 4553 ixgbe_free_receive_buffers(struct rx_ring *rxr) 4554 { 4555 struct adapter *adapter = rxr->adapter; 4556 struct ixgbe_rx_buf *rxbuf; 4557 4558 INIT_DEBUGOUT("free_receive_structures: begin"); 4559 4560 /* Cleanup any existing buffers */ 4561 if (rxr->rx_buffers != NULL) { 4562 for (int i = 0; i < adapter->num_rx_desc; i++) { 4563 rxbuf = &rxr->rx_buffers[i]; 4564 if (rxbuf->m_head != NULL) { 4565 ixgbe_dmamap_sync(rxr->htag, rxbuf->hmap, 4566 BUS_DMASYNC_POSTREAD); 4567 ixgbe_dmamap_unload(rxr->htag, rxbuf->hmap); 4568 rxbuf->m_head->m_flags |= M_PKTHDR; 4569 m_freem(rxbuf->m_head); 4570 } 4571 if (rxbuf->m_pack != NULL) { 4572 bus_dmamap_sync(rxr->ptag->dt_dmat, rxbuf->pmap, 4573 0, rxbuf->m_pack->m_pkthdr.len, 4574 BUS_DMASYNC_POSTREAD); 4575 ixgbe_dmamap_unload(rxr->ptag, rxbuf->pmap); 4576 rxbuf->m_pack->m_flags |= M_PKTHDR; 4577 m_freem(rxbuf->m_pack); 4578 } 4579 rxbuf->m_head = NULL; 4580 rxbuf->m_pack = NULL; 4581 if (rxbuf->hmap != NULL) { 4582 ixgbe_dmamap_destroy(rxr->htag, rxbuf->hmap); 4583 rxbuf->hmap = NULL; 4584 } 4585 if (rxbuf->pmap != NULL) { 4586 ixgbe_dmamap_destroy(rxr->ptag, rxbuf->pmap); 4587 rxbuf->pmap = NULL; 4588 } 4589 } 4590 if (rxr->rx_buffers != NULL) { 4591 free(rxr->rx_buffers, M_DEVBUF); 4592 rxr->rx_buffers = NULL; 4593 } 4594 } 4595 4596 if (rxr->htag != NULL) { 4597 ixgbe_dma_tag_destroy(rxr->htag); 4598 rxr->htag = NULL; 4599 } 4600 if (rxr->ptag != NULL) { 4601 ixgbe_dma_tag_destroy(rxr->ptag); 4602 rxr->ptag = NULL; 4603 } 4604 4605 return; 4606 } 4607 4608 static __inline void 4609 ixgbe_rx_input(struct rx_ring *rxr, struct ifnet *ifp, struct mbuf *m, u32 ptype) 4610 { 4611 int s; 4612 4613 #ifdef LRO 4614 struct adapter *adapter = ifp->if_softc; 4615 struct ethercom *ec = &adapter->osdep.ec; 4616 4617 /* 4618 * ATM LRO is only for IP/TCP packets and TCP checksum of the packet 4619 * should be computed by hardware. Also it should not have VLAN tag in 4620 * ethernet header. In case of IPv6 we do not yet support ext. hdrs. 4621 */ 4622 if (rxr->lro_enabled && 4623 (ec->ec_capenable & ETHERCAP_VLAN_HWTAGGING) != 0 && 4624 (ptype & IXGBE_RXDADV_PKTTYPE_ETQF) == 0 && 4625 ((ptype & (IXGBE_RXDADV_PKTTYPE_IPV4 | IXGBE_RXDADV_PKTTYPE_TCP)) == 4626 (IXGBE_RXDADV_PKTTYPE_IPV4 | IXGBE_RXDADV_PKTTYPE_TCP) || 4627 (ptype & (IXGBE_RXDADV_PKTTYPE_IPV6 | IXGBE_RXDADV_PKTTYPE_TCP)) == 4628 (IXGBE_RXDADV_PKTTYPE_IPV6 | IXGBE_RXDADV_PKTTYPE_TCP)) && 4629 (m->m_pkthdr.csum_flags & (CSUM_DATA_VALID | CSUM_PSEUDO_HDR)) == 4630 (CSUM_DATA_VALID | CSUM_PSEUDO_HDR)) { 4631 /* 4632 * Send to the stack if: 4633 ** - LRO not enabled, or 4634 ** - no LRO resources, or 4635 ** - lro enqueue fails 4636 */ 4637 if (rxr->lro.lro_cnt != 0) 4638 if (tcp_lro_rx(&rxr->lro, m, 0) == 0) 4639 return; 4640 } 4641 #endif /* LRO */ 4642 4643 IXGBE_RX_UNLOCK(rxr); 4644 4645 s = splnet(); 4646 /* Pass this up to any BPF listeners. */ 4647 bpf_mtap(ifp, m); 4648 (*ifp->if_input)(ifp, m); 4649 splx(s); 4650 4651 IXGBE_RX_LOCK(rxr); 4652 } 4653 4654 static __inline void 4655 ixgbe_rx_discard(struct rx_ring *rxr, int i) 4656 { 4657 struct ixgbe_rx_buf *rbuf; 4658 4659 rbuf = &rxr->rx_buffers[i]; 4660 4661 if (rbuf->fmp != NULL) {/* Partial chain ? */ 4662 rbuf->fmp->m_flags |= M_PKTHDR; 4663 m_freem(rbuf->fmp); 4664 rbuf->fmp = NULL; 4665 } 4666 4667 /* 4668 ** With advanced descriptors the writeback 4669 ** clobbers the buffer addrs, so its easier 4670 ** to just free the existing mbufs and take 4671 ** the normal refresh path to get new buffers 4672 ** and mapping. 4673 */ 4674 if (rbuf->m_head) { 4675 m_free(rbuf->m_head); 4676 rbuf->m_head = NULL; 4677 } 4678 4679 if (rbuf->m_pack) { 4680 m_free(rbuf->m_pack); 4681 rbuf->m_pack = NULL; 4682 } 4683 4684 return; 4685 } 4686 4687 4688 /********************************************************************* 4689 * 4690 * This routine executes in interrupt context. It replenishes 4691 * the mbufs in the descriptor and sends data which has been 4692 * dma'ed into host memory to upper layer. 4693 * 4694 * We loop at most count times if count is > 0, or until done if 4695 * count < 0. 4696 * 4697 * Return TRUE for more work, FALSE for all clean. 4698 *********************************************************************/ 4699 static bool 4700 ixgbe_rxeof(struct ix_queue *que, int count) 4701 { 4702 struct adapter *adapter = que->adapter; 4703 struct rx_ring *rxr = que->rxr; 4704 struct ifnet *ifp = adapter->ifp; 4705 #ifdef LRO 4706 struct lro_ctrl *lro = &rxr->lro; 4707 struct lro_entry *queued; 4708 #endif /* LRO */ 4709 int i, nextp, processed = 0; 4710 u32 staterr = 0; 4711 union ixgbe_adv_rx_desc *cur; 4712 struct ixgbe_rx_buf *rbuf, *nbuf; 4713 4714 IXGBE_RX_LOCK(rxr); 4715 4716 #ifdef DEV_NETMAP 4717 if (ifp->if_capenable & IFCAP_NETMAP) { 4718 /* 4719 * Same as the txeof routine: only wakeup clients on intr. 4720 * NKR_PENDINTR in nr_kflags is used to implement interrupt 4721 * mitigation (ixgbe_rxsync() will not look for new packets 4722 * unless NKR_PENDINTR is set). 4723 */ 4724 struct netmap_adapter *na = NA(ifp); 4725 4726 na->rx_rings[rxr->me].nr_kflags |= NKR_PENDINTR; 4727 selwakeuppri(&na->rx_rings[rxr->me].si, PI_NET); 4728 IXGBE_RX_UNLOCK(rxr); 4729 IXGBE_CORE_LOCK(adapter); 4730 selwakeuppri(&na->rx_si, PI_NET); 4731 IXGBE_CORE_UNLOCK(adapter); 4732 return (FALSE); 4733 } 4734 #endif /* DEV_NETMAP */ 4735 for (i = rxr->next_to_check; count != 0;) { 4736 struct mbuf *sendmp, *mh, *mp; 4737 u32 rsc, ptype; 4738 u16 hlen, plen, hdr; 4739 u16 vtag = 0; 4740 bool eop; 4741 4742 /* Sync the ring. */ 4743 ixgbe_dmamap_sync(rxr->rxdma.dma_tag, rxr->rxdma.dma_map, 4744 BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE); 4745 4746 cur = &rxr->rx_base[i]; 4747 staterr = le32toh(cur->wb.upper.status_error); 4748 4749 if ((staterr & IXGBE_RXD_STAT_DD) == 0) 4750 break; 4751 if ((ifp->if_flags & IFF_RUNNING) == 0) 4752 break; 4753 4754 count--; 4755 sendmp = NULL; 4756 nbuf = NULL; 4757 rsc = 0; 4758 cur->wb.upper.status_error = 0; 4759 rbuf = &rxr->rx_buffers[i]; 4760 mh = rbuf->m_head; 4761 mp = rbuf->m_pack; 4762 4763 plen = le16toh(cur->wb.upper.length); 4764 ptype = le32toh(cur->wb.lower.lo_dword.data) & 4765 IXGBE_RXDADV_PKTTYPE_MASK; 4766 hdr = le16toh(cur->wb.lower.lo_dword.hs_rss.hdr_info); 4767 eop = ((staterr & IXGBE_RXD_STAT_EOP) != 0); 4768 4769 /* Process vlan info */ 4770 if ((rxr->vtag_strip) && (staterr & IXGBE_RXD_STAT_VP)) 4771 vtag = le16toh(cur->wb.upper.vlan); 4772 4773 /* Make sure bad packets are discarded */ 4774 if (((staterr & IXGBE_RXDADV_ERR_FRAME_ERR_MASK) != 0) || 4775 (rxr->discard)) { 4776 ifp->if_ierrors++; 4777 rxr->rx_discarded.ev_count++; 4778 if (eop) 4779 rxr->discard = FALSE; 4780 else 4781 rxr->discard = TRUE; 4782 ixgbe_rx_discard(rxr, i); 4783 goto next_desc; 4784 } 4785 4786 /* 4787 ** On 82599 which supports a hardware 4788 ** LRO (called HW RSC), packets need 4789 ** not be fragmented across sequential 4790 ** descriptors, rather the next descriptor 4791 ** is indicated in bits of the descriptor. 4792 ** This also means that we might proceses 4793 ** more than one packet at a time, something 4794 ** that has never been true before, it 4795 ** required eliminating global chain pointers 4796 ** in favor of what we are doing here. -jfv 4797 */ 4798 if (!eop) { 4799 /* 4800 ** Figure out the next descriptor 4801 ** of this frame. 4802 */ 4803 if (rxr->hw_rsc == TRUE) { 4804 rsc = ixgbe_rsc_count(cur); 4805 rxr->rsc_num += (rsc - 1); 4806 } 4807 if (rsc) { /* Get hardware index */ 4808 nextp = ((staterr & 4809 IXGBE_RXDADV_NEXTP_MASK) >> 4810 IXGBE_RXDADV_NEXTP_SHIFT); 4811 } else { /* Just sequential */ 4812 nextp = i + 1; 4813 if (nextp == adapter->num_rx_desc) 4814 nextp = 0; 4815 } 4816 nbuf = &rxr->rx_buffers[nextp]; 4817 prefetch(nbuf); 4818 } 4819 /* 4820 ** The header mbuf is ONLY used when header 4821 ** split is enabled, otherwise we get normal 4822 ** behavior, ie, both header and payload 4823 ** are DMA'd into the payload buffer. 4824 ** 4825 ** Rather than using the fmp/lmp global pointers 4826 ** we now keep the head of a packet chain in the 4827 ** buffer struct and pass this along from one 4828 ** descriptor to the next, until we get EOP. 4829 */ 4830 if (rxr->hdr_split && (rbuf->fmp == NULL)) { 4831 /* This must be an initial descriptor */ 4832 hlen = (hdr & IXGBE_RXDADV_HDRBUFLEN_MASK) >> 4833 IXGBE_RXDADV_HDRBUFLEN_SHIFT; 4834 if (hlen > IXGBE_RX_HDR) 4835 hlen = IXGBE_RX_HDR; 4836 mh->m_len = hlen; 4837 mh->m_flags |= M_PKTHDR; 4838 mh->m_next = NULL; 4839 mh->m_pkthdr.len = mh->m_len; 4840 /* Null buf pointer so it is refreshed */ 4841 rbuf->m_head = NULL; 4842 /* 4843 ** Check the payload length, this 4844 ** could be zero if its a small 4845 ** packet. 4846 */ 4847 if (plen > 0) { 4848 mp->m_len = plen; 4849 mp->m_next = NULL; 4850 mp->m_flags &= ~M_PKTHDR; 4851 mh->m_next = mp; 4852 mh->m_pkthdr.len += mp->m_len; 4853 /* Null buf pointer so it is refreshed */ 4854 rbuf->m_pack = NULL; 4855 rxr->rx_split_packets.ev_count++; 4856 } 4857 /* 4858 ** Now create the forward 4859 ** chain so when complete 4860 ** we wont have to. 4861 */ 4862 if (eop == 0) { 4863 /* stash the chain head */ 4864 nbuf->fmp = mh; 4865 /* Make forward chain */ 4866 if (plen) 4867 mp->m_next = nbuf->m_pack; 4868 else 4869 mh->m_next = nbuf->m_pack; 4870 } else { 4871 /* Singlet, prepare to send */ 4872 sendmp = mh; 4873 /* If hardware handled vtag */ 4874 if (vtag) { 4875 /* XXX Do something reasonable on 4876 * error. 4877 */ 4878 #if 0 4879 printf("%s.%d: VLAN_INPUT_TAG\n", 4880 __func__, __LINE__); 4881 Debugger(); 4882 #endif 4883 VLAN_INPUT_TAG(ifp, sendmp, vtag, 4884 printf("%s: could not apply VLAN " 4885 "tag", __func__)); 4886 } 4887 } 4888 } else { 4889 /* 4890 ** Either no header split, or a 4891 ** secondary piece of a fragmented 4892 ** split packet. 4893 */ 4894 mp->m_len = plen; 4895 /* 4896 ** See if there is a stored head 4897 ** that determines what we are 4898 */ 4899 sendmp = rbuf->fmp; 4900 rbuf->m_pack = rbuf->fmp = NULL; 4901 4902 if (sendmp != NULL) { /* secondary frag */ 4903 mp->m_flags &= ~M_PKTHDR; 4904 sendmp->m_pkthdr.len += mp->m_len; 4905 } else { 4906 /* first desc of a non-ps chain */ 4907 sendmp = mp; 4908 sendmp->m_flags |= M_PKTHDR; 4909 sendmp->m_pkthdr.len = mp->m_len; 4910 if (staterr & IXGBE_RXD_STAT_VP) { 4911 /* XXX Do something reasonable on 4912 * error. 4913 */ 4914 #if 0 4915 printf("%s.%d: VLAN_INPUT_TAG\n", 4916 __func__, __LINE__); 4917 Debugger(); 4918 #endif 4919 VLAN_INPUT_TAG(ifp, sendmp, vtag, 4920 printf("%s: could not apply VLAN " 4921 "tag", __func__)); 4922 } 4923 } 4924 /* Pass the head pointer on */ 4925 if (eop == 0) { 4926 nbuf->fmp = sendmp; 4927 sendmp = NULL; 4928 mp->m_next = nbuf->m_pack; 4929 } 4930 } 4931 ++processed; 4932 /* Sending this frame? */ 4933 if (eop) { 4934 sendmp->m_pkthdr.rcvif = ifp; 4935 ifp->if_ipackets++; 4936 rxr->rx_packets.ev_count++; 4937 /* capture data for AIM */ 4938 rxr->bytes += sendmp->m_pkthdr.len; 4939 rxr->rx_bytes.ev_count += sendmp->m_pkthdr.len; 4940 if ((ifp->if_capenable & IFCAP_RXCSUM) != 0) { 4941 ixgbe_rx_checksum(staterr, sendmp, ptype, 4942 &adapter->stats); 4943 } 4944 #if __FreeBSD_version >= 800000 4945 sendmp->m_pkthdr.flowid = que->msix; 4946 sendmp->m_flags |= M_FLOWID; 4947 #endif 4948 } 4949 next_desc: 4950 ixgbe_dmamap_sync(rxr->rxdma.dma_tag, rxr->rxdma.dma_map, 4951 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); 4952 4953 /* Advance our pointers to the next descriptor. */ 4954 if (++i == adapter->num_rx_desc) 4955 i = 0; 4956 4957 /* Now send to the stack or do LRO */ 4958 if (sendmp != NULL) { 4959 rxr->next_to_check = i; 4960 ixgbe_rx_input(rxr, ifp, sendmp, ptype); 4961 i = rxr->next_to_check; 4962 } 4963 4964 /* Every 8 descriptors we go to refresh mbufs */ 4965 if (processed == 8) { 4966 ixgbe_refresh_mbufs(rxr, i); 4967 processed = 0; 4968 } 4969 } 4970 4971 /* Refresh any remaining buf structs */ 4972 if (ixgbe_rx_unrefreshed(rxr)) 4973 ixgbe_refresh_mbufs(rxr, i); 4974 4975 rxr->next_to_check = i; 4976 4977 #ifdef LRO 4978 /* 4979 * Flush any outstanding LRO work 4980 */ 4981 while ((queued = SLIST_FIRST(&lro->lro_active)) != NULL) { 4982 SLIST_REMOVE_HEAD(&lro->lro_active, next); 4983 tcp_lro_flush(lro, queued); 4984 } 4985 #endif /* LRO */ 4986 4987 IXGBE_RX_UNLOCK(rxr); 4988 4989 /* 4990 ** We still have cleaning to do? 4991 ** Schedule another interrupt if so. 4992 */ 4993 if ((staterr & IXGBE_RXD_STAT_DD) != 0) { 4994 ixgbe_rearm_queues(adapter, (u64)(1ULL << que->msix)); 4995 return true; 4996 } 4997 4998 return false; 4999 } 5000 5001 5002 /********************************************************************* 5003 * 5004 * Verify that the hardware indicated that the checksum is valid. 5005 * Inform the stack about the status of checksum so that stack 5006 * doesn't spend time verifying the checksum. 5007 * 5008 *********************************************************************/ 5009 static void 5010 ixgbe_rx_checksum(u32 staterr, struct mbuf * mp, u32 ptype, 5011 struct ixgbe_hw_stats *stats) 5012 { 5013 u16 status = (u16) staterr; 5014 u8 errors = (u8) (staterr >> 24); 5015 #if 0 5016 bool sctp = FALSE; 5017 5018 if ((ptype & IXGBE_RXDADV_PKTTYPE_ETQF) == 0 && 5019 (ptype & IXGBE_RXDADV_PKTTYPE_SCTP) != 0) 5020 sctp = TRUE; 5021 #endif 5022 5023 if (status & IXGBE_RXD_STAT_IPCS) { 5024 stats->ipcs.ev_count++; 5025 if (!(errors & IXGBE_RXD_ERR_IPE)) { 5026 /* IP Checksum Good */ 5027 mp->m_pkthdr.csum_flags = M_CSUM_IPv4; 5028 5029 } else { 5030 stats->ipcs_bad.ev_count++; 5031 mp->m_pkthdr.csum_flags = M_CSUM_IPv4|M_CSUM_IPv4_BAD; 5032 } 5033 } 5034 if (status & IXGBE_RXD_STAT_L4CS) { 5035 stats->l4cs.ev_count++; 5036 u16 type = M_CSUM_TCPv4|M_CSUM_TCPv6|M_CSUM_UDPv4|M_CSUM_UDPv6; 5037 if (!(errors & IXGBE_RXD_ERR_TCPE)) { 5038 mp->m_pkthdr.csum_flags |= type; 5039 } else { 5040 stats->l4cs_bad.ev_count++; 5041 mp->m_pkthdr.csum_flags |= type | M_CSUM_TCP_UDP_BAD; 5042 } 5043 } 5044 return; 5045 } 5046 5047 5048 #if 0 /* XXX Badly need to overhaul vlan(4) on NetBSD. */ 5049 /* 5050 ** This routine is run via an vlan config EVENT, 5051 ** it enables us to use the HW Filter table since 5052 ** we can get the vlan id. This just creates the 5053 ** entry in the soft version of the VFTA, init will 5054 ** repopulate the real table. 5055 */ 5056 static void 5057 ixgbe_register_vlan(void *arg, struct ifnet *ifp, u16 vtag) 5058 { 5059 struct adapter *adapter = ifp->if_softc; 5060 u16 index, bit; 5061 5062 if (ifp->if_softc != arg) /* Not our event */ 5063 return; 5064 5065 if ((vtag == 0) || (vtag > 4095)) /* Invalid */ 5066 return; 5067 5068 IXGBE_CORE_LOCK(adapter); 5069 index = (vtag >> 5) & 0x7F; 5070 bit = vtag & 0x1F; 5071 adapter->shadow_vfta[index] |= (1 << bit); 5072 ixgbe_init_locked(adapter); 5073 IXGBE_CORE_UNLOCK(adapter); 5074 } 5075 5076 /* 5077 ** This routine is run via an vlan 5078 ** unconfig EVENT, remove our entry 5079 ** in the soft vfta. 5080 */ 5081 static void 5082 ixgbe_unregister_vlan(void *arg, struct ifnet *ifp, u16 vtag) 5083 { 5084 struct adapter *adapter = ifp->if_softc; 5085 u16 index, bit; 5086 5087 if (ifp->if_softc != arg) 5088 return; 5089 5090 if ((vtag == 0) || (vtag > 4095)) /* Invalid */ 5091 return; 5092 5093 IXGBE_CORE_LOCK(adapter); 5094 index = (vtag >> 5) & 0x7F; 5095 bit = vtag & 0x1F; 5096 adapter->shadow_vfta[index] &= ~(1 << bit); 5097 /* Re-init to load the changes */ 5098 ixgbe_init_locked(adapter); 5099 IXGBE_CORE_UNLOCK(adapter); 5100 } 5101 #endif 5102 5103 static void 5104 ixgbe_setup_vlan_hw_support(struct adapter *adapter) 5105 { 5106 struct ethercom *ec = &adapter->osdep.ec; 5107 struct ixgbe_hw *hw = &adapter->hw; 5108 struct rx_ring *rxr; 5109 u32 ctrl; 5110 5111 /* 5112 ** We get here thru init_locked, meaning 5113 ** a soft reset, this has already cleared 5114 ** the VFTA and other state, so if there 5115 ** have been no vlan's registered do nothing. 5116 */ 5117 if (!VLAN_ATTACHED(&adapter->osdep.ec)) { 5118 return; 5119 } 5120 5121 /* 5122 ** A soft reset zero's out the VFTA, so 5123 ** we need to repopulate it now. 5124 */ 5125 for (int i = 0; i < IXGBE_VFTA_SIZE; i++) 5126 if (adapter->shadow_vfta[i] != 0) 5127 IXGBE_WRITE_REG(hw, IXGBE_VFTA(i), 5128 adapter->shadow_vfta[i]); 5129 5130 ctrl = IXGBE_READ_REG(hw, IXGBE_VLNCTRL); 5131 /* Enable the Filter Table if enabled */ 5132 if (ec->ec_capenable & ETHERCAP_VLAN_HWFILTER) { 5133 ctrl &= ~IXGBE_VLNCTRL_CFIEN; 5134 ctrl |= IXGBE_VLNCTRL_VFE; 5135 } 5136 if (hw->mac.type == ixgbe_mac_82598EB) 5137 ctrl |= IXGBE_VLNCTRL_VME; 5138 IXGBE_WRITE_REG(hw, IXGBE_VLNCTRL, ctrl); 5139 5140 /* Setup the queues for vlans */ 5141 for (int i = 0; i < adapter->num_queues; i++) { 5142 rxr = &adapter->rx_rings[i]; 5143 /* On 82599 the VLAN enable is per/queue in RXDCTL */ 5144 if (hw->mac.type != ixgbe_mac_82598EB) { 5145 ctrl = IXGBE_READ_REG(hw, IXGBE_RXDCTL(i)); 5146 ctrl |= IXGBE_RXDCTL_VME; 5147 IXGBE_WRITE_REG(hw, IXGBE_RXDCTL(i), ctrl); 5148 } 5149 rxr->vtag_strip = TRUE; 5150 } 5151 } 5152 5153 static void 5154 ixgbe_enable_intr(struct adapter *adapter) 5155 { 5156 struct ixgbe_hw *hw = &adapter->hw; 5157 struct ix_queue *que = adapter->queues; 5158 u32 mask = (IXGBE_EIMS_ENABLE_MASK & ~IXGBE_EIMS_RTX_QUEUE); 5159 5160 5161 /* Enable Fan Failure detection */ 5162 if (hw->device_id == IXGBE_DEV_ID_82598AT) 5163 mask |= IXGBE_EIMS_GPI_SDP1; 5164 else { 5165 mask |= IXGBE_EIMS_ECC; 5166 mask |= IXGBE_EIMS_GPI_SDP0; 5167 mask |= IXGBE_EIMS_GPI_SDP1; 5168 mask |= IXGBE_EIMS_GPI_SDP2; 5169 #ifdef IXGBE_FDIR 5170 mask |= IXGBE_EIMS_FLOW_DIR; 5171 #endif 5172 } 5173 5174 IXGBE_WRITE_REG(hw, IXGBE_EIMS, mask); 5175 5176 /* With RSS we use auto clear */ 5177 if (adapter->msix_mem) { 5178 mask = IXGBE_EIMS_ENABLE_MASK; 5179 /* Don't autoclear Link */ 5180 mask &= ~IXGBE_EIMS_OTHER; 5181 mask &= ~IXGBE_EIMS_LSC; 5182 IXGBE_WRITE_REG(hw, IXGBE_EIAC, mask); 5183 } 5184 5185 /* 5186 ** Now enable all queues, this is done separately to 5187 ** allow for handling the extended (beyond 32) MSIX 5188 ** vectors that can be used by 82599 5189 */ 5190 for (int i = 0; i < adapter->num_queues; i++, que++) 5191 ixgbe_enable_queue(adapter, que->msix); 5192 5193 IXGBE_WRITE_FLUSH(hw); 5194 5195 return; 5196 } 5197 5198 static void 5199 ixgbe_disable_intr(struct adapter *adapter) 5200 { 5201 if (adapter->msix_mem) 5202 IXGBE_WRITE_REG(&adapter->hw, IXGBE_EIAC, 0); 5203 if (adapter->hw.mac.type == ixgbe_mac_82598EB) { 5204 IXGBE_WRITE_REG(&adapter->hw, IXGBE_EIMC, ~0); 5205 } else { 5206 IXGBE_WRITE_REG(&adapter->hw, IXGBE_EIMC, 0xFFFF0000); 5207 IXGBE_WRITE_REG(&adapter->hw, IXGBE_EIMC_EX(0), ~0); 5208 IXGBE_WRITE_REG(&adapter->hw, IXGBE_EIMC_EX(1), ~0); 5209 } 5210 IXGBE_WRITE_FLUSH(&adapter->hw); 5211 return; 5212 } 5213 5214 u16 5215 ixgbe_read_pci_cfg(struct ixgbe_hw *hw, u32 reg) 5216 { 5217 switch (reg % 4) { 5218 case 0: 5219 return pci_conf_read(hw->back->pc, hw->back->tag, reg) & 5220 __BITS(15, 0); 5221 case 2: 5222 return __SHIFTOUT(pci_conf_read(hw->back->pc, hw->back->tag, 5223 reg - 2), __BITS(31, 16)); 5224 default: 5225 panic("%s: invalid register (%" PRIx32, __func__, reg); 5226 break; 5227 } 5228 } 5229 5230 void 5231 ixgbe_write_pci_cfg(struct ixgbe_hw *hw, u32 reg, u16 value) 5232 { 5233 pcireg_t old; 5234 5235 switch (reg % 4) { 5236 case 0: 5237 old = pci_conf_read(hw->back->pc, hw->back->tag, reg) & 5238 __BITS(31, 16); 5239 pci_conf_write(hw->back->pc, hw->back->tag, reg, value | old); 5240 break; 5241 case 2: 5242 old = pci_conf_read(hw->back->pc, hw->back->tag, reg - 2) & 5243 __BITS(15, 0); 5244 pci_conf_write(hw->back->pc, hw->back->tag, reg - 2, 5245 __SHIFTIN(value, __BITS(31, 16)) | old); 5246 break; 5247 default: 5248 panic("%s: invalid register (%" PRIx32, __func__, reg); 5249 break; 5250 } 5251 5252 return; 5253 } 5254 5255 /* 5256 ** Setup the correct IVAR register for a particular MSIX interrupt 5257 ** (yes this is all very magic and confusing :) 5258 ** - entry is the register array entry 5259 ** - vector is the MSIX vector for this queue 5260 ** - type is RX/TX/MISC 5261 */ 5262 static void 5263 ixgbe_set_ivar(struct adapter *adapter, u8 entry, u8 vector, s8 type) 5264 { 5265 struct ixgbe_hw *hw = &adapter->hw; 5266 u32 ivar, index; 5267 5268 vector |= IXGBE_IVAR_ALLOC_VAL; 5269 5270 switch (hw->mac.type) { 5271 5272 case ixgbe_mac_82598EB: 5273 if (type == -1) 5274 entry = IXGBE_IVAR_OTHER_CAUSES_INDEX; 5275 else 5276 entry += (type * 64); 5277 index = (entry >> 2) & 0x1F; 5278 ivar = IXGBE_READ_REG(hw, IXGBE_IVAR(index)); 5279 ivar &= ~(0xFF << (8 * (entry & 0x3))); 5280 ivar |= (vector << (8 * (entry & 0x3))); 5281 IXGBE_WRITE_REG(&adapter->hw, IXGBE_IVAR(index), ivar); 5282 break; 5283 5284 case ixgbe_mac_82599EB: 5285 case ixgbe_mac_X540: 5286 if (type == -1) { /* MISC IVAR */ 5287 index = (entry & 1) * 8; 5288 ivar = IXGBE_READ_REG(hw, IXGBE_IVAR_MISC); 5289 ivar &= ~(0xFF << index); 5290 ivar |= (vector << index); 5291 IXGBE_WRITE_REG(hw, IXGBE_IVAR_MISC, ivar); 5292 } else { /* RX/TX IVARS */ 5293 index = (16 * (entry & 1)) + (8 * type); 5294 ivar = IXGBE_READ_REG(hw, IXGBE_IVAR(entry >> 1)); 5295 ivar &= ~(0xFF << index); 5296 ivar |= (vector << index); 5297 IXGBE_WRITE_REG(hw, IXGBE_IVAR(entry >> 1), ivar); 5298 } 5299 5300 default: 5301 break; 5302 } 5303 } 5304 5305 static void 5306 ixgbe_configure_ivars(struct adapter *adapter) 5307 { 5308 struct ix_queue *que = adapter->queues; 5309 u32 newitr; 5310 5311 if (ixgbe_max_interrupt_rate > 0) 5312 newitr = (4000000 / ixgbe_max_interrupt_rate) & 0x0FF8; 5313 else 5314 newitr = 0; 5315 5316 for (int i = 0; i < adapter->num_queues; i++, que++) { 5317 /* First the RX queue entry */ 5318 ixgbe_set_ivar(adapter, i, que->msix, 0); 5319 /* ... and the TX */ 5320 ixgbe_set_ivar(adapter, i, que->msix, 1); 5321 /* Set an Initial EITR value */ 5322 IXGBE_WRITE_REG(&adapter->hw, 5323 IXGBE_EITR(que->msix), newitr); 5324 } 5325 5326 /* For the Link interrupt */ 5327 ixgbe_set_ivar(adapter, 1, adapter->linkvec, -1); 5328 } 5329 5330 /* 5331 ** ixgbe_sfp_probe - called in the local timer to 5332 ** determine if a port had optics inserted. 5333 */ 5334 static bool ixgbe_sfp_probe(struct adapter *adapter) 5335 { 5336 struct ixgbe_hw *hw = &adapter->hw; 5337 device_t dev = adapter->dev; 5338 bool result = FALSE; 5339 5340 if ((hw->phy.type == ixgbe_phy_nl) && 5341 (hw->phy.sfp_type == ixgbe_sfp_type_not_present)) { 5342 s32 ret = hw->phy.ops.identify_sfp(hw); 5343 if (ret) 5344 goto out; 5345 ret = hw->phy.ops.reset(hw); 5346 if (ret == IXGBE_ERR_SFP_NOT_SUPPORTED) { 5347 device_printf(dev,"Unsupported SFP+ module detected!"); 5348 device_printf(dev, "Reload driver with supported module.\n"); 5349 adapter->sfp_probe = FALSE; 5350 goto out; 5351 } else 5352 device_printf(dev,"SFP+ module detected!\n"); 5353 /* We now have supported optics */ 5354 adapter->sfp_probe = FALSE; 5355 /* Set the optics type so system reports correctly */ 5356 ixgbe_setup_optics(adapter); 5357 result = TRUE; 5358 } 5359 out: 5360 return (result); 5361 } 5362 5363 /* 5364 ** Tasklet handler for MSIX Link interrupts 5365 ** - do outside interrupt since it might sleep 5366 */ 5367 static void 5368 ixgbe_handle_link(void *context) 5369 { 5370 struct adapter *adapter = context; 5371 5372 if (ixgbe_check_link(&adapter->hw, 5373 &adapter->link_speed, &adapter->link_up, 0) == 0) 5374 ixgbe_update_link_status(adapter); 5375 } 5376 5377 /* 5378 ** Tasklet for handling SFP module interrupts 5379 */ 5380 static void 5381 ixgbe_handle_mod(void *context) 5382 { 5383 struct adapter *adapter = context; 5384 struct ixgbe_hw *hw = &adapter->hw; 5385 device_t dev = adapter->dev; 5386 u32 err; 5387 5388 err = hw->phy.ops.identify_sfp(hw); 5389 if (err == IXGBE_ERR_SFP_NOT_SUPPORTED) { 5390 device_printf(dev, 5391 "Unsupported SFP+ module type was detected.\n"); 5392 return; 5393 } 5394 err = hw->mac.ops.setup_sfp(hw); 5395 if (err == IXGBE_ERR_SFP_NOT_SUPPORTED) { 5396 device_printf(dev, 5397 "Setup failure - unsupported SFP+ module type.\n"); 5398 return; 5399 } 5400 softint_schedule(adapter->msf_si); 5401 return; 5402 } 5403 5404 5405 /* 5406 ** Tasklet for handling MSF (multispeed fiber) interrupts 5407 */ 5408 static void 5409 ixgbe_handle_msf(void *context) 5410 { 5411 struct adapter *adapter = context; 5412 struct ixgbe_hw *hw = &adapter->hw; 5413 u32 autoneg; 5414 bool negotiate; 5415 5416 autoneg = hw->phy.autoneg_advertised; 5417 if ((!autoneg) && (hw->mac.ops.get_link_capabilities)) 5418 hw->mac.ops.get_link_capabilities(hw, &autoneg, &negotiate); 5419 else 5420 negotiate = 0; 5421 if (hw->mac.ops.setup_link) 5422 hw->mac.ops.setup_link(hw, autoneg, negotiate, TRUE); 5423 return; 5424 } 5425 5426 #ifdef IXGBE_FDIR 5427 /* 5428 ** Tasklet for reinitializing the Flow Director filter table 5429 */ 5430 static void 5431 ixgbe_reinit_fdir(void *context) 5432 { 5433 struct adapter *adapter = context; 5434 struct ifnet *ifp = adapter->ifp; 5435 5436 if (adapter->fdir_reinit != 1) /* Shouldn't happen */ 5437 return; 5438 ixgbe_reinit_fdir_tables_82599(&adapter->hw); 5439 adapter->fdir_reinit = 0; 5440 /* re-enable flow director interrupts */ 5441 IXGBE_WRITE_REG(&adapter->hw, IXGBE_EIMS, IXGBE_EIMS_FLOW_DIR); 5442 /* Restart the interface */ 5443 ifp->if_flags |= IFF_RUNNING; 5444 return; 5445 } 5446 #endif 5447 5448 /********************************************************************** 5449 * 5450 * Update the board statistics counters. 5451 * 5452 **********************************************************************/ 5453 static void 5454 ixgbe_update_stats_counters(struct adapter *adapter) 5455 { 5456 struct ifnet *ifp = adapter->ifp; 5457 struct ixgbe_hw *hw = &adapter->hw; 5458 u32 missed_rx = 0, bprc, lxon, lxoff, total; 5459 u64 total_missed_rx = 0; 5460 5461 adapter->stats.crcerrs.ev_count += IXGBE_READ_REG(hw, IXGBE_CRCERRS); 5462 adapter->stats.illerrc.ev_count += IXGBE_READ_REG(hw, IXGBE_ILLERRC); 5463 adapter->stats.errbc.ev_count += IXGBE_READ_REG(hw, IXGBE_ERRBC); 5464 adapter->stats.mspdc.ev_count += IXGBE_READ_REG(hw, IXGBE_MSPDC); 5465 5466 for (int i = 0; i < __arraycount(adapter->stats.mpc); i++) { 5467 int j = i % adapter->num_queues; 5468 u32 mp; 5469 mp = IXGBE_READ_REG(hw, IXGBE_MPC(i)); 5470 /* missed_rx tallies misses for the gprc workaround */ 5471 missed_rx += mp; 5472 /* global total per queue */ 5473 adapter->stats.mpc[j].ev_count += mp; 5474 /* Running comprehensive total for stats display */ 5475 total_missed_rx += adapter->stats.mpc[j].ev_count; 5476 if (hw->mac.type == ixgbe_mac_82598EB) 5477 adapter->stats.rnbc[j] += 5478 IXGBE_READ_REG(hw, IXGBE_RNBC(i)); 5479 adapter->stats.pxontxc[j].ev_count += 5480 IXGBE_READ_REG(hw, IXGBE_PXONTXC(i)); 5481 adapter->stats.pxonrxc[j].ev_count += 5482 IXGBE_READ_REG(hw, IXGBE_PXONRXC(i)); 5483 adapter->stats.pxofftxc[j].ev_count += 5484 IXGBE_READ_REG(hw, IXGBE_PXOFFTXC(i)); 5485 adapter->stats.pxoffrxc[j].ev_count += 5486 IXGBE_READ_REG(hw, IXGBE_PXOFFRXC(i)); 5487 adapter->stats.pxon2offc[j].ev_count += 5488 IXGBE_READ_REG(hw, IXGBE_PXON2OFFCNT(i)); 5489 } 5490 for (int i = 0; i < __arraycount(adapter->stats.qprc); i++) { 5491 int j = i % adapter->num_queues; 5492 adapter->stats.qprc[j].ev_count += IXGBE_READ_REG(hw, IXGBE_QPRC(i)); 5493 adapter->stats.qptc[j].ev_count += IXGBE_READ_REG(hw, IXGBE_QPTC(i)); 5494 adapter->stats.qbrc[j].ev_count += IXGBE_READ_REG(hw, IXGBE_QBRC(i)); 5495 adapter->stats.qbrc[j].ev_count += 5496 ((u64)IXGBE_READ_REG(hw, IXGBE_QBRC(i)) << 32); 5497 adapter->stats.qbtc[j].ev_count += IXGBE_READ_REG(hw, IXGBE_QBTC(i)); 5498 adapter->stats.qbtc[j].ev_count += 5499 ((u64)IXGBE_READ_REG(hw, IXGBE_QBTC(i)) << 32); 5500 adapter->stats.qprdc[j].ev_count += IXGBE_READ_REG(hw, IXGBE_QPRDC(i)); 5501 } 5502 adapter->stats.mlfc.ev_count += IXGBE_READ_REG(hw, IXGBE_MLFC); 5503 adapter->stats.mrfc.ev_count += IXGBE_READ_REG(hw, IXGBE_MRFC); 5504 adapter->stats.rlec.ev_count += IXGBE_READ_REG(hw, IXGBE_RLEC); 5505 5506 /* Hardware workaround, gprc counts missed packets */ 5507 adapter->stats.gprc.ev_count += IXGBE_READ_REG(hw, IXGBE_GPRC) - missed_rx; 5508 5509 lxon = IXGBE_READ_REG(hw, IXGBE_LXONTXC); 5510 adapter->stats.lxontxc.ev_count += lxon; 5511 lxoff = IXGBE_READ_REG(hw, IXGBE_LXOFFTXC); 5512 adapter->stats.lxofftxc.ev_count += lxoff; 5513 total = lxon + lxoff; 5514 5515 if (hw->mac.type != ixgbe_mac_82598EB) { 5516 adapter->stats.gorc.ev_count += IXGBE_READ_REG(hw, IXGBE_GORCL) + 5517 ((u64)IXGBE_READ_REG(hw, IXGBE_GORCH) << 32); 5518 adapter->stats.gotc.ev_count += IXGBE_READ_REG(hw, IXGBE_GOTCL) + 5519 ((u64)IXGBE_READ_REG(hw, IXGBE_GOTCH) << 32) - total * ETHER_MIN_LEN; 5520 adapter->stats.tor.ev_count += IXGBE_READ_REG(hw, IXGBE_TORL) + 5521 ((u64)IXGBE_READ_REG(hw, IXGBE_TORH) << 32); 5522 adapter->stats.lxonrxc.ev_count += IXGBE_READ_REG(hw, IXGBE_LXONRXCNT); 5523 adapter->stats.lxoffrxc.ev_count += IXGBE_READ_REG(hw, IXGBE_LXOFFRXCNT); 5524 } else { 5525 adapter->stats.lxonrxc.ev_count += IXGBE_READ_REG(hw, IXGBE_LXONRXC); 5526 adapter->stats.lxoffrxc.ev_count += IXGBE_READ_REG(hw, IXGBE_LXOFFRXC); 5527 /* 82598 only has a counter in the high register */ 5528 adapter->stats.gorc.ev_count += IXGBE_READ_REG(hw, IXGBE_GORCH); 5529 adapter->stats.gotc.ev_count += IXGBE_READ_REG(hw, IXGBE_GOTCH) - total * ETHER_MIN_LEN; 5530 adapter->stats.tor.ev_count += IXGBE_READ_REG(hw, IXGBE_TORH); 5531 } 5532 5533 /* 5534 * Workaround: mprc hardware is incorrectly counting 5535 * broadcasts, so for now we subtract those. 5536 */ 5537 bprc = IXGBE_READ_REG(hw, IXGBE_BPRC); 5538 adapter->stats.bprc.ev_count += bprc; 5539 adapter->stats.mprc.ev_count += IXGBE_READ_REG(hw, IXGBE_MPRC) - ((hw->mac.type == ixgbe_mac_82598EB) ? bprc : 0); 5540 5541 adapter->stats.prc64.ev_count += IXGBE_READ_REG(hw, IXGBE_PRC64); 5542 adapter->stats.prc127.ev_count += IXGBE_READ_REG(hw, IXGBE_PRC127); 5543 adapter->stats.prc255.ev_count += IXGBE_READ_REG(hw, IXGBE_PRC255); 5544 adapter->stats.prc511.ev_count += IXGBE_READ_REG(hw, IXGBE_PRC511); 5545 adapter->stats.prc1023.ev_count += IXGBE_READ_REG(hw, IXGBE_PRC1023); 5546 adapter->stats.prc1522.ev_count += IXGBE_READ_REG(hw, IXGBE_PRC1522); 5547 5548 adapter->stats.gptc.ev_count += IXGBE_READ_REG(hw, IXGBE_GPTC) - total; 5549 adapter->stats.mptc.ev_count += IXGBE_READ_REG(hw, IXGBE_MPTC) - total; 5550 adapter->stats.ptc64.ev_count += IXGBE_READ_REG(hw, IXGBE_PTC64) - total; 5551 5552 adapter->stats.ruc.ev_count += IXGBE_READ_REG(hw, IXGBE_RUC); 5553 adapter->stats.rfc.ev_count += IXGBE_READ_REG(hw, IXGBE_RFC); 5554 adapter->stats.roc.ev_count += IXGBE_READ_REG(hw, IXGBE_ROC); 5555 adapter->stats.rjc.ev_count += IXGBE_READ_REG(hw, IXGBE_RJC); 5556 adapter->stats.mngprc.ev_count += IXGBE_READ_REG(hw, IXGBE_MNGPRC); 5557 adapter->stats.mngpdc.ev_count += IXGBE_READ_REG(hw, IXGBE_MNGPDC); 5558 adapter->stats.mngptc.ev_count += IXGBE_READ_REG(hw, IXGBE_MNGPTC); 5559 adapter->stats.tpr.ev_count += IXGBE_READ_REG(hw, IXGBE_TPR); 5560 adapter->stats.tpt.ev_count += IXGBE_READ_REG(hw, IXGBE_TPT); 5561 adapter->stats.ptc127.ev_count += IXGBE_READ_REG(hw, IXGBE_PTC127); 5562 adapter->stats.ptc255.ev_count += IXGBE_READ_REG(hw, IXGBE_PTC255); 5563 adapter->stats.ptc511.ev_count += IXGBE_READ_REG(hw, IXGBE_PTC511); 5564 adapter->stats.ptc1023.ev_count += IXGBE_READ_REG(hw, IXGBE_PTC1023); 5565 adapter->stats.ptc1522.ev_count += IXGBE_READ_REG(hw, IXGBE_PTC1522); 5566 adapter->stats.bptc.ev_count += IXGBE_READ_REG(hw, IXGBE_BPTC); 5567 adapter->stats.xec.ev_count += IXGBE_READ_REG(hw, IXGBE_XEC); 5568 adapter->stats.fccrc.ev_count += IXGBE_READ_REG(hw, IXGBE_FCCRC); 5569 adapter->stats.fclast.ev_count += IXGBE_READ_REG(hw, IXGBE_FCLAST); 5570 5571 /* Only read FCOE on 82599 */ 5572 if (hw->mac.type != ixgbe_mac_82598EB) { 5573 adapter->stats.fcoerpdc.ev_count += 5574 IXGBE_READ_REG(hw, IXGBE_FCOERPDC); 5575 adapter->stats.fcoeprc.ev_count += 5576 IXGBE_READ_REG(hw, IXGBE_FCOEPRC); 5577 adapter->stats.fcoeptc.ev_count += 5578 IXGBE_READ_REG(hw, IXGBE_FCOEPTC); 5579 adapter->stats.fcoedwrc.ev_count += 5580 IXGBE_READ_REG(hw, IXGBE_FCOEDWRC); 5581 adapter->stats.fcoedwtc.ev_count += 5582 IXGBE_READ_REG(hw, IXGBE_FCOEDWTC); 5583 } 5584 5585 /* Fill out the OS statistics structure */ 5586 ifp->if_ipackets = adapter->stats.gprc.ev_count; 5587 ifp->if_opackets = adapter->stats.gptc.ev_count; 5588 ifp->if_ibytes = adapter->stats.gorc.ev_count; 5589 ifp->if_obytes = adapter->stats.gotc.ev_count; 5590 ifp->if_imcasts = adapter->stats.mprc.ev_count; 5591 ifp->if_collisions = 0; 5592 5593 /* Rx Errors */ 5594 ifp->if_ierrors = total_missed_rx + adapter->stats.crcerrs.ev_count + 5595 adapter->stats.rlec.ev_count; 5596 } 5597 5598 /** ixgbe_sysctl_tdh_handler - Handler function 5599 * Retrieves the TDH value from the hardware 5600 */ 5601 static int 5602 ixgbe_sysctl_tdh_handler(SYSCTLFN_ARGS) 5603 { 5604 struct sysctlnode node; 5605 uint32_t val; 5606 struct tx_ring *txr; 5607 5608 node = *rnode; 5609 txr = (struct tx_ring *)node.sysctl_data; 5610 if (txr == NULL) 5611 return 0; 5612 val = IXGBE_READ_REG(&txr->adapter->hw, IXGBE_TDH(txr->me)); 5613 node.sysctl_data = &val; 5614 return sysctl_lookup(SYSCTLFN_CALL(&node)); 5615 } 5616 5617 /** ixgbe_sysctl_tdt_handler - Handler function 5618 * Retrieves the TDT value from the hardware 5619 */ 5620 static int 5621 ixgbe_sysctl_tdt_handler(SYSCTLFN_ARGS) 5622 { 5623 struct sysctlnode node; 5624 uint32_t val; 5625 struct tx_ring *txr; 5626 5627 node = *rnode; 5628 txr = (struct tx_ring *)node.sysctl_data; 5629 if (txr == NULL) 5630 return 0; 5631 val = IXGBE_READ_REG(&txr->adapter->hw, IXGBE_TDT(txr->me)); 5632 node.sysctl_data = &val; 5633 return sysctl_lookup(SYSCTLFN_CALL(&node)); 5634 } 5635 5636 /** ixgbe_sysctl_rdh_handler - Handler function 5637 * Retrieves the RDH value from the hardware 5638 */ 5639 static int 5640 ixgbe_sysctl_rdh_handler(SYSCTLFN_ARGS) 5641 { 5642 struct sysctlnode node; 5643 uint32_t val; 5644 struct rx_ring *rxr; 5645 5646 node = *rnode; 5647 rxr = (struct rx_ring *)node.sysctl_data; 5648 if (rxr == NULL) 5649 return 0; 5650 val = IXGBE_READ_REG(&rxr->adapter->hw, IXGBE_RDH(rxr->me)); 5651 node.sysctl_data = &val; 5652 return sysctl_lookup(SYSCTLFN_CALL(&node)); 5653 } 5654 5655 /** ixgbe_sysctl_rdt_handler - Handler function 5656 * Retrieves the RDT value from the hardware 5657 */ 5658 static int 5659 ixgbe_sysctl_rdt_handler(SYSCTLFN_ARGS) 5660 { 5661 struct sysctlnode node; 5662 uint32_t val; 5663 struct rx_ring *rxr; 5664 5665 node = *rnode; 5666 rxr = (struct rx_ring *)node.sysctl_data; 5667 if (rxr == NULL) 5668 return 0; 5669 val = IXGBE_READ_REG(&rxr->adapter->hw, IXGBE_RDT(rxr->me)); 5670 node.sysctl_data = &val; 5671 return sysctl_lookup(SYSCTLFN_CALL(&node)); 5672 } 5673 5674 static int 5675 ixgbe_sysctl_interrupt_rate_handler(SYSCTLFN_ARGS) 5676 { 5677 int error; 5678 struct sysctlnode node; 5679 struct ix_queue *que; 5680 uint32_t reg, usec, rate; 5681 5682 node = *rnode; 5683 que = (struct ix_queue *)node.sysctl_data; 5684 if (que == NULL) 5685 return 0; 5686 reg = IXGBE_READ_REG(&que->adapter->hw, IXGBE_EITR(que->msix)); 5687 usec = ((reg & 0x0FF8) >> 3); 5688 if (usec > 0) 5689 rate = 500000 / usec; 5690 else 5691 rate = 0; 5692 node.sysctl_data = &rate; 5693 error = sysctl_lookup(SYSCTLFN_CALL(&node)); 5694 if (error) 5695 return error; 5696 reg &= ~0xfff; /* default, no limitation */ 5697 ixgbe_max_interrupt_rate = 0; 5698 if (rate > 0 && rate < 500000) { 5699 if (rate < 1000) 5700 rate = 1000; 5701 ixgbe_max_interrupt_rate = rate; 5702 reg |= ((4000000/rate) & 0xff8 ); 5703 } 5704 IXGBE_WRITE_REG(&que->adapter->hw, IXGBE_EITR(que->msix), reg); 5705 return 0; 5706 } 5707 5708 const struct sysctlnode * 5709 ixgbe_sysctl_instance(struct adapter *adapter) 5710 { 5711 const char *dvname; 5712 struct sysctllog **log; 5713 int rc; 5714 const struct sysctlnode *rnode; 5715 5716 log = &adapter->sysctllog; 5717 dvname = device_xname(adapter->dev); 5718 5719 if ((rc = sysctl_createv(log, 0, NULL, &rnode, 5720 0, CTLTYPE_NODE, dvname, 5721 SYSCTL_DESCR("ixgbe information and settings"), 5722 NULL, 0, NULL, 0, CTL_HW, CTL_CREATE, CTL_EOL)) != 0) 5723 goto err; 5724 5725 return rnode; 5726 err: 5727 printf("%s: sysctl_createv failed, rc = %d\n", __func__, rc); 5728 return NULL; 5729 } 5730 5731 /* 5732 * Add sysctl variables, one per statistic, to the system. 5733 */ 5734 static void 5735 ixgbe_add_hw_stats(struct adapter *adapter) 5736 { 5737 device_t dev = adapter->dev; 5738 const struct sysctlnode *rnode, *cnode; 5739 struct sysctllog **log = &adapter->sysctllog; 5740 struct tx_ring *txr = adapter->tx_rings; 5741 struct rx_ring *rxr = adapter->rx_rings; 5742 struct ixgbe_hw *hw = &adapter->hw; 5743 5744 struct ixgbe_hw_stats *stats = &adapter->stats; 5745 5746 /* Driver Statistics */ 5747 #if 0 5748 /* These counters are not updated by the software */ 5749 SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "dropped", 5750 CTLFLAG_RD, &adapter->dropped_pkts, 5751 "Driver dropped packets"); 5752 SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "mbuf_header_failed", 5753 CTLFLAG_RD, &adapter->mbuf_header_failed, 5754 "???"); 5755 SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "mbuf_packet_failed", 5756 CTLFLAG_RD, &adapter->mbuf_packet_failed, 5757 "???"); 5758 SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "no_tx_map_avail", 5759 CTLFLAG_RD, &adapter->no_tx_map_avail, 5760 "???"); 5761 #endif 5762 evcnt_attach_dynamic(&adapter->handleq, EVCNT_TYPE_MISC, 5763 NULL, device_xname(dev), "Handled queue in softint"); 5764 evcnt_attach_dynamic(&adapter->req, EVCNT_TYPE_MISC, 5765 NULL, device_xname(dev), "Requeued in softint"); 5766 evcnt_attach_dynamic(&adapter->morerx, EVCNT_TYPE_MISC, 5767 NULL, device_xname(dev), "Interrupt handler more rx"); 5768 evcnt_attach_dynamic(&adapter->moretx, EVCNT_TYPE_MISC, 5769 NULL, device_xname(dev), "Interrupt handler more tx"); 5770 evcnt_attach_dynamic(&adapter->txloops, EVCNT_TYPE_MISC, 5771 NULL, device_xname(dev), "Interrupt handler tx loops"); 5772 evcnt_attach_dynamic(&adapter->efbig_tx_dma_setup, EVCNT_TYPE_MISC, 5773 NULL, device_xname(dev), "Driver tx dma soft fail EFBIG"); 5774 evcnt_attach_dynamic(&adapter->m_defrag_failed, EVCNT_TYPE_MISC, 5775 NULL, device_xname(dev), "m_defrag() failed"); 5776 evcnt_attach_dynamic(&adapter->efbig2_tx_dma_setup, EVCNT_TYPE_MISC, 5777 NULL, device_xname(dev), "Driver tx dma hard fail EFBIG"); 5778 evcnt_attach_dynamic(&adapter->einval_tx_dma_setup, EVCNT_TYPE_MISC, 5779 NULL, device_xname(dev), "Driver tx dma hard fail EINVAL"); 5780 evcnt_attach_dynamic(&adapter->other_tx_dma_setup, EVCNT_TYPE_MISC, 5781 NULL, device_xname(dev), "Driver tx dma hard fail other"); 5782 evcnt_attach_dynamic(&adapter->eagain_tx_dma_setup, EVCNT_TYPE_MISC, 5783 NULL, device_xname(dev), "Driver tx dma soft fail EAGAIN"); 5784 evcnt_attach_dynamic(&adapter->enomem_tx_dma_setup, EVCNT_TYPE_MISC, 5785 NULL, device_xname(dev), "Driver tx dma soft fail ENOMEM"); 5786 evcnt_attach_dynamic(&adapter->watchdog_events, EVCNT_TYPE_MISC, 5787 NULL, device_xname(dev), "Watchdog timeouts"); 5788 evcnt_attach_dynamic(&adapter->tso_err, EVCNT_TYPE_MISC, 5789 NULL, device_xname(dev), "TSO errors"); 5790 evcnt_attach_dynamic(&adapter->tso_tx, EVCNT_TYPE_MISC, 5791 NULL, device_xname(dev), "TSO"); 5792 evcnt_attach_dynamic(&adapter->link_irq, EVCNT_TYPE_MISC, 5793 NULL, device_xname(dev), "Link MSIX IRQ Handled"); 5794 5795 for (int i = 0; i < adapter->num_queues; i++, rxr++, txr++) { 5796 snprintf(adapter->queues[i].evnamebuf, 5797 sizeof(adapter->queues[i].evnamebuf), "%s queue%d", 5798 device_xname(dev), i); 5799 snprintf(adapter->queues[i].namebuf, 5800 sizeof(adapter->queues[i].namebuf), "queue%d", i); 5801 5802 if ((rnode = ixgbe_sysctl_instance(adapter)) == NULL) { 5803 aprint_error_dev(dev, "could not create sysctl root\n"); 5804 break; 5805 } 5806 5807 if (sysctl_createv(log, 0, &rnode, &rnode, 5808 0, CTLTYPE_NODE, 5809 adapter->queues[i].namebuf, SYSCTL_DESCR("Queue Name"), 5810 NULL, 0, NULL, 0, CTL_CREATE, CTL_EOL) != 0) 5811 break; 5812 5813 if (sysctl_createv(log, 0, &rnode, &cnode, 5814 CTLFLAG_READWRITE, CTLTYPE_INT, 5815 "interrupt_rate", SYSCTL_DESCR("Interrupt Rate"), 5816 ixgbe_sysctl_interrupt_rate_handler, 0, 5817 (void *)&adapter->queues[i], 0, CTL_CREATE, CTL_EOL) != 0) 5818 break; 5819 5820 if (sysctl_createv(log, 0, &rnode, &cnode, 5821 CTLFLAG_READONLY, CTLTYPE_QUAD, 5822 "irqs", SYSCTL_DESCR("irqs on this queue"), 5823 NULL, 0, &(adapter->queues[i].irqs), 5824 0, CTL_CREATE, CTL_EOL) != 0) 5825 break; 5826 5827 if (sysctl_createv(log, 0, &rnode, &cnode, 5828 CTLFLAG_READONLY, CTLTYPE_INT, 5829 "txd_head", SYSCTL_DESCR("Transmit Descriptor Head"), 5830 ixgbe_sysctl_tdh_handler, 0, (void *)txr, 5831 0, CTL_CREATE, CTL_EOL) != 0) 5832 break; 5833 5834 if (sysctl_createv(log, 0, &rnode, &cnode, 5835 CTLFLAG_READONLY, CTLTYPE_INT, 5836 "txd_tail", SYSCTL_DESCR("Transmit Descriptor Tail"), 5837 ixgbe_sysctl_tdt_handler, 0, (void *)txr, 5838 0, CTL_CREATE, CTL_EOL) != 0) 5839 break; 5840 5841 evcnt_attach_dynamic(&txr->no_desc_avail, EVCNT_TYPE_MISC, 5842 NULL, adapter->queues[i].evnamebuf, 5843 "Queue No Descriptor Available"); 5844 evcnt_attach_dynamic(&txr->total_packets, EVCNT_TYPE_MISC, 5845 NULL, adapter->queues[i].evnamebuf, 5846 "Queue Packets Transmitted"); 5847 5848 #ifdef LRO 5849 struct lro_ctrl *lro = &rxr->lro; 5850 #endif /* LRO */ 5851 5852 if (sysctl_createv(log, 0, &rnode, &cnode, 5853 CTLFLAG_READONLY, 5854 CTLTYPE_INT, 5855 "rxd_head", SYSCTL_DESCR("Receive Descriptor Head"), 5856 ixgbe_sysctl_rdh_handler, 0, (void *)rxr, 0, 5857 CTL_CREATE, CTL_EOL) != 0) 5858 break; 5859 5860 if (sysctl_createv(log, 0, &rnode, &cnode, 5861 CTLFLAG_READONLY, 5862 CTLTYPE_INT, 5863 "rxd_tail", SYSCTL_DESCR("Receive Descriptor Tail"), 5864 ixgbe_sysctl_rdt_handler, 0, (void *)rxr, 0, 5865 CTL_CREATE, CTL_EOL) != 0) 5866 break; 5867 5868 if (i < __arraycount(adapter->stats.mpc)) { 5869 evcnt_attach_dynamic(&adapter->stats.mpc[i], 5870 EVCNT_TYPE_MISC, NULL, adapter->queues[i].evnamebuf, 5871 "Missed Packet Count"); 5872 } 5873 if (i < __arraycount(adapter->stats.pxontxc)) { 5874 evcnt_attach_dynamic(&adapter->stats.pxontxc[i], 5875 EVCNT_TYPE_MISC, NULL, adapter->queues[i].evnamebuf, 5876 "pxontxc"); 5877 evcnt_attach_dynamic(&adapter->stats.pxonrxc[i], 5878 EVCNT_TYPE_MISC, NULL, adapter->queues[i].evnamebuf, 5879 "pxonrxc"); 5880 evcnt_attach_dynamic(&adapter->stats.pxofftxc[i], 5881 EVCNT_TYPE_MISC, NULL, adapter->queues[i].evnamebuf, 5882 "pxofftxc"); 5883 evcnt_attach_dynamic(&adapter->stats.pxoffrxc[i], 5884 EVCNT_TYPE_MISC, NULL, adapter->queues[i].evnamebuf, 5885 "pxoffrxc"); 5886 evcnt_attach_dynamic(&adapter->stats.pxon2offc[i], 5887 EVCNT_TYPE_MISC, NULL, adapter->queues[i].evnamebuf, 5888 "pxon2offc"); 5889 } 5890 if (i < __arraycount(adapter->stats.qprc)) { 5891 evcnt_attach_dynamic(&adapter->stats.qprc[i], 5892 EVCNT_TYPE_MISC, NULL, adapter->queues[i].evnamebuf, 5893 "qprc"); 5894 evcnt_attach_dynamic(&adapter->stats.qptc[i], 5895 EVCNT_TYPE_MISC, NULL, adapter->queues[i].evnamebuf, 5896 "qptc"); 5897 evcnt_attach_dynamic(&adapter->stats.qbrc[i], 5898 EVCNT_TYPE_MISC, NULL, adapter->queues[i].evnamebuf, 5899 "qbrc"); 5900 evcnt_attach_dynamic(&adapter->stats.qbtc[i], 5901 EVCNT_TYPE_MISC, NULL, adapter->queues[i].evnamebuf, 5902 "qbtc"); 5903 evcnt_attach_dynamic(&adapter->stats.qprdc[i], 5904 EVCNT_TYPE_MISC, NULL, adapter->queues[i].evnamebuf, 5905 "qprdc"); 5906 } 5907 5908 evcnt_attach_dynamic(&rxr->rx_packets, EVCNT_TYPE_MISC, 5909 NULL, adapter->queues[i].evnamebuf, "Queue Packets Received"); 5910 evcnt_attach_dynamic(&rxr->rx_bytes, EVCNT_TYPE_MISC, 5911 NULL, adapter->queues[i].evnamebuf, "Queue Bytes Received"); 5912 evcnt_attach_dynamic(&rxr->no_jmbuf, EVCNT_TYPE_MISC, 5913 NULL, adapter->queues[i].evnamebuf, "Rx no jumbo mbuf"); 5914 evcnt_attach_dynamic(&rxr->rx_discarded, EVCNT_TYPE_MISC, 5915 NULL, adapter->queues[i].evnamebuf, "Rx discarded"); 5916 evcnt_attach_dynamic(&rxr->rx_split_packets, EVCNT_TYPE_MISC, 5917 NULL, adapter->queues[i].evnamebuf, "Rx split packets"); 5918 evcnt_attach_dynamic(&rxr->rx_irq, EVCNT_TYPE_MISC, 5919 NULL, adapter->queues[i].evnamebuf, "Rx interrupts"); 5920 #ifdef LRO 5921 SYSCTL_ADD_INT(ctx, queue_list, OID_AUTO, "lro_queued", 5922 CTLFLAG_RD, &lro->lro_queued, 0, 5923 "LRO Queued"); 5924 SYSCTL_ADD_INT(ctx, queue_list, OID_AUTO, "lro_flushed", 5925 CTLFLAG_RD, &lro->lro_flushed, 0, 5926 "LRO Flushed"); 5927 #endif /* LRO */ 5928 } 5929 5930 /* MAC stats get the own sub node */ 5931 5932 5933 snprintf(stats->namebuf, 5934 sizeof(stats->namebuf), "%s MAC Statistics", device_xname(dev)); 5935 5936 evcnt_attach_dynamic(&stats->ipcs, EVCNT_TYPE_MISC, NULL, 5937 stats->namebuf, "rx csum offload - IP"); 5938 evcnt_attach_dynamic(&stats->l4cs, EVCNT_TYPE_MISC, NULL, 5939 stats->namebuf, "rx csum offload - L4"); 5940 evcnt_attach_dynamic(&stats->ipcs_bad, EVCNT_TYPE_MISC, NULL, 5941 stats->namebuf, "rx csum offload - IP bad"); 5942 evcnt_attach_dynamic(&stats->l4cs_bad, EVCNT_TYPE_MISC, NULL, 5943 stats->namebuf, "rx csum offload - L4 bad"); 5944 evcnt_attach_dynamic(&stats->intzero, EVCNT_TYPE_MISC, NULL, 5945 stats->namebuf, "Interrupt conditions zero"); 5946 evcnt_attach_dynamic(&stats->legint, EVCNT_TYPE_MISC, NULL, 5947 stats->namebuf, "Legacy interrupts"); 5948 evcnt_attach_dynamic(&stats->crcerrs, EVCNT_TYPE_MISC, NULL, 5949 stats->namebuf, "CRC Errors"); 5950 evcnt_attach_dynamic(&stats->illerrc, EVCNT_TYPE_MISC, NULL, 5951 stats->namebuf, "Illegal Byte Errors"); 5952 evcnt_attach_dynamic(&stats->errbc, EVCNT_TYPE_MISC, NULL, 5953 stats->namebuf, "Byte Errors"); 5954 evcnt_attach_dynamic(&stats->mspdc, EVCNT_TYPE_MISC, NULL, 5955 stats->namebuf, "MAC Short Packets Discarded"); 5956 evcnt_attach_dynamic(&stats->mlfc, EVCNT_TYPE_MISC, NULL, 5957 stats->namebuf, "MAC Local Faults"); 5958 evcnt_attach_dynamic(&stats->mrfc, EVCNT_TYPE_MISC, NULL, 5959 stats->namebuf, "MAC Remote Faults"); 5960 evcnt_attach_dynamic(&stats->rlec, EVCNT_TYPE_MISC, NULL, 5961 stats->namebuf, "Receive Length Errors"); 5962 evcnt_attach_dynamic(&stats->lxontxc, EVCNT_TYPE_MISC, NULL, 5963 stats->namebuf, "Link XON Transmitted"); 5964 evcnt_attach_dynamic(&stats->lxonrxc, EVCNT_TYPE_MISC, NULL, 5965 stats->namebuf, "Link XON Received"); 5966 evcnt_attach_dynamic(&stats->lxofftxc, EVCNT_TYPE_MISC, NULL, 5967 stats->namebuf, "Link XOFF Transmitted"); 5968 evcnt_attach_dynamic(&stats->lxoffrxc, EVCNT_TYPE_MISC, NULL, 5969 stats->namebuf, "Link XOFF Received"); 5970 5971 /* Packet Reception Stats */ 5972 evcnt_attach_dynamic(&stats->tor, EVCNT_TYPE_MISC, NULL, 5973 stats->namebuf, "Total Octets Received"); 5974 evcnt_attach_dynamic(&stats->gorc, EVCNT_TYPE_MISC, NULL, 5975 stats->namebuf, "Good Octets Received"); 5976 evcnt_attach_dynamic(&stats->tpr, EVCNT_TYPE_MISC, NULL, 5977 stats->namebuf, "Total Packets Received"); 5978 evcnt_attach_dynamic(&stats->gprc, EVCNT_TYPE_MISC, NULL, 5979 stats->namebuf, "Good Packets Received"); 5980 evcnt_attach_dynamic(&stats->mprc, EVCNT_TYPE_MISC, NULL, 5981 stats->namebuf, "Multicast Packets Received"); 5982 evcnt_attach_dynamic(&stats->bprc, EVCNT_TYPE_MISC, NULL, 5983 stats->namebuf, "Broadcast Packets Received"); 5984 evcnt_attach_dynamic(&stats->prc64, EVCNT_TYPE_MISC, NULL, 5985 stats->namebuf, "64 byte frames received "); 5986 evcnt_attach_dynamic(&stats->prc127, EVCNT_TYPE_MISC, NULL, 5987 stats->namebuf, "65-127 byte frames received"); 5988 evcnt_attach_dynamic(&stats->prc255, EVCNT_TYPE_MISC, NULL, 5989 stats->namebuf, "128-255 byte frames received"); 5990 evcnt_attach_dynamic(&stats->prc511, EVCNT_TYPE_MISC, NULL, 5991 stats->namebuf, "256-511 byte frames received"); 5992 evcnt_attach_dynamic(&stats->prc1023, EVCNT_TYPE_MISC, NULL, 5993 stats->namebuf, "512-1023 byte frames received"); 5994 evcnt_attach_dynamic(&stats->prc1522, EVCNT_TYPE_MISC, NULL, 5995 stats->namebuf, "1023-1522 byte frames received"); 5996 evcnt_attach_dynamic(&stats->ruc, EVCNT_TYPE_MISC, NULL, 5997 stats->namebuf, "Receive Undersized"); 5998 evcnt_attach_dynamic(&stats->rfc, EVCNT_TYPE_MISC, NULL, 5999 stats->namebuf, "Fragmented Packets Received "); 6000 evcnt_attach_dynamic(&stats->roc, EVCNT_TYPE_MISC, NULL, 6001 stats->namebuf, "Oversized Packets Received"); 6002 evcnt_attach_dynamic(&stats->rjc, EVCNT_TYPE_MISC, NULL, 6003 stats->namebuf, "Received Jabber"); 6004 evcnt_attach_dynamic(&stats->mngprc, EVCNT_TYPE_MISC, NULL, 6005 stats->namebuf, "Management Packets Received"); 6006 evcnt_attach_dynamic(&stats->xec, EVCNT_TYPE_MISC, NULL, 6007 stats->namebuf, "Checksum Errors"); 6008 6009 /* Packet Transmission Stats */ 6010 evcnt_attach_dynamic(&stats->gotc, EVCNT_TYPE_MISC, NULL, 6011 stats->namebuf, "Good Octets Transmitted"); 6012 evcnt_attach_dynamic(&stats->tpt, EVCNT_TYPE_MISC, NULL, 6013 stats->namebuf, "Total Packets Transmitted"); 6014 evcnt_attach_dynamic(&stats->gptc, EVCNT_TYPE_MISC, NULL, 6015 stats->namebuf, "Good Packets Transmitted"); 6016 evcnt_attach_dynamic(&stats->bptc, EVCNT_TYPE_MISC, NULL, 6017 stats->namebuf, "Broadcast Packets Transmitted"); 6018 evcnt_attach_dynamic(&stats->mptc, EVCNT_TYPE_MISC, NULL, 6019 stats->namebuf, "Multicast Packets Transmitted"); 6020 evcnt_attach_dynamic(&stats->mngptc, EVCNT_TYPE_MISC, NULL, 6021 stats->namebuf, "Management Packets Transmitted"); 6022 evcnt_attach_dynamic(&stats->ptc64, EVCNT_TYPE_MISC, NULL, 6023 stats->namebuf, "64 byte frames transmitted "); 6024 evcnt_attach_dynamic(&stats->ptc127, EVCNT_TYPE_MISC, NULL, 6025 stats->namebuf, "65-127 byte frames transmitted"); 6026 evcnt_attach_dynamic(&stats->ptc255, EVCNT_TYPE_MISC, NULL, 6027 stats->namebuf, "128-255 byte frames transmitted"); 6028 evcnt_attach_dynamic(&stats->ptc511, EVCNT_TYPE_MISC, NULL, 6029 stats->namebuf, "256-511 byte frames transmitted"); 6030 evcnt_attach_dynamic(&stats->ptc1023, EVCNT_TYPE_MISC, NULL, 6031 stats->namebuf, "512-1023 byte frames transmitted"); 6032 evcnt_attach_dynamic(&stats->ptc1522, EVCNT_TYPE_MISC, NULL, 6033 stats->namebuf, "1024-1522 byte frames transmitted"); 6034 6035 /* FC Stats */ 6036 evcnt_attach_dynamic(&stats->fccrc, EVCNT_TYPE_MISC, NULL, 6037 stats->namebuf, "FC CRC Errors"); 6038 evcnt_attach_dynamic(&stats->fclast, EVCNT_TYPE_MISC, NULL, 6039 stats->namebuf, "FC Last Error"); 6040 if (hw->mac.type != ixgbe_mac_82598EB) { 6041 evcnt_attach_dynamic(&stats->fcoerpdc, EVCNT_TYPE_MISC, NULL, 6042 stats->namebuf, "FCoE Packets Dropped"); 6043 evcnt_attach_dynamic(&stats->fcoeprc, EVCNT_TYPE_MISC, NULL, 6044 stats->namebuf, "FCoE Packets Received"); 6045 evcnt_attach_dynamic(&stats->fcoeptc, EVCNT_TYPE_MISC, NULL, 6046 stats->namebuf, "FCoE Packets Transmitted"); 6047 evcnt_attach_dynamic(&stats->fcoedwrc, EVCNT_TYPE_MISC, NULL, 6048 stats->namebuf, "FCoE DWords Received"); 6049 evcnt_attach_dynamic(&stats->fcoedwtc, EVCNT_TYPE_MISC, NULL, 6050 stats->namebuf, "FCoE DWords Transmitted"); 6051 } 6052 } 6053 6054 /* 6055 ** Set flow control using sysctl: 6056 ** Flow control values: 6057 ** 0 - off 6058 ** 1 - rx pause 6059 ** 2 - tx pause 6060 ** 3 - full 6061 */ 6062 static int 6063 ixgbe_set_flowcntl(SYSCTLFN_ARGS) 6064 { 6065 struct sysctlnode node; 6066 int error, last; 6067 struct adapter *adapter; 6068 6069 node = *rnode; 6070 adapter = (struct adapter *)node.sysctl_data; 6071 node.sysctl_data = &adapter->fc; 6072 last = adapter->fc; 6073 error = sysctl_lookup(SYSCTLFN_CALL(&node)); 6074 if (error != 0 || newp == NULL) 6075 return error; 6076 6077 /* Don't bother if it's not changed */ 6078 if (adapter->fc == last) 6079 return (0); 6080 6081 switch (adapter->fc) { 6082 case ixgbe_fc_rx_pause: 6083 case ixgbe_fc_tx_pause: 6084 case ixgbe_fc_full: 6085 adapter->hw.fc.requested_mode = adapter->fc; 6086 break; 6087 case ixgbe_fc_none: 6088 default: 6089 adapter->hw.fc.requested_mode = ixgbe_fc_none; 6090 } 6091 /* Don't autoneg if forcing a value */ 6092 adapter->hw.fc.disable_fc_autoneg = TRUE; 6093 ixgbe_fc_enable(&adapter->hw); 6094 return 0; 6095 } 6096 6097 static void 6098 ixgbe_add_rx_process_limit(struct adapter *adapter, const char *name, 6099 const char *description, int *limit, int value) 6100 { 6101 const struct sysctlnode *rnode, *cnode; 6102 struct sysctllog **log = &adapter->sysctllog; 6103 6104 *limit = value; 6105 6106 if ((rnode = ixgbe_sysctl_instance(adapter)) == NULL) 6107 aprint_error_dev(adapter->dev, 6108 "could not create sysctl root\n"); 6109 else if (sysctl_createv(log, 0, &rnode, &cnode, 6110 CTLFLAG_READWRITE, 6111 CTLTYPE_INT, 6112 name, SYSCTL_DESCR(description), 6113 NULL, 0, limit, 0, 6114 CTL_CREATE, CTL_EOL) != 0) { 6115 aprint_error_dev(adapter->dev, "%s: could not create sysctl", 6116 __func__); 6117 } 6118 } 6119 6120 /* 6121 ** Control link advertise speed: 6122 ** 1 - advertise only 1G 6123 ** 2 - advertise 100Mb 6124 ** 3 - advertise normal 6125 */ 6126 static int 6127 ixgbe_set_advertise(SYSCTLFN_ARGS) 6128 { 6129 struct sysctlnode node; 6130 int t, error = 0; 6131 struct adapter *adapter; 6132 device_t dev; 6133 struct ixgbe_hw *hw; 6134 ixgbe_link_speed speed, last; 6135 6136 node = *rnode; 6137 adapter = (struct adapter *)node.sysctl_data; 6138 dev = adapter->dev; 6139 hw = &adapter->hw; 6140 last = adapter->advertise; 6141 t = adapter->advertise; 6142 node.sysctl_data = &t; 6143 error = sysctl_lookup(SYSCTLFN_CALL(&node)); 6144 if (error != 0 || newp == NULL) 6145 return error; 6146 6147 if (adapter->advertise == last) /* no change */ 6148 return (0); 6149 6150 if (t == -1) 6151 return 0; 6152 6153 adapter->advertise = t; 6154 6155 if (!((hw->phy.media_type == ixgbe_media_type_copper) || 6156 (hw->phy.multispeed_fiber))) 6157 return 0; 6158 6159 if ((adapter->advertise == 2) && (hw->mac.type != ixgbe_mac_X540)) { 6160 device_printf(dev, "Set Advertise: 100Mb on X540 only\n"); 6161 return (error); 6162 } 6163 6164 if (adapter->advertise == 1) 6165 speed = IXGBE_LINK_SPEED_1GB_FULL; 6166 else if (adapter->advertise == 2) 6167 speed = IXGBE_LINK_SPEED_100_FULL; 6168 else if (adapter->advertise == 3) 6169 speed = IXGBE_LINK_SPEED_1GB_FULL | 6170 IXGBE_LINK_SPEED_10GB_FULL; 6171 else /* bogus value */ 6172 return (error); 6173 6174 hw->mac.autotry_restart = TRUE; 6175 hw->mac.ops.setup_link(hw, speed, TRUE, TRUE); 6176 6177 return 0; 6178 } 6179 6180 /* 6181 ** Thermal Shutdown Trigger 6182 ** - cause a Thermal Overtemp IRQ 6183 */ 6184 static int 6185 ixgbe_set_thermal_test(SYSCTLFN_ARGS) 6186 { 6187 struct sysctlnode node; 6188 int error, fire = 0; 6189 struct adapter *adapter; 6190 struct ixgbe_hw *hw; 6191 6192 node = *rnode; 6193 adapter = (struct adapter *)node.sysctl_data; 6194 hw = &adapter->hw; 6195 6196 if (hw->mac.type != ixgbe_mac_X540) 6197 return (0); 6198 6199 node.sysctl_data = &fire; 6200 error = sysctl_lookup(SYSCTLFN_CALL(&node)); 6201 if ((error) || (newp == NULL)) 6202 return (error); 6203 6204 if (fire) { 6205 u32 reg = IXGBE_READ_REG(hw, IXGBE_EICS); 6206 reg |= IXGBE_EICR_TS; 6207 IXGBE_WRITE_REG(hw, IXGBE_EICS, reg); 6208 } 6209 6210 return (0); 6211 } 6212
Indexes created Tue Sep 30 11:09:46 GMT 2025