ixgbe.c revision 1.22
1 /****************************************************************************** 2 3 Copyright (c) 2001-2013, Intel Corporation 4 All rights reserved. 5 6 Redistribution and use in source and binary forms, with or without 7 modification, are permitted provided that the following conditions are met: 8 9 1. Redistributions of source code must retain the above copyright notice, 10 this list of conditions and the following disclaimer. 11 12 2. Redistributions in binary form must reproduce the above copyright 13 notice, this list of conditions and the following disclaimer in the 14 documentation and/or other materials provided with the distribution. 15 16 3. Neither the name of the Intel Corporation nor the names of its 17 contributors may be used to endorse or promote products derived from 18 this software without specific prior written permission. 19 20 THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 21 AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 22 IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 23 ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE 24 LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 25 CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 26 SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 27 INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 28 CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 29 ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 30 POSSIBILITY OF SUCH DAMAGE. 31 32 ******************************************************************************/ 33 /* 34 * Copyright (c) 2011 The NetBSD Foundation, Inc. 35 * All rights reserved. 36 * 37 * This code is derived from software contributed to The NetBSD Foundation 38 * by Coyote Point Systems, Inc. 39 * 40 * Redistribution and use in source and binary forms, with or without 41 * modification, are permitted provided that the following conditions 42 * are met: 43 * 1. Redistributions of source code must retain the above copyright 44 * notice, this list of conditions and the following disclaimer. 45 * 2. Redistributions in binary form must reproduce the above copyright 46 * notice, this list of conditions and the following disclaimer in the 47 * documentation and/or other materials provided with the distribution. 48 * 49 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS 50 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED 51 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 52 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS 53 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 54 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 55 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 56 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 57 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 58 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 59 * POSSIBILITY OF SUCH DAMAGE. 60 */ 61 /*$FreeBSD: src/sys/dev/ixgbe/ixgbe.c,v 1.51 2011/04/25 23:34:21 jfv Exp $*/ 62 /*$NetBSD: ixgbe.c,v 1.22 2015/03/19 14:22:23 msaitoh Exp $*/ 63 64 #include "opt_inet.h" 65 #include "opt_inet6.h" 66 67 #include "ixgbe.h" 68 69 /********************************************************************* 70 * Set this to one to display debug statistics 71 *********************************************************************/ 72 int ixgbe_display_debug_stats = 0; 73 74 /********************************************************************* 75 * Driver version 76 *********************************************************************/ 77 char ixgbe_driver_version[] = "2.3.11"; 78 79 /********************************************************************* 80 * PCI Device ID Table 81 * 82 * Used by probe to select devices to load on 83 * Last field stores an index into ixgbe_strings 84 * Last entry must be all 0s 85 * 86 * { Vendor ID, Device ID, SubVendor ID, SubDevice ID, String Index } 87 *********************************************************************/ 88 89 static ixgbe_vendor_info_t ixgbe_vendor_info_array[] = 90 { 91 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598AF_DUAL_PORT, 0, 0, 0}, 92 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598AF_SINGLE_PORT, 0, 0, 0}, 93 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598EB_CX4, 0, 0, 0}, 94 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598AT, 0, 0, 0}, 95 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598AT2, 0, 0, 0}, 96 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598, 0, 0, 0}, 97 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598_DA_DUAL_PORT, 0, 0, 0}, 98 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598_CX4_DUAL_PORT, 0, 0, 0}, 99 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598EB_XF_LR, 0, 0, 0}, 100 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598_SR_DUAL_PORT_EM, 0, 0, 0}, 101 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598EB_SFP_LOM, 0, 0, 0}, 102 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_KX4, 0, 0, 0}, 103 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_KX4_MEZZ, 0, 0, 0}, 104 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_SFP, 0, 0, 0}, 105 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_XAUI_LOM, 0, 0, 0}, 106 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_CX4, 0, 0, 0}, 107 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_T3_LOM, 0, 0, 0}, 108 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_COMBO_BACKPLANE, 0, 0, 0}, 109 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_BACKPLANE_FCOE, 0, 0, 0}, 110 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_SFP_SF2, 0, 0, 0}, 111 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_SFP_FCOE, 0, 0, 0}, 112 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599EN_SFP, 0, 0, 0}, 113 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_SFP_SF_QP, 0, 0, 0}, 114 /* required last entry */ 115 {0, 0, 0, 0, 0} 116 }; 117 118 /********************************************************************* 119 * Table of branding strings 120 *********************************************************************/ 121 122 static const char *ixgbe_strings[] = { 123 "Intel(R) PRO/10GbE PCI-Express Network Driver" 124 }; 125 126 /********************************************************************* 127 * Function prototypes 128 *********************************************************************/ 129 static int ixgbe_probe(device_t, cfdata_t, void *); 130 static void ixgbe_attach(device_t, device_t, void *); 131 static int ixgbe_detach(device_t, int); 132 #if 0 133 static int ixgbe_shutdown(device_t); 134 #endif 135 static void ixgbe_start(struct ifnet *); 136 static void ixgbe_start_locked(struct tx_ring *, struct ifnet *); 137 #if __FreeBSD_version >= 800000 138 static int ixgbe_mq_start(struct ifnet *, struct mbuf *); 139 static int ixgbe_mq_start_locked(struct ifnet *, 140 struct tx_ring *, struct mbuf *); 141 static void ixgbe_qflush(struct ifnet *); 142 #endif 143 static int ixgbe_ioctl(struct ifnet *, u_long, void *); 144 static void ixgbe_ifstop(struct ifnet *, int); 145 static int ixgbe_init(struct ifnet *); 146 static void ixgbe_init_locked(struct adapter *); 147 static void ixgbe_stop(void *); 148 static void ixgbe_media_status(struct ifnet *, struct ifmediareq *); 149 static int ixgbe_media_change(struct ifnet *); 150 static void ixgbe_identify_hardware(struct adapter *); 151 static int ixgbe_allocate_pci_resources(struct adapter *, 152 const struct pci_attach_args *); 153 static int ixgbe_allocate_msix(struct adapter *, 154 const struct pci_attach_args *); 155 static int ixgbe_allocate_legacy(struct adapter *, 156 const struct pci_attach_args *); 157 static int ixgbe_allocate_queues(struct adapter *); 158 static int ixgbe_setup_msix(struct adapter *); 159 static void ixgbe_free_pci_resources(struct adapter *); 160 static void ixgbe_local_timer(void *); 161 static int ixgbe_setup_interface(device_t, struct adapter *); 162 static void ixgbe_config_link(struct adapter *); 163 164 static int ixgbe_allocate_transmit_buffers(struct tx_ring *); 165 static int ixgbe_setup_transmit_structures(struct adapter *); 166 static void ixgbe_setup_transmit_ring(struct tx_ring *); 167 static void ixgbe_initialize_transmit_units(struct adapter *); 168 static void ixgbe_free_transmit_structures(struct adapter *); 169 static void ixgbe_free_transmit_buffers(struct tx_ring *); 170 171 static int ixgbe_allocate_receive_buffers(struct rx_ring *); 172 static int ixgbe_setup_receive_structures(struct adapter *); 173 static int ixgbe_setup_receive_ring(struct rx_ring *); 174 static void ixgbe_initialize_receive_units(struct adapter *); 175 static void ixgbe_free_receive_structures(struct adapter *); 176 static void ixgbe_free_receive_buffers(struct rx_ring *); 177 static void ixgbe_setup_hw_rsc(struct rx_ring *); 178 179 static void ixgbe_enable_intr(struct adapter *); 180 static void ixgbe_disable_intr(struct adapter *); 181 static void ixgbe_update_stats_counters(struct adapter *); 182 static bool ixgbe_txeof(struct tx_ring *); 183 static bool ixgbe_rxeof(struct ix_queue *, int); 184 static void ixgbe_rx_checksum(u32, struct mbuf *, u32, 185 struct ixgbe_hw_stats *); 186 static void ixgbe_set_promisc(struct adapter *); 187 static void ixgbe_set_multi(struct adapter *); 188 static void ixgbe_update_link_status(struct adapter *); 189 static void ixgbe_refresh_mbufs(struct rx_ring *, int); 190 static int ixgbe_xmit(struct tx_ring *, struct mbuf *); 191 static int ixgbe_set_flowcntl(SYSCTLFN_PROTO); 192 static int ixgbe_set_advertise(SYSCTLFN_PROTO); 193 static int ixgbe_dma_malloc(struct adapter *, bus_size_t, 194 struct ixgbe_dma_alloc *, int); 195 static void ixgbe_dma_free(struct adapter *, struct ixgbe_dma_alloc *); 196 static void ixgbe_add_rx_process_limit(struct adapter *, const char *, 197 const char *, int *, int); 198 static u32 ixgbe_tx_ctx_setup(struct tx_ring *, struct mbuf *); 199 static bool ixgbe_tso_setup(struct tx_ring *, struct mbuf *, u32 *); 200 static void ixgbe_set_ivar(struct adapter *, u8, u8, s8); 201 static void ixgbe_configure_ivars(struct adapter *); 202 static u8 * ixgbe_mc_array_itr(struct ixgbe_hw *, u8 **, u32 *); 203 204 static void ixgbe_setup_vlan_hw_support(struct adapter *); 205 #if 0 206 static void ixgbe_register_vlan(void *, struct ifnet *, u16); 207 static void ixgbe_unregister_vlan(void *, struct ifnet *, u16); 208 #endif 209 210 static void ixgbe_add_hw_stats(struct adapter *adapter); 211 212 static __inline void ixgbe_rx_discard(struct rx_ring *, int); 213 static __inline void ixgbe_rx_input(struct rx_ring *, struct ifnet *, 214 struct mbuf *, u32); 215 216 /* Support for pluggable optic modules */ 217 static bool ixgbe_sfp_probe(struct adapter *); 218 static void ixgbe_setup_optics(struct adapter *); 219 220 /* Legacy (single vector interrupt handler */ 221 static int ixgbe_legacy_irq(void *); 222 223 #if defined(NETBSD_MSI_OR_MSIX) 224 /* The MSI/X Interrupt handlers */ 225 static void ixgbe_msix_que(void *); 226 static void ixgbe_msix_link(void *); 227 #endif 228 229 /* Software interrupts for deferred work */ 230 static void ixgbe_handle_que(void *); 231 static void ixgbe_handle_link(void *); 232 static void ixgbe_handle_msf(void *); 233 static void ixgbe_handle_mod(void *); 234 235 const struct sysctlnode *ixgbe_sysctl_instance(struct adapter *); 236 static ixgbe_vendor_info_t *ixgbe_lookup(const struct pci_attach_args *); 237 238 #ifdef IXGBE_FDIR 239 static void ixgbe_atr(struct tx_ring *, struct mbuf *); 240 static void ixgbe_reinit_fdir(void *, int); 241 #endif 242 243 /********************************************************************* 244 * FreeBSD Device Interface Entry Points 245 *********************************************************************/ 246 247 CFATTACH_DECL3_NEW(ixg, sizeof(struct adapter), 248 ixgbe_probe, ixgbe_attach, ixgbe_detach, NULL, NULL, NULL, 249 DVF_DETACH_SHUTDOWN); 250 251 #if 0 252 devclass_t ixgbe_devclass; 253 DRIVER_MODULE(ixgbe, pci, ixgbe_driver, ixgbe_devclass, 0, 0); 254 255 MODULE_DEPEND(ixgbe, pci, 1, 1, 1); 256 MODULE_DEPEND(ixgbe, ether, 1, 1, 1); 257 #endif 258 259 /* 260 ** TUNEABLE PARAMETERS: 261 */ 262 263 /* 264 ** AIM: Adaptive Interrupt Moderation 265 ** which means that the interrupt rate 266 ** is varied over time based on the 267 ** traffic for that interrupt vector 268 */ 269 static int ixgbe_enable_aim = TRUE; 270 #define TUNABLE_INT(__x, __y) 271 TUNABLE_INT("hw.ixgbe.enable_aim", &ixgbe_enable_aim); 272 273 static int ixgbe_max_interrupt_rate = (4000000 / IXGBE_LOW_LATENCY); 274 TUNABLE_INT("hw.ixgbe.max_interrupt_rate", &ixgbe_max_interrupt_rate); 275 276 /* How many packets rxeof tries to clean at a time */ 277 static int ixgbe_rx_process_limit = 256; 278 TUNABLE_INT("hw.ixgbe.rx_process_limit", &ixgbe_rx_process_limit); 279 280 /* Flow control setting, default to full */ 281 static int ixgbe_flow_control = ixgbe_fc_full; 282 TUNABLE_INT("hw.ixgbe.flow_control", &ixgbe_flow_control); 283 284 /* 285 ** Smart speed setting, default to on 286 ** this only works as a compile option 287 ** right now as its during attach, set 288 ** this to 'ixgbe_smart_speed_off' to 289 ** disable. 290 */ 291 static int ixgbe_smart_speed = ixgbe_smart_speed_on; 292 293 /* 294 * MSIX should be the default for best performance, 295 * but this allows it to be forced off for testing. 296 */ 297 static int ixgbe_enable_msix = 1; 298 TUNABLE_INT("hw.ixgbe.enable_msix", &ixgbe_enable_msix); 299 300 /* 301 * Header split: this causes the hardware to DMA 302 * the header into a separate mbuf from the payload, 303 * it can be a performance win in some workloads, but 304 * in others it actually hurts, its off by default. 305 */ 306 static int ixgbe_header_split = FALSE; 307 TUNABLE_INT("hw.ixgbe.hdr_split", &ixgbe_header_split); 308 309 #if defined(NETBSD_MSI_OR_MSIX) 310 /* 311 * Number of Queues, can be set to 0, 312 * it then autoconfigures based on the 313 * number of cpus with a max of 8. This 314 * can be overriden manually here. 315 */ 316 static int ixgbe_num_queues = 0; 317 TUNABLE_INT("hw.ixgbe.num_queues", &ixgbe_num_queues); 318 #endif 319 320 /* 321 ** Number of TX descriptors per ring, 322 ** setting higher than RX as this seems 323 ** the better performing choice. 324 */ 325 static int ixgbe_txd = PERFORM_TXD; 326 TUNABLE_INT("hw.ixgbe.txd", &ixgbe_txd); 327 328 /* Number of RX descriptors per ring */ 329 static int ixgbe_rxd = PERFORM_RXD; 330 TUNABLE_INT("hw.ixgbe.rxd", &ixgbe_rxd); 331 332 /* Keep running tab on them for sanity check */ 333 static int ixgbe_total_ports; 334 335 #ifdef IXGBE_FDIR 336 /* 337 ** For Flow Director: this is the 338 ** number of TX packets we sample 339 ** for the filter pool, this means 340 ** every 20th packet will be probed. 341 ** 342 ** This feature can be disabled by 343 ** setting this to 0. 344 */ 345 static int atr_sample_rate = 20; 346 /* 347 ** Flow Director actually 'steals' 348 ** part of the packet buffer as its 349 ** filter pool, this variable controls 350 ** how much it uses: 351 ** 0 = 64K, 1 = 128K, 2 = 256K 352 */ 353 static int fdir_pballoc = 1; 354 #endif 355 356 #ifdef DEV_NETMAP 357 /* 358 * The #ifdef DEV_NETMAP / #endif blocks in this file are meant to 359 * be a reference on how to implement netmap support in a driver. 360 * Additional comments are in ixgbe_netmap.h . 361 * 362 * <dev/netma/ixgbe_netmap.h> contains functions for netmap support 363 * that extend the standard driver. 364 */ 365 #include <dev/netmap/ixgbe_netmap.h> 366 #endif /* DEV_NETMAP */ 367 368 /********************************************************************* 369 * Device identification routine 370 * 371 * ixgbe_probe determines if the driver should be loaded on 372 * adapter based on PCI vendor/device id of the adapter. 373 * 374 * return 1 on success, 0 on failure 375 *********************************************************************/ 376 377 static int 378 ixgbe_probe(device_t dev, cfdata_t cf, void *aux) 379 { 380 const struct pci_attach_args *pa = aux; 381 382 return (ixgbe_lookup(pa) != NULL) ? 1 : 0; 383 } 384 385 static ixgbe_vendor_info_t * 386 ixgbe_lookup(const struct pci_attach_args *pa) 387 { 388 pcireg_t subid; 389 ixgbe_vendor_info_t *ent; 390 391 INIT_DEBUGOUT("ixgbe_probe: begin"); 392 393 if (PCI_VENDOR(pa->pa_id) != IXGBE_INTEL_VENDOR_ID) 394 return NULL; 395 396 subid = pci_conf_read(pa->pa_pc, pa->pa_tag, PCI_SUBSYS_ID_REG); 397 398 for (ent = ixgbe_vendor_info_array; ent->vendor_id != 0; ent++) { 399 if (PCI_VENDOR(pa->pa_id) == ent->vendor_id && 400 PCI_PRODUCT(pa->pa_id) == ent->device_id && 401 402 (PCI_SUBSYS_VENDOR(subid) == ent->subvendor_id || 403 ent->subvendor_id == 0) && 404 405 (PCI_SUBSYS_ID(subid) == ent->subdevice_id || 406 ent->subdevice_id == 0)) { 407 ++ixgbe_total_ports; 408 return ent; 409 } 410 } 411 return NULL; 412 } 413 414 415 static void 416 ixgbe_sysctl_attach(struct adapter *adapter) 417 { 418 struct sysctllog **log; 419 const struct sysctlnode *rnode, *cnode; 420 device_t dev; 421 422 dev = adapter->dev; 423 log = &adapter->sysctllog; 424 425 if ((rnode = ixgbe_sysctl_instance(adapter)) == NULL) { 426 aprint_error_dev(dev, "could not create sysctl root\n"); 427 return; 428 } 429 430 if (sysctl_createv(log, 0, &rnode, &cnode, 431 CTLFLAG_READONLY, CTLTYPE_INT, 432 "num_rx_desc", SYSCTL_DESCR("Number of rx descriptors"), 433 NULL, 0, &adapter->num_rx_desc, 0, CTL_CREATE, CTL_EOL) != 0) 434 aprint_error_dev(dev, "could not create sysctl\n"); 435 436 if (sysctl_createv(log, 0, &rnode, &cnode, 437 CTLFLAG_READONLY, CTLTYPE_INT, 438 "num_queues", SYSCTL_DESCR("Number of queues"), 439 NULL, 0, &adapter->num_queues, 0, CTL_CREATE, CTL_EOL) != 0) 440 aprint_error_dev(dev, "could not create sysctl\n"); 441 442 if (sysctl_createv(log, 0, &rnode, &cnode, 443 CTLFLAG_READWRITE, CTLTYPE_INT, 444 "flow_control", SYSCTL_DESCR("Flow Control"), 445 ixgbe_set_flowcntl, 0, (void *)adapter, 0, CTL_CREATE, CTL_EOL) != 0) 446 aprint_error_dev(dev, "could not create sysctl\n"); 447 448 if (sysctl_createv(log, 0, &rnode, &cnode, 449 CTLFLAG_READWRITE, CTLTYPE_INT, 450 "advertise_gig", SYSCTL_DESCR("1G Link"), 451 ixgbe_set_advertise, 0, (void *)adapter, 0, CTL_CREATE, CTL_EOL) != 0) 452 aprint_error_dev(dev, "could not create sysctl\n"); 453 454 /* XXX This is an *instance* sysctl controlling a *global* variable. 455 * XXX It's that way in the FreeBSD driver that this derives from. 456 */ 457 if (sysctl_createv(log, 0, &rnode, &cnode, 458 CTLFLAG_READWRITE, CTLTYPE_INT, 459 "enable_aim", SYSCTL_DESCR("Interrupt Moderation"), 460 NULL, 0, &ixgbe_enable_aim, 0, CTL_CREATE, CTL_EOL) != 0) 461 aprint_error_dev(dev, "could not create sysctl\n"); 462 } 463 464 /********************************************************************* 465 * Device initialization routine 466 * 467 * The attach entry point is called when the driver is being loaded. 468 * This routine identifies the type of hardware, allocates all resources 469 * and initializes the hardware. 470 * 471 * return 0 on success, positive on failure 472 *********************************************************************/ 473 474 static void 475 ixgbe_attach(device_t parent, device_t dev, void *aux) 476 { 477 struct adapter *adapter; 478 struct ixgbe_hw *hw; 479 int error = 0; 480 u16 csum; 481 u32 ctrl_ext; 482 ixgbe_vendor_info_t *ent; 483 const struct pci_attach_args *pa = aux; 484 485 INIT_DEBUGOUT("ixgbe_attach: begin"); 486 487 /* Allocate, clear, and link in our adapter structure */ 488 adapter = device_private(dev); 489 adapter->dev = adapter->osdep.dev = dev; 490 hw = &adapter->hw; 491 adapter->osdep.pc = pa->pa_pc; 492 adapter->osdep.tag = pa->pa_tag; 493 adapter->osdep.dmat = pa->pa_dmat; 494 495 ent = ixgbe_lookup(pa); 496 497 KASSERT(ent != NULL); 498 499 aprint_normal(": %s, Version - %s\n", 500 ixgbe_strings[ent->index], ixgbe_driver_version); 501 502 /* Core Lock Init*/ 503 IXGBE_CORE_LOCK_INIT(adapter, device_xname(dev)); 504 505 /* SYSCTL APIs */ 506 507 ixgbe_sysctl_attach(adapter); 508 509 /* Set up the timer callout */ 510 callout_init(&adapter->timer, 0); 511 512 /* Determine hardware revision */ 513 ixgbe_identify_hardware(adapter); 514 515 /* Do base PCI setup - map BAR0 */ 516 if (ixgbe_allocate_pci_resources(adapter, pa)) { 517 aprint_error_dev(dev, "Allocation of PCI resources failed\n"); 518 error = ENXIO; 519 goto err_out; 520 } 521 522 /* Do descriptor calc and sanity checks */ 523 if (((ixgbe_txd * sizeof(union ixgbe_adv_tx_desc)) % DBA_ALIGN) != 0 || 524 ixgbe_txd < MIN_TXD || ixgbe_txd > MAX_TXD) { 525 aprint_error_dev(dev, "TXD config issue, using default!\n"); 526 adapter->num_tx_desc = DEFAULT_TXD; 527 } else 528 adapter->num_tx_desc = ixgbe_txd; 529 530 /* 531 ** With many RX rings it is easy to exceed the 532 ** system mbuf allocation. Tuning nmbclusters 533 ** can alleviate this. 534 */ 535 if (nmbclusters > 0 ) { 536 int s; 537 s = (ixgbe_rxd * adapter->num_queues) * ixgbe_total_ports; 538 if (s > nmbclusters) { 539 aprint_error_dev(dev, "RX Descriptors exceed " 540 "system mbuf max, using default instead!\n"); 541 ixgbe_rxd = DEFAULT_RXD; 542 } 543 } 544 545 if (((ixgbe_rxd * sizeof(union ixgbe_adv_rx_desc)) % DBA_ALIGN) != 0 || 546 ixgbe_rxd < MIN_TXD || ixgbe_rxd > MAX_TXD) { 547 aprint_error_dev(dev, "RXD config issue, using default!\n"); 548 adapter->num_rx_desc = DEFAULT_RXD; 549 } else 550 adapter->num_rx_desc = ixgbe_rxd; 551 552 /* Allocate our TX/RX Queues */ 553 if (ixgbe_allocate_queues(adapter)) { 554 error = ENOMEM; 555 goto err_out; 556 } 557 558 /* Allocate multicast array memory. */ 559 adapter->mta = malloc(sizeof(u8) * IXGBE_ETH_LENGTH_OF_ADDRESS * 560 MAX_NUM_MULTICAST_ADDRESSES, M_DEVBUF, M_NOWAIT); 561 if (adapter->mta == NULL) { 562 aprint_error_dev(dev, "Cannot allocate multicast setup array\n"); 563 error = ENOMEM; 564 goto err_late; 565 } 566 567 /* Initialize the shared code */ 568 error = ixgbe_init_shared_code(hw); 569 if (error == IXGBE_ERR_SFP_NOT_PRESENT) { 570 /* 571 ** No optics in this port, set up 572 ** so the timer routine will probe 573 ** for later insertion. 574 */ 575 adapter->sfp_probe = TRUE; 576 error = 0; 577 } else if (error == IXGBE_ERR_SFP_NOT_SUPPORTED) { 578 aprint_error_dev(dev,"Unsupported SFP+ module detected!\n"); 579 error = EIO; 580 goto err_late; 581 } else if (error) { 582 aprint_error_dev(dev,"Unable to initialize the shared code\n"); 583 error = EIO; 584 goto err_late; 585 } 586 587 /* Make sure we have a good EEPROM before we read from it */ 588 if (ixgbe_validate_eeprom_checksum(&adapter->hw, &csum) < 0) { 589 aprint_error_dev(dev,"The EEPROM Checksum Is Not Valid\n"); 590 error = EIO; 591 goto err_late; 592 } 593 594 /* Get Hardware Flow Control setting */ 595 hw->fc.requested_mode = ixgbe_fc_full; 596 hw->fc.pause_time = IXGBE_FC_PAUSE; 597 hw->fc.low_water = IXGBE_FC_LO; 598 hw->fc.high_water = IXGBE_FC_HI; 599 hw->fc.send_xon = TRUE; 600 601 error = ixgbe_init_hw(hw); 602 if (error == IXGBE_ERR_EEPROM_VERSION) { 603 aprint_error_dev(dev, "This device is a pre-production adapter/" 604 "LOM. Please be aware there may be issues associated " 605 "with your hardware.\n If you are experiencing problems " 606 "please contact your Intel or hardware representative " 607 "who provided you with this hardware.\n"); 608 } else if (error == IXGBE_ERR_SFP_NOT_SUPPORTED) 609 aprint_error_dev(dev,"Unsupported SFP+ Module\n"); 610 611 if (error) { 612 error = EIO; 613 aprint_error_dev(dev,"Hardware Initialization Failure\n"); 614 goto err_late; 615 } 616 617 /* Detect and set physical type */ 618 ixgbe_setup_optics(adapter); 619 620 if ((adapter->msix > 1) && (ixgbe_enable_msix)) 621 error = ixgbe_allocate_msix(adapter, pa); 622 else 623 error = ixgbe_allocate_legacy(adapter, pa); 624 if (error) 625 goto err_late; 626 627 /* Setup OS specific network interface */ 628 if (ixgbe_setup_interface(dev, adapter) != 0) 629 goto err_late; 630 631 /* Sysctl for limiting the amount of work done in software interrupts */ 632 ixgbe_add_rx_process_limit(adapter, "rx_processing_limit", 633 "max number of rx packets to process", &adapter->rx_process_limit, 634 ixgbe_rx_process_limit); 635 636 /* Initialize statistics */ 637 ixgbe_update_stats_counters(adapter); 638 639 /* Print PCIE bus type/speed/width info */ 640 ixgbe_get_bus_info(hw); 641 aprint_normal_dev(dev,"PCI Express Bus: Speed %s %s\n", 642 ((hw->bus.speed == ixgbe_bus_speed_5000) ? "5.0Gb/s": 643 (hw->bus.speed == ixgbe_bus_speed_2500) ? "2.5Gb/s":"Unknown"), 644 (hw->bus.width == ixgbe_bus_width_pcie_x8) ? "Width x8" : 645 (hw->bus.width == ixgbe_bus_width_pcie_x4) ? "Width x4" : 646 (hw->bus.width == ixgbe_bus_width_pcie_x1) ? "Width x1" : 647 ("Unknown")); 648 649 if ((hw->bus.width <= ixgbe_bus_width_pcie_x4) && 650 (hw->bus.speed == ixgbe_bus_speed_2500)) { 651 aprint_error_dev(dev, "PCI-Express bandwidth available" 652 " for this card\n is not sufficient for" 653 " optimal performance.\n"); 654 aprint_error_dev(dev, "For optimal performance a x8 " 655 "PCIE, or x4 PCIE 2 slot is required.\n"); 656 } 657 658 /* let hardware know driver is loaded */ 659 ctrl_ext = IXGBE_READ_REG(hw, IXGBE_CTRL_EXT); 660 ctrl_ext |= IXGBE_CTRL_EXT_DRV_LOAD; 661 IXGBE_WRITE_REG(hw, IXGBE_CTRL_EXT, ctrl_ext); 662 663 ixgbe_add_hw_stats(adapter); 664 665 #ifdef DEV_NETMAP 666 ixgbe_netmap_attach(adapter); 667 #endif /* DEV_NETMAP */ 668 INIT_DEBUGOUT("ixgbe_attach: end"); 669 return; 670 err_late: 671 ixgbe_free_transmit_structures(adapter); 672 ixgbe_free_receive_structures(adapter); 673 err_out: 674 if (adapter->ifp != NULL) 675 if_free(adapter->ifp); 676 ixgbe_free_pci_resources(adapter); 677 if (adapter->mta != NULL) 678 free(adapter->mta, M_DEVBUF); 679 return; 680 681 } 682 683 /********************************************************************* 684 * Device removal routine 685 * 686 * The detach entry point is called when the driver is being removed. 687 * This routine stops the adapter and deallocates all the resources 688 * that were allocated for driver operation. 689 * 690 * return 0 on success, positive on failure 691 *********************************************************************/ 692 693 static int 694 ixgbe_detach(device_t dev, int flags) 695 { 696 struct adapter *adapter = device_private(dev); 697 struct tx_ring *txr = adapter->tx_rings; 698 struct rx_ring *rxr = adapter->rx_rings; 699 struct ixgbe_hw_stats *stats = &adapter->stats; 700 struct ix_queue *que = adapter->queues; 701 u32 ctrl_ext; 702 703 INIT_DEBUGOUT("ixgbe_detach: begin"); 704 705 /* Make sure VLANs are not using driver */ 706 if (!VLAN_ATTACHED(&adapter->osdep.ec)) 707 ; /* nothing to do: no VLANs */ 708 else if ((flags & (DETACH_SHUTDOWN|DETACH_FORCE)) != 0) 709 vlan_ifdetach(adapter->ifp); 710 else { 711 aprint_error_dev(dev, "VLANs in use\n"); 712 return EBUSY; 713 } 714 715 IXGBE_CORE_LOCK(adapter); 716 ixgbe_stop(adapter); 717 IXGBE_CORE_UNLOCK(adapter); 718 719 for (int i = 0; i < adapter->num_queues; i++, que++) { 720 softint_disestablish(que->que_si); 721 } 722 723 /* Drain the Link queue */ 724 softint_disestablish(adapter->link_si); 725 softint_disestablish(adapter->mod_si); 726 softint_disestablish(adapter->msf_si); 727 #ifdef IXGBE_FDIR 728 softint_disestablish(adapter->fdir_si); 729 #endif 730 731 /* let hardware know driver is unloading */ 732 ctrl_ext = IXGBE_READ_REG(&adapter->hw, IXGBE_CTRL_EXT); 733 ctrl_ext &= ~IXGBE_CTRL_EXT_DRV_LOAD; 734 IXGBE_WRITE_REG(&adapter->hw, IXGBE_CTRL_EXT, ctrl_ext); 735 736 ether_ifdetach(adapter->ifp); 737 callout_halt(&adapter->timer, NULL); 738 #ifdef DEV_NETMAP 739 netmap_detach(adapter->ifp); 740 #endif /* DEV_NETMAP */ 741 ixgbe_free_pci_resources(adapter); 742 #if 0 /* XXX the NetBSD port is probably missing something here */ 743 bus_generic_detach(dev); 744 #endif 745 if_detach(adapter->ifp); 746 747 sysctl_teardown(&adapter->sysctllog); 748 evcnt_detach(&adapter->handleq); 749 evcnt_detach(&adapter->req); 750 evcnt_detach(&adapter->morerx); 751 evcnt_detach(&adapter->moretx); 752 evcnt_detach(&adapter->txloops); 753 evcnt_detach(&adapter->efbig_tx_dma_setup); 754 evcnt_detach(&adapter->m_defrag_failed); 755 evcnt_detach(&adapter->efbig2_tx_dma_setup); 756 evcnt_detach(&adapter->einval_tx_dma_setup); 757 evcnt_detach(&adapter->other_tx_dma_setup); 758 evcnt_detach(&adapter->eagain_tx_dma_setup); 759 evcnt_detach(&adapter->enomem_tx_dma_setup); 760 evcnt_detach(&adapter->watchdog_events); 761 evcnt_detach(&adapter->tso_err); 762 evcnt_detach(&adapter->tso_tx); 763 evcnt_detach(&adapter->link_irq); 764 for (int i = 0; i < adapter->num_queues; i++, rxr++, txr++) { 765 evcnt_detach(&txr->no_desc_avail); 766 evcnt_detach(&txr->total_packets); 767 768 if (i < __arraycount(adapter->stats.mpc)) { 769 evcnt_detach(&adapter->stats.mpc[i]); 770 } 771 if (i < __arraycount(adapter->stats.pxontxc)) { 772 evcnt_detach(&adapter->stats.pxontxc[i]); 773 evcnt_detach(&adapter->stats.pxonrxc[i]); 774 evcnt_detach(&adapter->stats.pxofftxc[i]); 775 evcnt_detach(&adapter->stats.pxoffrxc[i]); 776 evcnt_detach(&adapter->stats.pxon2offc[i]); 777 } 778 if (i < __arraycount(adapter->stats.qprc)) { 779 evcnt_detach(&adapter->stats.qprc[i]); 780 evcnt_detach(&adapter->stats.qptc[i]); 781 evcnt_detach(&adapter->stats.qbrc[i]); 782 evcnt_detach(&adapter->stats.qbtc[i]); 783 evcnt_detach(&adapter->stats.qprdc[i]); 784 } 785 786 evcnt_detach(&rxr->rx_packets); 787 evcnt_detach(&rxr->rx_bytes); 788 evcnt_detach(&rxr->no_jmbuf); 789 evcnt_detach(&rxr->rx_discarded); 790 evcnt_detach(&rxr->rx_split_packets); 791 evcnt_detach(&rxr->rx_irq); 792 } 793 evcnt_detach(&stats->ipcs); 794 evcnt_detach(&stats->l4cs); 795 evcnt_detach(&stats->ipcs_bad); 796 evcnt_detach(&stats->l4cs_bad); 797 evcnt_detach(&stats->intzero); 798 evcnt_detach(&stats->legint); 799 evcnt_detach(&stats->crcerrs); 800 evcnt_detach(&stats->illerrc); 801 evcnt_detach(&stats->errbc); 802 evcnt_detach(&stats->mspdc); 803 evcnt_detach(&stats->mlfc); 804 evcnt_detach(&stats->mrfc); 805 evcnt_detach(&stats->rlec); 806 evcnt_detach(&stats->lxontxc); 807 evcnt_detach(&stats->lxonrxc); 808 evcnt_detach(&stats->lxofftxc); 809 evcnt_detach(&stats->lxoffrxc); 810 811 /* Packet Reception Stats */ 812 evcnt_detach(&stats->tor); 813 evcnt_detach(&stats->gorc); 814 evcnt_detach(&stats->tpr); 815 evcnt_detach(&stats->gprc); 816 evcnt_detach(&stats->mprc); 817 evcnt_detach(&stats->bprc); 818 evcnt_detach(&stats->prc64); 819 evcnt_detach(&stats->prc127); 820 evcnt_detach(&stats->prc255); 821 evcnt_detach(&stats->prc511); 822 evcnt_detach(&stats->prc1023); 823 evcnt_detach(&stats->prc1522); 824 evcnt_detach(&stats->ruc); 825 evcnt_detach(&stats->rfc); 826 evcnt_detach(&stats->roc); 827 evcnt_detach(&stats->rjc); 828 evcnt_detach(&stats->mngprc); 829 evcnt_detach(&stats->xec); 830 831 /* Packet Transmission Stats */ 832 evcnt_detach(&stats->gotc); 833 evcnt_detach(&stats->tpt); 834 evcnt_detach(&stats->gptc); 835 evcnt_detach(&stats->bptc); 836 evcnt_detach(&stats->mptc); 837 evcnt_detach(&stats->mngptc); 838 evcnt_detach(&stats->ptc64); 839 evcnt_detach(&stats->ptc127); 840 evcnt_detach(&stats->ptc255); 841 evcnt_detach(&stats->ptc511); 842 evcnt_detach(&stats->ptc1023); 843 evcnt_detach(&stats->ptc1522); 844 845 /* FC Stats */ 846 evcnt_detach(&stats->fccrc); 847 evcnt_detach(&stats->fclast); 848 if (adapter->hw.mac.type != ixgbe_mac_82598EB) { 849 evcnt_detach(&stats->fcoerpdc); 850 evcnt_detach(&stats->fcoeprc); 851 evcnt_detach(&stats->fcoeptc); 852 evcnt_detach(&stats->fcoedwrc); 853 evcnt_detach(&stats->fcoedwtc); 854 } 855 856 ixgbe_free_transmit_structures(adapter); 857 ixgbe_free_receive_structures(adapter); 858 free(adapter->mta, M_DEVBUF); 859 860 IXGBE_CORE_LOCK_DESTROY(adapter); 861 return (0); 862 } 863 864 /********************************************************************* 865 * 866 * Shutdown entry point 867 * 868 **********************************************************************/ 869 870 #if 0 /* XXX NetBSD ought to register something like this through pmf(9) */ 871 static int 872 ixgbe_shutdown(device_t dev) 873 { 874 struct adapter *adapter = device_private(dev); 875 IXGBE_CORE_LOCK(adapter); 876 ixgbe_stop(adapter); 877 IXGBE_CORE_UNLOCK(adapter); 878 return (0); 879 } 880 #endif 881 882 883 /********************************************************************* 884 * Transmit entry point 885 * 886 * ixgbe_start is called by the stack to initiate a transmit. 887 * The driver will remain in this routine as long as there are 888 * packets to transmit and transmit resources are available. 889 * In case resources are not available stack is notified and 890 * the packet is requeued. 891 **********************************************************************/ 892 893 static void 894 ixgbe_start_locked(struct tx_ring *txr, struct ifnet * ifp) 895 { 896 int rc; 897 struct mbuf *m_head; 898 struct adapter *adapter = txr->adapter; 899 900 IXGBE_TX_LOCK_ASSERT(txr); 901 902 if ((ifp->if_flags & (IFF_RUNNING|IFF_OACTIVE)) != 903 IFF_RUNNING) 904 return; 905 if (!adapter->link_active) 906 return; 907 908 while (!IFQ_IS_EMPTY(&ifp->if_snd)) { 909 910 IFQ_POLL(&ifp->if_snd, m_head); 911 if (m_head == NULL) 912 break; 913 914 if ((rc = ixgbe_xmit(txr, m_head)) == EAGAIN) { 915 ifp->if_flags |= IFF_OACTIVE; 916 break; 917 } 918 IFQ_DEQUEUE(&ifp->if_snd, m_head); 919 if (rc == EFBIG) { 920 struct mbuf *mtmp; 921 922 if ((mtmp = m_defrag(m_head, M_DONTWAIT)) != NULL) { 923 m_head = mtmp; 924 rc = ixgbe_xmit(txr, m_head); 925 if (rc != 0) 926 adapter->efbig2_tx_dma_setup.ev_count++; 927 } else 928 adapter->m_defrag_failed.ev_count++; 929 } 930 if (rc != 0) { 931 m_freem(m_head); 932 continue; 933 } 934 935 /* Send a copy of the frame to the BPF listener */ 936 bpf_mtap(ifp, m_head); 937 938 /* Set watchdog on */ 939 getmicrotime(&txr->watchdog_time); 940 txr->queue_status = IXGBE_QUEUE_WORKING; 941 942 } 943 return; 944 } 945 946 /* 947 * Legacy TX start - called by the stack, this 948 * always uses the first tx ring, and should 949 * not be used with multiqueue tx enabled. 950 */ 951 static void 952 ixgbe_start(struct ifnet *ifp) 953 { 954 struct adapter *adapter = ifp->if_softc; 955 struct tx_ring *txr = adapter->tx_rings; 956 957 if (ifp->if_flags & IFF_RUNNING) { 958 IXGBE_TX_LOCK(txr); 959 ixgbe_start_locked(txr, ifp); 960 IXGBE_TX_UNLOCK(txr); 961 } 962 return; 963 } 964 965 #if __FreeBSD_version >= 800000 966 /* 967 ** Multiqueue Transmit driver 968 ** 969 */ 970 static int 971 ixgbe_mq_start(struct ifnet *ifp, struct mbuf *m) 972 { 973 struct adapter *adapter = ifp->if_softc; 974 struct ix_queue *que; 975 struct tx_ring *txr; 976 int i = 0, err = 0; 977 978 /* Which queue to use */ 979 if ((m->m_flags & M_FLOWID) != 0) 980 i = m->m_pkthdr.flowid % adapter->num_queues; 981 982 txr = &adapter->tx_rings[i]; 983 que = &adapter->queues[i]; 984 985 if (IXGBE_TX_TRYLOCK(txr)) { 986 err = ixgbe_mq_start_locked(ifp, txr, m); 987 IXGBE_TX_UNLOCK(txr); 988 } else { 989 err = drbr_enqueue(ifp, txr->br, m); 990 softint_schedule(que->que_si); 991 } 992 993 return (err); 994 } 995 996 static int 997 ixgbe_mq_start_locked(struct ifnet *ifp, struct tx_ring *txr, struct mbuf *m) 998 { 999 struct adapter *adapter = txr->adapter; 1000 struct mbuf *next; 1001 int enqueued, err = 0; 1002 1003 if ((ifp->if_flags & (IFF_RUNNING | IFF_OACTIVE)) != 1004 IFF_RUNNING || adapter->link_active == 0) { 1005 if (m != NULL) 1006 err = drbr_enqueue(ifp, txr->br, m); 1007 return (err); 1008 } 1009 1010 enqueued = 0; 1011 if (m == NULL) { 1012 next = drbr_dequeue(ifp, txr->br); 1013 } else if (drbr_needs_enqueue(ifp, txr->br)) { 1014 if ((err = drbr_enqueue(ifp, txr->br, m)) != 0) 1015 return (err); 1016 next = drbr_dequeue(ifp, txr->br); 1017 } else 1018 next = m; 1019 1020 /* Process the queue */ 1021 while (next != NULL) { 1022 if ((err = ixgbe_xmit(txr, &next)) != 0) { 1023 if (next != NULL) 1024 err = drbr_enqueue(ifp, txr->br, next); 1025 break; 1026 } 1027 enqueued++; 1028 drbr_stats_update(ifp, next->m_pkthdr.len, next->m_flags); 1029 /* Send a copy of the frame to the BPF listener */ 1030 bpf_mtap(ifp, next); 1031 if ((ifp->if_flags & IFF_RUNNING) == 0) 1032 break; 1033 if (txr->tx_avail < IXGBE_TX_OP_THRESHOLD) 1034 ixgbe_txeof(txr); 1035 if (txr->tx_avail < IXGBE_TX_OP_THRESHOLD) { 1036 ifp->if_flags |= IFF_OACTIVE; 1037 break; 1038 } 1039 next = drbr_dequeue(ifp, txr->br); 1040 } 1041 1042 if (enqueued > 0) { 1043 /* Set watchdog on */ 1044 txr->queue_status = IXGBE_QUEUE_WORKING; 1045 getmicrotime(&txr->watchdog_time); 1046 } 1047 1048 return (err); 1049 } 1050 1051 /* 1052 ** Flush all ring buffers 1053 */ 1054 static void 1055 ixgbe_qflush(struct ifnet *ifp) 1056 { 1057 struct adapter *adapter = ifp->if_softc; 1058 struct tx_ring *txr = adapter->tx_rings; 1059 struct mbuf *m; 1060 1061 for (int i = 0; i < adapter->num_queues; i++, txr++) { 1062 IXGBE_TX_LOCK(txr); 1063 while ((m = buf_ring_dequeue_sc(txr->br)) != NULL) 1064 m_freem(m); 1065 IXGBE_TX_UNLOCK(txr); 1066 } 1067 if_qflush(ifp); 1068 } 1069 #endif /* __FreeBSD_version >= 800000 */ 1070 1071 static int 1072 ixgbe_ifflags_cb(struct ethercom *ec) 1073 { 1074 struct ifnet *ifp = &ec->ec_if; 1075 struct adapter *adapter = ifp->if_softc; 1076 int change = ifp->if_flags ^ adapter->if_flags, rc = 0; 1077 1078 IXGBE_CORE_LOCK(adapter); 1079 1080 if (change != 0) 1081 adapter->if_flags = ifp->if_flags; 1082 1083 if ((change & ~(IFF_CANTCHANGE|IFF_DEBUG)) != 0) 1084 rc = ENETRESET; 1085 else if ((change & (IFF_PROMISC | IFF_ALLMULTI)) != 0) 1086 ixgbe_set_promisc(adapter); 1087 1088 IXGBE_CORE_UNLOCK(adapter); 1089 1090 return rc; 1091 } 1092 1093 /********************************************************************* 1094 * Ioctl entry point 1095 * 1096 * ixgbe_ioctl is called when the user wants to configure the 1097 * interface. 1098 * 1099 * return 0 on success, positive on failure 1100 **********************************************************************/ 1101 1102 static int 1103 ixgbe_ioctl(struct ifnet * ifp, u_long command, void *data) 1104 { 1105 struct adapter *adapter = ifp->if_softc; 1106 struct ifcapreq *ifcr = data; 1107 struct ifreq *ifr = data; 1108 int error = 0; 1109 int l4csum_en; 1110 const int l4csum = IFCAP_CSUM_TCPv4_Rx|IFCAP_CSUM_UDPv4_Rx| 1111 IFCAP_CSUM_TCPv6_Rx|IFCAP_CSUM_UDPv6_Rx; 1112 1113 switch (command) { 1114 case SIOCSIFFLAGS: 1115 IOCTL_DEBUGOUT("ioctl: SIOCSIFFLAGS (Set Interface Flags)"); 1116 break; 1117 case SIOCADDMULTI: 1118 case SIOCDELMULTI: 1119 IOCTL_DEBUGOUT("ioctl: SIOC(ADD|DEL)MULTI"); 1120 break; 1121 case SIOCSIFMEDIA: 1122 case SIOCGIFMEDIA: 1123 IOCTL_DEBUGOUT("ioctl: SIOCxIFMEDIA (Get/Set Interface Media)"); 1124 break; 1125 case SIOCSIFCAP: 1126 IOCTL_DEBUGOUT("ioctl: SIOCSIFCAP (Set Capabilities)"); 1127 break; 1128 case SIOCSIFMTU: 1129 IOCTL_DEBUGOUT("ioctl: SIOCSIFMTU (Set Interface MTU)"); 1130 break; 1131 default: 1132 IOCTL_DEBUGOUT1("ioctl: UNKNOWN (0x%X)\n", (int)command); 1133 break; 1134 } 1135 1136 switch (command) { 1137 case SIOCSIFMEDIA: 1138 case SIOCGIFMEDIA: 1139 return ifmedia_ioctl(ifp, ifr, &adapter->media, command); 1140 case SIOCSIFCAP: 1141 /* Layer-4 Rx checksum offload has to be turned on and 1142 * off as a unit. 1143 */ 1144 l4csum_en = ifcr->ifcr_capenable & l4csum; 1145 if (l4csum_en != l4csum && l4csum_en != 0) 1146 return EINVAL; 1147 /*FALLTHROUGH*/ 1148 case SIOCADDMULTI: 1149 case SIOCDELMULTI: 1150 case SIOCSIFFLAGS: 1151 case SIOCSIFMTU: 1152 default: 1153 if ((error = ether_ioctl(ifp, command, data)) != ENETRESET) 1154 return error; 1155 if ((ifp->if_flags & IFF_RUNNING) == 0) 1156 ; 1157 else if (command == SIOCSIFCAP || command == SIOCSIFMTU) { 1158 IXGBE_CORE_LOCK(adapter); 1159 ixgbe_init_locked(adapter); 1160 IXGBE_CORE_UNLOCK(adapter); 1161 } else if (command == SIOCADDMULTI || command == SIOCDELMULTI) { 1162 /* 1163 * Multicast list has changed; set the hardware filter 1164 * accordingly. 1165 */ 1166 IXGBE_CORE_LOCK(adapter); 1167 ixgbe_disable_intr(adapter); 1168 ixgbe_set_multi(adapter); 1169 ixgbe_enable_intr(adapter); 1170 IXGBE_CORE_UNLOCK(adapter); 1171 } 1172 return 0; 1173 } 1174 } 1175 1176 /********************************************************************* 1177 * Init entry point 1178 * 1179 * This routine is used in two ways. It is used by the stack as 1180 * init entry point in network interface structure. It is also used 1181 * by the driver as a hw/sw initialization routine to get to a 1182 * consistent state. 1183 * 1184 * return 0 on success, positive on failure 1185 **********************************************************************/ 1186 #define IXGBE_MHADD_MFS_SHIFT 16 1187 1188 static void 1189 ixgbe_init_locked(struct adapter *adapter) 1190 { 1191 struct ifnet *ifp = adapter->ifp; 1192 device_t dev = adapter->dev; 1193 struct ixgbe_hw *hw = &adapter->hw; 1194 u32 k, txdctl, mhadd, gpie; 1195 u32 rxdctl, rxctrl; 1196 1197 /* XXX check IFF_UP and IFF_RUNNING, power-saving state! */ 1198 1199 KASSERT(mutex_owned(&adapter->core_mtx)); 1200 INIT_DEBUGOUT("ixgbe_init: begin"); 1201 hw->adapter_stopped = FALSE; 1202 ixgbe_stop_adapter(hw); 1203 callout_stop(&adapter->timer); 1204 1205 /* XXX I moved this here from the SIOCSIFMTU case in ixgbe_ioctl(). */ 1206 adapter->max_frame_size = 1207 ifp->if_mtu + ETHER_HDR_LEN + ETHER_CRC_LEN; 1208 1209 /* reprogram the RAR[0] in case user changed it. */ 1210 ixgbe_set_rar(hw, 0, adapter->hw.mac.addr, 0, IXGBE_RAH_AV); 1211 1212 /* Get the latest mac address, User can use a LAA */ 1213 memcpy(hw->mac.addr, CLLADDR(adapter->ifp->if_sadl), 1214 IXGBE_ETH_LENGTH_OF_ADDRESS); 1215 ixgbe_set_rar(hw, 0, hw->mac.addr, 0, 1); 1216 hw->addr_ctrl.rar_used_count = 1; 1217 1218 /* Prepare transmit descriptors and buffers */ 1219 if (ixgbe_setup_transmit_structures(adapter)) { 1220 device_printf(dev,"Could not setup transmit structures\n"); 1221 ixgbe_stop(adapter); 1222 return; 1223 } 1224 1225 ixgbe_init_hw(hw); 1226 ixgbe_initialize_transmit_units(adapter); 1227 1228 /* Setup Multicast table */ 1229 ixgbe_set_multi(adapter); 1230 1231 /* 1232 ** Determine the correct mbuf pool 1233 ** for doing jumbo/headersplit 1234 */ 1235 if (adapter->max_frame_size <= 2048) 1236 adapter->rx_mbuf_sz = MCLBYTES; 1237 else if (adapter->max_frame_size <= 4096) 1238 adapter->rx_mbuf_sz = MJUMPAGESIZE; 1239 else if (adapter->max_frame_size <= 9216) 1240 adapter->rx_mbuf_sz = MJUM9BYTES; 1241 else 1242 adapter->rx_mbuf_sz = MJUM16BYTES; 1243 1244 /* Prepare receive descriptors and buffers */ 1245 if (ixgbe_setup_receive_structures(adapter)) { 1246 device_printf(dev,"Could not setup receive structures\n"); 1247 ixgbe_stop(adapter); 1248 return; 1249 } 1250 1251 /* Configure RX settings */ 1252 ixgbe_initialize_receive_units(adapter); 1253 1254 gpie = IXGBE_READ_REG(&adapter->hw, IXGBE_GPIE); 1255 1256 /* Enable Fan Failure Interrupt */ 1257 gpie |= IXGBE_SDP1_GPIEN; 1258 1259 /* Add for Thermal detection */ 1260 if (hw->mac.type == ixgbe_mac_82599EB) 1261 gpie |= IXGBE_SDP2_GPIEN; 1262 1263 if (adapter->msix > 1) { 1264 /* Enable Enhanced MSIX mode */ 1265 gpie |= IXGBE_GPIE_MSIX_MODE; 1266 gpie |= IXGBE_GPIE_EIAME | IXGBE_GPIE_PBA_SUPPORT | 1267 IXGBE_GPIE_OCD; 1268 } 1269 IXGBE_WRITE_REG(hw, IXGBE_GPIE, gpie); 1270 1271 /* Set MTU size */ 1272 if (ifp->if_mtu > ETHERMTU) { 1273 mhadd = IXGBE_READ_REG(hw, IXGBE_MHADD); 1274 mhadd &= ~IXGBE_MHADD_MFS_MASK; 1275 mhadd |= adapter->max_frame_size << IXGBE_MHADD_MFS_SHIFT; 1276 IXGBE_WRITE_REG(hw, IXGBE_MHADD, mhadd); 1277 } 1278 1279 /* Now enable all the queues */ 1280 1281 for (int i = 0; i < adapter->num_queues; i++) { 1282 txdctl = IXGBE_READ_REG(hw, IXGBE_TXDCTL(i)); 1283 txdctl |= IXGBE_TXDCTL_ENABLE; 1284 /* Set WTHRESH to 8, burst writeback */ 1285 txdctl |= (8 << 16); 1286 IXGBE_WRITE_REG(hw, IXGBE_TXDCTL(i), txdctl); 1287 } 1288 1289 for (int i = 0; i < adapter->num_queues; i++) { 1290 rxdctl = IXGBE_READ_REG(hw, IXGBE_RXDCTL(i)); 1291 if (hw->mac.type == ixgbe_mac_82598EB) { 1292 /* 1293 ** PTHRESH = 21 1294 ** HTHRESH = 4 1295 ** WTHRESH = 8 1296 */ 1297 rxdctl &= ~0x3FFFFF; 1298 rxdctl |= 0x080420; 1299 } 1300 rxdctl |= IXGBE_RXDCTL_ENABLE; 1301 IXGBE_WRITE_REG(hw, IXGBE_RXDCTL(i), rxdctl); 1302 /* XXX I don't trust this loop, and I don't trust the 1303 * XXX memory barrier. What is this meant to do? --dyoung 1304 */ 1305 for (k = 0; k < 10; k++) { 1306 if (IXGBE_READ_REG(hw, IXGBE_RXDCTL(i)) & 1307 IXGBE_RXDCTL_ENABLE) 1308 break; 1309 else 1310 msec_delay(1); 1311 } 1312 wmb(); 1313 #ifdef DEV_NETMAP 1314 /* 1315 * In netmap mode, we must preserve the buffers made 1316 * available to userspace before the if_init() 1317 * (this is true by default on the TX side, because 1318 * init makes all buffers available to userspace). 1319 * 1320 * netmap_reset() and the device specific routines 1321 * (e.g. ixgbe_setup_receive_rings()) map these 1322 * buffers at the end of the NIC ring, so here we 1323 * must set the RDT (tail) register to make sure 1324 * they are not overwritten. 1325 * 1326 * In this driver the NIC ring starts at RDH = 0, 1327 * RDT points to the last slot available for reception (?), 1328 * so RDT = num_rx_desc - 1 means the whole ring is available. 1329 */ 1330 if (ifp->if_capenable & IFCAP_NETMAP) { 1331 struct netmap_adapter *na = NA(adapter->ifp); 1332 struct netmap_kring *kring = &na->rx_rings[i]; 1333 int t = na->num_rx_desc - 1 - kring->nr_hwavail; 1334 1335 IXGBE_WRITE_REG(hw, IXGBE_RDT(i), t); 1336 } else 1337 #endif /* DEV_NETMAP */ 1338 IXGBE_WRITE_REG(hw, IXGBE_RDT(i), adapter->num_rx_desc - 1); 1339 } 1340 1341 /* Set up VLAN support and filter */ 1342 ixgbe_setup_vlan_hw_support(adapter); 1343 1344 /* Enable Receive engine */ 1345 rxctrl = IXGBE_READ_REG(hw, IXGBE_RXCTRL); 1346 if (hw->mac.type == ixgbe_mac_82598EB) 1347 rxctrl |= IXGBE_RXCTRL_DMBYPS; 1348 rxctrl |= IXGBE_RXCTRL_RXEN; 1349 ixgbe_enable_rx_dma(hw, rxctrl); 1350 1351 callout_reset(&adapter->timer, hz, ixgbe_local_timer, adapter); 1352 1353 /* Set up MSI/X routing */ 1354 if (ixgbe_enable_msix) { 1355 ixgbe_configure_ivars(adapter); 1356 /* Set up auto-mask */ 1357 if (hw->mac.type == ixgbe_mac_82598EB) 1358 IXGBE_WRITE_REG(hw, IXGBE_EIAM, IXGBE_EICS_RTX_QUEUE); 1359 else { 1360 IXGBE_WRITE_REG(hw, IXGBE_EIAM_EX(0), 0xFFFFFFFF); 1361 IXGBE_WRITE_REG(hw, IXGBE_EIAM_EX(1), 0xFFFFFFFF); 1362 } 1363 } else { /* Simple settings for Legacy/MSI */ 1364 ixgbe_set_ivar(adapter, 0, 0, 0); 1365 ixgbe_set_ivar(adapter, 0, 0, 1); 1366 IXGBE_WRITE_REG(hw, IXGBE_EIAM, IXGBE_EICS_RTX_QUEUE); 1367 } 1368 1369 #ifdef IXGBE_FDIR 1370 /* Init Flow director */ 1371 if (hw->mac.type != ixgbe_mac_82598EB) 1372 ixgbe_init_fdir_signature_82599(&adapter->hw, fdir_pballoc); 1373 #endif 1374 1375 /* 1376 ** Check on any SFP devices that 1377 ** need to be kick-started 1378 */ 1379 if (hw->phy.type == ixgbe_phy_none) { 1380 int err = hw->phy.ops.identify(hw); 1381 if (err == IXGBE_ERR_SFP_NOT_SUPPORTED) { 1382 device_printf(dev, 1383 "Unsupported SFP+ module type was detected.\n"); 1384 return; 1385 } 1386 } 1387 1388 /* Set moderation on the Link interrupt */ 1389 IXGBE_WRITE_REG(hw, IXGBE_EITR(adapter->linkvec), IXGBE_LINK_ITR); 1390 1391 /* Config/Enable Link */ 1392 ixgbe_config_link(adapter); 1393 1394 /* And now turn on interrupts */ 1395 ixgbe_enable_intr(adapter); 1396 1397 /* Now inform the stack we're ready */ 1398 ifp->if_flags |= IFF_RUNNING; 1399 ifp->if_flags &= ~IFF_OACTIVE; 1400 1401 return; 1402 } 1403 1404 static int 1405 ixgbe_init(struct ifnet *ifp) 1406 { 1407 struct adapter *adapter = ifp->if_softc; 1408 1409 IXGBE_CORE_LOCK(adapter); 1410 ixgbe_init_locked(adapter); 1411 IXGBE_CORE_UNLOCK(adapter); 1412 return 0; /* XXX ixgbe_init_locked cannot fail? really? */ 1413 } 1414 1415 1416 /* 1417 ** 1418 ** MSIX Interrupt Handlers and Tasklets 1419 ** 1420 */ 1421 1422 static inline void 1423 ixgbe_enable_queue(struct adapter *adapter, u32 vector) 1424 { 1425 struct ixgbe_hw *hw = &adapter->hw; 1426 u64 queue = (u64)(1ULL << vector); 1427 u32 mask; 1428 1429 if (hw->mac.type == ixgbe_mac_82598EB) { 1430 mask = (IXGBE_EIMS_RTX_QUEUE & queue); 1431 IXGBE_WRITE_REG(hw, IXGBE_EIMS, mask); 1432 } else { 1433 mask = (queue & 0xFFFFFFFF); 1434 if (mask) 1435 IXGBE_WRITE_REG(hw, IXGBE_EIMS_EX(0), mask); 1436 mask = (queue >> 32); 1437 if (mask) 1438 IXGBE_WRITE_REG(hw, IXGBE_EIMS_EX(1), mask); 1439 } 1440 } 1441 1442 __unused static inline void 1443 ixgbe_disable_queue(struct adapter *adapter, u32 vector) 1444 { 1445 struct ixgbe_hw *hw = &adapter->hw; 1446 u64 queue = (u64)(1ULL << vector); 1447 u32 mask; 1448 1449 if (hw->mac.type == ixgbe_mac_82598EB) { 1450 mask = (IXGBE_EIMS_RTX_QUEUE & queue); 1451 IXGBE_WRITE_REG(hw, IXGBE_EIMC, mask); 1452 } else { 1453 mask = (queue & 0xFFFFFFFF); 1454 if (mask) 1455 IXGBE_WRITE_REG(hw, IXGBE_EIMC_EX(0), mask); 1456 mask = (queue >> 32); 1457 if (mask) 1458 IXGBE_WRITE_REG(hw, IXGBE_EIMC_EX(1), mask); 1459 } 1460 } 1461 1462 static inline void 1463 ixgbe_rearm_queues(struct adapter *adapter, u64 queues) 1464 { 1465 u32 mask; 1466 1467 if (adapter->hw.mac.type == ixgbe_mac_82598EB) { 1468 mask = (IXGBE_EIMS_RTX_QUEUE & queues); 1469 IXGBE_WRITE_REG(&adapter->hw, IXGBE_EICS, mask); 1470 } else { 1471 mask = (queues & 0xFFFFFFFF); 1472 IXGBE_WRITE_REG(&adapter->hw, IXGBE_EICS_EX(0), mask); 1473 mask = (queues >> 32); 1474 IXGBE_WRITE_REG(&adapter->hw, IXGBE_EICS_EX(1), mask); 1475 } 1476 } 1477 1478 1479 static void 1480 ixgbe_handle_que(void *context) 1481 { 1482 struct ix_queue *que = context; 1483 struct adapter *adapter = que->adapter; 1484 struct tx_ring *txr = que->txr; 1485 struct ifnet *ifp = adapter->ifp; 1486 bool more; 1487 1488 adapter->handleq.ev_count++; 1489 1490 if (ifp->if_flags & IFF_RUNNING) { 1491 more = ixgbe_rxeof(que, adapter->rx_process_limit); 1492 IXGBE_TX_LOCK(txr); 1493 ixgbe_txeof(txr); 1494 #if __FreeBSD_version >= 800000 1495 if (!drbr_empty(ifp, txr->br)) 1496 ixgbe_mq_start_locked(ifp, txr, NULL); 1497 #else 1498 if (!IFQ_IS_EMPTY(&ifp->if_snd)) 1499 ixgbe_start_locked(txr, ifp); 1500 #endif 1501 IXGBE_TX_UNLOCK(txr); 1502 if (more) { 1503 adapter->req.ev_count++; 1504 softint_schedule(que->que_si); 1505 return; 1506 } 1507 } 1508 1509 /* Reenable this interrupt */ 1510 ixgbe_enable_queue(adapter, que->msix); 1511 1512 return; 1513 } 1514 1515 1516 /********************************************************************* 1517 * 1518 * Legacy Interrupt Service routine 1519 * 1520 **********************************************************************/ 1521 1522 static int 1523 ixgbe_legacy_irq(void *arg) 1524 { 1525 struct ix_queue *que = arg; 1526 struct adapter *adapter = que->adapter; 1527 struct ifnet *ifp = adapter->ifp; 1528 struct ixgbe_hw *hw = &adapter->hw; 1529 struct tx_ring *txr = adapter->tx_rings; 1530 bool more_tx = false, more_rx = false; 1531 u32 reg_eicr, loop = MAX_LOOP; 1532 1533 reg_eicr = IXGBE_READ_REG(hw, IXGBE_EICR); 1534 1535 adapter->stats.legint.ev_count++; 1536 ++que->irqs; 1537 if (reg_eicr == 0) { 1538 adapter->stats.intzero.ev_count++; 1539 if ((ifp->if_flags & IFF_UP) != 0) 1540 ixgbe_enable_intr(adapter); 1541 return 0; 1542 } 1543 1544 if ((ifp->if_flags & IFF_RUNNING) != 0) { 1545 more_rx = ixgbe_rxeof(que, adapter->rx_process_limit); 1546 1547 IXGBE_TX_LOCK(txr); 1548 do { 1549 adapter->txloops.ev_count++; 1550 more_tx = ixgbe_txeof(txr); 1551 } while (loop-- && more_tx); 1552 IXGBE_TX_UNLOCK(txr); 1553 } 1554 1555 if (more_rx || more_tx) { 1556 if (more_rx) 1557 adapter->morerx.ev_count++; 1558 if (more_tx) 1559 adapter->moretx.ev_count++; 1560 softint_schedule(que->que_si); 1561 } 1562 1563 /* Check for fan failure */ 1564 if ((hw->phy.media_type == ixgbe_media_type_copper) && 1565 (reg_eicr & IXGBE_EICR_GPI_SDP1)) { 1566 device_printf(adapter->dev, "\nCRITICAL: FAN FAILURE!! " 1567 "REPLACE IMMEDIATELY!!\n"); 1568 IXGBE_WRITE_REG(hw, IXGBE_EIMS, IXGBE_EICR_GPI_SDP1); 1569 } 1570 1571 /* Link status change */ 1572 if (reg_eicr & IXGBE_EICR_LSC) 1573 softint_schedule(adapter->link_si); 1574 1575 ixgbe_enable_intr(adapter); 1576 return 1; 1577 } 1578 1579 1580 #if defined(NETBSD_MSI_OR_MSIX) 1581 /********************************************************************* 1582 * 1583 * MSIX Queue Interrupt Service routine 1584 * 1585 **********************************************************************/ 1586 void 1587 ixgbe_msix_que(void *arg) 1588 { 1589 struct ix_queue *que = arg; 1590 struct adapter *adapter = que->adapter; 1591 struct tx_ring *txr = que->txr; 1592 struct rx_ring *rxr = que->rxr; 1593 bool more_tx, more_rx; 1594 u32 newitr = 0; 1595 1596 ++que->irqs; 1597 1598 more_rx = ixgbe_rxeof(que, adapter->rx_process_limit); 1599 1600 IXGBE_TX_LOCK(txr); 1601 more_tx = ixgbe_txeof(txr); 1602 /* 1603 ** Make certain that if the stack 1604 ** has anything queued the task gets 1605 ** scheduled to handle it. 1606 */ 1607 #if __FreeBSD_version < 800000 1608 if (!IFQ_DRV_IS_EMPTY(&adapter->ifp->if_snd)) 1609 #else 1610 if (!drbr_empty(adapter->ifp, txr->br)) 1611 #endif 1612 more_tx = 1; 1613 IXGBE_TX_UNLOCK(txr); 1614 1615 /* Do AIM now? */ 1616 1617 if (ixgbe_enable_aim == FALSE) 1618 goto no_calc; 1619 /* 1620 ** Do Adaptive Interrupt Moderation: 1621 ** - Write out last calculated setting 1622 ** - Calculate based on average size over 1623 ** the last interval. 1624 */ 1625 if (que->eitr_setting) 1626 IXGBE_WRITE_REG(&adapter->hw, 1627 IXGBE_EITR(que->msix), que->eitr_setting); 1628 1629 que->eitr_setting = 0; 1630 1631 /* Idle, do nothing */ 1632 if ((txr->bytes == 0) && (rxr->bytes == 0)) 1633 goto no_calc; 1634 1635 if ((txr->bytes) && (txr->packets)) 1636 newitr = txr->bytes/txr->packets; 1637 if ((rxr->bytes) && (rxr->packets)) 1638 newitr = max(newitr, 1639 (rxr->bytes / rxr->packets)); 1640 newitr += 24; /* account for hardware frame, crc */ 1641 1642 /* set an upper boundary */ 1643 newitr = min(newitr, 3000); 1644 1645 /* Be nice to the mid range */ 1646 if ((newitr > 300) && (newitr < 1200)) 1647 newitr = (newitr / 3); 1648 else 1649 newitr = (newitr / 2); 1650 1651 if (adapter->hw.mac.type == ixgbe_mac_82598EB) 1652 newitr |= newitr << 16; 1653 else 1654 newitr |= IXGBE_EITR_CNT_WDIS; 1655 1656 /* save for next interrupt */ 1657 que->eitr_setting = newitr; 1658 1659 /* Reset state */ 1660 txr->bytes = 0; 1661 txr->packets = 0; 1662 rxr->bytes = 0; 1663 rxr->packets = 0; 1664 1665 no_calc: 1666 if (more_tx || more_rx) 1667 softint_schedule(que->que_si); 1668 else /* Reenable this interrupt */ 1669 ixgbe_enable_queue(adapter, que->msix); 1670 return; 1671 } 1672 1673 1674 static void 1675 ixgbe_msix_link(void *arg) 1676 { 1677 struct adapter *adapter = arg; 1678 struct ixgbe_hw *hw = &adapter->hw; 1679 u32 reg_eicr; 1680 1681 ++adapter->link_irq.ev_count; 1682 1683 /* First get the cause */ 1684 reg_eicr = IXGBE_READ_REG(hw, IXGBE_EICS); 1685 /* Clear interrupt with write */ 1686 IXGBE_WRITE_REG(hw, IXGBE_EICR, reg_eicr); 1687 1688 /* Link status change */ 1689 if (reg_eicr & IXGBE_EICR_LSC) 1690 softint_schedule(adapter->link_si); 1691 1692 if (adapter->hw.mac.type != ixgbe_mac_82598EB) { 1693 #ifdef IXGBE_FDIR 1694 if (reg_eicr & IXGBE_EICR_FLOW_DIR) { 1695 /* This is probably overkill :) */ 1696 if (!atomic_cmpset_int(&adapter->fdir_reinit, 0, 1)) 1697 return; 1698 /* Clear the interrupt */ 1699 IXGBE_WRITE_REG(hw, IXGBE_EICR, IXGBE_EICR_FLOW_DIR); 1700 /* Turn off the interface */ 1701 adapter->ifp->if_flags &= ~IFF_RUNNING; 1702 softint_schedule(adapter->fdir_si); 1703 } else 1704 #endif 1705 if (reg_eicr & IXGBE_EICR_ECC) { 1706 device_printf(adapter->dev, "\nCRITICAL: ECC ERROR!! " 1707 "Please Reboot!!\n"); 1708 IXGBE_WRITE_REG(hw, IXGBE_EICR, IXGBE_EICR_ECC); 1709 } else 1710 1711 if (reg_eicr & IXGBE_EICR_GPI_SDP1) { 1712 /* Clear the interrupt */ 1713 IXGBE_WRITE_REG(hw, IXGBE_EICR, IXGBE_EICR_GPI_SDP1); 1714 softint_schedule(adapter->msf_si); 1715 } else if (reg_eicr & IXGBE_EICR_GPI_SDP2) { 1716 /* Clear the interrupt */ 1717 IXGBE_WRITE_REG(hw, IXGBE_EICR, IXGBE_EICR_GPI_SDP2); 1718 softint_schedule(adapter->mod_si); 1719 } 1720 } 1721 1722 /* Check for fan failure */ 1723 if ((hw->device_id == IXGBE_DEV_ID_82598AT) && 1724 (reg_eicr & IXGBE_EICR_GPI_SDP1)) { 1725 device_printf(adapter->dev, "\nCRITICAL: FAN FAILURE!! " 1726 "REPLACE IMMEDIATELY!!\n"); 1727 IXGBE_WRITE_REG(hw, IXGBE_EICR, IXGBE_EICR_GPI_SDP1); 1728 } 1729 1730 IXGBE_WRITE_REG(&adapter->hw, IXGBE_EIMS, IXGBE_EIMS_OTHER); 1731 return; 1732 } 1733 #endif 1734 1735 /********************************************************************* 1736 * 1737 * Media Ioctl callback 1738 * 1739 * This routine is called whenever the user queries the status of 1740 * the interface using ifconfig. 1741 * 1742 **********************************************************************/ 1743 static void 1744 ixgbe_media_status(struct ifnet * ifp, struct ifmediareq * ifmr) 1745 { 1746 struct adapter *adapter = ifp->if_softc; 1747 1748 INIT_DEBUGOUT("ixgbe_media_status: begin"); 1749 IXGBE_CORE_LOCK(adapter); 1750 ixgbe_update_link_status(adapter); 1751 1752 ifmr->ifm_status = IFM_AVALID; 1753 ifmr->ifm_active = IFM_ETHER; 1754 1755 if (!adapter->link_active) { 1756 IXGBE_CORE_UNLOCK(adapter); 1757 return; 1758 } 1759 1760 ifmr->ifm_status |= IFM_ACTIVE; 1761 1762 switch (adapter->link_speed) { 1763 case IXGBE_LINK_SPEED_1GB_FULL: 1764 ifmr->ifm_active |= IFM_1000_T | IFM_FDX; 1765 break; 1766 case IXGBE_LINK_SPEED_10GB_FULL: 1767 ifmr->ifm_active |= adapter->optics | IFM_FDX; 1768 break; 1769 } 1770 1771 IXGBE_CORE_UNLOCK(adapter); 1772 1773 return; 1774 } 1775 1776 /********************************************************************* 1777 * 1778 * Media Ioctl callback 1779 * 1780 * This routine is called when the user changes speed/duplex using 1781 * media/mediopt option with ifconfig. 1782 * 1783 **********************************************************************/ 1784 static int 1785 ixgbe_media_change(struct ifnet * ifp) 1786 { 1787 struct adapter *adapter = ifp->if_softc; 1788 struct ifmedia *ifm = &adapter->media; 1789 1790 INIT_DEBUGOUT("ixgbe_media_change: begin"); 1791 1792 if (IFM_TYPE(ifm->ifm_media) != IFM_ETHER) 1793 return (EINVAL); 1794 1795 switch (IFM_SUBTYPE(ifm->ifm_media)) { 1796 case IFM_AUTO: 1797 adapter->hw.phy.autoneg_advertised = 1798 IXGBE_LINK_SPEED_1GB_FULL | IXGBE_LINK_SPEED_10GB_FULL; 1799 break; 1800 default: 1801 device_printf(adapter->dev, "Only auto media type\n"); 1802 return (EINVAL); 1803 } 1804 1805 return (0); 1806 } 1807 1808 /********************************************************************* 1809 * 1810 * This routine maps the mbufs to tx descriptors, allowing the 1811 * TX engine to transmit the packets. 1812 * - return 0 on success, positive on failure 1813 * 1814 **********************************************************************/ 1815 1816 static int 1817 ixgbe_xmit(struct tx_ring *txr, struct mbuf *m_head) 1818 { 1819 struct m_tag *mtag; 1820 struct adapter *adapter = txr->adapter; 1821 struct ethercom *ec = &adapter->osdep.ec; 1822 u32 olinfo_status = 0, cmd_type_len; 1823 u32 paylen = 0; 1824 int i, j, error; 1825 int first, last = 0; 1826 bus_dmamap_t map; 1827 struct ixgbe_tx_buf *txbuf; 1828 union ixgbe_adv_tx_desc *txd = NULL; 1829 1830 /* Basic descriptor defines */ 1831 cmd_type_len = (IXGBE_ADVTXD_DTYP_DATA | 1832 IXGBE_ADVTXD_DCMD_IFCS | IXGBE_ADVTXD_DCMD_DEXT); 1833 1834 if ((mtag = VLAN_OUTPUT_TAG(ec, m_head)) != NULL) 1835 cmd_type_len |= IXGBE_ADVTXD_DCMD_VLE; 1836 1837 /* 1838 * Important to capture the first descriptor 1839 * used because it will contain the index of 1840 * the one we tell the hardware to report back 1841 */ 1842 first = txr->next_avail_desc; 1843 txbuf = &txr->tx_buffers[first]; 1844 map = txbuf->map; 1845 1846 /* 1847 * Map the packet for DMA. 1848 */ 1849 error = bus_dmamap_load_mbuf(txr->txtag->dt_dmat, map, 1850 m_head, BUS_DMA_NOWAIT); 1851 1852 switch (error) { 1853 case EAGAIN: 1854 adapter->eagain_tx_dma_setup.ev_count++; 1855 return EAGAIN; 1856 case ENOMEM: 1857 adapter->enomem_tx_dma_setup.ev_count++; 1858 return EAGAIN; 1859 case EFBIG: 1860 adapter->efbig_tx_dma_setup.ev_count++; 1861 return error; 1862 case EINVAL: 1863 adapter->einval_tx_dma_setup.ev_count++; 1864 return error; 1865 default: 1866 adapter->other_tx_dma_setup.ev_count++; 1867 return error; 1868 case 0: 1869 break; 1870 } 1871 1872 /* Make certain there are enough descriptors */ 1873 if (map->dm_nsegs > txr->tx_avail - 2) { 1874 txr->no_desc_avail.ev_count++; 1875 ixgbe_dmamap_unload(txr->txtag, txbuf->map); 1876 return EAGAIN; 1877 } 1878 1879 /* 1880 ** Set up the appropriate offload context 1881 ** this becomes the first descriptor of 1882 ** a packet. 1883 */ 1884 if (m_head->m_pkthdr.csum_flags & (M_CSUM_TSOv4|M_CSUM_TSOv6)) { 1885 if (ixgbe_tso_setup(txr, m_head, &paylen)) { 1886 cmd_type_len |= IXGBE_ADVTXD_DCMD_TSE; 1887 olinfo_status |= IXGBE_TXD_POPTS_IXSM << 8; 1888 olinfo_status |= IXGBE_TXD_POPTS_TXSM << 8; 1889 olinfo_status |= paylen << IXGBE_ADVTXD_PAYLEN_SHIFT; 1890 ++adapter->tso_tx.ev_count; 1891 } else { 1892 ++adapter->tso_err.ev_count; 1893 /* XXX unload DMA map! --dyoung */ 1894 return ENXIO; 1895 } 1896 } else 1897 olinfo_status |= ixgbe_tx_ctx_setup(txr, m_head); 1898 1899 #ifdef IXGBE_IEEE1588 1900 /* This is changing soon to an mtag detection */ 1901 if (we detect this mbuf has a TSTAMP mtag) 1902 cmd_type_len |= IXGBE_ADVTXD_MAC_TSTAMP; 1903 #endif 1904 1905 #ifdef IXGBE_FDIR 1906 /* Do the flow director magic */ 1907 if ((txr->atr_sample) && (!adapter->fdir_reinit)) { 1908 ++txr->atr_count; 1909 if (txr->atr_count >= atr_sample_rate) { 1910 ixgbe_atr(txr, m_head); 1911 txr->atr_count = 0; 1912 } 1913 } 1914 #endif 1915 /* Record payload length */ 1916 if (paylen == 0) 1917 olinfo_status |= m_head->m_pkthdr.len << 1918 IXGBE_ADVTXD_PAYLEN_SHIFT; 1919 1920 i = txr->next_avail_desc; 1921 for (j = 0; j < map->dm_nsegs; j++) { 1922 bus_size_t seglen; 1923 bus_addr_t segaddr; 1924 1925 txbuf = &txr->tx_buffers[i]; 1926 txd = &txr->tx_base[i]; 1927 seglen = map->dm_segs[j].ds_len; 1928 segaddr = htole64(map->dm_segs[j].ds_addr); 1929 1930 txd->read.buffer_addr = segaddr; 1931 txd->read.cmd_type_len = htole32(txr->txd_cmd | 1932 cmd_type_len |seglen); 1933 txd->read.olinfo_status = htole32(olinfo_status); 1934 last = i; /* descriptor that will get completion IRQ */ 1935 1936 if (++i == adapter->num_tx_desc) 1937 i = 0; 1938 1939 txbuf->m_head = NULL; 1940 txbuf->eop_index = -1; 1941 } 1942 1943 txd->read.cmd_type_len |= 1944 htole32(IXGBE_TXD_CMD_EOP | IXGBE_TXD_CMD_RS); 1945 txr->tx_avail -= map->dm_nsegs; 1946 txr->next_avail_desc = i; 1947 1948 txbuf->m_head = m_head; 1949 /* Swap the dma map between the first and last descriptor */ 1950 txr->tx_buffers[first].map = txbuf->map; 1951 txbuf->map = map; 1952 bus_dmamap_sync(txr->txtag->dt_dmat, map, 0, m_head->m_pkthdr.len, 1953 BUS_DMASYNC_PREWRITE); 1954 1955 /* Set the index of the descriptor that will be marked done */ 1956 txbuf = &txr->tx_buffers[first]; 1957 txbuf->eop_index = last; 1958 1959 ixgbe_dmamap_sync(txr->txdma.dma_tag, txr->txdma.dma_map, 1960 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); 1961 /* 1962 * Advance the Transmit Descriptor Tail (Tdt), this tells the 1963 * hardware that this frame is available to transmit. 1964 */ 1965 ++txr->total_packets.ev_count; 1966 IXGBE_WRITE_REG(&adapter->hw, IXGBE_TDT(txr->me), i); 1967 1968 return 0; 1969 } 1970 1971 static void 1972 ixgbe_set_promisc(struct adapter *adapter) 1973 { 1974 u_int32_t reg_rctl; 1975 struct ifnet *ifp = adapter->ifp; 1976 1977 reg_rctl = IXGBE_READ_REG(&adapter->hw, IXGBE_FCTRL); 1978 reg_rctl &= (~IXGBE_FCTRL_UPE); 1979 reg_rctl &= (~IXGBE_FCTRL_MPE); 1980 IXGBE_WRITE_REG(&adapter->hw, IXGBE_FCTRL, reg_rctl); 1981 1982 if (ifp->if_flags & IFF_PROMISC) { 1983 reg_rctl |= (IXGBE_FCTRL_UPE | IXGBE_FCTRL_MPE); 1984 IXGBE_WRITE_REG(&adapter->hw, IXGBE_FCTRL, reg_rctl); 1985 } else if (ifp->if_flags & IFF_ALLMULTI) { 1986 reg_rctl |= IXGBE_FCTRL_MPE; 1987 reg_rctl &= ~IXGBE_FCTRL_UPE; 1988 IXGBE_WRITE_REG(&adapter->hw, IXGBE_FCTRL, reg_rctl); 1989 } 1990 return; 1991 } 1992 1993 1994 /********************************************************************* 1995 * Multicast Update 1996 * 1997 * This routine is called whenever multicast address list is updated. 1998 * 1999 **********************************************************************/ 2000 #define IXGBE_RAR_ENTRIES 16 2001 2002 static void 2003 ixgbe_set_multi(struct adapter *adapter) 2004 { 2005 struct ether_multi *enm; 2006 struct ether_multistep step; 2007 u32 fctrl; 2008 u8 *mta; 2009 u8 *update_ptr; 2010 int mcnt = 0; 2011 struct ethercom *ec = &adapter->osdep.ec; 2012 struct ifnet *ifp = adapter->ifp; 2013 2014 IOCTL_DEBUGOUT("ixgbe_set_multi: begin"); 2015 2016 mta = adapter->mta; 2017 bzero(mta, sizeof(u8) * IXGBE_ETH_LENGTH_OF_ADDRESS * 2018 MAX_NUM_MULTICAST_ADDRESSES); 2019 2020 fctrl = IXGBE_READ_REG(&adapter->hw, IXGBE_FCTRL); 2021 fctrl |= (IXGBE_FCTRL_UPE | IXGBE_FCTRL_MPE); 2022 if (ifp->if_flags & IFF_PROMISC) 2023 fctrl |= (IXGBE_FCTRL_UPE | IXGBE_FCTRL_MPE); 2024 else if (ifp->if_flags & IFF_ALLMULTI) { 2025 fctrl |= IXGBE_FCTRL_MPE; 2026 fctrl &= ~IXGBE_FCTRL_UPE; 2027 } else 2028 fctrl &= ~(IXGBE_FCTRL_UPE | IXGBE_FCTRL_MPE); 2029 2030 IXGBE_WRITE_REG(&adapter->hw, IXGBE_FCTRL, fctrl); 2031 2032 ETHER_FIRST_MULTI(step, ec, enm); 2033 while (enm != NULL) { 2034 if (memcmp(enm->enm_addrlo, enm->enm_addrhi, 2035 ETHER_ADDR_LEN) != 0) { 2036 fctrl |= IXGBE_FCTRL_MPE; 2037 IXGBE_WRITE_REG(&adapter->hw, IXGBE_FCTRL, fctrl); 2038 break; 2039 } 2040 bcopy(enm->enm_addrlo, 2041 &mta[mcnt * IXGBE_ETH_LENGTH_OF_ADDRESS], 2042 IXGBE_ETH_LENGTH_OF_ADDRESS); 2043 mcnt++; 2044 ETHER_NEXT_MULTI(step, enm); 2045 } 2046 2047 update_ptr = mta; 2048 ixgbe_update_mc_addr_list(&adapter->hw, 2049 update_ptr, mcnt, ixgbe_mc_array_itr); 2050 2051 return; 2052 } 2053 2054 /* 2055 * This is an iterator function now needed by the multicast 2056 * shared code. It simply feeds the shared code routine the 2057 * addresses in the array of ixgbe_set_multi() one by one. 2058 */ 2059 static u8 * 2060 ixgbe_mc_array_itr(struct ixgbe_hw *hw, u8 **update_ptr, u32 *vmdq) 2061 { 2062 u8 *addr = *update_ptr; 2063 u8 *newptr; 2064 *vmdq = 0; 2065 2066 newptr = addr + IXGBE_ETH_LENGTH_OF_ADDRESS; 2067 *update_ptr = newptr; 2068 return addr; 2069 } 2070 2071 2072 /********************************************************************* 2073 * Timer routine 2074 * 2075 * This routine checks for link status,updates statistics, 2076 * and runs the watchdog check. 2077 * 2078 **********************************************************************/ 2079 2080 static void 2081 ixgbe_local_timer1(void *arg) 2082 { 2083 struct adapter *adapter = arg; 2084 device_t dev = adapter->dev; 2085 struct tx_ring *txr = adapter->tx_rings; 2086 2087 KASSERT(mutex_owned(&adapter->core_mtx)); 2088 2089 /* Check for pluggable optics */ 2090 if (adapter->sfp_probe) 2091 if (!ixgbe_sfp_probe(adapter)) 2092 goto out; /* Nothing to do */ 2093 2094 ixgbe_update_link_status(adapter); 2095 ixgbe_update_stats_counters(adapter); 2096 2097 /* 2098 * If the interface has been paused 2099 * then don't do the watchdog check 2100 */ 2101 if (IXGBE_READ_REG(&adapter->hw, IXGBE_TFCS) & IXGBE_TFCS_TXOFF) 2102 goto out; 2103 2104 /* 2105 ** Check status on the TX queues for a hang 2106 */ 2107 for (int i = 0; i < adapter->num_queues; i++, txr++) 2108 if (txr->queue_status == IXGBE_QUEUE_HUNG) 2109 goto hung; 2110 2111 out: 2112 ixgbe_rearm_queues(adapter, adapter->que_mask); 2113 callout_reset(&adapter->timer, hz, ixgbe_local_timer, adapter); 2114 return; 2115 2116 hung: 2117 device_printf(adapter->dev, "Watchdog timeout -- resetting\n"); 2118 device_printf(dev,"Queue(%d) tdh = %d, hw tdt = %d\n", txr->me, 2119 IXGBE_READ_REG(&adapter->hw, IXGBE_TDH(txr->me)), 2120 IXGBE_READ_REG(&adapter->hw, IXGBE_TDT(txr->me))); 2121 device_printf(dev,"TX(%d) desc avail = %d," 2122 "Next TX to Clean = %d\n", 2123 txr->me, txr->tx_avail, txr->next_to_clean); 2124 adapter->ifp->if_flags &= ~IFF_RUNNING; 2125 adapter->watchdog_events.ev_count++; 2126 ixgbe_init_locked(adapter); 2127 } 2128 2129 static void 2130 ixgbe_local_timer(void *arg) 2131 { 2132 struct adapter *adapter = arg; 2133 2134 IXGBE_CORE_LOCK(adapter); 2135 ixgbe_local_timer1(adapter); 2136 IXGBE_CORE_UNLOCK(adapter); 2137 } 2138 2139 /* 2140 ** Note: this routine updates the OS on the link state 2141 ** the real check of the hardware only happens with 2142 ** a link interrupt. 2143 */ 2144 static void 2145 ixgbe_update_link_status(struct adapter *adapter) 2146 { 2147 struct ifnet *ifp = adapter->ifp; 2148 struct tx_ring *txr = adapter->tx_rings; 2149 device_t dev = adapter->dev; 2150 2151 2152 if (adapter->link_up){ 2153 if (adapter->link_active == FALSE) { 2154 if (bootverbose) 2155 device_printf(dev,"Link is up %d Gbps %s \n", 2156 ((adapter->link_speed == 128)? 10:1), 2157 "Full Duplex"); 2158 adapter->link_active = TRUE; 2159 if_link_state_change(ifp, LINK_STATE_UP); 2160 } 2161 } else { /* Link down */ 2162 if (adapter->link_active == TRUE) { 2163 if (bootverbose) 2164 device_printf(dev,"Link is Down\n"); 2165 if_link_state_change(ifp, LINK_STATE_DOWN); 2166 adapter->link_active = FALSE; 2167 for (int i = 0; i < adapter->num_queues; 2168 i++, txr++) 2169 txr->queue_status = IXGBE_QUEUE_IDLE; 2170 } 2171 } 2172 2173 return; 2174 } 2175 2176 2177 static void 2178 ixgbe_ifstop(struct ifnet *ifp, int disable) 2179 { 2180 struct adapter *adapter = ifp->if_softc; 2181 2182 IXGBE_CORE_LOCK(adapter); 2183 ixgbe_stop(adapter); 2184 IXGBE_CORE_UNLOCK(adapter); 2185 } 2186 2187 /********************************************************************* 2188 * 2189 * This routine disables all traffic on the adapter by issuing a 2190 * global reset on the MAC and deallocates TX/RX buffers. 2191 * 2192 **********************************************************************/ 2193 2194 static void 2195 ixgbe_stop(void *arg) 2196 { 2197 struct ifnet *ifp; 2198 struct adapter *adapter = arg; 2199 struct ixgbe_hw *hw = &adapter->hw; 2200 ifp = adapter->ifp; 2201 2202 KASSERT(mutex_owned(&adapter->core_mtx)); 2203 2204 INIT_DEBUGOUT("ixgbe_stop: begin\n"); 2205 ixgbe_disable_intr(adapter); 2206 2207 /* Tell the stack that the interface is no longer active */ 2208 ifp->if_flags &= ~(IFF_RUNNING | IFF_OACTIVE); 2209 2210 ixgbe_reset_hw(hw); 2211 hw->adapter_stopped = FALSE; 2212 ixgbe_stop_adapter(hw); 2213 /* Turn off the laser */ 2214 if (hw->phy.multispeed_fiber) 2215 ixgbe_disable_tx_laser(hw); 2216 callout_stop(&adapter->timer); 2217 2218 /* reprogram the RAR[0] in case user changed it. */ 2219 ixgbe_set_rar(&adapter->hw, 0, adapter->hw.mac.addr, 0, IXGBE_RAH_AV); 2220 2221 return; 2222 } 2223 2224 2225 /********************************************************************* 2226 * 2227 * Determine hardware revision. 2228 * 2229 **********************************************************************/ 2230 static void 2231 ixgbe_identify_hardware(struct adapter *adapter) 2232 { 2233 pcitag_t tag; 2234 pci_chipset_tag_t pc; 2235 pcireg_t subid, id; 2236 struct ixgbe_hw *hw = &adapter->hw; 2237 2238 pc = adapter->osdep.pc; 2239 tag = adapter->osdep.tag; 2240 2241 id = pci_conf_read(pc, tag, PCI_ID_REG); 2242 subid = pci_conf_read(pc, tag, PCI_SUBSYS_ID_REG); 2243 2244 /* Save off the information about this board */ 2245 hw->vendor_id = PCI_VENDOR(id); 2246 hw->device_id = PCI_PRODUCT(id); 2247 hw->revision_id = 2248 PCI_REVISION(pci_conf_read(pc, tag, PCI_CLASS_REG)); 2249 hw->subsystem_vendor_id = PCI_SUBSYS_VENDOR(subid); 2250 hw->subsystem_device_id = PCI_SUBSYS_ID(subid); 2251 2252 /* We need this here to set the num_segs below */ 2253 ixgbe_set_mac_type(hw); 2254 2255 /* Pick up the 82599 and VF settings */ 2256 if (hw->mac.type != ixgbe_mac_82598EB) { 2257 hw->phy.smart_speed = ixgbe_smart_speed; 2258 adapter->num_segs = IXGBE_82599_SCATTER; 2259 } else 2260 adapter->num_segs = IXGBE_82598_SCATTER; 2261 2262 return; 2263 } 2264 2265 /********************************************************************* 2266 * 2267 * Determine optic type 2268 * 2269 **********************************************************************/ 2270 static void 2271 ixgbe_setup_optics(struct adapter *adapter) 2272 { 2273 struct ixgbe_hw *hw = &adapter->hw; 2274 int layer; 2275 2276 layer = ixgbe_get_supported_physical_layer(hw); 2277 switch (layer) { 2278 case IXGBE_PHYSICAL_LAYER_10GBASE_T: 2279 adapter->optics = IFM_10G_T; 2280 break; 2281 case IXGBE_PHYSICAL_LAYER_1000BASE_T: 2282 adapter->optics = IFM_1000_T; 2283 break; 2284 case IXGBE_PHYSICAL_LAYER_10GBASE_LR: 2285 case IXGBE_PHYSICAL_LAYER_10GBASE_LRM: 2286 adapter->optics = IFM_10G_LR; 2287 break; 2288 case IXGBE_PHYSICAL_LAYER_10GBASE_SR: 2289 adapter->optics = IFM_10G_SR; 2290 break; 2291 case IXGBE_PHYSICAL_LAYER_10GBASE_KX4: 2292 case IXGBE_PHYSICAL_LAYER_10GBASE_CX4: 2293 adapter->optics = IFM_10G_CX4; 2294 break; 2295 case IXGBE_PHYSICAL_LAYER_SFP_PLUS_CU: 2296 adapter->optics = IFM_10G_TWINAX; 2297 break; 2298 case IXGBE_PHYSICAL_LAYER_1000BASE_KX: 2299 case IXGBE_PHYSICAL_LAYER_10GBASE_KR: 2300 case IXGBE_PHYSICAL_LAYER_10GBASE_XAUI: 2301 case IXGBE_PHYSICAL_LAYER_UNKNOWN: 2302 default: 2303 adapter->optics = IFM_ETHER | IFM_AUTO; 2304 break; 2305 } 2306 return; 2307 } 2308 2309 /********************************************************************* 2310 * 2311 * Setup the Legacy or MSI Interrupt handler 2312 * 2313 **********************************************************************/ 2314 static int 2315 ixgbe_allocate_legacy(struct adapter *adapter, const struct pci_attach_args *pa) 2316 { 2317 device_t dev = adapter->dev; 2318 struct ix_queue *que = adapter->queues; 2319 char intrbuf[PCI_INTRSTR_LEN]; 2320 #if 0 2321 int rid = 0; 2322 2323 /* MSI RID at 1 */ 2324 if (adapter->msix == 1) 2325 rid = 1; 2326 #endif 2327 2328 /* We allocate a single interrupt resource */ 2329 if (pci_intr_map(pa, &adapter->osdep.ih) != 0) { 2330 aprint_error_dev(dev, "unable to map interrupt\n"); 2331 return ENXIO; 2332 } else { 2333 aprint_normal_dev(dev, "interrupting at %s\n", 2334 pci_intr_string(adapter->osdep.pc, adapter->osdep.ih, 2335 intrbuf, sizeof(intrbuf))); 2336 } 2337 2338 /* 2339 * Try allocating a fast interrupt and the associated deferred 2340 * processing contexts. 2341 */ 2342 que->que_si = softint_establish(SOFTINT_NET, ixgbe_handle_que, que); 2343 2344 /* Tasklets for Link, SFP and Multispeed Fiber */ 2345 adapter->link_si = 2346 softint_establish(SOFTINT_NET, ixgbe_handle_link, adapter); 2347 adapter->mod_si = 2348 softint_establish(SOFTINT_NET, ixgbe_handle_mod, adapter); 2349 adapter->msf_si = 2350 softint_establish(SOFTINT_NET, ixgbe_handle_msf, adapter); 2351 2352 #ifdef IXGBE_FDIR 2353 adapter->fdir_si = 2354 softint_establish(SOFTINT_NET, ixgbe_reinit_fdir, adapter); 2355 #endif 2356 if (que->que_si == NULL || 2357 adapter->link_si == NULL || 2358 adapter->mod_si == NULL || 2359 #ifdef IXGBE_FDIR 2360 adapter->fdir_si == NULL || 2361 #endif 2362 adapter->msf_si == NULL) { 2363 aprint_error_dev(dev, 2364 "could not establish software interrupts\n"); 2365 return ENXIO; 2366 } 2367 2368 adapter->osdep.intr = pci_intr_establish(adapter->osdep.pc, 2369 adapter->osdep.ih, IPL_NET, ixgbe_legacy_irq, que); 2370 if (adapter->osdep.intr == NULL) { 2371 aprint_error_dev(dev, "failed to register interrupt handler\n"); 2372 softint_disestablish(que->que_si); 2373 softint_disestablish(adapter->link_si); 2374 softint_disestablish(adapter->mod_si); 2375 softint_disestablish(adapter->msf_si); 2376 #ifdef IXGBE_FDIR 2377 softint_disestablish(adapter->fdir_si); 2378 #endif 2379 return ENXIO; 2380 } 2381 /* For simplicity in the handlers */ 2382 adapter->que_mask = IXGBE_EIMS_ENABLE_MASK; 2383 2384 return (0); 2385 } 2386 2387 2388 /********************************************************************* 2389 * 2390 * Setup MSIX Interrupt resources and handlers 2391 * 2392 **********************************************************************/ 2393 static int 2394 ixgbe_allocate_msix(struct adapter *adapter, const struct pci_attach_args *pa) 2395 { 2396 #if !defined(NETBSD_MSI_OR_MSIX) 2397 return 0; 2398 #else 2399 device_t dev = adapter->dev; 2400 struct ix_queue *que = adapter->queues; 2401 int error, rid, vector = 0; 2402 2403 for (int i = 0; i < adapter->num_queues; i++, vector++, que++) { 2404 rid = vector + 1; 2405 que->res = bus_alloc_resource_any(dev, SYS_RES_IRQ, &rid, 2406 RF_SHAREABLE | RF_ACTIVE); 2407 if (que->res == NULL) { 2408 aprint_error_dev(dev,"Unable to allocate" 2409 " bus resource: que interrupt [%d]\n", vector); 2410 return (ENXIO); 2411 } 2412 /* Set the handler function */ 2413 error = bus_setup_intr(dev, que->res, 2414 INTR_TYPE_NET | INTR_MPSAFE, NULL, 2415 ixgbe_msix_que, que, &que->tag); 2416 if (error) { 2417 que->res = NULL; 2418 aprint_error_dev(dev, 2419 "Failed to register QUE handler\n"); 2420 return error; 2421 } 2422 #if __FreeBSD_version >= 800504 2423 bus_describe_intr(dev, que->res, que->tag, "que %d", i); 2424 #endif 2425 que->msix = vector; 2426 adapter->que_mask |= (u64)(1 << que->msix); 2427 /* 2428 ** Bind the msix vector, and thus the 2429 ** ring to the corresponding cpu. 2430 */ 2431 if (adapter->num_queues > 1) 2432 bus_bind_intr(dev, que->res, i); 2433 2434 que->que_si = softint_establish(ixgbe_handle_que, que); 2435 if (que->que_si == NULL) { 2436 aprint_error_dev(dev, 2437 "could not establish software interrupt\n"); 2438 } 2439 } 2440 2441 /* and Link */ 2442 rid = vector + 1; 2443 adapter->res = bus_alloc_resource_any(dev, 2444 SYS_RES_IRQ, &rid, RF_SHAREABLE | RF_ACTIVE); 2445 if (!adapter->res) { 2446 aprint_error_dev(dev,"Unable to allocate bus resource: " 2447 "Link interrupt [%d]\n", rid); 2448 return (ENXIO); 2449 } 2450 /* Set the link handler function */ 2451 error = bus_setup_intr(dev, adapter->res, 2452 INTR_TYPE_NET | INTR_MPSAFE, NULL, 2453 ixgbe_msix_link, adapter, &adapter->tag); 2454 if (error) { 2455 adapter->res = NULL; 2456 aprint_error_dev(dev, "Failed to register LINK handler\n"); 2457 return (error); 2458 } 2459 #if __FreeBSD_version >= 800504 2460 bus_describe_intr(dev, adapter->res, adapter->tag, "link"); 2461 #endif 2462 adapter->linkvec = vector; 2463 /* Tasklets for Link, SFP and Multispeed Fiber */ 2464 adapter->link_si = 2465 softint_establish(SOFTINT_NET, ixgbe_handle_link, adapter); 2466 adapter->mod_si = 2467 softint_establish(SOFTINT_NET, ixgbe_handle_mod, adapter); 2468 adapter->msf_si = 2469 softint_establish(SOFTINT_NET, ixgbe_handle_msf, adapter); 2470 #ifdef IXGBE_FDIR 2471 adapter->fdir_si = 2472 softint_establish(SOFTINT_NET, ixgbe_reinit_fdir, adapter); 2473 #endif 2474 2475 return (0); 2476 #endif 2477 } 2478 2479 /* 2480 * Setup Either MSI/X or MSI 2481 */ 2482 static int 2483 ixgbe_setup_msix(struct adapter *adapter) 2484 { 2485 #if !defined(NETBSD_MSI_OR_MSIX) 2486 return 0; 2487 #else 2488 device_t dev = adapter->dev; 2489 int rid, want, queues, msgs; 2490 2491 /* Override by tuneable */ 2492 if (ixgbe_enable_msix == 0) 2493 goto msi; 2494 2495 /* First try MSI/X */ 2496 rid = PCI_BAR(MSIX_82598_BAR); 2497 adapter->msix_mem = bus_alloc_resource_any(dev, 2498 SYS_RES_MEMORY, &rid, RF_ACTIVE); 2499 if (!adapter->msix_mem) { 2500 rid += 4; /* 82599 maps in higher BAR */ 2501 adapter->msix_mem = bus_alloc_resource_any(dev, 2502 SYS_RES_MEMORY, &rid, RF_ACTIVE); 2503 } 2504 if (!adapter->msix_mem) { 2505 /* May not be enabled */ 2506 device_printf(adapter->dev, 2507 "Unable to map MSIX table \n"); 2508 goto msi; 2509 } 2510 2511 msgs = pci_msix_count(dev); 2512 if (msgs == 0) { /* system has msix disabled */ 2513 bus_release_resource(dev, SYS_RES_MEMORY, 2514 rid, adapter->msix_mem); 2515 adapter->msix_mem = NULL; 2516 goto msi; 2517 } 2518 2519 /* Figure out a reasonable auto config value */ 2520 queues = (mp_ncpus > (msgs-1)) ? (msgs-1) : mp_ncpus; 2521 2522 if (ixgbe_num_queues != 0) 2523 queues = ixgbe_num_queues; 2524 /* Set max queues to 8 when autoconfiguring */ 2525 else if ((ixgbe_num_queues == 0) && (queues > 8)) 2526 queues = 8; 2527 2528 /* 2529 ** Want one vector (RX/TX pair) per queue 2530 ** plus an additional for Link. 2531 */ 2532 want = queues + 1; 2533 if (msgs >= want) 2534 msgs = want; 2535 else { 2536 device_printf(adapter->dev, 2537 "MSIX Configuration Problem, " 2538 "%d vectors but %d queues wanted!\n", 2539 msgs, want); 2540 return (0); /* Will go to Legacy setup */ 2541 } 2542 if ((msgs) && pci_alloc_msix(dev, &msgs) == 0) { 2543 device_printf(adapter->dev, 2544 "Using MSIX interrupts with %d vectors\n", msgs); 2545 adapter->num_queues = queues; 2546 return (msgs); 2547 } 2548 msi: 2549 msgs = pci_msi_count(dev); 2550 if (msgs == 1 && pci_alloc_msi(dev, &msgs) == 0) 2551 device_printf(adapter->dev,"Using an MSI interrupt\n"); 2552 else 2553 device_printf(adapter->dev,"Using a Legacy interrupt\n"); 2554 return (msgs); 2555 #endif 2556 } 2557 2558 2559 static int 2560 ixgbe_allocate_pci_resources(struct adapter *adapter, const struct pci_attach_args *pa) 2561 { 2562 pcireg_t memtype; 2563 device_t dev = adapter->dev; 2564 bus_addr_t addr; 2565 int flags; 2566 2567 memtype = pci_mapreg_type(pa->pa_pc, pa->pa_tag, PCI_BAR(0)); 2568 switch (memtype) { 2569 case PCI_MAPREG_TYPE_MEM | PCI_MAPREG_MEM_TYPE_32BIT: 2570 case PCI_MAPREG_TYPE_MEM | PCI_MAPREG_MEM_TYPE_64BIT: 2571 adapter->osdep.mem_bus_space_tag = pa->pa_memt; 2572 if (pci_mapreg_info(pa->pa_pc, pa->pa_tag, PCI_BAR(0), 2573 memtype, &addr, &adapter->osdep.mem_size, &flags) != 0) 2574 goto map_err; 2575 if ((flags & BUS_SPACE_MAP_PREFETCHABLE) != 0) { 2576 aprint_normal_dev(dev, "clearing prefetchable bit\n"); 2577 flags &= ~BUS_SPACE_MAP_PREFETCHABLE; 2578 } 2579 if (bus_space_map(adapter->osdep.mem_bus_space_tag, addr, 2580 adapter->osdep.mem_size, flags, 2581 &adapter->osdep.mem_bus_space_handle) != 0) { 2582 map_err: 2583 adapter->osdep.mem_size = 0; 2584 aprint_error_dev(dev, "unable to map BAR0\n"); 2585 return ENXIO; 2586 } 2587 break; 2588 default: 2589 aprint_error_dev(dev, "unexpected type on BAR0\n"); 2590 return ENXIO; 2591 } 2592 2593 /* Legacy defaults */ 2594 adapter->num_queues = 1; 2595 adapter->hw.back = &adapter->osdep; 2596 2597 /* 2598 ** Now setup MSI or MSI/X, should 2599 ** return us the number of supported 2600 ** vectors. (Will be 1 for MSI) 2601 */ 2602 adapter->msix = ixgbe_setup_msix(adapter); 2603 return (0); 2604 } 2605 2606 static void 2607 ixgbe_free_pci_resources(struct adapter * adapter) 2608 { 2609 #if defined(NETBSD_MSI_OR_MSIX) 2610 struct ix_queue *que = adapter->queues; 2611 device_t dev = adapter->dev; 2612 #endif 2613 int rid; 2614 2615 #if defined(NETBSD_MSI_OR_MSIX) 2616 int memrid; 2617 if (adapter->hw.mac.type == ixgbe_mac_82598EB) 2618 memrid = PCI_BAR(MSIX_82598_BAR); 2619 else 2620 memrid = PCI_BAR(MSIX_82599_BAR); 2621 2622 /* 2623 ** There is a slight possibility of a failure mode 2624 ** in attach that will result in entering this function 2625 ** before interrupt resources have been initialized, and 2626 ** in that case we do not want to execute the loops below 2627 ** We can detect this reliably by the state of the adapter 2628 ** res pointer. 2629 */ 2630 if (adapter->res == NULL) 2631 goto mem; 2632 2633 /* 2634 ** Release all msix queue resources: 2635 */ 2636 for (int i = 0; i < adapter->num_queues; i++, que++) { 2637 rid = que->msix + 1; 2638 if (que->tag != NULL) { 2639 bus_teardown_intr(dev, que->res, que->tag); 2640 que->tag = NULL; 2641 } 2642 if (que->res != NULL) 2643 bus_release_resource(dev, SYS_RES_IRQ, rid, que->res); 2644 } 2645 #endif 2646 2647 /* Clean the Legacy or Link interrupt last */ 2648 if (adapter->linkvec) /* we are doing MSIX */ 2649 rid = adapter->linkvec + 1; 2650 else 2651 (adapter->msix != 0) ? (rid = 1):(rid = 0); 2652 2653 pci_intr_disestablish(adapter->osdep.pc, adapter->osdep.intr); 2654 adapter->osdep.intr = NULL; 2655 2656 #if defined(NETBSD_MSI_OR_MSIX) 2657 mem: 2658 if (adapter->msix) 2659 pci_release_msi(dev); 2660 2661 if (adapter->msix_mem != NULL) 2662 bus_release_resource(dev, SYS_RES_MEMORY, 2663 memrid, adapter->msix_mem); 2664 #endif 2665 2666 if (adapter->osdep.mem_size != 0) { 2667 bus_space_unmap(adapter->osdep.mem_bus_space_tag, 2668 adapter->osdep.mem_bus_space_handle, 2669 adapter->osdep.mem_size); 2670 } 2671 2672 return; 2673 } 2674 2675 /********************************************************************* 2676 * 2677 * Setup networking device structure and register an interface. 2678 * 2679 **********************************************************************/ 2680 static int 2681 ixgbe_setup_interface(device_t dev, struct adapter *adapter) 2682 { 2683 struct ethercom *ec = &adapter->osdep.ec; 2684 struct ixgbe_hw *hw = &adapter->hw; 2685 struct ifnet *ifp; 2686 2687 INIT_DEBUGOUT("ixgbe_setup_interface: begin"); 2688 2689 ifp = adapter->ifp = &ec->ec_if; 2690 strlcpy(ifp->if_xname, device_xname(dev), IFNAMSIZ); 2691 ifp->if_baudrate = 1000000000; 2692 ifp->if_init = ixgbe_init; 2693 ifp->if_stop = ixgbe_ifstop; 2694 ifp->if_softc = adapter; 2695 ifp->if_flags = IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST; 2696 ifp->if_ioctl = ixgbe_ioctl; 2697 ifp->if_start = ixgbe_start; 2698 #if __FreeBSD_version >= 800000 2699 ifp->if_transmit = ixgbe_mq_start; 2700 ifp->if_qflush = ixgbe_qflush; 2701 #endif 2702 ifp->if_snd.ifq_maxlen = adapter->num_tx_desc - 2; 2703 2704 if_attach(ifp); 2705 ether_ifattach(ifp, adapter->hw.mac.addr); 2706 ether_set_ifflags_cb(ec, ixgbe_ifflags_cb); 2707 2708 adapter->max_frame_size = 2709 ifp->if_mtu + ETHER_HDR_LEN + ETHER_CRC_LEN; 2710 2711 /* 2712 * Tell the upper layer(s) we support long frames. 2713 */ 2714 ifp->if_hdrlen = sizeof(struct ether_vlan_header); 2715 2716 ifp->if_capabilities |= IFCAP_HWCSUM | IFCAP_TSOv4; 2717 ifp->if_capenable = 0; 2718 2719 ec->ec_capabilities |= ETHERCAP_VLAN_HWCSUM; 2720 ec->ec_capabilities |= ETHERCAP_JUMBO_MTU; 2721 ec->ec_capabilities |= ETHERCAP_VLAN_HWTAGGING 2722 | ETHERCAP_VLAN_MTU; 2723 ec->ec_capenable = ec->ec_capabilities; 2724 2725 /* Don't enable LRO by default */ 2726 ifp->if_capabilities |= IFCAP_LRO; 2727 2728 /* 2729 ** Don't turn this on by default, if vlans are 2730 ** created on another pseudo device (eg. lagg) 2731 ** then vlan events are not passed thru, breaking 2732 ** operation, but with HW FILTER off it works. If 2733 ** using vlans directly on the ixgbe driver you can 2734 ** enable this and get full hardware tag filtering. 2735 */ 2736 ec->ec_capabilities |= ETHERCAP_VLAN_HWFILTER; 2737 2738 /* 2739 * Specify the media types supported by this adapter and register 2740 * callbacks to update media and link information 2741 */ 2742 ifmedia_init(&adapter->media, IFM_IMASK, ixgbe_media_change, 2743 ixgbe_media_status); 2744 ifmedia_add(&adapter->media, IFM_ETHER | adapter->optics, 0, NULL); 2745 ifmedia_set(&adapter->media, IFM_ETHER | adapter->optics); 2746 if (hw->device_id == IXGBE_DEV_ID_82598AT) { 2747 ifmedia_add(&adapter->media, 2748 IFM_ETHER | IFM_1000_T | IFM_FDX, 0, NULL); 2749 ifmedia_add(&adapter->media, 2750 IFM_ETHER | IFM_1000_T, 0, NULL); 2751 } 2752 ifmedia_add(&adapter->media, IFM_ETHER | IFM_AUTO, 0, NULL); 2753 ifmedia_set(&adapter->media, IFM_ETHER | IFM_AUTO); 2754 2755 return (0); 2756 } 2757 2758 static void 2759 ixgbe_config_link(struct adapter *adapter) 2760 { 2761 struct ixgbe_hw *hw = &adapter->hw; 2762 u32 autoneg, err = 0; 2763 bool sfp, negotiate; 2764 2765 sfp = ixgbe_is_sfp(hw); 2766 2767 if (sfp) { 2768 void *ip; 2769 2770 if (hw->phy.multispeed_fiber) { 2771 hw->mac.ops.setup_sfp(hw); 2772 ixgbe_enable_tx_laser(hw); 2773 ip = adapter->msf_si; 2774 } else { 2775 ip = adapter->mod_si; 2776 } 2777 2778 kpreempt_disable(); 2779 softint_schedule(ip); 2780 kpreempt_enable(); 2781 } else { 2782 if (hw->mac.ops.check_link) 2783 err = ixgbe_check_link(hw, &autoneg, 2784 &adapter->link_up, FALSE); 2785 if (err) 2786 goto out; 2787 autoneg = hw->phy.autoneg_advertised; 2788 if ((!autoneg) && (hw->mac.ops.get_link_capabilities)) 2789 err = hw->mac.ops.get_link_capabilities(hw, 2790 &autoneg, &negotiate); 2791 else 2792 negotiate = 0; 2793 if (err) 2794 goto out; 2795 if (hw->mac.ops.setup_link) 2796 err = hw->mac.ops.setup_link(hw, autoneg, 2797 negotiate, adapter->link_up); 2798 } 2799 out: 2800 return; 2801 } 2802 2803 /******************************************************************** 2804 * Manage DMA'able memory. 2805 *******************************************************************/ 2806 2807 static int 2808 ixgbe_dma_malloc(struct adapter *adapter, const bus_size_t size, 2809 struct ixgbe_dma_alloc *dma, const int mapflags) 2810 { 2811 device_t dev = adapter->dev; 2812 int r, rsegs; 2813 2814 r = ixgbe_dma_tag_create(adapter->osdep.dmat, /* parent */ 2815 DBA_ALIGN, 0, /* alignment, bounds */ 2816 size, /* maxsize */ 2817 1, /* nsegments */ 2818 size, /* maxsegsize */ 2819 BUS_DMA_ALLOCNOW, /* flags */ 2820 &dma->dma_tag); 2821 if (r != 0) { 2822 aprint_error_dev(dev, 2823 "%s: ixgbe_dma_tag_create failed; error %d\n", __func__, r); 2824 goto fail_0; 2825 } 2826 2827 r = bus_dmamem_alloc(dma->dma_tag->dt_dmat, 2828 size, 2829 dma->dma_tag->dt_alignment, 2830 dma->dma_tag->dt_boundary, 2831 &dma->dma_seg, 1, &rsegs, BUS_DMA_NOWAIT); 2832 if (r != 0) { 2833 aprint_error_dev(dev, 2834 "%s: bus_dmamem_alloc failed; error %d\n", __func__, r); 2835 goto fail_1; 2836 } 2837 2838 r = bus_dmamem_map(dma->dma_tag->dt_dmat, &dma->dma_seg, rsegs, 2839 size, &dma->dma_vaddr, BUS_DMA_NOWAIT); 2840 if (r != 0) { 2841 aprint_error_dev(dev, "%s: bus_dmamem_map failed; error %d\n", 2842 __func__, r); 2843 goto fail_2; 2844 } 2845 2846 r = ixgbe_dmamap_create(dma->dma_tag, 0, &dma->dma_map); 2847 if (r != 0) { 2848 aprint_error_dev(dev, "%s: bus_dmamem_map failed; error %d\n", 2849 __func__, r); 2850 goto fail_3; 2851 } 2852 2853 r = bus_dmamap_load(dma->dma_tag->dt_dmat, dma->dma_map, dma->dma_vaddr, 2854 size, 2855 NULL, 2856 mapflags | BUS_DMA_NOWAIT); 2857 if (r != 0) { 2858 aprint_error_dev(dev, "%s: bus_dmamap_load failed; error %d\n", 2859 __func__, r); 2860 goto fail_4; 2861 } 2862 dma->dma_paddr = dma->dma_map->dm_segs[0].ds_addr; 2863 dma->dma_size = size; 2864 return 0; 2865 fail_4: 2866 ixgbe_dmamap_destroy(dma->dma_tag, dma->dma_map); 2867 fail_3: 2868 bus_dmamem_unmap(dma->dma_tag->dt_dmat, dma->dma_vaddr, size); 2869 fail_2: 2870 bus_dmamem_free(dma->dma_tag->dt_dmat, &dma->dma_seg, rsegs); 2871 fail_1: 2872 ixgbe_dma_tag_destroy(dma->dma_tag); 2873 fail_0: 2874 return r; 2875 } 2876 2877 static void 2878 ixgbe_dma_free(struct adapter *adapter, struct ixgbe_dma_alloc *dma) 2879 { 2880 bus_dmamap_sync(dma->dma_tag->dt_dmat, dma->dma_map, 0, dma->dma_size, 2881 BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE); 2882 ixgbe_dmamap_unload(dma->dma_tag, dma->dma_map); 2883 bus_dmamem_free(dma->dma_tag->dt_dmat, &dma->dma_seg, 1); 2884 ixgbe_dma_tag_destroy(dma->dma_tag); 2885 } 2886 2887 2888 /********************************************************************* 2889 * 2890 * Allocate memory for the transmit and receive rings, and then 2891 * the descriptors associated with each, called only once at attach. 2892 * 2893 **********************************************************************/ 2894 static int 2895 ixgbe_allocate_queues(struct adapter *adapter) 2896 { 2897 device_t dev = adapter->dev; 2898 struct ix_queue *que; 2899 struct tx_ring *txr; 2900 struct rx_ring *rxr; 2901 int rsize, tsize, error = IXGBE_SUCCESS; 2902 int txconf = 0, rxconf = 0; 2903 2904 /* First allocate the top level queue structs */ 2905 if (!(adapter->queues = 2906 (struct ix_queue *) malloc(sizeof(struct ix_queue) * 2907 adapter->num_queues, M_DEVBUF, M_NOWAIT | M_ZERO))) { 2908 aprint_error_dev(dev, "Unable to allocate queue memory\n"); 2909 error = ENOMEM; 2910 goto fail; 2911 } 2912 2913 /* First allocate the TX ring struct memory */ 2914 if (!(adapter->tx_rings = 2915 (struct tx_ring *) malloc(sizeof(struct tx_ring) * 2916 adapter->num_queues, M_DEVBUF, M_NOWAIT | M_ZERO))) { 2917 aprint_error_dev(dev, "Unable to allocate TX ring memory\n"); 2918 error = ENOMEM; 2919 goto tx_fail; 2920 } 2921 2922 /* Next allocate the RX */ 2923 if (!(adapter->rx_rings = 2924 (struct rx_ring *) malloc(sizeof(struct rx_ring) * 2925 adapter->num_queues, M_DEVBUF, M_NOWAIT | M_ZERO))) { 2926 aprint_error_dev(dev, "Unable to allocate RX ring memory\n"); 2927 error = ENOMEM; 2928 goto rx_fail; 2929 } 2930 2931 /* For the ring itself */ 2932 tsize = roundup2(adapter->num_tx_desc * 2933 sizeof(union ixgbe_adv_tx_desc), DBA_ALIGN); 2934 2935 /* 2936 * Now set up the TX queues, txconf is needed to handle the 2937 * possibility that things fail midcourse and we need to 2938 * undo memory gracefully 2939 */ 2940 for (int i = 0; i < adapter->num_queues; i++, txconf++) { 2941 /* Set up some basics */ 2942 txr = &adapter->tx_rings[i]; 2943 txr->adapter = adapter; 2944 txr->me = i; 2945 2946 /* Initialize the TX side lock */ 2947 snprintf(txr->mtx_name, sizeof(txr->mtx_name), "%s:tx(%d)", 2948 device_xname(dev), txr->me); 2949 mutex_init(&txr->tx_mtx, MUTEX_DEFAULT, IPL_NET); 2950 2951 if (ixgbe_dma_malloc(adapter, tsize, 2952 &txr->txdma, BUS_DMA_NOWAIT)) { 2953 aprint_error_dev(dev, 2954 "Unable to allocate TX Descriptor memory\n"); 2955 error = ENOMEM; 2956 goto err_tx_desc; 2957 } 2958 txr->tx_base = (union ixgbe_adv_tx_desc *)txr->txdma.dma_vaddr; 2959 bzero((void *)txr->tx_base, tsize); 2960 2961 /* Now allocate transmit buffers for the ring */ 2962 if (ixgbe_allocate_transmit_buffers(txr)) { 2963 aprint_error_dev(dev, 2964 "Critical Failure setting up transmit buffers\n"); 2965 error = ENOMEM; 2966 goto err_tx_desc; 2967 } 2968 #if __FreeBSD_version >= 800000 2969 /* Allocate a buf ring */ 2970 txr->br = buf_ring_alloc(IXGBE_BR_SIZE, M_DEVBUF, 2971 M_WAITOK, &txr->tx_mtx); 2972 if (txr->br == NULL) { 2973 aprint_error_dev(dev, 2974 "Critical Failure setting up buf ring\n"); 2975 error = ENOMEM; 2976 goto err_tx_desc; 2977 } 2978 #endif 2979 } 2980 2981 /* 2982 * Next the RX queues... 2983 */ 2984 rsize = roundup2(adapter->num_rx_desc * 2985 sizeof(union ixgbe_adv_rx_desc), DBA_ALIGN); 2986 for (int i = 0; i < adapter->num_queues; i++, rxconf++) { 2987 rxr = &adapter->rx_rings[i]; 2988 /* Set up some basics */ 2989 rxr->adapter = adapter; 2990 rxr->me = i; 2991 2992 /* Initialize the RX side lock */ 2993 snprintf(rxr->mtx_name, sizeof(rxr->mtx_name), "%s:rx(%d)", 2994 device_xname(dev), rxr->me); 2995 mutex_init(&rxr->rx_mtx, MUTEX_DEFAULT, IPL_NET); 2996 2997 if (ixgbe_dma_malloc(adapter, rsize, 2998 &rxr->rxdma, BUS_DMA_NOWAIT)) { 2999 aprint_error_dev(dev, 3000 "Unable to allocate RxDescriptor memory\n"); 3001 error = ENOMEM; 3002 goto err_rx_desc; 3003 } 3004 rxr->rx_base = (union ixgbe_adv_rx_desc *)rxr->rxdma.dma_vaddr; 3005 bzero((void *)rxr->rx_base, rsize); 3006 3007 /* Allocate receive buffers for the ring*/ 3008 if (ixgbe_allocate_receive_buffers(rxr)) { 3009 aprint_error_dev(dev, 3010 "Critical Failure setting up receive buffers\n"); 3011 error = ENOMEM; 3012 goto err_rx_desc; 3013 } 3014 } 3015 3016 /* 3017 ** Finally set up the queue holding structs 3018 */ 3019 for (int i = 0; i < adapter->num_queues; i++) { 3020 que = &adapter->queues[i]; 3021 que->adapter = adapter; 3022 que->txr = &adapter->tx_rings[i]; 3023 que->rxr = &adapter->rx_rings[i]; 3024 } 3025 3026 return (0); 3027 3028 err_rx_desc: 3029 for (rxr = adapter->rx_rings; rxconf > 0; rxr++, rxconf--) 3030 ixgbe_dma_free(adapter, &rxr->rxdma); 3031 err_tx_desc: 3032 for (txr = adapter->tx_rings; txconf > 0; txr++, txconf--) 3033 ixgbe_dma_free(adapter, &txr->txdma); 3034 free(adapter->rx_rings, M_DEVBUF); 3035 rx_fail: 3036 free(adapter->tx_rings, M_DEVBUF); 3037 tx_fail: 3038 free(adapter->queues, M_DEVBUF); 3039 fail: 3040 return (error); 3041 } 3042 3043 /********************************************************************* 3044 * 3045 * Allocate memory for tx_buffer structures. The tx_buffer stores all 3046 * the information needed to transmit a packet on the wire. This is 3047 * called only once at attach, setup is done every reset. 3048 * 3049 **********************************************************************/ 3050 static int 3051 ixgbe_allocate_transmit_buffers(struct tx_ring *txr) 3052 { 3053 struct adapter *adapter = txr->adapter; 3054 device_t dev = adapter->dev; 3055 struct ixgbe_tx_buf *txbuf; 3056 int error, i; 3057 3058 /* 3059 * Setup DMA descriptor areas. 3060 */ 3061 if ((error = ixgbe_dma_tag_create(adapter->osdep.dmat, /* parent */ 3062 1, 0, /* alignment, bounds */ 3063 IXGBE_TSO_SIZE, /* maxsize */ 3064 adapter->num_segs, /* nsegments */ 3065 PAGE_SIZE, /* maxsegsize */ 3066 0, /* flags */ 3067 &txr->txtag))) { 3068 aprint_error_dev(dev,"Unable to allocate TX DMA tag\n"); 3069 goto fail; 3070 } 3071 3072 if (!(txr->tx_buffers = 3073 (struct ixgbe_tx_buf *) malloc(sizeof(struct ixgbe_tx_buf) * 3074 adapter->num_tx_desc, M_DEVBUF, M_NOWAIT | M_ZERO))) { 3075 aprint_error_dev(dev, "Unable to allocate tx_buffer memory\n"); 3076 error = ENOMEM; 3077 goto fail; 3078 } 3079 3080 /* Create the descriptor buffer dma maps */ 3081 txbuf = txr->tx_buffers; 3082 for (i = 0; i < adapter->num_tx_desc; i++, txbuf++) { 3083 error = ixgbe_dmamap_create(txr->txtag, 0, &txbuf->map); 3084 if (error != 0) { 3085 aprint_error_dev(dev, "Unable to create TX DMA map\n"); 3086 goto fail; 3087 } 3088 } 3089 3090 return 0; 3091 fail: 3092 /* We free all, it handles case where we are in the middle */ 3093 ixgbe_free_transmit_structures(adapter); 3094 return (error); 3095 } 3096 3097 /********************************************************************* 3098 * 3099 * Initialize a transmit ring. 3100 * 3101 **********************************************************************/ 3102 static void 3103 ixgbe_setup_transmit_ring(struct tx_ring *txr) 3104 { 3105 struct adapter *adapter = txr->adapter; 3106 struct ixgbe_tx_buf *txbuf; 3107 int i; 3108 #ifdef DEV_NETMAP 3109 struct netmap_adapter *na = NA(adapter->ifp); 3110 struct netmap_slot *slot; 3111 #endif /* DEV_NETMAP */ 3112 3113 /* Clear the old ring contents */ 3114 IXGBE_TX_LOCK(txr); 3115 #ifdef DEV_NETMAP 3116 /* 3117 * (under lock): if in netmap mode, do some consistency 3118 * checks and set slot to entry 0 of the netmap ring. 3119 */ 3120 slot = netmap_reset(na, NR_TX, txr->me, 0); 3121 #endif /* DEV_NETMAP */ 3122 bzero((void *)txr->tx_base, 3123 (sizeof(union ixgbe_adv_tx_desc)) * adapter->num_tx_desc); 3124 /* Reset indices */ 3125 txr->next_avail_desc = 0; 3126 txr->next_to_clean = 0; 3127 3128 /* Free any existing tx buffers. */ 3129 txbuf = txr->tx_buffers; 3130 for (i = 0; i < adapter->num_tx_desc; i++, txbuf++) { 3131 if (txbuf->m_head != NULL) { 3132 bus_dmamap_sync(txr->txtag->dt_dmat, txbuf->map, 3133 0, txbuf->m_head->m_pkthdr.len, 3134 BUS_DMASYNC_POSTWRITE); 3135 ixgbe_dmamap_unload(txr->txtag, txbuf->map); 3136 m_freem(txbuf->m_head); 3137 txbuf->m_head = NULL; 3138 } 3139 #ifdef DEV_NETMAP 3140 /* 3141 * In netmap mode, set the map for the packet buffer. 3142 * NOTE: Some drivers (not this one) also need to set 3143 * the physical buffer address in the NIC ring. 3144 * Slots in the netmap ring (indexed by "si") are 3145 * kring->nkr_hwofs positions "ahead" wrt the 3146 * corresponding slot in the NIC ring. In some drivers 3147 * (not here) nkr_hwofs can be negative. When computing 3148 * si = i + kring->nkr_hwofs make sure to handle wraparounds. 3149 */ 3150 if (slot) { 3151 int si = i + na->tx_rings[txr->me].nkr_hwofs; 3152 3153 if (si >= na->num_tx_desc) 3154 si -= na->num_tx_desc; 3155 netmap_load_map(txr->txtag, txbuf->map, NMB(slot + si)); 3156 } 3157 #endif /* DEV_NETMAP */ 3158 /* Clear the EOP index */ 3159 txbuf->eop_index = -1; 3160 } 3161 3162 #ifdef IXGBE_FDIR 3163 /* Set the rate at which we sample packets */ 3164 if (adapter->hw.mac.type != ixgbe_mac_82598EB) 3165 txr->atr_sample = atr_sample_rate; 3166 #endif 3167 3168 /* Set number of descriptors available */ 3169 txr->tx_avail = adapter->num_tx_desc; 3170 3171 ixgbe_dmamap_sync(txr->txdma.dma_tag, txr->txdma.dma_map, 3172 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); 3173 IXGBE_TX_UNLOCK(txr); 3174 } 3175 3176 /********************************************************************* 3177 * 3178 * Initialize all transmit rings. 3179 * 3180 **********************************************************************/ 3181 static int 3182 ixgbe_setup_transmit_structures(struct adapter *adapter) 3183 { 3184 struct tx_ring *txr = adapter->tx_rings; 3185 3186 for (int i = 0; i < adapter->num_queues; i++, txr++) 3187 ixgbe_setup_transmit_ring(txr); 3188 3189 return (0); 3190 } 3191 3192 /********************************************************************* 3193 * 3194 * Enable transmit unit. 3195 * 3196 **********************************************************************/ 3197 static void 3198 ixgbe_initialize_transmit_units(struct adapter *adapter) 3199 { 3200 struct tx_ring *txr = adapter->tx_rings; 3201 struct ixgbe_hw *hw = &adapter->hw; 3202 3203 /* Setup the Base and Length of the Tx Descriptor Ring */ 3204 3205 for (int i = 0; i < adapter->num_queues; i++, txr++) { 3206 u64 tdba = txr->txdma.dma_paddr; 3207 u32 txctrl; 3208 3209 IXGBE_WRITE_REG(hw, IXGBE_TDBAL(i), 3210 (tdba & 0x00000000ffffffffULL)); 3211 IXGBE_WRITE_REG(hw, IXGBE_TDBAH(i), (tdba >> 32)); 3212 IXGBE_WRITE_REG(hw, IXGBE_TDLEN(i), 3213 adapter->num_tx_desc * sizeof(struct ixgbe_legacy_tx_desc)); 3214 3215 /* Setup the HW Tx Head and Tail descriptor pointers */ 3216 IXGBE_WRITE_REG(hw, IXGBE_TDH(i), 0); 3217 IXGBE_WRITE_REG(hw, IXGBE_TDT(i), 0); 3218 3219 /* Setup Transmit Descriptor Cmd Settings */ 3220 txr->txd_cmd = IXGBE_TXD_CMD_IFCS; 3221 txr->queue_status = IXGBE_QUEUE_IDLE; 3222 3223 /* Disable Head Writeback */ 3224 switch (hw->mac.type) { 3225 case ixgbe_mac_82598EB: 3226 txctrl = IXGBE_READ_REG(hw, IXGBE_DCA_TXCTRL(i)); 3227 break; 3228 case ixgbe_mac_82599EB: 3229 default: 3230 txctrl = IXGBE_READ_REG(hw, IXGBE_DCA_TXCTRL_82599(i)); 3231 break; 3232 } 3233 txctrl &= ~IXGBE_DCA_TXCTRL_TX_WB_RO_EN; 3234 switch (hw->mac.type) { 3235 case ixgbe_mac_82598EB: 3236 IXGBE_WRITE_REG(hw, IXGBE_DCA_TXCTRL(i), txctrl); 3237 break; 3238 case ixgbe_mac_82599EB: 3239 default: 3240 IXGBE_WRITE_REG(hw, IXGBE_DCA_TXCTRL_82599(i), txctrl); 3241 break; 3242 } 3243 3244 } 3245 3246 if (hw->mac.type != ixgbe_mac_82598EB) { 3247 u32 dmatxctl, rttdcs; 3248 dmatxctl = IXGBE_READ_REG(hw, IXGBE_DMATXCTL); 3249 dmatxctl |= IXGBE_DMATXCTL_TE; 3250 IXGBE_WRITE_REG(hw, IXGBE_DMATXCTL, dmatxctl); 3251 /* Disable arbiter to set MTQC */ 3252 rttdcs = IXGBE_READ_REG(hw, IXGBE_RTTDCS); 3253 rttdcs |= IXGBE_RTTDCS_ARBDIS; 3254 IXGBE_WRITE_REG(hw, IXGBE_RTTDCS, rttdcs); 3255 IXGBE_WRITE_REG(hw, IXGBE_MTQC, IXGBE_MTQC_64Q_1PB); 3256 rttdcs &= ~IXGBE_RTTDCS_ARBDIS; 3257 IXGBE_WRITE_REG(hw, IXGBE_RTTDCS, rttdcs); 3258 } 3259 3260 return; 3261 } 3262 3263 /********************************************************************* 3264 * 3265 * Free all transmit rings. 3266 * 3267 **********************************************************************/ 3268 static void 3269 ixgbe_free_transmit_structures(struct adapter *adapter) 3270 { 3271 struct tx_ring *txr = adapter->tx_rings; 3272 3273 for (int i = 0; i < adapter->num_queues; i++, txr++) { 3274 ixgbe_free_transmit_buffers(txr); 3275 ixgbe_dma_free(adapter, &txr->txdma); 3276 IXGBE_TX_LOCK_DESTROY(txr); 3277 } 3278 free(adapter->tx_rings, M_DEVBUF); 3279 } 3280 3281 /********************************************************************* 3282 * 3283 * Free transmit ring related data structures. 3284 * 3285 **********************************************************************/ 3286 static void 3287 ixgbe_free_transmit_buffers(struct tx_ring *txr) 3288 { 3289 struct adapter *adapter = txr->adapter; 3290 struct ixgbe_tx_buf *tx_buffer; 3291 int i; 3292 3293 INIT_DEBUGOUT("free_transmit_ring: begin"); 3294 3295 if (txr->tx_buffers == NULL) 3296 return; 3297 3298 tx_buffer = txr->tx_buffers; 3299 for (i = 0; i < adapter->num_tx_desc; i++, tx_buffer++) { 3300 if (tx_buffer->m_head != NULL) { 3301 bus_dmamap_sync(txr->txtag->dt_dmat, tx_buffer->map, 3302 0, tx_buffer->m_head->m_pkthdr.len, 3303 BUS_DMASYNC_POSTWRITE); 3304 ixgbe_dmamap_unload(txr->txtag, tx_buffer->map); 3305 m_freem(tx_buffer->m_head); 3306 tx_buffer->m_head = NULL; 3307 if (tx_buffer->map != NULL) { 3308 ixgbe_dmamap_destroy(txr->txtag, 3309 tx_buffer->map); 3310 tx_buffer->map = NULL; 3311 } 3312 } else if (tx_buffer->map != NULL) { 3313 ixgbe_dmamap_unload(txr->txtag, tx_buffer->map); 3314 ixgbe_dmamap_destroy(txr->txtag, tx_buffer->map); 3315 tx_buffer->map = NULL; 3316 } 3317 } 3318 #if __FreeBSD_version >= 800000 3319 if (txr->br != NULL) 3320 buf_ring_free(txr->br, M_DEVBUF); 3321 #endif 3322 if (txr->tx_buffers != NULL) { 3323 free(txr->tx_buffers, M_DEVBUF); 3324 txr->tx_buffers = NULL; 3325 } 3326 if (txr->txtag != NULL) { 3327 ixgbe_dma_tag_destroy(txr->txtag); 3328 txr->txtag = NULL; 3329 } 3330 return; 3331 } 3332 3333 /********************************************************************* 3334 * 3335 * Advanced Context Descriptor setup for VLAN or L4 CSUM 3336 * 3337 **********************************************************************/ 3338 3339 static u32 3340 ixgbe_tx_ctx_setup(struct tx_ring *txr, struct mbuf *mp) 3341 { 3342 struct m_tag *mtag; 3343 struct adapter *adapter = txr->adapter; 3344 struct ethercom *ec = &adapter->osdep.ec; 3345 struct ixgbe_adv_tx_context_desc *TXD; 3346 struct ixgbe_tx_buf *tx_buffer; 3347 u32 olinfo = 0, vlan_macip_lens = 0, type_tucmd_mlhl = 0; 3348 struct ether_vlan_header *eh; 3349 struct ip ip; 3350 struct ip6_hdr ip6; 3351 int ehdrlen, ip_hlen = 0; 3352 u16 etype; 3353 u8 ipproto __diagused = 0; 3354 bool offload; 3355 int ctxd = txr->next_avail_desc; 3356 u16 vtag = 0; 3357 3358 offload = ((mp->m_pkthdr.csum_flags & M_CSUM_OFFLOAD) != 0); 3359 3360 tx_buffer = &txr->tx_buffers[ctxd]; 3361 TXD = (struct ixgbe_adv_tx_context_desc *) &txr->tx_base[ctxd]; 3362 3363 /* 3364 ** In advanced descriptors the vlan tag must 3365 ** be placed into the descriptor itself. 3366 */ 3367 if ((mtag = VLAN_OUTPUT_TAG(ec, mp)) != NULL) { 3368 vtag = htole16(VLAN_TAG_VALUE(mtag) & 0xffff); 3369 vlan_macip_lens |= (vtag << IXGBE_ADVTXD_VLAN_SHIFT); 3370 } else if (!offload) 3371 return 0; 3372 3373 /* 3374 * Determine where frame payload starts. 3375 * Jump over vlan headers if already present, 3376 * helpful for QinQ too. 3377 */ 3378 KASSERT(mp->m_len >= offsetof(struct ether_vlan_header, evl_tag)); 3379 eh = mtod(mp, struct ether_vlan_header *); 3380 if (eh->evl_encap_proto == htons(ETHERTYPE_VLAN)) { 3381 KASSERT(mp->m_len >= sizeof(struct ether_vlan_header)); 3382 etype = ntohs(eh->evl_proto); 3383 ehdrlen = ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN; 3384 } else { 3385 etype = ntohs(eh->evl_encap_proto); 3386 ehdrlen = ETHER_HDR_LEN; 3387 } 3388 3389 /* Set the ether header length */ 3390 vlan_macip_lens |= ehdrlen << IXGBE_ADVTXD_MACLEN_SHIFT; 3391 3392 switch (etype) { 3393 case ETHERTYPE_IP: 3394 m_copydata(mp, ehdrlen, sizeof(ip), &ip); 3395 ip_hlen = ip.ip_hl << 2; 3396 ipproto = ip.ip_p; 3397 #if 0 3398 ip.ip_sum = 0; 3399 m_copyback(mp, ehdrlen, sizeof(ip), &ip); 3400 #else 3401 KASSERT((mp->m_pkthdr.csum_flags & M_CSUM_IPv4) == 0 || 3402 ip.ip_sum == 0); 3403 #endif 3404 type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_IPV4; 3405 break; 3406 case ETHERTYPE_IPV6: 3407 m_copydata(mp, ehdrlen, sizeof(ip6), &ip6); 3408 ip_hlen = sizeof(ip6); 3409 ipproto = ip6.ip6_nxt; 3410 type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_IPV6; 3411 break; 3412 default: 3413 break; 3414 } 3415 3416 if ((mp->m_pkthdr.csum_flags & M_CSUM_IPv4) != 0) 3417 olinfo |= IXGBE_TXD_POPTS_IXSM << 8; 3418 3419 vlan_macip_lens |= ip_hlen; 3420 type_tucmd_mlhl |= IXGBE_ADVTXD_DCMD_DEXT | IXGBE_ADVTXD_DTYP_CTXT; 3421 3422 if (mp->m_pkthdr.csum_flags & (M_CSUM_TCPv4|M_CSUM_TCPv6)) { 3423 type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_L4T_TCP; 3424 olinfo |= IXGBE_TXD_POPTS_TXSM << 8; 3425 KASSERT(ipproto == IPPROTO_TCP); 3426 } else if (mp->m_pkthdr.csum_flags & (M_CSUM_UDPv4|M_CSUM_UDPv6)) { 3427 type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_L4T_UDP; 3428 olinfo |= IXGBE_TXD_POPTS_TXSM << 8; 3429 KASSERT(ipproto == IPPROTO_UDP); 3430 } 3431 3432 /* Now copy bits into descriptor */ 3433 TXD->vlan_macip_lens |= htole32(vlan_macip_lens); 3434 TXD->type_tucmd_mlhl |= htole32(type_tucmd_mlhl); 3435 TXD->seqnum_seed = htole32(0); 3436 TXD->mss_l4len_idx = htole32(0); 3437 3438 tx_buffer->m_head = NULL; 3439 tx_buffer->eop_index = -1; 3440 3441 /* We've consumed the first desc, adjust counters */ 3442 if (++ctxd == adapter->num_tx_desc) 3443 ctxd = 0; 3444 txr->next_avail_desc = ctxd; 3445 --txr->tx_avail; 3446 3447 return olinfo; 3448 } 3449 3450 /********************************************************************** 3451 * 3452 * Setup work for hardware segmentation offload (TSO) on 3453 * adapters using advanced tx descriptors 3454 * 3455 **********************************************************************/ 3456 static bool 3457 ixgbe_tso_setup(struct tx_ring *txr, struct mbuf *mp, u32 *paylen) 3458 { 3459 struct m_tag *mtag; 3460 struct adapter *adapter = txr->adapter; 3461 struct ethercom *ec = &adapter->osdep.ec; 3462 struct ixgbe_adv_tx_context_desc *TXD; 3463 struct ixgbe_tx_buf *tx_buffer; 3464 u32 vlan_macip_lens = 0, type_tucmd_mlhl = 0; 3465 u32 mss_l4len_idx = 0; 3466 u16 vtag = 0; 3467 int ctxd, ehdrlen, hdrlen, ip_hlen, tcp_hlen; 3468 struct ether_vlan_header *eh; 3469 struct ip *ip; 3470 struct tcphdr *th; 3471 3472 3473 /* 3474 * Determine where frame payload starts. 3475 * Jump over vlan headers if already present 3476 */ 3477 eh = mtod(mp, struct ether_vlan_header *); 3478 if (eh->evl_encap_proto == htons(ETHERTYPE_VLAN)) 3479 ehdrlen = ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN; 3480 else 3481 ehdrlen = ETHER_HDR_LEN; 3482 3483 /* Ensure we have at least the IP+TCP header in the first mbuf. */ 3484 if (mp->m_len < ehdrlen + sizeof(struct ip) + sizeof(struct tcphdr)) 3485 return FALSE; 3486 3487 ctxd = txr->next_avail_desc; 3488 tx_buffer = &txr->tx_buffers[ctxd]; 3489 TXD = (struct ixgbe_adv_tx_context_desc *) &txr->tx_base[ctxd]; 3490 3491 ip = (struct ip *)(mp->m_data + ehdrlen); 3492 if (ip->ip_p != IPPROTO_TCP) 3493 return FALSE; /* 0 */ 3494 ip->ip_sum = 0; 3495 ip_hlen = ip->ip_hl << 2; 3496 th = (struct tcphdr *)((char *)ip + ip_hlen); 3497 /* XXX Educated guess: FreeBSD's in_pseudo == NetBSD's in_cksum_phdr */ 3498 th->th_sum = in_cksum_phdr(ip->ip_src.s_addr, 3499 ip->ip_dst.s_addr, htons(IPPROTO_TCP)); 3500 tcp_hlen = th->th_off << 2; 3501 hdrlen = ehdrlen + ip_hlen + tcp_hlen; 3502 3503 /* This is used in the transmit desc in encap */ 3504 *paylen = mp->m_pkthdr.len - hdrlen; 3505 3506 /* VLAN MACLEN IPLEN */ 3507 if ((mtag = VLAN_OUTPUT_TAG(ec, mp)) != NULL) { 3508 vtag = htole16(VLAN_TAG_VALUE(mtag) & 0xffff); 3509 vlan_macip_lens |= (vtag << IXGBE_ADVTXD_VLAN_SHIFT); 3510 } 3511 3512 vlan_macip_lens |= ehdrlen << IXGBE_ADVTXD_MACLEN_SHIFT; 3513 vlan_macip_lens |= ip_hlen; 3514 TXD->vlan_macip_lens |= htole32(vlan_macip_lens); 3515 3516 /* ADV DTYPE TUCMD */ 3517 type_tucmd_mlhl |= IXGBE_ADVTXD_DCMD_DEXT | IXGBE_ADVTXD_DTYP_CTXT; 3518 type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_L4T_TCP; 3519 type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_IPV4; 3520 TXD->type_tucmd_mlhl |= htole32(type_tucmd_mlhl); 3521 3522 3523 /* MSS L4LEN IDX */ 3524 mss_l4len_idx |= (mp->m_pkthdr.segsz << IXGBE_ADVTXD_MSS_SHIFT); 3525 mss_l4len_idx |= (tcp_hlen << IXGBE_ADVTXD_L4LEN_SHIFT); 3526 TXD->mss_l4len_idx = htole32(mss_l4len_idx); 3527 3528 TXD->seqnum_seed = htole32(0); 3529 tx_buffer->m_head = NULL; 3530 tx_buffer->eop_index = -1; 3531 3532 if (++ctxd == adapter->num_tx_desc) 3533 ctxd = 0; 3534 3535 txr->tx_avail--; 3536 txr->next_avail_desc = ctxd; 3537 return TRUE; 3538 } 3539 3540 #ifdef IXGBE_FDIR 3541 /* 3542 ** This routine parses packet headers so that Flow 3543 ** Director can make a hashed filter table entry 3544 ** allowing traffic flows to be identified and kept 3545 ** on the same cpu. This would be a performance 3546 ** hit, but we only do it at IXGBE_FDIR_RATE of 3547 ** packets. 3548 */ 3549 static void 3550 ixgbe_atr(struct tx_ring *txr, struct mbuf *mp) 3551 { 3552 struct adapter *adapter = txr->adapter; 3553 struct ix_queue *que; 3554 struct ip *ip; 3555 struct tcphdr *th; 3556 struct udphdr *uh; 3557 struct ether_vlan_header *eh; 3558 union ixgbe_atr_hash_dword input = {.dword = 0}; 3559 union ixgbe_atr_hash_dword common = {.dword = 0}; 3560 int ehdrlen, ip_hlen; 3561 u16 etype; 3562 3563 eh = mtod(mp, struct ether_vlan_header *); 3564 if (eh->evl_encap_proto == htons(ETHERTYPE_VLAN)) { 3565 ehdrlen = ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN; 3566 etype = eh->evl_proto; 3567 } else { 3568 ehdrlen = ETHER_HDR_LEN; 3569 etype = eh->evl_encap_proto; 3570 } 3571 3572 /* Only handling IPv4 */ 3573 if (etype != htons(ETHERTYPE_IP)) 3574 return; 3575 3576 ip = (struct ip *)(mp->m_data + ehdrlen); 3577 ip_hlen = ip->ip_hl << 2; 3578 3579 /* check if we're UDP or TCP */ 3580 switch (ip->ip_p) { 3581 case IPPROTO_TCP: 3582 th = (struct tcphdr *)((char *)ip + ip_hlen); 3583 /* src and dst are inverted */ 3584 common.port.dst ^= th->th_sport; 3585 common.port.src ^= th->th_dport; 3586 input.formatted.flow_type ^= IXGBE_ATR_FLOW_TYPE_TCPV4; 3587 break; 3588 case IPPROTO_UDP: 3589 uh = (struct udphdr *)((char *)ip + ip_hlen); 3590 /* src and dst are inverted */ 3591 common.port.dst ^= uh->uh_sport; 3592 common.port.src ^= uh->uh_dport; 3593 input.formatted.flow_type ^= IXGBE_ATR_FLOW_TYPE_UDPV4; 3594 break; 3595 default: 3596 return; 3597 } 3598 3599 input.formatted.vlan_id = htobe16(mp->m_pkthdr.ether_vtag); 3600 if (mp->m_pkthdr.ether_vtag) 3601 common.flex_bytes ^= htons(ETHERTYPE_VLAN); 3602 else 3603 common.flex_bytes ^= etype; 3604 common.ip ^= ip->ip_src.s_addr ^ ip->ip_dst.s_addr; 3605 3606 que = &adapter->queues[txr->me]; 3607 /* 3608 ** This assumes the Rx queue and Tx 3609 ** queue are bound to the same CPU 3610 */ 3611 ixgbe_fdir_add_signature_filter_82599(&adapter->hw, 3612 input, common, que->msix); 3613 } 3614 #endif /* IXGBE_FDIR */ 3615 3616 /********************************************************************** 3617 * 3618 * Examine each tx_buffer in the used queue. If the hardware is done 3619 * processing the packet then free associated resources. The 3620 * tx_buffer is put back on the free queue. 3621 * 3622 **********************************************************************/ 3623 static bool 3624 ixgbe_txeof(struct tx_ring *txr) 3625 { 3626 struct adapter *adapter = txr->adapter; 3627 struct ifnet *ifp = adapter->ifp; 3628 u32 first, last, done, processed; 3629 struct ixgbe_tx_buf *tx_buffer; 3630 struct ixgbe_legacy_tx_desc *tx_desc, *eop_desc; 3631 struct timeval now, elapsed; 3632 3633 KASSERT(mutex_owned(&txr->tx_mtx)); 3634 3635 #ifdef DEV_NETMAP 3636 if (ifp->if_capenable & IFCAP_NETMAP) { 3637 struct netmap_adapter *na = NA(ifp); 3638 struct netmap_kring *kring = &na->tx_rings[txr->me]; 3639 3640 tx_desc = (struct ixgbe_legacy_tx_desc *)txr->tx_base; 3641 3642 bus_dmamap_sync(txr->txdma.dma_tag, txr->txdma.dma_map, 3643 BUS_DMASYNC_POSTREAD); 3644 /* 3645 * In netmap mode, all the work is done in the context 3646 * of the client thread. Interrupt handlers only wake up 3647 * clients, which may be sleeping on individual rings 3648 * or on a global resource for all rings. 3649 * To implement tx interrupt mitigation, we wake up the client 3650 * thread roughly every half ring, even if the NIC interrupts 3651 * more frequently. This is implemented as follows: 3652 * - ixgbe_txsync() sets kring->nr_kflags with the index of 3653 * the slot that should wake up the thread (nkr_num_slots 3654 * means the user thread should not be woken up); 3655 * - the driver ignores tx interrupts unless netmap_mitigate=0 3656 * or the slot has the DD bit set. 3657 * 3658 * When the driver has separate locks, we need to 3659 * release and re-acquire txlock to avoid deadlocks. 3660 * XXX see if we can find a better way. 3661 */ 3662 if (!netmap_mitigate || 3663 (kring->nr_kflags < kring->nkr_num_slots && 3664 tx_desc[kring->nr_kflags].upper.fields.status & IXGBE_TXD_STAT_DD)) { 3665 kring->nr_kflags = kring->nkr_num_slots; 3666 selwakeuppri(&na->tx_rings[txr->me].si, PI_NET); 3667 IXGBE_TX_UNLOCK(txr); 3668 IXGBE_CORE_LOCK(adapter); 3669 selwakeuppri(&na->tx_rings[na->num_queues + 1].si, PI_NET); 3670 IXGBE_CORE_UNLOCK(adapter); 3671 IXGBE_TX_LOCK(txr); 3672 } 3673 return FALSE; 3674 } 3675 #endif /* DEV_NETMAP */ 3676 3677 if (txr->tx_avail == adapter->num_tx_desc) { 3678 txr->queue_status = IXGBE_QUEUE_IDLE; 3679 return false; 3680 } 3681 3682 processed = 0; 3683 first = txr->next_to_clean; 3684 tx_buffer = &txr->tx_buffers[first]; 3685 /* For cleanup we just use legacy struct */ 3686 tx_desc = (struct ixgbe_legacy_tx_desc *)&txr->tx_base[first]; 3687 last = tx_buffer->eop_index; 3688 if (last == -1) 3689 return false; 3690 eop_desc = (struct ixgbe_legacy_tx_desc *)&txr->tx_base[last]; 3691 3692 /* 3693 ** Get the index of the first descriptor 3694 ** BEYOND the EOP and call that 'done'. 3695 ** I do this so the comparison in the 3696 ** inner while loop below can be simple 3697 */ 3698 if (++last == adapter->num_tx_desc) last = 0; 3699 done = last; 3700 3701 ixgbe_dmamap_sync(txr->txdma.dma_tag, txr->txdma.dma_map, 3702 BUS_DMASYNC_POSTREAD); 3703 /* 3704 ** Only the EOP descriptor of a packet now has the DD 3705 ** bit set, this is what we look for... 3706 */ 3707 while (eop_desc->upper.fields.status & IXGBE_TXD_STAT_DD) { 3708 /* We clean the range of the packet */ 3709 while (first != done) { 3710 tx_desc->upper.data = 0; 3711 tx_desc->lower.data = 0; 3712 tx_desc->buffer_addr = 0; 3713 ++txr->tx_avail; 3714 ++processed; 3715 3716 if (tx_buffer->m_head) { 3717 txr->bytes += 3718 tx_buffer->m_head->m_pkthdr.len; 3719 bus_dmamap_sync(txr->txtag->dt_dmat, 3720 tx_buffer->map, 3721 0, tx_buffer->m_head->m_pkthdr.len, 3722 BUS_DMASYNC_POSTWRITE); 3723 ixgbe_dmamap_unload(txr->txtag, tx_buffer->map); 3724 m_freem(tx_buffer->m_head); 3725 tx_buffer->m_head = NULL; 3726 } 3727 tx_buffer->eop_index = -1; 3728 getmicrotime(&txr->watchdog_time); 3729 3730 if (++first == adapter->num_tx_desc) 3731 first = 0; 3732 3733 tx_buffer = &txr->tx_buffers[first]; 3734 tx_desc = 3735 (struct ixgbe_legacy_tx_desc *)&txr->tx_base[first]; 3736 } 3737 ++txr->packets; 3738 ++ifp->if_opackets; 3739 /* See if there is more work now */ 3740 last = tx_buffer->eop_index; 3741 if (last != -1) { 3742 eop_desc = 3743 (struct ixgbe_legacy_tx_desc *)&txr->tx_base[last]; 3744 /* Get next done point */ 3745 if (++last == adapter->num_tx_desc) last = 0; 3746 done = last; 3747 } else 3748 break; 3749 } 3750 ixgbe_dmamap_sync(txr->txdma.dma_tag, txr->txdma.dma_map, 3751 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); 3752 3753 txr->next_to_clean = first; 3754 3755 /* 3756 ** Watchdog calculation, we know there's 3757 ** work outstanding or the first return 3758 ** would have been taken, so none processed 3759 ** for too long indicates a hang. 3760 */ 3761 getmicrotime(&now); 3762 timersub(&now, &txr->watchdog_time, &elapsed); 3763 if (!processed && tvtohz(&elapsed) > IXGBE_WATCHDOG) 3764 txr->queue_status = IXGBE_QUEUE_HUNG; 3765 3766 /* 3767 * If we have enough room, clear IFF_OACTIVE to tell the stack that 3768 * it is OK to send packets. If there are no pending descriptors, 3769 * clear the timeout. Otherwise, if some descriptors have been freed, 3770 * restart the timeout. 3771 */ 3772 if (txr->tx_avail > IXGBE_TX_CLEANUP_THRESHOLD) { 3773 ifp->if_flags &= ~IFF_OACTIVE; 3774 if (txr->tx_avail == adapter->num_tx_desc) { 3775 txr->queue_status = IXGBE_QUEUE_IDLE; 3776 return false; 3777 } 3778 } 3779 3780 return true; 3781 } 3782 3783 /********************************************************************* 3784 * 3785 * Refresh mbuf buffers for RX descriptor rings 3786 * - now keeps its own state so discards due to resource 3787 * exhaustion are unnecessary, if an mbuf cannot be obtained 3788 * it just returns, keeping its placeholder, thus it can simply 3789 * be recalled to try again. 3790 * 3791 **********************************************************************/ 3792 static void 3793 ixgbe_refresh_mbufs(struct rx_ring *rxr, int limit) 3794 { 3795 struct adapter *adapter = rxr->adapter; 3796 struct ixgbe_rx_buf *rxbuf; 3797 struct mbuf *mh, *mp; 3798 int i, j, error; 3799 bool refreshed = false; 3800 3801 i = j = rxr->next_to_refresh; 3802 /* Control the loop with one beyond */ 3803 if (++j == adapter->num_rx_desc) 3804 j = 0; 3805 3806 while (j != limit) { 3807 rxbuf = &rxr->rx_buffers[i]; 3808 if (rxr->hdr_split == FALSE) 3809 goto no_split; 3810 3811 if (rxbuf->m_head == NULL) { 3812 mh = m_gethdr(M_DONTWAIT, MT_DATA); 3813 if (mh == NULL) 3814 goto update; 3815 } else 3816 mh = rxbuf->m_head; 3817 3818 mh->m_pkthdr.len = mh->m_len = MHLEN; 3819 mh->m_len = MHLEN; 3820 mh->m_flags |= M_PKTHDR; 3821 /* Get the memory mapping */ 3822 error = bus_dmamap_load_mbuf(rxr->htag->dt_dmat, 3823 rxbuf->hmap, mh, BUS_DMA_NOWAIT); 3824 if (error != 0) { 3825 printf("Refresh mbufs: hdr dmamap load" 3826 " failure - %d\n", error); 3827 m_free(mh); 3828 rxbuf->m_head = NULL; 3829 goto update; 3830 } 3831 rxbuf->m_head = mh; 3832 ixgbe_dmamap_sync(rxr->htag, rxbuf->hmap, BUS_DMASYNC_PREREAD); 3833 rxr->rx_base[i].read.hdr_addr = 3834 htole64(rxbuf->hmap->dm_segs[0].ds_addr); 3835 3836 no_split: 3837 if (rxbuf->m_pack == NULL) { 3838 mp = ixgbe_getjcl(&adapter->jcl_head, M_DONTWAIT, 3839 MT_DATA, M_PKTHDR, adapter->rx_mbuf_sz); 3840 if (mp == NULL) { 3841 rxr->no_jmbuf.ev_count++; 3842 goto update; 3843 } 3844 } else 3845 mp = rxbuf->m_pack; 3846 3847 mp->m_pkthdr.len = mp->m_len = adapter->rx_mbuf_sz; 3848 /* Get the memory mapping */ 3849 error = bus_dmamap_load_mbuf(rxr->ptag->dt_dmat, 3850 rxbuf->pmap, mp, BUS_DMA_NOWAIT); 3851 if (error != 0) { 3852 printf("Refresh mbufs: payload dmamap load" 3853 " failure - %d\n", error); 3854 m_free(mp); 3855 rxbuf->m_pack = NULL; 3856 goto update; 3857 } 3858 rxbuf->m_pack = mp; 3859 bus_dmamap_sync(rxr->ptag->dt_dmat, rxbuf->pmap, 3860 0, mp->m_pkthdr.len, BUS_DMASYNC_PREREAD); 3861 rxr->rx_base[i].read.pkt_addr = 3862 htole64(rxbuf->pmap->dm_segs[0].ds_addr); 3863 3864 refreshed = true; 3865 /* Next is precalculated */ 3866 i = j; 3867 rxr->next_to_refresh = i; 3868 if (++j == adapter->num_rx_desc) 3869 j = 0; 3870 } 3871 update: 3872 if (refreshed) /* Update hardware tail index */ 3873 IXGBE_WRITE_REG(&adapter->hw, 3874 IXGBE_RDT(rxr->me), rxr->next_to_refresh); 3875 return; 3876 } 3877 3878 /********************************************************************* 3879 * 3880 * Allocate memory for rx_buffer structures. Since we use one 3881 * rx_buffer per received packet, the maximum number of rx_buffer's 3882 * that we'll need is equal to the number of receive descriptors 3883 * that we've allocated. 3884 * 3885 **********************************************************************/ 3886 static int 3887 ixgbe_allocate_receive_buffers(struct rx_ring *rxr) 3888 { 3889 struct adapter *adapter = rxr->adapter; 3890 device_t dev = adapter->dev; 3891 struct ixgbe_rx_buf *rxbuf; 3892 int i, bsize, error; 3893 3894 bsize = sizeof(struct ixgbe_rx_buf) * adapter->num_rx_desc; 3895 if (!(rxr->rx_buffers = 3896 (struct ixgbe_rx_buf *) malloc(bsize, 3897 M_DEVBUF, M_NOWAIT | M_ZERO))) { 3898 aprint_error_dev(dev, "Unable to allocate rx_buffer memory\n"); 3899 error = ENOMEM; 3900 goto fail; 3901 } 3902 3903 if ((error = ixgbe_dma_tag_create(adapter->osdep.dmat, /* parent */ 3904 1, 0, /* alignment, bounds */ 3905 MSIZE, /* maxsize */ 3906 1, /* nsegments */ 3907 MSIZE, /* maxsegsize */ 3908 0, /* flags */ 3909 &rxr->htag))) { 3910 aprint_error_dev(dev, "Unable to create RX DMA tag\n"); 3911 goto fail; 3912 } 3913 3914 if ((error = ixgbe_dma_tag_create(adapter->osdep.dmat, /* parent */ 3915 1, 0, /* alignment, bounds */ 3916 MJUM16BYTES, /* maxsize */ 3917 1, /* nsegments */ 3918 MJUM16BYTES, /* maxsegsize */ 3919 0, /* flags */ 3920 &rxr->ptag))) { 3921 aprint_error_dev(dev, "Unable to create RX DMA tag\n"); 3922 goto fail; 3923 } 3924 3925 for (i = 0; i < adapter->num_rx_desc; i++, rxbuf++) { 3926 rxbuf = &rxr->rx_buffers[i]; 3927 error = ixgbe_dmamap_create(rxr->htag, 3928 BUS_DMA_NOWAIT, &rxbuf->hmap); 3929 if (error) { 3930 aprint_error_dev(dev, "Unable to create RX head map\n"); 3931 goto fail; 3932 } 3933 error = ixgbe_dmamap_create(rxr->ptag, 3934 BUS_DMA_NOWAIT, &rxbuf->pmap); 3935 if (error) { 3936 aprint_error_dev(dev, "Unable to create RX pkt map\n"); 3937 goto fail; 3938 } 3939 } 3940 3941 return (0); 3942 3943 fail: 3944 /* Frees all, but can handle partial completion */ 3945 ixgbe_free_receive_structures(adapter); 3946 return (error); 3947 } 3948 3949 /* 3950 ** Used to detect a descriptor that has 3951 ** been merged by Hardware RSC. 3952 */ 3953 static inline u32 3954 ixgbe_rsc_count(union ixgbe_adv_rx_desc *rx) 3955 { 3956 return (le32toh(rx->wb.lower.lo_dword.data) & 3957 IXGBE_RXDADV_RSCCNT_MASK) >> IXGBE_RXDADV_RSCCNT_SHIFT; 3958 } 3959 3960 /********************************************************************* 3961 * 3962 * Initialize Hardware RSC (LRO) feature on 82599 3963 * for an RX ring, this is toggled by the LRO capability 3964 * even though it is transparent to the stack. 3965 * 3966 **********************************************************************/ 3967 static void 3968 ixgbe_setup_hw_rsc(struct rx_ring *rxr) 3969 { 3970 struct adapter *adapter = rxr->adapter; 3971 struct ixgbe_hw *hw = &adapter->hw; 3972 u32 rscctrl, rdrxctl; 3973 3974 rdrxctl = IXGBE_READ_REG(hw, IXGBE_RDRXCTL); 3975 rdrxctl &= ~IXGBE_RDRXCTL_RSCFRSTSIZE; 3976 rdrxctl |= IXGBE_RDRXCTL_CRCSTRIP; 3977 rdrxctl |= IXGBE_RDRXCTL_RSCACKC; 3978 IXGBE_WRITE_REG(hw, IXGBE_RDRXCTL, rdrxctl); 3979 3980 rscctrl = IXGBE_READ_REG(hw, IXGBE_RSCCTL(rxr->me)); 3981 rscctrl |= IXGBE_RSCCTL_RSCEN; 3982 /* 3983 ** Limit the total number of descriptors that 3984 ** can be combined, so it does not exceed 64K 3985 */ 3986 if (adapter->rx_mbuf_sz == MCLBYTES) 3987 rscctrl |= IXGBE_RSCCTL_MAXDESC_16; 3988 else if (adapter->rx_mbuf_sz == MJUMPAGESIZE) 3989 rscctrl |= IXGBE_RSCCTL_MAXDESC_8; 3990 else if (adapter->rx_mbuf_sz == MJUM9BYTES) 3991 rscctrl |= IXGBE_RSCCTL_MAXDESC_4; 3992 else /* Using 16K cluster */ 3993 rscctrl |= IXGBE_RSCCTL_MAXDESC_1; 3994 3995 IXGBE_WRITE_REG(hw, IXGBE_RSCCTL(rxr->me), rscctrl); 3996 3997 /* Enable TCP header recognition */ 3998 IXGBE_WRITE_REG(hw, IXGBE_PSRTYPE(0), 3999 (IXGBE_READ_REG(hw, IXGBE_PSRTYPE(0)) | 4000 IXGBE_PSRTYPE_TCPHDR)); 4001 4002 /* Disable RSC for ACK packets */ 4003 IXGBE_WRITE_REG(hw, IXGBE_RSCDBU, 4004 (IXGBE_RSCDBU_RSCACKDIS | IXGBE_READ_REG(hw, IXGBE_RSCDBU))); 4005 4006 rxr->hw_rsc = TRUE; 4007 } 4008 4009 4010 static void 4011 ixgbe_free_receive_ring(struct rx_ring *rxr) 4012 { 4013 struct adapter *adapter; 4014 struct ixgbe_rx_buf *rxbuf; 4015 int i; 4016 4017 adapter = rxr->adapter; 4018 for (i = 0; i < adapter->num_rx_desc; i++) { 4019 rxbuf = &rxr->rx_buffers[i]; 4020 if (rxbuf->m_head != NULL) { 4021 ixgbe_dmamap_sync(rxr->htag, rxbuf->hmap, 4022 BUS_DMASYNC_POSTREAD); 4023 ixgbe_dmamap_unload(rxr->htag, rxbuf->hmap); 4024 rxbuf->m_head->m_flags |= M_PKTHDR; 4025 m_freem(rxbuf->m_head); 4026 } 4027 if (rxbuf->m_pack != NULL) { 4028 bus_dmamap_sync(rxr->ptag->dt_dmat, rxbuf->pmap, 4029 0, rxbuf->m_pack->m_pkthdr.len, 4030 BUS_DMASYNC_POSTREAD); 4031 ixgbe_dmamap_unload(rxr->ptag, rxbuf->pmap); 4032 rxbuf->m_pack->m_flags |= M_PKTHDR; 4033 m_freem(rxbuf->m_pack); 4034 } 4035 rxbuf->m_head = NULL; 4036 rxbuf->m_pack = NULL; 4037 } 4038 } 4039 4040 4041 /********************************************************************* 4042 * 4043 * Initialize a receive ring and its buffers. 4044 * 4045 **********************************************************************/ 4046 static int 4047 ixgbe_setup_receive_ring(struct rx_ring *rxr) 4048 { 4049 struct adapter *adapter; 4050 struct ifnet *ifp; 4051 struct ixgbe_rx_buf *rxbuf; 4052 #ifdef LRO 4053 struct lro_ctrl *lro = &rxr->lro; 4054 #endif /* LRO */ 4055 int rsize, error = 0; 4056 #ifdef DEV_NETMAP 4057 struct netmap_adapter *na = NA(rxr->adapter->ifp); 4058 struct netmap_slot *slot; 4059 #endif /* DEV_NETMAP */ 4060 4061 adapter = rxr->adapter; 4062 ifp = adapter->ifp; 4063 4064 /* Clear the ring contents */ 4065 IXGBE_RX_LOCK(rxr); 4066 #ifdef DEV_NETMAP 4067 /* same as in ixgbe_setup_transmit_ring() */ 4068 slot = netmap_reset(na, NR_RX, rxr->me, 0); 4069 #endif /* DEV_NETMAP */ 4070 rsize = roundup2(adapter->num_rx_desc * 4071 sizeof(union ixgbe_adv_rx_desc), DBA_ALIGN); 4072 bzero((void *)rxr->rx_base, rsize); 4073 4074 /* Free current RX buffer structs and their mbufs */ 4075 ixgbe_free_receive_ring(rxr); 4076 4077 IXGBE_RX_UNLOCK(rxr); 4078 4079 /* Now reinitialize our supply of jumbo mbufs. The number 4080 * or size of jumbo mbufs may have changed. 4081 */ 4082 ixgbe_jcl_reinit(&adapter->jcl_head, rxr->ptag->dt_dmat, 4083 2 * adapter->num_rx_desc, adapter->rx_mbuf_sz); 4084 4085 IXGBE_RX_LOCK(rxr); 4086 4087 /* Configure header split? */ 4088 if (ixgbe_header_split) 4089 rxr->hdr_split = TRUE; 4090 4091 /* Now replenish the mbufs */ 4092 for (int j = 0; j != adapter->num_rx_desc; ++j) { 4093 struct mbuf *mh, *mp; 4094 4095 rxbuf = &rxr->rx_buffers[j]; 4096 #ifdef DEV_NETMAP 4097 /* 4098 * In netmap mode, fill the map and set the buffer 4099 * address in the NIC ring, considering the offset 4100 * between the netmap and NIC rings (see comment in 4101 * ixgbe_setup_transmit_ring() ). No need to allocate 4102 * an mbuf, so end the block with a continue; 4103 */ 4104 if (slot) { 4105 int sj = j + na->rx_rings[rxr->me].nkr_hwofs; 4106 uint64_t paddr; 4107 void *addr; 4108 4109 if (sj >= na->num_rx_desc) 4110 sj -= na->num_rx_desc; 4111 addr = PNMB(slot + sj, &paddr); 4112 netmap_load_map(rxr->ptag, rxbuf->pmap, addr); 4113 /* Update descriptor */ 4114 rxr->rx_base[j].read.pkt_addr = htole64(paddr); 4115 continue; 4116 } 4117 #endif /* DEV_NETMAP */ 4118 /* 4119 ** Don't allocate mbufs if not 4120 ** doing header split, its wasteful 4121 */ 4122 if (rxr->hdr_split == FALSE) 4123 goto skip_head; 4124 4125 /* First the header */ 4126 rxbuf->m_head = m_gethdr(M_DONTWAIT, MT_DATA); 4127 if (rxbuf->m_head == NULL) { 4128 error = ENOBUFS; 4129 goto fail; 4130 } 4131 m_adj(rxbuf->m_head, ETHER_ALIGN); 4132 mh = rxbuf->m_head; 4133 mh->m_len = mh->m_pkthdr.len = MHLEN; 4134 mh->m_flags |= M_PKTHDR; 4135 /* Get the memory mapping */ 4136 error = bus_dmamap_load_mbuf(rxr->htag->dt_dmat, 4137 rxbuf->hmap, rxbuf->m_head, BUS_DMA_NOWAIT); 4138 if (error != 0) /* Nothing elegant to do here */ 4139 goto fail; 4140 bus_dmamap_sync(rxr->htag->dt_dmat, rxbuf->hmap, 4141 0, mh->m_pkthdr.len, BUS_DMASYNC_PREREAD); 4142 /* Update descriptor */ 4143 rxr->rx_base[j].read.hdr_addr = 4144 htole64(rxbuf->hmap->dm_segs[0].ds_addr); 4145 4146 skip_head: 4147 /* Now the payload cluster */ 4148 rxbuf->m_pack = ixgbe_getjcl(&adapter->jcl_head, M_DONTWAIT, 4149 MT_DATA, M_PKTHDR, adapter->rx_mbuf_sz); 4150 if (rxbuf->m_pack == NULL) { 4151 error = ENOBUFS; 4152 goto fail; 4153 } 4154 mp = rxbuf->m_pack; 4155 mp->m_pkthdr.len = mp->m_len = adapter->rx_mbuf_sz; 4156 /* Get the memory mapping */ 4157 error = bus_dmamap_load_mbuf(rxr->ptag->dt_dmat, 4158 rxbuf->pmap, mp, BUS_DMA_NOWAIT); 4159 if (error != 0) 4160 goto fail; 4161 bus_dmamap_sync(rxr->ptag->dt_dmat, rxbuf->pmap, 4162 0, adapter->rx_mbuf_sz, BUS_DMASYNC_PREREAD); 4163 /* Update descriptor */ 4164 rxr->rx_base[j].read.pkt_addr = 4165 htole64(rxbuf->pmap->dm_segs[0].ds_addr); 4166 } 4167 4168 4169 /* Setup our descriptor indices */ 4170 rxr->next_to_check = 0; 4171 rxr->next_to_refresh = 0; 4172 rxr->lro_enabled = FALSE; 4173 rxr->rx_split_packets.ev_count = 0; 4174 rxr->rx_bytes.ev_count = 0; 4175 rxr->discard = FALSE; 4176 4177 ixgbe_dmamap_sync(rxr->rxdma.dma_tag, rxr->rxdma.dma_map, 4178 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); 4179 4180 /* 4181 ** Now set up the LRO interface: 4182 ** 82598 uses software LRO, the 4183 ** 82599 uses a hardware assist. 4184 */ 4185 if ((adapter->hw.mac.type != ixgbe_mac_82598EB) && 4186 (ifp->if_capenable & IFCAP_RXCSUM) && 4187 (ifp->if_capenable & IFCAP_LRO)) 4188 ixgbe_setup_hw_rsc(rxr); 4189 #ifdef LRO 4190 else if (ifp->if_capenable & IFCAP_LRO) { 4191 device_t dev = adapter->dev; 4192 int err = tcp_lro_init(lro); 4193 if (err) { 4194 device_printf(dev, "LRO Initialization failed!\n"); 4195 goto fail; 4196 } 4197 INIT_DEBUGOUT("RX Soft LRO Initialized\n"); 4198 rxr->lro_enabled = TRUE; 4199 lro->ifp = adapter->ifp; 4200 } 4201 #endif /* LRO */ 4202 4203 IXGBE_RX_UNLOCK(rxr); 4204 return (0); 4205 4206 fail: 4207 ixgbe_free_receive_ring(rxr); 4208 IXGBE_RX_UNLOCK(rxr); 4209 return (error); 4210 } 4211 4212 /********************************************************************* 4213 * 4214 * Initialize all receive rings. 4215 * 4216 **********************************************************************/ 4217 static int 4218 ixgbe_setup_receive_structures(struct adapter *adapter) 4219 { 4220 struct rx_ring *rxr = adapter->rx_rings; 4221 int j; 4222 4223 for (j = 0; j < adapter->num_queues; j++, rxr++) 4224 if (ixgbe_setup_receive_ring(rxr)) 4225 goto fail; 4226 4227 return (0); 4228 fail: 4229 /* 4230 * Free RX buffers allocated so far, we will only handle 4231 * the rings that completed, the failing case will have 4232 * cleaned up for itself. 'j' failed, so its the terminus. 4233 */ 4234 for (int i = 0; i < j; ++i) { 4235 rxr = &adapter->rx_rings[i]; 4236 ixgbe_free_receive_ring(rxr); 4237 } 4238 4239 return (ENOBUFS); 4240 } 4241 4242 /********************************************************************* 4243 * 4244 * Setup receive registers and features. 4245 * 4246 **********************************************************************/ 4247 #define IXGBE_SRRCTL_BSIZEHDRSIZE_SHIFT 2 4248 4249 #define BSIZEPKT_ROUNDUP ((1<<IXGBE_SRRCTL_BSIZEPKT_SHIFT)-1) 4250 4251 static void 4252 ixgbe_initialize_receive_units(struct adapter *adapter) 4253 { 4254 int i; 4255 struct rx_ring *rxr = adapter->rx_rings; 4256 struct ixgbe_hw *hw = &adapter->hw; 4257 struct ifnet *ifp = adapter->ifp; 4258 u32 bufsz, rxctrl, fctrl, srrctl, rxcsum; 4259 u32 reta, mrqc = 0, hlreg, r[10]; 4260 4261 4262 /* 4263 * Make sure receives are disabled while 4264 * setting up the descriptor ring 4265 */ 4266 rxctrl = IXGBE_READ_REG(hw, IXGBE_RXCTRL); 4267 IXGBE_WRITE_REG(hw, IXGBE_RXCTRL, 4268 rxctrl & ~IXGBE_RXCTRL_RXEN); 4269 4270 /* Enable broadcasts */ 4271 fctrl = IXGBE_READ_REG(hw, IXGBE_FCTRL); 4272 fctrl |= IXGBE_FCTRL_BAM; 4273 fctrl |= IXGBE_FCTRL_DPF; 4274 fctrl |= IXGBE_FCTRL_PMCF; 4275 IXGBE_WRITE_REG(hw, IXGBE_FCTRL, fctrl); 4276 4277 /* Set for Jumbo Frames? */ 4278 hlreg = IXGBE_READ_REG(hw, IXGBE_HLREG0); 4279 if (ifp->if_mtu > ETHERMTU) 4280 hlreg |= IXGBE_HLREG0_JUMBOEN; 4281 else 4282 hlreg &= ~IXGBE_HLREG0_JUMBOEN; 4283 IXGBE_WRITE_REG(hw, IXGBE_HLREG0, hlreg); 4284 4285 bufsz = (adapter->rx_mbuf_sz + BSIZEPKT_ROUNDUP) >> IXGBE_SRRCTL_BSIZEPKT_SHIFT; 4286 4287 for (i = 0; i < adapter->num_queues; i++, rxr++) { 4288 u64 rdba = rxr->rxdma.dma_paddr; 4289 4290 /* Setup the Base and Length of the Rx Descriptor Ring */ 4291 IXGBE_WRITE_REG(hw, IXGBE_RDBAL(i), 4292 (rdba & 0x00000000ffffffffULL)); 4293 IXGBE_WRITE_REG(hw, IXGBE_RDBAH(i), (rdba >> 32)); 4294 IXGBE_WRITE_REG(hw, IXGBE_RDLEN(i), 4295 adapter->num_rx_desc * sizeof(union ixgbe_adv_rx_desc)); 4296 4297 /* Set up the SRRCTL register */ 4298 srrctl = IXGBE_READ_REG(hw, IXGBE_SRRCTL(i)); 4299 srrctl &= ~IXGBE_SRRCTL_BSIZEHDR_MASK; 4300 srrctl &= ~IXGBE_SRRCTL_BSIZEPKT_MASK; 4301 srrctl |= bufsz; 4302 if (rxr->hdr_split) { 4303 /* Use a standard mbuf for the header */ 4304 srrctl |= ((IXGBE_RX_HDR << 4305 IXGBE_SRRCTL_BSIZEHDRSIZE_SHIFT) 4306 & IXGBE_SRRCTL_BSIZEHDR_MASK); 4307 srrctl |= IXGBE_SRRCTL_DESCTYPE_HDR_SPLIT_ALWAYS; 4308 } else 4309 srrctl |= IXGBE_SRRCTL_DESCTYPE_ADV_ONEBUF; 4310 IXGBE_WRITE_REG(hw, IXGBE_SRRCTL(i), srrctl); 4311 4312 /* Setup the HW Rx Head and Tail Descriptor Pointers */ 4313 IXGBE_WRITE_REG(hw, IXGBE_RDH(i), 0); 4314 IXGBE_WRITE_REG(hw, IXGBE_RDT(i), 0); 4315 } 4316 4317 if (adapter->hw.mac.type != ixgbe_mac_82598EB) { 4318 u32 psrtype = IXGBE_PSRTYPE_TCPHDR | 4319 IXGBE_PSRTYPE_UDPHDR | 4320 IXGBE_PSRTYPE_IPV4HDR | 4321 IXGBE_PSRTYPE_IPV6HDR; 4322 IXGBE_WRITE_REG(hw, IXGBE_PSRTYPE(0), psrtype); 4323 } 4324 4325 rxcsum = IXGBE_READ_REG(hw, IXGBE_RXCSUM); 4326 4327 /* Setup RSS */ 4328 if (adapter->num_queues > 1) { 4329 int j; 4330 reta = 0; 4331 4332 /* set up random bits */ 4333 cprng_fast(&r, sizeof(r)); 4334 4335 /* Set up the redirection table */ 4336 for (i = 0, j = 0; i < 128; i++, j++) { 4337 if (j == adapter->num_queues) j = 0; 4338 reta = (reta << 8) | (j * 0x11); 4339 if ((i & 3) == 3) 4340 IXGBE_WRITE_REG(hw, IXGBE_RETA(i >> 2), reta); 4341 } 4342 4343 /* Now fill our hash function seeds */ 4344 for (i = 0; i < 10; i++) 4345 IXGBE_WRITE_REG(hw, IXGBE_RSSRK(i), r[i]); 4346 4347 /* Perform hash on these packet types */ 4348 mrqc = IXGBE_MRQC_RSSEN 4349 | IXGBE_MRQC_RSS_FIELD_IPV4 4350 | IXGBE_MRQC_RSS_FIELD_IPV4_TCP 4351 | IXGBE_MRQC_RSS_FIELD_IPV4_UDP 4352 | IXGBE_MRQC_RSS_FIELD_IPV6_EX_TCP 4353 | IXGBE_MRQC_RSS_FIELD_IPV6_EX 4354 | IXGBE_MRQC_RSS_FIELD_IPV6 4355 | IXGBE_MRQC_RSS_FIELD_IPV6_TCP 4356 | IXGBE_MRQC_RSS_FIELD_IPV6_UDP 4357 | IXGBE_MRQC_RSS_FIELD_IPV6_EX_UDP; 4358 IXGBE_WRITE_REG(hw, IXGBE_MRQC, mrqc); 4359 4360 /* RSS and RX IPP Checksum are mutually exclusive */ 4361 rxcsum |= IXGBE_RXCSUM_PCSD; 4362 } 4363 4364 if (ifp->if_capenable & IFCAP_RXCSUM) 4365 rxcsum |= IXGBE_RXCSUM_PCSD; 4366 4367 if (!(rxcsum & IXGBE_RXCSUM_PCSD)) 4368 rxcsum |= IXGBE_RXCSUM_IPPCSE; 4369 4370 IXGBE_WRITE_REG(hw, IXGBE_RXCSUM, rxcsum); 4371 4372 return; 4373 } 4374 4375 /********************************************************************* 4376 * 4377 * Free all receive rings. 4378 * 4379 **********************************************************************/ 4380 static void 4381 ixgbe_free_receive_structures(struct adapter *adapter) 4382 { 4383 struct rx_ring *rxr = adapter->rx_rings; 4384 4385 for (int i = 0; i < adapter->num_queues; i++, rxr++) { 4386 #ifdef LRO 4387 struct lro_ctrl *lro = &rxr->lro; 4388 #endif /* LRO */ 4389 ixgbe_free_receive_buffers(rxr); 4390 #ifdef LRO 4391 /* Free LRO memory */ 4392 tcp_lro_free(lro); 4393 #endif /* LRO */ 4394 /* Free the ring memory as well */ 4395 ixgbe_dma_free(adapter, &rxr->rxdma); 4396 IXGBE_RX_LOCK_DESTROY(rxr); 4397 } 4398 4399 free(adapter->rx_rings, M_DEVBUF); 4400 } 4401 4402 4403 /********************************************************************* 4404 * 4405 * Free receive ring data structures 4406 * 4407 **********************************************************************/ 4408 static void 4409 ixgbe_free_receive_buffers(struct rx_ring *rxr) 4410 { 4411 struct adapter *adapter = rxr->adapter; 4412 struct ixgbe_rx_buf *rxbuf; 4413 4414 INIT_DEBUGOUT("free_receive_structures: begin"); 4415 4416 /* Cleanup any existing buffers */ 4417 if (rxr->rx_buffers != NULL) { 4418 for (int i = 0; i < adapter->num_rx_desc; i++) { 4419 rxbuf = &rxr->rx_buffers[i]; 4420 if (rxbuf->m_head != NULL) { 4421 ixgbe_dmamap_sync(rxr->htag, rxbuf->hmap, 4422 BUS_DMASYNC_POSTREAD); 4423 ixgbe_dmamap_unload(rxr->htag, rxbuf->hmap); 4424 rxbuf->m_head->m_flags |= M_PKTHDR; 4425 m_freem(rxbuf->m_head); 4426 } 4427 if (rxbuf->m_pack != NULL) { 4428 bus_dmamap_sync(rxr->ptag->dt_dmat, rxbuf->pmap, 4429 0, rxbuf->m_pack->m_pkthdr.len, 4430 BUS_DMASYNC_POSTREAD); 4431 ixgbe_dmamap_unload(rxr->ptag, rxbuf->pmap); 4432 rxbuf->m_pack->m_flags |= M_PKTHDR; 4433 m_freem(rxbuf->m_pack); 4434 } 4435 rxbuf->m_head = NULL; 4436 rxbuf->m_pack = NULL; 4437 if (rxbuf->hmap != NULL) { 4438 ixgbe_dmamap_destroy(rxr->htag, rxbuf->hmap); 4439 rxbuf->hmap = NULL; 4440 } 4441 if (rxbuf->pmap != NULL) { 4442 ixgbe_dmamap_destroy(rxr->ptag, rxbuf->pmap); 4443 rxbuf->pmap = NULL; 4444 } 4445 } 4446 if (rxr->rx_buffers != NULL) { 4447 free(rxr->rx_buffers, M_DEVBUF); 4448 rxr->rx_buffers = NULL; 4449 } 4450 } 4451 4452 if (rxr->htag != NULL) { 4453 ixgbe_dma_tag_destroy(rxr->htag); 4454 rxr->htag = NULL; 4455 } 4456 if (rxr->ptag != NULL) { 4457 ixgbe_dma_tag_destroy(rxr->ptag); 4458 rxr->ptag = NULL; 4459 } 4460 4461 return; 4462 } 4463 4464 static __inline void 4465 ixgbe_rx_input(struct rx_ring *rxr, struct ifnet *ifp, struct mbuf *m, u32 ptype) 4466 { 4467 int s; 4468 4469 #ifdef LRO 4470 struct adapter *adapter = ifp->if_softc; 4471 struct ethercom *ec = &adapter->osdep.ec; 4472 4473 /* 4474 * ATM LRO is only for IPv4/TCP packets and TCP checksum of the packet 4475 * should be computed by hardware. Also it should not have VLAN tag in 4476 * ethernet header. 4477 */ 4478 if (rxr->lro_enabled && 4479 (ec->ec_capenable & ETHERCAP_VLAN_HWTAGGING) != 0 && 4480 (ptype & IXGBE_RXDADV_PKTTYPE_ETQF) == 0 && 4481 (ptype & (IXGBE_RXDADV_PKTTYPE_IPV4 | IXGBE_RXDADV_PKTTYPE_TCP)) == 4482 (IXGBE_RXDADV_PKTTYPE_IPV4 | IXGBE_RXDADV_PKTTYPE_TCP) && 4483 (m->m_pkthdr.csum_flags & (CSUM_DATA_VALID | CSUM_PSEUDO_HDR)) == 4484 (CSUM_DATA_VALID | CSUM_PSEUDO_HDR)) { 4485 /* 4486 * Send to the stack if: 4487 ** - LRO not enabled, or 4488 ** - no LRO resources, or 4489 ** - lro enqueue fails 4490 */ 4491 if (rxr->lro.lro_cnt != 0) 4492 if (tcp_lro_rx(&rxr->lro, m, 0) == 0) 4493 return; 4494 } 4495 #endif /* LRO */ 4496 4497 IXGBE_RX_UNLOCK(rxr); 4498 4499 s = splnet(); 4500 /* Pass this up to any BPF listeners. */ 4501 bpf_mtap(ifp, m); 4502 (*ifp->if_input)(ifp, m); 4503 splx(s); 4504 4505 IXGBE_RX_LOCK(rxr); 4506 } 4507 4508 static __inline void 4509 ixgbe_rx_discard(struct rx_ring *rxr, int i) 4510 { 4511 struct ixgbe_rx_buf *rbuf; 4512 4513 rbuf = &rxr->rx_buffers[i]; 4514 4515 if (rbuf->fmp != NULL) {/* Partial chain ? */ 4516 rbuf->fmp->m_flags |= M_PKTHDR; 4517 m_freem(rbuf->fmp); 4518 rbuf->fmp = NULL; 4519 } 4520 4521 /* 4522 ** With advanced descriptors the writeback 4523 ** clobbers the buffer addrs, so its easier 4524 ** to just free the existing mbufs and take 4525 ** the normal refresh path to get new buffers 4526 ** and mapping. 4527 */ 4528 if (rbuf->m_head) { 4529 m_free(rbuf->m_head); 4530 rbuf->m_head = NULL; 4531 } 4532 4533 if (rbuf->m_pack) { 4534 m_free(rbuf->m_pack); 4535 rbuf->m_pack = NULL; 4536 } 4537 4538 return; 4539 } 4540 4541 4542 /********************************************************************* 4543 * 4544 * This routine executes in interrupt context. It replenishes 4545 * the mbufs in the descriptor and sends data which has been 4546 * dma'ed into host memory to upper layer. 4547 * 4548 * We loop at most count times if count is > 0, or until done if 4549 * count < 0. 4550 * 4551 * Return TRUE for more work, FALSE for all clean. 4552 *********************************************************************/ 4553 static bool 4554 ixgbe_rxeof(struct ix_queue *que, int count) 4555 { 4556 struct adapter *adapter = que->adapter; 4557 struct rx_ring *rxr = que->rxr; 4558 struct ifnet *ifp = adapter->ifp; 4559 #ifdef LRO 4560 struct lro_ctrl *lro = &rxr->lro; 4561 struct lro_entry *queued; 4562 #endif /* LRO */ 4563 int i, nextp, processed = 0; 4564 u32 staterr = 0; 4565 union ixgbe_adv_rx_desc *cur; 4566 struct ixgbe_rx_buf *rbuf, *nbuf; 4567 4568 IXGBE_RX_LOCK(rxr); 4569 4570 #ifdef DEV_NETMAP 4571 if (ifp->if_capenable & IFCAP_NETMAP) { 4572 /* 4573 * Same as the txeof routine: only wakeup clients on intr. 4574 * NKR_PENDINTR in nr_kflags is used to implement interrupt 4575 * mitigation (ixgbe_rxsync() will not look for new packets 4576 * unless NKR_PENDINTR is set). 4577 */ 4578 struct netmap_adapter *na = NA(ifp); 4579 4580 na->rx_rings[rxr->me].nr_kflags |= NKR_PENDINTR; 4581 selwakeuppri(&na->rx_rings[rxr->me].si, PI_NET); 4582 IXGBE_RX_UNLOCK(rxr); 4583 IXGBE_CORE_LOCK(adapter); 4584 selwakeuppri(&na->rx_rings[na->num_queues + 1].si, PI_NET); 4585 IXGBE_CORE_UNLOCK(adapter); 4586 return (FALSE); 4587 } 4588 #endif /* DEV_NETMAP */ 4589 for (i = rxr->next_to_check; count != 0;) { 4590 struct mbuf *sendmp, *mh, *mp; 4591 u32 rsc, ptype; 4592 u16 hlen, plen, hdr, vtag; 4593 bool eop; 4594 4595 /* Sync the ring. */ 4596 ixgbe_dmamap_sync(rxr->rxdma.dma_tag, rxr->rxdma.dma_map, 4597 BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE); 4598 4599 cur = &rxr->rx_base[i]; 4600 staterr = le32toh(cur->wb.upper.status_error); 4601 4602 if ((staterr & IXGBE_RXD_STAT_DD) == 0) 4603 break; 4604 if ((ifp->if_flags & IFF_RUNNING) == 0) 4605 break; 4606 4607 count--; 4608 sendmp = NULL; 4609 nbuf = NULL; 4610 rsc = 0; 4611 cur->wb.upper.status_error = 0; 4612 rbuf = &rxr->rx_buffers[i]; 4613 mh = rbuf->m_head; 4614 mp = rbuf->m_pack; 4615 4616 plen = le16toh(cur->wb.upper.length); 4617 ptype = le32toh(cur->wb.lower.lo_dword.data) & 4618 IXGBE_RXDADV_PKTTYPE_MASK; 4619 hdr = le16toh(cur->wb.lower.lo_dword.hs_rss.hdr_info); 4620 vtag = le16toh(cur->wb.upper.vlan); 4621 eop = ((staterr & IXGBE_RXD_STAT_EOP) != 0); 4622 4623 /* Make sure bad packets are discarded */ 4624 if (((staterr & IXGBE_RXDADV_ERR_FRAME_ERR_MASK) != 0) || 4625 (rxr->discard)) { 4626 ifp->if_ierrors++; 4627 rxr->rx_discarded.ev_count++; 4628 if (eop) 4629 rxr->discard = FALSE; 4630 else 4631 rxr->discard = TRUE; 4632 ixgbe_rx_discard(rxr, i); 4633 goto next_desc; 4634 } 4635 4636 /* 4637 ** On 82599 which supports a hardware 4638 ** LRO (called HW RSC), packets need 4639 ** not be fragmented across sequential 4640 ** descriptors, rather the next descriptor 4641 ** is indicated in bits of the descriptor. 4642 ** This also means that we might proceses 4643 ** more than one packet at a time, something 4644 ** that has never been true before, it 4645 ** required eliminating global chain pointers 4646 ** in favor of what we are doing here. -jfv 4647 */ 4648 if (!eop) { 4649 /* 4650 ** Figure out the next descriptor 4651 ** of this frame. 4652 */ 4653 if (rxr->hw_rsc == TRUE) { 4654 rsc = ixgbe_rsc_count(cur); 4655 rxr->rsc_num += (rsc - 1); 4656 } 4657 if (rsc) { /* Get hardware index */ 4658 nextp = ((staterr & 4659 IXGBE_RXDADV_NEXTP_MASK) >> 4660 IXGBE_RXDADV_NEXTP_SHIFT); 4661 } else { /* Just sequential */ 4662 nextp = i + 1; 4663 if (nextp == adapter->num_rx_desc) 4664 nextp = 0; 4665 } 4666 nbuf = &rxr->rx_buffers[nextp]; 4667 prefetch(nbuf); 4668 } 4669 /* 4670 ** The header mbuf is ONLY used when header 4671 ** split is enabled, otherwise we get normal 4672 ** behavior, ie, both header and payload 4673 ** are DMA'd into the payload buffer. 4674 ** 4675 ** Rather than using the fmp/lmp global pointers 4676 ** we now keep the head of a packet chain in the 4677 ** buffer struct and pass this along from one 4678 ** descriptor to the next, until we get EOP. 4679 */ 4680 if (rxr->hdr_split && (rbuf->fmp == NULL)) { 4681 /* This must be an initial descriptor */ 4682 hlen = (hdr & IXGBE_RXDADV_HDRBUFLEN_MASK) >> 4683 IXGBE_RXDADV_HDRBUFLEN_SHIFT; 4684 if (hlen > IXGBE_RX_HDR) 4685 hlen = IXGBE_RX_HDR; 4686 mh->m_len = hlen; 4687 mh->m_flags |= M_PKTHDR; 4688 mh->m_next = NULL; 4689 mh->m_pkthdr.len = mh->m_len; 4690 /* Null buf pointer so it is refreshed */ 4691 rbuf->m_head = NULL; 4692 /* 4693 ** Check the payload length, this 4694 ** could be zero if its a small 4695 ** packet. 4696 */ 4697 if (plen > 0) { 4698 mp->m_len = plen; 4699 mp->m_next = NULL; 4700 mp->m_flags &= ~M_PKTHDR; 4701 mh->m_next = mp; 4702 mh->m_pkthdr.len += mp->m_len; 4703 /* Null buf pointer so it is refreshed */ 4704 rbuf->m_pack = NULL; 4705 rxr->rx_split_packets.ev_count++; 4706 } 4707 /* 4708 ** Now create the forward 4709 ** chain so when complete 4710 ** we wont have to. 4711 */ 4712 if (eop == 0) { 4713 /* stash the chain head */ 4714 nbuf->fmp = mh; 4715 /* Make forward chain */ 4716 if (plen) 4717 mp->m_next = nbuf->m_pack; 4718 else 4719 mh->m_next = nbuf->m_pack; 4720 } else { 4721 /* Singlet, prepare to send */ 4722 sendmp = mh; 4723 if (VLAN_ATTACHED(&adapter->osdep.ec) && 4724 (staterr & IXGBE_RXD_STAT_VP)) { 4725 /* XXX Do something reasonable on 4726 * error. 4727 */ 4728 #if 0 4729 printf("%s.%d: VLAN_INPUT_TAG\n", 4730 __func__, __LINE__); 4731 Debugger(); 4732 #endif 4733 VLAN_INPUT_TAG(ifp, sendmp, vtag, 4734 printf("%s: could not apply VLAN " 4735 "tag", __func__)); 4736 } 4737 } 4738 } else { 4739 /* 4740 ** Either no header split, or a 4741 ** secondary piece of a fragmented 4742 ** split packet. 4743 */ 4744 mp->m_len = plen; 4745 /* 4746 ** See if there is a stored head 4747 ** that determines what we are 4748 */ 4749 sendmp = rbuf->fmp; 4750 rbuf->m_pack = rbuf->fmp = NULL; 4751 4752 if (sendmp != NULL) { /* secondary frag */ 4753 mp->m_flags &= ~M_PKTHDR; 4754 sendmp->m_pkthdr.len += mp->m_len; 4755 } else { 4756 /* first desc of a non-ps chain */ 4757 sendmp = mp; 4758 sendmp->m_flags |= M_PKTHDR; 4759 sendmp->m_pkthdr.len = mp->m_len; 4760 if (staterr & IXGBE_RXD_STAT_VP) { 4761 /* XXX Do something reasonable on 4762 * error. 4763 */ 4764 #if 0 4765 printf("%s.%d: VLAN_INPUT_TAG\n", 4766 __func__, __LINE__); 4767 Debugger(); 4768 #endif 4769 VLAN_INPUT_TAG(ifp, sendmp, vtag, 4770 printf("%s: could not apply VLAN " 4771 "tag", __func__)); 4772 } 4773 } 4774 /* Pass the head pointer on */ 4775 if (eop == 0) { 4776 nbuf->fmp = sendmp; 4777 sendmp = NULL; 4778 mp->m_next = nbuf->m_pack; 4779 } 4780 } 4781 ++processed; 4782 /* Sending this frame? */ 4783 if (eop) { 4784 sendmp->m_pkthdr.rcvif = ifp; 4785 ifp->if_ipackets++; 4786 rxr->rx_packets.ev_count++; 4787 /* capture data for AIM */ 4788 rxr->bytes += sendmp->m_pkthdr.len; 4789 rxr->rx_bytes.ev_count += sendmp->m_pkthdr.len; 4790 if ((ifp->if_capenable & IFCAP_RXCSUM) != 0) { 4791 ixgbe_rx_checksum(staterr, sendmp, ptype, 4792 &adapter->stats); 4793 } 4794 #if __FreeBSD_version >= 800000 4795 sendmp->m_pkthdr.flowid = que->msix; 4796 sendmp->m_flags |= M_FLOWID; 4797 #endif 4798 } 4799 next_desc: 4800 ixgbe_dmamap_sync(rxr->rxdma.dma_tag, rxr->rxdma.dma_map, 4801 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); 4802 4803 /* Advance our pointers to the next descriptor. */ 4804 if (++i == adapter->num_rx_desc) 4805 i = 0; 4806 4807 /* Now send to the stack or do LRO */ 4808 if (sendmp != NULL) { 4809 rxr->next_to_check = i; 4810 ixgbe_rx_input(rxr, ifp, sendmp, ptype); 4811 i = rxr->next_to_check; 4812 } 4813 4814 /* Every 8 descriptors we go to refresh mbufs */ 4815 if (processed == 8) { 4816 ixgbe_refresh_mbufs(rxr, i); 4817 processed = 0; 4818 } 4819 } 4820 4821 /* Refresh any remaining buf structs */ 4822 if (ixgbe_rx_unrefreshed(rxr)) 4823 ixgbe_refresh_mbufs(rxr, i); 4824 4825 rxr->next_to_check = i; 4826 4827 #ifdef LRO 4828 /* 4829 * Flush any outstanding LRO work 4830 */ 4831 while ((queued = SLIST_FIRST(&lro->lro_active)) != NULL) { 4832 SLIST_REMOVE_HEAD(&lro->lro_active, next); 4833 tcp_lro_flush(lro, queued); 4834 } 4835 #endif /* LRO */ 4836 4837 IXGBE_RX_UNLOCK(rxr); 4838 4839 /* 4840 ** We still have cleaning to do? 4841 ** Schedule another interrupt if so. 4842 */ 4843 if ((staterr & IXGBE_RXD_STAT_DD) != 0) { 4844 ixgbe_rearm_queues(adapter, (u64)(1ULL << que->msix)); 4845 return true; 4846 } 4847 4848 return false; 4849 } 4850 4851 4852 /********************************************************************* 4853 * 4854 * Verify that the hardware indicated that the checksum is valid. 4855 * Inform the stack about the status of checksum so that stack 4856 * doesn't spend time verifying the checksum. 4857 * 4858 *********************************************************************/ 4859 static void 4860 ixgbe_rx_checksum(u32 staterr, struct mbuf * mp, u32 ptype, 4861 struct ixgbe_hw_stats *stats) 4862 { 4863 u16 status = (u16) staterr; 4864 u8 errors = (u8) (staterr >> 24); 4865 #if 0 4866 bool sctp = FALSE; 4867 4868 if ((ptype & IXGBE_RXDADV_PKTTYPE_ETQF) == 0 && 4869 (ptype & IXGBE_RXDADV_PKTTYPE_SCTP) != 0) 4870 sctp = TRUE; 4871 #endif 4872 4873 if (status & IXGBE_RXD_STAT_IPCS) { 4874 stats->ipcs.ev_count++; 4875 if (!(errors & IXGBE_RXD_ERR_IPE)) { 4876 /* IP Checksum Good */ 4877 mp->m_pkthdr.csum_flags = M_CSUM_IPv4; 4878 4879 } else { 4880 stats->ipcs_bad.ev_count++; 4881 mp->m_pkthdr.csum_flags = M_CSUM_IPv4|M_CSUM_IPv4_BAD; 4882 } 4883 } 4884 if (status & IXGBE_RXD_STAT_L4CS) { 4885 stats->l4cs.ev_count++; 4886 u16 type = M_CSUM_TCPv4|M_CSUM_TCPv6|M_CSUM_UDPv4|M_CSUM_UDPv6; 4887 if (!(errors & IXGBE_RXD_ERR_TCPE)) { 4888 mp->m_pkthdr.csum_flags |= type; 4889 } else { 4890 stats->l4cs_bad.ev_count++; 4891 mp->m_pkthdr.csum_flags |= type | M_CSUM_TCP_UDP_BAD; 4892 } 4893 } 4894 return; 4895 } 4896 4897 4898 #if 0 /* XXX Badly need to overhaul vlan(4) on NetBSD. */ 4899 /* 4900 ** This routine is run via an vlan config EVENT, 4901 ** it enables us to use the HW Filter table since 4902 ** we can get the vlan id. This just creates the 4903 ** entry in the soft version of the VFTA, init will 4904 ** repopulate the real table. 4905 */ 4906 static void 4907 ixgbe_register_vlan(void *arg, struct ifnet *ifp, u16 vtag) 4908 { 4909 struct adapter *adapter = ifp->if_softc; 4910 u16 index, bit; 4911 4912 if (ifp->if_softc != arg) /* Not our event */ 4913 return; 4914 4915 if ((vtag == 0) || (vtag > 4095)) /* Invalid */ 4916 return; 4917 4918 IXGBE_CORE_LOCK(adapter); 4919 index = (vtag >> 5) & 0x7F; 4920 bit = vtag & 0x1F; 4921 adapter->shadow_vfta[index] |= (1 << bit); 4922 ixgbe_init_locked(adapter); 4923 IXGBE_CORE_UNLOCK(adapter); 4924 } 4925 4926 /* 4927 ** This routine is run via an vlan 4928 ** unconfig EVENT, remove our entry 4929 ** in the soft vfta. 4930 */ 4931 static void 4932 ixgbe_unregister_vlan(void *arg, struct ifnet *ifp, u16 vtag) 4933 { 4934 struct adapter *adapter = ifp->if_softc; 4935 u16 index, bit; 4936 4937 if (ifp->if_softc != arg) 4938 return; 4939 4940 if ((vtag == 0) || (vtag > 4095)) /* Invalid */ 4941 return; 4942 4943 IXGBE_CORE_LOCK(adapter); 4944 index = (vtag >> 5) & 0x7F; 4945 bit = vtag & 0x1F; 4946 adapter->shadow_vfta[index] &= ~(1 << bit); 4947 /* Re-init to load the changes */ 4948 ixgbe_init_locked(adapter); 4949 IXGBE_CORE_UNLOCK(adapter); 4950 } 4951 #endif 4952 4953 static void 4954 ixgbe_setup_vlan_hw_support(struct adapter *adapter) 4955 { 4956 struct ethercom *ec = &adapter->osdep.ec; 4957 struct ixgbe_hw *hw = &adapter->hw; 4958 u32 ctrl; 4959 4960 /* 4961 ** We get here thru init_locked, meaning 4962 ** a soft reset, this has already cleared 4963 ** the VFTA and other state, so if there 4964 ** have been no vlan's registered do nothing. 4965 */ 4966 if (!VLAN_ATTACHED(&adapter->osdep.ec)) { 4967 return; 4968 } 4969 4970 /* 4971 ** A soft reset zero's out the VFTA, so 4972 ** we need to repopulate it now. 4973 */ 4974 for (int i = 0; i < IXGBE_VFTA_SIZE; i++) 4975 if (adapter->shadow_vfta[i] != 0) 4976 IXGBE_WRITE_REG(hw, IXGBE_VFTA(i), 4977 adapter->shadow_vfta[i]); 4978 4979 ctrl = IXGBE_READ_REG(hw, IXGBE_VLNCTRL); 4980 /* Enable the Filter Table if enabled */ 4981 if (ec->ec_capenable & ETHERCAP_VLAN_HWFILTER) { 4982 ctrl &= ~IXGBE_VLNCTRL_CFIEN; 4983 ctrl |= IXGBE_VLNCTRL_VFE; 4984 } 4985 if (hw->mac.type == ixgbe_mac_82598EB) 4986 ctrl |= IXGBE_VLNCTRL_VME; 4987 IXGBE_WRITE_REG(hw, IXGBE_VLNCTRL, ctrl); 4988 4989 /* On 82599 the VLAN enable is per/queue in RXDCTL */ 4990 if (hw->mac.type != ixgbe_mac_82598EB) 4991 for (int i = 0; i < adapter->num_queues; i++) { 4992 ctrl = IXGBE_READ_REG(hw, IXGBE_RXDCTL(i)); 4993 ctrl |= IXGBE_RXDCTL_VME; 4994 IXGBE_WRITE_REG(hw, IXGBE_RXDCTL(i), ctrl); 4995 } 4996 } 4997 4998 static void 4999 ixgbe_enable_intr(struct adapter *adapter) 5000 { 5001 struct ixgbe_hw *hw = &adapter->hw; 5002 struct ix_queue *que = adapter->queues; 5003 u32 mask = (IXGBE_EIMS_ENABLE_MASK & ~IXGBE_EIMS_RTX_QUEUE); 5004 5005 5006 /* Enable Fan Failure detection */ 5007 if (hw->device_id == IXGBE_DEV_ID_82598AT) 5008 mask |= IXGBE_EIMS_GPI_SDP1; 5009 else { 5010 mask |= IXGBE_EIMS_ECC; 5011 mask |= IXGBE_EIMS_GPI_SDP1; 5012 mask |= IXGBE_EIMS_GPI_SDP2; 5013 #ifdef IXGBE_FDIR 5014 mask |= IXGBE_EIMS_FLOW_DIR; 5015 #endif 5016 } 5017 5018 IXGBE_WRITE_REG(hw, IXGBE_EIMS, mask); 5019 5020 /* With RSS we use auto clear */ 5021 if (adapter->msix_mem) { 5022 mask = IXGBE_EIMS_ENABLE_MASK; 5023 /* Don't autoclear Link */ 5024 mask &= ~IXGBE_EIMS_OTHER; 5025 mask &= ~IXGBE_EIMS_LSC; 5026 IXGBE_WRITE_REG(hw, IXGBE_EIAC, mask); 5027 } 5028 5029 /* 5030 ** Now enable all queues, this is done separately to 5031 ** allow for handling the extended (beyond 32) MSIX 5032 ** vectors that can be used by 82599 5033 */ 5034 for (int i = 0; i < adapter->num_queues; i++, que++) 5035 ixgbe_enable_queue(adapter, que->msix); 5036 5037 IXGBE_WRITE_FLUSH(hw); 5038 5039 return; 5040 } 5041 5042 static void 5043 ixgbe_disable_intr(struct adapter *adapter) 5044 { 5045 if (adapter->msix_mem) 5046 IXGBE_WRITE_REG(&adapter->hw, IXGBE_EIAC, 0); 5047 if (adapter->hw.mac.type == ixgbe_mac_82598EB) { 5048 IXGBE_WRITE_REG(&adapter->hw, IXGBE_EIMC, ~0); 5049 } else { 5050 IXGBE_WRITE_REG(&adapter->hw, IXGBE_EIMC, 0xFFFF0000); 5051 IXGBE_WRITE_REG(&adapter->hw, IXGBE_EIMC_EX(0), ~0); 5052 IXGBE_WRITE_REG(&adapter->hw, IXGBE_EIMC_EX(1), ~0); 5053 } 5054 IXGBE_WRITE_FLUSH(&adapter->hw); 5055 return; 5056 } 5057 5058 u16 5059 ixgbe_read_pci_cfg(struct ixgbe_hw *hw, u32 reg) 5060 { 5061 switch (reg % 4) { 5062 case 0: 5063 return pci_conf_read(hw->back->pc, hw->back->tag, reg) & 5064 __BITS(15, 0); 5065 case 2: 5066 return __SHIFTOUT(pci_conf_read(hw->back->pc, hw->back->tag, 5067 reg - 2), __BITS(31, 16)); 5068 default: 5069 panic("%s: invalid register (%" PRIx32, __func__, reg); 5070 break; 5071 } 5072 } 5073 5074 void 5075 ixgbe_write_pci_cfg(struct ixgbe_hw *hw, u32 reg, u16 value) 5076 { 5077 pcireg_t old; 5078 5079 switch (reg % 4) { 5080 case 0: 5081 old = pci_conf_read(hw->back->pc, hw->back->tag, reg) & 5082 __BITS(31, 16); 5083 pci_conf_write(hw->back->pc, hw->back->tag, reg, value | old); 5084 break; 5085 case 2: 5086 old = pci_conf_read(hw->back->pc, hw->back->tag, reg - 2) & 5087 __BITS(15, 0); 5088 pci_conf_write(hw->back->pc, hw->back->tag, reg - 2, 5089 __SHIFTIN(value, __BITS(31, 16)) | old); 5090 break; 5091 default: 5092 panic("%s: invalid register (%" PRIx32, __func__, reg); 5093 break; 5094 } 5095 5096 return; 5097 } 5098 5099 /* 5100 ** Setup the correct IVAR register for a particular MSIX interrupt 5101 ** (yes this is all very magic and confusing :) 5102 ** - entry is the register array entry 5103 ** - vector is the MSIX vector for this queue 5104 ** - type is RX/TX/MISC 5105 */ 5106 static void 5107 ixgbe_set_ivar(struct adapter *adapter, u8 entry, u8 vector, s8 type) 5108 { 5109 struct ixgbe_hw *hw = &adapter->hw; 5110 u32 ivar, index; 5111 5112 vector |= IXGBE_IVAR_ALLOC_VAL; 5113 5114 switch (hw->mac.type) { 5115 5116 case ixgbe_mac_82598EB: 5117 if (type == -1) 5118 entry = IXGBE_IVAR_OTHER_CAUSES_INDEX; 5119 else 5120 entry += (type * 64); 5121 index = (entry >> 2) & 0x1F; 5122 ivar = IXGBE_READ_REG(hw, IXGBE_IVAR(index)); 5123 ivar &= ~(0xFF << (8 * (entry & 0x3))); 5124 ivar |= (vector << (8 * (entry & 0x3))); 5125 IXGBE_WRITE_REG(&adapter->hw, IXGBE_IVAR(index), ivar); 5126 break; 5127 5128 case ixgbe_mac_82599EB: 5129 if (type == -1) { /* MISC IVAR */ 5130 index = (entry & 1) * 8; 5131 ivar = IXGBE_READ_REG(hw, IXGBE_IVAR_MISC); 5132 ivar &= ~(0xFF << index); 5133 ivar |= (vector << index); 5134 IXGBE_WRITE_REG(hw, IXGBE_IVAR_MISC, ivar); 5135 } else { /* RX/TX IVARS */ 5136 index = (16 * (entry & 1)) + (8 * type); 5137 ivar = IXGBE_READ_REG(hw, IXGBE_IVAR(entry >> 1)); 5138 ivar &= ~(0xFF << index); 5139 ivar |= (vector << index); 5140 IXGBE_WRITE_REG(hw, IXGBE_IVAR(entry >> 1), ivar); 5141 } 5142 5143 default: 5144 break; 5145 } 5146 } 5147 5148 static void 5149 ixgbe_configure_ivars(struct adapter *adapter) 5150 { 5151 struct ix_queue *que = adapter->queues; 5152 u32 newitr; 5153 5154 if (ixgbe_max_interrupt_rate > 0) 5155 newitr = (4000000 / ixgbe_max_interrupt_rate) & 0x0FF8; 5156 else 5157 newitr = 0; 5158 5159 for (int i = 0; i < adapter->num_queues; i++, que++) { 5160 /* First the RX queue entry */ 5161 ixgbe_set_ivar(adapter, i, que->msix, 0); 5162 /* ... and the TX */ 5163 ixgbe_set_ivar(adapter, i, que->msix, 1); 5164 /* Set an Initial EITR value */ 5165 IXGBE_WRITE_REG(&adapter->hw, 5166 IXGBE_EITR(que->msix), newitr); 5167 } 5168 5169 /* For the Link interrupt */ 5170 ixgbe_set_ivar(adapter, 1, adapter->linkvec, -1); 5171 } 5172 5173 /* 5174 ** ixgbe_sfp_probe - called in the local timer to 5175 ** determine if a port had optics inserted. 5176 */ 5177 static bool ixgbe_sfp_probe(struct adapter *adapter) 5178 { 5179 struct ixgbe_hw *hw = &adapter->hw; 5180 device_t dev = adapter->dev; 5181 bool result = FALSE; 5182 5183 if ((hw->phy.type == ixgbe_phy_nl) && 5184 (hw->phy.sfp_type == ixgbe_sfp_type_not_present)) { 5185 s32 ret = hw->phy.ops.identify_sfp(hw); 5186 if (ret) 5187 goto out; 5188 ret = hw->phy.ops.reset(hw); 5189 if (ret == IXGBE_ERR_SFP_NOT_SUPPORTED) { 5190 device_printf(dev,"Unsupported SFP+ module detected!"); 5191 device_printf(dev, "Reload driver with supported module.\n"); 5192 adapter->sfp_probe = FALSE; 5193 goto out; 5194 } else 5195 device_printf(dev,"SFP+ module detected!\n"); 5196 /* We now have supported optics */ 5197 adapter->sfp_probe = FALSE; 5198 /* Set the optics type so system reports correctly */ 5199 ixgbe_setup_optics(adapter); 5200 result = TRUE; 5201 } 5202 out: 5203 return (result); 5204 } 5205 5206 /* 5207 ** Tasklet handler for MSIX Link interrupts 5208 ** - do outside interrupt since it might sleep 5209 */ 5210 static void 5211 ixgbe_handle_link(void *context) 5212 { 5213 struct adapter *adapter = context; 5214 5215 if (ixgbe_check_link(&adapter->hw, 5216 &adapter->link_speed, &adapter->link_up, 0) == 0) 5217 ixgbe_update_link_status(adapter); 5218 } 5219 5220 /* 5221 ** Tasklet for handling SFP module interrupts 5222 */ 5223 static void 5224 ixgbe_handle_mod(void *context) 5225 { 5226 struct adapter *adapter = context; 5227 struct ixgbe_hw *hw = &adapter->hw; 5228 device_t dev = adapter->dev; 5229 u32 err; 5230 5231 err = hw->phy.ops.identify_sfp(hw); 5232 if (err == IXGBE_ERR_SFP_NOT_SUPPORTED) { 5233 device_printf(dev, 5234 "Unsupported SFP+ module type was detected.\n"); 5235 return; 5236 } 5237 err = hw->mac.ops.setup_sfp(hw); 5238 if (err == IXGBE_ERR_SFP_NOT_SUPPORTED) { 5239 device_printf(dev, 5240 "Setup failure - unsupported SFP+ module type.\n"); 5241 return; 5242 } 5243 softint_schedule(adapter->msf_si); 5244 return; 5245 } 5246 5247 5248 /* 5249 ** Tasklet for handling MSF (multispeed fiber) interrupts 5250 */ 5251 static void 5252 ixgbe_handle_msf(void *context) 5253 { 5254 struct adapter *adapter = context; 5255 struct ixgbe_hw *hw = &adapter->hw; 5256 u32 autoneg; 5257 bool negotiate; 5258 5259 autoneg = hw->phy.autoneg_advertised; 5260 if ((!autoneg) && (hw->mac.ops.get_link_capabilities)) 5261 hw->mac.ops.get_link_capabilities(hw, &autoneg, &negotiate); 5262 else 5263 negotiate = 0; 5264 if (hw->mac.ops.setup_link) 5265 hw->mac.ops.setup_link(hw, autoneg, negotiate, TRUE); 5266 return; 5267 } 5268 5269 #ifdef IXGBE_FDIR 5270 /* 5271 ** Tasklet for reinitializing the Flow Director filter table 5272 */ 5273 static void 5274 ixgbe_reinit_fdir(void *context) 5275 { 5276 struct adapter *adapter = context; 5277 struct ifnet *ifp = adapter->ifp; 5278 5279 if (adapter->fdir_reinit != 1) /* Shouldn't happen */ 5280 return; 5281 ixgbe_reinit_fdir_tables_82599(&adapter->hw); 5282 adapter->fdir_reinit = 0; 5283 /* Restart the interface */ 5284 ifp->if_flags |= IFF_RUNNING; 5285 return; 5286 } 5287 #endif 5288 5289 /********************************************************************** 5290 * 5291 * Update the board statistics counters. 5292 * 5293 **********************************************************************/ 5294 static void 5295 ixgbe_update_stats_counters(struct adapter *adapter) 5296 { 5297 struct ifnet *ifp = adapter->ifp; 5298 struct ixgbe_hw *hw = &adapter->hw; 5299 u32 missed_rx = 0, bprc, lxon, lxoff, total; 5300 u64 total_missed_rx = 0; 5301 5302 adapter->stats.crcerrs.ev_count += IXGBE_READ_REG(hw, IXGBE_CRCERRS); 5303 adapter->stats.illerrc.ev_count += IXGBE_READ_REG(hw, IXGBE_ILLERRC); 5304 adapter->stats.errbc.ev_count += IXGBE_READ_REG(hw, IXGBE_ERRBC); 5305 adapter->stats.mspdc.ev_count += IXGBE_READ_REG(hw, IXGBE_MSPDC); 5306 5307 for (int i = 0; i < __arraycount(adapter->stats.mpc); i++) { 5308 int j = i % adapter->num_queues; 5309 u32 mp; 5310 mp = IXGBE_READ_REG(hw, IXGBE_MPC(i)); 5311 /* missed_rx tallies misses for the gprc workaround */ 5312 missed_rx += mp; 5313 /* global total per queue */ 5314 adapter->stats.mpc[j].ev_count += mp; 5315 /* Running comprehensive total for stats display */ 5316 total_missed_rx += adapter->stats.mpc[j].ev_count; 5317 if (hw->mac.type == ixgbe_mac_82598EB) 5318 adapter->stats.rnbc[j] += 5319 IXGBE_READ_REG(hw, IXGBE_RNBC(i)); 5320 adapter->stats.pxontxc[j].ev_count += 5321 IXGBE_READ_REG(hw, IXGBE_PXONTXC(i)); 5322 adapter->stats.pxonrxc[j].ev_count += 5323 IXGBE_READ_REG(hw, IXGBE_PXONRXC(i)); 5324 adapter->stats.pxofftxc[j].ev_count += 5325 IXGBE_READ_REG(hw, IXGBE_PXOFFTXC(i)); 5326 adapter->stats.pxoffrxc[j].ev_count += 5327 IXGBE_READ_REG(hw, IXGBE_PXOFFRXC(i)); 5328 adapter->stats.pxon2offc[j].ev_count += 5329 IXGBE_READ_REG(hw, IXGBE_PXON2OFFCNT(i)); 5330 } 5331 for (int i = 0; i < __arraycount(adapter->stats.qprc); i++) { 5332 int j = i % adapter->num_queues; 5333 adapter->stats.qprc[j].ev_count += IXGBE_READ_REG(hw, IXGBE_QPRC(i)); 5334 adapter->stats.qptc[j].ev_count += IXGBE_READ_REG(hw, IXGBE_QPTC(i)); 5335 adapter->stats.qbrc[j].ev_count += IXGBE_READ_REG(hw, IXGBE_QBRC(i)); 5336 adapter->stats.qbrc[j].ev_count += 5337 ((u64)IXGBE_READ_REG(hw, IXGBE_QBRC(i)) << 32); 5338 adapter->stats.qbtc[j].ev_count += IXGBE_READ_REG(hw, IXGBE_QBTC(i)); 5339 adapter->stats.qbtc[j].ev_count += 5340 ((u64)IXGBE_READ_REG(hw, IXGBE_QBTC(i)) << 32); 5341 adapter->stats.qprdc[j].ev_count += IXGBE_READ_REG(hw, IXGBE_QPRDC(i)); 5342 } 5343 adapter->stats.mlfc.ev_count += IXGBE_READ_REG(hw, IXGBE_MLFC); 5344 adapter->stats.mrfc.ev_count += IXGBE_READ_REG(hw, IXGBE_MRFC); 5345 adapter->stats.rlec.ev_count += IXGBE_READ_REG(hw, IXGBE_RLEC); 5346 5347 /* Hardware workaround, gprc counts missed packets */ 5348 adapter->stats.gprc.ev_count += IXGBE_READ_REG(hw, IXGBE_GPRC) - missed_rx; 5349 5350 lxon = IXGBE_READ_REG(hw, IXGBE_LXONTXC); 5351 adapter->stats.lxontxc.ev_count += lxon; 5352 lxoff = IXGBE_READ_REG(hw, IXGBE_LXOFFTXC); 5353 adapter->stats.lxofftxc.ev_count += lxoff; 5354 total = lxon + lxoff; 5355 5356 if (hw->mac.type != ixgbe_mac_82598EB) { 5357 adapter->stats.gorc.ev_count += IXGBE_READ_REG(hw, IXGBE_GORCL) + 5358 ((u64)IXGBE_READ_REG(hw, IXGBE_GORCH) << 32); 5359 adapter->stats.gotc.ev_count += IXGBE_READ_REG(hw, IXGBE_GOTCL) + 5360 ((u64)IXGBE_READ_REG(hw, IXGBE_GOTCH) << 32) - total * ETHER_MIN_LEN; 5361 adapter->stats.tor.ev_count += IXGBE_READ_REG(hw, IXGBE_TORL) + 5362 ((u64)IXGBE_READ_REG(hw, IXGBE_TORH) << 32); 5363 adapter->stats.lxonrxc.ev_count += IXGBE_READ_REG(hw, IXGBE_LXONRXCNT); 5364 adapter->stats.lxoffrxc.ev_count += IXGBE_READ_REG(hw, IXGBE_LXOFFRXCNT); 5365 } else { 5366 adapter->stats.lxonrxc.ev_count += IXGBE_READ_REG(hw, IXGBE_LXONRXC); 5367 adapter->stats.lxoffrxc.ev_count += IXGBE_READ_REG(hw, IXGBE_LXOFFRXC); 5368 /* 82598 only has a counter in the high register */ 5369 adapter->stats.gorc.ev_count += IXGBE_READ_REG(hw, IXGBE_GORCH); 5370 adapter->stats.gotc.ev_count += IXGBE_READ_REG(hw, IXGBE_GOTCH) - total * ETHER_MIN_LEN; 5371 adapter->stats.tor.ev_count += IXGBE_READ_REG(hw, IXGBE_TORH); 5372 } 5373 5374 /* 5375 * Workaround: mprc hardware is incorrectly counting 5376 * broadcasts, so for now we subtract those. 5377 */ 5378 bprc = IXGBE_READ_REG(hw, IXGBE_BPRC); 5379 adapter->stats.bprc.ev_count += bprc; 5380 adapter->stats.mprc.ev_count += IXGBE_READ_REG(hw, IXGBE_MPRC) - ((hw->mac.type == ixgbe_mac_82598EB) ? bprc : 0); 5381 5382 adapter->stats.prc64.ev_count += IXGBE_READ_REG(hw, IXGBE_PRC64); 5383 adapter->stats.prc127.ev_count += IXGBE_READ_REG(hw, IXGBE_PRC127); 5384 adapter->stats.prc255.ev_count += IXGBE_READ_REG(hw, IXGBE_PRC255); 5385 adapter->stats.prc511.ev_count += IXGBE_READ_REG(hw, IXGBE_PRC511); 5386 adapter->stats.prc1023.ev_count += IXGBE_READ_REG(hw, IXGBE_PRC1023); 5387 adapter->stats.prc1522.ev_count += IXGBE_READ_REG(hw, IXGBE_PRC1522); 5388 5389 adapter->stats.gptc.ev_count += IXGBE_READ_REG(hw, IXGBE_GPTC) - total; 5390 adapter->stats.mptc.ev_count += IXGBE_READ_REG(hw, IXGBE_MPTC) - total; 5391 adapter->stats.ptc64.ev_count += IXGBE_READ_REG(hw, IXGBE_PTC64) - total; 5392 5393 adapter->stats.ruc.ev_count += IXGBE_READ_REG(hw, IXGBE_RUC); 5394 adapter->stats.rfc.ev_count += IXGBE_READ_REG(hw, IXGBE_RFC); 5395 adapter->stats.roc.ev_count += IXGBE_READ_REG(hw, IXGBE_ROC); 5396 adapter->stats.rjc.ev_count += IXGBE_READ_REG(hw, IXGBE_RJC); 5397 adapter->stats.mngprc.ev_count += IXGBE_READ_REG(hw, IXGBE_MNGPRC); 5398 adapter->stats.mngpdc.ev_count += IXGBE_READ_REG(hw, IXGBE_MNGPDC); 5399 adapter->stats.mngptc.ev_count += IXGBE_READ_REG(hw, IXGBE_MNGPTC); 5400 adapter->stats.tpr.ev_count += IXGBE_READ_REG(hw, IXGBE_TPR); 5401 adapter->stats.tpt.ev_count += IXGBE_READ_REG(hw, IXGBE_TPT); 5402 adapter->stats.ptc127.ev_count += IXGBE_READ_REG(hw, IXGBE_PTC127); 5403 adapter->stats.ptc255.ev_count += IXGBE_READ_REG(hw, IXGBE_PTC255); 5404 adapter->stats.ptc511.ev_count += IXGBE_READ_REG(hw, IXGBE_PTC511); 5405 adapter->stats.ptc1023.ev_count += IXGBE_READ_REG(hw, IXGBE_PTC1023); 5406 adapter->stats.ptc1522.ev_count += IXGBE_READ_REG(hw, IXGBE_PTC1522); 5407 adapter->stats.bptc.ev_count += IXGBE_READ_REG(hw, IXGBE_BPTC); 5408 adapter->stats.xec.ev_count += IXGBE_READ_REG(hw, IXGBE_XEC); 5409 adapter->stats.fccrc.ev_count += IXGBE_READ_REG(hw, IXGBE_FCCRC); 5410 adapter->stats.fclast.ev_count += IXGBE_READ_REG(hw, IXGBE_FCLAST); 5411 5412 /* Only read FCOE on 82599 */ 5413 if (hw->mac.type != ixgbe_mac_82598EB) { 5414 adapter->stats.fcoerpdc.ev_count += 5415 IXGBE_READ_REG(hw, IXGBE_FCOERPDC); 5416 adapter->stats.fcoeprc.ev_count += 5417 IXGBE_READ_REG(hw, IXGBE_FCOEPRC); 5418 adapter->stats.fcoeptc.ev_count += 5419 IXGBE_READ_REG(hw, IXGBE_FCOEPTC); 5420 adapter->stats.fcoedwrc.ev_count += 5421 IXGBE_READ_REG(hw, IXGBE_FCOEDWRC); 5422 adapter->stats.fcoedwtc.ev_count += 5423 IXGBE_READ_REG(hw, IXGBE_FCOEDWTC); 5424 } 5425 5426 /* Fill out the OS statistics structure */ 5427 ifp->if_ipackets = adapter->stats.gprc.ev_count; 5428 ifp->if_opackets = adapter->stats.gptc.ev_count; 5429 ifp->if_ibytes = adapter->stats.gorc.ev_count; 5430 ifp->if_obytes = adapter->stats.gotc.ev_count; 5431 ifp->if_imcasts = adapter->stats.mprc.ev_count; 5432 ifp->if_collisions = 0; 5433 5434 /* Rx Errors */ 5435 ifp->if_ierrors = total_missed_rx + adapter->stats.crcerrs.ev_count + 5436 adapter->stats.rlec.ev_count; 5437 } 5438 5439 /** ixgbe_sysctl_tdh_handler - Handler function 5440 * Retrieves the TDH value from the hardware 5441 */ 5442 static int 5443 ixgbe_sysctl_tdh_handler(SYSCTLFN_ARGS) 5444 { 5445 struct sysctlnode node; 5446 uint32_t val; 5447 struct tx_ring *txr; 5448 5449 node = *rnode; 5450 txr = (struct tx_ring *)node.sysctl_data; 5451 if (txr == NULL) 5452 return 0; 5453 val = IXGBE_READ_REG(&txr->adapter->hw, IXGBE_TDH(txr->me)); 5454 node.sysctl_data = &val; 5455 return sysctl_lookup(SYSCTLFN_CALL(&node)); 5456 } 5457 5458 /** ixgbe_sysctl_tdt_handler - Handler function 5459 * Retrieves the TDT value from the hardware 5460 */ 5461 static int 5462 ixgbe_sysctl_tdt_handler(SYSCTLFN_ARGS) 5463 { 5464 struct sysctlnode node; 5465 uint32_t val; 5466 struct tx_ring *txr; 5467 5468 node = *rnode; 5469 txr = (struct tx_ring *)node.sysctl_data; 5470 if (txr == NULL) 5471 return 0; 5472 val = IXGBE_READ_REG(&txr->adapter->hw, IXGBE_TDT(txr->me)); 5473 node.sysctl_data = &val; 5474 return sysctl_lookup(SYSCTLFN_CALL(&node)); 5475 } 5476 5477 /** ixgbe_sysctl_rdh_handler - Handler function 5478 * Retrieves the RDH value from the hardware 5479 */ 5480 static int 5481 ixgbe_sysctl_rdh_handler(SYSCTLFN_ARGS) 5482 { 5483 struct sysctlnode node; 5484 uint32_t val; 5485 struct rx_ring *rxr; 5486 5487 node = *rnode; 5488 rxr = (struct rx_ring *)node.sysctl_data; 5489 if (rxr == NULL) 5490 return 0; 5491 val = IXGBE_READ_REG(&rxr->adapter->hw, IXGBE_RDH(rxr->me)); 5492 node.sysctl_data = &val; 5493 return sysctl_lookup(SYSCTLFN_CALL(&node)); 5494 } 5495 5496 /** ixgbe_sysctl_rdt_handler - Handler function 5497 * Retrieves the RDT value from the hardware 5498 */ 5499 static int 5500 ixgbe_sysctl_rdt_handler(SYSCTLFN_ARGS) 5501 { 5502 struct sysctlnode node; 5503 uint32_t val; 5504 struct rx_ring *rxr; 5505 5506 node = *rnode; 5507 rxr = (struct rx_ring *)node.sysctl_data; 5508 if (rxr == NULL) 5509 return 0; 5510 val = IXGBE_READ_REG(&rxr->adapter->hw, IXGBE_RDT(rxr->me)); 5511 node.sysctl_data = &val; 5512 return sysctl_lookup(SYSCTLFN_CALL(&node)); 5513 } 5514 5515 static int 5516 ixgbe_sysctl_interrupt_rate_handler(SYSCTLFN_ARGS) 5517 { 5518 int error; 5519 struct sysctlnode node; 5520 struct ix_queue *que; 5521 uint32_t reg, usec, rate; 5522 5523 node = *rnode; 5524 que = (struct ix_queue *)node.sysctl_data; 5525 if (que == NULL) 5526 return 0; 5527 reg = IXGBE_READ_REG(&que->adapter->hw, IXGBE_EITR(que->msix)); 5528 usec = ((reg & 0x0FF8) >> 3); 5529 if (usec > 0) 5530 rate = 500000 / usec; 5531 else 5532 rate = 0; 5533 node.sysctl_data = &rate; 5534 error = sysctl_lookup(SYSCTLFN_CALL(&node)); 5535 if (error) 5536 return error; 5537 reg &= ~0xfff; /* default, no limitation */ 5538 ixgbe_max_interrupt_rate = 0; 5539 if (rate > 0 && rate < 500000) { 5540 if (rate < 1000) 5541 rate = 1000; 5542 ixgbe_max_interrupt_rate = rate; 5543 reg |= ((4000000/rate) & 0xff8 ); 5544 } 5545 IXGBE_WRITE_REG(&que->adapter->hw, IXGBE_EITR(que->msix), reg); 5546 return 0; 5547 } 5548 5549 const struct sysctlnode * 5550 ixgbe_sysctl_instance(struct adapter *adapter) 5551 { 5552 const char *dvname; 5553 struct sysctllog **log; 5554 int rc; 5555 const struct sysctlnode *rnode; 5556 5557 log = &adapter->sysctllog; 5558 dvname = device_xname(adapter->dev); 5559 5560 if ((rc = sysctl_createv(log, 0, NULL, &rnode, 5561 0, CTLTYPE_NODE, dvname, 5562 SYSCTL_DESCR("ixgbe information and settings"), 5563 NULL, 0, NULL, 0, CTL_HW, CTL_CREATE, CTL_EOL)) != 0) 5564 goto err; 5565 5566 return rnode; 5567 err: 5568 printf("%s: sysctl_createv failed, rc = %d\n", __func__, rc); 5569 return NULL; 5570 } 5571 5572 /* 5573 * Add sysctl variables, one per statistic, to the system. 5574 */ 5575 static void 5576 ixgbe_add_hw_stats(struct adapter *adapter) 5577 { 5578 device_t dev = adapter->dev; 5579 const struct sysctlnode *rnode, *cnode; 5580 struct sysctllog **log = &adapter->sysctllog; 5581 struct tx_ring *txr = adapter->tx_rings; 5582 struct rx_ring *rxr = adapter->rx_rings; 5583 struct ixgbe_hw *hw = &adapter->hw; 5584 5585 struct ixgbe_hw_stats *stats = &adapter->stats; 5586 5587 /* Driver Statistics */ 5588 #if 0 5589 /* These counters are not updated by the software */ 5590 SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "dropped", 5591 CTLFLAG_RD, &adapter->dropped_pkts, 5592 "Driver dropped packets"); 5593 SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "mbuf_header_failed", 5594 CTLFLAG_RD, &adapter->mbuf_header_failed, 5595 "???"); 5596 SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "mbuf_packet_failed", 5597 CTLFLAG_RD, &adapter->mbuf_packet_failed, 5598 "???"); 5599 SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "no_tx_map_avail", 5600 CTLFLAG_RD, &adapter->no_tx_map_avail, 5601 "???"); 5602 #endif 5603 evcnt_attach_dynamic(&adapter->handleq, EVCNT_TYPE_MISC, 5604 NULL, device_xname(dev), "Handled queue in softint"); 5605 evcnt_attach_dynamic(&adapter->req, EVCNT_TYPE_MISC, 5606 NULL, device_xname(dev), "Requeued in softint"); 5607 evcnt_attach_dynamic(&adapter->morerx, EVCNT_TYPE_MISC, 5608 NULL, device_xname(dev), "Interrupt handler more rx"); 5609 evcnt_attach_dynamic(&adapter->moretx, EVCNT_TYPE_MISC, 5610 NULL, device_xname(dev), "Interrupt handler more tx"); 5611 evcnt_attach_dynamic(&adapter->txloops, EVCNT_TYPE_MISC, 5612 NULL, device_xname(dev), "Interrupt handler tx loops"); 5613 evcnt_attach_dynamic(&adapter->efbig_tx_dma_setup, EVCNT_TYPE_MISC, 5614 NULL, device_xname(dev), "Driver tx dma soft fail EFBIG"); 5615 evcnt_attach_dynamic(&adapter->m_defrag_failed, EVCNT_TYPE_MISC, 5616 NULL, device_xname(dev), "m_defrag() failed"); 5617 evcnt_attach_dynamic(&adapter->efbig2_tx_dma_setup, EVCNT_TYPE_MISC, 5618 NULL, device_xname(dev), "Driver tx dma hard fail EFBIG"); 5619 evcnt_attach_dynamic(&adapter->einval_tx_dma_setup, EVCNT_TYPE_MISC, 5620 NULL, device_xname(dev), "Driver tx dma hard fail EINVAL"); 5621 evcnt_attach_dynamic(&adapter->other_tx_dma_setup, EVCNT_TYPE_MISC, 5622 NULL, device_xname(dev), "Driver tx dma hard fail other"); 5623 evcnt_attach_dynamic(&adapter->eagain_tx_dma_setup, EVCNT_TYPE_MISC, 5624 NULL, device_xname(dev), "Driver tx dma soft fail EAGAIN"); 5625 evcnt_attach_dynamic(&adapter->enomem_tx_dma_setup, EVCNT_TYPE_MISC, 5626 NULL, device_xname(dev), "Driver tx dma soft fail ENOMEM"); 5627 evcnt_attach_dynamic(&adapter->watchdog_events, EVCNT_TYPE_MISC, 5628 NULL, device_xname(dev), "Watchdog timeouts"); 5629 evcnt_attach_dynamic(&adapter->tso_err, EVCNT_TYPE_MISC, 5630 NULL, device_xname(dev), "TSO errors"); 5631 evcnt_attach_dynamic(&adapter->tso_tx, EVCNT_TYPE_MISC, 5632 NULL, device_xname(dev), "TSO"); 5633 evcnt_attach_dynamic(&adapter->link_irq, EVCNT_TYPE_MISC, 5634 NULL, device_xname(dev), "Link MSIX IRQ Handled"); 5635 5636 for (int i = 0; i < adapter->num_queues; i++, rxr++, txr++) { 5637 snprintf(adapter->queues[i].evnamebuf, 5638 sizeof(adapter->queues[i].evnamebuf), "%s queue%d", 5639 device_xname(dev), i); 5640 snprintf(adapter->queues[i].namebuf, 5641 sizeof(adapter->queues[i].namebuf), "queue%d", i); 5642 5643 if ((rnode = ixgbe_sysctl_instance(adapter)) == NULL) { 5644 aprint_error_dev(dev, "could not create sysctl root\n"); 5645 break; 5646 } 5647 5648 if (sysctl_createv(log, 0, &rnode, &rnode, 5649 0, CTLTYPE_NODE, 5650 adapter->queues[i].namebuf, SYSCTL_DESCR("Queue Name"), 5651 NULL, 0, NULL, 0, CTL_CREATE, CTL_EOL) != 0) 5652 break; 5653 5654 if (sysctl_createv(log, 0, &rnode, &cnode, 5655 CTLFLAG_READWRITE, CTLTYPE_INT, 5656 "interrupt_rate", SYSCTL_DESCR("Interrupt Rate"), 5657 ixgbe_sysctl_interrupt_rate_handler, 0, 5658 (void *)&adapter->queues[i], 0, CTL_CREATE, CTL_EOL) != 0) 5659 break; 5660 5661 if (sysctl_createv(log, 0, &rnode, &cnode, 5662 CTLFLAG_READONLY, CTLTYPE_QUAD, 5663 "irqs", SYSCTL_DESCR("irqs on this queue"), 5664 NULL, 0, &(adapter->queues[i].irqs), 5665 0, CTL_CREATE, CTL_EOL) != 0) 5666 break; 5667 5668 if (sysctl_createv(log, 0, &rnode, &cnode, 5669 CTLFLAG_READONLY, CTLTYPE_INT, 5670 "txd_head", SYSCTL_DESCR("Transmit Descriptor Head"), 5671 ixgbe_sysctl_tdh_handler, 0, (void *)txr, 5672 0, CTL_CREATE, CTL_EOL) != 0) 5673 break; 5674 5675 if (sysctl_createv(log, 0, &rnode, &cnode, 5676 CTLFLAG_READONLY, CTLTYPE_INT, 5677 "txd_tail", SYSCTL_DESCR("Transmit Descriptor Tail"), 5678 ixgbe_sysctl_tdt_handler, 0, (void *)txr, 5679 0, CTL_CREATE, CTL_EOL) != 0) 5680 break; 5681 5682 evcnt_attach_dynamic(&txr->no_desc_avail, EVCNT_TYPE_MISC, 5683 NULL, adapter->queues[i].evnamebuf, 5684 "Queue No Descriptor Available"); 5685 evcnt_attach_dynamic(&txr->total_packets, EVCNT_TYPE_MISC, 5686 NULL, adapter->queues[i].evnamebuf, 5687 "Queue Packets Transmitted"); 5688 5689 #ifdef LRO 5690 struct lro_ctrl *lro = &rxr->lro; 5691 #endif /* LRO */ 5692 5693 if (sysctl_createv(log, 0, &rnode, &cnode, 5694 CTLFLAG_READONLY, 5695 CTLTYPE_INT, 5696 "rxd_head", SYSCTL_DESCR("Receive Descriptor Head"), 5697 ixgbe_sysctl_rdh_handler, 0, (void *)rxr, 0, 5698 CTL_CREATE, CTL_EOL) != 0) 5699 break; 5700 5701 if (sysctl_createv(log, 0, &rnode, &cnode, 5702 CTLFLAG_READONLY, 5703 CTLTYPE_INT, 5704 "rxd_tail", SYSCTL_DESCR("Receive Descriptor Tail"), 5705 ixgbe_sysctl_rdt_handler, 0, (void *)rxr, 0, 5706 CTL_CREATE, CTL_EOL) != 0) 5707 break; 5708 5709 if (i < __arraycount(adapter->stats.mpc)) { 5710 evcnt_attach_dynamic(&adapter->stats.mpc[i], 5711 EVCNT_TYPE_MISC, NULL, adapter->queues[i].evnamebuf, 5712 "Missed Packet Count"); 5713 } 5714 if (i < __arraycount(adapter->stats.pxontxc)) { 5715 evcnt_attach_dynamic(&adapter->stats.pxontxc[i], 5716 EVCNT_TYPE_MISC, NULL, adapter->queues[i].evnamebuf, 5717 "pxontxc"); 5718 evcnt_attach_dynamic(&adapter->stats.pxonrxc[i], 5719 EVCNT_TYPE_MISC, NULL, adapter->queues[i].evnamebuf, 5720 "pxonrxc"); 5721 evcnt_attach_dynamic(&adapter->stats.pxofftxc[i], 5722 EVCNT_TYPE_MISC, NULL, adapter->queues[i].evnamebuf, 5723 "pxofftxc"); 5724 evcnt_attach_dynamic(&adapter->stats.pxoffrxc[i], 5725 EVCNT_TYPE_MISC, NULL, adapter->queues[i].evnamebuf, 5726 "pxoffrxc"); 5727 evcnt_attach_dynamic(&adapter->stats.pxon2offc[i], 5728 EVCNT_TYPE_MISC, NULL, adapter->queues[i].evnamebuf, 5729 "pxon2offc"); 5730 } 5731 if (i < __arraycount(adapter->stats.qprc)) { 5732 evcnt_attach_dynamic(&adapter->stats.qprc[i], 5733 EVCNT_TYPE_MISC, NULL, adapter->queues[i].evnamebuf, 5734 "qprc"); 5735 evcnt_attach_dynamic(&adapter->stats.qptc[i], 5736 EVCNT_TYPE_MISC, NULL, adapter->queues[i].evnamebuf, 5737 "qptc"); 5738 evcnt_attach_dynamic(&adapter->stats.qbrc[i], 5739 EVCNT_TYPE_MISC, NULL, adapter->queues[i].evnamebuf, 5740 "qbrc"); 5741 evcnt_attach_dynamic(&adapter->stats.qbtc[i], 5742 EVCNT_TYPE_MISC, NULL, adapter->queues[i].evnamebuf, 5743 "qbtc"); 5744 evcnt_attach_dynamic(&adapter->stats.qprdc[i], 5745 EVCNT_TYPE_MISC, NULL, adapter->queues[i].evnamebuf, 5746 "qprdc"); 5747 } 5748 5749 evcnt_attach_dynamic(&rxr->rx_packets, EVCNT_TYPE_MISC, 5750 NULL, adapter->queues[i].evnamebuf, "Queue Packets Received"); 5751 evcnt_attach_dynamic(&rxr->rx_bytes, EVCNT_TYPE_MISC, 5752 NULL, adapter->queues[i].evnamebuf, "Queue Bytes Received"); 5753 evcnt_attach_dynamic(&rxr->no_jmbuf, EVCNT_TYPE_MISC, 5754 NULL, adapter->queues[i].evnamebuf, "Rx no jumbo mbuf"); 5755 evcnt_attach_dynamic(&rxr->rx_discarded, EVCNT_TYPE_MISC, 5756 NULL, adapter->queues[i].evnamebuf, "Rx discarded"); 5757 evcnt_attach_dynamic(&rxr->rx_split_packets, EVCNT_TYPE_MISC, 5758 NULL, adapter->queues[i].evnamebuf, "Rx split packets"); 5759 evcnt_attach_dynamic(&rxr->rx_irq, EVCNT_TYPE_MISC, 5760 NULL, adapter->queues[i].evnamebuf, "Rx interrupts"); 5761 #ifdef LRO 5762 SYSCTL_ADD_INT(ctx, queue_list, OID_AUTO, "lro_queued", 5763 CTLFLAG_RD, &lro->lro_queued, 0, 5764 "LRO Queued"); 5765 SYSCTL_ADD_INT(ctx, queue_list, OID_AUTO, "lro_flushed", 5766 CTLFLAG_RD, &lro->lro_flushed, 0, 5767 "LRO Flushed"); 5768 #endif /* LRO */ 5769 } 5770 5771 /* MAC stats get the own sub node */ 5772 5773 5774 snprintf(stats->namebuf, 5775 sizeof(stats->namebuf), "%s MAC Statistics", device_xname(dev)); 5776 5777 evcnt_attach_dynamic(&stats->ipcs, EVCNT_TYPE_MISC, NULL, 5778 stats->namebuf, "rx csum offload - IP"); 5779 evcnt_attach_dynamic(&stats->l4cs, EVCNT_TYPE_MISC, NULL, 5780 stats->namebuf, "rx csum offload - L4"); 5781 evcnt_attach_dynamic(&stats->ipcs_bad, EVCNT_TYPE_MISC, NULL, 5782 stats->namebuf, "rx csum offload - IP bad"); 5783 evcnt_attach_dynamic(&stats->l4cs_bad, EVCNT_TYPE_MISC, NULL, 5784 stats->namebuf, "rx csum offload - L4 bad"); 5785 evcnt_attach_dynamic(&stats->intzero, EVCNT_TYPE_MISC, NULL, 5786 stats->namebuf, "Interrupt conditions zero"); 5787 evcnt_attach_dynamic(&stats->legint, EVCNT_TYPE_MISC, NULL, 5788 stats->namebuf, "Legacy interrupts"); 5789 evcnt_attach_dynamic(&stats->crcerrs, EVCNT_TYPE_MISC, NULL, 5790 stats->namebuf, "CRC Errors"); 5791 evcnt_attach_dynamic(&stats->illerrc, EVCNT_TYPE_MISC, NULL, 5792 stats->namebuf, "Illegal Byte Errors"); 5793 evcnt_attach_dynamic(&stats->errbc, EVCNT_TYPE_MISC, NULL, 5794 stats->namebuf, "Byte Errors"); 5795 evcnt_attach_dynamic(&stats->mspdc, EVCNT_TYPE_MISC, NULL, 5796 stats->namebuf, "MAC Short Packets Discarded"); 5797 evcnt_attach_dynamic(&stats->mlfc, EVCNT_TYPE_MISC, NULL, 5798 stats->namebuf, "MAC Local Faults"); 5799 evcnt_attach_dynamic(&stats->mrfc, EVCNT_TYPE_MISC, NULL, 5800 stats->namebuf, "MAC Remote Faults"); 5801 evcnt_attach_dynamic(&stats->rlec, EVCNT_TYPE_MISC, NULL, 5802 stats->namebuf, "Receive Length Errors"); 5803 evcnt_attach_dynamic(&stats->lxontxc, EVCNT_TYPE_MISC, NULL, 5804 stats->namebuf, "Link XON Transmitted"); 5805 evcnt_attach_dynamic(&stats->lxonrxc, EVCNT_TYPE_MISC, NULL, 5806 stats->namebuf, "Link XON Received"); 5807 evcnt_attach_dynamic(&stats->lxofftxc, EVCNT_TYPE_MISC, NULL, 5808 stats->namebuf, "Link XOFF Transmitted"); 5809 evcnt_attach_dynamic(&stats->lxoffrxc, EVCNT_TYPE_MISC, NULL, 5810 stats->namebuf, "Link XOFF Received"); 5811 5812 /* Packet Reception Stats */ 5813 evcnt_attach_dynamic(&stats->tor, EVCNT_TYPE_MISC, NULL, 5814 stats->namebuf, "Total Octets Received"); 5815 evcnt_attach_dynamic(&stats->gorc, EVCNT_TYPE_MISC, NULL, 5816 stats->namebuf, "Good Octets Received"); 5817 evcnt_attach_dynamic(&stats->tpr, EVCNT_TYPE_MISC, NULL, 5818 stats->namebuf, "Total Packets Received"); 5819 evcnt_attach_dynamic(&stats->gprc, EVCNT_TYPE_MISC, NULL, 5820 stats->namebuf, "Good Packets Received"); 5821 evcnt_attach_dynamic(&stats->mprc, EVCNT_TYPE_MISC, NULL, 5822 stats->namebuf, "Multicast Packets Received"); 5823 evcnt_attach_dynamic(&stats->bprc, EVCNT_TYPE_MISC, NULL, 5824 stats->namebuf, "Broadcast Packets Received"); 5825 evcnt_attach_dynamic(&stats->prc64, EVCNT_TYPE_MISC, NULL, 5826 stats->namebuf, "64 byte frames received "); 5827 evcnt_attach_dynamic(&stats->prc127, EVCNT_TYPE_MISC, NULL, 5828 stats->namebuf, "65-127 byte frames received"); 5829 evcnt_attach_dynamic(&stats->prc255, EVCNT_TYPE_MISC, NULL, 5830 stats->namebuf, "128-255 byte frames received"); 5831 evcnt_attach_dynamic(&stats->prc511, EVCNT_TYPE_MISC, NULL, 5832 stats->namebuf, "256-511 byte frames received"); 5833 evcnt_attach_dynamic(&stats->prc1023, EVCNT_TYPE_MISC, NULL, 5834 stats->namebuf, "512-1023 byte frames received"); 5835 evcnt_attach_dynamic(&stats->prc1522, EVCNT_TYPE_MISC, NULL, 5836 stats->namebuf, "1023-1522 byte frames received"); 5837 evcnt_attach_dynamic(&stats->ruc, EVCNT_TYPE_MISC, NULL, 5838 stats->namebuf, "Receive Undersized"); 5839 evcnt_attach_dynamic(&stats->rfc, EVCNT_TYPE_MISC, NULL, 5840 stats->namebuf, "Fragmented Packets Received "); 5841 evcnt_attach_dynamic(&stats->roc, EVCNT_TYPE_MISC, NULL, 5842 stats->namebuf, "Oversized Packets Received"); 5843 evcnt_attach_dynamic(&stats->rjc, EVCNT_TYPE_MISC, NULL, 5844 stats->namebuf, "Received Jabber"); 5845 evcnt_attach_dynamic(&stats->mngprc, EVCNT_TYPE_MISC, NULL, 5846 stats->namebuf, "Management Packets Received"); 5847 evcnt_attach_dynamic(&stats->xec, EVCNT_TYPE_MISC, NULL, 5848 stats->namebuf, "Checksum Errors"); 5849 5850 /* Packet Transmission Stats */ 5851 evcnt_attach_dynamic(&stats->gotc, EVCNT_TYPE_MISC, NULL, 5852 stats->namebuf, "Good Octets Transmitted"); 5853 evcnt_attach_dynamic(&stats->tpt, EVCNT_TYPE_MISC, NULL, 5854 stats->namebuf, "Total Packets Transmitted"); 5855 evcnt_attach_dynamic(&stats->gptc, EVCNT_TYPE_MISC, NULL, 5856 stats->namebuf, "Good Packets Transmitted"); 5857 evcnt_attach_dynamic(&stats->bptc, EVCNT_TYPE_MISC, NULL, 5858 stats->namebuf, "Broadcast Packets Transmitted"); 5859 evcnt_attach_dynamic(&stats->mptc, EVCNT_TYPE_MISC, NULL, 5860 stats->namebuf, "Multicast Packets Transmitted"); 5861 evcnt_attach_dynamic(&stats->mngptc, EVCNT_TYPE_MISC, NULL, 5862 stats->namebuf, "Management Packets Transmitted"); 5863 evcnt_attach_dynamic(&stats->ptc64, EVCNT_TYPE_MISC, NULL, 5864 stats->namebuf, "64 byte frames transmitted "); 5865 evcnt_attach_dynamic(&stats->ptc127, EVCNT_TYPE_MISC, NULL, 5866 stats->namebuf, "65-127 byte frames transmitted"); 5867 evcnt_attach_dynamic(&stats->ptc255, EVCNT_TYPE_MISC, NULL, 5868 stats->namebuf, "128-255 byte frames transmitted"); 5869 evcnt_attach_dynamic(&stats->ptc511, EVCNT_TYPE_MISC, NULL, 5870 stats->namebuf, "256-511 byte frames transmitted"); 5871 evcnt_attach_dynamic(&stats->ptc1023, EVCNT_TYPE_MISC, NULL, 5872 stats->namebuf, "512-1023 byte frames transmitted"); 5873 evcnt_attach_dynamic(&stats->ptc1522, EVCNT_TYPE_MISC, NULL, 5874 stats->namebuf, "1024-1522 byte frames transmitted"); 5875 5876 /* FC Stats */ 5877 evcnt_attach_dynamic(&stats->fccrc, EVCNT_TYPE_MISC, NULL, 5878 stats->namebuf, "FC CRC Errors"); 5879 evcnt_attach_dynamic(&stats->fclast, EVCNT_TYPE_MISC, NULL, 5880 stats->namebuf, "FC Last Error"); 5881 if (hw->mac.type != ixgbe_mac_82598EB) { 5882 evcnt_attach_dynamic(&stats->fcoerpdc, EVCNT_TYPE_MISC, NULL, 5883 stats->namebuf, "FCoE Packets Dropped"); 5884 evcnt_attach_dynamic(&stats->fcoeprc, EVCNT_TYPE_MISC, NULL, 5885 stats->namebuf, "FCoE Packets Received"); 5886 evcnt_attach_dynamic(&stats->fcoeptc, EVCNT_TYPE_MISC, NULL, 5887 stats->namebuf, "FCoE Packets Transmitted"); 5888 evcnt_attach_dynamic(&stats->fcoedwrc, EVCNT_TYPE_MISC, NULL, 5889 stats->namebuf, "FCoE DWords Received"); 5890 evcnt_attach_dynamic(&stats->fcoedwtc, EVCNT_TYPE_MISC, NULL, 5891 stats->namebuf, "FCoE DWords Transmitted"); 5892 } 5893 } 5894 5895 /* 5896 ** Set flow control using sysctl: 5897 ** Flow control values: 5898 ** 0 - off 5899 ** 1 - rx pause 5900 ** 2 - tx pause 5901 ** 3 - full 5902 */ 5903 static int 5904 ixgbe_set_flowcntl(SYSCTLFN_ARGS) 5905 { 5906 struct sysctlnode node; 5907 int error; 5908 int last = ixgbe_flow_control; 5909 struct adapter *adapter; 5910 5911 node = *rnode; 5912 adapter = (struct adapter *)node.sysctl_data; 5913 node.sysctl_data = &ixgbe_flow_control; 5914 error = sysctl_lookup(SYSCTLFN_CALL(&node)); 5915 if (error != 0 || newp == NULL) 5916 return error; 5917 5918 /* Don't bother if it's not changed */ 5919 if (ixgbe_flow_control == last) 5920 return (0); 5921 5922 switch (ixgbe_flow_control) { 5923 case ixgbe_fc_rx_pause: 5924 case ixgbe_fc_tx_pause: 5925 case ixgbe_fc_full: 5926 adapter->hw.fc.requested_mode = ixgbe_flow_control; 5927 break; 5928 case ixgbe_fc_none: 5929 default: 5930 adapter->hw.fc.requested_mode = ixgbe_fc_none; 5931 } 5932 5933 ixgbe_fc_enable(&adapter->hw, 0); 5934 return 0; 5935 } 5936 5937 static void 5938 ixgbe_add_rx_process_limit(struct adapter *adapter, const char *name, 5939 const char *description, int *limit, int value) 5940 { 5941 const struct sysctlnode *rnode, *cnode; 5942 struct sysctllog **log = &adapter->sysctllog; 5943 5944 *limit = value; 5945 5946 if ((rnode = ixgbe_sysctl_instance(adapter)) == NULL) 5947 aprint_error_dev(adapter->dev, 5948 "could not create sysctl root\n"); 5949 else if (sysctl_createv(log, 0, &rnode, &cnode, 5950 CTLFLAG_READWRITE, 5951 CTLTYPE_INT, 5952 name, SYSCTL_DESCR(description), 5953 NULL, 0, limit, 0, 5954 CTL_CREATE, CTL_EOL) != 0) { 5955 aprint_error_dev(adapter->dev, "%s: could not create sysctl", 5956 __func__); 5957 } 5958 } 5959 5960 /* 5961 ** Control link advertise speed: 5962 ** 0 - normal 5963 ** 1 - advertise only 1G 5964 */ 5965 static int 5966 ixgbe_set_advertise(SYSCTLFN_ARGS) 5967 { 5968 struct sysctlnode node; 5969 int t, error = 0; 5970 struct adapter *adapter; 5971 struct ixgbe_hw *hw; 5972 ixgbe_link_speed speed, last; 5973 5974 node = *rnode; 5975 adapter = (struct adapter *)node.sysctl_data; 5976 t = adapter->advertise; 5977 node.sysctl_data = &t; 5978 error = sysctl_lookup(SYSCTLFN_CALL(&node)); 5979 if (error != 0 || newp == NULL) 5980 return error; 5981 5982 if (t == -1) 5983 return 0; 5984 5985 adapter->advertise = t; 5986 5987 hw = &adapter->hw; 5988 last = hw->phy.autoneg_advertised; 5989 5990 if (!((hw->phy.media_type == ixgbe_media_type_copper) || 5991 (hw->phy.multispeed_fiber))) 5992 return 0; 5993 5994 if (adapter->advertise == 1) 5995 speed = IXGBE_LINK_SPEED_1GB_FULL; 5996 else 5997 speed = IXGBE_LINK_SPEED_1GB_FULL | 5998 IXGBE_LINK_SPEED_10GB_FULL; 5999 6000 if (speed == last) /* no change */ 6001 return 0; 6002 6003 hw->mac.autotry_restart = TRUE; 6004 hw->mac.ops.setup_link(hw, speed, TRUE, TRUE); 6005 6006 return 0; 6007 } 6008
Indexes created Sun Sep 28 07:09:56 GMT 2025