ixgbe.c revision 1.11
1 /****************************************************************************** 2 3 Copyright (c) 2001-2011, Intel Corporation 4 All rights reserved. 5 6 Redistribution and use in source and binary forms, with or without 7 modification, are permitted provided that the following conditions are met: 8 9 1. Redistributions of source code must retain the above copyright notice, 10 this list of conditions and the following disclaimer. 11 12 2. Redistributions in binary form must reproduce the above copyright 13 notice, this list of conditions and the following disclaimer in the 14 documentation and/or other materials provided with the distribution. 15 16 3. Neither the name of the Intel Corporation nor the names of its 17 contributors may be used to endorse or promote products derived from 18 this software without specific prior written permission. 19 20 THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 21 AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 22 IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 23 ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE 24 LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 25 CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 26 SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 27 INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 28 CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 29 ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 30 POSSIBILITY OF SUCH DAMAGE. 31 32 ******************************************************************************/ 33 /* 34 * Copyright (c) 2011 The NetBSD Foundation, Inc. 35 * All rights reserved. 36 * 37 * This code is derived from software contributed to The NetBSD Foundation 38 * by Coyote Point Systems, Inc. 39 * 40 * Redistribution and use in source and binary forms, with or without 41 * modification, are permitted provided that the following conditions 42 * are met: 43 * 1. Redistributions of source code must retain the above copyright 44 * notice, this list of conditions and the following disclaimer. 45 * 2. Redistributions in binary form must reproduce the above copyright 46 * notice, this list of conditions and the following disclaimer in the 47 * documentation and/or other materials provided with the distribution. 48 * 49 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS 50 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED 51 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 52 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS 53 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 54 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 55 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 56 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 57 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 58 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 59 * POSSIBILITY OF SUCH DAMAGE. 60 */ 61 /*$FreeBSD: src/sys/dev/ixgbe/ixgbe.c,v 1.51 2011/04/25 23:34:21 jfv Exp $*/ 62 /*$NetBSD: ixgbe.c,v 1.11 2014/04/11 06:50:26 joerg Exp $*/ 63 64 #include "opt_inet.h" 65 66 #include "ixgbe.h" 67 68 /********************************************************************* 69 * Set this to one to display debug statistics 70 *********************************************************************/ 71 int ixgbe_display_debug_stats = 0; 72 73 /********************************************************************* 74 * Driver version 75 *********************************************************************/ 76 char ixgbe_driver_version[] = "2.3.10"; 77 78 /********************************************************************* 79 * PCI Device ID Table 80 * 81 * Used by probe to select devices to load on 82 * Last field stores an index into ixgbe_strings 83 * Last entry must be all 0s 84 * 85 * { Vendor ID, Device ID, SubVendor ID, SubDevice ID, String Index } 86 *********************************************************************/ 87 88 static ixgbe_vendor_info_t ixgbe_vendor_info_array[] = 89 { 90 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598AF_DUAL_PORT, 0, 0, 0}, 91 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598AF_SINGLE_PORT, 0, 0, 0}, 92 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598EB_CX4, 0, 0, 0}, 93 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598AT, 0, 0, 0}, 94 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598AT2, 0, 0, 0}, 95 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598, 0, 0, 0}, 96 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598_DA_DUAL_PORT, 0, 0, 0}, 97 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598_CX4_DUAL_PORT, 0, 0, 0}, 98 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598EB_XF_LR, 0, 0, 0}, 99 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598_SR_DUAL_PORT_EM, 0, 0, 0}, 100 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598EB_SFP_LOM, 0, 0, 0}, 101 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_KX4, 0, 0, 0}, 102 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_KX4_MEZZ, 0, 0, 0}, 103 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_SFP, 0, 0, 0}, 104 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_XAUI_LOM, 0, 0, 0}, 105 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_CX4, 0, 0, 0}, 106 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_T3_LOM, 0, 0, 0}, 107 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_COMBO_BACKPLANE, 0, 0, 0}, 108 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_BACKPLANE_FCOE, 0, 0, 0}, 109 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_SFP_FCOE, 0, 0, 0}, 110 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_SFP_DELL, 0, 0, 0}, 111 /* required last entry */ 112 {0, 0, 0, 0, 0} 113 }; 114 115 /********************************************************************* 116 * Table of branding strings 117 *********************************************************************/ 118 119 static const char *ixgbe_strings[] = { 120 "Intel(R) PRO/10GbE PCI-Express Network Driver" 121 }; 122 123 /********************************************************************* 124 * Function prototypes 125 *********************************************************************/ 126 static int ixgbe_probe(device_t, cfdata_t, void *); 127 static void ixgbe_attach(device_t, device_t, void *); 128 static int ixgbe_detach(device_t, int); 129 #if 0 130 static int ixgbe_shutdown(device_t); 131 #endif 132 static void ixgbe_start(struct ifnet *); 133 static void ixgbe_start_locked(struct tx_ring *, struct ifnet *); 134 #if __FreeBSD_version >= 800000 135 static int ixgbe_mq_start(struct ifnet *, struct mbuf *); 136 static int ixgbe_mq_start_locked(struct ifnet *, 137 struct tx_ring *, struct mbuf *); 138 static void ixgbe_qflush(struct ifnet *); 139 #endif 140 static int ixgbe_ioctl(struct ifnet *, u_long, void *); 141 static void ixgbe_ifstop(struct ifnet *, int); 142 static int ixgbe_init(struct ifnet *); 143 static void ixgbe_init_locked(struct adapter *); 144 static void ixgbe_stop(void *); 145 static void ixgbe_media_status(struct ifnet *, struct ifmediareq *); 146 static int ixgbe_media_change(struct ifnet *); 147 static void ixgbe_identify_hardware(struct adapter *); 148 static int ixgbe_allocate_pci_resources(struct adapter *, 149 const struct pci_attach_args *); 150 static int ixgbe_allocate_msix(struct adapter *, 151 const struct pci_attach_args *); 152 static int ixgbe_allocate_legacy(struct adapter *, 153 const struct pci_attach_args *); 154 static int ixgbe_allocate_queues(struct adapter *); 155 static int ixgbe_setup_msix(struct adapter *); 156 static void ixgbe_free_pci_resources(struct adapter *); 157 static void ixgbe_local_timer(void *); 158 static int ixgbe_setup_interface(device_t, struct adapter *); 159 static void ixgbe_config_link(struct adapter *); 160 161 static int ixgbe_allocate_transmit_buffers(struct tx_ring *); 162 static int ixgbe_setup_transmit_structures(struct adapter *); 163 static void ixgbe_setup_transmit_ring(struct tx_ring *); 164 static void ixgbe_initialize_transmit_units(struct adapter *); 165 static void ixgbe_free_transmit_structures(struct adapter *); 166 static void ixgbe_free_transmit_buffers(struct tx_ring *); 167 168 static int ixgbe_allocate_receive_buffers(struct rx_ring *); 169 static int ixgbe_setup_receive_structures(struct adapter *); 170 static int ixgbe_setup_receive_ring(struct rx_ring *); 171 static void ixgbe_initialize_receive_units(struct adapter *); 172 static void ixgbe_free_receive_structures(struct adapter *); 173 static void ixgbe_free_receive_buffers(struct rx_ring *); 174 static void ixgbe_setup_hw_rsc(struct rx_ring *); 175 176 static void ixgbe_enable_intr(struct adapter *); 177 static void ixgbe_disable_intr(struct adapter *); 178 static void ixgbe_update_stats_counters(struct adapter *); 179 static bool ixgbe_txeof(struct tx_ring *); 180 static bool ixgbe_rxeof(struct ix_queue *, int); 181 static void ixgbe_rx_checksum(u32, struct mbuf *, u32, 182 struct ixgbe_hw_stats *); 183 static void ixgbe_set_promisc(struct adapter *); 184 static void ixgbe_set_multi(struct adapter *); 185 static void ixgbe_update_link_status(struct adapter *); 186 static void ixgbe_refresh_mbufs(struct rx_ring *, int); 187 static int ixgbe_xmit(struct tx_ring *, struct mbuf *); 188 static int ixgbe_set_flowcntl(SYSCTLFN_PROTO); 189 static int ixgbe_set_advertise(SYSCTLFN_PROTO); 190 static int ixgbe_dma_malloc(struct adapter *, bus_size_t, 191 struct ixgbe_dma_alloc *, int); 192 static void ixgbe_dma_free(struct adapter *, struct ixgbe_dma_alloc *); 193 static void ixgbe_add_rx_process_limit(struct adapter *, const char *, 194 const char *, int *, int); 195 static u32 ixgbe_tx_ctx_setup(struct tx_ring *, struct mbuf *); 196 static bool ixgbe_tso_setup(struct tx_ring *, struct mbuf *, u32 *); 197 static void ixgbe_set_ivar(struct adapter *, u8, u8, s8); 198 static void ixgbe_configure_ivars(struct adapter *); 199 static u8 * ixgbe_mc_array_itr(struct ixgbe_hw *, u8 **, u32 *); 200 201 static void ixgbe_setup_vlan_hw_support(struct adapter *); 202 #if 0 203 static void ixgbe_register_vlan(void *, struct ifnet *, u16); 204 static void ixgbe_unregister_vlan(void *, struct ifnet *, u16); 205 #endif 206 207 static void ixgbe_add_hw_stats(struct adapter *adapter); 208 209 static __inline void ixgbe_rx_discard(struct rx_ring *, int); 210 static __inline void ixgbe_rx_input(struct rx_ring *, struct ifnet *, 211 struct mbuf *, u32); 212 213 /* Support for pluggable optic modules */ 214 static bool ixgbe_sfp_probe(struct adapter *); 215 static void ixgbe_setup_optics(struct adapter *); 216 217 /* Legacy (single vector interrupt handler */ 218 static int ixgbe_legacy_irq(void *); 219 220 #if defined(NETBSD_MSI_OR_MSIX) 221 /* The MSI/X Interrupt handlers */ 222 static void ixgbe_msix_que(void *); 223 static void ixgbe_msix_link(void *); 224 #endif 225 226 /* Software interrupts for deferred work */ 227 static void ixgbe_handle_que(void *); 228 static void ixgbe_handle_link(void *); 229 static void ixgbe_handle_msf(void *); 230 static void ixgbe_handle_mod(void *); 231 232 const struct sysctlnode *ixgbe_sysctl_instance(struct adapter *); 233 static ixgbe_vendor_info_t *ixgbe_lookup(const struct pci_attach_args *); 234 235 #ifdef IXGBE_FDIR 236 static void ixgbe_atr(struct tx_ring *, struct mbuf *); 237 static void ixgbe_reinit_fdir(void *, int); 238 #endif 239 240 /********************************************************************* 241 * FreeBSD Device Interface Entry Points 242 *********************************************************************/ 243 244 CFATTACH_DECL3_NEW(ixg, sizeof(struct adapter), 245 ixgbe_probe, ixgbe_attach, ixgbe_detach, NULL, NULL, NULL, 246 DVF_DETACH_SHUTDOWN); 247 248 #if 0 249 devclass_t ixgbe_devclass; 250 DRIVER_MODULE(ixgbe, pci, ixgbe_driver, ixgbe_devclass, 0, 0); 251 252 MODULE_DEPEND(ixgbe, pci, 1, 1, 1); 253 MODULE_DEPEND(ixgbe, ether, 1, 1, 1); 254 #endif 255 256 /* 257 ** TUNEABLE PARAMETERS: 258 */ 259 260 /* 261 ** AIM: Adaptive Interrupt Moderation 262 ** which means that the interrupt rate 263 ** is varied over time based on the 264 ** traffic for that interrupt vector 265 */ 266 static int ixgbe_enable_aim = TRUE; 267 #define TUNABLE_INT(__x, __y) 268 TUNABLE_INT("hw.ixgbe.enable_aim", &ixgbe_enable_aim); 269 270 static int ixgbe_max_interrupt_rate = (8000000 / IXGBE_LOW_LATENCY); 271 TUNABLE_INT("hw.ixgbe.max_interrupt_rate", &ixgbe_max_interrupt_rate); 272 273 /* How many packets rxeof tries to clean at a time */ 274 static int ixgbe_rx_process_limit = 256; 275 TUNABLE_INT("hw.ixgbe.rx_process_limit", &ixgbe_rx_process_limit); 276 277 /* Flow control setting, default to full */ 278 static int ixgbe_flow_control = ixgbe_fc_full; 279 TUNABLE_INT("hw.ixgbe.flow_control", &ixgbe_flow_control); 280 281 /* 282 ** Smart speed setting, default to on 283 ** this only works as a compile option 284 ** right now as its during attach, set 285 ** this to 'ixgbe_smart_speed_off' to 286 ** disable. 287 */ 288 static int ixgbe_smart_speed = ixgbe_smart_speed_on; 289 290 /* 291 * MSIX should be the default for best performance, 292 * but this allows it to be forced off for testing. 293 */ 294 static int ixgbe_enable_msix = 1; 295 TUNABLE_INT("hw.ixgbe.enable_msix", &ixgbe_enable_msix); 296 297 /* 298 * Header split: this causes the hardware to DMA 299 * the header into a separate mbuf from the payload, 300 * it can be a performance win in some workloads, but 301 * in others it actually hurts, its off by default. 302 */ 303 static bool ixgbe_header_split = FALSE; 304 TUNABLE_INT("hw.ixgbe.hdr_split", &ixgbe_header_split); 305 306 #if defined(NETBSD_MSI_OR_MSIX) 307 /* 308 * Number of Queues, can be set to 0, 309 * it then autoconfigures based on the 310 * number of cpus with a max of 8. This 311 * can be overriden manually here. 312 */ 313 static int ixgbe_num_queues = 0; 314 TUNABLE_INT("hw.ixgbe.num_queues", &ixgbe_num_queues); 315 #endif 316 317 /* 318 ** Number of TX descriptors per ring, 319 ** setting higher than RX as this seems 320 ** the better performing choice. 321 */ 322 static int ixgbe_txd = PERFORM_TXD; 323 TUNABLE_INT("hw.ixgbe.txd", &ixgbe_txd); 324 325 /* Number of RX descriptors per ring */ 326 static int ixgbe_rxd = PERFORM_RXD; 327 TUNABLE_INT("hw.ixgbe.rxd", &ixgbe_rxd); 328 329 /* Keep running tab on them for sanity check */ 330 static int ixgbe_total_ports; 331 332 #ifdef IXGBE_FDIR 333 /* 334 ** For Flow Director: this is the 335 ** number of TX packets we sample 336 ** for the filter pool, this means 337 ** every 20th packet will be probed. 338 ** 339 ** This feature can be disabled by 340 ** setting this to 0. 341 */ 342 static int atr_sample_rate = 20; 343 /* 344 ** Flow Director actually 'steals' 345 ** part of the packet buffer as its 346 ** filter pool, this variable controls 347 ** how much it uses: 348 ** 0 = 64K, 1 = 128K, 2 = 256K 349 */ 350 static int fdir_pballoc = 1; 351 #endif 352 353 /********************************************************************* 354 * Device identification routine 355 * 356 * ixgbe_probe determines if the driver should be loaded on 357 * adapter based on PCI vendor/device id of the adapter. 358 * 359 * return 1 on success, 0 on failure 360 *********************************************************************/ 361 362 static int 363 ixgbe_probe(device_t dev, cfdata_t cf, void *aux) 364 { 365 const struct pci_attach_args *pa = aux; 366 367 return (ixgbe_lookup(pa) != NULL) ? 1 : 0; 368 } 369 370 static ixgbe_vendor_info_t * 371 ixgbe_lookup(const struct pci_attach_args *pa) 372 { 373 pcireg_t subid; 374 ixgbe_vendor_info_t *ent; 375 376 INIT_DEBUGOUT("ixgbe_probe: begin"); 377 378 if (PCI_VENDOR(pa->pa_id) != IXGBE_INTEL_VENDOR_ID) 379 return NULL; 380 381 subid = pci_conf_read(pa->pa_pc, pa->pa_tag, PCI_SUBSYS_ID_REG); 382 383 for (ent = ixgbe_vendor_info_array; ent->vendor_id != 0; ent++) { 384 if (PCI_VENDOR(pa->pa_id) == ent->vendor_id && 385 PCI_PRODUCT(pa->pa_id) == ent->device_id && 386 387 (PCI_SUBSYS_VENDOR(subid) == ent->subvendor_id || 388 ent->subvendor_id == 0) && 389 390 (PCI_SUBSYS_ID(subid) == ent->subdevice_id || 391 ent->subdevice_id == 0)) { 392 ++ixgbe_total_ports; 393 return ent; 394 } 395 } 396 return NULL; 397 } 398 399 400 static void 401 ixgbe_sysctl_attach(struct adapter *adapter) 402 { 403 struct sysctllog **log; 404 const struct sysctlnode *rnode, *cnode; 405 device_t dev; 406 407 dev = adapter->dev; 408 log = &adapter->sysctllog; 409 410 if ((rnode = ixgbe_sysctl_instance(adapter)) == NULL) { 411 aprint_error_dev(dev, "could not create sysctl root\n"); 412 return; 413 } 414 415 if (sysctl_createv(log, 0, &rnode, &cnode, 416 CTLFLAG_READONLY, CTLTYPE_INT, 417 "num_rx_desc", SYSCTL_DESCR("Number of rx descriptors"), 418 NULL, 0, &adapter->num_rx_desc, 0, CTL_CREATE, CTL_EOL) != 0) 419 aprint_error_dev(dev, "could not create sysctl\n"); 420 421 if (sysctl_createv(log, 0, &rnode, &cnode, 422 CTLFLAG_READONLY, CTLTYPE_INT, 423 "num_queues", SYSCTL_DESCR("Number of queues"), 424 NULL, 0, &adapter->num_queues, 0, CTL_CREATE, CTL_EOL) != 0) 425 aprint_error_dev(dev, "could not create sysctl\n"); 426 427 if (sysctl_createv(log, 0, &rnode, &cnode, 428 CTLFLAG_READWRITE, CTLTYPE_INT, 429 "flow_control", SYSCTL_DESCR("Flow Control"), 430 ixgbe_set_flowcntl, 0, (void *)adapter, 0, CTL_CREATE, CTL_EOL) != 0) 431 aprint_error_dev(dev, "could not create sysctl\n"); 432 433 if (sysctl_createv(log, 0, &rnode, &cnode, 434 CTLFLAG_READWRITE, CTLTYPE_INT, 435 "advertise_gig", SYSCTL_DESCR("1G Link"), 436 ixgbe_set_advertise, 0, (void *)adapter, 0, CTL_CREATE, CTL_EOL) != 0) 437 aprint_error_dev(dev, "could not create sysctl\n"); 438 439 /* XXX This is an *instance* sysctl controlling a *global* variable. 440 * XXX It's that way in the FreeBSD driver that this derives from. 441 */ 442 if (sysctl_createv(log, 0, &rnode, &cnode, 443 CTLFLAG_READWRITE, CTLTYPE_INT, 444 "enable_aim", SYSCTL_DESCR("Interrupt Moderation"), 445 NULL, 0, &ixgbe_enable_aim, 0, CTL_CREATE, CTL_EOL) != 0) 446 aprint_error_dev(dev, "could not create sysctl\n"); 447 } 448 449 /********************************************************************* 450 * Device initialization routine 451 * 452 * The attach entry point is called when the driver is being loaded. 453 * This routine identifies the type of hardware, allocates all resources 454 * and initializes the hardware. 455 * 456 * return 0 on success, positive on failure 457 *********************************************************************/ 458 459 static void 460 ixgbe_attach(device_t parent, device_t dev, void *aux) 461 { 462 struct adapter *adapter; 463 struct ixgbe_hw *hw; 464 int error = 0; 465 u16 csum; 466 u32 ctrl_ext; 467 ixgbe_vendor_info_t *ent; 468 const struct pci_attach_args *pa = aux; 469 470 INIT_DEBUGOUT("ixgbe_attach: begin"); 471 472 /* Allocate, clear, and link in our adapter structure */ 473 adapter = device_private(dev); 474 adapter->dev = adapter->osdep.dev = dev; 475 hw = &adapter->hw; 476 adapter->osdep.pc = pa->pa_pc; 477 adapter->osdep.tag = pa->pa_tag; 478 adapter->osdep.dmat = pa->pa_dmat; 479 480 ent = ixgbe_lookup(pa); 481 482 KASSERT(ent != NULL); 483 484 aprint_normal(": %s, Version - %s\n", 485 ixgbe_strings[ent->index], ixgbe_driver_version); 486 487 /* Core Lock Init*/ 488 IXGBE_CORE_LOCK_INIT(adapter, device_xname(dev)); 489 490 /* SYSCTL APIs */ 491 492 ixgbe_sysctl_attach(adapter); 493 494 /* Set up the timer callout */ 495 callout_init(&adapter->timer, 0); 496 497 /* Determine hardware revision */ 498 ixgbe_identify_hardware(adapter); 499 500 /* Do base PCI setup - map BAR0 */ 501 if (ixgbe_allocate_pci_resources(adapter, pa)) { 502 aprint_error_dev(dev, "Allocation of PCI resources failed\n"); 503 error = ENXIO; 504 goto err_out; 505 } 506 507 /* Do descriptor calc and sanity checks */ 508 if (((ixgbe_txd * sizeof(union ixgbe_adv_tx_desc)) % DBA_ALIGN) != 0 || 509 ixgbe_txd < MIN_TXD || ixgbe_txd > MAX_TXD) { 510 aprint_error_dev(dev, "TXD config issue, using default!\n"); 511 adapter->num_tx_desc = DEFAULT_TXD; 512 } else 513 adapter->num_tx_desc = ixgbe_txd; 514 515 /* 516 ** With many RX rings it is easy to exceed the 517 ** system mbuf allocation. Tuning nmbclusters 518 ** can alleviate this. 519 */ 520 if (nmbclusters > 0 ) { 521 int s; 522 s = (ixgbe_rxd * adapter->num_queues) * ixgbe_total_ports; 523 if (s > nmbclusters) { 524 aprint_error_dev(dev, "RX Descriptors exceed " 525 "system mbuf max, using default instead!\n"); 526 ixgbe_rxd = DEFAULT_RXD; 527 } 528 } 529 530 if (((ixgbe_rxd * sizeof(union ixgbe_adv_rx_desc)) % DBA_ALIGN) != 0 || 531 ixgbe_rxd < MIN_TXD || ixgbe_rxd > MAX_TXD) { 532 aprint_error_dev(dev, "RXD config issue, using default!\n"); 533 adapter->num_rx_desc = DEFAULT_RXD; 534 } else 535 adapter->num_rx_desc = ixgbe_rxd; 536 537 /* Allocate our TX/RX Queues */ 538 if (ixgbe_allocate_queues(adapter)) { 539 error = ENOMEM; 540 goto err_out; 541 } 542 543 /* Allocate multicast array memory. */ 544 adapter->mta = malloc(sizeof(u8) * IXGBE_ETH_LENGTH_OF_ADDRESS * 545 MAX_NUM_MULTICAST_ADDRESSES, M_DEVBUF, M_NOWAIT); 546 if (adapter->mta == NULL) { 547 aprint_error_dev(dev, "Cannot allocate multicast setup array\n"); 548 error = ENOMEM; 549 goto err_late; 550 } 551 552 /* Initialize the shared code */ 553 error = ixgbe_init_shared_code(hw); 554 if (error == IXGBE_ERR_SFP_NOT_PRESENT) { 555 /* 556 ** No optics in this port, set up 557 ** so the timer routine will probe 558 ** for later insertion. 559 */ 560 adapter->sfp_probe = TRUE; 561 error = 0; 562 } else if (error == IXGBE_ERR_SFP_NOT_SUPPORTED) { 563 aprint_error_dev(dev,"Unsupported SFP+ module detected!\n"); 564 error = EIO; 565 goto err_late; 566 } else if (error) { 567 aprint_error_dev(dev,"Unable to initialize the shared code\n"); 568 error = EIO; 569 goto err_late; 570 } 571 572 /* Make sure we have a good EEPROM before we read from it */ 573 if (ixgbe_validate_eeprom_checksum(&adapter->hw, &csum) < 0) { 574 aprint_error_dev(dev,"The EEPROM Checksum Is Not Valid\n"); 575 error = EIO; 576 goto err_late; 577 } 578 579 /* Get Hardware Flow Control setting */ 580 hw->fc.requested_mode = ixgbe_fc_full; 581 hw->fc.pause_time = IXGBE_FC_PAUSE; 582 hw->fc.low_water = IXGBE_FC_LO; 583 hw->fc.high_water = IXGBE_FC_HI; 584 hw->fc.send_xon = TRUE; 585 586 error = ixgbe_init_hw(hw); 587 if (error == IXGBE_ERR_EEPROM_VERSION) { 588 aprint_error_dev(dev, "This device is a pre-production adapter/" 589 "LOM. Please be aware there may be issues associated " 590 "with your hardware.\n If you are experiencing problems " 591 "please contact your Intel or hardware representative " 592 "who provided you with this hardware.\n"); 593 } else if (error == IXGBE_ERR_SFP_NOT_SUPPORTED) 594 aprint_error_dev(dev,"Unsupported SFP+ Module\n"); 595 596 if (error) { 597 error = EIO; 598 aprint_error_dev(dev,"Hardware Initialization Failure\n"); 599 goto err_late; 600 } 601 602 /* Detect and set physical type */ 603 ixgbe_setup_optics(adapter); 604 605 if ((adapter->msix > 1) && (ixgbe_enable_msix)) 606 error = ixgbe_allocate_msix(adapter, pa); 607 else 608 error = ixgbe_allocate_legacy(adapter, pa); 609 if (error) 610 goto err_late; 611 612 /* Setup OS specific network interface */ 613 if (ixgbe_setup_interface(dev, adapter) != 0) 614 goto err_late; 615 616 /* Sysctl for limiting the amount of work done in software interrupts */ 617 ixgbe_add_rx_process_limit(adapter, "rx_processing_limit", 618 "max number of rx packets to process", &adapter->rx_process_limit, 619 ixgbe_rx_process_limit); 620 621 /* Initialize statistics */ 622 ixgbe_update_stats_counters(adapter); 623 624 /* Print PCIE bus type/speed/width info */ 625 ixgbe_get_bus_info(hw); 626 aprint_normal_dev(dev,"PCI Express Bus: Speed %s %s\n", 627 ((hw->bus.speed == ixgbe_bus_speed_5000) ? "5.0Gb/s": 628 (hw->bus.speed == ixgbe_bus_speed_2500) ? "2.5Gb/s":"Unknown"), 629 (hw->bus.width == ixgbe_bus_width_pcie_x8) ? "Width x8" : 630 (hw->bus.width == ixgbe_bus_width_pcie_x4) ? "Width x4" : 631 (hw->bus.width == ixgbe_bus_width_pcie_x1) ? "Width x1" : 632 ("Unknown")); 633 634 if ((hw->bus.width <= ixgbe_bus_width_pcie_x4) && 635 (hw->bus.speed == ixgbe_bus_speed_2500)) { 636 aprint_error_dev(dev, "PCI-Express bandwidth available" 637 " for this card\n is not sufficient for" 638 " optimal performance.\n"); 639 aprint_error_dev(dev, "For optimal performance a x8 " 640 "PCIE, or x4 PCIE 2 slot is required.\n"); 641 } 642 643 /* let hardware know driver is loaded */ 644 ctrl_ext = IXGBE_READ_REG(hw, IXGBE_CTRL_EXT); 645 ctrl_ext |= IXGBE_CTRL_EXT_DRV_LOAD; 646 IXGBE_WRITE_REG(hw, IXGBE_CTRL_EXT, ctrl_ext); 647 648 ixgbe_add_hw_stats(adapter); 649 650 INIT_DEBUGOUT("ixgbe_attach: end"); 651 return; 652 err_late: 653 ixgbe_free_transmit_structures(adapter); 654 ixgbe_free_receive_structures(adapter); 655 err_out: 656 if (adapter->ifp != NULL) 657 if_free(adapter->ifp); 658 ixgbe_free_pci_resources(adapter); 659 if (adapter->mta != NULL) 660 free(adapter->mta, M_DEVBUF); 661 return; 662 663 } 664 665 /********************************************************************* 666 * Device removal routine 667 * 668 * The detach entry point is called when the driver is being removed. 669 * This routine stops the adapter and deallocates all the resources 670 * that were allocated for driver operation. 671 * 672 * return 0 on success, positive on failure 673 *********************************************************************/ 674 675 static int 676 ixgbe_detach(device_t dev, int flags) 677 { 678 struct adapter *adapter = device_private(dev); 679 struct tx_ring *txr = adapter->tx_rings; 680 struct rx_ring *rxr = adapter->rx_rings; 681 struct ixgbe_hw_stats *stats = &adapter->stats; 682 struct ix_queue *que = adapter->queues; 683 u32 ctrl_ext; 684 685 INIT_DEBUGOUT("ixgbe_detach: begin"); 686 687 /* Make sure VLANs are not using driver */ 688 if (!VLAN_ATTACHED(&adapter->osdep.ec)) 689 ; /* nothing to do: no VLANs */ 690 else if ((flags & (DETACH_SHUTDOWN|DETACH_FORCE)) != 0) 691 vlan_ifdetach(adapter->ifp); 692 else { 693 aprint_error_dev(dev, "VLANs in use\n"); 694 return EBUSY; 695 } 696 697 IXGBE_CORE_LOCK(adapter); 698 ixgbe_stop(adapter); 699 IXGBE_CORE_UNLOCK(adapter); 700 701 for (int i = 0; i < adapter->num_queues; i++, que++) { 702 softint_disestablish(que->que_si); 703 } 704 705 /* Drain the Link queue */ 706 softint_disestablish(adapter->link_si); 707 softint_disestablish(adapter->mod_si); 708 softint_disestablish(adapter->msf_si); 709 #ifdef IXGBE_FDIR 710 softint_disestablish(adapter->fdir_si); 711 #endif 712 713 /* let hardware know driver is unloading */ 714 ctrl_ext = IXGBE_READ_REG(&adapter->hw, IXGBE_CTRL_EXT); 715 ctrl_ext &= ~IXGBE_CTRL_EXT_DRV_LOAD; 716 IXGBE_WRITE_REG(&adapter->hw, IXGBE_CTRL_EXT, ctrl_ext); 717 718 ether_ifdetach(adapter->ifp); 719 callout_halt(&adapter->timer, NULL); 720 ixgbe_free_pci_resources(adapter); 721 #if 0 /* XXX the NetBSD port is probably missing something here */ 722 bus_generic_detach(dev); 723 #endif 724 if_detach(adapter->ifp); 725 726 sysctl_teardown(&adapter->sysctllog); 727 evcnt_detach(&adapter->handleq); 728 evcnt_detach(&adapter->req); 729 evcnt_detach(&adapter->morerx); 730 evcnt_detach(&adapter->moretx); 731 evcnt_detach(&adapter->txloops); 732 evcnt_detach(&adapter->efbig_tx_dma_setup); 733 evcnt_detach(&adapter->m_defrag_failed); 734 evcnt_detach(&adapter->efbig2_tx_dma_setup); 735 evcnt_detach(&adapter->einval_tx_dma_setup); 736 evcnt_detach(&adapter->other_tx_dma_setup); 737 evcnt_detach(&adapter->eagain_tx_dma_setup); 738 evcnt_detach(&adapter->enomem_tx_dma_setup); 739 evcnt_detach(&adapter->watchdog_events); 740 evcnt_detach(&adapter->tso_err); 741 evcnt_detach(&adapter->tso_tx); 742 evcnt_detach(&adapter->link_irq); 743 for (int i = 0; i < adapter->num_queues; i++, rxr++, txr++) { 744 evcnt_detach(&txr->no_desc_avail); 745 evcnt_detach(&txr->total_packets); 746 747 if (i < __arraycount(adapter->stats.mpc)) { 748 evcnt_detach(&adapter->stats.mpc[i]); 749 } 750 if (i < __arraycount(adapter->stats.pxontxc)) { 751 evcnt_detach(&adapter->stats.pxontxc[i]); 752 evcnt_detach(&adapter->stats.pxonrxc[i]); 753 evcnt_detach(&adapter->stats.pxofftxc[i]); 754 evcnt_detach(&adapter->stats.pxoffrxc[i]); 755 evcnt_detach(&adapter->stats.pxon2offc[i]); 756 } 757 if (i < __arraycount(adapter->stats.qprc)) { 758 evcnt_detach(&adapter->stats.qprc[i]); 759 evcnt_detach(&adapter->stats.qptc[i]); 760 evcnt_detach(&adapter->stats.qbrc[i]); 761 evcnt_detach(&adapter->stats.qbtc[i]); 762 evcnt_detach(&adapter->stats.qprdc[i]); 763 } 764 765 evcnt_detach(&rxr->rx_packets); 766 evcnt_detach(&rxr->rx_bytes); 767 evcnt_detach(&rxr->no_jmbuf); 768 evcnt_detach(&rxr->rx_discarded); 769 evcnt_detach(&rxr->rx_split_packets); 770 evcnt_detach(&rxr->rx_irq); 771 } 772 evcnt_detach(&stats->ipcs); 773 evcnt_detach(&stats->l4cs); 774 evcnt_detach(&stats->ipcs_bad); 775 evcnt_detach(&stats->l4cs_bad); 776 evcnt_detach(&stats->intzero); 777 evcnt_detach(&stats->legint); 778 evcnt_detach(&stats->crcerrs); 779 evcnt_detach(&stats->illerrc); 780 evcnt_detach(&stats->errbc); 781 evcnt_detach(&stats->mspdc); 782 evcnt_detach(&stats->mlfc); 783 evcnt_detach(&stats->mrfc); 784 evcnt_detach(&stats->rlec); 785 evcnt_detach(&stats->lxontxc); 786 evcnt_detach(&stats->lxonrxc); 787 evcnt_detach(&stats->lxofftxc); 788 evcnt_detach(&stats->lxoffrxc); 789 790 /* Packet Reception Stats */ 791 evcnt_detach(&stats->tor); 792 evcnt_detach(&stats->gorc); 793 evcnt_detach(&stats->tpr); 794 evcnt_detach(&stats->gprc); 795 evcnt_detach(&stats->mprc); 796 evcnt_detach(&stats->bprc); 797 evcnt_detach(&stats->prc64); 798 evcnt_detach(&stats->prc127); 799 evcnt_detach(&stats->prc255); 800 evcnt_detach(&stats->prc511); 801 evcnt_detach(&stats->prc1023); 802 evcnt_detach(&stats->prc1522); 803 evcnt_detach(&stats->ruc); 804 evcnt_detach(&stats->rfc); 805 evcnt_detach(&stats->roc); 806 evcnt_detach(&stats->rjc); 807 evcnt_detach(&stats->mngprc); 808 evcnt_detach(&stats->mngptc); 809 evcnt_detach(&stats->xec); 810 811 /* Packet Transmission Stats */ 812 evcnt_detach(&stats->gotc); 813 evcnt_detach(&stats->tpt); 814 evcnt_detach(&stats->gptc); 815 evcnt_detach(&stats->bptc); 816 evcnt_detach(&stats->mptc); 817 evcnt_detach(&stats->mngptc); 818 evcnt_detach(&stats->ptc64); 819 evcnt_detach(&stats->ptc127); 820 evcnt_detach(&stats->ptc255); 821 evcnt_detach(&stats->ptc511); 822 evcnt_detach(&stats->ptc1023); 823 evcnt_detach(&stats->ptc1522); 824 825 /* FC Stats */ 826 evcnt_detach(&stats->fccrc); 827 evcnt_detach(&stats->fclast); 828 if (adapter->hw.mac.type != ixgbe_mac_82598EB) { 829 evcnt_detach(&stats->fcoerpdc); 830 evcnt_detach(&stats->fcoeprc); 831 evcnt_detach(&stats->fcoeptc); 832 evcnt_detach(&stats->fcoedwrc); 833 evcnt_detach(&stats->fcoedwtc); 834 } 835 836 ixgbe_free_transmit_structures(adapter); 837 ixgbe_free_receive_structures(adapter); 838 free(adapter->mta, M_DEVBUF); 839 840 IXGBE_CORE_LOCK_DESTROY(adapter); 841 return (0); 842 } 843 844 /********************************************************************* 845 * 846 * Shutdown entry point 847 * 848 **********************************************************************/ 849 850 #if 0 /* XXX NetBSD ought to register something like this through pmf(9) */ 851 static int 852 ixgbe_shutdown(device_t dev) 853 { 854 struct adapter *adapter = device_private(dev); 855 IXGBE_CORE_LOCK(adapter); 856 ixgbe_stop(adapter); 857 IXGBE_CORE_UNLOCK(adapter); 858 return (0); 859 } 860 #endif 861 862 863 /********************************************************************* 864 * Transmit entry point 865 * 866 * ixgbe_start is called by the stack to initiate a transmit. 867 * The driver will remain in this routine as long as there are 868 * packets to transmit and transmit resources are available. 869 * In case resources are not available stack is notified and 870 * the packet is requeued. 871 **********************************************************************/ 872 873 static void 874 ixgbe_start_locked(struct tx_ring *txr, struct ifnet * ifp) 875 { 876 int rc; 877 struct mbuf *m_head; 878 struct adapter *adapter = txr->adapter; 879 880 IXGBE_TX_LOCK_ASSERT(txr); 881 882 if ((ifp->if_flags & (IFF_RUNNING|IFF_OACTIVE)) != 883 IFF_RUNNING) 884 return; 885 if (!adapter->link_active) 886 return; 887 888 while (!IFQ_IS_EMPTY(&ifp->if_snd)) { 889 890 IFQ_POLL(&ifp->if_snd, m_head); 891 if (m_head == NULL) 892 break; 893 894 if ((rc = ixgbe_xmit(txr, m_head)) == EAGAIN) { 895 ifp->if_flags |= IFF_OACTIVE; 896 break; 897 } 898 IFQ_DEQUEUE(&ifp->if_snd, m_head); 899 if (rc == EFBIG) { 900 struct mbuf *mtmp; 901 902 if ((mtmp = m_defrag(m_head, M_DONTWAIT)) != NULL) { 903 m_head = mtmp; 904 rc = ixgbe_xmit(txr, m_head); 905 if (rc != 0) 906 adapter->efbig2_tx_dma_setup.ev_count++; 907 } else 908 adapter->m_defrag_failed.ev_count++; 909 } 910 if (rc != 0) { 911 m_freem(m_head); 912 continue; 913 } 914 915 /* Send a copy of the frame to the BPF listener */ 916 bpf_mtap(ifp, m_head); 917 918 /* Set watchdog on */ 919 getmicrotime(&txr->watchdog_time); 920 txr->queue_status = IXGBE_QUEUE_WORKING; 921 922 } 923 return; 924 } 925 926 /* 927 * Legacy TX start - called by the stack, this 928 * always uses the first tx ring, and should 929 * not be used with multiqueue tx enabled. 930 */ 931 static void 932 ixgbe_start(struct ifnet *ifp) 933 { 934 struct adapter *adapter = ifp->if_softc; 935 struct tx_ring *txr = adapter->tx_rings; 936 937 if (ifp->if_flags & IFF_RUNNING) { 938 IXGBE_TX_LOCK(txr); 939 ixgbe_start_locked(txr, ifp); 940 IXGBE_TX_UNLOCK(txr); 941 } 942 return; 943 } 944 945 #if __FreeBSD_version >= 800000 946 /* 947 ** Multiqueue Transmit driver 948 ** 949 */ 950 static int 951 ixgbe_mq_start(struct ifnet *ifp, struct mbuf *m) 952 { 953 struct adapter *adapter = ifp->if_softc; 954 struct ix_queue *que; 955 struct tx_ring *txr; 956 int i = 0, err = 0; 957 958 /* Which queue to use */ 959 if ((m->m_flags & M_FLOWID) != 0) 960 i = m->m_pkthdr.flowid % adapter->num_queues; 961 962 txr = &adapter->tx_rings[i]; 963 que = &adapter->queues[i]; 964 965 if (IXGBE_TX_TRYLOCK(txr)) { 966 err = ixgbe_mq_start_locked(ifp, txr, m); 967 IXGBE_TX_UNLOCK(txr); 968 } else { 969 err = drbr_enqueue(ifp, txr->br, m); 970 softint_schedule(que->que_si); 971 } 972 973 return (err); 974 } 975 976 static int 977 ixgbe_mq_start_locked(struct ifnet *ifp, struct tx_ring *txr, struct mbuf *m) 978 { 979 struct adapter *adapter = txr->adapter; 980 struct mbuf *next; 981 int enqueued, err = 0; 982 983 if ((ifp->if_flags & (IFF_RUNNING | IFF_OACTIVE)) != 984 IFF_RUNNING || adapter->link_active == 0) { 985 if (m != NULL) 986 err = drbr_enqueue(ifp, txr->br, m); 987 return (err); 988 } 989 990 enqueued = 0; 991 if (m == NULL) { 992 next = drbr_dequeue(ifp, txr->br); 993 } else if (drbr_needs_enqueue(ifp, txr->br)) { 994 if ((err = drbr_enqueue(ifp, txr->br, m)) != 0) 995 return (err); 996 next = drbr_dequeue(ifp, txr->br); 997 } else 998 next = m; 999 1000 /* Process the queue */ 1001 while (next != NULL) { 1002 if ((err = ixgbe_xmit(txr, &next)) != 0) { 1003 if (next != NULL) 1004 err = drbr_enqueue(ifp, txr->br, next); 1005 break; 1006 } 1007 enqueued++; 1008 drbr_stats_update(ifp, next->m_pkthdr.len, next->m_flags); 1009 /* Send a copy of the frame to the BPF listener */ 1010 bpf_mtap(ifp, next); 1011 if ((ifp->if_flags & IFF_RUNNING) == 0) 1012 break; 1013 if (txr->tx_avail < IXGBE_TX_OP_THRESHOLD) 1014 ixgbe_txeof(txr); 1015 if (txr->tx_avail < IXGBE_TX_OP_THRESHOLD) { 1016 ifp->if_flags |= IFF_OACTIVE; 1017 break; 1018 } 1019 next = drbr_dequeue(ifp, txr->br); 1020 } 1021 1022 if (enqueued > 0) { 1023 /* Set watchdog on */ 1024 txr->queue_status = IXGBE_QUEUE_WORKING; 1025 getmicrotime(&txr->watchdog_time); 1026 } 1027 1028 return (err); 1029 } 1030 1031 /* 1032 ** Flush all ring buffers 1033 */ 1034 static void 1035 ixgbe_qflush(struct ifnet *ifp) 1036 { 1037 struct adapter *adapter = ifp->if_softc; 1038 struct tx_ring *txr = adapter->tx_rings; 1039 struct mbuf *m; 1040 1041 for (int i = 0; i < adapter->num_queues; i++, txr++) { 1042 IXGBE_TX_LOCK(txr); 1043 while ((m = buf_ring_dequeue_sc(txr->br)) != NULL) 1044 m_freem(m); 1045 IXGBE_TX_UNLOCK(txr); 1046 } 1047 if_qflush(ifp); 1048 } 1049 #endif /* __FreeBSD_version >= 800000 */ 1050 1051 static int 1052 ixgbe_ifflags_cb(struct ethercom *ec) 1053 { 1054 struct ifnet *ifp = &ec->ec_if; 1055 struct adapter *adapter = ifp->if_softc; 1056 int change = ifp->if_flags ^ adapter->if_flags, rc = 0; 1057 1058 IXGBE_CORE_LOCK(adapter); 1059 1060 if (change != 0) 1061 adapter->if_flags = ifp->if_flags; 1062 1063 if ((change & ~(IFF_CANTCHANGE|IFF_DEBUG)) != 0) 1064 rc = ENETRESET; 1065 else if ((change & (IFF_PROMISC | IFF_ALLMULTI)) != 0) 1066 ixgbe_set_promisc(adapter); 1067 1068 IXGBE_CORE_UNLOCK(adapter); 1069 1070 return rc; 1071 } 1072 1073 /********************************************************************* 1074 * Ioctl entry point 1075 * 1076 * ixgbe_ioctl is called when the user wants to configure the 1077 * interface. 1078 * 1079 * return 0 on success, positive on failure 1080 **********************************************************************/ 1081 1082 static int 1083 ixgbe_ioctl(struct ifnet * ifp, u_long command, void *data) 1084 { 1085 struct adapter *adapter = ifp->if_softc; 1086 struct ifcapreq *ifcr = data; 1087 struct ifreq *ifr = data; 1088 int error = 0; 1089 int l4csum_en; 1090 const int l4csum = IFCAP_CSUM_TCPv4_Rx|IFCAP_CSUM_UDPv4_Rx| 1091 IFCAP_CSUM_TCPv6_Rx|IFCAP_CSUM_UDPv6_Rx; 1092 1093 switch (command) { 1094 case SIOCSIFFLAGS: 1095 IOCTL_DEBUGOUT("ioctl: SIOCSIFFLAGS (Set Interface Flags)"); 1096 break; 1097 case SIOCADDMULTI: 1098 case SIOCDELMULTI: 1099 IOCTL_DEBUGOUT("ioctl: SIOC(ADD|DEL)MULTI"); 1100 break; 1101 case SIOCSIFMEDIA: 1102 case SIOCGIFMEDIA: 1103 IOCTL_DEBUGOUT("ioctl: SIOCxIFMEDIA (Get/Set Interface Media)"); 1104 break; 1105 case SIOCSIFCAP: 1106 IOCTL_DEBUGOUT("ioctl: SIOCSIFCAP (Set Capabilities)"); 1107 break; 1108 case SIOCSIFMTU: 1109 IOCTL_DEBUGOUT("ioctl: SIOCSIFMTU (Set Interface MTU)"); 1110 break; 1111 default: 1112 IOCTL_DEBUGOUT1("ioctl: UNKNOWN (0x%X)\n", (int)command); 1113 break; 1114 } 1115 1116 switch (command) { 1117 case SIOCSIFMEDIA: 1118 case SIOCGIFMEDIA: 1119 return ifmedia_ioctl(ifp, ifr, &adapter->media, command); 1120 case SIOCSIFCAP: 1121 /* Layer-4 Rx checksum offload has to be turned on and 1122 * off as a unit. 1123 */ 1124 l4csum_en = ifcr->ifcr_capenable & l4csum; 1125 if (l4csum_en != l4csum && l4csum_en != 0) 1126 return EINVAL; 1127 /*FALLTHROUGH*/ 1128 case SIOCADDMULTI: 1129 case SIOCDELMULTI: 1130 case SIOCSIFFLAGS: 1131 case SIOCSIFMTU: 1132 default: 1133 if ((error = ether_ioctl(ifp, command, data)) != ENETRESET) 1134 return error; 1135 if ((ifp->if_flags & IFF_RUNNING) == 0) 1136 ; 1137 else if (command == SIOCSIFCAP || command == SIOCSIFMTU) { 1138 IXGBE_CORE_LOCK(adapter); 1139 ixgbe_init_locked(adapter); 1140 IXGBE_CORE_UNLOCK(adapter); 1141 } else if (command == SIOCADDMULTI || command == SIOCDELMULTI) { 1142 /* 1143 * Multicast list has changed; set the hardware filter 1144 * accordingly. 1145 */ 1146 IXGBE_CORE_LOCK(adapter); 1147 ixgbe_disable_intr(adapter); 1148 ixgbe_set_multi(adapter); 1149 ixgbe_enable_intr(adapter); 1150 IXGBE_CORE_UNLOCK(adapter); 1151 } 1152 return 0; 1153 } 1154 } 1155 1156 /********************************************************************* 1157 * Init entry point 1158 * 1159 * This routine is used in two ways. It is used by the stack as 1160 * init entry point in network interface structure. It is also used 1161 * by the driver as a hw/sw initialization routine to get to a 1162 * consistent state. 1163 * 1164 * return 0 on success, positive on failure 1165 **********************************************************************/ 1166 #define IXGBE_MHADD_MFS_SHIFT 16 1167 1168 static void 1169 ixgbe_init_locked(struct adapter *adapter) 1170 { 1171 struct ifnet *ifp = adapter->ifp; 1172 device_t dev = adapter->dev; 1173 struct ixgbe_hw *hw = &adapter->hw; 1174 u32 k, txdctl, mhadd, gpie; 1175 u32 rxdctl, rxctrl; 1176 1177 /* XXX check IFF_UP and IFF_RUNNING, power-saving state! */ 1178 1179 KASSERT(mutex_owned(&adapter->core_mtx)); 1180 INIT_DEBUGOUT("ixgbe_init: begin"); 1181 hw->adapter_stopped = FALSE; 1182 ixgbe_stop_adapter(hw); 1183 callout_stop(&adapter->timer); 1184 1185 /* XXX I moved this here from the SIOCSIFMTU case in ixgbe_ioctl(). */ 1186 adapter->max_frame_size = 1187 ifp->if_mtu + ETHER_HDR_LEN + ETHER_CRC_LEN; 1188 1189 /* reprogram the RAR[0] in case user changed it. */ 1190 ixgbe_set_rar(hw, 0, adapter->hw.mac.addr, 0, IXGBE_RAH_AV); 1191 1192 /* Get the latest mac address, User can use a LAA */ 1193 memcpy(hw->mac.addr, CLLADDR(adapter->ifp->if_sadl), 1194 IXGBE_ETH_LENGTH_OF_ADDRESS); 1195 ixgbe_set_rar(hw, 0, hw->mac.addr, 0, 1); 1196 hw->addr_ctrl.rar_used_count = 1; 1197 1198 /* Prepare transmit descriptors and buffers */ 1199 if (ixgbe_setup_transmit_structures(adapter)) { 1200 device_printf(dev,"Could not setup transmit structures\n"); 1201 ixgbe_stop(adapter); 1202 return; 1203 } 1204 1205 ixgbe_init_hw(hw); 1206 ixgbe_initialize_transmit_units(adapter); 1207 1208 /* Setup Multicast table */ 1209 ixgbe_set_multi(adapter); 1210 1211 /* 1212 ** Determine the correct mbuf pool 1213 ** for doing jumbo/headersplit 1214 */ 1215 if (adapter->max_frame_size <= 2048) 1216 adapter->rx_mbuf_sz = MCLBYTES; 1217 else if (adapter->max_frame_size <= 4096) 1218 adapter->rx_mbuf_sz = MJUMPAGESIZE; 1219 else if (adapter->max_frame_size <= 9216) 1220 adapter->rx_mbuf_sz = MJUM9BYTES; 1221 else 1222 adapter->rx_mbuf_sz = MJUM16BYTES; 1223 1224 /* Prepare receive descriptors and buffers */ 1225 if (ixgbe_setup_receive_structures(adapter)) { 1226 device_printf(dev,"Could not setup receive structures\n"); 1227 ixgbe_stop(adapter); 1228 return; 1229 } 1230 1231 /* Configure RX settings */ 1232 ixgbe_initialize_receive_units(adapter); 1233 1234 gpie = IXGBE_READ_REG(&adapter->hw, IXGBE_GPIE); 1235 1236 /* Enable Fan Failure Interrupt */ 1237 gpie |= IXGBE_SDP1_GPIEN; 1238 1239 /* Add for Thermal detection */ 1240 if (hw->mac.type == ixgbe_mac_82599EB) 1241 gpie |= IXGBE_SDP2_GPIEN; 1242 1243 if (adapter->msix > 1) { 1244 /* Enable Enhanced MSIX mode */ 1245 gpie |= IXGBE_GPIE_MSIX_MODE; 1246 gpie |= IXGBE_GPIE_EIAME | IXGBE_GPIE_PBA_SUPPORT | 1247 IXGBE_GPIE_OCD; 1248 } 1249 IXGBE_WRITE_REG(hw, IXGBE_GPIE, gpie); 1250 1251 /* Set MTU size */ 1252 if (ifp->if_mtu > ETHERMTU) { 1253 mhadd = IXGBE_READ_REG(hw, IXGBE_MHADD); 1254 mhadd &= ~IXGBE_MHADD_MFS_MASK; 1255 mhadd |= adapter->max_frame_size << IXGBE_MHADD_MFS_SHIFT; 1256 IXGBE_WRITE_REG(hw, IXGBE_MHADD, mhadd); 1257 } 1258 1259 /* Now enable all the queues */ 1260 1261 for (int i = 0; i < adapter->num_queues; i++) { 1262 txdctl = IXGBE_READ_REG(hw, IXGBE_TXDCTL(i)); 1263 txdctl |= IXGBE_TXDCTL_ENABLE; 1264 /* Set WTHRESH to 8, burst writeback */ 1265 txdctl |= (8 << 16); 1266 IXGBE_WRITE_REG(hw, IXGBE_TXDCTL(i), txdctl); 1267 } 1268 1269 for (int i = 0; i < adapter->num_queues; i++) { 1270 rxdctl = IXGBE_READ_REG(hw, IXGBE_RXDCTL(i)); 1271 if (hw->mac.type == ixgbe_mac_82598EB) { 1272 /* 1273 ** PTHRESH = 21 1274 ** HTHRESH = 4 1275 ** WTHRESH = 8 1276 */ 1277 rxdctl &= ~0x3FFFFF; 1278 rxdctl |= 0x080420; 1279 } 1280 rxdctl |= IXGBE_RXDCTL_ENABLE; 1281 IXGBE_WRITE_REG(hw, IXGBE_RXDCTL(i), rxdctl); 1282 /* XXX I don't trust this loop, and I don't trust the 1283 * XXX memory barrier. What is this meant to do? --dyoung 1284 */ 1285 for (k = 0; k < 10; k++) { 1286 if (IXGBE_READ_REG(hw, IXGBE_RXDCTL(i)) & 1287 IXGBE_RXDCTL_ENABLE) 1288 break; 1289 else 1290 msec_delay(1); 1291 } 1292 wmb(); 1293 IXGBE_WRITE_REG(hw, IXGBE_RDT(i), adapter->num_rx_desc - 1); 1294 } 1295 1296 /* Set up VLAN support and filter */ 1297 ixgbe_setup_vlan_hw_support(adapter); 1298 1299 /* Enable Receive engine */ 1300 rxctrl = IXGBE_READ_REG(hw, IXGBE_RXCTRL); 1301 if (hw->mac.type == ixgbe_mac_82598EB) 1302 rxctrl |= IXGBE_RXCTRL_DMBYPS; 1303 rxctrl |= IXGBE_RXCTRL_RXEN; 1304 ixgbe_enable_rx_dma(hw, rxctrl); 1305 1306 callout_reset(&adapter->timer, hz, ixgbe_local_timer, adapter); 1307 1308 /* Set up MSI/X routing */ 1309 if (ixgbe_enable_msix) { 1310 ixgbe_configure_ivars(adapter); 1311 /* Set up auto-mask */ 1312 if (hw->mac.type == ixgbe_mac_82598EB) 1313 IXGBE_WRITE_REG(hw, IXGBE_EIAM, IXGBE_EICS_RTX_QUEUE); 1314 else { 1315 IXGBE_WRITE_REG(hw, IXGBE_EIAM_EX(0), 0xFFFFFFFF); 1316 IXGBE_WRITE_REG(hw, IXGBE_EIAM_EX(1), 0xFFFFFFFF); 1317 } 1318 } else { /* Simple settings for Legacy/MSI */ 1319 ixgbe_set_ivar(adapter, 0, 0, 0); 1320 ixgbe_set_ivar(adapter, 0, 0, 1); 1321 IXGBE_WRITE_REG(hw, IXGBE_EIAM, IXGBE_EICS_RTX_QUEUE); 1322 } 1323 1324 #ifdef IXGBE_FDIR 1325 /* Init Flow director */ 1326 if (hw->mac.type != ixgbe_mac_82598EB) 1327 ixgbe_init_fdir_signature_82599(&adapter->hw, fdir_pballoc); 1328 #endif 1329 1330 /* 1331 ** Check on any SFP devices that 1332 ** need to be kick-started 1333 */ 1334 if (hw->phy.type == ixgbe_phy_none) { 1335 int err = hw->phy.ops.identify(hw); 1336 if (err == IXGBE_ERR_SFP_NOT_SUPPORTED) { 1337 device_printf(dev, 1338 "Unsupported SFP+ module type was detected.\n"); 1339 return; 1340 } 1341 } 1342 1343 /* Set moderation on the Link interrupt */ 1344 IXGBE_WRITE_REG(hw, IXGBE_EITR(adapter->linkvec), IXGBE_LINK_ITR); 1345 1346 /* Config/Enable Link */ 1347 ixgbe_config_link(adapter); 1348 1349 /* And now turn on interrupts */ 1350 ixgbe_enable_intr(adapter); 1351 1352 /* Now inform the stack we're ready */ 1353 ifp->if_flags |= IFF_RUNNING; 1354 ifp->if_flags &= ~IFF_OACTIVE; 1355 1356 return; 1357 } 1358 1359 static int 1360 ixgbe_init(struct ifnet *ifp) 1361 { 1362 struct adapter *adapter = ifp->if_softc; 1363 1364 IXGBE_CORE_LOCK(adapter); 1365 ixgbe_init_locked(adapter); 1366 IXGBE_CORE_UNLOCK(adapter); 1367 return 0; /* XXX ixgbe_init_locked cannot fail? really? */ 1368 } 1369 1370 1371 /* 1372 ** 1373 ** MSIX Interrupt Handlers and Tasklets 1374 ** 1375 */ 1376 1377 static inline void 1378 ixgbe_enable_queue(struct adapter *adapter, u32 vector) 1379 { 1380 struct ixgbe_hw *hw = &adapter->hw; 1381 u64 queue = (u64)(1 << vector); 1382 u32 mask; 1383 1384 if (hw->mac.type == ixgbe_mac_82598EB) { 1385 mask = (IXGBE_EIMS_RTX_QUEUE & queue); 1386 IXGBE_WRITE_REG(hw, IXGBE_EIMS, mask); 1387 } else { 1388 mask = (queue & 0xFFFFFFFF); 1389 if (mask) 1390 IXGBE_WRITE_REG(hw, IXGBE_EIMS_EX(0), mask); 1391 mask = (queue >> 32); 1392 if (mask) 1393 IXGBE_WRITE_REG(hw, IXGBE_EIMS_EX(1), mask); 1394 } 1395 } 1396 1397 __unused static inline void 1398 ixgbe_disable_queue(struct adapter *adapter, u32 vector) 1399 { 1400 struct ixgbe_hw *hw = &adapter->hw; 1401 u64 queue = (u64)(1 << vector); 1402 u32 mask; 1403 1404 if (hw->mac.type == ixgbe_mac_82598EB) { 1405 mask = (IXGBE_EIMS_RTX_QUEUE & queue); 1406 IXGBE_WRITE_REG(hw, IXGBE_EIMC, mask); 1407 } else { 1408 mask = (queue & 0xFFFFFFFF); 1409 if (mask) 1410 IXGBE_WRITE_REG(hw, IXGBE_EIMC_EX(0), mask); 1411 mask = (queue >> 32); 1412 if (mask) 1413 IXGBE_WRITE_REG(hw, IXGBE_EIMC_EX(1), mask); 1414 } 1415 } 1416 1417 static inline void 1418 ixgbe_rearm_queues(struct adapter *adapter, u64 queues) 1419 { 1420 u32 mask; 1421 1422 if (adapter->hw.mac.type == ixgbe_mac_82598EB) { 1423 mask = (IXGBE_EIMS_RTX_QUEUE & queues); 1424 IXGBE_WRITE_REG(&adapter->hw, IXGBE_EICS, mask); 1425 } else { 1426 mask = (queues & 0xFFFFFFFF); 1427 IXGBE_WRITE_REG(&adapter->hw, IXGBE_EICS_EX(0), mask); 1428 mask = (queues >> 32); 1429 IXGBE_WRITE_REG(&adapter->hw, IXGBE_EICS_EX(1), mask); 1430 } 1431 } 1432 1433 1434 static void 1435 ixgbe_handle_que(void *context) 1436 { 1437 struct ix_queue *que = context; 1438 struct adapter *adapter = que->adapter; 1439 struct tx_ring *txr = que->txr; 1440 struct ifnet *ifp = adapter->ifp; 1441 bool more; 1442 1443 adapter->handleq.ev_count++; 1444 1445 if (ifp->if_flags & IFF_RUNNING) { 1446 more = ixgbe_rxeof(que, adapter->rx_process_limit); 1447 IXGBE_TX_LOCK(txr); 1448 ixgbe_txeof(txr); 1449 #if __FreeBSD_version >= 800000 1450 if (!drbr_empty(ifp, txr->br)) 1451 ixgbe_mq_start_locked(ifp, txr, NULL); 1452 #else 1453 if (!IFQ_IS_EMPTY(&ifp->if_snd)) 1454 ixgbe_start_locked(txr, ifp); 1455 #endif 1456 IXGBE_TX_UNLOCK(txr); 1457 if (more) { 1458 adapter->req.ev_count++; 1459 softint_schedule(que->que_si); 1460 return; 1461 } 1462 } 1463 1464 /* Reenable this interrupt */ 1465 ixgbe_enable_queue(adapter, que->msix); 1466 1467 return; 1468 } 1469 1470 1471 /********************************************************************* 1472 * 1473 * Legacy Interrupt Service routine 1474 * 1475 **********************************************************************/ 1476 1477 static int 1478 ixgbe_legacy_irq(void *arg) 1479 { 1480 struct ix_queue *que = arg; 1481 struct adapter *adapter = que->adapter; 1482 struct ixgbe_hw *hw = &adapter->hw; 1483 struct tx_ring *txr = adapter->tx_rings; 1484 bool more_tx, more_rx; 1485 u32 reg_eicr, loop = MAX_LOOP; 1486 1487 1488 reg_eicr = IXGBE_READ_REG(hw, IXGBE_EICR); 1489 1490 adapter->stats.legint.ev_count++; 1491 ++que->irqs; 1492 if (reg_eicr == 0) { 1493 adapter->stats.intzero.ev_count++; 1494 ixgbe_enable_intr(adapter); 1495 return 0; 1496 } 1497 1498 more_rx = ixgbe_rxeof(que, adapter->rx_process_limit); 1499 1500 IXGBE_TX_LOCK(txr); 1501 do { 1502 adapter->txloops.ev_count++; 1503 more_tx = ixgbe_txeof(txr); 1504 } while (loop-- && more_tx); 1505 IXGBE_TX_UNLOCK(txr); 1506 1507 if (more_rx || more_tx) { 1508 if (more_rx) 1509 adapter->morerx.ev_count++; 1510 if (more_tx) 1511 adapter->moretx.ev_count++; 1512 softint_schedule(que->que_si); 1513 } 1514 1515 /* Check for fan failure */ 1516 if ((hw->phy.media_type == ixgbe_media_type_copper) && 1517 (reg_eicr & IXGBE_EICR_GPI_SDP1)) { 1518 device_printf(adapter->dev, "\nCRITICAL: FAN FAILURE!! " 1519 "REPLACE IMMEDIATELY!!\n"); 1520 IXGBE_WRITE_REG(hw, IXGBE_EIMS, IXGBE_EICR_GPI_SDP1); 1521 } 1522 1523 /* Link status change */ 1524 if (reg_eicr & IXGBE_EICR_LSC) 1525 softint_schedule(adapter->link_si); 1526 1527 ixgbe_enable_intr(adapter); 1528 return 1; 1529 } 1530 1531 1532 #if defined(NETBSD_MSI_OR_MSIX) 1533 /********************************************************************* 1534 * 1535 * MSI Queue Interrupt Service routine 1536 * 1537 **********************************************************************/ 1538 void 1539 ixgbe_msix_que(void *arg) 1540 { 1541 struct ix_queue *que = arg; 1542 struct adapter *adapter = que->adapter; 1543 struct tx_ring *txr = que->txr; 1544 struct rx_ring *rxr = que->rxr; 1545 bool more_tx, more_rx; 1546 u32 newitr = 0; 1547 1548 ++que->irqs; 1549 1550 more_rx = ixgbe_rxeof(que, adapter->rx_process_limit); 1551 1552 IXGBE_TX_LOCK(txr); 1553 more_tx = ixgbe_txeof(txr); 1554 IXGBE_TX_UNLOCK(txr); 1555 1556 /* Do AIM now? */ 1557 1558 if (ixgbe_enable_aim == FALSE) 1559 goto no_calc; 1560 /* 1561 ** Do Adaptive Interrupt Moderation: 1562 ** - Write out last calculated setting 1563 ** - Calculate based on average size over 1564 ** the last interval. 1565 */ 1566 if (que->eitr_setting) 1567 IXGBE_WRITE_REG(&adapter->hw, 1568 IXGBE_EITR(que->msix), que->eitr_setting); 1569 1570 que->eitr_setting = 0; 1571 1572 /* Idle, do nothing */ 1573 if ((txr->bytes == 0) && (rxr->bytes == 0)) 1574 goto no_calc; 1575 1576 if ((txr->bytes) && (txr->packets)) 1577 newitr = txr->bytes/txr->packets; 1578 if ((rxr->bytes) && (rxr->packets)) 1579 newitr = max(newitr, 1580 (rxr->bytes / rxr->packets)); 1581 newitr += 24; /* account for hardware frame, crc */ 1582 1583 /* set an upper boundary */ 1584 newitr = min(newitr, 3000); 1585 1586 /* Be nice to the mid range */ 1587 if ((newitr > 300) && (newitr < 1200)) 1588 newitr = (newitr / 3); 1589 else 1590 newitr = (newitr / 2); 1591 1592 if (adapter->hw.mac.type == ixgbe_mac_82598EB) 1593 newitr |= newitr << 16; 1594 else 1595 newitr |= IXGBE_EITR_CNT_WDIS; 1596 1597 /* save for next interrupt */ 1598 que->eitr_setting = newitr; 1599 1600 /* Reset state */ 1601 txr->bytes = 0; 1602 txr->packets = 0; 1603 rxr->bytes = 0; 1604 rxr->packets = 0; 1605 1606 no_calc: 1607 if (more_tx || more_rx) 1608 softint_schedule(que->que_si); 1609 else /* Reenable this interrupt */ 1610 ixgbe_enable_queue(adapter, que->msix); 1611 return; 1612 } 1613 1614 1615 static void 1616 ixgbe_msix_link(void *arg) 1617 { 1618 struct adapter *adapter = arg; 1619 struct ixgbe_hw *hw = &adapter->hw; 1620 u32 reg_eicr; 1621 1622 ++adapter->link_irq.ev_count; 1623 1624 /* First get the cause */ 1625 reg_eicr = IXGBE_READ_REG(hw, IXGBE_EICS); 1626 /* Clear interrupt with write */ 1627 IXGBE_WRITE_REG(hw, IXGBE_EICR, reg_eicr); 1628 1629 /* Link status change */ 1630 if (reg_eicr & IXGBE_EICR_LSC) 1631 softint_schedule(adapter->link_si); 1632 1633 if (adapter->hw.mac.type != ixgbe_mac_82598EB) { 1634 #ifdef IXGBE_FDIR 1635 if (reg_eicr & IXGBE_EICR_FLOW_DIR) { 1636 /* This is probably overkill :) */ 1637 if (!atomic_cmpset_int(&adapter->fdir_reinit, 0, 1)) 1638 return; 1639 /* Clear the interrupt */ 1640 IXGBE_WRITE_REG(hw, IXGBE_EICR, IXGBE_EICR_FLOW_DIR); 1641 /* Turn off the interface */ 1642 adapter->ifp->if_flags &= ~IFF_RUNNING; 1643 softint_schedule(adapter->fdir_si); 1644 } else 1645 #endif 1646 if (reg_eicr & IXGBE_EICR_ECC) { 1647 device_printf(adapter->dev, "\nCRITICAL: ECC ERROR!! " 1648 "Please Reboot!!\n"); 1649 IXGBE_WRITE_REG(hw, IXGBE_EICR, IXGBE_EICR_ECC); 1650 } else 1651 1652 if (reg_eicr & IXGBE_EICR_GPI_SDP1) { 1653 /* Clear the interrupt */ 1654 IXGBE_WRITE_REG(hw, IXGBE_EICR, IXGBE_EICR_GPI_SDP1); 1655 softint_schedule(adapter->msf_si); 1656 } else if (reg_eicr & IXGBE_EICR_GPI_SDP2) { 1657 /* Clear the interrupt */ 1658 IXGBE_WRITE_REG(hw, IXGBE_EICR, IXGBE_EICR_GPI_SDP2); 1659 softint_schedule(adapter->mod_si); 1660 } 1661 } 1662 1663 /* Check for fan failure */ 1664 if ((hw->device_id == IXGBE_DEV_ID_82598AT) && 1665 (reg_eicr & IXGBE_EICR_GPI_SDP1)) { 1666 device_printf(adapter->dev, "\nCRITICAL: FAN FAILURE!! " 1667 "REPLACE IMMEDIATELY!!\n"); 1668 IXGBE_WRITE_REG(hw, IXGBE_EICR, IXGBE_EICR_GPI_SDP1); 1669 } 1670 1671 IXGBE_WRITE_REG(&adapter->hw, IXGBE_EIMS, IXGBE_EIMS_OTHER); 1672 return; 1673 } 1674 #endif 1675 1676 /********************************************************************* 1677 * 1678 * Media Ioctl callback 1679 * 1680 * This routine is called whenever the user queries the status of 1681 * the interface using ifconfig. 1682 * 1683 **********************************************************************/ 1684 static void 1685 ixgbe_media_status(struct ifnet * ifp, struct ifmediareq * ifmr) 1686 { 1687 struct adapter *adapter = ifp->if_softc; 1688 1689 INIT_DEBUGOUT("ixgbe_media_status: begin"); 1690 IXGBE_CORE_LOCK(adapter); 1691 ixgbe_update_link_status(adapter); 1692 1693 ifmr->ifm_status = IFM_AVALID; 1694 ifmr->ifm_active = IFM_ETHER; 1695 1696 if (!adapter->link_active) { 1697 IXGBE_CORE_UNLOCK(adapter); 1698 return; 1699 } 1700 1701 ifmr->ifm_status |= IFM_ACTIVE; 1702 1703 switch (adapter->link_speed) { 1704 case IXGBE_LINK_SPEED_1GB_FULL: 1705 ifmr->ifm_active |= IFM_1000_T | IFM_FDX; 1706 break; 1707 case IXGBE_LINK_SPEED_10GB_FULL: 1708 ifmr->ifm_active |= adapter->optics | IFM_FDX; 1709 break; 1710 } 1711 1712 IXGBE_CORE_UNLOCK(adapter); 1713 1714 return; 1715 } 1716 1717 /********************************************************************* 1718 * 1719 * Media Ioctl callback 1720 * 1721 * This routine is called when the user changes speed/duplex using 1722 * media/mediopt option with ifconfig. 1723 * 1724 **********************************************************************/ 1725 static int 1726 ixgbe_media_change(struct ifnet * ifp) 1727 { 1728 struct adapter *adapter = ifp->if_softc; 1729 struct ifmedia *ifm = &adapter->media; 1730 1731 INIT_DEBUGOUT("ixgbe_media_change: begin"); 1732 1733 if (IFM_TYPE(ifm->ifm_media) != IFM_ETHER) 1734 return (EINVAL); 1735 1736 switch (IFM_SUBTYPE(ifm->ifm_media)) { 1737 case IFM_AUTO: 1738 adapter->hw.phy.autoneg_advertised = 1739 IXGBE_LINK_SPEED_1GB_FULL | IXGBE_LINK_SPEED_10GB_FULL; 1740 break; 1741 default: 1742 device_printf(adapter->dev, "Only auto media type\n"); 1743 return (EINVAL); 1744 } 1745 1746 return (0); 1747 } 1748 1749 /********************************************************************* 1750 * 1751 * This routine maps the mbufs to tx descriptors, allowing the 1752 * TX engine to transmit the packets. 1753 * - return 0 on success, positive on failure 1754 * 1755 **********************************************************************/ 1756 1757 static int 1758 ixgbe_xmit(struct tx_ring *txr, struct mbuf *m_head) 1759 { 1760 struct m_tag *mtag; 1761 struct adapter *adapter = txr->adapter; 1762 struct ethercom *ec = &adapter->osdep.ec; 1763 u32 olinfo_status = 0, cmd_type_len; 1764 u32 paylen = 0; 1765 int i, j, error; 1766 int first, last = 0; 1767 bus_dmamap_t map; 1768 struct ixgbe_tx_buf *txbuf; 1769 union ixgbe_adv_tx_desc *txd = NULL; 1770 1771 /* Basic descriptor defines */ 1772 cmd_type_len = (IXGBE_ADVTXD_DTYP_DATA | 1773 IXGBE_ADVTXD_DCMD_IFCS | IXGBE_ADVTXD_DCMD_DEXT); 1774 1775 if ((mtag = VLAN_OUTPUT_TAG(ec, m_head)) != NULL) 1776 cmd_type_len |= IXGBE_ADVTXD_DCMD_VLE; 1777 1778 /* 1779 * Important to capture the first descriptor 1780 * used because it will contain the index of 1781 * the one we tell the hardware to report back 1782 */ 1783 first = txr->next_avail_desc; 1784 txbuf = &txr->tx_buffers[first]; 1785 map = txbuf->map; 1786 1787 /* 1788 * Map the packet for DMA. 1789 */ 1790 error = bus_dmamap_load_mbuf(txr->txtag->dt_dmat, map, 1791 m_head, BUS_DMA_NOWAIT); 1792 1793 switch (error) { 1794 case EAGAIN: 1795 adapter->eagain_tx_dma_setup.ev_count++; 1796 return EAGAIN; 1797 case ENOMEM: 1798 adapter->enomem_tx_dma_setup.ev_count++; 1799 return EAGAIN; 1800 case EFBIG: 1801 adapter->efbig_tx_dma_setup.ev_count++; 1802 return error; 1803 case EINVAL: 1804 adapter->einval_tx_dma_setup.ev_count++; 1805 return error; 1806 default: 1807 adapter->other_tx_dma_setup.ev_count++; 1808 return error; 1809 case 0: 1810 break; 1811 } 1812 1813 /* Make certain there are enough descriptors */ 1814 if (map->dm_nsegs > txr->tx_avail - 2) { 1815 txr->no_desc_avail.ev_count++; 1816 ixgbe_dmamap_unload(txr->txtag, txbuf->map); 1817 return EAGAIN; 1818 } 1819 1820 /* 1821 ** Set up the appropriate offload context 1822 ** this becomes the first descriptor of 1823 ** a packet. 1824 */ 1825 if (m_head->m_pkthdr.csum_flags & (M_CSUM_TSOv4|M_CSUM_TSOv6)) { 1826 if (ixgbe_tso_setup(txr, m_head, &paylen)) { 1827 cmd_type_len |= IXGBE_ADVTXD_DCMD_TSE; 1828 olinfo_status |= IXGBE_TXD_POPTS_IXSM << 8; 1829 olinfo_status |= IXGBE_TXD_POPTS_TXSM << 8; 1830 olinfo_status |= paylen << IXGBE_ADVTXD_PAYLEN_SHIFT; 1831 ++adapter->tso_tx.ev_count; 1832 } else { 1833 ++adapter->tso_err.ev_count; 1834 /* XXX unload DMA map! --dyoung */ 1835 return ENXIO; 1836 } 1837 } else 1838 olinfo_status |= ixgbe_tx_ctx_setup(txr, m_head); 1839 1840 #ifdef IXGBE_IEEE1588 1841 /* This is changing soon to an mtag detection */ 1842 if (we detect this mbuf has a TSTAMP mtag) 1843 cmd_type_len |= IXGBE_ADVTXD_MAC_TSTAMP; 1844 #endif 1845 1846 #ifdef IXGBE_FDIR 1847 /* Do the flow director magic */ 1848 if ((txr->atr_sample) && (!adapter->fdir_reinit)) { 1849 ++txr->atr_count; 1850 if (txr->atr_count >= atr_sample_rate) { 1851 ixgbe_atr(txr, m_head); 1852 txr->atr_count = 0; 1853 } 1854 } 1855 #endif 1856 /* Record payload length */ 1857 if (paylen == 0) 1858 olinfo_status |= m_head->m_pkthdr.len << 1859 IXGBE_ADVTXD_PAYLEN_SHIFT; 1860 1861 i = txr->next_avail_desc; 1862 for (j = 0; j < map->dm_nsegs; j++) { 1863 bus_size_t seglen; 1864 bus_addr_t segaddr; 1865 1866 txbuf = &txr->tx_buffers[i]; 1867 txd = &txr->tx_base[i]; 1868 seglen = map->dm_segs[j].ds_len; 1869 segaddr = htole64(map->dm_segs[j].ds_addr); 1870 1871 txd->read.buffer_addr = segaddr; 1872 txd->read.cmd_type_len = htole32(txr->txd_cmd | 1873 cmd_type_len |seglen); 1874 txd->read.olinfo_status = htole32(olinfo_status); 1875 last = i; /* descriptor that will get completion IRQ */ 1876 1877 if (++i == adapter->num_tx_desc) 1878 i = 0; 1879 1880 txbuf->m_head = NULL; 1881 txbuf->eop_index = -1; 1882 } 1883 1884 txd->read.cmd_type_len |= 1885 htole32(IXGBE_TXD_CMD_EOP | IXGBE_TXD_CMD_RS); 1886 txr->tx_avail -= map->dm_nsegs; 1887 txr->next_avail_desc = i; 1888 1889 txbuf->m_head = m_head; 1890 /* We exchange the maps instead of copying because otherwise 1891 * we end up with many pointers to the same map and we free 1892 * one map twice in ixgbe_free_transmit_structures(). Who 1893 * knows what other problems this caused. --dyoung 1894 */ 1895 txr->tx_buffers[first].map = txbuf->map; 1896 txbuf->map = map; 1897 bus_dmamap_sync(txr->txtag->dt_dmat, map, 0, m_head->m_pkthdr.len, 1898 BUS_DMASYNC_PREWRITE); 1899 1900 /* Set the index of the descriptor that will be marked done */ 1901 txbuf = &txr->tx_buffers[first]; 1902 txbuf->eop_index = last; 1903 1904 ixgbe_dmamap_sync(txr->txdma.dma_tag, txr->txdma.dma_map, 1905 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); 1906 /* 1907 * Advance the Transmit Descriptor Tail (Tdt), this tells the 1908 * hardware that this frame is available to transmit. 1909 */ 1910 ++txr->total_packets.ev_count; 1911 IXGBE_WRITE_REG(&adapter->hw, IXGBE_TDT(txr->me), i); 1912 1913 return 0; 1914 } 1915 1916 static void 1917 ixgbe_set_promisc(struct adapter *adapter) 1918 { 1919 u_int32_t reg_rctl; 1920 struct ifnet *ifp = adapter->ifp; 1921 1922 reg_rctl = IXGBE_READ_REG(&adapter->hw, IXGBE_FCTRL); 1923 reg_rctl &= (~IXGBE_FCTRL_UPE); 1924 reg_rctl &= (~IXGBE_FCTRL_MPE); 1925 IXGBE_WRITE_REG(&adapter->hw, IXGBE_FCTRL, reg_rctl); 1926 1927 if (ifp->if_flags & IFF_PROMISC) { 1928 reg_rctl |= (IXGBE_FCTRL_UPE | IXGBE_FCTRL_MPE); 1929 IXGBE_WRITE_REG(&adapter->hw, IXGBE_FCTRL, reg_rctl); 1930 } else if (ifp->if_flags & IFF_ALLMULTI) { 1931 reg_rctl |= IXGBE_FCTRL_MPE; 1932 reg_rctl &= ~IXGBE_FCTRL_UPE; 1933 IXGBE_WRITE_REG(&adapter->hw, IXGBE_FCTRL, reg_rctl); 1934 } 1935 return; 1936 } 1937 1938 1939 /********************************************************************* 1940 * Multicast Update 1941 * 1942 * This routine is called whenever multicast address list is updated. 1943 * 1944 **********************************************************************/ 1945 #define IXGBE_RAR_ENTRIES 16 1946 1947 static void 1948 ixgbe_set_multi(struct adapter *adapter) 1949 { 1950 struct ether_multi *enm; 1951 struct ether_multistep step; 1952 u32 fctrl; 1953 u8 *mta; 1954 u8 *update_ptr; 1955 int mcnt = 0; 1956 struct ethercom *ec = &adapter->osdep.ec; 1957 struct ifnet *ifp = adapter->ifp; 1958 1959 IOCTL_DEBUGOUT("ixgbe_set_multi: begin"); 1960 1961 mta = adapter->mta; 1962 bzero(mta, sizeof(u8) * IXGBE_ETH_LENGTH_OF_ADDRESS * 1963 MAX_NUM_MULTICAST_ADDRESSES); 1964 1965 fctrl = IXGBE_READ_REG(&adapter->hw, IXGBE_FCTRL); 1966 fctrl |= (IXGBE_FCTRL_UPE | IXGBE_FCTRL_MPE); 1967 if (ifp->if_flags & IFF_PROMISC) 1968 fctrl |= (IXGBE_FCTRL_UPE | IXGBE_FCTRL_MPE); 1969 else if (ifp->if_flags & IFF_ALLMULTI) { 1970 fctrl |= IXGBE_FCTRL_MPE; 1971 fctrl &= ~IXGBE_FCTRL_UPE; 1972 } else 1973 fctrl &= ~(IXGBE_FCTRL_UPE | IXGBE_FCTRL_MPE); 1974 1975 IXGBE_WRITE_REG(&adapter->hw, IXGBE_FCTRL, fctrl); 1976 1977 ETHER_FIRST_MULTI(step, ec, enm); 1978 while (enm != NULL) { 1979 if (memcmp(enm->enm_addrlo, enm->enm_addrhi, 1980 ETHER_ADDR_LEN) != 0) { 1981 fctrl |= IXGBE_FCTRL_MPE; 1982 IXGBE_WRITE_REG(&adapter->hw, IXGBE_FCTRL, fctrl); 1983 break; 1984 } 1985 bcopy(enm->enm_addrlo, 1986 &mta[mcnt * IXGBE_ETH_LENGTH_OF_ADDRESS], 1987 IXGBE_ETH_LENGTH_OF_ADDRESS); 1988 mcnt++; 1989 ETHER_NEXT_MULTI(step, enm); 1990 } 1991 1992 update_ptr = mta; 1993 ixgbe_update_mc_addr_list(&adapter->hw, 1994 update_ptr, mcnt, ixgbe_mc_array_itr); 1995 1996 return; 1997 } 1998 1999 /* 2000 * This is an iterator function now needed by the multicast 2001 * shared code. It simply feeds the shared code routine the 2002 * addresses in the array of ixgbe_set_multi() one by one. 2003 */ 2004 static u8 * 2005 ixgbe_mc_array_itr(struct ixgbe_hw *hw, u8 **update_ptr, u32 *vmdq) 2006 { 2007 u8 *addr = *update_ptr; 2008 u8 *newptr; 2009 *vmdq = 0; 2010 2011 newptr = addr + IXGBE_ETH_LENGTH_OF_ADDRESS; 2012 *update_ptr = newptr; 2013 return addr; 2014 } 2015 2016 2017 /********************************************************************* 2018 * Timer routine 2019 * 2020 * This routine checks for link status,updates statistics, 2021 * and runs the watchdog check. 2022 * 2023 **********************************************************************/ 2024 2025 static void 2026 ixgbe_local_timer1(void *arg) 2027 { 2028 struct adapter *adapter = arg; 2029 device_t dev = adapter->dev; 2030 struct tx_ring *txr = adapter->tx_rings; 2031 2032 KASSERT(mutex_owned(&adapter->core_mtx)); 2033 2034 /* Check for pluggable optics */ 2035 if (adapter->sfp_probe) 2036 if (!ixgbe_sfp_probe(adapter)) 2037 goto out; /* Nothing to do */ 2038 2039 ixgbe_update_link_status(adapter); 2040 ixgbe_update_stats_counters(adapter); 2041 2042 /* 2043 * If the interface has been paused 2044 * then don't do the watchdog check 2045 */ 2046 if (IXGBE_READ_REG(&adapter->hw, IXGBE_TFCS) & IXGBE_TFCS_TXOFF) 2047 goto out; 2048 2049 /* 2050 ** Check status on the TX queues for a hang 2051 */ 2052 for (int i = 0; i < adapter->num_queues; i++, txr++) 2053 if (txr->queue_status == IXGBE_QUEUE_HUNG) 2054 goto hung; 2055 2056 out: 2057 ixgbe_rearm_queues(adapter, adapter->que_mask); 2058 callout_reset(&adapter->timer, hz, ixgbe_local_timer, adapter); 2059 return; 2060 2061 hung: 2062 device_printf(adapter->dev, "Watchdog timeout -- resetting\n"); 2063 device_printf(dev,"Queue(%d) tdh = %d, hw tdt = %d\n", txr->me, 2064 IXGBE_READ_REG(&adapter->hw, IXGBE_TDH(txr->me)), 2065 IXGBE_READ_REG(&adapter->hw, IXGBE_TDT(txr->me))); 2066 device_printf(dev,"TX(%d) desc avail = %d," 2067 "Next TX to Clean = %d\n", 2068 txr->me, txr->tx_avail, txr->next_to_clean); 2069 adapter->ifp->if_flags &= ~IFF_RUNNING; 2070 adapter->watchdog_events.ev_count++; 2071 ixgbe_init_locked(adapter); 2072 } 2073 2074 static void 2075 ixgbe_local_timer(void *arg) 2076 { 2077 struct adapter *adapter = arg; 2078 2079 IXGBE_CORE_LOCK(adapter); 2080 ixgbe_local_timer1(adapter); 2081 IXGBE_CORE_UNLOCK(adapter); 2082 } 2083 2084 /* 2085 ** Note: this routine updates the OS on the link state 2086 ** the real check of the hardware only happens with 2087 ** a link interrupt. 2088 */ 2089 static void 2090 ixgbe_update_link_status(struct adapter *adapter) 2091 { 2092 struct ifnet *ifp = adapter->ifp; 2093 struct tx_ring *txr = adapter->tx_rings; 2094 device_t dev = adapter->dev; 2095 2096 2097 if (adapter->link_up){ 2098 if (adapter->link_active == FALSE) { 2099 if (bootverbose) 2100 device_printf(dev,"Link is up %d Gbps %s \n", 2101 ((adapter->link_speed == 128)? 10:1), 2102 "Full Duplex"); 2103 adapter->link_active = TRUE; 2104 if_link_state_change(ifp, LINK_STATE_UP); 2105 } 2106 } else { /* Link down */ 2107 if (adapter->link_active == TRUE) { 2108 if (bootverbose) 2109 device_printf(dev,"Link is Down\n"); 2110 if_link_state_change(ifp, LINK_STATE_DOWN); 2111 adapter->link_active = FALSE; 2112 for (int i = 0; i < adapter->num_queues; 2113 i++, txr++) 2114 txr->queue_status = IXGBE_QUEUE_IDLE; 2115 } 2116 } 2117 2118 return; 2119 } 2120 2121 2122 static void 2123 ixgbe_ifstop(struct ifnet *ifp, int disable) 2124 { 2125 struct adapter *adapter = ifp->if_softc; 2126 2127 IXGBE_CORE_LOCK(adapter); 2128 ixgbe_stop(adapter); 2129 IXGBE_CORE_UNLOCK(adapter); 2130 } 2131 2132 /********************************************************************* 2133 * 2134 * This routine disables all traffic on the adapter by issuing a 2135 * global reset on the MAC and deallocates TX/RX buffers. 2136 * 2137 **********************************************************************/ 2138 2139 static void 2140 ixgbe_stop(void *arg) 2141 { 2142 struct ifnet *ifp; 2143 struct adapter *adapter = arg; 2144 struct ixgbe_hw *hw = &adapter->hw; 2145 ifp = adapter->ifp; 2146 2147 KASSERT(mutex_owned(&adapter->core_mtx)); 2148 2149 INIT_DEBUGOUT("ixgbe_stop: begin\n"); 2150 ixgbe_disable_intr(adapter); 2151 2152 /* Tell the stack that the interface is no longer active */ 2153 ifp->if_flags &= ~(IFF_RUNNING | IFF_OACTIVE); 2154 2155 ixgbe_reset_hw(hw); 2156 hw->adapter_stopped = FALSE; 2157 ixgbe_stop_adapter(hw); 2158 /* Turn off the laser */ 2159 if (hw->phy.multispeed_fiber) 2160 ixgbe_disable_tx_laser(hw); 2161 callout_stop(&adapter->timer); 2162 2163 /* reprogram the RAR[0] in case user changed it. */ 2164 ixgbe_set_rar(&adapter->hw, 0, adapter->hw.mac.addr, 0, IXGBE_RAH_AV); 2165 2166 return; 2167 } 2168 2169 2170 /********************************************************************* 2171 * 2172 * Determine hardware revision. 2173 * 2174 **********************************************************************/ 2175 static void 2176 ixgbe_identify_hardware(struct adapter *adapter) 2177 { 2178 pcitag_t tag; 2179 pci_chipset_tag_t pc; 2180 pcireg_t subid, id; 2181 struct ixgbe_hw *hw = &adapter->hw; 2182 2183 pc = adapter->osdep.pc; 2184 tag = adapter->osdep.tag; 2185 2186 id = pci_conf_read(pc, tag, PCI_ID_REG); 2187 subid = pci_conf_read(pc, tag, PCI_SUBSYS_ID_REG); 2188 2189 /* Save off the information about this board */ 2190 hw->vendor_id = PCI_VENDOR(id); 2191 hw->device_id = PCI_PRODUCT(id); 2192 hw->revision_id = 2193 PCI_REVISION(pci_conf_read(pc, tag, PCI_CLASS_REG)); 2194 hw->subsystem_vendor_id = PCI_SUBSYS_VENDOR(subid); 2195 hw->subsystem_device_id = PCI_SUBSYS_ID(subid); 2196 2197 /* We need this here to set the num_segs below */ 2198 ixgbe_set_mac_type(hw); 2199 2200 /* Pick up the 82599 and VF settings */ 2201 if (hw->mac.type != ixgbe_mac_82598EB) { 2202 hw->phy.smart_speed = ixgbe_smart_speed; 2203 adapter->num_segs = IXGBE_82599_SCATTER; 2204 } else 2205 adapter->num_segs = IXGBE_82598_SCATTER; 2206 2207 return; 2208 } 2209 2210 /********************************************************************* 2211 * 2212 * Determine optic type 2213 * 2214 **********************************************************************/ 2215 static void 2216 ixgbe_setup_optics(struct adapter *adapter) 2217 { 2218 struct ixgbe_hw *hw = &adapter->hw; 2219 int layer; 2220 2221 layer = ixgbe_get_supported_physical_layer(hw); 2222 switch (layer) { 2223 case IXGBE_PHYSICAL_LAYER_10GBASE_T: 2224 adapter->optics = IFM_10G_T; 2225 break; 2226 case IXGBE_PHYSICAL_LAYER_1000BASE_T: 2227 adapter->optics = IFM_1000_T; 2228 break; 2229 case IXGBE_PHYSICAL_LAYER_10GBASE_LR: 2230 case IXGBE_PHYSICAL_LAYER_10GBASE_LRM: 2231 adapter->optics = IFM_10G_LR; 2232 break; 2233 case IXGBE_PHYSICAL_LAYER_10GBASE_SR: 2234 adapter->optics = IFM_10G_SR; 2235 break; 2236 case IXGBE_PHYSICAL_LAYER_10GBASE_KX4: 2237 case IXGBE_PHYSICAL_LAYER_10GBASE_CX4: 2238 adapter->optics = IFM_10G_CX4; 2239 break; 2240 case IXGBE_PHYSICAL_LAYER_SFP_PLUS_CU: 2241 adapter->optics = IFM_10G_TWINAX; 2242 break; 2243 case IXGBE_PHYSICAL_LAYER_1000BASE_KX: 2244 case IXGBE_PHYSICAL_LAYER_10GBASE_KR: 2245 case IXGBE_PHYSICAL_LAYER_10GBASE_XAUI: 2246 case IXGBE_PHYSICAL_LAYER_UNKNOWN: 2247 default: 2248 adapter->optics = IFM_ETHER | IFM_AUTO; 2249 break; 2250 } 2251 return; 2252 } 2253 2254 /********************************************************************* 2255 * 2256 * Setup the Legacy or MSI Interrupt handler 2257 * 2258 **********************************************************************/ 2259 static int 2260 ixgbe_allocate_legacy(struct adapter *adapter, const struct pci_attach_args *pa) 2261 { 2262 device_t dev = adapter->dev; 2263 struct ix_queue *que = adapter->queues; 2264 char intrbuf[PCI_INTRSTR_LEN]; 2265 #if 0 2266 int rid = 0; 2267 2268 /* MSI RID at 1 */ 2269 if (adapter->msix == 1) 2270 rid = 1; 2271 #endif 2272 2273 /* We allocate a single interrupt resource */ 2274 if (pci_intr_map(pa, &adapter->osdep.ih) != 0) { 2275 aprint_error_dev(dev, "unable to map interrupt\n"); 2276 return ENXIO; 2277 } else { 2278 aprint_normal_dev(dev, "interrupting at %s\n", 2279 pci_intr_string(adapter->osdep.pc, adapter->osdep.ih, intrbuf, sizeof(intrbuf))); 2280 } 2281 2282 /* 2283 * Try allocating a fast interrupt and the associated deferred 2284 * processing contexts. 2285 */ 2286 que->que_si = softint_establish(SOFTINT_NET, ixgbe_handle_que, que); 2287 2288 /* Tasklets for Link, SFP and Multispeed Fiber */ 2289 adapter->link_si = 2290 softint_establish(SOFTINT_NET, ixgbe_handle_link, adapter); 2291 adapter->mod_si = 2292 softint_establish(SOFTINT_NET, ixgbe_handle_mod, adapter); 2293 adapter->msf_si = 2294 softint_establish(SOFTINT_NET, ixgbe_handle_msf, adapter); 2295 2296 #ifdef IXGBE_FDIR 2297 adapter->fdir_si = 2298 softint_establish(SOFTINT_NET, ixgbe_reinit_fdir, adapter); 2299 #endif 2300 if (que->que_si == NULL || 2301 adapter->link_si == NULL || 2302 adapter->mod_si == NULL || 2303 #ifdef IXGBE_FDIR 2304 adapter->fdir_si == NULL || 2305 #endif 2306 adapter->msf_si == NULL) { 2307 aprint_error_dev(dev, 2308 "could not establish software interrupts\n"); 2309 return ENXIO; 2310 } 2311 2312 adapter->osdep.intr = pci_intr_establish(adapter->osdep.pc, 2313 adapter->osdep.ih, IPL_NET, ixgbe_legacy_irq, que); 2314 if (adapter->osdep.intr == NULL) { 2315 aprint_error_dev(dev, "failed to register interrupt handler\n"); 2316 softint_disestablish(que->que_si); 2317 softint_disestablish(adapter->link_si); 2318 softint_disestablish(adapter->mod_si); 2319 softint_disestablish(adapter->msf_si); 2320 #ifdef IXGBE_FDIR 2321 softint_disestablish(adapter->fdir_si); 2322 #endif 2323 return ENXIO; 2324 } 2325 /* For simplicity in the handlers */ 2326 adapter->que_mask = IXGBE_EIMS_ENABLE_MASK; 2327 2328 return (0); 2329 } 2330 2331 2332 /********************************************************************* 2333 * 2334 * Setup MSIX Interrupt resources and handlers 2335 * 2336 **********************************************************************/ 2337 static int 2338 ixgbe_allocate_msix(struct adapter *adapter, const struct pci_attach_args *pa) 2339 { 2340 #if !defined(NETBSD_MSI_OR_MSIX) 2341 return 0; 2342 #else 2343 device_t dev = adapter->dev; 2344 struct ix_queue *que = adapter->queues; 2345 int error, rid, vector = 0; 2346 2347 for (int i = 0; i < adapter->num_queues; i++, vector++, que++) { 2348 rid = vector + 1; 2349 que->res = bus_alloc_resource_any(dev, SYS_RES_IRQ, &rid, 2350 RF_SHAREABLE | RF_ACTIVE); 2351 if (que->res == NULL) { 2352 aprint_error_dev(dev,"Unable to allocate" 2353 " bus resource: que interrupt [%d]\n", vector); 2354 return (ENXIO); 2355 } 2356 /* Set the handler function */ 2357 error = bus_setup_intr(dev, que->res, 2358 INTR_TYPE_NET | INTR_MPSAFE, NULL, 2359 ixgbe_msix_que, que, &que->tag); 2360 if (error) { 2361 que->res = NULL; 2362 aprint_error_dev(dev, 2363 "Failed to register QUE handler\n"); 2364 return error; 2365 } 2366 #if __FreeBSD_version >= 800504 2367 bus_describe_intr(dev, que->res, que->tag, "que %d", i); 2368 #endif 2369 que->msix = vector; 2370 adapter->que_mask |= (u64)(1 << que->msix); 2371 /* 2372 ** Bind the msix vector, and thus the 2373 ** ring to the corresponding cpu. 2374 */ 2375 if (adapter->num_queues > 1) 2376 bus_bind_intr(dev, que->res, i); 2377 2378 que->que_si = softint_establish(ixgbe_handle_que, que); 2379 if (que->que_si == NULL) { 2380 aprint_error_dev(dev, 2381 "could not establish software interrupt\n"); 2382 } 2383 } 2384 2385 /* and Link */ 2386 rid = vector + 1; 2387 adapter->res = bus_alloc_resource_any(dev, 2388 SYS_RES_IRQ, &rid, RF_SHAREABLE | RF_ACTIVE); 2389 if (!adapter->res) { 2390 aprint_error_dev(dev,"Unable to allocate bus resource: " 2391 "Link interrupt [%d]\n", rid); 2392 return (ENXIO); 2393 } 2394 /* Set the link handler function */ 2395 error = bus_setup_intr(dev, adapter->res, 2396 INTR_TYPE_NET | INTR_MPSAFE, NULL, 2397 ixgbe_msix_link, adapter, &adapter->tag); 2398 if (error) { 2399 adapter->res = NULL; 2400 aprint_error_dev(dev, "Failed to register LINK handler\n"); 2401 return (error); 2402 } 2403 #if __FreeBSD_version >= 800504 2404 bus_describe_intr(dev, adapter->res, adapter->tag, "link"); 2405 #endif 2406 adapter->linkvec = vector; 2407 /* Tasklets for Link, SFP and Multispeed Fiber */ 2408 adapter->link_si = 2409 softint_establish(SOFTINT_NET, ixgbe_handle_link, adapter); 2410 adapter->mod_si = 2411 softint_establish(SOFTINT_NET, ixgbe_handle_mod, adapter); 2412 adapter->msf_si = 2413 softint_establish(SOFTINT_NET, ixgbe_handle_msf, adapter); 2414 #ifdef IXGBE_FDIR 2415 adapter->fdir_si = 2416 softint_establish(SOFTINT_NET, ixgbe_reinit_fdir, adapter); 2417 #endif 2418 2419 return (0); 2420 #endif 2421 } 2422 2423 /* 2424 * Setup Either MSI/X or MSI 2425 */ 2426 static int 2427 ixgbe_setup_msix(struct adapter *adapter) 2428 { 2429 #if !defined(NETBSD_MSI_OR_MSIX) 2430 return 0; 2431 #else 2432 device_t dev = adapter->dev; 2433 int rid, want, queues, msgs; 2434 2435 /* Override by tuneable */ 2436 if (ixgbe_enable_msix == 0) 2437 goto msi; 2438 2439 /* First try MSI/X */ 2440 rid = PCI_BAR(MSIX_82598_BAR); 2441 adapter->msix_mem = bus_alloc_resource_any(dev, 2442 SYS_RES_MEMORY, &rid, RF_ACTIVE); 2443 if (!adapter->msix_mem) { 2444 rid += 4; /* 82599 maps in higher BAR */ 2445 adapter->msix_mem = bus_alloc_resource_any(dev, 2446 SYS_RES_MEMORY, &rid, RF_ACTIVE); 2447 } 2448 if (!adapter->msix_mem) { 2449 /* May not be enabled */ 2450 device_printf(adapter->dev, 2451 "Unable to map MSIX table \n"); 2452 goto msi; 2453 } 2454 2455 msgs = pci_msix_count(dev); 2456 if (msgs == 0) { /* system has msix disabled */ 2457 bus_release_resource(dev, SYS_RES_MEMORY, 2458 rid, adapter->msix_mem); 2459 adapter->msix_mem = NULL; 2460 goto msi; 2461 } 2462 2463 /* Figure out a reasonable auto config value */ 2464 queues = (mp_ncpus > (msgs-1)) ? (msgs-1) : mp_ncpus; 2465 2466 if (ixgbe_num_queues != 0) 2467 queues = ixgbe_num_queues; 2468 /* Set max queues to 8 when autoconfiguring */ 2469 else if ((ixgbe_num_queues == 0) && (queues > 8)) 2470 queues = 8; 2471 2472 /* 2473 ** Want one vector (RX/TX pair) per queue 2474 ** plus an additional for Link. 2475 */ 2476 want = queues + 1; 2477 if (msgs >= want) 2478 msgs = want; 2479 else { 2480 device_printf(adapter->dev, 2481 "MSIX Configuration Problem, " 2482 "%d vectors but %d queues wanted!\n", 2483 msgs, want); 2484 return (0); /* Will go to Legacy setup */ 2485 } 2486 if ((msgs) && pci_alloc_msix(dev, &msgs) == 0) { 2487 device_printf(adapter->dev, 2488 "Using MSIX interrupts with %d vectors\n", msgs); 2489 adapter->num_queues = queues; 2490 return (msgs); 2491 } 2492 msi: 2493 msgs = pci_msi_count(dev); 2494 if (msgs == 1 && pci_alloc_msi(dev, &msgs) == 0) 2495 device_printf(adapter->dev,"Using MSI interrupt\n"); 2496 return (msgs); 2497 #endif 2498 } 2499 2500 2501 static int 2502 ixgbe_allocate_pci_resources(struct adapter *adapter, const struct pci_attach_args *pa) 2503 { 2504 pcireg_t memtype; 2505 device_t dev = adapter->dev; 2506 bus_addr_t addr; 2507 int flags; 2508 2509 memtype = pci_mapreg_type(pa->pa_pc, pa->pa_tag, PCI_BAR(0)); 2510 switch (memtype) { 2511 case PCI_MAPREG_TYPE_MEM | PCI_MAPREG_MEM_TYPE_32BIT: 2512 case PCI_MAPREG_TYPE_MEM | PCI_MAPREG_MEM_TYPE_64BIT: 2513 adapter->osdep.mem_bus_space_tag = pa->pa_memt; 2514 if (pci_mapreg_info(pa->pa_pc, pa->pa_tag, PCI_BAR(0), 2515 memtype, &addr, &adapter->osdep.mem_size, &flags) != 0) 2516 goto map_err; 2517 if ((flags & BUS_SPACE_MAP_PREFETCHABLE) != 0) { 2518 aprint_normal_dev(dev, "clearing prefetchable bit\n"); 2519 flags &= ~BUS_SPACE_MAP_PREFETCHABLE; 2520 } 2521 if (bus_space_map(adapter->osdep.mem_bus_space_tag, addr, 2522 adapter->osdep.mem_size, flags, 2523 &adapter->osdep.mem_bus_space_handle) != 0) { 2524 map_err: 2525 adapter->osdep.mem_size = 0; 2526 aprint_error_dev(dev, "unable to map BAR0\n"); 2527 return ENXIO; 2528 } 2529 break; 2530 default: 2531 aprint_error_dev(dev, "unexpected type on BAR0\n"); 2532 return ENXIO; 2533 } 2534 2535 /* Legacy defaults */ 2536 adapter->num_queues = 1; 2537 adapter->hw.back = &adapter->osdep; 2538 2539 /* 2540 ** Now setup MSI or MSI/X, should 2541 ** return us the number of supported 2542 ** vectors. (Will be 1 for MSI) 2543 */ 2544 adapter->msix = ixgbe_setup_msix(adapter); 2545 return (0); 2546 } 2547 2548 static void 2549 ixgbe_free_pci_resources(struct adapter * adapter) 2550 { 2551 #if defined(NETBSD_MSI_OR_MSIX) 2552 struct ix_queue *que = adapter->queues; 2553 #endif 2554 device_t dev = adapter->dev; 2555 int rid; 2556 2557 #if defined(NETBSD_MSI_OR_MSIX) 2558 int memrid; 2559 if (adapter->hw.mac.type == ixgbe_mac_82598EB) 2560 memrid = PCI_BAR(MSIX_82598_BAR); 2561 else 2562 memrid = PCI_BAR(MSIX_82599_BAR); 2563 2564 /* 2565 ** There is a slight possibility of a failure mode 2566 ** in attach that will result in entering this function 2567 ** before interrupt resources have been initialized, and 2568 ** in that case we do not want to execute the loops below 2569 ** We can detect this reliably by the state of the adapter 2570 ** res pointer. 2571 */ 2572 if (adapter->res == NULL) 2573 goto mem; 2574 2575 /* 2576 ** Release all msix queue resources: 2577 */ 2578 for (int i = 0; i < adapter->num_queues; i++, que++) { 2579 rid = que->msix + 1; 2580 if (que->tag != NULL) { 2581 bus_teardown_intr(dev, que->res, que->tag); 2582 que->tag = NULL; 2583 } 2584 if (que->res != NULL) 2585 bus_release_resource(dev, SYS_RES_IRQ, rid, que->res); 2586 } 2587 #endif 2588 2589 /* Clean the Legacy or Link interrupt last */ 2590 if (adapter->linkvec) /* we are doing MSIX */ 2591 rid = adapter->linkvec + 1; 2592 else 2593 (adapter->msix != 0) ? (rid = 1):(rid = 0); 2594 2595 printf("%s: disestablishing interrupt handler\n", device_xname(dev)); 2596 pci_intr_disestablish(adapter->osdep.pc, adapter->osdep.intr); 2597 adapter->osdep.intr = NULL; 2598 2599 #if defined(NETBSD_MSI_OR_MSIX) 2600 mem: 2601 if (adapter->msix) 2602 pci_release_msi(dev); 2603 2604 if (adapter->msix_mem != NULL) 2605 bus_release_resource(dev, SYS_RES_MEMORY, 2606 memrid, adapter->msix_mem); 2607 #endif 2608 2609 if (adapter->osdep.mem_size != 0) { 2610 bus_space_unmap(adapter->osdep.mem_bus_space_tag, 2611 adapter->osdep.mem_bus_space_handle, 2612 adapter->osdep.mem_size); 2613 } 2614 2615 return; 2616 } 2617 2618 /********************************************************************* 2619 * 2620 * Setup networking device structure and register an interface. 2621 * 2622 **********************************************************************/ 2623 static int 2624 ixgbe_setup_interface(device_t dev, struct adapter *adapter) 2625 { 2626 struct ethercom *ec = &adapter->osdep.ec; 2627 struct ixgbe_hw *hw = &adapter->hw; 2628 struct ifnet *ifp; 2629 2630 INIT_DEBUGOUT("ixgbe_setup_interface: begin"); 2631 2632 ifp = adapter->ifp = &ec->ec_if; 2633 strlcpy(ifp->if_xname, device_xname(dev), IFNAMSIZ); 2634 ifp->if_mtu = ETHERMTU; 2635 ifp->if_baudrate = 1000000000; 2636 ifp->if_init = ixgbe_init; 2637 ifp->if_stop = ixgbe_ifstop; 2638 ifp->if_softc = adapter; 2639 ifp->if_flags = IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST; 2640 ifp->if_ioctl = ixgbe_ioctl; 2641 ifp->if_start = ixgbe_start; 2642 #if __FreeBSD_version >= 800000 2643 ifp->if_transmit = ixgbe_mq_start; 2644 ifp->if_qflush = ixgbe_qflush; 2645 #endif 2646 ifp->if_snd.ifq_maxlen = adapter->num_tx_desc - 2; 2647 2648 if_attach(ifp); 2649 ether_ifattach(ifp, adapter->hw.mac.addr); 2650 ether_set_ifflags_cb(ec, ixgbe_ifflags_cb); 2651 2652 adapter->max_frame_size = 2653 ifp->if_mtu + ETHER_HDR_LEN + ETHER_CRC_LEN; 2654 2655 /* 2656 * Tell the upper layer(s) we support long frames. 2657 */ 2658 ifp->if_hdrlen = sizeof(struct ether_vlan_header); 2659 2660 ifp->if_capabilities |= IFCAP_HWCSUM | IFCAP_TSOv4; 2661 ifp->if_capenable = 0; 2662 2663 ec->ec_capabilities |= ETHERCAP_VLAN_HWCSUM; 2664 ec->ec_capabilities |= ETHERCAP_VLAN_HWTAGGING | ETHERCAP_VLAN_MTU; 2665 ec->ec_capabilities |= ETHERCAP_JUMBO_MTU; 2666 ec->ec_capenable = ec->ec_capabilities; 2667 2668 /* Don't enable LRO by default */ 2669 ifp->if_capabilities |= IFCAP_LRO; 2670 2671 /* 2672 ** Dont turn this on by default, if vlans are 2673 ** created on another pseudo device (eg. lagg) 2674 ** then vlan events are not passed thru, breaking 2675 ** operation, but with HW FILTER off it works. If 2676 ** using vlans directly on the em driver you can 2677 ** enable this and get full hardware tag filtering. 2678 */ 2679 ec->ec_capabilities |= ETHERCAP_VLAN_HWFILTER; 2680 2681 /* 2682 * Specify the media types supported by this adapter and register 2683 * callbacks to update media and link information 2684 */ 2685 ifmedia_init(&adapter->media, IFM_IMASK, ixgbe_media_change, 2686 ixgbe_media_status); 2687 ifmedia_add(&adapter->media, IFM_ETHER | adapter->optics, 0, NULL); 2688 ifmedia_set(&adapter->media, IFM_ETHER | adapter->optics); 2689 if (hw->device_id == IXGBE_DEV_ID_82598AT) { 2690 ifmedia_add(&adapter->media, 2691 IFM_ETHER | IFM_1000_T | IFM_FDX, 0, NULL); 2692 ifmedia_add(&adapter->media, 2693 IFM_ETHER | IFM_1000_T, 0, NULL); 2694 } 2695 ifmedia_add(&adapter->media, IFM_ETHER | IFM_AUTO, 0, NULL); 2696 ifmedia_set(&adapter->media, IFM_ETHER | IFM_AUTO); 2697 2698 return (0); 2699 } 2700 2701 static void 2702 ixgbe_config_link(struct adapter *adapter) 2703 { 2704 struct ixgbe_hw *hw = &adapter->hw; 2705 u32 autoneg, err = 0; 2706 bool sfp, negotiate; 2707 2708 sfp = ixgbe_is_sfp(hw); 2709 2710 if (sfp) { 2711 if (hw->phy.multispeed_fiber) { 2712 hw->mac.ops.setup_sfp(hw); 2713 ixgbe_enable_tx_laser(hw); 2714 softint_schedule(adapter->msf_si); 2715 } else { 2716 softint_schedule(adapter->mod_si); 2717 } 2718 } else { 2719 if (hw->mac.ops.check_link) 2720 err = ixgbe_check_link(hw, &autoneg, 2721 &adapter->link_up, FALSE); 2722 if (err) 2723 goto out; 2724 autoneg = hw->phy.autoneg_advertised; 2725 if ((!autoneg) && (hw->mac.ops.get_link_capabilities)) 2726 err = hw->mac.ops.get_link_capabilities(hw, 2727 &autoneg, &negotiate); 2728 if (err) 2729 goto out; 2730 if (hw->mac.ops.setup_link) 2731 err = hw->mac.ops.setup_link(hw, autoneg, 2732 negotiate, adapter->link_up); 2733 } 2734 out: 2735 return; 2736 } 2737 2738 /******************************************************************** 2739 * Manage DMA'able memory. 2740 *******************************************************************/ 2741 2742 static int 2743 ixgbe_dma_malloc(struct adapter *adapter, const bus_size_t size, 2744 struct ixgbe_dma_alloc *dma, const int mapflags) 2745 { 2746 device_t dev = adapter->dev; 2747 int r, rsegs; 2748 2749 r = ixgbe_dma_tag_create(adapter->osdep.dmat, /* parent */ 2750 DBA_ALIGN, 0, /* alignment, bounds */ 2751 size, /* maxsize */ 2752 1, /* nsegments */ 2753 size, /* maxsegsize */ 2754 BUS_DMA_ALLOCNOW, /* flags */ 2755 &dma->dma_tag); 2756 if (r != 0) { 2757 aprint_error_dev(dev, 2758 "%s: ixgbe_dma_tag_create failed; error %d\n", __func__, r); 2759 goto fail_0; 2760 } 2761 2762 r = bus_dmamem_alloc(dma->dma_tag->dt_dmat, 2763 size, 2764 dma->dma_tag->dt_alignment, 2765 dma->dma_tag->dt_boundary, 2766 &dma->dma_seg, 1, &rsegs, BUS_DMA_NOWAIT); 2767 if (r != 0) { 2768 aprint_error_dev(dev, 2769 "%s: bus_dmamem_alloc failed; error %d\n", __func__, r); 2770 goto fail_1; 2771 } 2772 2773 r = bus_dmamem_map(dma->dma_tag->dt_dmat, &dma->dma_seg, rsegs, 2774 size, &dma->dma_vaddr, BUS_DMA_NOWAIT); 2775 if (r != 0) { 2776 aprint_error_dev(dev, "%s: bus_dmamem_map failed; error %d\n", 2777 __func__, r); 2778 goto fail_2; 2779 } 2780 2781 r = ixgbe_dmamap_create(dma->dma_tag, 0, &dma->dma_map); 2782 if (r != 0) { 2783 aprint_error_dev(dev, "%s: bus_dmamem_map failed; error %d\n", 2784 __func__, r); 2785 goto fail_3; 2786 } 2787 2788 r = bus_dmamap_load(dma->dma_tag->dt_dmat, dma->dma_map, dma->dma_vaddr, 2789 size, 2790 NULL, 2791 mapflags | BUS_DMA_NOWAIT); 2792 if (r != 0) { 2793 aprint_error_dev(dev, "%s: bus_dmamap_load failed; error %d\n", 2794 __func__, r); 2795 goto fail_4; 2796 } 2797 dma->dma_paddr = dma->dma_map->dm_segs[0].ds_addr; 2798 dma->dma_size = size; 2799 return 0; 2800 fail_4: 2801 ixgbe_dmamap_destroy(dma->dma_tag, dma->dma_map); 2802 fail_3: 2803 bus_dmamem_unmap(dma->dma_tag->dt_dmat, dma->dma_vaddr, size); 2804 fail_2: 2805 bus_dmamem_free(dma->dma_tag->dt_dmat, &dma->dma_seg, rsegs); 2806 fail_1: 2807 ixgbe_dma_tag_destroy(dma->dma_tag); 2808 fail_0: 2809 return r; 2810 } 2811 2812 static void 2813 ixgbe_dma_free(struct adapter *adapter, struct ixgbe_dma_alloc *dma) 2814 { 2815 bus_dmamap_sync(dma->dma_tag->dt_dmat, dma->dma_map, 0, dma->dma_size, 2816 BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE); 2817 ixgbe_dmamap_unload(dma->dma_tag, dma->dma_map); 2818 bus_dmamem_free(dma->dma_tag->dt_dmat, &dma->dma_seg, 1); 2819 ixgbe_dma_tag_destroy(dma->dma_tag); 2820 } 2821 2822 2823 /********************************************************************* 2824 * 2825 * Allocate memory for the transmit and receive rings, and then 2826 * the descriptors associated with each, called only once at attach. 2827 * 2828 **********************************************************************/ 2829 static int 2830 ixgbe_allocate_queues(struct adapter *adapter) 2831 { 2832 device_t dev = adapter->dev; 2833 struct ix_queue *que; 2834 struct tx_ring *txr; 2835 struct rx_ring *rxr; 2836 int rsize, tsize, error = IXGBE_SUCCESS; 2837 int txconf = 0, rxconf = 0; 2838 2839 /* First allocate the top level queue structs */ 2840 if (!(adapter->queues = 2841 (struct ix_queue *) malloc(sizeof(struct ix_queue) * 2842 adapter->num_queues, M_DEVBUF, M_NOWAIT | M_ZERO))) { 2843 aprint_error_dev(dev, "Unable to allocate queue memory\n"); 2844 error = ENOMEM; 2845 goto fail; 2846 } 2847 2848 /* First allocate the TX ring struct memory */ 2849 if (!(adapter->tx_rings = 2850 (struct tx_ring *) malloc(sizeof(struct tx_ring) * 2851 adapter->num_queues, M_DEVBUF, M_NOWAIT | M_ZERO))) { 2852 aprint_error_dev(dev, "Unable to allocate TX ring memory\n"); 2853 error = ENOMEM; 2854 goto tx_fail; 2855 } 2856 2857 /* Next allocate the RX */ 2858 if (!(adapter->rx_rings = 2859 (struct rx_ring *) malloc(sizeof(struct rx_ring) * 2860 adapter->num_queues, M_DEVBUF, M_NOWAIT | M_ZERO))) { 2861 aprint_error_dev(dev, "Unable to allocate RX ring memory\n"); 2862 error = ENOMEM; 2863 goto rx_fail; 2864 } 2865 2866 /* For the ring itself */ 2867 tsize = roundup2(adapter->num_tx_desc * 2868 sizeof(union ixgbe_adv_tx_desc), DBA_ALIGN); 2869 2870 /* 2871 * Now set up the TX queues, txconf is needed to handle the 2872 * possibility that things fail midcourse and we need to 2873 * undo memory gracefully 2874 */ 2875 for (int i = 0; i < adapter->num_queues; i++, txconf++) { 2876 /* Set up some basics */ 2877 txr = &adapter->tx_rings[i]; 2878 txr->adapter = adapter; 2879 txr->me = i; 2880 2881 /* Initialize the TX side lock */ 2882 snprintf(txr->mtx_name, sizeof(txr->mtx_name), "%s:tx(%d)", 2883 device_xname(dev), txr->me); 2884 mutex_init(&txr->tx_mtx, MUTEX_DEFAULT, IPL_NET); 2885 2886 if (ixgbe_dma_malloc(adapter, tsize, 2887 &txr->txdma, BUS_DMA_NOWAIT)) { 2888 aprint_error_dev(dev, 2889 "Unable to allocate TX Descriptor memory\n"); 2890 error = ENOMEM; 2891 goto err_tx_desc; 2892 } 2893 txr->tx_base = (union ixgbe_adv_tx_desc *)txr->txdma.dma_vaddr; 2894 bzero((void *)txr->tx_base, tsize); 2895 2896 /* Now allocate transmit buffers for the ring */ 2897 if (ixgbe_allocate_transmit_buffers(txr)) { 2898 aprint_error_dev(dev, 2899 "Critical Failure setting up transmit buffers\n"); 2900 error = ENOMEM; 2901 goto err_tx_desc; 2902 } 2903 #if __FreeBSD_version >= 800000 2904 /* Allocate a buf ring */ 2905 txr->br = buf_ring_alloc(IXGBE_BR_SIZE, M_DEVBUF, 2906 M_WAITOK, &txr->tx_mtx); 2907 if (txr->br == NULL) { 2908 aprint_error_dev(dev, 2909 "Critical Failure setting up buf ring\n"); 2910 error = ENOMEM; 2911 goto err_tx_desc; 2912 } 2913 #endif 2914 } 2915 2916 /* 2917 * Next the RX queues... 2918 */ 2919 rsize = roundup2(adapter->num_rx_desc * 2920 sizeof(union ixgbe_adv_rx_desc), DBA_ALIGN); 2921 for (int i = 0; i < adapter->num_queues; i++, rxconf++) { 2922 rxr = &adapter->rx_rings[i]; 2923 /* Set up some basics */ 2924 rxr->adapter = adapter; 2925 rxr->me = i; 2926 2927 /* Initialize the RX side lock */ 2928 snprintf(rxr->mtx_name, sizeof(rxr->mtx_name), "%s:rx(%d)", 2929 device_xname(dev), rxr->me); 2930 mutex_init(&rxr->rx_mtx, MUTEX_DEFAULT, IPL_NET); 2931 2932 if (ixgbe_dma_malloc(adapter, rsize, 2933 &rxr->rxdma, BUS_DMA_NOWAIT)) { 2934 aprint_error_dev(dev, 2935 "Unable to allocate RxDescriptor memory\n"); 2936 error = ENOMEM; 2937 goto err_rx_desc; 2938 } 2939 rxr->rx_base = (union ixgbe_adv_rx_desc *)rxr->rxdma.dma_vaddr; 2940 bzero((void *)rxr->rx_base, rsize); 2941 2942 /* Allocate receive buffers for the ring*/ 2943 if (ixgbe_allocate_receive_buffers(rxr)) { 2944 aprint_error_dev(dev, 2945 "Critical Failure setting up receive buffers\n"); 2946 error = ENOMEM; 2947 goto err_rx_desc; 2948 } 2949 } 2950 2951 /* 2952 ** Finally set up the queue holding structs 2953 */ 2954 for (int i = 0; i < adapter->num_queues; i++) { 2955 que = &adapter->queues[i]; 2956 que->adapter = adapter; 2957 que->txr = &adapter->tx_rings[i]; 2958 que->rxr = &adapter->rx_rings[i]; 2959 } 2960 2961 return (0); 2962 2963 err_rx_desc: 2964 for (rxr = adapter->rx_rings; rxconf > 0; rxr++, rxconf--) 2965 ixgbe_dma_free(adapter, &rxr->rxdma); 2966 err_tx_desc: 2967 for (txr = adapter->tx_rings; txconf > 0; txr++, txconf--) 2968 ixgbe_dma_free(adapter, &txr->txdma); 2969 free(adapter->rx_rings, M_DEVBUF); 2970 rx_fail: 2971 free(adapter->tx_rings, M_DEVBUF); 2972 tx_fail: 2973 free(adapter->queues, M_DEVBUF); 2974 fail: 2975 return (error); 2976 } 2977 2978 /********************************************************************* 2979 * 2980 * Allocate memory for tx_buffer structures. The tx_buffer stores all 2981 * the information needed to transmit a packet on the wire. This is 2982 * called only once at attach, setup is done every reset. 2983 * 2984 **********************************************************************/ 2985 static int 2986 ixgbe_allocate_transmit_buffers(struct tx_ring *txr) 2987 { 2988 struct adapter *adapter = txr->adapter; 2989 device_t dev = adapter->dev; 2990 struct ixgbe_tx_buf *txbuf; 2991 int error, i; 2992 2993 /* 2994 * Setup DMA descriptor areas. 2995 */ 2996 if ((error = ixgbe_dma_tag_create(adapter->osdep.dmat, /* parent */ 2997 1, 0, /* alignment, bounds */ 2998 IXGBE_TSO_SIZE, /* maxsize */ 2999 adapter->num_segs, /* nsegments */ 3000 PAGE_SIZE, /* maxsegsize */ 3001 0, /* flags */ 3002 &txr->txtag))) { 3003 aprint_error_dev(dev,"Unable to allocate TX DMA tag\n"); 3004 goto fail; 3005 } 3006 3007 if (!(txr->tx_buffers = 3008 (struct ixgbe_tx_buf *) malloc(sizeof(struct ixgbe_tx_buf) * 3009 adapter->num_tx_desc, M_DEVBUF, M_NOWAIT | M_ZERO))) { 3010 aprint_error_dev(dev, "Unable to allocate tx_buffer memory\n"); 3011 error = ENOMEM; 3012 goto fail; 3013 } 3014 3015 /* Create the descriptor buffer dma maps */ 3016 txbuf = txr->tx_buffers; 3017 for (i = 0; i < adapter->num_tx_desc; i++, txbuf++) { 3018 error = ixgbe_dmamap_create(txr->txtag, 0, &txbuf->map); 3019 if (error != 0) { 3020 aprint_error_dev(dev, "Unable to create TX DMA map\n"); 3021 goto fail; 3022 } 3023 } 3024 3025 return 0; 3026 fail: 3027 /* We free all, it handles case where we are in the middle */ 3028 ixgbe_free_transmit_structures(adapter); 3029 return (error); 3030 } 3031 3032 /********************************************************************* 3033 * 3034 * Initialize a transmit ring. 3035 * 3036 **********************************************************************/ 3037 static void 3038 ixgbe_setup_transmit_ring(struct tx_ring *txr) 3039 { 3040 struct adapter *adapter = txr->adapter; 3041 struct ixgbe_tx_buf *txbuf; 3042 int i; 3043 3044 /* Clear the old ring contents */ 3045 IXGBE_TX_LOCK(txr); 3046 bzero((void *)txr->tx_base, 3047 (sizeof(union ixgbe_adv_tx_desc)) * adapter->num_tx_desc); 3048 /* Reset indices */ 3049 txr->next_avail_desc = 0; 3050 txr->next_to_clean = 0; 3051 3052 /* Free any existing tx buffers. */ 3053 txbuf = txr->tx_buffers; 3054 for (i = 0; i < adapter->num_tx_desc; i++, txbuf++) { 3055 if (txbuf->m_head != NULL) { 3056 bus_dmamap_sync(txr->txtag->dt_dmat, txbuf->map, 3057 0, txbuf->m_head->m_pkthdr.len, 3058 BUS_DMASYNC_POSTWRITE); 3059 ixgbe_dmamap_unload(txr->txtag, txbuf->map); 3060 m_freem(txbuf->m_head); 3061 txbuf->m_head = NULL; 3062 } 3063 /* Clear the EOP index */ 3064 txbuf->eop_index = -1; 3065 } 3066 3067 #ifdef IXGBE_FDIR 3068 /* Set the rate at which we sample packets */ 3069 if (adapter->hw.mac.type != ixgbe_mac_82598EB) 3070 txr->atr_sample = atr_sample_rate; 3071 #endif 3072 3073 /* Set number of descriptors available */ 3074 txr->tx_avail = adapter->num_tx_desc; 3075 3076 ixgbe_dmamap_sync(txr->txdma.dma_tag, txr->txdma.dma_map, 3077 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); 3078 IXGBE_TX_UNLOCK(txr); 3079 } 3080 3081 /********************************************************************* 3082 * 3083 * Initialize all transmit rings. 3084 * 3085 **********************************************************************/ 3086 static int 3087 ixgbe_setup_transmit_structures(struct adapter *adapter) 3088 { 3089 struct tx_ring *txr = adapter->tx_rings; 3090 3091 for (int i = 0; i < adapter->num_queues; i++, txr++) 3092 ixgbe_setup_transmit_ring(txr); 3093 3094 return (0); 3095 } 3096 3097 /********************************************************************* 3098 * 3099 * Enable transmit unit. 3100 * 3101 **********************************************************************/ 3102 static void 3103 ixgbe_initialize_transmit_units(struct adapter *adapter) 3104 { 3105 struct tx_ring *txr = adapter->tx_rings; 3106 struct ixgbe_hw *hw = &adapter->hw; 3107 3108 /* Setup the Base and Length of the Tx Descriptor Ring */ 3109 3110 for (int i = 0; i < adapter->num_queues; i++, txr++) { 3111 u64 tdba = txr->txdma.dma_paddr; 3112 u32 txctrl; 3113 3114 IXGBE_WRITE_REG(hw, IXGBE_TDBAL(i), 3115 (tdba & 0x00000000ffffffffULL)); 3116 IXGBE_WRITE_REG(hw, IXGBE_TDBAH(i), (tdba >> 32)); 3117 IXGBE_WRITE_REG(hw, IXGBE_TDLEN(i), 3118 adapter->num_tx_desc * sizeof(struct ixgbe_legacy_tx_desc)); 3119 3120 /* Setup the HW Tx Head and Tail descriptor pointers */ 3121 IXGBE_WRITE_REG(hw, IXGBE_TDH(i), 0); 3122 IXGBE_WRITE_REG(hw, IXGBE_TDT(i), 0); 3123 3124 /* Setup Transmit Descriptor Cmd Settings */ 3125 txr->txd_cmd = IXGBE_TXD_CMD_IFCS; 3126 txr->queue_status = IXGBE_QUEUE_IDLE; 3127 3128 /* Disable Head Writeback */ 3129 switch (hw->mac.type) { 3130 case ixgbe_mac_82598EB: 3131 txctrl = IXGBE_READ_REG(hw, IXGBE_DCA_TXCTRL(i)); 3132 break; 3133 case ixgbe_mac_82599EB: 3134 default: 3135 txctrl = IXGBE_READ_REG(hw, IXGBE_DCA_TXCTRL_82599(i)); 3136 break; 3137 } 3138 txctrl &= ~IXGBE_DCA_TXCTRL_TX_WB_RO_EN; 3139 switch (hw->mac.type) { 3140 case ixgbe_mac_82598EB: 3141 IXGBE_WRITE_REG(hw, IXGBE_DCA_TXCTRL(i), txctrl); 3142 break; 3143 case ixgbe_mac_82599EB: 3144 default: 3145 IXGBE_WRITE_REG(hw, IXGBE_DCA_TXCTRL_82599(i), txctrl); 3146 break; 3147 } 3148 3149 } 3150 3151 if (hw->mac.type != ixgbe_mac_82598EB) { 3152 u32 dmatxctl, rttdcs; 3153 dmatxctl = IXGBE_READ_REG(hw, IXGBE_DMATXCTL); 3154 dmatxctl |= IXGBE_DMATXCTL_TE; 3155 IXGBE_WRITE_REG(hw, IXGBE_DMATXCTL, dmatxctl); 3156 /* Disable arbiter to set MTQC */ 3157 rttdcs = IXGBE_READ_REG(hw, IXGBE_RTTDCS); 3158 rttdcs |= IXGBE_RTTDCS_ARBDIS; 3159 IXGBE_WRITE_REG(hw, IXGBE_RTTDCS, rttdcs); 3160 IXGBE_WRITE_REG(hw, IXGBE_MTQC, IXGBE_MTQC_64Q_1PB); 3161 rttdcs &= ~IXGBE_RTTDCS_ARBDIS; 3162 IXGBE_WRITE_REG(hw, IXGBE_RTTDCS, rttdcs); 3163 } 3164 3165 return; 3166 } 3167 3168 /********************************************************************* 3169 * 3170 * Free all transmit rings. 3171 * 3172 **********************************************************************/ 3173 static void 3174 ixgbe_free_transmit_structures(struct adapter *adapter) 3175 { 3176 struct tx_ring *txr = adapter->tx_rings; 3177 3178 for (int i = 0; i < adapter->num_queues; i++, txr++) { 3179 IXGBE_TX_LOCK(txr); 3180 ixgbe_free_transmit_buffers(txr); 3181 ixgbe_dma_free(adapter, &txr->txdma); 3182 IXGBE_TX_UNLOCK(txr); 3183 IXGBE_TX_LOCK_DESTROY(txr); 3184 } 3185 free(adapter->tx_rings, M_DEVBUF); 3186 } 3187 3188 /********************************************************************* 3189 * 3190 * Free transmit ring related data structures. 3191 * 3192 **********************************************************************/ 3193 static void 3194 ixgbe_free_transmit_buffers(struct tx_ring *txr) 3195 { 3196 struct adapter *adapter = txr->adapter; 3197 struct ixgbe_tx_buf *tx_buffer; 3198 int i; 3199 3200 INIT_DEBUGOUT("free_transmit_ring: begin"); 3201 3202 if (txr->tx_buffers == NULL) 3203 return; 3204 3205 tx_buffer = txr->tx_buffers; 3206 for (i = 0; i < adapter->num_tx_desc; i++, tx_buffer++) { 3207 if (tx_buffer->m_head != NULL) { 3208 bus_dmamap_sync(txr->txtag->dt_dmat, tx_buffer->map, 3209 0, tx_buffer->m_head->m_pkthdr.len, 3210 BUS_DMASYNC_POSTWRITE); 3211 ixgbe_dmamap_unload(txr->txtag, tx_buffer->map); 3212 m_freem(tx_buffer->m_head); 3213 tx_buffer->m_head = NULL; 3214 if (tx_buffer->map != NULL) { 3215 ixgbe_dmamap_destroy(txr->txtag, 3216 tx_buffer->map); 3217 tx_buffer->map = NULL; 3218 } 3219 } else if (tx_buffer->map != NULL) { 3220 ixgbe_dmamap_unload(txr->txtag, tx_buffer->map); 3221 ixgbe_dmamap_destroy(txr->txtag, tx_buffer->map); 3222 tx_buffer->map = NULL; 3223 } 3224 } 3225 #if __FreeBSD_version >= 800000 3226 if (txr->br != NULL) 3227 buf_ring_free(txr->br, M_DEVBUF); 3228 #endif 3229 if (txr->tx_buffers != NULL) { 3230 free(txr->tx_buffers, M_DEVBUF); 3231 txr->tx_buffers = NULL; 3232 } 3233 if (txr->txtag != NULL) { 3234 ixgbe_dma_tag_destroy(txr->txtag); 3235 txr->txtag = NULL; 3236 } 3237 return; 3238 } 3239 3240 /********************************************************************* 3241 * 3242 * Advanced Context Descriptor setup for VLAN or L4 CSUM 3243 * 3244 **********************************************************************/ 3245 3246 static u32 3247 ixgbe_tx_ctx_setup(struct tx_ring *txr, struct mbuf *mp) 3248 { 3249 struct m_tag *mtag; 3250 struct adapter *adapter = txr->adapter; 3251 struct ethercom *ec = &adapter->osdep.ec; 3252 struct ixgbe_adv_tx_context_desc *TXD; 3253 struct ixgbe_tx_buf *tx_buffer; 3254 u32 olinfo = 0, vlan_macip_lens = 0, type_tucmd_mlhl = 0; 3255 struct ether_vlan_header *eh; 3256 struct ip ip; 3257 struct ip6_hdr ip6; 3258 int ehdrlen, ip_hlen = 0; 3259 u16 etype; 3260 u8 ipproto = 0; 3261 bool offload; 3262 int ctxd = txr->next_avail_desc; 3263 u16 vtag = 0; 3264 3265 offload = ((mp->m_pkthdr.csum_flags & M_CSUM_OFFLOAD) != 0); 3266 3267 tx_buffer = &txr->tx_buffers[ctxd]; 3268 TXD = (struct ixgbe_adv_tx_context_desc *) &txr->tx_base[ctxd]; 3269 3270 /* 3271 ** In advanced descriptors the vlan tag must 3272 ** be placed into the descriptor itself. 3273 */ 3274 if ((mtag = VLAN_OUTPUT_TAG(ec, mp)) != NULL) { 3275 vtag = htole16(VLAN_TAG_VALUE(mtag) & 0xffff); 3276 vlan_macip_lens |= (vtag << IXGBE_ADVTXD_VLAN_SHIFT); 3277 } else if (!offload) 3278 return 0; 3279 3280 /* 3281 * Determine where frame payload starts. 3282 * Jump over vlan headers if already present, 3283 * helpful for QinQ too. 3284 */ 3285 KASSERT(mp->m_len >= offsetof(struct ether_vlan_header, evl_tag)); 3286 eh = mtod(mp, struct ether_vlan_header *); 3287 if (eh->evl_encap_proto == htons(ETHERTYPE_VLAN)) { 3288 KASSERT(mp->m_len >= sizeof(struct ether_vlan_header)); 3289 etype = ntohs(eh->evl_proto); 3290 ehdrlen = ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN; 3291 } else { 3292 etype = ntohs(eh->evl_encap_proto); 3293 ehdrlen = ETHER_HDR_LEN; 3294 } 3295 3296 /* Set the ether header length */ 3297 vlan_macip_lens |= ehdrlen << IXGBE_ADVTXD_MACLEN_SHIFT; 3298 3299 switch (etype) { 3300 case ETHERTYPE_IP: 3301 m_copydata(mp, ehdrlen, sizeof(ip), &ip); 3302 ip_hlen = ip.ip_hl << 2; 3303 ipproto = ip.ip_p; 3304 #if 0 3305 ip.ip_sum = 0; 3306 m_copyback(mp, ehdrlen, sizeof(ip), &ip); 3307 #else 3308 KASSERT((mp->m_pkthdr.csum_flags & M_CSUM_IPv4) == 0 || 3309 ip.ip_sum == 0); 3310 #endif 3311 type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_IPV4; 3312 break; 3313 case ETHERTYPE_IPV6: 3314 m_copydata(mp, ehdrlen, sizeof(ip6), &ip6); 3315 ip_hlen = sizeof(ip6); 3316 ipproto = ip6.ip6_nxt; 3317 type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_IPV6; 3318 break; 3319 default: 3320 break; 3321 } 3322 3323 if ((mp->m_pkthdr.csum_flags & M_CSUM_IPv4) != 0) 3324 olinfo |= IXGBE_TXD_POPTS_IXSM << 8; 3325 3326 vlan_macip_lens |= ip_hlen; 3327 type_tucmd_mlhl |= IXGBE_ADVTXD_DCMD_DEXT | IXGBE_ADVTXD_DTYP_CTXT; 3328 3329 if (mp->m_pkthdr.csum_flags & (M_CSUM_TCPv4|M_CSUM_TCPv6)) { 3330 type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_L4T_TCP; 3331 olinfo |= IXGBE_TXD_POPTS_TXSM << 8; 3332 KASSERT(ipproto == IPPROTO_TCP); 3333 } else if (mp->m_pkthdr.csum_flags & (M_CSUM_UDPv4|M_CSUM_UDPv6)) { 3334 type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_L4T_UDP; 3335 olinfo |= IXGBE_TXD_POPTS_TXSM << 8; 3336 KASSERT(ipproto == IPPROTO_UDP); 3337 } 3338 3339 /* Now copy bits into descriptor */ 3340 TXD->vlan_macip_lens |= htole32(vlan_macip_lens); 3341 TXD->type_tucmd_mlhl |= htole32(type_tucmd_mlhl); 3342 TXD->seqnum_seed = htole32(0); 3343 TXD->mss_l4len_idx = htole32(0); 3344 3345 tx_buffer->m_head = NULL; 3346 tx_buffer->eop_index = -1; 3347 3348 /* We've consumed the first desc, adjust counters */ 3349 if (++ctxd == adapter->num_tx_desc) 3350 ctxd = 0; 3351 txr->next_avail_desc = ctxd; 3352 --txr->tx_avail; 3353 3354 return olinfo; 3355 } 3356 3357 /********************************************************************** 3358 * 3359 * Setup work for hardware segmentation offload (TSO) on 3360 * adapters using advanced tx descriptors 3361 * 3362 **********************************************************************/ 3363 static bool 3364 ixgbe_tso_setup(struct tx_ring *txr, struct mbuf *mp, u32 *paylen) 3365 { 3366 struct m_tag *mtag; 3367 struct adapter *adapter = txr->adapter; 3368 struct ethercom *ec = &adapter->osdep.ec; 3369 struct ixgbe_adv_tx_context_desc *TXD; 3370 struct ixgbe_tx_buf *tx_buffer; 3371 u32 vlan_macip_lens = 0, type_tucmd_mlhl = 0; 3372 u32 mss_l4len_idx = 0; 3373 u16 vtag = 0; 3374 int ctxd, ehdrlen, hdrlen, ip_hlen, tcp_hlen; 3375 struct ether_vlan_header *eh; 3376 struct ip *ip; 3377 struct tcphdr *th; 3378 3379 3380 /* 3381 * Determine where frame payload starts. 3382 * Jump over vlan headers if already present 3383 */ 3384 eh = mtod(mp, struct ether_vlan_header *); 3385 if (eh->evl_encap_proto == htons(ETHERTYPE_VLAN)) 3386 ehdrlen = ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN; 3387 else 3388 ehdrlen = ETHER_HDR_LEN; 3389 3390 /* Ensure we have at least the IP+TCP header in the first mbuf. */ 3391 if (mp->m_len < ehdrlen + sizeof(struct ip) + sizeof(struct tcphdr)) 3392 return FALSE; 3393 3394 ctxd = txr->next_avail_desc; 3395 tx_buffer = &txr->tx_buffers[ctxd]; 3396 TXD = (struct ixgbe_adv_tx_context_desc *) &txr->tx_base[ctxd]; 3397 3398 ip = (struct ip *)(mp->m_data + ehdrlen); 3399 if (ip->ip_p != IPPROTO_TCP) 3400 return FALSE; /* 0 */ 3401 ip->ip_sum = 0; 3402 ip_hlen = ip->ip_hl << 2; 3403 th = (struct tcphdr *)((char *)ip + ip_hlen); 3404 /* XXX Educated guess: FreeBSD's in_pseudo == NetBSD's in_cksum_phdr */ 3405 th->th_sum = in_cksum_phdr(ip->ip_src.s_addr, 3406 ip->ip_dst.s_addr, htons(IPPROTO_TCP)); 3407 tcp_hlen = th->th_off << 2; 3408 hdrlen = ehdrlen + ip_hlen + tcp_hlen; 3409 3410 /* This is used in the transmit desc in encap */ 3411 *paylen = mp->m_pkthdr.len - hdrlen; 3412 3413 /* VLAN MACLEN IPLEN */ 3414 if ((mtag = VLAN_OUTPUT_TAG(ec, mp)) != NULL) { 3415 vtag = htole16(VLAN_TAG_VALUE(mtag) & 0xffff); 3416 vlan_macip_lens |= (vtag << IXGBE_ADVTXD_VLAN_SHIFT); 3417 } 3418 3419 vlan_macip_lens |= ehdrlen << IXGBE_ADVTXD_MACLEN_SHIFT; 3420 vlan_macip_lens |= ip_hlen; 3421 TXD->vlan_macip_lens |= htole32(vlan_macip_lens); 3422 3423 /* ADV DTYPE TUCMD */ 3424 type_tucmd_mlhl |= IXGBE_ADVTXD_DCMD_DEXT | IXGBE_ADVTXD_DTYP_CTXT; 3425 type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_L4T_TCP; 3426 type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_IPV4; 3427 TXD->type_tucmd_mlhl |= htole32(type_tucmd_mlhl); 3428 3429 3430 /* MSS L4LEN IDX */ 3431 mss_l4len_idx |= (mp->m_pkthdr.segsz << IXGBE_ADVTXD_MSS_SHIFT); 3432 mss_l4len_idx |= (tcp_hlen << IXGBE_ADVTXD_L4LEN_SHIFT); 3433 TXD->mss_l4len_idx = htole32(mss_l4len_idx); 3434 3435 TXD->seqnum_seed = htole32(0); 3436 tx_buffer->m_head = NULL; 3437 tx_buffer->eop_index = -1; 3438 3439 if (++ctxd == adapter->num_tx_desc) 3440 ctxd = 0; 3441 3442 txr->tx_avail--; 3443 txr->next_avail_desc = ctxd; 3444 return TRUE; 3445 } 3446 3447 #ifdef IXGBE_FDIR 3448 /* 3449 ** This routine parses packet headers so that Flow 3450 ** Director can make a hashed filter table entry 3451 ** allowing traffic flows to be identified and kept 3452 ** on the same cpu. This would be a performance 3453 ** hit, but we only do it at IXGBE_FDIR_RATE of 3454 ** packets. 3455 */ 3456 static void 3457 ixgbe_atr(struct tx_ring *txr, struct mbuf *mp) 3458 { 3459 struct adapter *adapter = txr->adapter; 3460 struct ix_queue *que; 3461 struct ip *ip; 3462 struct tcphdr *th; 3463 struct udphdr *uh; 3464 struct ether_vlan_header *eh; 3465 union ixgbe_atr_hash_dword input = {.dword = 0}; 3466 union ixgbe_atr_hash_dword common = {.dword = 0}; 3467 int ehdrlen, ip_hlen; 3468 u16 etype; 3469 3470 eh = mtod(mp, struct ether_vlan_header *); 3471 if (eh->evl_encap_proto == htons(ETHERTYPE_VLAN)) { 3472 ehdrlen = ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN; 3473 etype = eh->evl_proto; 3474 } else { 3475 ehdrlen = ETHER_HDR_LEN; 3476 etype = eh->evl_encap_proto; 3477 } 3478 3479 /* Only handling IPv4 */ 3480 if (etype != htons(ETHERTYPE_IP)) 3481 return; 3482 3483 ip = (struct ip *)(mp->m_data + ehdrlen); 3484 ip_hlen = ip->ip_hl << 2; 3485 3486 /* check if we're UDP or TCP */ 3487 switch (ip->ip_p) { 3488 case IPPROTO_TCP: 3489 th = (struct tcphdr *)((char *)ip + ip_hlen); 3490 /* src and dst are inverted */ 3491 common.port.dst ^= th->th_sport; 3492 common.port.src ^= th->th_dport; 3493 input.formatted.flow_type ^= IXGBE_ATR_FLOW_TYPE_TCPV4; 3494 break; 3495 case IPPROTO_UDP: 3496 uh = (struct udphdr *)((char *)ip + ip_hlen); 3497 /* src and dst are inverted */ 3498 common.port.dst ^= uh->uh_sport; 3499 common.port.src ^= uh->uh_dport; 3500 input.formatted.flow_type ^= IXGBE_ATR_FLOW_TYPE_UDPV4; 3501 break; 3502 default: 3503 return; 3504 } 3505 3506 input.formatted.vlan_id = htobe16(mp->m_pkthdr.ether_vtag); 3507 if (mp->m_pkthdr.ether_vtag) 3508 common.flex_bytes ^= htons(ETHERTYPE_VLAN); 3509 else 3510 common.flex_bytes ^= etype; 3511 common.ip ^= ip->ip_src.s_addr ^ ip->ip_dst.s_addr; 3512 3513 que = &adapter->queues[txr->me]; 3514 /* 3515 ** This assumes the Rx queue and Tx 3516 ** queue are bound to the same CPU 3517 */ 3518 ixgbe_fdir_add_signature_filter_82599(&adapter->hw, 3519 input, common, que->msix); 3520 } 3521 #endif /* IXGBE_FDIR */ 3522 3523 /********************************************************************** 3524 * 3525 * Examine each tx_buffer in the used queue. If the hardware is done 3526 * processing the packet then free associated resources. The 3527 * tx_buffer is put back on the free queue. 3528 * 3529 **********************************************************************/ 3530 static bool 3531 ixgbe_txeof(struct tx_ring *txr) 3532 { 3533 struct adapter *adapter = txr->adapter; 3534 struct ifnet *ifp = adapter->ifp; 3535 u32 first, last, done, processed; 3536 struct ixgbe_tx_buf *tx_buffer; 3537 struct ixgbe_legacy_tx_desc *tx_desc, *eop_desc; 3538 struct timeval now, elapsed; 3539 3540 KASSERT(mutex_owned(&txr->tx_mtx)); 3541 3542 if (txr->tx_avail == adapter->num_tx_desc) { 3543 txr->queue_status = IXGBE_QUEUE_IDLE; 3544 return false; 3545 } 3546 3547 processed = 0; 3548 first = txr->next_to_clean; 3549 tx_buffer = &txr->tx_buffers[first]; 3550 /* For cleanup we just use legacy struct */ 3551 tx_desc = (struct ixgbe_legacy_tx_desc *)&txr->tx_base[first]; 3552 last = tx_buffer->eop_index; 3553 if (last == -1) 3554 return false; 3555 eop_desc = (struct ixgbe_legacy_tx_desc *)&txr->tx_base[last]; 3556 3557 /* 3558 ** Get the index of the first descriptor 3559 ** BEYOND the EOP and call that 'done'. 3560 ** I do this so the comparison in the 3561 ** inner while loop below can be simple 3562 */ 3563 if (++last == adapter->num_tx_desc) last = 0; 3564 done = last; 3565 3566 ixgbe_dmamap_sync(txr->txdma.dma_tag, txr->txdma.dma_map, 3567 BUS_DMASYNC_POSTREAD); 3568 /* 3569 ** Only the EOP descriptor of a packet now has the DD 3570 ** bit set, this is what we look for... 3571 */ 3572 while (eop_desc->upper.fields.status & IXGBE_TXD_STAT_DD) { 3573 /* We clean the range of the packet */ 3574 while (first != done) { 3575 tx_desc->upper.data = 0; 3576 tx_desc->lower.data = 0; 3577 tx_desc->buffer_addr = 0; 3578 ++txr->tx_avail; 3579 ++processed; 3580 3581 if (tx_buffer->m_head) { 3582 txr->bytes += 3583 tx_buffer->m_head->m_pkthdr.len; 3584 bus_dmamap_sync(txr->txtag->dt_dmat, 3585 tx_buffer->map, 3586 0, tx_buffer->m_head->m_pkthdr.len, 3587 BUS_DMASYNC_POSTWRITE); 3588 ixgbe_dmamap_unload(txr->txtag, tx_buffer->map); 3589 m_freem(tx_buffer->m_head); 3590 tx_buffer->m_head = NULL; 3591 } 3592 tx_buffer->eop_index = -1; 3593 getmicrotime(&txr->watchdog_time); 3594 3595 if (++first == adapter->num_tx_desc) 3596 first = 0; 3597 3598 tx_buffer = &txr->tx_buffers[first]; 3599 tx_desc = 3600 (struct ixgbe_legacy_tx_desc *)&txr->tx_base[first]; 3601 } 3602 ++txr->packets; 3603 ++ifp->if_opackets; 3604 /* See if there is more work now */ 3605 last = tx_buffer->eop_index; 3606 if (last != -1) { 3607 eop_desc = 3608 (struct ixgbe_legacy_tx_desc *)&txr->tx_base[last]; 3609 /* Get next done point */ 3610 if (++last == adapter->num_tx_desc) last = 0; 3611 done = last; 3612 } else 3613 break; 3614 } 3615 ixgbe_dmamap_sync(txr->txdma.dma_tag, txr->txdma.dma_map, 3616 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); 3617 3618 txr->next_to_clean = first; 3619 3620 /* 3621 ** Watchdog calculation, we know there's 3622 ** work outstanding or the first return 3623 ** would have been taken, so none processed 3624 ** for too long indicates a hang. 3625 */ 3626 getmicrotime(&now); 3627 timersub(&now, &txr->watchdog_time, &elapsed); 3628 if (!processed && tvtohz(&elapsed) > IXGBE_WATCHDOG) 3629 txr->queue_status = IXGBE_QUEUE_HUNG; 3630 3631 /* 3632 * If we have enough room, clear IFF_OACTIVE to tell the stack that 3633 * it is OK to send packets. If there are no pending descriptors, 3634 * clear the timeout. Otherwise, if some descriptors have been freed, 3635 * restart the timeout. 3636 */ 3637 if (txr->tx_avail > IXGBE_TX_CLEANUP_THRESHOLD) { 3638 ifp->if_flags &= ~IFF_OACTIVE; 3639 if (txr->tx_avail == adapter->num_tx_desc) { 3640 txr->queue_status = IXGBE_QUEUE_IDLE; 3641 return false; 3642 } 3643 } 3644 3645 return true; 3646 } 3647 3648 /********************************************************************* 3649 * 3650 * Refresh mbuf buffers for RX descriptor rings 3651 * - now keeps its own state so discards due to resource 3652 * exhaustion are unnecessary, if an mbuf cannot be obtained 3653 * it just returns, keeping its placeholder, thus it can simply 3654 * be recalled to try again. 3655 * 3656 **********************************************************************/ 3657 static void 3658 ixgbe_refresh_mbufs(struct rx_ring *rxr, int limit) 3659 { 3660 struct adapter *adapter = rxr->adapter; 3661 struct ixgbe_rx_buf *rxbuf; 3662 struct mbuf *mh, *mp; 3663 int i, j, error; 3664 bool refreshed = false; 3665 3666 i = j = rxr->next_to_refresh; 3667 /* Control the loop with one beyond */ 3668 if (++j == adapter->num_rx_desc) 3669 j = 0; 3670 3671 while (j != limit) { 3672 rxbuf = &rxr->rx_buffers[i]; 3673 if (rxr->hdr_split == FALSE) 3674 goto no_split; 3675 3676 if (rxbuf->m_head == NULL) { 3677 mh = m_gethdr(M_DONTWAIT, MT_DATA); 3678 if (mh == NULL) 3679 goto update; 3680 } else 3681 mh = rxbuf->m_head; 3682 3683 mh->m_pkthdr.len = mh->m_len = MHLEN; 3684 mh->m_len = MHLEN; 3685 mh->m_flags |= M_PKTHDR; 3686 /* Get the memory mapping */ 3687 error = bus_dmamap_load_mbuf(rxr->htag->dt_dmat, 3688 rxbuf->hmap, mh, BUS_DMA_NOWAIT); 3689 if (error != 0) { 3690 printf("Refresh mbufs: hdr dmamap load" 3691 " failure - %d\n", error); 3692 m_free(mh); 3693 rxbuf->m_head = NULL; 3694 goto update; 3695 } 3696 rxbuf->m_head = mh; 3697 ixgbe_dmamap_sync(rxr->htag, rxbuf->hmap, BUS_DMASYNC_PREREAD); 3698 rxr->rx_base[i].read.hdr_addr = 3699 htole64(rxbuf->hmap->dm_segs[0].ds_addr); 3700 3701 no_split: 3702 if (rxbuf->m_pack == NULL) { 3703 mp = ixgbe_getjcl(&adapter->jcl_head, M_DONTWAIT, 3704 MT_DATA, M_PKTHDR, adapter->rx_mbuf_sz); 3705 if (mp == NULL) { 3706 rxr->no_jmbuf.ev_count++; 3707 goto update; 3708 } 3709 } else 3710 mp = rxbuf->m_pack; 3711 3712 mp->m_pkthdr.len = mp->m_len = adapter->rx_mbuf_sz; 3713 /* Get the memory mapping */ 3714 error = bus_dmamap_load_mbuf(rxr->ptag->dt_dmat, 3715 rxbuf->pmap, mp, BUS_DMA_NOWAIT); 3716 if (error != 0) { 3717 printf("Refresh mbufs: payload dmamap load" 3718 " failure - %d\n", error); 3719 m_free(mp); 3720 rxbuf->m_pack = NULL; 3721 goto update; 3722 } 3723 rxbuf->m_pack = mp; 3724 bus_dmamap_sync(rxr->ptag->dt_dmat, rxbuf->pmap, 3725 0, mp->m_pkthdr.len, BUS_DMASYNC_PREREAD); 3726 rxr->rx_base[i].read.pkt_addr = 3727 htole64(rxbuf->pmap->dm_segs[0].ds_addr); 3728 3729 refreshed = true; 3730 /* Next is precalculated */ 3731 i = j; 3732 rxr->next_to_refresh = i; 3733 if (++j == adapter->num_rx_desc) 3734 j = 0; 3735 } 3736 update: 3737 if (refreshed) /* Update hardware tail index */ 3738 IXGBE_WRITE_REG(&adapter->hw, 3739 IXGBE_RDT(rxr->me), rxr->next_to_refresh); 3740 return; 3741 } 3742 3743 /********************************************************************* 3744 * 3745 * Allocate memory for rx_buffer structures. Since we use one 3746 * rx_buffer per received packet, the maximum number of rx_buffer's 3747 * that we'll need is equal to the number of receive descriptors 3748 * that we've allocated. 3749 * 3750 **********************************************************************/ 3751 static int 3752 ixgbe_allocate_receive_buffers(struct rx_ring *rxr) 3753 { 3754 struct adapter *adapter = rxr->adapter; 3755 device_t dev = adapter->dev; 3756 struct ixgbe_rx_buf *rxbuf; 3757 int i, bsize, error; 3758 3759 bsize = sizeof(struct ixgbe_rx_buf) * adapter->num_rx_desc; 3760 if (!(rxr->rx_buffers = 3761 (struct ixgbe_rx_buf *) malloc(bsize, 3762 M_DEVBUF, M_NOWAIT | M_ZERO))) { 3763 aprint_error_dev(dev, "Unable to allocate rx_buffer memory\n"); 3764 error = ENOMEM; 3765 goto fail; 3766 } 3767 3768 if ((error = ixgbe_dma_tag_create(adapter->osdep.dmat, /* parent */ 3769 1, 0, /* alignment, bounds */ 3770 MSIZE, /* maxsize */ 3771 1, /* nsegments */ 3772 MSIZE, /* maxsegsize */ 3773 0, /* flags */ 3774 &rxr->htag))) { 3775 aprint_error_dev(dev, "Unable to create RX DMA tag\n"); 3776 goto fail; 3777 } 3778 3779 if ((error = ixgbe_dma_tag_create(adapter->osdep.dmat, /* parent */ 3780 1, 0, /* alignment, bounds */ 3781 MJUM16BYTES, /* maxsize */ 3782 1, /* nsegments */ 3783 MJUM16BYTES, /* maxsegsize */ 3784 0, /* flags */ 3785 &rxr->ptag))) { 3786 aprint_error_dev(dev, "Unable to create RX DMA tag\n"); 3787 goto fail; 3788 } 3789 3790 for (i = 0; i < adapter->num_rx_desc; i++, rxbuf++) { 3791 rxbuf = &rxr->rx_buffers[i]; 3792 error = ixgbe_dmamap_create(rxr->htag, 3793 BUS_DMA_NOWAIT, &rxbuf->hmap); 3794 if (error) { 3795 aprint_error_dev(dev, "Unable to create RX head map\n"); 3796 goto fail; 3797 } 3798 error = ixgbe_dmamap_create(rxr->ptag, 3799 BUS_DMA_NOWAIT, &rxbuf->pmap); 3800 if (error) { 3801 aprint_error_dev(dev, "Unable to create RX pkt map\n"); 3802 goto fail; 3803 } 3804 } 3805 3806 return (0); 3807 3808 fail: 3809 /* Frees all, but can handle partial completion */ 3810 ixgbe_free_receive_structures(adapter); 3811 return (error); 3812 } 3813 3814 /* 3815 ** Used to detect a descriptor that has 3816 ** been merged by Hardware RSC. 3817 */ 3818 static inline u32 3819 ixgbe_rsc_count(union ixgbe_adv_rx_desc *rx) 3820 { 3821 return (le32toh(rx->wb.lower.lo_dword.data) & 3822 IXGBE_RXDADV_RSCCNT_MASK) >> IXGBE_RXDADV_RSCCNT_SHIFT; 3823 } 3824 3825 /********************************************************************* 3826 * 3827 * Initialize Hardware RSC (LRO) feature on 82599 3828 * for an RX ring, this is toggled by the LRO capability 3829 * even though it is transparent to the stack. 3830 * 3831 **********************************************************************/ 3832 static void 3833 ixgbe_setup_hw_rsc(struct rx_ring *rxr) 3834 { 3835 struct adapter *adapter = rxr->adapter; 3836 struct ixgbe_hw *hw = &adapter->hw; 3837 u32 rscctrl, rdrxctl; 3838 3839 rdrxctl = IXGBE_READ_REG(hw, IXGBE_RDRXCTL); 3840 rdrxctl &= ~IXGBE_RDRXCTL_RSCFRSTSIZE; 3841 rdrxctl |= IXGBE_RDRXCTL_CRCSTRIP; 3842 rdrxctl |= IXGBE_RDRXCTL_RSCACKC; 3843 IXGBE_WRITE_REG(hw, IXGBE_RDRXCTL, rdrxctl); 3844 3845 rscctrl = IXGBE_READ_REG(hw, IXGBE_RSCCTL(rxr->me)); 3846 rscctrl |= IXGBE_RSCCTL_RSCEN; 3847 /* 3848 ** Limit the total number of descriptors that 3849 ** can be combined, so it does not exceed 64K 3850 */ 3851 if (adapter->rx_mbuf_sz == MCLBYTES) 3852 rscctrl |= IXGBE_RSCCTL_MAXDESC_16; 3853 else if (adapter->rx_mbuf_sz == MJUMPAGESIZE) 3854 rscctrl |= IXGBE_RSCCTL_MAXDESC_8; 3855 else if (adapter->rx_mbuf_sz == MJUM9BYTES) 3856 rscctrl |= IXGBE_RSCCTL_MAXDESC_4; 3857 else /* Using 16K cluster */ 3858 rscctrl |= IXGBE_RSCCTL_MAXDESC_1; 3859 3860 IXGBE_WRITE_REG(hw, IXGBE_RSCCTL(rxr->me), rscctrl); 3861 3862 /* Enable TCP header recognition */ 3863 IXGBE_WRITE_REG(hw, IXGBE_PSRTYPE(0), 3864 (IXGBE_READ_REG(hw, IXGBE_PSRTYPE(0)) | 3865 IXGBE_PSRTYPE_TCPHDR)); 3866 3867 /* Disable RSC for ACK packets */ 3868 IXGBE_WRITE_REG(hw, IXGBE_RSCDBU, 3869 (IXGBE_RSCDBU_RSCACKDIS | IXGBE_READ_REG(hw, IXGBE_RSCDBU))); 3870 3871 rxr->hw_rsc = TRUE; 3872 } 3873 3874 3875 static void 3876 ixgbe_free_receive_ring(struct rx_ring *rxr) 3877 { 3878 struct adapter *adapter; 3879 struct ixgbe_rx_buf *rxbuf; 3880 int i; 3881 3882 adapter = rxr->adapter; 3883 for (i = 0; i < adapter->num_rx_desc; i++) { 3884 rxbuf = &rxr->rx_buffers[i]; 3885 if (rxbuf->m_head != NULL) { 3886 ixgbe_dmamap_sync(rxr->htag, rxbuf->hmap, 3887 BUS_DMASYNC_POSTREAD); 3888 ixgbe_dmamap_unload(rxr->htag, rxbuf->hmap); 3889 rxbuf->m_head->m_flags |= M_PKTHDR; 3890 m_freem(rxbuf->m_head); 3891 } 3892 if (rxbuf->m_pack != NULL) { 3893 bus_dmamap_sync(rxr->ptag->dt_dmat, rxbuf->pmap, 3894 0, rxbuf->m_pack->m_pkthdr.len, 3895 BUS_DMASYNC_POSTREAD); 3896 ixgbe_dmamap_unload(rxr->ptag, rxbuf->pmap); 3897 rxbuf->m_pack->m_flags |= M_PKTHDR; 3898 m_freem(rxbuf->m_pack); 3899 } 3900 rxbuf->m_head = NULL; 3901 rxbuf->m_pack = NULL; 3902 } 3903 } 3904 3905 3906 /********************************************************************* 3907 * 3908 * Initialize a receive ring and its buffers. 3909 * 3910 **********************************************************************/ 3911 static int 3912 ixgbe_setup_receive_ring(struct rx_ring *rxr) 3913 { 3914 struct adapter *adapter; 3915 struct ifnet *ifp; 3916 struct ixgbe_rx_buf *rxbuf; 3917 #ifdef LRO 3918 struct lro_ctrl *lro = &rxr->lro; 3919 #endif /* LRO */ 3920 int rsize, error = 0; 3921 3922 adapter = rxr->adapter; 3923 ifp = adapter->ifp; 3924 3925 /* Clear the ring contents */ 3926 IXGBE_RX_LOCK(rxr); 3927 rsize = roundup2(adapter->num_rx_desc * 3928 sizeof(union ixgbe_adv_rx_desc), DBA_ALIGN); 3929 bzero((void *)rxr->rx_base, rsize); 3930 3931 /* Free current RX buffer structs and their mbufs */ 3932 ixgbe_free_receive_ring(rxr); 3933 3934 /* Now reinitialize our supply of jumbo mbufs. The number 3935 * or size of jumbo mbufs may have changed. 3936 */ 3937 ixgbe_jcl_reinit(&adapter->jcl_head, rxr->ptag->dt_dmat, 3938 2 * adapter->num_rx_desc, adapter->rx_mbuf_sz); 3939 3940 /* Configure header split? */ 3941 if (ixgbe_header_split) 3942 rxr->hdr_split = TRUE; 3943 3944 /* Now replenish the mbufs */ 3945 for (int j = 0; j != adapter->num_rx_desc; ++j) { 3946 struct mbuf *mh, *mp; 3947 3948 rxbuf = &rxr->rx_buffers[j]; 3949 /* 3950 ** Don't allocate mbufs if not 3951 ** doing header split, its wasteful 3952 */ 3953 if (rxr->hdr_split == FALSE) 3954 goto skip_head; 3955 3956 /* First the header */ 3957 rxbuf->m_head = m_gethdr(M_DONTWAIT, MT_DATA); 3958 if (rxbuf->m_head == NULL) { 3959 error = ENOBUFS; 3960 goto fail; 3961 } 3962 m_adj(rxbuf->m_head, ETHER_ALIGN); 3963 mh = rxbuf->m_head; 3964 mh->m_len = mh->m_pkthdr.len = MHLEN; 3965 mh->m_flags |= M_PKTHDR; 3966 /* Get the memory mapping */ 3967 error = bus_dmamap_load_mbuf(rxr->htag->dt_dmat, 3968 rxbuf->hmap, rxbuf->m_head, BUS_DMA_NOWAIT); 3969 if (error != 0) /* Nothing elegant to do here */ 3970 goto fail; 3971 bus_dmamap_sync(rxr->htag->dt_dmat, rxbuf->hmap, 3972 0, mh->m_pkthdr.len, BUS_DMASYNC_PREREAD); 3973 /* Update descriptor */ 3974 rxr->rx_base[j].read.hdr_addr = 3975 htole64(rxbuf->hmap->dm_segs[0].ds_addr); 3976 3977 skip_head: 3978 /* Now the payload cluster */ 3979 rxbuf->m_pack = ixgbe_getjcl(&adapter->jcl_head, M_DONTWAIT, 3980 MT_DATA, M_PKTHDR, adapter->rx_mbuf_sz); 3981 if (rxbuf->m_pack == NULL) { 3982 error = ENOBUFS; 3983 goto fail; 3984 } 3985 mp = rxbuf->m_pack; 3986 mp->m_pkthdr.len = mp->m_len = adapter->rx_mbuf_sz; 3987 /* Get the memory mapping */ 3988 error = bus_dmamap_load_mbuf(rxr->ptag->dt_dmat, 3989 rxbuf->pmap, mp, BUS_DMA_NOWAIT); 3990 if (error != 0) 3991 goto fail; 3992 bus_dmamap_sync(rxr->ptag->dt_dmat, rxbuf->pmap, 3993 0, adapter->rx_mbuf_sz, BUS_DMASYNC_PREREAD); 3994 /* Update descriptor */ 3995 rxr->rx_base[j].read.pkt_addr = 3996 htole64(rxbuf->pmap->dm_segs[0].ds_addr); 3997 } 3998 3999 4000 /* Setup our descriptor indices */ 4001 rxr->next_to_check = 0; 4002 rxr->next_to_refresh = 0; 4003 rxr->lro_enabled = FALSE; 4004 rxr->rx_split_packets.ev_count = 0; 4005 rxr->rx_bytes.ev_count = 0; 4006 rxr->discard = FALSE; 4007 4008 ixgbe_dmamap_sync(rxr->rxdma.dma_tag, rxr->rxdma.dma_map, 4009 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); 4010 4011 /* 4012 ** Now set up the LRO interface: 4013 ** 82598 uses software LRO, the 4014 ** 82599 uses a hardware assist. 4015 */ 4016 if ((adapter->hw.mac.type != ixgbe_mac_82598EB) && 4017 (ifp->if_capenable & IFCAP_RXCSUM) && 4018 (ifp->if_capenable & IFCAP_LRO)) 4019 ixgbe_setup_hw_rsc(rxr); 4020 #ifdef LRO 4021 else if (ifp->if_capenable & IFCAP_LRO) { 4022 device_t dev = adapter->dev; 4023 int err = tcp_lro_init(lro); 4024 if (err) { 4025 device_printf(dev, "LRO Initialization failed!\n"); 4026 goto fail; 4027 } 4028 INIT_DEBUGOUT("RX Soft LRO Initialized\n"); 4029 rxr->lro_enabled = TRUE; 4030 lro->ifp = adapter->ifp; 4031 } 4032 #endif /* LRO */ 4033 4034 IXGBE_RX_UNLOCK(rxr); 4035 return (0); 4036 4037 fail: 4038 ixgbe_free_receive_ring(rxr); 4039 IXGBE_RX_UNLOCK(rxr); 4040 return (error); 4041 } 4042 4043 /********************************************************************* 4044 * 4045 * Initialize all receive rings. 4046 * 4047 **********************************************************************/ 4048 static int 4049 ixgbe_setup_receive_structures(struct adapter *adapter) 4050 { 4051 struct rx_ring *rxr = adapter->rx_rings; 4052 int j; 4053 4054 for (j = 0; j < adapter->num_queues; j++, rxr++) 4055 if (ixgbe_setup_receive_ring(rxr)) 4056 goto fail; 4057 4058 return (0); 4059 fail: 4060 /* 4061 * Free RX buffers allocated so far, we will only handle 4062 * the rings that completed, the failing case will have 4063 * cleaned up for itself. 'j' failed, so its the terminus. 4064 */ 4065 for (int i = 0; i < j; ++i) { 4066 rxr = &adapter->rx_rings[i]; 4067 ixgbe_free_receive_ring(rxr); 4068 } 4069 4070 return (ENOBUFS); 4071 } 4072 4073 /********************************************************************* 4074 * 4075 * Setup receive registers and features. 4076 * 4077 **********************************************************************/ 4078 #define IXGBE_SRRCTL_BSIZEHDRSIZE_SHIFT 2 4079 4080 static void 4081 ixgbe_initialize_receive_units(struct adapter *adapter) 4082 { 4083 int i; 4084 struct rx_ring *rxr = adapter->rx_rings; 4085 struct ixgbe_hw *hw = &adapter->hw; 4086 struct ifnet *ifp = adapter->ifp; 4087 u32 bufsz, rxctrl, fctrl, srrctl, rxcsum; 4088 u32 reta, mrqc = 0, hlreg, r[10]; 4089 4090 4091 /* 4092 * Make sure receives are disabled while 4093 * setting up the descriptor ring 4094 */ 4095 rxctrl = IXGBE_READ_REG(hw, IXGBE_RXCTRL); 4096 IXGBE_WRITE_REG(hw, IXGBE_RXCTRL, 4097 rxctrl & ~IXGBE_RXCTRL_RXEN); 4098 4099 /* Enable broadcasts */ 4100 fctrl = IXGBE_READ_REG(hw, IXGBE_FCTRL); 4101 fctrl |= IXGBE_FCTRL_BAM; 4102 fctrl |= IXGBE_FCTRL_DPF; 4103 fctrl |= IXGBE_FCTRL_PMCF; 4104 IXGBE_WRITE_REG(hw, IXGBE_FCTRL, fctrl); 4105 4106 /* Set for Jumbo Frames? */ 4107 hlreg = IXGBE_READ_REG(hw, IXGBE_HLREG0); 4108 if (ifp->if_mtu > ETHERMTU) 4109 hlreg |= IXGBE_HLREG0_JUMBOEN; 4110 else 4111 hlreg &= ~IXGBE_HLREG0_JUMBOEN; 4112 IXGBE_WRITE_REG(hw, IXGBE_HLREG0, hlreg); 4113 4114 bufsz = adapter->rx_mbuf_sz >> IXGBE_SRRCTL_BSIZEPKT_SHIFT; 4115 4116 for (i = 0; i < adapter->num_queues; i++, rxr++) { 4117 u64 rdba = rxr->rxdma.dma_paddr; 4118 4119 /* Setup the Base and Length of the Rx Descriptor Ring */ 4120 IXGBE_WRITE_REG(hw, IXGBE_RDBAL(i), 4121 (rdba & 0x00000000ffffffffULL)); 4122 IXGBE_WRITE_REG(hw, IXGBE_RDBAH(i), (rdba >> 32)); 4123 IXGBE_WRITE_REG(hw, IXGBE_RDLEN(i), 4124 adapter->num_rx_desc * sizeof(union ixgbe_adv_rx_desc)); 4125 4126 /* Set up the SRRCTL register */ 4127 srrctl = IXGBE_READ_REG(hw, IXGBE_SRRCTL(i)); 4128 srrctl &= ~IXGBE_SRRCTL_BSIZEHDR_MASK; 4129 srrctl &= ~IXGBE_SRRCTL_BSIZEPKT_MASK; 4130 srrctl |= bufsz; 4131 if (rxr->hdr_split) { 4132 /* Use a standard mbuf for the header */ 4133 srrctl |= ((IXGBE_RX_HDR << 4134 IXGBE_SRRCTL_BSIZEHDRSIZE_SHIFT) 4135 & IXGBE_SRRCTL_BSIZEHDR_MASK); 4136 srrctl |= IXGBE_SRRCTL_DESCTYPE_HDR_SPLIT_ALWAYS; 4137 } else 4138 srrctl |= IXGBE_SRRCTL_DESCTYPE_ADV_ONEBUF; 4139 IXGBE_WRITE_REG(hw, IXGBE_SRRCTL(i), srrctl); 4140 4141 /* Setup the HW Rx Head and Tail Descriptor Pointers */ 4142 IXGBE_WRITE_REG(hw, IXGBE_RDH(i), 0); 4143 IXGBE_WRITE_REG(hw, IXGBE_RDT(i), 0); 4144 } 4145 4146 if (adapter->hw.mac.type != ixgbe_mac_82598EB) { 4147 u32 psrtype = IXGBE_PSRTYPE_TCPHDR | 4148 IXGBE_PSRTYPE_UDPHDR | 4149 IXGBE_PSRTYPE_IPV4HDR | 4150 IXGBE_PSRTYPE_IPV6HDR; 4151 IXGBE_WRITE_REG(hw, IXGBE_PSRTYPE(0), psrtype); 4152 } 4153 4154 rxcsum = IXGBE_READ_REG(hw, IXGBE_RXCSUM); 4155 4156 /* Setup RSS */ 4157 if (adapter->num_queues > 1) { 4158 int j; 4159 reta = 0; 4160 4161 /* set up random bits */ 4162 cprng_fast(&r, sizeof(r)); 4163 4164 /* Set up the redirection table */ 4165 for (i = 0, j = 0; i < 128; i++, j++) { 4166 if (j == adapter->num_queues) j = 0; 4167 reta = (reta << 8) | (j * 0x11); 4168 if ((i & 3) == 3) 4169 IXGBE_WRITE_REG(hw, IXGBE_RETA(i >> 2), reta); 4170 } 4171 4172 /* Now fill our hash function seeds */ 4173 for (i = 0; i < 10; i++) 4174 IXGBE_WRITE_REG(hw, IXGBE_RSSRK(i), r[i]); 4175 4176 /* Perform hash on these packet types */ 4177 mrqc = IXGBE_MRQC_RSSEN 4178 | IXGBE_MRQC_RSS_FIELD_IPV4 4179 | IXGBE_MRQC_RSS_FIELD_IPV4_TCP 4180 | IXGBE_MRQC_RSS_FIELD_IPV4_UDP 4181 | IXGBE_MRQC_RSS_FIELD_IPV6_EX_TCP 4182 | IXGBE_MRQC_RSS_FIELD_IPV6_EX 4183 | IXGBE_MRQC_RSS_FIELD_IPV6 4184 | IXGBE_MRQC_RSS_FIELD_IPV6_TCP 4185 | IXGBE_MRQC_RSS_FIELD_IPV6_UDP 4186 | IXGBE_MRQC_RSS_FIELD_IPV6_EX_UDP; 4187 IXGBE_WRITE_REG(hw, IXGBE_MRQC, mrqc); 4188 4189 /* RSS and RX IPP Checksum are mutually exclusive */ 4190 rxcsum |= IXGBE_RXCSUM_PCSD; 4191 } 4192 4193 if (ifp->if_capenable & IFCAP_RXCSUM) 4194 rxcsum |= IXGBE_RXCSUM_PCSD; 4195 4196 if (!(rxcsum & IXGBE_RXCSUM_PCSD)) 4197 rxcsum |= IXGBE_RXCSUM_IPPCSE; 4198 4199 IXGBE_WRITE_REG(hw, IXGBE_RXCSUM, rxcsum); 4200 4201 return; 4202 } 4203 4204 /********************************************************************* 4205 * 4206 * Free all receive rings. 4207 * 4208 **********************************************************************/ 4209 static void 4210 ixgbe_free_receive_structures(struct adapter *adapter) 4211 { 4212 struct rx_ring *rxr = adapter->rx_rings; 4213 4214 for (int i = 0; i < adapter->num_queues; i++, rxr++) { 4215 #ifdef LRO 4216 struct lro_ctrl *lro = &rxr->lro; 4217 #endif /* LRO */ 4218 ixgbe_free_receive_buffers(rxr); 4219 #ifdef LRO 4220 /* Free LRO memory */ 4221 tcp_lro_free(lro); 4222 #endif /* LRO */ 4223 /* Free the ring memory as well */ 4224 ixgbe_dma_free(adapter, &rxr->rxdma); 4225 } 4226 4227 free(adapter->rx_rings, M_DEVBUF); 4228 } 4229 4230 4231 /********************************************************************* 4232 * 4233 * Free receive ring data structures 4234 * 4235 **********************************************************************/ 4236 static void 4237 ixgbe_free_receive_buffers(struct rx_ring *rxr) 4238 { 4239 struct adapter *adapter = rxr->adapter; 4240 struct ixgbe_rx_buf *rxbuf; 4241 4242 INIT_DEBUGOUT("free_receive_structures: begin"); 4243 4244 /* Cleanup any existing buffers */ 4245 if (rxr->rx_buffers != NULL) { 4246 for (int i = 0; i < adapter->num_rx_desc; i++) { 4247 rxbuf = &rxr->rx_buffers[i]; 4248 if (rxbuf->m_head != NULL) { 4249 ixgbe_dmamap_sync(rxr->htag, rxbuf->hmap, 4250 BUS_DMASYNC_POSTREAD); 4251 ixgbe_dmamap_unload(rxr->htag, rxbuf->hmap); 4252 rxbuf->m_head->m_flags |= M_PKTHDR; 4253 m_freem(rxbuf->m_head); 4254 } 4255 if (rxbuf->m_pack != NULL) { 4256 bus_dmamap_sync(rxr->ptag->dt_dmat, rxbuf->pmap, 4257 0, rxbuf->m_pack->m_pkthdr.len, 4258 BUS_DMASYNC_POSTREAD); 4259 ixgbe_dmamap_unload(rxr->ptag, rxbuf->pmap); 4260 rxbuf->m_pack->m_flags |= M_PKTHDR; 4261 m_freem(rxbuf->m_pack); 4262 } 4263 rxbuf->m_head = NULL; 4264 rxbuf->m_pack = NULL; 4265 if (rxbuf->hmap != NULL) { 4266 ixgbe_dmamap_destroy(rxr->htag, rxbuf->hmap); 4267 rxbuf->hmap = NULL; 4268 } 4269 if (rxbuf->pmap != NULL) { 4270 ixgbe_dmamap_destroy(rxr->ptag, rxbuf->pmap); 4271 rxbuf->pmap = NULL; 4272 } 4273 } 4274 if (rxr->rx_buffers != NULL) { 4275 free(rxr->rx_buffers, M_DEVBUF); 4276 rxr->rx_buffers = NULL; 4277 } 4278 } 4279 4280 if (rxr->htag != NULL) { 4281 ixgbe_dma_tag_destroy(rxr->htag); 4282 rxr->htag = NULL; 4283 } 4284 if (rxr->ptag != NULL) { 4285 ixgbe_dma_tag_destroy(rxr->ptag); 4286 rxr->ptag = NULL; 4287 } 4288 4289 return; 4290 } 4291 4292 static __inline void 4293 ixgbe_rx_input(struct rx_ring *rxr, struct ifnet *ifp, struct mbuf *m, u32 ptype) 4294 { 4295 int s; 4296 4297 #ifdef LRO 4298 struct adapter *adapter = ifp->if_softc; 4299 struct ethercom *ec = &adapter->osdep.ec; 4300 4301 /* 4302 * ATM LRO is only for IPv4/TCP packets and TCP checksum of the packet 4303 * should be computed by hardware. Also it should not have VLAN tag in 4304 * ethernet header. 4305 */ 4306 if (rxr->lro_enabled && 4307 (ec->ec_capenable & ETHERCAP_VLAN_HWTAGGING) != 0 && 4308 (ptype & IXGBE_RXDADV_PKTTYPE_ETQF) == 0 && 4309 (ptype & (IXGBE_RXDADV_PKTTYPE_IPV4 | IXGBE_RXDADV_PKTTYPE_TCP)) == 4310 (IXGBE_RXDADV_PKTTYPE_IPV4 | IXGBE_RXDADV_PKTTYPE_TCP) && 4311 (m->m_pkthdr.csum_flags & (CSUM_DATA_VALID | CSUM_PSEUDO_HDR)) == 4312 (CSUM_DATA_VALID | CSUM_PSEUDO_HDR)) { 4313 /* 4314 * Send to the stack if: 4315 ** - LRO not enabled, or 4316 ** - no LRO resources, or 4317 ** - lro enqueue fails 4318 */ 4319 if (rxr->lro.lro_cnt != 0) 4320 if (tcp_lro_rx(&rxr->lro, m, 0) == 0) 4321 return; 4322 } 4323 #endif /* LRO */ 4324 4325 IXGBE_RX_UNLOCK(rxr); 4326 4327 s = splnet(); 4328 /* Pass this up to any BPF listeners. */ 4329 bpf_mtap(ifp, m); 4330 (*ifp->if_input)(ifp, m); 4331 splx(s); 4332 4333 IXGBE_RX_LOCK(rxr); 4334 } 4335 4336 static __inline void 4337 ixgbe_rx_discard(struct rx_ring *rxr, int i) 4338 { 4339 struct ixgbe_rx_buf *rbuf; 4340 4341 rbuf = &rxr->rx_buffers[i]; 4342 4343 if (rbuf->fmp != NULL) {/* Partial chain ? */ 4344 rbuf->fmp->m_flags |= M_PKTHDR; 4345 m_freem(rbuf->fmp); 4346 rbuf->fmp = NULL; 4347 } 4348 4349 /* 4350 ** With advanced descriptors the writeback 4351 ** clobbers the buffer addrs, so its easier 4352 ** to just free the existing mbufs and take 4353 ** the normal refresh path to get new buffers 4354 ** and mapping. 4355 */ 4356 if (rbuf->m_head) { 4357 m_free(rbuf->m_head); 4358 rbuf->m_head = NULL; 4359 } 4360 4361 if (rbuf->m_pack) { 4362 m_free(rbuf->m_pack); 4363 rbuf->m_pack = NULL; 4364 } 4365 4366 return; 4367 } 4368 4369 4370 /********************************************************************* 4371 * 4372 * This routine executes in interrupt context. It replenishes 4373 * the mbufs in the descriptor and sends data which has been 4374 * dma'ed into host memory to upper layer. 4375 * 4376 * We loop at most count times if count is > 0, or until done if 4377 * count < 0. 4378 * 4379 * Return TRUE for more work, FALSE for all clean. 4380 *********************************************************************/ 4381 static bool 4382 ixgbe_rxeof(struct ix_queue *que, int count) 4383 { 4384 struct adapter *adapter = que->adapter; 4385 struct rx_ring *rxr = que->rxr; 4386 struct ifnet *ifp = adapter->ifp; 4387 #ifdef LRO 4388 struct lro_ctrl *lro = &rxr->lro; 4389 struct lro_entry *queued; 4390 #endif /* LRO */ 4391 int i, nextp, processed = 0; 4392 u32 staterr = 0; 4393 union ixgbe_adv_rx_desc *cur; 4394 struct ixgbe_rx_buf *rbuf, *nbuf; 4395 4396 IXGBE_RX_LOCK(rxr); 4397 4398 for (i = rxr->next_to_check; count != 0;) { 4399 struct mbuf *sendmp, *mh, *mp; 4400 u32 rsc, ptype; 4401 u16 hlen, plen, hdr, vtag; 4402 bool eop; 4403 4404 /* Sync the ring. */ 4405 ixgbe_dmamap_sync(rxr->rxdma.dma_tag, rxr->rxdma.dma_map, 4406 BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE); 4407 4408 cur = &rxr->rx_base[i]; 4409 staterr = le32toh(cur->wb.upper.status_error); 4410 4411 if ((staterr & IXGBE_RXD_STAT_DD) == 0) 4412 break; 4413 if ((ifp->if_flags & IFF_RUNNING) == 0) 4414 break; 4415 4416 count--; 4417 sendmp = NULL; 4418 nbuf = NULL; 4419 rsc = 0; 4420 cur->wb.upper.status_error = 0; 4421 rbuf = &rxr->rx_buffers[i]; 4422 mh = rbuf->m_head; 4423 mp = rbuf->m_pack; 4424 4425 plen = le16toh(cur->wb.upper.length); 4426 ptype = le32toh(cur->wb.lower.lo_dword.data) & 4427 IXGBE_RXDADV_PKTTYPE_MASK; 4428 hdr = le16toh(cur->wb.lower.lo_dword.hs_rss.hdr_info); 4429 vtag = le16toh(cur->wb.upper.vlan); 4430 eop = ((staterr & IXGBE_RXD_STAT_EOP) != 0); 4431 4432 /* Make sure bad packets are discarded */ 4433 if (((staterr & IXGBE_RXDADV_ERR_FRAME_ERR_MASK) != 0) || 4434 (rxr->discard)) { 4435 ifp->if_ierrors++; 4436 rxr->rx_discarded.ev_count++; 4437 if (eop) 4438 rxr->discard = FALSE; 4439 else 4440 rxr->discard = TRUE; 4441 ixgbe_rx_discard(rxr, i); 4442 goto next_desc; 4443 } 4444 4445 /* 4446 ** On 82599 which supports a hardware 4447 ** LRO (called HW RSC), packets need 4448 ** not be fragmented across sequential 4449 ** descriptors, rather the next descriptor 4450 ** is indicated in bits of the descriptor. 4451 ** This also means that we might proceses 4452 ** more than one packet at a time, something 4453 ** that has never been true before, it 4454 ** required eliminating global chain pointers 4455 ** in favor of what we are doing here. -jfv 4456 */ 4457 if (!eop) { 4458 /* 4459 ** Figure out the next descriptor 4460 ** of this frame. 4461 */ 4462 if (rxr->hw_rsc == TRUE) { 4463 rsc = ixgbe_rsc_count(cur); 4464 rxr->rsc_num += (rsc - 1); 4465 } 4466 if (rsc) { /* Get hardware index */ 4467 nextp = ((staterr & 4468 IXGBE_RXDADV_NEXTP_MASK) >> 4469 IXGBE_RXDADV_NEXTP_SHIFT); 4470 } else { /* Just sequential */ 4471 nextp = i + 1; 4472 if (nextp == adapter->num_rx_desc) 4473 nextp = 0; 4474 } 4475 nbuf = &rxr->rx_buffers[nextp]; 4476 prefetch(nbuf); 4477 } 4478 /* 4479 ** The header mbuf is ONLY used when header 4480 ** split is enabled, otherwise we get normal 4481 ** behavior, ie, both header and payload 4482 ** are DMA'd into the payload buffer. 4483 ** 4484 ** Rather than using the fmp/lmp global pointers 4485 ** we now keep the head of a packet chain in the 4486 ** buffer struct and pass this along from one 4487 ** descriptor to the next, until we get EOP. 4488 */ 4489 if (rxr->hdr_split && (rbuf->fmp == NULL)) { 4490 /* This must be an initial descriptor */ 4491 hlen = (hdr & IXGBE_RXDADV_HDRBUFLEN_MASK) >> 4492 IXGBE_RXDADV_HDRBUFLEN_SHIFT; 4493 if (hlen > IXGBE_RX_HDR) 4494 hlen = IXGBE_RX_HDR; 4495 mh->m_len = hlen; 4496 mh->m_flags |= M_PKTHDR; 4497 mh->m_next = NULL; 4498 mh->m_pkthdr.len = mh->m_len; 4499 /* Null buf pointer so it is refreshed */ 4500 rbuf->m_head = NULL; 4501 /* 4502 ** Check the payload length, this 4503 ** could be zero if its a small 4504 ** packet. 4505 */ 4506 if (plen > 0) { 4507 mp->m_len = plen; 4508 mp->m_next = NULL; 4509 mp->m_flags &= ~M_PKTHDR; 4510 mh->m_next = mp; 4511 mh->m_pkthdr.len += mp->m_len; 4512 /* Null buf pointer so it is refreshed */ 4513 rbuf->m_pack = NULL; 4514 rxr->rx_split_packets.ev_count++; 4515 } 4516 /* 4517 ** Now create the forward 4518 ** chain so when complete 4519 ** we wont have to. 4520 */ 4521 if (eop == 0) { 4522 /* stash the chain head */ 4523 nbuf->fmp = mh; 4524 /* Make forward chain */ 4525 if (plen) 4526 mp->m_next = nbuf->m_pack; 4527 else 4528 mh->m_next = nbuf->m_pack; 4529 } else { 4530 /* Singlet, prepare to send */ 4531 sendmp = mh; 4532 if (VLAN_ATTACHED(&adapter->osdep.ec) && 4533 (staterr & IXGBE_RXD_STAT_VP)) { 4534 /* XXX Do something reasonable on 4535 * error. 4536 */ 4537 #if 0 4538 printf("%s.%d: VLAN_INPUT_TAG\n", 4539 __func__, __LINE__); 4540 Debugger(); 4541 #endif 4542 VLAN_INPUT_TAG(ifp, sendmp, vtag, 4543 printf("%s: could not apply VLAN " 4544 "tag", __func__)); 4545 } 4546 } 4547 } else { 4548 /* 4549 ** Either no header split, or a 4550 ** secondary piece of a fragmented 4551 ** split packet. 4552 */ 4553 mp->m_len = plen; 4554 /* 4555 ** See if there is a stored head 4556 ** that determines what we are 4557 */ 4558 sendmp = rbuf->fmp; 4559 rbuf->m_pack = rbuf->fmp = NULL; 4560 4561 if (sendmp != NULL) /* secondary frag */ 4562 sendmp->m_pkthdr.len += mp->m_len; 4563 else { 4564 /* first desc of a non-ps chain */ 4565 sendmp = mp; 4566 sendmp->m_flags |= M_PKTHDR; 4567 sendmp->m_pkthdr.len = mp->m_len; 4568 if (staterr & IXGBE_RXD_STAT_VP) { 4569 /* XXX Do something reasonable on 4570 * error. 4571 */ 4572 #if 0 4573 printf("%s.%d: VLAN_INPUT_TAG\n", 4574 __func__, __LINE__); 4575 Debugger(); 4576 #endif 4577 VLAN_INPUT_TAG(ifp, sendmp, vtag, 4578 printf("%s: could not apply VLAN " 4579 "tag", __func__)); 4580 } 4581 } 4582 /* Pass the head pointer on */ 4583 if (eop == 0) { 4584 nbuf->fmp = sendmp; 4585 sendmp = NULL; 4586 mp->m_next = nbuf->m_pack; 4587 } 4588 } 4589 ++processed; 4590 /* Sending this frame? */ 4591 if (eop) { 4592 sendmp->m_pkthdr.rcvif = ifp; 4593 ifp->if_ipackets++; 4594 rxr->rx_packets.ev_count++; 4595 /* capture data for AIM */ 4596 rxr->bytes += sendmp->m_pkthdr.len; 4597 rxr->rx_bytes.ev_count += sendmp->m_pkthdr.len; 4598 if ((ifp->if_capenable & IFCAP_RXCSUM) != 0) { 4599 ixgbe_rx_checksum(staterr, sendmp, ptype, 4600 &adapter->stats); 4601 } 4602 #if __FreeBSD_version >= 800000 4603 sendmp->m_pkthdr.flowid = que->msix; 4604 sendmp->m_flags |= M_FLOWID; 4605 #endif 4606 } 4607 next_desc: 4608 ixgbe_dmamap_sync(rxr->rxdma.dma_tag, rxr->rxdma.dma_map, 4609 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); 4610 4611 /* Advance our pointers to the next descriptor. */ 4612 if (++i == adapter->num_rx_desc) 4613 i = 0; 4614 4615 /* Now send to the stack or do LRO */ 4616 if (sendmp != NULL) { 4617 rxr->next_to_check = i; 4618 ixgbe_rx_input(rxr, ifp, sendmp, ptype); 4619 i = rxr->next_to_check; 4620 } 4621 4622 /* Every 8 descriptors we go to refresh mbufs */ 4623 if (processed == 8) { 4624 ixgbe_refresh_mbufs(rxr, i); 4625 processed = 0; 4626 } 4627 } 4628 4629 /* Refresh any remaining buf structs */ 4630 if (ixgbe_rx_unrefreshed(rxr)) 4631 ixgbe_refresh_mbufs(rxr, i); 4632 4633 rxr->next_to_check = i; 4634 4635 #ifdef LRO 4636 /* 4637 * Flush any outstanding LRO work 4638 */ 4639 while ((queued = SLIST_FIRST(&lro->lro_active)) != NULL) { 4640 SLIST_REMOVE_HEAD(&lro->lro_active, next); 4641 tcp_lro_flush(lro, queued); 4642 } 4643 #endif /* LRO */ 4644 4645 IXGBE_RX_UNLOCK(rxr); 4646 4647 /* 4648 ** We still have cleaning to do? 4649 ** Schedule another interrupt if so. 4650 */ 4651 if ((staterr & IXGBE_RXD_STAT_DD) != 0) { 4652 ixgbe_rearm_queues(adapter, (u64)(1 << que->msix)); 4653 return true; 4654 } 4655 4656 return false; 4657 } 4658 4659 4660 /********************************************************************* 4661 * 4662 * Verify that the hardware indicated that the checksum is valid. 4663 * Inform the stack about the status of checksum so that stack 4664 * doesn't spend time verifying the checksum. 4665 * 4666 *********************************************************************/ 4667 static void 4668 ixgbe_rx_checksum(u32 staterr, struct mbuf * mp, u32 ptype, 4669 struct ixgbe_hw_stats *stats) 4670 { 4671 u16 status = (u16) staterr; 4672 u8 errors = (u8) (staterr >> 24); 4673 #if 0 4674 bool sctp = FALSE; 4675 4676 if ((ptype & IXGBE_RXDADV_PKTTYPE_ETQF) == 0 && 4677 (ptype & IXGBE_RXDADV_PKTTYPE_SCTP) != 0) 4678 sctp = TRUE; 4679 #endif 4680 4681 if (status & IXGBE_RXD_STAT_IPCS) { 4682 stats->ipcs.ev_count++; 4683 if (!(errors & IXGBE_RXD_ERR_IPE)) { 4684 /* IP Checksum Good */ 4685 mp->m_pkthdr.csum_flags = M_CSUM_IPv4; 4686 4687 } else { 4688 stats->ipcs_bad.ev_count++; 4689 mp->m_pkthdr.csum_flags = M_CSUM_IPv4|M_CSUM_IPv4_BAD; 4690 } 4691 } 4692 if (status & IXGBE_RXD_STAT_L4CS) { 4693 stats->l4cs.ev_count++; 4694 u16 type = M_CSUM_TCPv4|M_CSUM_TCPv6|M_CSUM_UDPv4|M_CSUM_UDPv6; 4695 if (!(errors & IXGBE_RXD_ERR_TCPE)) { 4696 mp->m_pkthdr.csum_flags |= type; 4697 } else { 4698 stats->l4cs_bad.ev_count++; 4699 mp->m_pkthdr.csum_flags |= type | M_CSUM_TCP_UDP_BAD; 4700 } 4701 } 4702 return; 4703 } 4704 4705 4706 #if 0 /* XXX Badly need to overhaul vlan(4) on NetBSD. */ 4707 /* 4708 ** This routine is run via an vlan config EVENT, 4709 ** it enables us to use the HW Filter table since 4710 ** we can get the vlan id. This just creates the 4711 ** entry in the soft version of the VFTA, init will 4712 ** repopulate the real table. 4713 */ 4714 static void 4715 ixgbe_register_vlan(void *arg, struct ifnet *ifp, u16 vtag) 4716 { 4717 struct adapter *adapter = ifp->if_softc; 4718 u16 index, bit; 4719 4720 if (ifp->if_softc != arg) /* Not our event */ 4721 return; 4722 4723 if ((vtag == 0) || (vtag > 4095)) /* Invalid */ 4724 return; 4725 4726 IXGBE_CORE_LOCK(adapter); 4727 index = (vtag >> 5) & 0x7F; 4728 bit = vtag & 0x1F; 4729 adapter->shadow_vfta[index] |= (1 << bit); 4730 ixgbe_init_locked(adapter); 4731 IXGBE_CORE_UNLOCK(adapter); 4732 } 4733 4734 /* 4735 ** This routine is run via an vlan 4736 ** unconfig EVENT, remove our entry 4737 ** in the soft vfta. 4738 */ 4739 static void 4740 ixgbe_unregister_vlan(void *arg, struct ifnet *ifp, u16 vtag) 4741 { 4742 struct adapter *adapter = ifp->if_softc; 4743 u16 index, bit; 4744 4745 if (ifp->if_softc != arg) 4746 return; 4747 4748 if ((vtag == 0) || (vtag > 4095)) /* Invalid */ 4749 return; 4750 4751 IXGBE_CORE_LOCK(adapter); 4752 index = (vtag >> 5) & 0x7F; 4753 bit = vtag & 0x1F; 4754 adapter->shadow_vfta[index] &= ~(1 << bit); 4755 /* Re-init to load the changes */ 4756 ixgbe_init_locked(adapter); 4757 IXGBE_CORE_UNLOCK(adapter); 4758 } 4759 #endif 4760 4761 static void 4762 ixgbe_setup_vlan_hw_support(struct adapter *adapter) 4763 { 4764 struct ethercom *ec = &adapter->osdep.ec; 4765 struct ixgbe_hw *hw = &adapter->hw; 4766 u32 ctrl; 4767 4768 /* 4769 ** We get here thru init_locked, meaning 4770 ** a soft reset, this has already cleared 4771 ** the VFTA and other state, so if there 4772 ** have been no vlan's registered do nothing. 4773 */ 4774 if (!VLAN_ATTACHED(&adapter->osdep.ec)) { 4775 return; 4776 } 4777 4778 /* 4779 ** A soft reset zero's out the VFTA, so 4780 ** we need to repopulate it now. 4781 */ 4782 for (int i = 0; i < IXGBE_VFTA_SIZE; i++) 4783 if (adapter->shadow_vfta[i] != 0) 4784 IXGBE_WRITE_REG(hw, IXGBE_VFTA(i), 4785 adapter->shadow_vfta[i]); 4786 4787 ctrl = IXGBE_READ_REG(hw, IXGBE_VLNCTRL); 4788 /* Enable the Filter Table if enabled */ 4789 if (ec->ec_capenable & ETHERCAP_VLAN_HWFILTER) { 4790 ctrl &= ~IXGBE_VLNCTRL_CFIEN; 4791 ctrl |= IXGBE_VLNCTRL_VFE; 4792 } 4793 if (hw->mac.type == ixgbe_mac_82598EB) 4794 ctrl |= IXGBE_VLNCTRL_VME; 4795 IXGBE_WRITE_REG(hw, IXGBE_VLNCTRL, ctrl); 4796 4797 /* On 82599 the VLAN enable is per/queue in RXDCTL */ 4798 if (hw->mac.type != ixgbe_mac_82598EB) 4799 for (int i = 0; i < adapter->num_queues; i++) { 4800 ctrl = IXGBE_READ_REG(hw, IXGBE_RXDCTL(i)); 4801 ctrl |= IXGBE_RXDCTL_VME; 4802 IXGBE_WRITE_REG(hw, IXGBE_RXDCTL(i), ctrl); 4803 } 4804 } 4805 4806 static void 4807 ixgbe_enable_intr(struct adapter *adapter) 4808 { 4809 struct ixgbe_hw *hw = &adapter->hw; 4810 struct ix_queue *que = adapter->queues; 4811 u32 mask = (IXGBE_EIMS_ENABLE_MASK & ~IXGBE_EIMS_RTX_QUEUE); 4812 4813 4814 /* Enable Fan Failure detection */ 4815 if (hw->device_id == IXGBE_DEV_ID_82598AT) 4816 mask |= IXGBE_EIMS_GPI_SDP1; 4817 else { 4818 mask |= IXGBE_EIMS_ECC; 4819 mask |= IXGBE_EIMS_GPI_SDP1; 4820 mask |= IXGBE_EIMS_GPI_SDP2; 4821 #ifdef IXGBE_FDIR 4822 mask |= IXGBE_EIMS_FLOW_DIR; 4823 #endif 4824 } 4825 4826 IXGBE_WRITE_REG(hw, IXGBE_EIMS, mask); 4827 4828 /* With RSS we use auto clear */ 4829 if (adapter->msix_mem) { 4830 mask = IXGBE_EIMS_ENABLE_MASK; 4831 /* Don't autoclear Link */ 4832 mask &= ~IXGBE_EIMS_OTHER; 4833 mask &= ~IXGBE_EIMS_LSC; 4834 IXGBE_WRITE_REG(hw, IXGBE_EIAC, mask); 4835 } 4836 4837 /* 4838 ** Now enable all queues, this is done separately to 4839 ** allow for handling the extended (beyond 32) MSIX 4840 ** vectors that can be used by 82599 4841 */ 4842 for (int i = 0; i < adapter->num_queues; i++, que++) 4843 ixgbe_enable_queue(adapter, que->msix); 4844 4845 IXGBE_WRITE_FLUSH(hw); 4846 4847 return; 4848 } 4849 4850 static void 4851 ixgbe_disable_intr(struct adapter *adapter) 4852 { 4853 if (adapter->msix_mem) 4854 IXGBE_WRITE_REG(&adapter->hw, IXGBE_EIAC, 0); 4855 if (adapter->hw.mac.type == ixgbe_mac_82598EB) { 4856 IXGBE_WRITE_REG(&adapter->hw, IXGBE_EIMC, ~0); 4857 } else { 4858 IXGBE_WRITE_REG(&adapter->hw, IXGBE_EIMC, 0xFFFF0000); 4859 IXGBE_WRITE_REG(&adapter->hw, IXGBE_EIMC_EX(0), ~0); 4860 IXGBE_WRITE_REG(&adapter->hw, IXGBE_EIMC_EX(1), ~0); 4861 } 4862 IXGBE_WRITE_FLUSH(&adapter->hw); 4863 return; 4864 } 4865 4866 u16 4867 ixgbe_read_pci_cfg(struct ixgbe_hw *hw, u32 reg) 4868 { 4869 switch (reg % 4) { 4870 case 0: 4871 return pci_conf_read(hw->back->pc, hw->back->tag, reg) & 4872 __BITS(15, 0); 4873 case 2: 4874 return __SHIFTOUT(pci_conf_read(hw->back->pc, hw->back->tag, 4875 reg - 2), __BITS(31, 16)); 4876 default: 4877 panic("%s: invalid register (%" PRIx32, __func__, reg); 4878 break; 4879 } 4880 } 4881 4882 void 4883 ixgbe_write_pci_cfg(struct ixgbe_hw *hw, u32 reg, u16 value) 4884 { 4885 pcireg_t old; 4886 4887 switch (reg % 4) { 4888 case 0: 4889 old = pci_conf_read(hw->back->pc, hw->back->tag, reg) & 4890 __BITS(31, 16); 4891 pci_conf_write(hw->back->pc, hw->back->tag, reg, value | old); 4892 break; 4893 case 2: 4894 old = pci_conf_read(hw->back->pc, hw->back->tag, reg - 2) & 4895 __BITS(15, 0); 4896 pci_conf_write(hw->back->pc, hw->back->tag, reg - 2, 4897 __SHIFTIN(value, __BITS(31, 16)) | old); 4898 break; 4899 default: 4900 panic("%s: invalid register (%" PRIx32, __func__, reg); 4901 break; 4902 } 4903 4904 return; 4905 } 4906 4907 /* 4908 ** Setup the correct IVAR register for a particular MSIX interrupt 4909 ** (yes this is all very magic and confusing :) 4910 ** - entry is the register array entry 4911 ** - vector is the MSIX vector for this queue 4912 ** - type is RX/TX/MISC 4913 */ 4914 static void 4915 ixgbe_set_ivar(struct adapter *adapter, u8 entry, u8 vector, s8 type) 4916 { 4917 struct ixgbe_hw *hw = &adapter->hw; 4918 u32 ivar, index; 4919 4920 vector |= IXGBE_IVAR_ALLOC_VAL; 4921 4922 switch (hw->mac.type) { 4923 4924 case ixgbe_mac_82598EB: 4925 if (type == -1) 4926 entry = IXGBE_IVAR_OTHER_CAUSES_INDEX; 4927 else 4928 entry += (type * 64); 4929 index = (entry >> 2) & 0x1F; 4930 ivar = IXGBE_READ_REG(hw, IXGBE_IVAR(index)); 4931 ivar &= ~(0xFF << (8 * (entry & 0x3))); 4932 ivar |= (vector << (8 * (entry & 0x3))); 4933 IXGBE_WRITE_REG(&adapter->hw, IXGBE_IVAR(index), ivar); 4934 break; 4935 4936 case ixgbe_mac_82599EB: 4937 if (type == -1) { /* MISC IVAR */ 4938 index = (entry & 1) * 8; 4939 ivar = IXGBE_READ_REG(hw, IXGBE_IVAR_MISC); 4940 ivar &= ~(0xFF << index); 4941 ivar |= (vector << index); 4942 IXGBE_WRITE_REG(hw, IXGBE_IVAR_MISC, ivar); 4943 } else { /* RX/TX IVARS */ 4944 index = (16 * (entry & 1)) + (8 * type); 4945 ivar = IXGBE_READ_REG(hw, IXGBE_IVAR(entry >> 1)); 4946 ivar &= ~(0xFF << index); 4947 ivar |= (vector << index); 4948 IXGBE_WRITE_REG(hw, IXGBE_IVAR(entry >> 1), ivar); 4949 } 4950 4951 default: 4952 break; 4953 } 4954 } 4955 4956 static void 4957 ixgbe_configure_ivars(struct adapter *adapter) 4958 { 4959 struct ix_queue *que = adapter->queues; 4960 u32 newitr; 4961 4962 if (ixgbe_max_interrupt_rate > 0) 4963 newitr = (8000000 / ixgbe_max_interrupt_rate) & 0x0FF8; 4964 else 4965 newitr = 0; 4966 4967 for (int i = 0; i < adapter->num_queues; i++, que++) { 4968 /* First the RX queue entry */ 4969 ixgbe_set_ivar(adapter, i, que->msix, 0); 4970 /* ... and the TX */ 4971 ixgbe_set_ivar(adapter, i, que->msix, 1); 4972 /* Set an Initial EITR value */ 4973 IXGBE_WRITE_REG(&adapter->hw, 4974 IXGBE_EITR(que->msix), newitr); 4975 } 4976 4977 /* For the Link interrupt */ 4978 ixgbe_set_ivar(adapter, 1, adapter->linkvec, -1); 4979 } 4980 4981 /* 4982 ** ixgbe_sfp_probe - called in the local timer to 4983 ** determine if a port had optics inserted. 4984 */ 4985 static bool ixgbe_sfp_probe(struct adapter *adapter) 4986 { 4987 struct ixgbe_hw *hw = &adapter->hw; 4988 device_t dev = adapter->dev; 4989 bool result = FALSE; 4990 4991 if ((hw->phy.type == ixgbe_phy_nl) && 4992 (hw->phy.sfp_type == ixgbe_sfp_type_not_present)) { 4993 s32 ret = hw->phy.ops.identify_sfp(hw); 4994 if (ret) 4995 goto out; 4996 ret = hw->phy.ops.reset(hw); 4997 if (ret == IXGBE_ERR_SFP_NOT_SUPPORTED) { 4998 device_printf(dev,"Unsupported SFP+ module detected!"); 4999 device_printf(dev, "Reload driver with supported module.\n"); 5000 adapter->sfp_probe = FALSE; 5001 goto out; 5002 } else 5003 device_printf(dev,"SFP+ module detected!\n"); 5004 /* We now have supported optics */ 5005 adapter->sfp_probe = FALSE; 5006 /* Set the optics type so system reports correctly */ 5007 ixgbe_setup_optics(adapter); 5008 result = TRUE; 5009 } 5010 out: 5011 return (result); 5012 } 5013 5014 /* 5015 ** Tasklet handler for MSIX Link interrupts 5016 ** - do outside interrupt since it might sleep 5017 */ 5018 static void 5019 ixgbe_handle_link(void *context) 5020 { 5021 struct adapter *adapter = context; 5022 5023 ixgbe_check_link(&adapter->hw, 5024 &adapter->link_speed, &adapter->link_up, 0); 5025 ixgbe_update_link_status(adapter); 5026 } 5027 5028 /* 5029 ** Tasklet for handling SFP module interrupts 5030 */ 5031 static void 5032 ixgbe_handle_mod(void *context) 5033 { 5034 struct adapter *adapter = context; 5035 struct ixgbe_hw *hw = &adapter->hw; 5036 device_t dev = adapter->dev; 5037 u32 err; 5038 5039 err = hw->phy.ops.identify_sfp(hw); 5040 if (err == IXGBE_ERR_SFP_NOT_SUPPORTED) { 5041 device_printf(dev, 5042 "Unsupported SFP+ module type was detected.\n"); 5043 return; 5044 } 5045 err = hw->mac.ops.setup_sfp(hw); 5046 if (err == IXGBE_ERR_SFP_NOT_SUPPORTED) { 5047 device_printf(dev, 5048 "Setup failure - unsupported SFP+ module type.\n"); 5049 return; 5050 } 5051 softint_schedule(adapter->msf_si); 5052 return; 5053 } 5054 5055 5056 /* 5057 ** Tasklet for handling MSF (multispeed fiber) interrupts 5058 */ 5059 static void 5060 ixgbe_handle_msf(void *context) 5061 { 5062 struct adapter *adapter = context; 5063 struct ixgbe_hw *hw = &adapter->hw; 5064 u32 autoneg; 5065 bool negotiate; 5066 5067 autoneg = hw->phy.autoneg_advertised; 5068 if ((!autoneg) && (hw->mac.ops.get_link_capabilities)) 5069 hw->mac.ops.get_link_capabilities(hw, &autoneg, &negotiate); 5070 if (hw->mac.ops.setup_link) 5071 hw->mac.ops.setup_link(hw, autoneg, negotiate, TRUE); 5072 return; 5073 } 5074 5075 #ifdef IXGBE_FDIR 5076 /* 5077 ** Tasklet for reinitializing the Flow Director filter table 5078 */ 5079 static void 5080 ixgbe_reinit_fdir(void *context) 5081 { 5082 struct adapter *adapter = context; 5083 struct ifnet *ifp = adapter->ifp; 5084 5085 if (adapter->fdir_reinit != 1) /* Shouldn't happen */ 5086 return; 5087 ixgbe_reinit_fdir_tables_82599(&adapter->hw); 5088 adapter->fdir_reinit = 0; 5089 /* Restart the interface */ 5090 ifp->if_flags |= IFF_RUNNING; 5091 return; 5092 } 5093 #endif 5094 5095 /********************************************************************** 5096 * 5097 * Update the board statistics counters. 5098 * 5099 **********************************************************************/ 5100 static void 5101 ixgbe_update_stats_counters(struct adapter *adapter) 5102 { 5103 struct ifnet *ifp = adapter->ifp; 5104 struct ixgbe_hw *hw = &adapter->hw; 5105 u32 missed_rx = 0, bprc, lxon, lxoff, total; 5106 u64 total_missed_rx = 0; 5107 5108 adapter->stats.crcerrs.ev_count += IXGBE_READ_REG(hw, IXGBE_CRCERRS); 5109 adapter->stats.illerrc.ev_count += IXGBE_READ_REG(hw, IXGBE_ILLERRC); 5110 adapter->stats.errbc.ev_count += IXGBE_READ_REG(hw, IXGBE_ERRBC); 5111 adapter->stats.mspdc.ev_count += IXGBE_READ_REG(hw, IXGBE_MSPDC); 5112 5113 for (int i = 0; i < __arraycount(adapter->stats.mpc); i++) { 5114 int j = i % adapter->num_queues; 5115 u32 mp; 5116 mp = IXGBE_READ_REG(hw, IXGBE_MPC(i)); 5117 /* missed_rx tallies misses for the gprc workaround */ 5118 missed_rx += mp; 5119 /* global total per queue */ 5120 adapter->stats.mpc[j].ev_count += mp; 5121 /* Running comprehensive total for stats display */ 5122 total_missed_rx += adapter->stats.mpc[j].ev_count; 5123 if (hw->mac.type == ixgbe_mac_82598EB) 5124 adapter->stats.rnbc[j] += 5125 IXGBE_READ_REG(hw, IXGBE_RNBC(i)); 5126 adapter->stats.pxontxc[j].ev_count += 5127 IXGBE_READ_REG(hw, IXGBE_PXONTXC(i)); 5128 adapter->stats.pxonrxc[j].ev_count += 5129 IXGBE_READ_REG(hw, IXGBE_PXONRXC(i)); 5130 adapter->stats.pxofftxc[j].ev_count += 5131 IXGBE_READ_REG(hw, IXGBE_PXOFFTXC(i)); 5132 adapter->stats.pxoffrxc[j].ev_count += 5133 IXGBE_READ_REG(hw, IXGBE_PXOFFRXC(i)); 5134 adapter->stats.pxon2offc[j].ev_count += 5135 IXGBE_READ_REG(hw, IXGBE_PXON2OFFCNT(i)); 5136 } 5137 for (int i = 0; i < __arraycount(adapter->stats.qprc); i++) { 5138 int j = i % adapter->num_queues; 5139 adapter->stats.qprc[j].ev_count += IXGBE_READ_REG(hw, IXGBE_QPRC(i)); 5140 adapter->stats.qptc[j].ev_count += IXGBE_READ_REG(hw, IXGBE_QPTC(i)); 5141 adapter->stats.qbrc[j].ev_count += IXGBE_READ_REG(hw, IXGBE_QBRC(i)); 5142 adapter->stats.qbrc[j].ev_count += 5143 ((u64)IXGBE_READ_REG(hw, IXGBE_QBRC(i)) << 32); 5144 adapter->stats.qbtc[j].ev_count += IXGBE_READ_REG(hw, IXGBE_QBTC(i)); 5145 adapter->stats.qbtc[j].ev_count += 5146 ((u64)IXGBE_READ_REG(hw, IXGBE_QBTC(i)) << 32); 5147 adapter->stats.qprdc[j].ev_count += IXGBE_READ_REG(hw, IXGBE_QPRDC(i)); 5148 } 5149 adapter->stats.mlfc.ev_count += IXGBE_READ_REG(hw, IXGBE_MLFC); 5150 adapter->stats.mrfc.ev_count += IXGBE_READ_REG(hw, IXGBE_MRFC); 5151 adapter->stats.rlec.ev_count += IXGBE_READ_REG(hw, IXGBE_RLEC); 5152 5153 /* Hardware workaround, gprc counts missed packets */ 5154 adapter->stats.gprc.ev_count += IXGBE_READ_REG(hw, IXGBE_GPRC) - missed_rx; 5155 5156 lxon = IXGBE_READ_REG(hw, IXGBE_LXONTXC); 5157 adapter->stats.lxontxc.ev_count += lxon; 5158 lxoff = IXGBE_READ_REG(hw, IXGBE_LXOFFTXC); 5159 adapter->stats.lxofftxc.ev_count += lxoff; 5160 total = lxon + lxoff; 5161 5162 if (hw->mac.type != ixgbe_mac_82598EB) { 5163 adapter->stats.gorc.ev_count += IXGBE_READ_REG(hw, IXGBE_GORCL) + 5164 ((u64)IXGBE_READ_REG(hw, IXGBE_GORCH) << 32); 5165 adapter->stats.gotc.ev_count += IXGBE_READ_REG(hw, IXGBE_GOTCL) + 5166 ((u64)IXGBE_READ_REG(hw, IXGBE_GOTCH) << 32) - total * ETHER_MIN_LEN; 5167 adapter->stats.tor.ev_count += IXGBE_READ_REG(hw, IXGBE_TORL) + 5168 ((u64)IXGBE_READ_REG(hw, IXGBE_TORH) << 32); 5169 adapter->stats.lxonrxc.ev_count += IXGBE_READ_REG(hw, IXGBE_LXONRXCNT); 5170 adapter->stats.lxoffrxc.ev_count += IXGBE_READ_REG(hw, IXGBE_LXOFFRXCNT); 5171 } else { 5172 adapter->stats.lxonrxc.ev_count += IXGBE_READ_REG(hw, IXGBE_LXONRXC); 5173 adapter->stats.lxoffrxc.ev_count += IXGBE_READ_REG(hw, IXGBE_LXOFFRXC); 5174 /* 82598 only has a counter in the high register */ 5175 adapter->stats.gorc.ev_count += IXGBE_READ_REG(hw, IXGBE_GORCH); 5176 adapter->stats.gotc.ev_count += IXGBE_READ_REG(hw, IXGBE_GOTCH) - total * ETHER_MIN_LEN; 5177 adapter->stats.tor.ev_count += IXGBE_READ_REG(hw, IXGBE_TORH); 5178 } 5179 5180 /* 5181 * Workaround: mprc hardware is incorrectly counting 5182 * broadcasts, so for now we subtract those. 5183 */ 5184 bprc = IXGBE_READ_REG(hw, IXGBE_BPRC); 5185 adapter->stats.bprc.ev_count += bprc; 5186 adapter->stats.mprc.ev_count += IXGBE_READ_REG(hw, IXGBE_MPRC) - ((hw->mac.type == ixgbe_mac_82598EB) ? bprc : 0); 5187 5188 adapter->stats.prc64.ev_count += IXGBE_READ_REG(hw, IXGBE_PRC64); 5189 adapter->stats.prc127.ev_count += IXGBE_READ_REG(hw, IXGBE_PRC127); 5190 adapter->stats.prc255.ev_count += IXGBE_READ_REG(hw, IXGBE_PRC255); 5191 adapter->stats.prc511.ev_count += IXGBE_READ_REG(hw, IXGBE_PRC511); 5192 adapter->stats.prc1023.ev_count += IXGBE_READ_REG(hw, IXGBE_PRC1023); 5193 adapter->stats.prc1522.ev_count += IXGBE_READ_REG(hw, IXGBE_PRC1522); 5194 5195 adapter->stats.gptc.ev_count += IXGBE_READ_REG(hw, IXGBE_GPTC) - total; 5196 adapter->stats.mptc.ev_count += IXGBE_READ_REG(hw, IXGBE_MPTC) - total; 5197 adapter->stats.ptc64.ev_count += IXGBE_READ_REG(hw, IXGBE_PTC64) - total; 5198 5199 adapter->stats.ruc.ev_count += IXGBE_READ_REG(hw, IXGBE_RUC); 5200 adapter->stats.rfc.ev_count += IXGBE_READ_REG(hw, IXGBE_RFC); 5201 adapter->stats.roc.ev_count += IXGBE_READ_REG(hw, IXGBE_ROC); 5202 adapter->stats.rjc.ev_count += IXGBE_READ_REG(hw, IXGBE_RJC); 5203 adapter->stats.mngprc.ev_count += IXGBE_READ_REG(hw, IXGBE_MNGPRC); 5204 adapter->stats.mngpdc.ev_count += IXGBE_READ_REG(hw, IXGBE_MNGPDC); 5205 adapter->stats.mngptc.ev_count += IXGBE_READ_REG(hw, IXGBE_MNGPTC); 5206 adapter->stats.tpr.ev_count += IXGBE_READ_REG(hw, IXGBE_TPR); 5207 adapter->stats.tpt.ev_count += IXGBE_READ_REG(hw, IXGBE_TPT); 5208 adapter->stats.ptc127.ev_count += IXGBE_READ_REG(hw, IXGBE_PTC127); 5209 adapter->stats.ptc255.ev_count += IXGBE_READ_REG(hw, IXGBE_PTC255); 5210 adapter->stats.ptc511.ev_count += IXGBE_READ_REG(hw, IXGBE_PTC511); 5211 adapter->stats.ptc1023.ev_count += IXGBE_READ_REG(hw, IXGBE_PTC1023); 5212 adapter->stats.ptc1522.ev_count += IXGBE_READ_REG(hw, IXGBE_PTC1522); 5213 adapter->stats.bptc.ev_count += IXGBE_READ_REG(hw, IXGBE_BPTC); 5214 adapter->stats.xec.ev_count += IXGBE_READ_REG(hw, IXGBE_XEC); 5215 adapter->stats.fccrc.ev_count += IXGBE_READ_REG(hw, IXGBE_FCCRC); 5216 adapter->stats.fclast.ev_count += IXGBE_READ_REG(hw, IXGBE_FCLAST); 5217 5218 /* Only read FCOE on 82599 */ 5219 if (hw->mac.type != ixgbe_mac_82598EB) { 5220 adapter->stats.fcoerpdc.ev_count += 5221 IXGBE_READ_REG(hw, IXGBE_FCOERPDC); 5222 adapter->stats.fcoeprc.ev_count += 5223 IXGBE_READ_REG(hw, IXGBE_FCOEPRC); 5224 adapter->stats.fcoeptc.ev_count += 5225 IXGBE_READ_REG(hw, IXGBE_FCOEPTC); 5226 adapter->stats.fcoedwrc.ev_count += 5227 IXGBE_READ_REG(hw, IXGBE_FCOEDWRC); 5228 adapter->stats.fcoedwtc.ev_count += 5229 IXGBE_READ_REG(hw, IXGBE_FCOEDWTC); 5230 } 5231 5232 /* Fill out the OS statistics structure */ 5233 ifp->if_ipackets = adapter->stats.gprc.ev_count; 5234 ifp->if_opackets = adapter->stats.gptc.ev_count; 5235 ifp->if_ibytes = adapter->stats.gorc.ev_count; 5236 ifp->if_obytes = adapter->stats.gotc.ev_count; 5237 ifp->if_imcasts = adapter->stats.mprc.ev_count; 5238 ifp->if_collisions = 0; 5239 5240 /* Rx Errors */ 5241 ifp->if_ierrors = total_missed_rx + adapter->stats.crcerrs.ev_count + 5242 adapter->stats.rlec.ev_count; 5243 } 5244 5245 /** ixgbe_sysctl_tdh_handler - Handler function 5246 * Retrieves the TDH value from the hardware 5247 */ 5248 static int 5249 ixgbe_sysctl_tdh_handler(SYSCTLFN_ARGS) 5250 { 5251 struct sysctlnode node; 5252 uint32_t val; 5253 struct tx_ring *txr; 5254 5255 node = *rnode; 5256 txr = (struct tx_ring *)node.sysctl_data; 5257 if (txr == NULL) 5258 return 0; 5259 val = IXGBE_READ_REG(&txr->adapter->hw, IXGBE_TDH(txr->me)); 5260 node.sysctl_data = &val; 5261 return sysctl_lookup(SYSCTLFN_CALL(&node)); 5262 } 5263 5264 /** ixgbe_sysctl_tdt_handler - Handler function 5265 * Retrieves the TDT value from the hardware 5266 */ 5267 static int 5268 ixgbe_sysctl_tdt_handler(SYSCTLFN_ARGS) 5269 { 5270 struct sysctlnode node; 5271 uint32_t val; 5272 struct tx_ring *txr; 5273 5274 node = *rnode; 5275 txr = (struct tx_ring *)node.sysctl_data; 5276 if (txr == NULL) 5277 return 0; 5278 val = IXGBE_READ_REG(&txr->adapter->hw, IXGBE_TDT(txr->me)); 5279 node.sysctl_data = &val; 5280 return sysctl_lookup(SYSCTLFN_CALL(&node)); 5281 } 5282 5283 /** ixgbe_sysctl_rdh_handler - Handler function 5284 * Retrieves the RDH value from the hardware 5285 */ 5286 static int 5287 ixgbe_sysctl_rdh_handler(SYSCTLFN_ARGS) 5288 { 5289 struct sysctlnode node; 5290 uint32_t val; 5291 struct rx_ring *rxr; 5292 5293 node = *rnode; 5294 rxr = (struct rx_ring *)node.sysctl_data; 5295 if (rxr == NULL) 5296 return 0; 5297 val = IXGBE_READ_REG(&rxr->adapter->hw, IXGBE_RDH(rxr->me)); 5298 node.sysctl_data = &val; 5299 return sysctl_lookup(SYSCTLFN_CALL(&node)); 5300 } 5301 5302 /** ixgbe_sysctl_rdt_handler - Handler function 5303 * Retrieves the RDT value from the hardware 5304 */ 5305 static int 5306 ixgbe_sysctl_rdt_handler(SYSCTLFN_ARGS) 5307 { 5308 struct sysctlnode node; 5309 uint32_t val; 5310 struct rx_ring *rxr; 5311 5312 node = *rnode; 5313 rxr = (struct rx_ring *)node.sysctl_data; 5314 if (rxr == NULL) 5315 return 0; 5316 val = IXGBE_READ_REG(&rxr->adapter->hw, IXGBE_RDT(rxr->me)); 5317 node.sysctl_data = &val; 5318 return sysctl_lookup(SYSCTLFN_CALL(&node)); 5319 } 5320 5321 static int 5322 ixgbe_sysctl_interrupt_rate_handler(SYSCTLFN_ARGS) 5323 { 5324 struct sysctlnode node; 5325 struct ix_queue *que; 5326 uint32_t reg, usec, rate; 5327 5328 node = *rnode; 5329 que = (struct ix_queue *)node.sysctl_data; 5330 if (que == NULL) 5331 return 0; 5332 reg = IXGBE_READ_REG(&que->adapter->hw, IXGBE_EITR(que->msix)); 5333 usec = ((reg & 0x0FF8) >> 3); 5334 if (usec > 0) 5335 rate = 1000000 / usec; 5336 else 5337 rate = 0; 5338 node.sysctl_data = &rate; 5339 return sysctl_lookup(SYSCTLFN_CALL(&node)); 5340 } 5341 5342 const struct sysctlnode * 5343 ixgbe_sysctl_instance(struct adapter *adapter) 5344 { 5345 const char *dvname; 5346 struct sysctllog **log; 5347 int rc; 5348 const struct sysctlnode *rnode; 5349 5350 log = &adapter->sysctllog; 5351 dvname = device_xname(adapter->dev); 5352 5353 if ((rc = sysctl_createv(log, 0, NULL, &rnode, 5354 0, CTLTYPE_NODE, dvname, 5355 SYSCTL_DESCR("ixgbe information and settings"), 5356 NULL, 0, NULL, 0, CTL_HW, CTL_CREATE, CTL_EOL)) != 0) 5357 goto err; 5358 5359 return rnode; 5360 err: 5361 printf("%s: sysctl_createv failed, rc = %d\n", __func__, rc); 5362 return NULL; 5363 } 5364 5365 /* 5366 * Add sysctl variables, one per statistic, to the system. 5367 */ 5368 static void 5369 ixgbe_add_hw_stats(struct adapter *adapter) 5370 { 5371 device_t dev = adapter->dev; 5372 const struct sysctlnode *rnode, *cnode; 5373 struct sysctllog **log = &adapter->sysctllog; 5374 struct tx_ring *txr = adapter->tx_rings; 5375 struct rx_ring *rxr = adapter->rx_rings; 5376 struct ixgbe_hw *hw = &adapter->hw; 5377 5378 struct ixgbe_hw_stats *stats = &adapter->stats; 5379 5380 /* Driver Statistics */ 5381 #if 0 5382 /* These counters are not updated by the software */ 5383 SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "dropped", 5384 CTLFLAG_RD, &adapter->dropped_pkts, 5385 "Driver dropped packets"); 5386 SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "mbuf_header_failed", 5387 CTLFLAG_RD, &adapter->mbuf_header_failed, 5388 "???"); 5389 SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "mbuf_packet_failed", 5390 CTLFLAG_RD, &adapter->mbuf_packet_failed, 5391 "???"); 5392 SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "no_tx_map_avail", 5393 CTLFLAG_RD, &adapter->no_tx_map_avail, 5394 "???"); 5395 #endif 5396 evcnt_attach_dynamic(&adapter->handleq, EVCNT_TYPE_MISC, 5397 NULL, device_xname(dev), "Handled queue in softint"); 5398 evcnt_attach_dynamic(&adapter->req, EVCNT_TYPE_MISC, 5399 NULL, device_xname(dev), "Requeued in softint"); 5400 evcnt_attach_dynamic(&adapter->morerx, EVCNT_TYPE_MISC, 5401 NULL, device_xname(dev), "Interrupt handler more rx"); 5402 evcnt_attach_dynamic(&adapter->moretx, EVCNT_TYPE_MISC, 5403 NULL, device_xname(dev), "Interrupt handler more tx"); 5404 evcnt_attach_dynamic(&adapter->txloops, EVCNT_TYPE_MISC, 5405 NULL, device_xname(dev), "Interrupt handler tx loops"); 5406 evcnt_attach_dynamic(&adapter->efbig_tx_dma_setup, EVCNT_TYPE_MISC, 5407 NULL, device_xname(dev), "Driver tx dma soft fail EFBIG"); 5408 evcnt_attach_dynamic(&adapter->m_defrag_failed, EVCNT_TYPE_MISC, 5409 NULL, device_xname(dev), "m_defrag() failed"); 5410 evcnt_attach_dynamic(&adapter->efbig2_tx_dma_setup, EVCNT_TYPE_MISC, 5411 NULL, device_xname(dev), "Driver tx dma hard fail EFBIG"); 5412 evcnt_attach_dynamic(&adapter->einval_tx_dma_setup, EVCNT_TYPE_MISC, 5413 NULL, device_xname(dev), "Driver tx dma hard fail EINVAL"); 5414 evcnt_attach_dynamic(&adapter->other_tx_dma_setup, EVCNT_TYPE_MISC, 5415 NULL, device_xname(dev), "Driver tx dma hard fail other"); 5416 evcnt_attach_dynamic(&adapter->eagain_tx_dma_setup, EVCNT_TYPE_MISC, 5417 NULL, device_xname(dev), "Driver tx dma soft fail EAGAIN"); 5418 evcnt_attach_dynamic(&adapter->enomem_tx_dma_setup, EVCNT_TYPE_MISC, 5419 NULL, device_xname(dev), "Driver tx dma soft fail ENOMEM"); 5420 evcnt_attach_dynamic(&adapter->watchdog_events, EVCNT_TYPE_MISC, 5421 NULL, device_xname(dev), "Watchdog timeouts"); 5422 evcnt_attach_dynamic(&adapter->tso_err, EVCNT_TYPE_MISC, 5423 NULL, device_xname(dev), "TSO errors"); 5424 evcnt_attach_dynamic(&adapter->tso_tx, EVCNT_TYPE_MISC, 5425 NULL, device_xname(dev), "TSO"); 5426 evcnt_attach_dynamic(&adapter->link_irq, EVCNT_TYPE_MISC, 5427 NULL, device_xname(dev), "Link MSIX IRQ Handled"); 5428 5429 for (int i = 0; i < adapter->num_queues; i++, rxr++, txr++) { 5430 snprintf(adapter->queues[i].evnamebuf, 5431 sizeof(adapter->queues[i].evnamebuf), "%s queue%d", 5432 device_xname(dev), i); 5433 snprintf(adapter->queues[i].namebuf, 5434 sizeof(adapter->queues[i].namebuf), "queue%d", i); 5435 5436 if ((rnode = ixgbe_sysctl_instance(adapter)) == NULL) { 5437 aprint_error_dev(dev, "could not create sysctl root\n"); 5438 break; 5439 } 5440 5441 if (sysctl_createv(log, 0, &rnode, &rnode, 5442 0, CTLTYPE_NODE, 5443 adapter->queues[i].namebuf, SYSCTL_DESCR("Queue Name"), 5444 NULL, 0, NULL, 0, CTL_CREATE, CTL_EOL) != 0) 5445 break; 5446 5447 if (sysctl_createv(log, 0, &rnode, &cnode, 5448 CTLFLAG_READONLY, CTLTYPE_INT, 5449 "interrupt_rate", SYSCTL_DESCR("Interrupt Rate"), 5450 ixgbe_sysctl_interrupt_rate_handler, 0, 5451 (void *)&adapter->queues[i], 0, CTL_CREATE, CTL_EOL) != 0) 5452 break; 5453 5454 if (sysctl_createv(log, 0, &rnode, &cnode, 5455 CTLFLAG_READONLY, CTLTYPE_INT, 5456 "txd_head", SYSCTL_DESCR("Transmit Descriptor Head"), 5457 ixgbe_sysctl_tdh_handler, 0, (void *)txr, 5458 0, CTL_CREATE, CTL_EOL) != 0) 5459 break; 5460 5461 if (sysctl_createv(log, 0, &rnode, &cnode, 5462 CTLFLAG_READONLY, CTLTYPE_INT, 5463 "txd_tail", SYSCTL_DESCR("Transmit Descriptor Tail"), 5464 ixgbe_sysctl_tdt_handler, 0, (void *)txr, 5465 0, CTL_CREATE, CTL_EOL) != 0) 5466 break; 5467 5468 evcnt_attach_dynamic(&txr->no_desc_avail, EVCNT_TYPE_MISC, 5469 NULL, adapter->queues[i].evnamebuf, 5470 "Queue No Descriptor Available"); 5471 evcnt_attach_dynamic(&txr->total_packets, EVCNT_TYPE_MISC, 5472 NULL, adapter->queues[i].evnamebuf, 5473 "Queue Packets Transmitted"); 5474 5475 #ifdef LRO 5476 struct lro_ctrl *lro = &rxr->lro; 5477 #endif /* LRO */ 5478 5479 if (sysctl_createv(log, 0, &rnode, &cnode, 5480 CTLFLAG_READONLY, 5481 CTLTYPE_INT, 5482 "rxd_head", SYSCTL_DESCR("Receive Descriptor Head"), 5483 ixgbe_sysctl_rdh_handler, 0, (void *)rxr, 0, 5484 CTL_CREATE, CTL_EOL) != 0) 5485 break; 5486 5487 if (sysctl_createv(log, 0, &rnode, &cnode, 5488 CTLFLAG_READONLY, 5489 CTLTYPE_INT, 5490 "rxd_tail", SYSCTL_DESCR("Receive Descriptor Tail"), 5491 ixgbe_sysctl_rdt_handler, 0, (void *)rxr, 0, 5492 CTL_CREATE, CTL_EOL) != 0) 5493 break; 5494 5495 if (i < __arraycount(adapter->stats.mpc)) { 5496 evcnt_attach_dynamic(&adapter->stats.mpc[i], 5497 EVCNT_TYPE_MISC, NULL, adapter->queues[i].evnamebuf, 5498 "Missed Packet Count"); 5499 } 5500 if (i < __arraycount(adapter->stats.pxontxc)) { 5501 evcnt_attach_dynamic(&adapter->stats.pxontxc[i], 5502 EVCNT_TYPE_MISC, NULL, adapter->queues[i].evnamebuf, 5503 "pxontxc"); 5504 evcnt_attach_dynamic(&adapter->stats.pxonrxc[i], 5505 EVCNT_TYPE_MISC, NULL, adapter->queues[i].evnamebuf, 5506 "pxonrxc"); 5507 evcnt_attach_dynamic(&adapter->stats.pxofftxc[i], 5508 EVCNT_TYPE_MISC, NULL, adapter->queues[i].evnamebuf, 5509 "pxofftxc"); 5510 evcnt_attach_dynamic(&adapter->stats.pxoffrxc[i], 5511 EVCNT_TYPE_MISC, NULL, adapter->queues[i].evnamebuf, 5512 "pxoffrxc"); 5513 evcnt_attach_dynamic(&adapter->stats.pxon2offc[i], 5514 EVCNT_TYPE_MISC, NULL, adapter->queues[i].evnamebuf, 5515 "pxon2offc"); 5516 } 5517 if (i < __arraycount(adapter->stats.qprc)) { 5518 evcnt_attach_dynamic(&adapter->stats.qprc[i], 5519 EVCNT_TYPE_MISC, NULL, adapter->queues[i].evnamebuf, 5520 "qprc"); 5521 evcnt_attach_dynamic(&adapter->stats.qptc[i], 5522 EVCNT_TYPE_MISC, NULL, adapter->queues[i].evnamebuf, 5523 "qptc"); 5524 evcnt_attach_dynamic(&adapter->stats.qbrc[i], 5525 EVCNT_TYPE_MISC, NULL, adapter->queues[i].evnamebuf, 5526 "qbrc"); 5527 evcnt_attach_dynamic(&adapter->stats.qbtc[i], 5528 EVCNT_TYPE_MISC, NULL, adapter->queues[i].evnamebuf, 5529 "qbtc"); 5530 evcnt_attach_dynamic(&adapter->stats.qprdc[i], 5531 EVCNT_TYPE_MISC, NULL, adapter->queues[i].evnamebuf, 5532 "qprdc"); 5533 } 5534 5535 evcnt_attach_dynamic(&rxr->rx_packets, EVCNT_TYPE_MISC, 5536 NULL, adapter->queues[i].evnamebuf, "Queue Packets Received"); 5537 evcnt_attach_dynamic(&rxr->rx_bytes, EVCNT_TYPE_MISC, 5538 NULL, adapter->queues[i].evnamebuf, "Queue Bytes Received"); 5539 evcnt_attach_dynamic(&rxr->no_jmbuf, EVCNT_TYPE_MISC, 5540 NULL, adapter->queues[i].evnamebuf, "Rx no jumbo mbuf"); 5541 evcnt_attach_dynamic(&rxr->rx_discarded, EVCNT_TYPE_MISC, 5542 NULL, adapter->queues[i].evnamebuf, "Rx discarded"); 5543 evcnt_attach_dynamic(&rxr->rx_split_packets, EVCNT_TYPE_MISC, 5544 NULL, adapter->queues[i].evnamebuf, "Rx split packets"); 5545 evcnt_attach_dynamic(&rxr->rx_irq, EVCNT_TYPE_MISC, 5546 NULL, adapter->queues[i].evnamebuf, "Rx interrupts"); 5547 #ifdef LRO 5548 SYSCTL_ADD_INT(ctx, queue_list, OID_AUTO, "lro_queued", 5549 CTLFLAG_RD, &lro->lro_queued, 0, 5550 "LRO Queued"); 5551 SYSCTL_ADD_INT(ctx, queue_list, OID_AUTO, "lro_flushed", 5552 CTLFLAG_RD, &lro->lro_flushed, 0, 5553 "LRO Flushed"); 5554 #endif /* LRO */ 5555 } 5556 5557 /* MAC stats get the own sub node */ 5558 5559 5560 snprintf(stats->namebuf, 5561 sizeof(stats->namebuf), "%s MAC Statistics", device_xname(dev)); 5562 5563 evcnt_attach_dynamic(&stats->ipcs, EVCNT_TYPE_MISC, NULL, 5564 stats->namebuf, "rx csum offload - IP"); 5565 evcnt_attach_dynamic(&stats->l4cs, EVCNT_TYPE_MISC, NULL, 5566 stats->namebuf, "rx csum offload - L4"); 5567 evcnt_attach_dynamic(&stats->ipcs_bad, EVCNT_TYPE_MISC, NULL, 5568 stats->namebuf, "rx csum offload - IP bad"); 5569 evcnt_attach_dynamic(&stats->l4cs_bad, EVCNT_TYPE_MISC, NULL, 5570 stats->namebuf, "rx csum offload - L4 bad"); 5571 evcnt_attach_dynamic(&stats->intzero, EVCNT_TYPE_MISC, NULL, 5572 stats->namebuf, "Interrupt conditions zero"); 5573 evcnt_attach_dynamic(&stats->legint, EVCNT_TYPE_MISC, NULL, 5574 stats->namebuf, "Legacy interrupts"); 5575 evcnt_attach_dynamic(&stats->crcerrs, EVCNT_TYPE_MISC, NULL, 5576 stats->namebuf, "CRC Errors"); 5577 evcnt_attach_dynamic(&stats->illerrc, EVCNT_TYPE_MISC, NULL, 5578 stats->namebuf, "Illegal Byte Errors"); 5579 evcnt_attach_dynamic(&stats->errbc, EVCNT_TYPE_MISC, NULL, 5580 stats->namebuf, "Byte Errors"); 5581 evcnt_attach_dynamic(&stats->mspdc, EVCNT_TYPE_MISC, NULL, 5582 stats->namebuf, "MAC Short Packets Discarded"); 5583 evcnt_attach_dynamic(&stats->mlfc, EVCNT_TYPE_MISC, NULL, 5584 stats->namebuf, "MAC Local Faults"); 5585 evcnt_attach_dynamic(&stats->mrfc, EVCNT_TYPE_MISC, NULL, 5586 stats->namebuf, "MAC Remote Faults"); 5587 evcnt_attach_dynamic(&stats->rlec, EVCNT_TYPE_MISC, NULL, 5588 stats->namebuf, "Receive Length Errors"); 5589 evcnt_attach_dynamic(&stats->lxontxc, EVCNT_TYPE_MISC, NULL, 5590 stats->namebuf, "Link XON Transmitted"); 5591 evcnt_attach_dynamic(&stats->lxonrxc, EVCNT_TYPE_MISC, NULL, 5592 stats->namebuf, "Link XON Received"); 5593 evcnt_attach_dynamic(&stats->lxofftxc, EVCNT_TYPE_MISC, NULL, 5594 stats->namebuf, "Link XOFF Transmitted"); 5595 evcnt_attach_dynamic(&stats->lxoffrxc, EVCNT_TYPE_MISC, NULL, 5596 stats->namebuf, "Link XOFF Received"); 5597 5598 /* Packet Reception Stats */ 5599 evcnt_attach_dynamic(&stats->tor, EVCNT_TYPE_MISC, NULL, 5600 stats->namebuf, "Total Octets Received"); 5601 evcnt_attach_dynamic(&stats->gorc, EVCNT_TYPE_MISC, NULL, 5602 stats->namebuf, "Good Octets Received"); 5603 evcnt_attach_dynamic(&stats->tpr, EVCNT_TYPE_MISC, NULL, 5604 stats->namebuf, "Total Packets Received"); 5605 evcnt_attach_dynamic(&stats->gprc, EVCNT_TYPE_MISC, NULL, 5606 stats->namebuf, "Good Packets Received"); 5607 evcnt_attach_dynamic(&stats->mprc, EVCNT_TYPE_MISC, NULL, 5608 stats->namebuf, "Multicast Packets Received"); 5609 evcnt_attach_dynamic(&stats->bprc, EVCNT_TYPE_MISC, NULL, 5610 stats->namebuf, "Broadcast Packets Received"); 5611 evcnt_attach_dynamic(&stats->prc64, EVCNT_TYPE_MISC, NULL, 5612 stats->namebuf, "64 byte frames received "); 5613 evcnt_attach_dynamic(&stats->prc127, EVCNT_TYPE_MISC, NULL, 5614 stats->namebuf, "65-127 byte frames received"); 5615 evcnt_attach_dynamic(&stats->prc255, EVCNT_TYPE_MISC, NULL, 5616 stats->namebuf, "128-255 byte frames received"); 5617 evcnt_attach_dynamic(&stats->prc511, EVCNT_TYPE_MISC, NULL, 5618 stats->namebuf, "256-511 byte frames received"); 5619 evcnt_attach_dynamic(&stats->prc1023, EVCNT_TYPE_MISC, NULL, 5620 stats->namebuf, "512-1023 byte frames received"); 5621 evcnt_attach_dynamic(&stats->prc1522, EVCNT_TYPE_MISC, NULL, 5622 stats->namebuf, "1023-1522 byte frames received"); 5623 evcnt_attach_dynamic(&stats->ruc, EVCNT_TYPE_MISC, NULL, 5624 stats->namebuf, "Receive Undersized"); 5625 evcnt_attach_dynamic(&stats->rfc, EVCNT_TYPE_MISC, NULL, 5626 stats->namebuf, "Fragmented Packets Received "); 5627 evcnt_attach_dynamic(&stats->roc, EVCNT_TYPE_MISC, NULL, 5628 stats->namebuf, "Oversized Packets Received"); 5629 evcnt_attach_dynamic(&stats->rjc, EVCNT_TYPE_MISC, NULL, 5630 stats->namebuf, "Received Jabber"); 5631 evcnt_attach_dynamic(&stats->mngprc, EVCNT_TYPE_MISC, NULL, 5632 stats->namebuf, "Management Packets Received"); 5633 evcnt_attach_dynamic(&stats->mngptc, EVCNT_TYPE_MISC, NULL, 5634 stats->namebuf, "Management Packets Dropped"); 5635 evcnt_attach_dynamic(&stats->xec, EVCNT_TYPE_MISC, NULL, 5636 stats->namebuf, "Checksum Errors"); 5637 5638 /* Packet Transmission Stats */ 5639 evcnt_attach_dynamic(&stats->gotc, EVCNT_TYPE_MISC, NULL, 5640 stats->namebuf, "Good Octets Transmitted"); 5641 evcnt_attach_dynamic(&stats->tpt, EVCNT_TYPE_MISC, NULL, 5642 stats->namebuf, "Total Packets Transmitted"); 5643 evcnt_attach_dynamic(&stats->gptc, EVCNT_TYPE_MISC, NULL, 5644 stats->namebuf, "Good Packets Transmitted"); 5645 evcnt_attach_dynamic(&stats->bptc, EVCNT_TYPE_MISC, NULL, 5646 stats->namebuf, "Broadcast Packets Transmitted"); 5647 evcnt_attach_dynamic(&stats->mptc, EVCNT_TYPE_MISC, NULL, 5648 stats->namebuf, "Multicast Packets Transmitted"); 5649 evcnt_attach_dynamic(&stats->mngptc, EVCNT_TYPE_MISC, NULL, 5650 stats->namebuf, "Management Packets Transmitted"); 5651 evcnt_attach_dynamic(&stats->ptc64, EVCNT_TYPE_MISC, NULL, 5652 stats->namebuf, "64 byte frames transmitted "); 5653 evcnt_attach_dynamic(&stats->ptc127, EVCNT_TYPE_MISC, NULL, 5654 stats->namebuf, "65-127 byte frames transmitted"); 5655 evcnt_attach_dynamic(&stats->ptc255, EVCNT_TYPE_MISC, NULL, 5656 stats->namebuf, "128-255 byte frames transmitted"); 5657 evcnt_attach_dynamic(&stats->ptc511, EVCNT_TYPE_MISC, NULL, 5658 stats->namebuf, "256-511 byte frames transmitted"); 5659 evcnt_attach_dynamic(&stats->ptc1023, EVCNT_TYPE_MISC, NULL, 5660 stats->namebuf, "512-1023 byte frames transmitted"); 5661 evcnt_attach_dynamic(&stats->ptc1522, EVCNT_TYPE_MISC, NULL, 5662 stats->namebuf, "1024-1522 byte frames transmitted"); 5663 5664 /* FC Stats */ 5665 evcnt_attach_dynamic(&stats->fccrc, EVCNT_TYPE_MISC, NULL, 5666 stats->namebuf, "FC CRC Errors"); 5667 evcnt_attach_dynamic(&stats->fclast, EVCNT_TYPE_MISC, NULL, 5668 stats->namebuf, "FC Last Error"); 5669 if (hw->mac.type != ixgbe_mac_82598EB) { 5670 evcnt_attach_dynamic(&stats->fcoerpdc, EVCNT_TYPE_MISC, NULL, 5671 stats->namebuf, "FCoE Packets Dropped"); 5672 evcnt_attach_dynamic(&stats->fcoeprc, EVCNT_TYPE_MISC, NULL, 5673 stats->namebuf, "FCoE Packets Received"); 5674 evcnt_attach_dynamic(&stats->fcoeptc, EVCNT_TYPE_MISC, NULL, 5675 stats->namebuf, "FCoE Packets Transmitted"); 5676 evcnt_attach_dynamic(&stats->fcoedwrc, EVCNT_TYPE_MISC, NULL, 5677 stats->namebuf, "FCoE DWords Received"); 5678 evcnt_attach_dynamic(&stats->fcoedwtc, EVCNT_TYPE_MISC, NULL, 5679 stats->namebuf, "FCoE DWords Transmitted"); 5680 } 5681 } 5682 5683 /* 5684 ** Set flow control using sysctl: 5685 ** Flow control values: 5686 ** 0 - off 5687 ** 1 - rx pause 5688 ** 2 - tx pause 5689 ** 3 - full 5690 */ 5691 static int 5692 ixgbe_set_flowcntl(SYSCTLFN_ARGS) 5693 { 5694 struct sysctlnode node; 5695 int error; 5696 int last = ixgbe_flow_control; 5697 struct adapter *adapter; 5698 5699 node = *rnode; 5700 adapter = (struct adapter *)node.sysctl_data; 5701 node.sysctl_data = &ixgbe_flow_control; 5702 error = sysctl_lookup(SYSCTLFN_CALL(&node)); 5703 if (error != 0 || newp == NULL) 5704 return error; 5705 5706 /* Don't bother if it's not changed */ 5707 if (ixgbe_flow_control == last) 5708 return (0); 5709 5710 switch (ixgbe_flow_control) { 5711 case ixgbe_fc_rx_pause: 5712 case ixgbe_fc_tx_pause: 5713 case ixgbe_fc_full: 5714 adapter->hw.fc.requested_mode = ixgbe_flow_control; 5715 break; 5716 case ixgbe_fc_none: 5717 default: 5718 adapter->hw.fc.requested_mode = ixgbe_fc_none; 5719 } 5720 5721 ixgbe_fc_enable(&adapter->hw, 0); 5722 return 0; 5723 } 5724 5725 static void 5726 ixgbe_add_rx_process_limit(struct adapter *adapter, const char *name, 5727 const char *description, int *limit, int value) 5728 { 5729 const struct sysctlnode *rnode, *cnode; 5730 struct sysctllog **log = &adapter->sysctllog; 5731 5732 *limit = value; 5733 5734 if ((rnode = ixgbe_sysctl_instance(adapter)) == NULL) 5735 aprint_error_dev(adapter->dev, 5736 "could not create sysctl root\n"); 5737 else if (sysctl_createv(log, 0, &rnode, &cnode, 5738 CTLFLAG_READWRITE, 5739 CTLTYPE_INT, 5740 name, SYSCTL_DESCR(description), 5741 NULL, 0, limit, 0, 5742 CTL_CREATE, CTL_EOL) != 0) { 5743 aprint_error_dev(adapter->dev, "%s: could not create sysctl", 5744 __func__); 5745 } 5746 } 5747 5748 /* 5749 ** Control link advertise speed: 5750 ** 0 - normal 5751 ** 1 - advertise only 1G 5752 */ 5753 static int 5754 ixgbe_set_advertise(SYSCTLFN_ARGS) 5755 { 5756 struct sysctlnode node; 5757 int t, error; 5758 struct adapter *adapter; 5759 struct ixgbe_hw *hw; 5760 ixgbe_link_speed speed, last; 5761 5762 node = *rnode; 5763 adapter = (struct adapter *)node.sysctl_data; 5764 t = adapter->advertise; 5765 node.sysctl_data = &t; 5766 error = sysctl_lookup(SYSCTLFN_CALL(&node)); 5767 if (error != 0 || newp == NULL) 5768 return error; 5769 5770 if (t == -1) 5771 return 0; 5772 5773 adapter->advertise = t; 5774 5775 hw = &adapter->hw; 5776 last = hw->phy.autoneg_advertised; 5777 5778 if (!((hw->phy.media_type == ixgbe_media_type_copper) || 5779 (hw->phy.multispeed_fiber))) 5780 return 0; 5781 5782 if (adapter->advertise == 1) 5783 speed = IXGBE_LINK_SPEED_1GB_FULL; 5784 else 5785 speed = IXGBE_LINK_SPEED_1GB_FULL | 5786 IXGBE_LINK_SPEED_10GB_FULL; 5787 5788 if (speed == last) /* no change */ 5789 return 0; 5790 5791 hw->mac.autotry_restart = TRUE; 5792 hw->mac.ops.setup_link(hw, speed, TRUE, TRUE); 5793 5794 return 0; 5795 } 5796
Indexes created Sun Sep 28 07:09:56 GMT 2025