ixgbe.c revision 1.14.2.2
1 /****************************************************************************** 2 3 Copyright (c) 2001-2011, Intel Corporation 4 All rights reserved. 5 6 Redistribution and use in source and binary forms, with or without 7 modification, are permitted provided that the following conditions are met: 8 9 1. Redistributions of source code must retain the above copyright notice, 10 this list of conditions and the following disclaimer. 11 12 2. Redistributions in binary form must reproduce the above copyright 13 notice, this list of conditions and the following disclaimer in the 14 documentation and/or other materials provided with the distribution. 15 16 3. Neither the name of the Intel Corporation nor the names of its 17 contributors may be used to endorse or promote products derived from 18 this software without specific prior written permission. 19 20 THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 21 AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 22 IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 23 ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE 24 LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 25 CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 26 SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 27 INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 28 CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 29 ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 30 POSSIBILITY OF SUCH DAMAGE. 31 32 ******************************************************************************/ 33 /* 34 * Copyright (c) 2011 The NetBSD Foundation, Inc. 35 * All rights reserved. 36 * 37 * This code is derived from software contributed to The NetBSD Foundation 38 * by Coyote Point Systems, Inc. 39 * 40 * Redistribution and use in source and binary forms, with or without 41 * modification, are permitted provided that the following conditions 42 * are met: 43 * 1. Redistributions of source code must retain the above copyright 44 * notice, this list of conditions and the following disclaimer. 45 * 2. Redistributions in binary form must reproduce the above copyright 46 * notice, this list of conditions and the following disclaimer in the 47 * documentation and/or other materials provided with the distribution. 48 * 49 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS 50 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED 51 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 52 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS 53 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 54 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 55 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 56 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 57 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 58 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 59 * POSSIBILITY OF SUCH DAMAGE. 60 */ 61 /*$FreeBSD: src/sys/dev/ixgbe/ixgbe.c,v 1.51 2011/04/25 23:34:21 jfv Exp $*/ 62 /*$NetBSD: ixgbe.c,v 1.14.2.2 2015/02/24 10:41:09 martin Exp $*/ 63 64 #include "opt_inet.h" 65 66 #include "ixgbe.h" 67 68 /********************************************************************* 69 * Set this to one to display debug statistics 70 *********************************************************************/ 71 int ixgbe_display_debug_stats = 0; 72 73 /********************************************************************* 74 * Driver version 75 *********************************************************************/ 76 char ixgbe_driver_version[] = "2.3.10"; 77 78 /********************************************************************* 79 * PCI Device ID Table 80 * 81 * Used by probe to select devices to load on 82 * Last field stores an index into ixgbe_strings 83 * Last entry must be all 0s 84 * 85 * { Vendor ID, Device ID, SubVendor ID, SubDevice ID, String Index } 86 *********************************************************************/ 87 88 static ixgbe_vendor_info_t ixgbe_vendor_info_array[] = 89 { 90 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598AF_DUAL_PORT, 0, 0, 0}, 91 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598AF_SINGLE_PORT, 0, 0, 0}, 92 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598EB_CX4, 0, 0, 0}, 93 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598AT, 0, 0, 0}, 94 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598AT2, 0, 0, 0}, 95 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598, 0, 0, 0}, 96 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598_DA_DUAL_PORT, 0, 0, 0}, 97 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598_CX4_DUAL_PORT, 0, 0, 0}, 98 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598EB_XF_LR, 0, 0, 0}, 99 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598_SR_DUAL_PORT_EM, 0, 0, 0}, 100 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598EB_SFP_LOM, 0, 0, 0}, 101 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_KX4, 0, 0, 0}, 102 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_KX4_MEZZ, 0, 0, 0}, 103 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_SFP, 0, 0, 0}, 104 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_XAUI_LOM, 0, 0, 0}, 105 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_CX4, 0, 0, 0}, 106 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_T3_LOM, 0, 0, 0}, 107 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_COMBO_BACKPLANE, 0, 0, 0}, 108 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_BACKPLANE_FCOE, 0, 0, 0}, 109 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_SFP_FCOE, 0, 0, 0}, 110 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_SFP_DELL, 0, 0, 0}, 111 /* required last entry */ 112 {0, 0, 0, 0, 0} 113 }; 114 115 /********************************************************************* 116 * Table of branding strings 117 *********************************************************************/ 118 119 static const char *ixgbe_strings[] = { 120 "Intel(R) PRO/10GbE PCI-Express Network Driver" 121 }; 122 123 /********************************************************************* 124 * Function prototypes 125 *********************************************************************/ 126 static int ixgbe_probe(device_t, cfdata_t, void *); 127 static void ixgbe_attach(device_t, device_t, void *); 128 static int ixgbe_detach(device_t, int); 129 #if 0 130 static int ixgbe_shutdown(device_t); 131 #endif 132 static void ixgbe_start(struct ifnet *); 133 static void ixgbe_start_locked(struct tx_ring *, struct ifnet *); 134 #if __FreeBSD_version >= 800000 135 static int ixgbe_mq_start(struct ifnet *, struct mbuf *); 136 static int ixgbe_mq_start_locked(struct ifnet *, 137 struct tx_ring *, struct mbuf *); 138 static void ixgbe_qflush(struct ifnet *); 139 #endif 140 static int ixgbe_ioctl(struct ifnet *, u_long, void *); 141 static void ixgbe_ifstop(struct ifnet *, int); 142 static int ixgbe_init(struct ifnet *); 143 static void ixgbe_init_locked(struct adapter *); 144 static void ixgbe_stop(void *); 145 static void ixgbe_media_status(struct ifnet *, struct ifmediareq *); 146 static int ixgbe_media_change(struct ifnet *); 147 static void ixgbe_identify_hardware(struct adapter *); 148 static int ixgbe_allocate_pci_resources(struct adapter *, 149 const struct pci_attach_args *); 150 static int ixgbe_allocate_msix(struct adapter *, 151 const struct pci_attach_args *); 152 static int ixgbe_allocate_legacy(struct adapter *, 153 const struct pci_attach_args *); 154 static int ixgbe_allocate_queues(struct adapter *); 155 static int ixgbe_setup_msix(struct adapter *); 156 static void ixgbe_free_pci_resources(struct adapter *); 157 static void ixgbe_local_timer(void *); 158 static int ixgbe_setup_interface(device_t, struct adapter *); 159 static void ixgbe_config_link(struct adapter *); 160 161 static int ixgbe_allocate_transmit_buffers(struct tx_ring *); 162 static int ixgbe_setup_transmit_structures(struct adapter *); 163 static void ixgbe_setup_transmit_ring(struct tx_ring *); 164 static void ixgbe_initialize_transmit_units(struct adapter *); 165 static void ixgbe_free_transmit_structures(struct adapter *); 166 static void ixgbe_free_transmit_buffers(struct tx_ring *); 167 168 static int ixgbe_allocate_receive_buffers(struct rx_ring *); 169 static int ixgbe_setup_receive_structures(struct adapter *); 170 static int ixgbe_setup_receive_ring(struct rx_ring *); 171 static void ixgbe_initialize_receive_units(struct adapter *); 172 static void ixgbe_free_receive_structures(struct adapter *); 173 static void ixgbe_free_receive_buffers(struct rx_ring *); 174 static void ixgbe_setup_hw_rsc(struct rx_ring *); 175 176 static void ixgbe_enable_intr(struct adapter *); 177 static void ixgbe_disable_intr(struct adapter *); 178 static void ixgbe_update_stats_counters(struct adapter *); 179 static bool ixgbe_txeof(struct tx_ring *); 180 static bool ixgbe_rxeof(struct ix_queue *, int); 181 static void ixgbe_rx_checksum(u32, struct mbuf *, u32, 182 struct ixgbe_hw_stats *); 183 static void ixgbe_set_promisc(struct adapter *); 184 static void ixgbe_set_multi(struct adapter *); 185 static void ixgbe_update_link_status(struct adapter *); 186 static void ixgbe_refresh_mbufs(struct rx_ring *, int); 187 static int ixgbe_xmit(struct tx_ring *, struct mbuf *); 188 static int ixgbe_set_flowcntl(SYSCTLFN_PROTO); 189 static int ixgbe_set_advertise(SYSCTLFN_PROTO); 190 static int ixgbe_dma_malloc(struct adapter *, bus_size_t, 191 struct ixgbe_dma_alloc *, int); 192 static void ixgbe_dma_free(struct adapter *, struct ixgbe_dma_alloc *); 193 static void ixgbe_add_rx_process_limit(struct adapter *, const char *, 194 const char *, int *, int); 195 static u32 ixgbe_tx_ctx_setup(struct tx_ring *, struct mbuf *); 196 static bool ixgbe_tso_setup(struct tx_ring *, struct mbuf *, u32 *); 197 static void ixgbe_set_ivar(struct adapter *, u8, u8, s8); 198 static void ixgbe_configure_ivars(struct adapter *); 199 static u8 * ixgbe_mc_array_itr(struct ixgbe_hw *, u8 **, u32 *); 200 201 static void ixgbe_setup_vlan_hw_support(struct adapter *); 202 #if 0 203 static void ixgbe_register_vlan(void *, struct ifnet *, u16); 204 static void ixgbe_unregister_vlan(void *, struct ifnet *, u16); 205 #endif 206 207 static void ixgbe_add_hw_stats(struct adapter *adapter); 208 209 static __inline void ixgbe_rx_discard(struct rx_ring *, int); 210 static __inline void ixgbe_rx_input(struct rx_ring *, struct ifnet *, 211 struct mbuf *, u32); 212 213 /* Support for pluggable optic modules */ 214 static bool ixgbe_sfp_probe(struct adapter *); 215 static void ixgbe_setup_optics(struct adapter *); 216 217 /* Legacy (single vector interrupt handler */ 218 static int ixgbe_legacy_irq(void *); 219 220 #if defined(NETBSD_MSI_OR_MSIX) 221 /* The MSI/X Interrupt handlers */ 222 static void ixgbe_msix_que(void *); 223 static void ixgbe_msix_link(void *); 224 #endif 225 226 /* Software interrupts for deferred work */ 227 static void ixgbe_handle_que(void *); 228 static void ixgbe_handle_link(void *); 229 static void ixgbe_handle_msf(void *); 230 static void ixgbe_handle_mod(void *); 231 232 const struct sysctlnode *ixgbe_sysctl_instance(struct adapter *); 233 static ixgbe_vendor_info_t *ixgbe_lookup(const struct pci_attach_args *); 234 235 #ifdef IXGBE_FDIR 236 static void ixgbe_atr(struct tx_ring *, struct mbuf *); 237 static void ixgbe_reinit_fdir(void *, int); 238 #endif 239 240 /********************************************************************* 241 * FreeBSD Device Interface Entry Points 242 *********************************************************************/ 243 244 CFATTACH_DECL3_NEW(ixg, sizeof(struct adapter), 245 ixgbe_probe, ixgbe_attach, ixgbe_detach, NULL, NULL, NULL, 246 DVF_DETACH_SHUTDOWN); 247 248 #if 0 249 devclass_t ixgbe_devclass; 250 DRIVER_MODULE(ixgbe, pci, ixgbe_driver, ixgbe_devclass, 0, 0); 251 252 MODULE_DEPEND(ixgbe, pci, 1, 1, 1); 253 MODULE_DEPEND(ixgbe, ether, 1, 1, 1); 254 #endif 255 256 /* 257 ** TUNEABLE PARAMETERS: 258 */ 259 260 /* 261 ** AIM: Adaptive Interrupt Moderation 262 ** which means that the interrupt rate 263 ** is varied over time based on the 264 ** traffic for that interrupt vector 265 */ 266 static int ixgbe_enable_aim = TRUE; 267 #define TUNABLE_INT(__x, __y) 268 TUNABLE_INT("hw.ixgbe.enable_aim", &ixgbe_enable_aim); 269 270 static int ixgbe_max_interrupt_rate = (8000000 / IXGBE_LOW_LATENCY); 271 TUNABLE_INT("hw.ixgbe.max_interrupt_rate", &ixgbe_max_interrupt_rate); 272 273 /* How many packets rxeof tries to clean at a time */ 274 static int ixgbe_rx_process_limit = 256; 275 TUNABLE_INT("hw.ixgbe.rx_process_limit", &ixgbe_rx_process_limit); 276 277 /* Flow control setting, default to full */ 278 static int ixgbe_flow_control = ixgbe_fc_full; 279 TUNABLE_INT("hw.ixgbe.flow_control", &ixgbe_flow_control); 280 281 /* 282 ** Smart speed setting, default to on 283 ** this only works as a compile option 284 ** right now as its during attach, set 285 ** this to 'ixgbe_smart_speed_off' to 286 ** disable. 287 */ 288 static int ixgbe_smart_speed = ixgbe_smart_speed_on; 289 290 /* 291 * MSIX should be the default for best performance, 292 * but this allows it to be forced off for testing. 293 */ 294 static int ixgbe_enable_msix = 1; 295 TUNABLE_INT("hw.ixgbe.enable_msix", &ixgbe_enable_msix); 296 297 /* 298 * Header split: this causes the hardware to DMA 299 * the header into a separate mbuf from the payload, 300 * it can be a performance win in some workloads, but 301 * in others it actually hurts, its off by default. 302 */ 303 static bool ixgbe_header_split = FALSE; 304 TUNABLE_INT("hw.ixgbe.hdr_split", &ixgbe_header_split); 305 306 #if defined(NETBSD_MSI_OR_MSIX) 307 /* 308 * Number of Queues, can be set to 0, 309 * it then autoconfigures based on the 310 * number of cpus with a max of 8. This 311 * can be overriden manually here. 312 */ 313 static int ixgbe_num_queues = 0; 314 TUNABLE_INT("hw.ixgbe.num_queues", &ixgbe_num_queues); 315 #endif 316 317 /* 318 ** Number of TX descriptors per ring, 319 ** setting higher than RX as this seems 320 ** the better performing choice. 321 */ 322 static int ixgbe_txd = PERFORM_TXD; 323 TUNABLE_INT("hw.ixgbe.txd", &ixgbe_txd); 324 325 /* Number of RX descriptors per ring */ 326 static int ixgbe_rxd = PERFORM_RXD; 327 TUNABLE_INT("hw.ixgbe.rxd", &ixgbe_rxd); 328 329 /* Keep running tab on them for sanity check */ 330 static int ixgbe_total_ports; 331 332 #ifdef IXGBE_FDIR 333 /* 334 ** For Flow Director: this is the 335 ** number of TX packets we sample 336 ** for the filter pool, this means 337 ** every 20th packet will be probed. 338 ** 339 ** This feature can be disabled by 340 ** setting this to 0. 341 */ 342 static int atr_sample_rate = 20; 343 /* 344 ** Flow Director actually 'steals' 345 ** part of the packet buffer as its 346 ** filter pool, this variable controls 347 ** how much it uses: 348 ** 0 = 64K, 1 = 128K, 2 = 256K 349 */ 350 static int fdir_pballoc = 1; 351 #endif 352 353 /********************************************************************* 354 * Device identification routine 355 * 356 * ixgbe_probe determines if the driver should be loaded on 357 * adapter based on PCI vendor/device id of the adapter. 358 * 359 * return 1 on success, 0 on failure 360 *********************************************************************/ 361 362 static int 363 ixgbe_probe(device_t dev, cfdata_t cf, void *aux) 364 { 365 const struct pci_attach_args *pa = aux; 366 367 return (ixgbe_lookup(pa) != NULL) ? 1 : 0; 368 } 369 370 static ixgbe_vendor_info_t * 371 ixgbe_lookup(const struct pci_attach_args *pa) 372 { 373 pcireg_t subid; 374 ixgbe_vendor_info_t *ent; 375 376 INIT_DEBUGOUT("ixgbe_probe: begin"); 377 378 if (PCI_VENDOR(pa->pa_id) != IXGBE_INTEL_VENDOR_ID) 379 return NULL; 380 381 subid = pci_conf_read(pa->pa_pc, pa->pa_tag, PCI_SUBSYS_ID_REG); 382 383 for (ent = ixgbe_vendor_info_array; ent->vendor_id != 0; ent++) { 384 if (PCI_VENDOR(pa->pa_id) == ent->vendor_id && 385 PCI_PRODUCT(pa->pa_id) == ent->device_id && 386 387 (PCI_SUBSYS_VENDOR(subid) == ent->subvendor_id || 388 ent->subvendor_id == 0) && 389 390 (PCI_SUBSYS_ID(subid) == ent->subdevice_id || 391 ent->subdevice_id == 0)) { 392 ++ixgbe_total_ports; 393 return ent; 394 } 395 } 396 return NULL; 397 } 398 399 400 static void 401 ixgbe_sysctl_attach(struct adapter *adapter) 402 { 403 struct sysctllog **log; 404 const struct sysctlnode *rnode, *cnode; 405 device_t dev; 406 407 dev = adapter->dev; 408 log = &adapter->sysctllog; 409 410 if ((rnode = ixgbe_sysctl_instance(adapter)) == NULL) { 411 aprint_error_dev(dev, "could not create sysctl root\n"); 412 return; 413 } 414 415 if (sysctl_createv(log, 0, &rnode, &cnode, 416 CTLFLAG_READONLY, CTLTYPE_INT, 417 "num_rx_desc", SYSCTL_DESCR("Number of rx descriptors"), 418 NULL, 0, &adapter->num_rx_desc, 0, CTL_CREATE, CTL_EOL) != 0) 419 aprint_error_dev(dev, "could not create sysctl\n"); 420 421 if (sysctl_createv(log, 0, &rnode, &cnode, 422 CTLFLAG_READONLY, CTLTYPE_INT, 423 "num_queues", SYSCTL_DESCR("Number of queues"), 424 NULL, 0, &adapter->num_queues, 0, CTL_CREATE, CTL_EOL) != 0) 425 aprint_error_dev(dev, "could not create sysctl\n"); 426 427 if (sysctl_createv(log, 0, &rnode, &cnode, 428 CTLFLAG_READWRITE, CTLTYPE_INT, 429 "flow_control", SYSCTL_DESCR("Flow Control"), 430 ixgbe_set_flowcntl, 0, (void *)adapter, 0, CTL_CREATE, CTL_EOL) != 0) 431 aprint_error_dev(dev, "could not create sysctl\n"); 432 433 if (sysctl_createv(log, 0, &rnode, &cnode, 434 CTLFLAG_READWRITE, CTLTYPE_INT, 435 "advertise_gig", SYSCTL_DESCR("1G Link"), 436 ixgbe_set_advertise, 0, (void *)adapter, 0, CTL_CREATE, CTL_EOL) != 0) 437 aprint_error_dev(dev, "could not create sysctl\n"); 438 439 /* XXX This is an *instance* sysctl controlling a *global* variable. 440 * XXX It's that way in the FreeBSD driver that this derives from. 441 */ 442 if (sysctl_createv(log, 0, &rnode, &cnode, 443 CTLFLAG_READWRITE, CTLTYPE_INT, 444 "enable_aim", SYSCTL_DESCR("Interrupt Moderation"), 445 NULL, 0, &ixgbe_enable_aim, 0, CTL_CREATE, CTL_EOL) != 0) 446 aprint_error_dev(dev, "could not create sysctl\n"); 447 } 448 449 /********************************************************************* 450 * Device initialization routine 451 * 452 * The attach entry point is called when the driver is being loaded. 453 * This routine identifies the type of hardware, allocates all resources 454 * and initializes the hardware. 455 * 456 * return 0 on success, positive on failure 457 *********************************************************************/ 458 459 static void 460 ixgbe_attach(device_t parent, device_t dev, void *aux) 461 { 462 struct adapter *adapter; 463 struct ixgbe_hw *hw; 464 int error = 0; 465 u16 csum; 466 u32 ctrl_ext; 467 ixgbe_vendor_info_t *ent; 468 const struct pci_attach_args *pa = aux; 469 470 INIT_DEBUGOUT("ixgbe_attach: begin"); 471 472 /* Allocate, clear, and link in our adapter structure */ 473 adapter = device_private(dev); 474 adapter->dev = adapter->osdep.dev = dev; 475 hw = &adapter->hw; 476 adapter->osdep.pc = pa->pa_pc; 477 adapter->osdep.tag = pa->pa_tag; 478 adapter->osdep.dmat = pa->pa_dmat; 479 480 ent = ixgbe_lookup(pa); 481 482 KASSERT(ent != NULL); 483 484 aprint_normal(": %s, Version - %s\n", 485 ixgbe_strings[ent->index], ixgbe_driver_version); 486 487 /* Core Lock Init*/ 488 IXGBE_CORE_LOCK_INIT(adapter, device_xname(dev)); 489 490 /* SYSCTL APIs */ 491 492 ixgbe_sysctl_attach(adapter); 493 494 /* Set up the timer callout */ 495 callout_init(&adapter->timer, 0); 496 497 /* Determine hardware revision */ 498 ixgbe_identify_hardware(adapter); 499 500 /* Do base PCI setup - map BAR0 */ 501 if (ixgbe_allocate_pci_resources(adapter, pa)) { 502 aprint_error_dev(dev, "Allocation of PCI resources failed\n"); 503 error = ENXIO; 504 goto err_out; 505 } 506 507 /* Do descriptor calc and sanity checks */ 508 if (((ixgbe_txd * sizeof(union ixgbe_adv_tx_desc)) % DBA_ALIGN) != 0 || 509 ixgbe_txd < MIN_TXD || ixgbe_txd > MAX_TXD) { 510 aprint_error_dev(dev, "TXD config issue, using default!\n"); 511 adapter->num_tx_desc = DEFAULT_TXD; 512 } else 513 adapter->num_tx_desc = ixgbe_txd; 514 515 /* 516 ** With many RX rings it is easy to exceed the 517 ** system mbuf allocation. Tuning nmbclusters 518 ** can alleviate this. 519 */ 520 if (nmbclusters > 0 ) { 521 int s; 522 s = (ixgbe_rxd * adapter->num_queues) * ixgbe_total_ports; 523 if (s > nmbclusters) { 524 aprint_error_dev(dev, "RX Descriptors exceed " 525 "system mbuf max, using default instead!\n"); 526 ixgbe_rxd = DEFAULT_RXD; 527 } 528 } 529 530 if (((ixgbe_rxd * sizeof(union ixgbe_adv_rx_desc)) % DBA_ALIGN) != 0 || 531 ixgbe_rxd < MIN_TXD || ixgbe_rxd > MAX_TXD) { 532 aprint_error_dev(dev, "RXD config issue, using default!\n"); 533 adapter->num_rx_desc = DEFAULT_RXD; 534 } else 535 adapter->num_rx_desc = ixgbe_rxd; 536 537 /* Allocate our TX/RX Queues */ 538 if (ixgbe_allocate_queues(adapter)) { 539 error = ENOMEM; 540 goto err_out; 541 } 542 543 /* Allocate multicast array memory. */ 544 adapter->mta = malloc(sizeof(u8) * IXGBE_ETH_LENGTH_OF_ADDRESS * 545 MAX_NUM_MULTICAST_ADDRESSES, M_DEVBUF, M_NOWAIT); 546 if (adapter->mta == NULL) { 547 aprint_error_dev(dev, "Cannot allocate multicast setup array\n"); 548 error = ENOMEM; 549 goto err_late; 550 } 551 552 /* Initialize the shared code */ 553 error = ixgbe_init_shared_code(hw); 554 if (error == IXGBE_ERR_SFP_NOT_PRESENT) { 555 /* 556 ** No optics in this port, set up 557 ** so the timer routine will probe 558 ** for later insertion. 559 */ 560 adapter->sfp_probe = TRUE; 561 error = 0; 562 } else if (error == IXGBE_ERR_SFP_NOT_SUPPORTED) { 563 aprint_error_dev(dev,"Unsupported SFP+ module detected!\n"); 564 error = EIO; 565 goto err_late; 566 } else if (error) { 567 aprint_error_dev(dev,"Unable to initialize the shared code\n"); 568 error = EIO; 569 goto err_late; 570 } 571 572 /* Make sure we have a good EEPROM before we read from it */ 573 if (ixgbe_validate_eeprom_checksum(&adapter->hw, &csum) < 0) { 574 aprint_error_dev(dev,"The EEPROM Checksum Is Not Valid\n"); 575 error = EIO; 576 goto err_late; 577 } 578 579 /* Get Hardware Flow Control setting */ 580 hw->fc.requested_mode = ixgbe_fc_full; 581 hw->fc.pause_time = IXGBE_FC_PAUSE; 582 hw->fc.low_water = IXGBE_FC_LO; 583 hw->fc.high_water = IXGBE_FC_HI; 584 hw->fc.send_xon = TRUE; 585 586 error = ixgbe_init_hw(hw); 587 if (error == IXGBE_ERR_EEPROM_VERSION) { 588 aprint_error_dev(dev, "This device is a pre-production adapter/" 589 "LOM. Please be aware there may be issues associated " 590 "with your hardware.\n If you are experiencing problems " 591 "please contact your Intel or hardware representative " 592 "who provided you with this hardware.\n"); 593 } else if (error == IXGBE_ERR_SFP_NOT_SUPPORTED) 594 aprint_error_dev(dev,"Unsupported SFP+ Module\n"); 595 596 if (error) { 597 error = EIO; 598 aprint_error_dev(dev,"Hardware Initialization Failure\n"); 599 goto err_late; 600 } 601 602 /* Detect and set physical type */ 603 ixgbe_setup_optics(adapter); 604 605 if ((adapter->msix > 1) && (ixgbe_enable_msix)) 606 error = ixgbe_allocate_msix(adapter, pa); 607 else 608 error = ixgbe_allocate_legacy(adapter, pa); 609 if (error) 610 goto err_late; 611 612 /* Setup OS specific network interface */ 613 if (ixgbe_setup_interface(dev, adapter) != 0) 614 goto err_late; 615 616 /* Sysctl for limiting the amount of work done in software interrupts */ 617 ixgbe_add_rx_process_limit(adapter, "rx_processing_limit", 618 "max number of rx packets to process", &adapter->rx_process_limit, 619 ixgbe_rx_process_limit); 620 621 /* Initialize statistics */ 622 ixgbe_update_stats_counters(adapter); 623 624 /* Print PCIE bus type/speed/width info */ 625 ixgbe_get_bus_info(hw); 626 aprint_normal_dev(dev,"PCI Express Bus: Speed %s %s\n", 627 ((hw->bus.speed == ixgbe_bus_speed_5000) ? "5.0Gb/s": 628 (hw->bus.speed == ixgbe_bus_speed_2500) ? "2.5Gb/s":"Unknown"), 629 (hw->bus.width == ixgbe_bus_width_pcie_x8) ? "Width x8" : 630 (hw->bus.width == ixgbe_bus_width_pcie_x4) ? "Width x4" : 631 (hw->bus.width == ixgbe_bus_width_pcie_x1) ? "Width x1" : 632 ("Unknown")); 633 634 if ((hw->bus.width <= ixgbe_bus_width_pcie_x4) && 635 (hw->bus.speed == ixgbe_bus_speed_2500)) { 636 aprint_error_dev(dev, "PCI-Express bandwidth available" 637 " for this card\n is not sufficient for" 638 " optimal performance.\n"); 639 aprint_error_dev(dev, "For optimal performance a x8 " 640 "PCIE, or x4 PCIE 2 slot is required.\n"); 641 } 642 643 /* let hardware know driver is loaded */ 644 ctrl_ext = IXGBE_READ_REG(hw, IXGBE_CTRL_EXT); 645 ctrl_ext |= IXGBE_CTRL_EXT_DRV_LOAD; 646 IXGBE_WRITE_REG(hw, IXGBE_CTRL_EXT, ctrl_ext); 647 648 ixgbe_add_hw_stats(adapter); 649 650 INIT_DEBUGOUT("ixgbe_attach: end"); 651 return; 652 err_late: 653 ixgbe_free_transmit_structures(adapter); 654 ixgbe_free_receive_structures(adapter); 655 err_out: 656 if (adapter->ifp != NULL) 657 if_free(adapter->ifp); 658 ixgbe_free_pci_resources(adapter); 659 if (adapter->mta != NULL) 660 free(adapter->mta, M_DEVBUF); 661 return; 662 663 } 664 665 /********************************************************************* 666 * Device removal routine 667 * 668 * The detach entry point is called when the driver is being removed. 669 * This routine stops the adapter and deallocates all the resources 670 * that were allocated for driver operation. 671 * 672 * return 0 on success, positive on failure 673 *********************************************************************/ 674 675 static int 676 ixgbe_detach(device_t dev, int flags) 677 { 678 struct adapter *adapter = device_private(dev); 679 struct tx_ring *txr = adapter->tx_rings; 680 struct rx_ring *rxr = adapter->rx_rings; 681 struct ixgbe_hw_stats *stats = &adapter->stats; 682 struct ix_queue *que = adapter->queues; 683 u32 ctrl_ext; 684 685 INIT_DEBUGOUT("ixgbe_detach: begin"); 686 687 /* Make sure VLANs are not using driver */ 688 if (!VLAN_ATTACHED(&adapter->osdep.ec)) 689 ; /* nothing to do: no VLANs */ 690 else if ((flags & (DETACH_SHUTDOWN|DETACH_FORCE)) != 0) 691 vlan_ifdetach(adapter->ifp); 692 else { 693 aprint_error_dev(dev, "VLANs in use\n"); 694 return EBUSY; 695 } 696 697 IXGBE_CORE_LOCK(adapter); 698 ixgbe_stop(adapter); 699 IXGBE_CORE_UNLOCK(adapter); 700 701 for (int i = 0; i < adapter->num_queues; i++, que++) { 702 softint_disestablish(que->que_si); 703 } 704 705 /* Drain the Link queue */ 706 softint_disestablish(adapter->link_si); 707 softint_disestablish(adapter->mod_si); 708 softint_disestablish(adapter->msf_si); 709 #ifdef IXGBE_FDIR 710 softint_disestablish(adapter->fdir_si); 711 #endif 712 713 /* let hardware know driver is unloading */ 714 ctrl_ext = IXGBE_READ_REG(&adapter->hw, IXGBE_CTRL_EXT); 715 ctrl_ext &= ~IXGBE_CTRL_EXT_DRV_LOAD; 716 IXGBE_WRITE_REG(&adapter->hw, IXGBE_CTRL_EXT, ctrl_ext); 717 718 ether_ifdetach(adapter->ifp); 719 callout_halt(&adapter->timer, NULL); 720 ixgbe_free_pci_resources(adapter); 721 #if 0 /* XXX the NetBSD port is probably missing something here */ 722 bus_generic_detach(dev); 723 #endif 724 if_detach(adapter->ifp); 725 726 sysctl_teardown(&adapter->sysctllog); 727 evcnt_detach(&adapter->handleq); 728 evcnt_detach(&adapter->req); 729 evcnt_detach(&adapter->morerx); 730 evcnt_detach(&adapter->moretx); 731 evcnt_detach(&adapter->txloops); 732 evcnt_detach(&adapter->efbig_tx_dma_setup); 733 evcnt_detach(&adapter->m_defrag_failed); 734 evcnt_detach(&adapter->efbig2_tx_dma_setup); 735 evcnt_detach(&adapter->einval_tx_dma_setup); 736 evcnt_detach(&adapter->other_tx_dma_setup); 737 evcnt_detach(&adapter->eagain_tx_dma_setup); 738 evcnt_detach(&adapter->enomem_tx_dma_setup); 739 evcnt_detach(&adapter->watchdog_events); 740 evcnt_detach(&adapter->tso_err); 741 evcnt_detach(&adapter->tso_tx); 742 evcnt_detach(&adapter->link_irq); 743 for (int i = 0; i < adapter->num_queues; i++, rxr++, txr++) { 744 evcnt_detach(&txr->no_desc_avail); 745 evcnt_detach(&txr->total_packets); 746 747 if (i < __arraycount(adapter->stats.mpc)) { 748 evcnt_detach(&adapter->stats.mpc[i]); 749 } 750 if (i < __arraycount(adapter->stats.pxontxc)) { 751 evcnt_detach(&adapter->stats.pxontxc[i]); 752 evcnt_detach(&adapter->stats.pxonrxc[i]); 753 evcnt_detach(&adapter->stats.pxofftxc[i]); 754 evcnt_detach(&adapter->stats.pxoffrxc[i]); 755 evcnt_detach(&adapter->stats.pxon2offc[i]); 756 } 757 if (i < __arraycount(adapter->stats.qprc)) { 758 evcnt_detach(&adapter->stats.qprc[i]); 759 evcnt_detach(&adapter->stats.qptc[i]); 760 evcnt_detach(&adapter->stats.qbrc[i]); 761 evcnt_detach(&adapter->stats.qbtc[i]); 762 evcnt_detach(&adapter->stats.qprdc[i]); 763 } 764 765 evcnt_detach(&rxr->rx_packets); 766 evcnt_detach(&rxr->rx_bytes); 767 evcnt_detach(&rxr->no_jmbuf); 768 evcnt_detach(&rxr->rx_discarded); 769 evcnt_detach(&rxr->rx_split_packets); 770 evcnt_detach(&rxr->rx_irq); 771 } 772 evcnt_detach(&stats->ipcs); 773 evcnt_detach(&stats->l4cs); 774 evcnt_detach(&stats->ipcs_bad); 775 evcnt_detach(&stats->l4cs_bad); 776 evcnt_detach(&stats->intzero); 777 evcnt_detach(&stats->legint); 778 evcnt_detach(&stats->crcerrs); 779 evcnt_detach(&stats->illerrc); 780 evcnt_detach(&stats->errbc); 781 evcnt_detach(&stats->mspdc); 782 evcnt_detach(&stats->mlfc); 783 evcnt_detach(&stats->mrfc); 784 evcnt_detach(&stats->rlec); 785 evcnt_detach(&stats->lxontxc); 786 evcnt_detach(&stats->lxonrxc); 787 evcnt_detach(&stats->lxofftxc); 788 evcnt_detach(&stats->lxoffrxc); 789 790 /* Packet Reception Stats */ 791 evcnt_detach(&stats->tor); 792 evcnt_detach(&stats->gorc); 793 evcnt_detach(&stats->tpr); 794 evcnt_detach(&stats->gprc); 795 evcnt_detach(&stats->mprc); 796 evcnt_detach(&stats->bprc); 797 evcnt_detach(&stats->prc64); 798 evcnt_detach(&stats->prc127); 799 evcnt_detach(&stats->prc255); 800 evcnt_detach(&stats->prc511); 801 evcnt_detach(&stats->prc1023); 802 evcnt_detach(&stats->prc1522); 803 evcnt_detach(&stats->ruc); 804 evcnt_detach(&stats->rfc); 805 evcnt_detach(&stats->roc); 806 evcnt_detach(&stats->rjc); 807 evcnt_detach(&stats->mngprc); 808 evcnt_detach(&stats->xec); 809 810 /* Packet Transmission Stats */ 811 evcnt_detach(&stats->gotc); 812 evcnt_detach(&stats->tpt); 813 evcnt_detach(&stats->gptc); 814 evcnt_detach(&stats->bptc); 815 evcnt_detach(&stats->mptc); 816 evcnt_detach(&stats->mngptc); 817 evcnt_detach(&stats->ptc64); 818 evcnt_detach(&stats->ptc127); 819 evcnt_detach(&stats->ptc255); 820 evcnt_detach(&stats->ptc511); 821 evcnt_detach(&stats->ptc1023); 822 evcnt_detach(&stats->ptc1522); 823 824 /* FC Stats */ 825 evcnt_detach(&stats->fccrc); 826 evcnt_detach(&stats->fclast); 827 if (adapter->hw.mac.type != ixgbe_mac_82598EB) { 828 evcnt_detach(&stats->fcoerpdc); 829 evcnt_detach(&stats->fcoeprc); 830 evcnt_detach(&stats->fcoeptc); 831 evcnt_detach(&stats->fcoedwrc); 832 evcnt_detach(&stats->fcoedwtc); 833 } 834 835 ixgbe_free_transmit_structures(adapter); 836 ixgbe_free_receive_structures(adapter); 837 free(adapter->mta, M_DEVBUF); 838 839 IXGBE_CORE_LOCK_DESTROY(adapter); 840 return (0); 841 } 842 843 /********************************************************************* 844 * 845 * Shutdown entry point 846 * 847 **********************************************************************/ 848 849 #if 0 /* XXX NetBSD ought to register something like this through pmf(9) */ 850 static int 851 ixgbe_shutdown(device_t dev) 852 { 853 struct adapter *adapter = device_private(dev); 854 IXGBE_CORE_LOCK(adapter); 855 ixgbe_stop(adapter); 856 IXGBE_CORE_UNLOCK(adapter); 857 return (0); 858 } 859 #endif 860 861 862 /********************************************************************* 863 * Transmit entry point 864 * 865 * ixgbe_start is called by the stack to initiate a transmit. 866 * The driver will remain in this routine as long as there are 867 * packets to transmit and transmit resources are available. 868 * In case resources are not available stack is notified and 869 * the packet is requeued. 870 **********************************************************************/ 871 872 static void 873 ixgbe_start_locked(struct tx_ring *txr, struct ifnet * ifp) 874 { 875 int rc; 876 struct mbuf *m_head; 877 struct adapter *adapter = txr->adapter; 878 879 IXGBE_TX_LOCK_ASSERT(txr); 880 881 if ((ifp->if_flags & (IFF_RUNNING|IFF_OACTIVE)) != 882 IFF_RUNNING) 883 return; 884 if (!adapter->link_active) 885 return; 886 887 while (!IFQ_IS_EMPTY(&ifp->if_snd)) { 888 889 IFQ_POLL(&ifp->if_snd, m_head); 890 if (m_head == NULL) 891 break; 892 893 if ((rc = ixgbe_xmit(txr, m_head)) == EAGAIN) { 894 ifp->if_flags |= IFF_OACTIVE; 895 break; 896 } 897 IFQ_DEQUEUE(&ifp->if_snd, m_head); 898 if (rc == EFBIG) { 899 struct mbuf *mtmp; 900 901 if ((mtmp = m_defrag(m_head, M_DONTWAIT)) != NULL) { 902 m_head = mtmp; 903 rc = ixgbe_xmit(txr, m_head); 904 if (rc != 0) 905 adapter->efbig2_tx_dma_setup.ev_count++; 906 } else 907 adapter->m_defrag_failed.ev_count++; 908 } 909 if (rc != 0) { 910 m_freem(m_head); 911 continue; 912 } 913 914 /* Send a copy of the frame to the BPF listener */ 915 bpf_mtap(ifp, m_head); 916 917 /* Set watchdog on */ 918 getmicrotime(&txr->watchdog_time); 919 txr->queue_status = IXGBE_QUEUE_WORKING; 920 921 } 922 return; 923 } 924 925 /* 926 * Legacy TX start - called by the stack, this 927 * always uses the first tx ring, and should 928 * not be used with multiqueue tx enabled. 929 */ 930 static void 931 ixgbe_start(struct ifnet *ifp) 932 { 933 struct adapter *adapter = ifp->if_softc; 934 struct tx_ring *txr = adapter->tx_rings; 935 936 if (ifp->if_flags & IFF_RUNNING) { 937 IXGBE_TX_LOCK(txr); 938 ixgbe_start_locked(txr, ifp); 939 IXGBE_TX_UNLOCK(txr); 940 } 941 return; 942 } 943 944 #if __FreeBSD_version >= 800000 945 /* 946 ** Multiqueue Transmit driver 947 ** 948 */ 949 static int 950 ixgbe_mq_start(struct ifnet *ifp, struct mbuf *m) 951 { 952 struct adapter *adapter = ifp->if_softc; 953 struct ix_queue *que; 954 struct tx_ring *txr; 955 int i = 0, err = 0; 956 957 /* Which queue to use */ 958 if ((m->m_flags & M_FLOWID) != 0) 959 i = m->m_pkthdr.flowid % adapter->num_queues; 960 961 txr = &adapter->tx_rings[i]; 962 que = &adapter->queues[i]; 963 964 if (IXGBE_TX_TRYLOCK(txr)) { 965 err = ixgbe_mq_start_locked(ifp, txr, m); 966 IXGBE_TX_UNLOCK(txr); 967 } else { 968 err = drbr_enqueue(ifp, txr->br, m); 969 softint_schedule(que->que_si); 970 } 971 972 return (err); 973 } 974 975 static int 976 ixgbe_mq_start_locked(struct ifnet *ifp, struct tx_ring *txr, struct mbuf *m) 977 { 978 struct adapter *adapter = txr->adapter; 979 struct mbuf *next; 980 int enqueued, err = 0; 981 982 if ((ifp->if_flags & (IFF_RUNNING | IFF_OACTIVE)) != 983 IFF_RUNNING || adapter->link_active == 0) { 984 if (m != NULL) 985 err = drbr_enqueue(ifp, txr->br, m); 986 return (err); 987 } 988 989 enqueued = 0; 990 if (m == NULL) { 991 next = drbr_dequeue(ifp, txr->br); 992 } else if (drbr_needs_enqueue(ifp, txr->br)) { 993 if ((err = drbr_enqueue(ifp, txr->br, m)) != 0) 994 return (err); 995 next = drbr_dequeue(ifp, txr->br); 996 } else 997 next = m; 998 999 /* Process the queue */ 1000 while (next != NULL) { 1001 if ((err = ixgbe_xmit(txr, &next)) != 0) { 1002 if (next != NULL) 1003 err = drbr_enqueue(ifp, txr->br, next); 1004 break; 1005 } 1006 enqueued++; 1007 drbr_stats_update(ifp, next->m_pkthdr.len, next->m_flags); 1008 /* Send a copy of the frame to the BPF listener */ 1009 bpf_mtap(ifp, next); 1010 if ((ifp->if_flags & IFF_RUNNING) == 0) 1011 break; 1012 if (txr->tx_avail < IXGBE_TX_OP_THRESHOLD) 1013 ixgbe_txeof(txr); 1014 if (txr->tx_avail < IXGBE_TX_OP_THRESHOLD) { 1015 ifp->if_flags |= IFF_OACTIVE; 1016 break; 1017 } 1018 next = drbr_dequeue(ifp, txr->br); 1019 } 1020 1021 if (enqueued > 0) { 1022 /* Set watchdog on */ 1023 txr->queue_status = IXGBE_QUEUE_WORKING; 1024 getmicrotime(&txr->watchdog_time); 1025 } 1026 1027 return (err); 1028 } 1029 1030 /* 1031 ** Flush all ring buffers 1032 */ 1033 static void 1034 ixgbe_qflush(struct ifnet *ifp) 1035 { 1036 struct adapter *adapter = ifp->if_softc; 1037 struct tx_ring *txr = adapter->tx_rings; 1038 struct mbuf *m; 1039 1040 for (int i = 0; i < adapter->num_queues; i++, txr++) { 1041 IXGBE_TX_LOCK(txr); 1042 while ((m = buf_ring_dequeue_sc(txr->br)) != NULL) 1043 m_freem(m); 1044 IXGBE_TX_UNLOCK(txr); 1045 } 1046 if_qflush(ifp); 1047 } 1048 #endif /* __FreeBSD_version >= 800000 */ 1049 1050 static int 1051 ixgbe_ifflags_cb(struct ethercom *ec) 1052 { 1053 struct ifnet *ifp = &ec->ec_if; 1054 struct adapter *adapter = ifp->if_softc; 1055 int change = ifp->if_flags ^ adapter->if_flags, rc = 0; 1056 1057 IXGBE_CORE_LOCK(adapter); 1058 1059 if (change != 0) 1060 adapter->if_flags = ifp->if_flags; 1061 1062 if ((change & ~(IFF_CANTCHANGE|IFF_DEBUG)) != 0) 1063 rc = ENETRESET; 1064 else if ((change & (IFF_PROMISC | IFF_ALLMULTI)) != 0) 1065 ixgbe_set_promisc(adapter); 1066 1067 IXGBE_CORE_UNLOCK(adapter); 1068 1069 return rc; 1070 } 1071 1072 /********************************************************************* 1073 * Ioctl entry point 1074 * 1075 * ixgbe_ioctl is called when the user wants to configure the 1076 * interface. 1077 * 1078 * return 0 on success, positive on failure 1079 **********************************************************************/ 1080 1081 static int 1082 ixgbe_ioctl(struct ifnet * ifp, u_long command, void *data) 1083 { 1084 struct adapter *adapter = ifp->if_softc; 1085 struct ifcapreq *ifcr = data; 1086 struct ifreq *ifr = data; 1087 int error = 0; 1088 int l4csum_en; 1089 const int l4csum = IFCAP_CSUM_TCPv4_Rx|IFCAP_CSUM_UDPv4_Rx| 1090 IFCAP_CSUM_TCPv6_Rx|IFCAP_CSUM_UDPv6_Rx; 1091 1092 switch (command) { 1093 case SIOCSIFFLAGS: 1094 IOCTL_DEBUGOUT("ioctl: SIOCSIFFLAGS (Set Interface Flags)"); 1095 break; 1096 case SIOCADDMULTI: 1097 case SIOCDELMULTI: 1098 IOCTL_DEBUGOUT("ioctl: SIOC(ADD|DEL)MULTI"); 1099 break; 1100 case SIOCSIFMEDIA: 1101 case SIOCGIFMEDIA: 1102 IOCTL_DEBUGOUT("ioctl: SIOCxIFMEDIA (Get/Set Interface Media)"); 1103 break; 1104 case SIOCSIFCAP: 1105 IOCTL_DEBUGOUT("ioctl: SIOCSIFCAP (Set Capabilities)"); 1106 break; 1107 case SIOCSIFMTU: 1108 IOCTL_DEBUGOUT("ioctl: SIOCSIFMTU (Set Interface MTU)"); 1109 break; 1110 default: 1111 IOCTL_DEBUGOUT1("ioctl: UNKNOWN (0x%X)\n", (int)command); 1112 break; 1113 } 1114 1115 switch (command) { 1116 case SIOCSIFMEDIA: 1117 case SIOCGIFMEDIA: 1118 return ifmedia_ioctl(ifp, ifr, &adapter->media, command); 1119 case SIOCSIFCAP: 1120 /* Layer-4 Rx checksum offload has to be turned on and 1121 * off as a unit. 1122 */ 1123 l4csum_en = ifcr->ifcr_capenable & l4csum; 1124 if (l4csum_en != l4csum && l4csum_en != 0) 1125 return EINVAL; 1126 /*FALLTHROUGH*/ 1127 case SIOCADDMULTI: 1128 case SIOCDELMULTI: 1129 case SIOCSIFFLAGS: 1130 case SIOCSIFMTU: 1131 default: 1132 if ((error = ether_ioctl(ifp, command, data)) != ENETRESET) 1133 return error; 1134 if ((ifp->if_flags & IFF_RUNNING) == 0) 1135 ; 1136 else if (command == SIOCSIFCAP || command == SIOCSIFMTU) { 1137 IXGBE_CORE_LOCK(adapter); 1138 ixgbe_init_locked(adapter); 1139 IXGBE_CORE_UNLOCK(adapter); 1140 } else if (command == SIOCADDMULTI || command == SIOCDELMULTI) { 1141 /* 1142 * Multicast list has changed; set the hardware filter 1143 * accordingly. 1144 */ 1145 IXGBE_CORE_LOCK(adapter); 1146 ixgbe_disable_intr(adapter); 1147 ixgbe_set_multi(adapter); 1148 ixgbe_enable_intr(adapter); 1149 IXGBE_CORE_UNLOCK(adapter); 1150 } 1151 return 0; 1152 } 1153 } 1154 1155 /********************************************************************* 1156 * Init entry point 1157 * 1158 * This routine is used in two ways. It is used by the stack as 1159 * init entry point in network interface structure. It is also used 1160 * by the driver as a hw/sw initialization routine to get to a 1161 * consistent state. 1162 * 1163 * return 0 on success, positive on failure 1164 **********************************************************************/ 1165 #define IXGBE_MHADD_MFS_SHIFT 16 1166 1167 static void 1168 ixgbe_init_locked(struct adapter *adapter) 1169 { 1170 struct ifnet *ifp = adapter->ifp; 1171 device_t dev = adapter->dev; 1172 struct ixgbe_hw *hw = &adapter->hw; 1173 u32 k, txdctl, mhadd, gpie; 1174 u32 rxdctl, rxctrl; 1175 1176 /* XXX check IFF_UP and IFF_RUNNING, power-saving state! */ 1177 1178 KASSERT(mutex_owned(&adapter->core_mtx)); 1179 INIT_DEBUGOUT("ixgbe_init: begin"); 1180 hw->adapter_stopped = FALSE; 1181 ixgbe_stop_adapter(hw); 1182 callout_stop(&adapter->timer); 1183 1184 /* XXX I moved this here from the SIOCSIFMTU case in ixgbe_ioctl(). */ 1185 adapter->max_frame_size = 1186 ifp->if_mtu + ETHER_HDR_LEN + ETHER_CRC_LEN; 1187 1188 /* reprogram the RAR[0] in case user changed it. */ 1189 ixgbe_set_rar(hw, 0, adapter->hw.mac.addr, 0, IXGBE_RAH_AV); 1190 1191 /* Get the latest mac address, User can use a LAA */ 1192 memcpy(hw->mac.addr, CLLADDR(adapter->ifp->if_sadl), 1193 IXGBE_ETH_LENGTH_OF_ADDRESS); 1194 ixgbe_set_rar(hw, 0, hw->mac.addr, 0, 1); 1195 hw->addr_ctrl.rar_used_count = 1; 1196 1197 /* Prepare transmit descriptors and buffers */ 1198 if (ixgbe_setup_transmit_structures(adapter)) { 1199 device_printf(dev,"Could not setup transmit structures\n"); 1200 ixgbe_stop(adapter); 1201 return; 1202 } 1203 1204 ixgbe_init_hw(hw); 1205 ixgbe_initialize_transmit_units(adapter); 1206 1207 /* Setup Multicast table */ 1208 ixgbe_set_multi(adapter); 1209 1210 /* 1211 ** Determine the correct mbuf pool 1212 ** for doing jumbo/headersplit 1213 */ 1214 if (adapter->max_frame_size <= 2048) 1215 adapter->rx_mbuf_sz = MCLBYTES; 1216 else if (adapter->max_frame_size <= 4096) 1217 adapter->rx_mbuf_sz = MJUMPAGESIZE; 1218 else if (adapter->max_frame_size <= 9216) 1219 adapter->rx_mbuf_sz = MJUM9BYTES; 1220 else 1221 adapter->rx_mbuf_sz = MJUM16BYTES; 1222 1223 /* Prepare receive descriptors and buffers */ 1224 if (ixgbe_setup_receive_structures(adapter)) { 1225 device_printf(dev,"Could not setup receive structures\n"); 1226 ixgbe_stop(adapter); 1227 return; 1228 } 1229 1230 /* Configure RX settings */ 1231 ixgbe_initialize_receive_units(adapter); 1232 1233 gpie = IXGBE_READ_REG(&adapter->hw, IXGBE_GPIE); 1234 1235 /* Enable Fan Failure Interrupt */ 1236 gpie |= IXGBE_SDP1_GPIEN; 1237 1238 /* Add for Thermal detection */ 1239 if (hw->mac.type == ixgbe_mac_82599EB) 1240 gpie |= IXGBE_SDP2_GPIEN; 1241 1242 if (adapter->msix > 1) { 1243 /* Enable Enhanced MSIX mode */ 1244 gpie |= IXGBE_GPIE_MSIX_MODE; 1245 gpie |= IXGBE_GPIE_EIAME | IXGBE_GPIE_PBA_SUPPORT | 1246 IXGBE_GPIE_OCD; 1247 } 1248 IXGBE_WRITE_REG(hw, IXGBE_GPIE, gpie); 1249 1250 /* Set MTU size */ 1251 if (ifp->if_mtu > ETHERMTU) { 1252 mhadd = IXGBE_READ_REG(hw, IXGBE_MHADD); 1253 mhadd &= ~IXGBE_MHADD_MFS_MASK; 1254 mhadd |= adapter->max_frame_size << IXGBE_MHADD_MFS_SHIFT; 1255 IXGBE_WRITE_REG(hw, IXGBE_MHADD, mhadd); 1256 } 1257 1258 /* Now enable all the queues */ 1259 1260 for (int i = 0; i < adapter->num_queues; i++) { 1261 txdctl = IXGBE_READ_REG(hw, IXGBE_TXDCTL(i)); 1262 txdctl |= IXGBE_TXDCTL_ENABLE; 1263 /* Set WTHRESH to 8, burst writeback */ 1264 txdctl |= (8 << 16); 1265 IXGBE_WRITE_REG(hw, IXGBE_TXDCTL(i), txdctl); 1266 } 1267 1268 for (int i = 0; i < adapter->num_queues; i++) { 1269 rxdctl = IXGBE_READ_REG(hw, IXGBE_RXDCTL(i)); 1270 if (hw->mac.type == ixgbe_mac_82598EB) { 1271 /* 1272 ** PTHRESH = 21 1273 ** HTHRESH = 4 1274 ** WTHRESH = 8 1275 */ 1276 rxdctl &= ~0x3FFFFF; 1277 rxdctl |= 0x080420; 1278 } 1279 rxdctl |= IXGBE_RXDCTL_ENABLE; 1280 IXGBE_WRITE_REG(hw, IXGBE_RXDCTL(i), rxdctl); 1281 /* XXX I don't trust this loop, and I don't trust the 1282 * XXX memory barrier. What is this meant to do? --dyoung 1283 */ 1284 for (k = 0; k < 10; k++) { 1285 if (IXGBE_READ_REG(hw, IXGBE_RXDCTL(i)) & 1286 IXGBE_RXDCTL_ENABLE) 1287 break; 1288 else 1289 msec_delay(1); 1290 } 1291 wmb(); 1292 IXGBE_WRITE_REG(hw, IXGBE_RDT(i), adapter->num_rx_desc - 1); 1293 } 1294 1295 /* Set up VLAN support and filter */ 1296 ixgbe_setup_vlan_hw_support(adapter); 1297 1298 /* Enable Receive engine */ 1299 rxctrl = IXGBE_READ_REG(hw, IXGBE_RXCTRL); 1300 if (hw->mac.type == ixgbe_mac_82598EB) 1301 rxctrl |= IXGBE_RXCTRL_DMBYPS; 1302 rxctrl |= IXGBE_RXCTRL_RXEN; 1303 ixgbe_enable_rx_dma(hw, rxctrl); 1304 1305 callout_reset(&adapter->timer, hz, ixgbe_local_timer, adapter); 1306 1307 /* Set up MSI/X routing */ 1308 if (ixgbe_enable_msix) { 1309 ixgbe_configure_ivars(adapter); 1310 /* Set up auto-mask */ 1311 if (hw->mac.type == ixgbe_mac_82598EB) 1312 IXGBE_WRITE_REG(hw, IXGBE_EIAM, IXGBE_EICS_RTX_QUEUE); 1313 else { 1314 IXGBE_WRITE_REG(hw, IXGBE_EIAM_EX(0), 0xFFFFFFFF); 1315 IXGBE_WRITE_REG(hw, IXGBE_EIAM_EX(1), 0xFFFFFFFF); 1316 } 1317 } else { /* Simple settings for Legacy/MSI */ 1318 ixgbe_set_ivar(adapter, 0, 0, 0); 1319 ixgbe_set_ivar(adapter, 0, 0, 1); 1320 IXGBE_WRITE_REG(hw, IXGBE_EIAM, IXGBE_EICS_RTX_QUEUE); 1321 } 1322 1323 #ifdef IXGBE_FDIR 1324 /* Init Flow director */ 1325 if (hw->mac.type != ixgbe_mac_82598EB) 1326 ixgbe_init_fdir_signature_82599(&adapter->hw, fdir_pballoc); 1327 #endif 1328 1329 /* 1330 ** Check on any SFP devices that 1331 ** need to be kick-started 1332 */ 1333 if (hw->phy.type == ixgbe_phy_none) { 1334 int err = hw->phy.ops.identify(hw); 1335 if (err == IXGBE_ERR_SFP_NOT_SUPPORTED) { 1336 device_printf(dev, 1337 "Unsupported SFP+ module type was detected.\n"); 1338 return; 1339 } 1340 } 1341 1342 /* Set moderation on the Link interrupt */ 1343 IXGBE_WRITE_REG(hw, IXGBE_EITR(adapter->linkvec), IXGBE_LINK_ITR); 1344 1345 /* Config/Enable Link */ 1346 ixgbe_config_link(adapter); 1347 1348 /* And now turn on interrupts */ 1349 ixgbe_enable_intr(adapter); 1350 1351 /* Now inform the stack we're ready */ 1352 ifp->if_flags |= IFF_RUNNING; 1353 ifp->if_flags &= ~IFF_OACTIVE; 1354 1355 return; 1356 } 1357 1358 static int 1359 ixgbe_init(struct ifnet *ifp) 1360 { 1361 struct adapter *adapter = ifp->if_softc; 1362 1363 IXGBE_CORE_LOCK(adapter); 1364 ixgbe_init_locked(adapter); 1365 IXGBE_CORE_UNLOCK(adapter); 1366 return 0; /* XXX ixgbe_init_locked cannot fail? really? */ 1367 } 1368 1369 1370 /* 1371 ** 1372 ** MSIX Interrupt Handlers and Tasklets 1373 ** 1374 */ 1375 1376 static inline void 1377 ixgbe_enable_queue(struct adapter *adapter, u32 vector) 1378 { 1379 struct ixgbe_hw *hw = &adapter->hw; 1380 u64 queue = (u64)(1ULL << vector); 1381 u32 mask; 1382 1383 if (hw->mac.type == ixgbe_mac_82598EB) { 1384 mask = (IXGBE_EIMS_RTX_QUEUE & queue); 1385 IXGBE_WRITE_REG(hw, IXGBE_EIMS, mask); 1386 } else { 1387 mask = (queue & 0xFFFFFFFF); 1388 if (mask) 1389 IXGBE_WRITE_REG(hw, IXGBE_EIMS_EX(0), mask); 1390 mask = (queue >> 32); 1391 if (mask) 1392 IXGBE_WRITE_REG(hw, IXGBE_EIMS_EX(1), mask); 1393 } 1394 } 1395 1396 __unused static inline void 1397 ixgbe_disable_queue(struct adapter *adapter, u32 vector) 1398 { 1399 struct ixgbe_hw *hw = &adapter->hw; 1400 u64 queue = (u64)(1ULL << vector); 1401 u32 mask; 1402 1403 if (hw->mac.type == ixgbe_mac_82598EB) { 1404 mask = (IXGBE_EIMS_RTX_QUEUE & queue); 1405 IXGBE_WRITE_REG(hw, IXGBE_EIMC, mask); 1406 } else { 1407 mask = (queue & 0xFFFFFFFF); 1408 if (mask) 1409 IXGBE_WRITE_REG(hw, IXGBE_EIMC_EX(0), mask); 1410 mask = (queue >> 32); 1411 if (mask) 1412 IXGBE_WRITE_REG(hw, IXGBE_EIMC_EX(1), mask); 1413 } 1414 } 1415 1416 static inline void 1417 ixgbe_rearm_queues(struct adapter *adapter, u64 queues) 1418 { 1419 u32 mask; 1420 1421 if (adapter->hw.mac.type == ixgbe_mac_82598EB) { 1422 mask = (IXGBE_EIMS_RTX_QUEUE & queues); 1423 IXGBE_WRITE_REG(&adapter->hw, IXGBE_EICS, mask); 1424 } else { 1425 mask = (queues & 0xFFFFFFFF); 1426 IXGBE_WRITE_REG(&adapter->hw, IXGBE_EICS_EX(0), mask); 1427 mask = (queues >> 32); 1428 IXGBE_WRITE_REG(&adapter->hw, IXGBE_EICS_EX(1), mask); 1429 } 1430 } 1431 1432 1433 static void 1434 ixgbe_handle_que(void *context) 1435 { 1436 struct ix_queue *que = context; 1437 struct adapter *adapter = que->adapter; 1438 struct tx_ring *txr = que->txr; 1439 struct ifnet *ifp = adapter->ifp; 1440 bool more; 1441 1442 adapter->handleq.ev_count++; 1443 1444 if (ifp->if_flags & IFF_RUNNING) { 1445 more = ixgbe_rxeof(que, adapter->rx_process_limit); 1446 IXGBE_TX_LOCK(txr); 1447 ixgbe_txeof(txr); 1448 #if __FreeBSD_version >= 800000 1449 if (!drbr_empty(ifp, txr->br)) 1450 ixgbe_mq_start_locked(ifp, txr, NULL); 1451 #else 1452 if (!IFQ_IS_EMPTY(&ifp->if_snd)) 1453 ixgbe_start_locked(txr, ifp); 1454 #endif 1455 IXGBE_TX_UNLOCK(txr); 1456 if (more) { 1457 adapter->req.ev_count++; 1458 softint_schedule(que->que_si); 1459 return; 1460 } 1461 } 1462 1463 /* Reenable this interrupt */ 1464 ixgbe_enable_queue(adapter, que->msix); 1465 1466 return; 1467 } 1468 1469 1470 /********************************************************************* 1471 * 1472 * Legacy Interrupt Service routine 1473 * 1474 **********************************************************************/ 1475 1476 static int 1477 ixgbe_legacy_irq(void *arg) 1478 { 1479 struct ix_queue *que = arg; 1480 struct adapter *adapter = que->adapter; 1481 struct ifnet *ifp = adapter->ifp; 1482 struct ixgbe_hw *hw = &adapter->hw; 1483 struct tx_ring *txr = adapter->tx_rings; 1484 bool more_tx = false, more_rx = false; 1485 u32 reg_eicr, loop = MAX_LOOP; 1486 1487 reg_eicr = IXGBE_READ_REG(hw, IXGBE_EICR); 1488 1489 adapter->stats.legint.ev_count++; 1490 ++que->irqs; 1491 if (reg_eicr == 0) { 1492 adapter->stats.intzero.ev_count++; 1493 if ((ifp->if_flags & IFF_UP) != 0) 1494 ixgbe_enable_intr(adapter); 1495 return 0; 1496 } 1497 1498 if ((ifp->if_flags & IFF_RUNNING) != 0) { 1499 more_rx = ixgbe_rxeof(que, adapter->rx_process_limit); 1500 1501 IXGBE_TX_LOCK(txr); 1502 do { 1503 adapter->txloops.ev_count++; 1504 more_tx = ixgbe_txeof(txr); 1505 } while (loop-- && more_tx); 1506 IXGBE_TX_UNLOCK(txr); 1507 } 1508 1509 if (more_rx || more_tx) { 1510 if (more_rx) 1511 adapter->morerx.ev_count++; 1512 if (more_tx) 1513 adapter->moretx.ev_count++; 1514 softint_schedule(que->que_si); 1515 } 1516 1517 /* Check for fan failure */ 1518 if ((hw->phy.media_type == ixgbe_media_type_copper) && 1519 (reg_eicr & IXGBE_EICR_GPI_SDP1)) { 1520 device_printf(adapter->dev, "\nCRITICAL: FAN FAILURE!! " 1521 "REPLACE IMMEDIATELY!!\n"); 1522 IXGBE_WRITE_REG(hw, IXGBE_EIMS, IXGBE_EICR_GPI_SDP1); 1523 } 1524 1525 /* Link status change */ 1526 if (reg_eicr & IXGBE_EICR_LSC) 1527 softint_schedule(adapter->link_si); 1528 1529 ixgbe_enable_intr(adapter); 1530 return 1; 1531 } 1532 1533 1534 #if defined(NETBSD_MSI_OR_MSIX) 1535 /********************************************************************* 1536 * 1537 * MSI Queue Interrupt Service routine 1538 * 1539 **********************************************************************/ 1540 void 1541 ixgbe_msix_que(void *arg) 1542 { 1543 struct ix_queue *que = arg; 1544 struct adapter *adapter = que->adapter; 1545 struct tx_ring *txr = que->txr; 1546 struct rx_ring *rxr = que->rxr; 1547 bool more_tx, more_rx; 1548 u32 newitr = 0; 1549 1550 ++que->irqs; 1551 1552 more_rx = ixgbe_rxeof(que, adapter->rx_process_limit); 1553 1554 IXGBE_TX_LOCK(txr); 1555 more_tx = ixgbe_txeof(txr); 1556 IXGBE_TX_UNLOCK(txr); 1557 1558 /* Do AIM now? */ 1559 1560 if (ixgbe_enable_aim == FALSE) 1561 goto no_calc; 1562 /* 1563 ** Do Adaptive Interrupt Moderation: 1564 ** - Write out last calculated setting 1565 ** - Calculate based on average size over 1566 ** the last interval. 1567 */ 1568 if (que->eitr_setting) 1569 IXGBE_WRITE_REG(&adapter->hw, 1570 IXGBE_EITR(que->msix), que->eitr_setting); 1571 1572 que->eitr_setting = 0; 1573 1574 /* Idle, do nothing */ 1575 if ((txr->bytes == 0) && (rxr->bytes == 0)) 1576 goto no_calc; 1577 1578 if ((txr->bytes) && (txr->packets)) 1579 newitr = txr->bytes/txr->packets; 1580 if ((rxr->bytes) && (rxr->packets)) 1581 newitr = max(newitr, 1582 (rxr->bytes / rxr->packets)); 1583 newitr += 24; /* account for hardware frame, crc */ 1584 1585 /* set an upper boundary */ 1586 newitr = min(newitr, 3000); 1587 1588 /* Be nice to the mid range */ 1589 if ((newitr > 300) && (newitr < 1200)) 1590 newitr = (newitr / 3); 1591 else 1592 newitr = (newitr / 2); 1593 1594 if (adapter->hw.mac.type == ixgbe_mac_82598EB) 1595 newitr |= newitr << 16; 1596 else 1597 newitr |= IXGBE_EITR_CNT_WDIS; 1598 1599 /* save for next interrupt */ 1600 que->eitr_setting = newitr; 1601 1602 /* Reset state */ 1603 txr->bytes = 0; 1604 txr->packets = 0; 1605 rxr->bytes = 0; 1606 rxr->packets = 0; 1607 1608 no_calc: 1609 if (more_tx || more_rx) 1610 softint_schedule(que->que_si); 1611 else /* Reenable this interrupt */ 1612 ixgbe_enable_queue(adapter, que->msix); 1613 return; 1614 } 1615 1616 1617 static void 1618 ixgbe_msix_link(void *arg) 1619 { 1620 struct adapter *adapter = arg; 1621 struct ixgbe_hw *hw = &adapter->hw; 1622 u32 reg_eicr; 1623 1624 ++adapter->link_irq.ev_count; 1625 1626 /* First get the cause */ 1627 reg_eicr = IXGBE_READ_REG(hw, IXGBE_EICS); 1628 /* Clear interrupt with write */ 1629 IXGBE_WRITE_REG(hw, IXGBE_EICR, reg_eicr); 1630 1631 /* Link status change */ 1632 if (reg_eicr & IXGBE_EICR_LSC) 1633 softint_schedule(adapter->link_si); 1634 1635 if (adapter->hw.mac.type != ixgbe_mac_82598EB) { 1636 #ifdef IXGBE_FDIR 1637 if (reg_eicr & IXGBE_EICR_FLOW_DIR) { 1638 /* This is probably overkill :) */ 1639 if (!atomic_cmpset_int(&adapter->fdir_reinit, 0, 1)) 1640 return; 1641 /* Clear the interrupt */ 1642 IXGBE_WRITE_REG(hw, IXGBE_EICR, IXGBE_EICR_FLOW_DIR); 1643 /* Turn off the interface */ 1644 adapter->ifp->if_flags &= ~IFF_RUNNING; 1645 softint_schedule(adapter->fdir_si); 1646 } else 1647 #endif 1648 if (reg_eicr & IXGBE_EICR_ECC) { 1649 device_printf(adapter->dev, "\nCRITICAL: ECC ERROR!! " 1650 "Please Reboot!!\n"); 1651 IXGBE_WRITE_REG(hw, IXGBE_EICR, IXGBE_EICR_ECC); 1652 } else 1653 1654 if (reg_eicr & IXGBE_EICR_GPI_SDP1) { 1655 /* Clear the interrupt */ 1656 IXGBE_WRITE_REG(hw, IXGBE_EICR, IXGBE_EICR_GPI_SDP1); 1657 softint_schedule(adapter->msf_si); 1658 } else if (reg_eicr & IXGBE_EICR_GPI_SDP2) { 1659 /* Clear the interrupt */ 1660 IXGBE_WRITE_REG(hw, IXGBE_EICR, IXGBE_EICR_GPI_SDP2); 1661 softint_schedule(adapter->mod_si); 1662 } 1663 } 1664 1665 /* Check for fan failure */ 1666 if ((hw->device_id == IXGBE_DEV_ID_82598AT) && 1667 (reg_eicr & IXGBE_EICR_GPI_SDP1)) { 1668 device_printf(adapter->dev, "\nCRITICAL: FAN FAILURE!! " 1669 "REPLACE IMMEDIATELY!!\n"); 1670 IXGBE_WRITE_REG(hw, IXGBE_EICR, IXGBE_EICR_GPI_SDP1); 1671 } 1672 1673 IXGBE_WRITE_REG(&adapter->hw, IXGBE_EIMS, IXGBE_EIMS_OTHER); 1674 return; 1675 } 1676 #endif 1677 1678 /********************************************************************* 1679 * 1680 * Media Ioctl callback 1681 * 1682 * This routine is called whenever the user queries the status of 1683 * the interface using ifconfig. 1684 * 1685 **********************************************************************/ 1686 static void 1687 ixgbe_media_status(struct ifnet * ifp, struct ifmediareq * ifmr) 1688 { 1689 struct adapter *adapter = ifp->if_softc; 1690 1691 INIT_DEBUGOUT("ixgbe_media_status: begin"); 1692 IXGBE_CORE_LOCK(adapter); 1693 ixgbe_update_link_status(adapter); 1694 1695 ifmr->ifm_status = IFM_AVALID; 1696 ifmr->ifm_active = IFM_ETHER; 1697 1698 if (!adapter->link_active) { 1699 IXGBE_CORE_UNLOCK(adapter); 1700 return; 1701 } 1702 1703 ifmr->ifm_status |= IFM_ACTIVE; 1704 1705 switch (adapter->link_speed) { 1706 case IXGBE_LINK_SPEED_1GB_FULL: 1707 ifmr->ifm_active |= IFM_1000_T | IFM_FDX; 1708 break; 1709 case IXGBE_LINK_SPEED_10GB_FULL: 1710 ifmr->ifm_active |= adapter->optics | IFM_FDX; 1711 break; 1712 } 1713 1714 IXGBE_CORE_UNLOCK(adapter); 1715 1716 return; 1717 } 1718 1719 /********************************************************************* 1720 * 1721 * Media Ioctl callback 1722 * 1723 * This routine is called when the user changes speed/duplex using 1724 * media/mediopt option with ifconfig. 1725 * 1726 **********************************************************************/ 1727 static int 1728 ixgbe_media_change(struct ifnet * ifp) 1729 { 1730 struct adapter *adapter = ifp->if_softc; 1731 struct ifmedia *ifm = &adapter->media; 1732 1733 INIT_DEBUGOUT("ixgbe_media_change: begin"); 1734 1735 if (IFM_TYPE(ifm->ifm_media) != IFM_ETHER) 1736 return (EINVAL); 1737 1738 switch (IFM_SUBTYPE(ifm->ifm_media)) { 1739 case IFM_AUTO: 1740 adapter->hw.phy.autoneg_advertised = 1741 IXGBE_LINK_SPEED_1GB_FULL | IXGBE_LINK_SPEED_10GB_FULL; 1742 break; 1743 default: 1744 device_printf(adapter->dev, "Only auto media type\n"); 1745 return (EINVAL); 1746 } 1747 1748 return (0); 1749 } 1750 1751 /********************************************************************* 1752 * 1753 * This routine maps the mbufs to tx descriptors, allowing the 1754 * TX engine to transmit the packets. 1755 * - return 0 on success, positive on failure 1756 * 1757 **********************************************************************/ 1758 1759 static int 1760 ixgbe_xmit(struct tx_ring *txr, struct mbuf *m_head) 1761 { 1762 struct m_tag *mtag; 1763 struct adapter *adapter = txr->adapter; 1764 struct ethercom *ec = &adapter->osdep.ec; 1765 u32 olinfo_status = 0, cmd_type_len; 1766 u32 paylen = 0; 1767 int i, j, error; 1768 int first, last = 0; 1769 bus_dmamap_t map; 1770 struct ixgbe_tx_buf *txbuf; 1771 union ixgbe_adv_tx_desc *txd = NULL; 1772 1773 /* Basic descriptor defines */ 1774 cmd_type_len = (IXGBE_ADVTXD_DTYP_DATA | 1775 IXGBE_ADVTXD_DCMD_IFCS | IXGBE_ADVTXD_DCMD_DEXT); 1776 1777 if ((mtag = VLAN_OUTPUT_TAG(ec, m_head)) != NULL) 1778 cmd_type_len |= IXGBE_ADVTXD_DCMD_VLE; 1779 1780 /* 1781 * Important to capture the first descriptor 1782 * used because it will contain the index of 1783 * the one we tell the hardware to report back 1784 */ 1785 first = txr->next_avail_desc; 1786 txbuf = &txr->tx_buffers[first]; 1787 map = txbuf->map; 1788 1789 /* 1790 * Map the packet for DMA. 1791 */ 1792 error = bus_dmamap_load_mbuf(txr->txtag->dt_dmat, map, 1793 m_head, BUS_DMA_NOWAIT); 1794 1795 switch (error) { 1796 case EAGAIN: 1797 adapter->eagain_tx_dma_setup.ev_count++; 1798 return EAGAIN; 1799 case ENOMEM: 1800 adapter->enomem_tx_dma_setup.ev_count++; 1801 return EAGAIN; 1802 case EFBIG: 1803 adapter->efbig_tx_dma_setup.ev_count++; 1804 return error; 1805 case EINVAL: 1806 adapter->einval_tx_dma_setup.ev_count++; 1807 return error; 1808 default: 1809 adapter->other_tx_dma_setup.ev_count++; 1810 return error; 1811 case 0: 1812 break; 1813 } 1814 1815 /* Make certain there are enough descriptors */ 1816 if (map->dm_nsegs > txr->tx_avail - 2) { 1817 txr->no_desc_avail.ev_count++; 1818 ixgbe_dmamap_unload(txr->txtag, txbuf->map); 1819 return EAGAIN; 1820 } 1821 1822 /* 1823 ** Set up the appropriate offload context 1824 ** this becomes the first descriptor of 1825 ** a packet. 1826 */ 1827 if (m_head->m_pkthdr.csum_flags & (M_CSUM_TSOv4|M_CSUM_TSOv6)) { 1828 if (ixgbe_tso_setup(txr, m_head, &paylen)) { 1829 cmd_type_len |= IXGBE_ADVTXD_DCMD_TSE; 1830 olinfo_status |= IXGBE_TXD_POPTS_IXSM << 8; 1831 olinfo_status |= IXGBE_TXD_POPTS_TXSM << 8; 1832 olinfo_status |= paylen << IXGBE_ADVTXD_PAYLEN_SHIFT; 1833 ++adapter->tso_tx.ev_count; 1834 } else { 1835 ++adapter->tso_err.ev_count; 1836 /* XXX unload DMA map! --dyoung */ 1837 return ENXIO; 1838 } 1839 } else 1840 olinfo_status |= ixgbe_tx_ctx_setup(txr, m_head); 1841 1842 #ifdef IXGBE_IEEE1588 1843 /* This is changing soon to an mtag detection */ 1844 if (we detect this mbuf has a TSTAMP mtag) 1845 cmd_type_len |= IXGBE_ADVTXD_MAC_TSTAMP; 1846 #endif 1847 1848 #ifdef IXGBE_FDIR 1849 /* Do the flow director magic */ 1850 if ((txr->atr_sample) && (!adapter->fdir_reinit)) { 1851 ++txr->atr_count; 1852 if (txr->atr_count >= atr_sample_rate) { 1853 ixgbe_atr(txr, m_head); 1854 txr->atr_count = 0; 1855 } 1856 } 1857 #endif 1858 /* Record payload length */ 1859 if (paylen == 0) 1860 olinfo_status |= m_head->m_pkthdr.len << 1861 IXGBE_ADVTXD_PAYLEN_SHIFT; 1862 1863 i = txr->next_avail_desc; 1864 for (j = 0; j < map->dm_nsegs; j++) { 1865 bus_size_t seglen; 1866 bus_addr_t segaddr; 1867 1868 txbuf = &txr->tx_buffers[i]; 1869 txd = &txr->tx_base[i]; 1870 seglen = map->dm_segs[j].ds_len; 1871 segaddr = htole64(map->dm_segs[j].ds_addr); 1872 1873 txd->read.buffer_addr = segaddr; 1874 txd->read.cmd_type_len = htole32(txr->txd_cmd | 1875 cmd_type_len |seglen); 1876 txd->read.olinfo_status = htole32(olinfo_status); 1877 last = i; /* descriptor that will get completion IRQ */ 1878 1879 if (++i == adapter->num_tx_desc) 1880 i = 0; 1881 1882 txbuf->m_head = NULL; 1883 txbuf->eop_index = -1; 1884 } 1885 1886 txd->read.cmd_type_len |= 1887 htole32(IXGBE_TXD_CMD_EOP | IXGBE_TXD_CMD_RS); 1888 txr->tx_avail -= map->dm_nsegs; 1889 txr->next_avail_desc = i; 1890 1891 txbuf->m_head = m_head; 1892 /* We exchange the maps instead of copying because otherwise 1893 * we end up with many pointers to the same map and we free 1894 * one map twice in ixgbe_free_transmit_structures(). Who 1895 * knows what other problems this caused. --dyoung 1896 */ 1897 txr->tx_buffers[first].map = txbuf->map; 1898 txbuf->map = map; 1899 bus_dmamap_sync(txr->txtag->dt_dmat, map, 0, m_head->m_pkthdr.len, 1900 BUS_DMASYNC_PREWRITE); 1901 1902 /* Set the index of the descriptor that will be marked done */ 1903 txbuf = &txr->tx_buffers[first]; 1904 txbuf->eop_index = last; 1905 1906 ixgbe_dmamap_sync(txr->txdma.dma_tag, txr->txdma.dma_map, 1907 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); 1908 /* 1909 * Advance the Transmit Descriptor Tail (Tdt), this tells the 1910 * hardware that this frame is available to transmit. 1911 */ 1912 ++txr->total_packets.ev_count; 1913 IXGBE_WRITE_REG(&adapter->hw, IXGBE_TDT(txr->me), i); 1914 1915 return 0; 1916 } 1917 1918 static void 1919 ixgbe_set_promisc(struct adapter *adapter) 1920 { 1921 u_int32_t reg_rctl; 1922 struct ifnet *ifp = adapter->ifp; 1923 1924 reg_rctl = IXGBE_READ_REG(&adapter->hw, IXGBE_FCTRL); 1925 reg_rctl &= (~IXGBE_FCTRL_UPE); 1926 reg_rctl &= (~IXGBE_FCTRL_MPE); 1927 IXGBE_WRITE_REG(&adapter->hw, IXGBE_FCTRL, reg_rctl); 1928 1929 if (ifp->if_flags & IFF_PROMISC) { 1930 reg_rctl |= (IXGBE_FCTRL_UPE | IXGBE_FCTRL_MPE); 1931 IXGBE_WRITE_REG(&adapter->hw, IXGBE_FCTRL, reg_rctl); 1932 } else if (ifp->if_flags & IFF_ALLMULTI) { 1933 reg_rctl |= IXGBE_FCTRL_MPE; 1934 reg_rctl &= ~IXGBE_FCTRL_UPE; 1935 IXGBE_WRITE_REG(&adapter->hw, IXGBE_FCTRL, reg_rctl); 1936 } 1937 return; 1938 } 1939 1940 1941 /********************************************************************* 1942 * Multicast Update 1943 * 1944 * This routine is called whenever multicast address list is updated. 1945 * 1946 **********************************************************************/ 1947 #define IXGBE_RAR_ENTRIES 16 1948 1949 static void 1950 ixgbe_set_multi(struct adapter *adapter) 1951 { 1952 struct ether_multi *enm; 1953 struct ether_multistep step; 1954 u32 fctrl; 1955 u8 *mta; 1956 u8 *update_ptr; 1957 int mcnt = 0; 1958 struct ethercom *ec = &adapter->osdep.ec; 1959 struct ifnet *ifp = adapter->ifp; 1960 1961 IOCTL_DEBUGOUT("ixgbe_set_multi: begin"); 1962 1963 mta = adapter->mta; 1964 bzero(mta, sizeof(u8) * IXGBE_ETH_LENGTH_OF_ADDRESS * 1965 MAX_NUM_MULTICAST_ADDRESSES); 1966 1967 fctrl = IXGBE_READ_REG(&adapter->hw, IXGBE_FCTRL); 1968 fctrl |= (IXGBE_FCTRL_UPE | IXGBE_FCTRL_MPE); 1969 if (ifp->if_flags & IFF_PROMISC) 1970 fctrl |= (IXGBE_FCTRL_UPE | IXGBE_FCTRL_MPE); 1971 else if (ifp->if_flags & IFF_ALLMULTI) { 1972 fctrl |= IXGBE_FCTRL_MPE; 1973 fctrl &= ~IXGBE_FCTRL_UPE; 1974 } else 1975 fctrl &= ~(IXGBE_FCTRL_UPE | IXGBE_FCTRL_MPE); 1976 1977 IXGBE_WRITE_REG(&adapter->hw, IXGBE_FCTRL, fctrl); 1978 1979 ETHER_FIRST_MULTI(step, ec, enm); 1980 while (enm != NULL) { 1981 if (memcmp(enm->enm_addrlo, enm->enm_addrhi, 1982 ETHER_ADDR_LEN) != 0) { 1983 fctrl |= IXGBE_FCTRL_MPE; 1984 IXGBE_WRITE_REG(&adapter->hw, IXGBE_FCTRL, fctrl); 1985 break; 1986 } 1987 bcopy(enm->enm_addrlo, 1988 &mta[mcnt * IXGBE_ETH_LENGTH_OF_ADDRESS], 1989 IXGBE_ETH_LENGTH_OF_ADDRESS); 1990 mcnt++; 1991 ETHER_NEXT_MULTI(step, enm); 1992 } 1993 1994 update_ptr = mta; 1995 ixgbe_update_mc_addr_list(&adapter->hw, 1996 update_ptr, mcnt, ixgbe_mc_array_itr); 1997 1998 return; 1999 } 2000 2001 /* 2002 * This is an iterator function now needed by the multicast 2003 * shared code. It simply feeds the shared code routine the 2004 * addresses in the array of ixgbe_set_multi() one by one. 2005 */ 2006 static u8 * 2007 ixgbe_mc_array_itr(struct ixgbe_hw *hw, u8 **update_ptr, u32 *vmdq) 2008 { 2009 u8 *addr = *update_ptr; 2010 u8 *newptr; 2011 *vmdq = 0; 2012 2013 newptr = addr + IXGBE_ETH_LENGTH_OF_ADDRESS; 2014 *update_ptr = newptr; 2015 return addr; 2016 } 2017 2018 2019 /********************************************************************* 2020 * Timer routine 2021 * 2022 * This routine checks for link status,updates statistics, 2023 * and runs the watchdog check. 2024 * 2025 **********************************************************************/ 2026 2027 static void 2028 ixgbe_local_timer1(void *arg) 2029 { 2030 struct adapter *adapter = arg; 2031 device_t dev = adapter->dev; 2032 struct tx_ring *txr = adapter->tx_rings; 2033 2034 KASSERT(mutex_owned(&adapter->core_mtx)); 2035 2036 /* Check for pluggable optics */ 2037 if (adapter->sfp_probe) 2038 if (!ixgbe_sfp_probe(adapter)) 2039 goto out; /* Nothing to do */ 2040 2041 ixgbe_update_link_status(adapter); 2042 ixgbe_update_stats_counters(adapter); 2043 2044 /* 2045 * If the interface has been paused 2046 * then don't do the watchdog check 2047 */ 2048 if (IXGBE_READ_REG(&adapter->hw, IXGBE_TFCS) & IXGBE_TFCS_TXOFF) 2049 goto out; 2050 2051 /* 2052 ** Check status on the TX queues for a hang 2053 */ 2054 for (int i = 0; i < adapter->num_queues; i++, txr++) 2055 if (txr->queue_status == IXGBE_QUEUE_HUNG) 2056 goto hung; 2057 2058 out: 2059 ixgbe_rearm_queues(adapter, adapter->que_mask); 2060 callout_reset(&adapter->timer, hz, ixgbe_local_timer, adapter); 2061 return; 2062 2063 hung: 2064 device_printf(adapter->dev, "Watchdog timeout -- resetting\n"); 2065 device_printf(dev,"Queue(%d) tdh = %d, hw tdt = %d\n", txr->me, 2066 IXGBE_READ_REG(&adapter->hw, IXGBE_TDH(txr->me)), 2067 IXGBE_READ_REG(&adapter->hw, IXGBE_TDT(txr->me))); 2068 device_printf(dev,"TX(%d) desc avail = %d," 2069 "Next TX to Clean = %d\n", 2070 txr->me, txr->tx_avail, txr->next_to_clean); 2071 adapter->ifp->if_flags &= ~IFF_RUNNING; 2072 adapter->watchdog_events.ev_count++; 2073 ixgbe_init_locked(adapter); 2074 } 2075 2076 static void 2077 ixgbe_local_timer(void *arg) 2078 { 2079 struct adapter *adapter = arg; 2080 2081 IXGBE_CORE_LOCK(adapter); 2082 ixgbe_local_timer1(adapter); 2083 IXGBE_CORE_UNLOCK(adapter); 2084 } 2085 2086 /* 2087 ** Note: this routine updates the OS on the link state 2088 ** the real check of the hardware only happens with 2089 ** a link interrupt. 2090 */ 2091 static void 2092 ixgbe_update_link_status(struct adapter *adapter) 2093 { 2094 struct ifnet *ifp = adapter->ifp; 2095 struct tx_ring *txr = adapter->tx_rings; 2096 device_t dev = adapter->dev; 2097 2098 2099 if (adapter->link_up){ 2100 if (adapter->link_active == FALSE) { 2101 if (bootverbose) 2102 device_printf(dev,"Link is up %d Gbps %s \n", 2103 ((adapter->link_speed == 128)? 10:1), 2104 "Full Duplex"); 2105 adapter->link_active = TRUE; 2106 if_link_state_change(ifp, LINK_STATE_UP); 2107 } 2108 } else { /* Link down */ 2109 if (adapter->link_active == TRUE) { 2110 if (bootverbose) 2111 device_printf(dev,"Link is Down\n"); 2112 if_link_state_change(ifp, LINK_STATE_DOWN); 2113 adapter->link_active = FALSE; 2114 for (int i = 0; i < adapter->num_queues; 2115 i++, txr++) 2116 txr->queue_status = IXGBE_QUEUE_IDLE; 2117 } 2118 } 2119 2120 return; 2121 } 2122 2123 2124 static void 2125 ixgbe_ifstop(struct ifnet *ifp, int disable) 2126 { 2127 struct adapter *adapter = ifp->if_softc; 2128 2129 IXGBE_CORE_LOCK(adapter); 2130 ixgbe_stop(adapter); 2131 IXGBE_CORE_UNLOCK(adapter); 2132 } 2133 2134 /********************************************************************* 2135 * 2136 * This routine disables all traffic on the adapter by issuing a 2137 * global reset on the MAC and deallocates TX/RX buffers. 2138 * 2139 **********************************************************************/ 2140 2141 static void 2142 ixgbe_stop(void *arg) 2143 { 2144 struct ifnet *ifp; 2145 struct adapter *adapter = arg; 2146 struct ixgbe_hw *hw = &adapter->hw; 2147 ifp = adapter->ifp; 2148 2149 KASSERT(mutex_owned(&adapter->core_mtx)); 2150 2151 INIT_DEBUGOUT("ixgbe_stop: begin\n"); 2152 ixgbe_disable_intr(adapter); 2153 2154 /* Tell the stack that the interface is no longer active */ 2155 ifp->if_flags &= ~(IFF_RUNNING | IFF_OACTIVE); 2156 2157 ixgbe_reset_hw(hw); 2158 hw->adapter_stopped = FALSE; 2159 ixgbe_stop_adapter(hw); 2160 /* Turn off the laser */ 2161 if (hw->phy.multispeed_fiber) 2162 ixgbe_disable_tx_laser(hw); 2163 callout_stop(&adapter->timer); 2164 2165 /* reprogram the RAR[0] in case user changed it. */ 2166 ixgbe_set_rar(&adapter->hw, 0, adapter->hw.mac.addr, 0, IXGBE_RAH_AV); 2167 2168 return; 2169 } 2170 2171 2172 /********************************************************************* 2173 * 2174 * Determine hardware revision. 2175 * 2176 **********************************************************************/ 2177 static void 2178 ixgbe_identify_hardware(struct adapter *adapter) 2179 { 2180 pcitag_t tag; 2181 pci_chipset_tag_t pc; 2182 pcireg_t subid, id; 2183 struct ixgbe_hw *hw = &adapter->hw; 2184 2185 pc = adapter->osdep.pc; 2186 tag = adapter->osdep.tag; 2187 2188 id = pci_conf_read(pc, tag, PCI_ID_REG); 2189 subid = pci_conf_read(pc, tag, PCI_SUBSYS_ID_REG); 2190 2191 /* Save off the information about this board */ 2192 hw->vendor_id = PCI_VENDOR(id); 2193 hw->device_id = PCI_PRODUCT(id); 2194 hw->revision_id = 2195 PCI_REVISION(pci_conf_read(pc, tag, PCI_CLASS_REG)); 2196 hw->subsystem_vendor_id = PCI_SUBSYS_VENDOR(subid); 2197 hw->subsystem_device_id = PCI_SUBSYS_ID(subid); 2198 2199 /* We need this here to set the num_segs below */ 2200 ixgbe_set_mac_type(hw); 2201 2202 /* Pick up the 82599 and VF settings */ 2203 if (hw->mac.type != ixgbe_mac_82598EB) { 2204 hw->phy.smart_speed = ixgbe_smart_speed; 2205 adapter->num_segs = IXGBE_82599_SCATTER; 2206 } else 2207 adapter->num_segs = IXGBE_82598_SCATTER; 2208 2209 return; 2210 } 2211 2212 /********************************************************************* 2213 * 2214 * Determine optic type 2215 * 2216 **********************************************************************/ 2217 static void 2218 ixgbe_setup_optics(struct adapter *adapter) 2219 { 2220 struct ixgbe_hw *hw = &adapter->hw; 2221 int layer; 2222 2223 layer = ixgbe_get_supported_physical_layer(hw); 2224 switch (layer) { 2225 case IXGBE_PHYSICAL_LAYER_10GBASE_T: 2226 adapter->optics = IFM_10G_T; 2227 break; 2228 case IXGBE_PHYSICAL_LAYER_1000BASE_T: 2229 adapter->optics = IFM_1000_T; 2230 break; 2231 case IXGBE_PHYSICAL_LAYER_10GBASE_LR: 2232 case IXGBE_PHYSICAL_LAYER_10GBASE_LRM: 2233 adapter->optics = IFM_10G_LR; 2234 break; 2235 case IXGBE_PHYSICAL_LAYER_10GBASE_SR: 2236 adapter->optics = IFM_10G_SR; 2237 break; 2238 case IXGBE_PHYSICAL_LAYER_10GBASE_KX4: 2239 case IXGBE_PHYSICAL_LAYER_10GBASE_CX4: 2240 adapter->optics = IFM_10G_CX4; 2241 break; 2242 case IXGBE_PHYSICAL_LAYER_SFP_PLUS_CU: 2243 adapter->optics = IFM_10G_TWINAX; 2244 break; 2245 case IXGBE_PHYSICAL_LAYER_1000BASE_KX: 2246 case IXGBE_PHYSICAL_LAYER_10GBASE_KR: 2247 case IXGBE_PHYSICAL_LAYER_10GBASE_XAUI: 2248 case IXGBE_PHYSICAL_LAYER_UNKNOWN: 2249 default: 2250 adapter->optics = IFM_ETHER | IFM_AUTO; 2251 break; 2252 } 2253 return; 2254 } 2255 2256 /********************************************************************* 2257 * 2258 * Setup the Legacy or MSI Interrupt handler 2259 * 2260 **********************************************************************/ 2261 static int 2262 ixgbe_allocate_legacy(struct adapter *adapter, const struct pci_attach_args *pa) 2263 { 2264 device_t dev = adapter->dev; 2265 struct ix_queue *que = adapter->queues; 2266 char intrbuf[PCI_INTRSTR_LEN]; 2267 #if 0 2268 int rid = 0; 2269 2270 /* MSI RID at 1 */ 2271 if (adapter->msix == 1) 2272 rid = 1; 2273 #endif 2274 2275 /* We allocate a single interrupt resource */ 2276 if (pci_intr_map(pa, &adapter->osdep.ih) != 0) { 2277 aprint_error_dev(dev, "unable to map interrupt\n"); 2278 return ENXIO; 2279 } else { 2280 aprint_normal_dev(dev, "interrupting at %s\n", 2281 pci_intr_string(adapter->osdep.pc, adapter->osdep.ih, 2282 intrbuf, sizeof(intrbuf))); 2283 } 2284 2285 /* 2286 * Try allocating a fast interrupt and the associated deferred 2287 * processing contexts. 2288 */ 2289 que->que_si = softint_establish(SOFTINT_NET, ixgbe_handle_que, que); 2290 2291 /* Tasklets for Link, SFP and Multispeed Fiber */ 2292 adapter->link_si = 2293 softint_establish(SOFTINT_NET, ixgbe_handle_link, adapter); 2294 adapter->mod_si = 2295 softint_establish(SOFTINT_NET, ixgbe_handle_mod, adapter); 2296 adapter->msf_si = 2297 softint_establish(SOFTINT_NET, ixgbe_handle_msf, adapter); 2298 2299 #ifdef IXGBE_FDIR 2300 adapter->fdir_si = 2301 softint_establish(SOFTINT_NET, ixgbe_reinit_fdir, adapter); 2302 #endif 2303 if (que->que_si == NULL || 2304 adapter->link_si == NULL || 2305 adapter->mod_si == NULL || 2306 #ifdef IXGBE_FDIR 2307 adapter->fdir_si == NULL || 2308 #endif 2309 adapter->msf_si == NULL) { 2310 aprint_error_dev(dev, 2311 "could not establish software interrupts\n"); 2312 return ENXIO; 2313 } 2314 2315 adapter->osdep.intr = pci_intr_establish(adapter->osdep.pc, 2316 adapter->osdep.ih, IPL_NET, ixgbe_legacy_irq, que); 2317 if (adapter->osdep.intr == NULL) { 2318 aprint_error_dev(dev, "failed to register interrupt handler\n"); 2319 softint_disestablish(que->que_si); 2320 softint_disestablish(adapter->link_si); 2321 softint_disestablish(adapter->mod_si); 2322 softint_disestablish(adapter->msf_si); 2323 #ifdef IXGBE_FDIR 2324 softint_disestablish(adapter->fdir_si); 2325 #endif 2326 return ENXIO; 2327 } 2328 /* For simplicity in the handlers */ 2329 adapter->que_mask = IXGBE_EIMS_ENABLE_MASK; 2330 2331 return (0); 2332 } 2333 2334 2335 /********************************************************************* 2336 * 2337 * Setup MSIX Interrupt resources and handlers 2338 * 2339 **********************************************************************/ 2340 static int 2341 ixgbe_allocate_msix(struct adapter *adapter, const struct pci_attach_args *pa) 2342 { 2343 #if !defined(NETBSD_MSI_OR_MSIX) 2344 return 0; 2345 #else 2346 device_t dev = adapter->dev; 2347 struct ix_queue *que = adapter->queues; 2348 int error, rid, vector = 0; 2349 2350 for (int i = 0; i < adapter->num_queues; i++, vector++, que++) { 2351 rid = vector + 1; 2352 que->res = bus_alloc_resource_any(dev, SYS_RES_IRQ, &rid, 2353 RF_SHAREABLE | RF_ACTIVE); 2354 if (que->res == NULL) { 2355 aprint_error_dev(dev,"Unable to allocate" 2356 " bus resource: que interrupt [%d]\n", vector); 2357 return (ENXIO); 2358 } 2359 /* Set the handler function */ 2360 error = bus_setup_intr(dev, que->res, 2361 INTR_TYPE_NET | INTR_MPSAFE, NULL, 2362 ixgbe_msix_que, que, &que->tag); 2363 if (error) { 2364 que->res = NULL; 2365 aprint_error_dev(dev, 2366 "Failed to register QUE handler\n"); 2367 return error; 2368 } 2369 #if __FreeBSD_version >= 800504 2370 bus_describe_intr(dev, que->res, que->tag, "que %d", i); 2371 #endif 2372 que->msix = vector; 2373 adapter->que_mask |= (u64)(1 << que->msix); 2374 /* 2375 ** Bind the msix vector, and thus the 2376 ** ring to the corresponding cpu. 2377 */ 2378 if (adapter->num_queues > 1) 2379 bus_bind_intr(dev, que->res, i); 2380 2381 que->que_si = softint_establish(ixgbe_handle_que, que); 2382 if (que->que_si == NULL) { 2383 aprint_error_dev(dev, 2384 "could not establish software interrupt\n"); 2385 } 2386 } 2387 2388 /* and Link */ 2389 rid = vector + 1; 2390 adapter->res = bus_alloc_resource_any(dev, 2391 SYS_RES_IRQ, &rid, RF_SHAREABLE | RF_ACTIVE); 2392 if (!adapter->res) { 2393 aprint_error_dev(dev,"Unable to allocate bus resource: " 2394 "Link interrupt [%d]\n", rid); 2395 return (ENXIO); 2396 } 2397 /* Set the link handler function */ 2398 error = bus_setup_intr(dev, adapter->res, 2399 INTR_TYPE_NET | INTR_MPSAFE, NULL, 2400 ixgbe_msix_link, adapter, &adapter->tag); 2401 if (error) { 2402 adapter->res = NULL; 2403 aprint_error_dev(dev, "Failed to register LINK handler\n"); 2404 return (error); 2405 } 2406 #if __FreeBSD_version >= 800504 2407 bus_describe_intr(dev, adapter->res, adapter->tag, "link"); 2408 #endif 2409 adapter->linkvec = vector; 2410 /* Tasklets for Link, SFP and Multispeed Fiber */ 2411 adapter->link_si = 2412 softint_establish(SOFTINT_NET, ixgbe_handle_link, adapter); 2413 adapter->mod_si = 2414 softint_establish(SOFTINT_NET, ixgbe_handle_mod, adapter); 2415 adapter->msf_si = 2416 softint_establish(SOFTINT_NET, ixgbe_handle_msf, adapter); 2417 #ifdef IXGBE_FDIR 2418 adapter->fdir_si = 2419 softint_establish(SOFTINT_NET, ixgbe_reinit_fdir, adapter); 2420 #endif 2421 2422 return (0); 2423 #endif 2424 } 2425 2426 /* 2427 * Setup Either MSI/X or MSI 2428 */ 2429 static int 2430 ixgbe_setup_msix(struct adapter *adapter) 2431 { 2432 #if !defined(NETBSD_MSI_OR_MSIX) 2433 return 0; 2434 #else 2435 device_t dev = adapter->dev; 2436 int rid, want, queues, msgs; 2437 2438 /* Override by tuneable */ 2439 if (ixgbe_enable_msix == 0) 2440 goto msi; 2441 2442 /* First try MSI/X */ 2443 rid = PCI_BAR(MSIX_82598_BAR); 2444 adapter->msix_mem = bus_alloc_resource_any(dev, 2445 SYS_RES_MEMORY, &rid, RF_ACTIVE); 2446 if (!adapter->msix_mem) { 2447 rid += 4; /* 82599 maps in higher BAR */ 2448 adapter->msix_mem = bus_alloc_resource_any(dev, 2449 SYS_RES_MEMORY, &rid, RF_ACTIVE); 2450 } 2451 if (!adapter->msix_mem) { 2452 /* May not be enabled */ 2453 device_printf(adapter->dev, 2454 "Unable to map MSIX table \n"); 2455 goto msi; 2456 } 2457 2458 msgs = pci_msix_count(dev); 2459 if (msgs == 0) { /* system has msix disabled */ 2460 bus_release_resource(dev, SYS_RES_MEMORY, 2461 rid, adapter->msix_mem); 2462 adapter->msix_mem = NULL; 2463 goto msi; 2464 } 2465 2466 /* Figure out a reasonable auto config value */ 2467 queues = (mp_ncpus > (msgs-1)) ? (msgs-1) : mp_ncpus; 2468 2469 if (ixgbe_num_queues != 0) 2470 queues = ixgbe_num_queues; 2471 /* Set max queues to 8 when autoconfiguring */ 2472 else if ((ixgbe_num_queues == 0) && (queues > 8)) 2473 queues = 8; 2474 2475 /* 2476 ** Want one vector (RX/TX pair) per queue 2477 ** plus an additional for Link. 2478 */ 2479 want = queues + 1; 2480 if (msgs >= want) 2481 msgs = want; 2482 else { 2483 device_printf(adapter->dev, 2484 "MSIX Configuration Problem, " 2485 "%d vectors but %d queues wanted!\n", 2486 msgs, want); 2487 return (0); /* Will go to Legacy setup */ 2488 } 2489 if ((msgs) && pci_alloc_msix(dev, &msgs) == 0) { 2490 device_printf(adapter->dev, 2491 "Using MSIX interrupts with %d vectors\n", msgs); 2492 adapter->num_queues = queues; 2493 return (msgs); 2494 } 2495 msi: 2496 msgs = pci_msi_count(dev); 2497 if (msgs == 1 && pci_alloc_msi(dev, &msgs) == 0) 2498 device_printf(adapter->dev,"Using MSI interrupt\n"); 2499 return (msgs); 2500 #endif 2501 } 2502 2503 2504 static int 2505 ixgbe_allocate_pci_resources(struct adapter *adapter, const struct pci_attach_args *pa) 2506 { 2507 pcireg_t memtype; 2508 device_t dev = adapter->dev; 2509 bus_addr_t addr; 2510 int flags; 2511 2512 memtype = pci_mapreg_type(pa->pa_pc, pa->pa_tag, PCI_BAR(0)); 2513 switch (memtype) { 2514 case PCI_MAPREG_TYPE_MEM | PCI_MAPREG_MEM_TYPE_32BIT: 2515 case PCI_MAPREG_TYPE_MEM | PCI_MAPREG_MEM_TYPE_64BIT: 2516 adapter->osdep.mem_bus_space_tag = pa->pa_memt; 2517 if (pci_mapreg_info(pa->pa_pc, pa->pa_tag, PCI_BAR(0), 2518 memtype, &addr, &adapter->osdep.mem_size, &flags) != 0) 2519 goto map_err; 2520 if ((flags & BUS_SPACE_MAP_PREFETCHABLE) != 0) { 2521 aprint_normal_dev(dev, "clearing prefetchable bit\n"); 2522 flags &= ~BUS_SPACE_MAP_PREFETCHABLE; 2523 } 2524 if (bus_space_map(adapter->osdep.mem_bus_space_tag, addr, 2525 adapter->osdep.mem_size, flags, 2526 &adapter->osdep.mem_bus_space_handle) != 0) { 2527 map_err: 2528 adapter->osdep.mem_size = 0; 2529 aprint_error_dev(dev, "unable to map BAR0\n"); 2530 return ENXIO; 2531 } 2532 break; 2533 default: 2534 aprint_error_dev(dev, "unexpected type on BAR0\n"); 2535 return ENXIO; 2536 } 2537 2538 /* Legacy defaults */ 2539 adapter->num_queues = 1; 2540 adapter->hw.back = &adapter->osdep; 2541 2542 /* 2543 ** Now setup MSI or MSI/X, should 2544 ** return us the number of supported 2545 ** vectors. (Will be 1 for MSI) 2546 */ 2547 adapter->msix = ixgbe_setup_msix(adapter); 2548 return (0); 2549 } 2550 2551 static void 2552 ixgbe_free_pci_resources(struct adapter * adapter) 2553 { 2554 #if defined(NETBSD_MSI_OR_MSIX) 2555 struct ix_queue *que = adapter->queues; 2556 device_t dev = adapter->dev; 2557 #endif 2558 int rid; 2559 2560 #if defined(NETBSD_MSI_OR_MSIX) 2561 int memrid; 2562 if (adapter->hw.mac.type == ixgbe_mac_82598EB) 2563 memrid = PCI_BAR(MSIX_82598_BAR); 2564 else 2565 memrid = PCI_BAR(MSIX_82599_BAR); 2566 2567 /* 2568 ** There is a slight possibility of a failure mode 2569 ** in attach that will result in entering this function 2570 ** before interrupt resources have been initialized, and 2571 ** in that case we do not want to execute the loops below 2572 ** We can detect this reliably by the state of the adapter 2573 ** res pointer. 2574 */ 2575 if (adapter->res == NULL) 2576 goto mem; 2577 2578 /* 2579 ** Release all msix queue resources: 2580 */ 2581 for (int i = 0; i < adapter->num_queues; i++, que++) { 2582 rid = que->msix + 1; 2583 if (que->tag != NULL) { 2584 bus_teardown_intr(dev, que->res, que->tag); 2585 que->tag = NULL; 2586 } 2587 if (que->res != NULL) 2588 bus_release_resource(dev, SYS_RES_IRQ, rid, que->res); 2589 } 2590 #endif 2591 2592 /* Clean the Legacy or Link interrupt last */ 2593 if (adapter->linkvec) /* we are doing MSIX */ 2594 rid = adapter->linkvec + 1; 2595 else 2596 (adapter->msix != 0) ? (rid = 1):(rid = 0); 2597 2598 pci_intr_disestablish(adapter->osdep.pc, adapter->osdep.intr); 2599 adapter->osdep.intr = NULL; 2600 2601 #if defined(NETBSD_MSI_OR_MSIX) 2602 mem: 2603 if (adapter->msix) 2604 pci_release_msi(dev); 2605 2606 if (adapter->msix_mem != NULL) 2607 bus_release_resource(dev, SYS_RES_MEMORY, 2608 memrid, adapter->msix_mem); 2609 #endif 2610 2611 if (adapter->osdep.mem_size != 0) { 2612 bus_space_unmap(adapter->osdep.mem_bus_space_tag, 2613 adapter->osdep.mem_bus_space_handle, 2614 adapter->osdep.mem_size); 2615 } 2616 2617 return; 2618 } 2619 2620 /********************************************************************* 2621 * 2622 * Setup networking device structure and register an interface. 2623 * 2624 **********************************************************************/ 2625 static int 2626 ixgbe_setup_interface(device_t dev, struct adapter *adapter) 2627 { 2628 struct ethercom *ec = &adapter->osdep.ec; 2629 struct ixgbe_hw *hw = &adapter->hw; 2630 struct ifnet *ifp; 2631 2632 INIT_DEBUGOUT("ixgbe_setup_interface: begin"); 2633 2634 ifp = adapter->ifp = &ec->ec_if; 2635 strlcpy(ifp->if_xname, device_xname(dev), IFNAMSIZ); 2636 ifp->if_mtu = ETHERMTU; 2637 ifp->if_baudrate = 1000000000; 2638 ifp->if_init = ixgbe_init; 2639 ifp->if_stop = ixgbe_ifstop; 2640 ifp->if_softc = adapter; 2641 ifp->if_flags = IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST; 2642 ifp->if_ioctl = ixgbe_ioctl; 2643 ifp->if_start = ixgbe_start; 2644 #if __FreeBSD_version >= 800000 2645 ifp->if_transmit = ixgbe_mq_start; 2646 ifp->if_qflush = ixgbe_qflush; 2647 #endif 2648 ifp->if_snd.ifq_maxlen = adapter->num_tx_desc - 2; 2649 2650 if_attach(ifp); 2651 ether_ifattach(ifp, adapter->hw.mac.addr); 2652 ether_set_ifflags_cb(ec, ixgbe_ifflags_cb); 2653 2654 adapter->max_frame_size = 2655 ifp->if_mtu + ETHER_HDR_LEN + ETHER_CRC_LEN; 2656 2657 /* 2658 * Tell the upper layer(s) we support long frames. 2659 */ 2660 ifp->if_hdrlen = sizeof(struct ether_vlan_header); 2661 2662 ifp->if_capabilities |= IFCAP_HWCSUM | IFCAP_TSOv4; 2663 ifp->if_capenable = 0; 2664 2665 ec->ec_capabilities |= ETHERCAP_VLAN_HWCSUM; 2666 ec->ec_capabilities |= ETHERCAP_VLAN_HWTAGGING | ETHERCAP_VLAN_MTU; 2667 ec->ec_capabilities |= ETHERCAP_JUMBO_MTU; 2668 ec->ec_capenable = ec->ec_capabilities; 2669 2670 /* Don't enable LRO by default */ 2671 ifp->if_capabilities |= IFCAP_LRO; 2672 2673 /* 2674 ** Dont turn this on by default, if vlans are 2675 ** created on another pseudo device (eg. lagg) 2676 ** then vlan events are not passed thru, breaking 2677 ** operation, but with HW FILTER off it works. If 2678 ** using vlans directly on the em driver you can 2679 ** enable this and get full hardware tag filtering. 2680 */ 2681 ec->ec_capabilities |= ETHERCAP_VLAN_HWFILTER; 2682 2683 /* 2684 * Specify the media types supported by this adapter and register 2685 * callbacks to update media and link information 2686 */ 2687 ifmedia_init(&adapter->media, IFM_IMASK, ixgbe_media_change, 2688 ixgbe_media_status); 2689 ifmedia_add(&adapter->media, IFM_ETHER | adapter->optics, 0, NULL); 2690 ifmedia_set(&adapter->media, IFM_ETHER | adapter->optics); 2691 if (hw->device_id == IXGBE_DEV_ID_82598AT) { 2692 ifmedia_add(&adapter->media, 2693 IFM_ETHER | IFM_1000_T | IFM_FDX, 0, NULL); 2694 ifmedia_add(&adapter->media, 2695 IFM_ETHER | IFM_1000_T, 0, NULL); 2696 } 2697 ifmedia_add(&adapter->media, IFM_ETHER | IFM_AUTO, 0, NULL); 2698 ifmedia_set(&adapter->media, IFM_ETHER | IFM_AUTO); 2699 2700 return (0); 2701 } 2702 2703 static void 2704 ixgbe_config_link(struct adapter *adapter) 2705 { 2706 struct ixgbe_hw *hw = &adapter->hw; 2707 u32 autoneg, err = 0; 2708 bool sfp, negotiate; 2709 2710 sfp = ixgbe_is_sfp(hw); 2711 2712 if (sfp) { 2713 void *ip; 2714 2715 if (hw->phy.multispeed_fiber) { 2716 hw->mac.ops.setup_sfp(hw); 2717 ixgbe_enable_tx_laser(hw); 2718 ip = adapter->msf_si; 2719 } else { 2720 ip = adapter->mod_si; 2721 } 2722 2723 kpreempt_disable(); 2724 softint_schedule(ip); 2725 kpreempt_enable(); 2726 } else { 2727 if (hw->mac.ops.check_link) 2728 err = ixgbe_check_link(hw, &autoneg, 2729 &adapter->link_up, FALSE); 2730 if (err) 2731 goto out; 2732 autoneg = hw->phy.autoneg_advertised; 2733 if ((!autoneg) && (hw->mac.ops.get_link_capabilities)) 2734 err = hw->mac.ops.get_link_capabilities(hw, 2735 &autoneg, &negotiate); 2736 else 2737 negotiate = 0; 2738 if (err) 2739 goto out; 2740 if (hw->mac.ops.setup_link) 2741 err = hw->mac.ops.setup_link(hw, autoneg, 2742 negotiate, adapter->link_up); 2743 } 2744 out: 2745 return; 2746 } 2747 2748 /******************************************************************** 2749 * Manage DMA'able memory. 2750 *******************************************************************/ 2751 2752 static int 2753 ixgbe_dma_malloc(struct adapter *adapter, const bus_size_t size, 2754 struct ixgbe_dma_alloc *dma, const int mapflags) 2755 { 2756 device_t dev = adapter->dev; 2757 int r, rsegs; 2758 2759 r = ixgbe_dma_tag_create(adapter->osdep.dmat, /* parent */ 2760 DBA_ALIGN, 0, /* alignment, bounds */ 2761 size, /* maxsize */ 2762 1, /* nsegments */ 2763 size, /* maxsegsize */ 2764 BUS_DMA_ALLOCNOW, /* flags */ 2765 &dma->dma_tag); 2766 if (r != 0) { 2767 aprint_error_dev(dev, 2768 "%s: ixgbe_dma_tag_create failed; error %d\n", __func__, r); 2769 goto fail_0; 2770 } 2771 2772 r = bus_dmamem_alloc(dma->dma_tag->dt_dmat, 2773 size, 2774 dma->dma_tag->dt_alignment, 2775 dma->dma_tag->dt_boundary, 2776 &dma->dma_seg, 1, &rsegs, BUS_DMA_NOWAIT); 2777 if (r != 0) { 2778 aprint_error_dev(dev, 2779 "%s: bus_dmamem_alloc failed; error %d\n", __func__, r); 2780 goto fail_1; 2781 } 2782 2783 r = bus_dmamem_map(dma->dma_tag->dt_dmat, &dma->dma_seg, rsegs, 2784 size, &dma->dma_vaddr, BUS_DMA_NOWAIT); 2785 if (r != 0) { 2786 aprint_error_dev(dev, "%s: bus_dmamem_map failed; error %d\n", 2787 __func__, r); 2788 goto fail_2; 2789 } 2790 2791 r = ixgbe_dmamap_create(dma->dma_tag, 0, &dma->dma_map); 2792 if (r != 0) { 2793 aprint_error_dev(dev, "%s: bus_dmamem_map failed; error %d\n", 2794 __func__, r); 2795 goto fail_3; 2796 } 2797 2798 r = bus_dmamap_load(dma->dma_tag->dt_dmat, dma->dma_map, dma->dma_vaddr, 2799 size, 2800 NULL, 2801 mapflags | BUS_DMA_NOWAIT); 2802 if (r != 0) { 2803 aprint_error_dev(dev, "%s: bus_dmamap_load failed; error %d\n", 2804 __func__, r); 2805 goto fail_4; 2806 } 2807 dma->dma_paddr = dma->dma_map->dm_segs[0].ds_addr; 2808 dma->dma_size = size; 2809 return 0; 2810 fail_4: 2811 ixgbe_dmamap_destroy(dma->dma_tag, dma->dma_map); 2812 fail_3: 2813 bus_dmamem_unmap(dma->dma_tag->dt_dmat, dma->dma_vaddr, size); 2814 fail_2: 2815 bus_dmamem_free(dma->dma_tag->dt_dmat, &dma->dma_seg, rsegs); 2816 fail_1: 2817 ixgbe_dma_tag_destroy(dma->dma_tag); 2818 fail_0: 2819 return r; 2820 } 2821 2822 static void 2823 ixgbe_dma_free(struct adapter *adapter, struct ixgbe_dma_alloc *dma) 2824 { 2825 bus_dmamap_sync(dma->dma_tag->dt_dmat, dma->dma_map, 0, dma->dma_size, 2826 BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE); 2827 ixgbe_dmamap_unload(dma->dma_tag, dma->dma_map); 2828 bus_dmamem_free(dma->dma_tag->dt_dmat, &dma->dma_seg, 1); 2829 ixgbe_dma_tag_destroy(dma->dma_tag); 2830 } 2831 2832 2833 /********************************************************************* 2834 * 2835 * Allocate memory for the transmit and receive rings, and then 2836 * the descriptors associated with each, called only once at attach. 2837 * 2838 **********************************************************************/ 2839 static int 2840 ixgbe_allocate_queues(struct adapter *adapter) 2841 { 2842 device_t dev = adapter->dev; 2843 struct ix_queue *que; 2844 struct tx_ring *txr; 2845 struct rx_ring *rxr; 2846 int rsize, tsize, error = IXGBE_SUCCESS; 2847 int txconf = 0, rxconf = 0; 2848 2849 /* First allocate the top level queue structs */ 2850 if (!(adapter->queues = 2851 (struct ix_queue *) malloc(sizeof(struct ix_queue) * 2852 adapter->num_queues, M_DEVBUF, M_NOWAIT | M_ZERO))) { 2853 aprint_error_dev(dev, "Unable to allocate queue memory\n"); 2854 error = ENOMEM; 2855 goto fail; 2856 } 2857 2858 /* First allocate the TX ring struct memory */ 2859 if (!(adapter->tx_rings = 2860 (struct tx_ring *) malloc(sizeof(struct tx_ring) * 2861 adapter->num_queues, M_DEVBUF, M_NOWAIT | M_ZERO))) { 2862 aprint_error_dev(dev, "Unable to allocate TX ring memory\n"); 2863 error = ENOMEM; 2864 goto tx_fail; 2865 } 2866 2867 /* Next allocate the RX */ 2868 if (!(adapter->rx_rings = 2869 (struct rx_ring *) malloc(sizeof(struct rx_ring) * 2870 adapter->num_queues, M_DEVBUF, M_NOWAIT | M_ZERO))) { 2871 aprint_error_dev(dev, "Unable to allocate RX ring memory\n"); 2872 error = ENOMEM; 2873 goto rx_fail; 2874 } 2875 2876 /* For the ring itself */ 2877 tsize = roundup2(adapter->num_tx_desc * 2878 sizeof(union ixgbe_adv_tx_desc), DBA_ALIGN); 2879 2880 /* 2881 * Now set up the TX queues, txconf is needed to handle the 2882 * possibility that things fail midcourse and we need to 2883 * undo memory gracefully 2884 */ 2885 for (int i = 0; i < adapter->num_queues; i++, txconf++) { 2886 /* Set up some basics */ 2887 txr = &adapter->tx_rings[i]; 2888 txr->adapter = adapter; 2889 txr->me = i; 2890 2891 /* Initialize the TX side lock */ 2892 snprintf(txr->mtx_name, sizeof(txr->mtx_name), "%s:tx(%d)", 2893 device_xname(dev), txr->me); 2894 mutex_init(&txr->tx_mtx, MUTEX_DEFAULT, IPL_NET); 2895 2896 if (ixgbe_dma_malloc(adapter, tsize, 2897 &txr->txdma, BUS_DMA_NOWAIT)) { 2898 aprint_error_dev(dev, 2899 "Unable to allocate TX Descriptor memory\n"); 2900 error = ENOMEM; 2901 goto err_tx_desc; 2902 } 2903 txr->tx_base = (union ixgbe_adv_tx_desc *)txr->txdma.dma_vaddr; 2904 bzero((void *)txr->tx_base, tsize); 2905 2906 /* Now allocate transmit buffers for the ring */ 2907 if (ixgbe_allocate_transmit_buffers(txr)) { 2908 aprint_error_dev(dev, 2909 "Critical Failure setting up transmit buffers\n"); 2910 error = ENOMEM; 2911 goto err_tx_desc; 2912 } 2913 #if __FreeBSD_version >= 800000 2914 /* Allocate a buf ring */ 2915 txr->br = buf_ring_alloc(IXGBE_BR_SIZE, M_DEVBUF, 2916 M_WAITOK, &txr->tx_mtx); 2917 if (txr->br == NULL) { 2918 aprint_error_dev(dev, 2919 "Critical Failure setting up buf ring\n"); 2920 error = ENOMEM; 2921 goto err_tx_desc; 2922 } 2923 #endif 2924 } 2925 2926 /* 2927 * Next the RX queues... 2928 */ 2929 rsize = roundup2(adapter->num_rx_desc * 2930 sizeof(union ixgbe_adv_rx_desc), DBA_ALIGN); 2931 for (int i = 0; i < adapter->num_queues; i++, rxconf++) { 2932 rxr = &adapter->rx_rings[i]; 2933 /* Set up some basics */ 2934 rxr->adapter = adapter; 2935 rxr->me = i; 2936 2937 /* Initialize the RX side lock */ 2938 snprintf(rxr->mtx_name, sizeof(rxr->mtx_name), "%s:rx(%d)", 2939 device_xname(dev), rxr->me); 2940 mutex_init(&rxr->rx_mtx, MUTEX_DEFAULT, IPL_NET); 2941 2942 if (ixgbe_dma_malloc(adapter, rsize, 2943 &rxr->rxdma, BUS_DMA_NOWAIT)) { 2944 aprint_error_dev(dev, 2945 "Unable to allocate RxDescriptor memory\n"); 2946 error = ENOMEM; 2947 goto err_rx_desc; 2948 } 2949 rxr->rx_base = (union ixgbe_adv_rx_desc *)rxr->rxdma.dma_vaddr; 2950 bzero((void *)rxr->rx_base, rsize); 2951 2952 /* Allocate receive buffers for the ring*/ 2953 if (ixgbe_allocate_receive_buffers(rxr)) { 2954 aprint_error_dev(dev, 2955 "Critical Failure setting up receive buffers\n"); 2956 error = ENOMEM; 2957 goto err_rx_desc; 2958 } 2959 } 2960 2961 /* 2962 ** Finally set up the queue holding structs 2963 */ 2964 for (int i = 0; i < adapter->num_queues; i++) { 2965 que = &adapter->queues[i]; 2966 que->adapter = adapter; 2967 que->txr = &adapter->tx_rings[i]; 2968 que->rxr = &adapter->rx_rings[i]; 2969 } 2970 2971 return (0); 2972 2973 err_rx_desc: 2974 for (rxr = adapter->rx_rings; rxconf > 0; rxr++, rxconf--) 2975 ixgbe_dma_free(adapter, &rxr->rxdma); 2976 err_tx_desc: 2977 for (txr = adapter->tx_rings; txconf > 0; txr++, txconf--) 2978 ixgbe_dma_free(adapter, &txr->txdma); 2979 free(adapter->rx_rings, M_DEVBUF); 2980 rx_fail: 2981 free(adapter->tx_rings, M_DEVBUF); 2982 tx_fail: 2983 free(adapter->queues, M_DEVBUF); 2984 fail: 2985 return (error); 2986 } 2987 2988 /********************************************************************* 2989 * 2990 * Allocate memory for tx_buffer structures. The tx_buffer stores all 2991 * the information needed to transmit a packet on the wire. This is 2992 * called only once at attach, setup is done every reset. 2993 * 2994 **********************************************************************/ 2995 static int 2996 ixgbe_allocate_transmit_buffers(struct tx_ring *txr) 2997 { 2998 struct adapter *adapter = txr->adapter; 2999 device_t dev = adapter->dev; 3000 struct ixgbe_tx_buf *txbuf; 3001 int error, i; 3002 3003 /* 3004 * Setup DMA descriptor areas. 3005 */ 3006 if ((error = ixgbe_dma_tag_create(adapter->osdep.dmat, /* parent */ 3007 1, 0, /* alignment, bounds */ 3008 IXGBE_TSO_SIZE, /* maxsize */ 3009 adapter->num_segs, /* nsegments */ 3010 PAGE_SIZE, /* maxsegsize */ 3011 0, /* flags */ 3012 &txr->txtag))) { 3013 aprint_error_dev(dev,"Unable to allocate TX DMA tag\n"); 3014 goto fail; 3015 } 3016 3017 if (!(txr->tx_buffers = 3018 (struct ixgbe_tx_buf *) malloc(sizeof(struct ixgbe_tx_buf) * 3019 adapter->num_tx_desc, M_DEVBUF, M_NOWAIT | M_ZERO))) { 3020 aprint_error_dev(dev, "Unable to allocate tx_buffer memory\n"); 3021 error = ENOMEM; 3022 goto fail; 3023 } 3024 3025 /* Create the descriptor buffer dma maps */ 3026 txbuf = txr->tx_buffers; 3027 for (i = 0; i < adapter->num_tx_desc; i++, txbuf++) { 3028 error = ixgbe_dmamap_create(txr->txtag, 0, &txbuf->map); 3029 if (error != 0) { 3030 aprint_error_dev(dev, "Unable to create TX DMA map\n"); 3031 goto fail; 3032 } 3033 } 3034 3035 return 0; 3036 fail: 3037 /* We free all, it handles case where we are in the middle */ 3038 ixgbe_free_transmit_structures(adapter); 3039 return (error); 3040 } 3041 3042 /********************************************************************* 3043 * 3044 * Initialize a transmit ring. 3045 * 3046 **********************************************************************/ 3047 static void 3048 ixgbe_setup_transmit_ring(struct tx_ring *txr) 3049 { 3050 struct adapter *adapter = txr->adapter; 3051 struct ixgbe_tx_buf *txbuf; 3052 int i; 3053 3054 /* Clear the old ring contents */ 3055 IXGBE_TX_LOCK(txr); 3056 bzero((void *)txr->tx_base, 3057 (sizeof(union ixgbe_adv_tx_desc)) * adapter->num_tx_desc); 3058 /* Reset indices */ 3059 txr->next_avail_desc = 0; 3060 txr->next_to_clean = 0; 3061 3062 /* Free any existing tx buffers. */ 3063 txbuf = txr->tx_buffers; 3064 for (i = 0; i < adapter->num_tx_desc; i++, txbuf++) { 3065 if (txbuf->m_head != NULL) { 3066 bus_dmamap_sync(txr->txtag->dt_dmat, txbuf->map, 3067 0, txbuf->m_head->m_pkthdr.len, 3068 BUS_DMASYNC_POSTWRITE); 3069 ixgbe_dmamap_unload(txr->txtag, txbuf->map); 3070 m_freem(txbuf->m_head); 3071 txbuf->m_head = NULL; 3072 } 3073 /* Clear the EOP index */ 3074 txbuf->eop_index = -1; 3075 } 3076 3077 #ifdef IXGBE_FDIR 3078 /* Set the rate at which we sample packets */ 3079 if (adapter->hw.mac.type != ixgbe_mac_82598EB) 3080 txr->atr_sample = atr_sample_rate; 3081 #endif 3082 3083 /* Set number of descriptors available */ 3084 txr->tx_avail = adapter->num_tx_desc; 3085 3086 ixgbe_dmamap_sync(txr->txdma.dma_tag, txr->txdma.dma_map, 3087 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); 3088 IXGBE_TX_UNLOCK(txr); 3089 } 3090 3091 /********************************************************************* 3092 * 3093 * Initialize all transmit rings. 3094 * 3095 **********************************************************************/ 3096 static int 3097 ixgbe_setup_transmit_structures(struct adapter *adapter) 3098 { 3099 struct tx_ring *txr = adapter->tx_rings; 3100 3101 for (int i = 0; i < adapter->num_queues; i++, txr++) 3102 ixgbe_setup_transmit_ring(txr); 3103 3104 return (0); 3105 } 3106 3107 /********************************************************************* 3108 * 3109 * Enable transmit unit. 3110 * 3111 **********************************************************************/ 3112 static void 3113 ixgbe_initialize_transmit_units(struct adapter *adapter) 3114 { 3115 struct tx_ring *txr = adapter->tx_rings; 3116 struct ixgbe_hw *hw = &adapter->hw; 3117 3118 /* Setup the Base and Length of the Tx Descriptor Ring */ 3119 3120 for (int i = 0; i < adapter->num_queues; i++, txr++) { 3121 u64 tdba = txr->txdma.dma_paddr; 3122 u32 txctrl; 3123 3124 IXGBE_WRITE_REG(hw, IXGBE_TDBAL(i), 3125 (tdba & 0x00000000ffffffffULL)); 3126 IXGBE_WRITE_REG(hw, IXGBE_TDBAH(i), (tdba >> 32)); 3127 IXGBE_WRITE_REG(hw, IXGBE_TDLEN(i), 3128 adapter->num_tx_desc * sizeof(struct ixgbe_legacy_tx_desc)); 3129 3130 /* Setup the HW Tx Head and Tail descriptor pointers */ 3131 IXGBE_WRITE_REG(hw, IXGBE_TDH(i), 0); 3132 IXGBE_WRITE_REG(hw, IXGBE_TDT(i), 0); 3133 3134 /* Setup Transmit Descriptor Cmd Settings */ 3135 txr->txd_cmd = IXGBE_TXD_CMD_IFCS; 3136 txr->queue_status = IXGBE_QUEUE_IDLE; 3137 3138 /* Disable Head Writeback */ 3139 switch (hw->mac.type) { 3140 case ixgbe_mac_82598EB: 3141 txctrl = IXGBE_READ_REG(hw, IXGBE_DCA_TXCTRL(i)); 3142 break; 3143 case ixgbe_mac_82599EB: 3144 default: 3145 txctrl = IXGBE_READ_REG(hw, IXGBE_DCA_TXCTRL_82599(i)); 3146 break; 3147 } 3148 txctrl &= ~IXGBE_DCA_TXCTRL_TX_WB_RO_EN; 3149 switch (hw->mac.type) { 3150 case ixgbe_mac_82598EB: 3151 IXGBE_WRITE_REG(hw, IXGBE_DCA_TXCTRL(i), txctrl); 3152 break; 3153 case ixgbe_mac_82599EB: 3154 default: 3155 IXGBE_WRITE_REG(hw, IXGBE_DCA_TXCTRL_82599(i), txctrl); 3156 break; 3157 } 3158 3159 } 3160 3161 if (hw->mac.type != ixgbe_mac_82598EB) { 3162 u32 dmatxctl, rttdcs; 3163 dmatxctl = IXGBE_READ_REG(hw, IXGBE_DMATXCTL); 3164 dmatxctl |= IXGBE_DMATXCTL_TE; 3165 IXGBE_WRITE_REG(hw, IXGBE_DMATXCTL, dmatxctl); 3166 /* Disable arbiter to set MTQC */ 3167 rttdcs = IXGBE_READ_REG(hw, IXGBE_RTTDCS); 3168 rttdcs |= IXGBE_RTTDCS_ARBDIS; 3169 IXGBE_WRITE_REG(hw, IXGBE_RTTDCS, rttdcs); 3170 IXGBE_WRITE_REG(hw, IXGBE_MTQC, IXGBE_MTQC_64Q_1PB); 3171 rttdcs &= ~IXGBE_RTTDCS_ARBDIS; 3172 IXGBE_WRITE_REG(hw, IXGBE_RTTDCS, rttdcs); 3173 } 3174 3175 return; 3176 } 3177 3178 /********************************************************************* 3179 * 3180 * Free all transmit rings. 3181 * 3182 **********************************************************************/ 3183 static void 3184 ixgbe_free_transmit_structures(struct adapter *adapter) 3185 { 3186 struct tx_ring *txr = adapter->tx_rings; 3187 3188 for (int i = 0; i < adapter->num_queues; i++, txr++) { 3189 ixgbe_free_transmit_buffers(txr); 3190 ixgbe_dma_free(adapter, &txr->txdma); 3191 IXGBE_TX_LOCK_DESTROY(txr); 3192 } 3193 free(adapter->tx_rings, M_DEVBUF); 3194 } 3195 3196 /********************************************************************* 3197 * 3198 * Free transmit ring related data structures. 3199 * 3200 **********************************************************************/ 3201 static void 3202 ixgbe_free_transmit_buffers(struct tx_ring *txr) 3203 { 3204 struct adapter *adapter = txr->adapter; 3205 struct ixgbe_tx_buf *tx_buffer; 3206 int i; 3207 3208 INIT_DEBUGOUT("free_transmit_ring: begin"); 3209 3210 if (txr->tx_buffers == NULL) 3211 return; 3212 3213 tx_buffer = txr->tx_buffers; 3214 for (i = 0; i < adapter->num_tx_desc; i++, tx_buffer++) { 3215 if (tx_buffer->m_head != NULL) { 3216 bus_dmamap_sync(txr->txtag->dt_dmat, tx_buffer->map, 3217 0, tx_buffer->m_head->m_pkthdr.len, 3218 BUS_DMASYNC_POSTWRITE); 3219 ixgbe_dmamap_unload(txr->txtag, tx_buffer->map); 3220 m_freem(tx_buffer->m_head); 3221 tx_buffer->m_head = NULL; 3222 if (tx_buffer->map != NULL) { 3223 ixgbe_dmamap_destroy(txr->txtag, 3224 tx_buffer->map); 3225 tx_buffer->map = NULL; 3226 } 3227 } else if (tx_buffer->map != NULL) { 3228 ixgbe_dmamap_unload(txr->txtag, tx_buffer->map); 3229 ixgbe_dmamap_destroy(txr->txtag, tx_buffer->map); 3230 tx_buffer->map = NULL; 3231 } 3232 } 3233 #if __FreeBSD_version >= 800000 3234 if (txr->br != NULL) 3235 buf_ring_free(txr->br, M_DEVBUF); 3236 #endif 3237 if (txr->tx_buffers != NULL) { 3238 free(txr->tx_buffers, M_DEVBUF); 3239 txr->tx_buffers = NULL; 3240 } 3241 if (txr->txtag != NULL) { 3242 ixgbe_dma_tag_destroy(txr->txtag); 3243 txr->txtag = NULL; 3244 } 3245 return; 3246 } 3247 3248 /********************************************************************* 3249 * 3250 * Advanced Context Descriptor setup for VLAN or L4 CSUM 3251 * 3252 **********************************************************************/ 3253 3254 static u32 3255 ixgbe_tx_ctx_setup(struct tx_ring *txr, struct mbuf *mp) 3256 { 3257 struct m_tag *mtag; 3258 struct adapter *adapter = txr->adapter; 3259 struct ethercom *ec = &adapter->osdep.ec; 3260 struct ixgbe_adv_tx_context_desc *TXD; 3261 struct ixgbe_tx_buf *tx_buffer; 3262 u32 olinfo = 0, vlan_macip_lens = 0, type_tucmd_mlhl = 0; 3263 struct ether_vlan_header *eh; 3264 struct ip ip; 3265 struct ip6_hdr ip6; 3266 int ehdrlen, ip_hlen = 0; 3267 u16 etype; 3268 u8 ipproto __diagused = 0; 3269 bool offload; 3270 int ctxd = txr->next_avail_desc; 3271 u16 vtag = 0; 3272 3273 offload = ((mp->m_pkthdr.csum_flags & M_CSUM_OFFLOAD) != 0); 3274 3275 tx_buffer = &txr->tx_buffers[ctxd]; 3276 TXD = (struct ixgbe_adv_tx_context_desc *) &txr->tx_base[ctxd]; 3277 3278 /* 3279 ** In advanced descriptors the vlan tag must 3280 ** be placed into the descriptor itself. 3281 */ 3282 if ((mtag = VLAN_OUTPUT_TAG(ec, mp)) != NULL) { 3283 vtag = htole16(VLAN_TAG_VALUE(mtag) & 0xffff); 3284 vlan_macip_lens |= (vtag << IXGBE_ADVTXD_VLAN_SHIFT); 3285 } else if (!offload) 3286 return 0; 3287 3288 /* 3289 * Determine where frame payload starts. 3290 * Jump over vlan headers if already present, 3291 * helpful for QinQ too. 3292 */ 3293 KASSERT(mp->m_len >= offsetof(struct ether_vlan_header, evl_tag)); 3294 eh = mtod(mp, struct ether_vlan_header *); 3295 if (eh->evl_encap_proto == htons(ETHERTYPE_VLAN)) { 3296 KASSERT(mp->m_len >= sizeof(struct ether_vlan_header)); 3297 etype = ntohs(eh->evl_proto); 3298 ehdrlen = ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN; 3299 } else { 3300 etype = ntohs(eh->evl_encap_proto); 3301 ehdrlen = ETHER_HDR_LEN; 3302 } 3303 3304 /* Set the ether header length */ 3305 vlan_macip_lens |= ehdrlen << IXGBE_ADVTXD_MACLEN_SHIFT; 3306 3307 switch (etype) { 3308 case ETHERTYPE_IP: 3309 m_copydata(mp, ehdrlen, sizeof(ip), &ip); 3310 ip_hlen = ip.ip_hl << 2; 3311 ipproto = ip.ip_p; 3312 #if 0 3313 ip.ip_sum = 0; 3314 m_copyback(mp, ehdrlen, sizeof(ip), &ip); 3315 #else 3316 KASSERT((mp->m_pkthdr.csum_flags & M_CSUM_IPv4) == 0 || 3317 ip.ip_sum == 0); 3318 #endif 3319 type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_IPV4; 3320 break; 3321 case ETHERTYPE_IPV6: 3322 m_copydata(mp, ehdrlen, sizeof(ip6), &ip6); 3323 ip_hlen = sizeof(ip6); 3324 ipproto = ip6.ip6_nxt; 3325 type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_IPV6; 3326 break; 3327 default: 3328 break; 3329 } 3330 3331 if ((mp->m_pkthdr.csum_flags & M_CSUM_IPv4) != 0) 3332 olinfo |= IXGBE_TXD_POPTS_IXSM << 8; 3333 3334 vlan_macip_lens |= ip_hlen; 3335 type_tucmd_mlhl |= IXGBE_ADVTXD_DCMD_DEXT | IXGBE_ADVTXD_DTYP_CTXT; 3336 3337 if (mp->m_pkthdr.csum_flags & (M_CSUM_TCPv4|M_CSUM_TCPv6)) { 3338 type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_L4T_TCP; 3339 olinfo |= IXGBE_TXD_POPTS_TXSM << 8; 3340 KASSERT(ipproto == IPPROTO_TCP); 3341 } else if (mp->m_pkthdr.csum_flags & (M_CSUM_UDPv4|M_CSUM_UDPv6)) { 3342 type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_L4T_UDP; 3343 olinfo |= IXGBE_TXD_POPTS_TXSM << 8; 3344 KASSERT(ipproto == IPPROTO_UDP); 3345 } 3346 3347 /* Now copy bits into descriptor */ 3348 TXD->vlan_macip_lens |= htole32(vlan_macip_lens); 3349 TXD->type_tucmd_mlhl |= htole32(type_tucmd_mlhl); 3350 TXD->seqnum_seed = htole32(0); 3351 TXD->mss_l4len_idx = htole32(0); 3352 3353 tx_buffer->m_head = NULL; 3354 tx_buffer->eop_index = -1; 3355 3356 /* We've consumed the first desc, adjust counters */ 3357 if (++ctxd == adapter->num_tx_desc) 3358 ctxd = 0; 3359 txr->next_avail_desc = ctxd; 3360 --txr->tx_avail; 3361 3362 return olinfo; 3363 } 3364 3365 /********************************************************************** 3366 * 3367 * Setup work for hardware segmentation offload (TSO) on 3368 * adapters using advanced tx descriptors 3369 * 3370 **********************************************************************/ 3371 static bool 3372 ixgbe_tso_setup(struct tx_ring *txr, struct mbuf *mp, u32 *paylen) 3373 { 3374 struct m_tag *mtag; 3375 struct adapter *adapter = txr->adapter; 3376 struct ethercom *ec = &adapter->osdep.ec; 3377 struct ixgbe_adv_tx_context_desc *TXD; 3378 struct ixgbe_tx_buf *tx_buffer; 3379 u32 vlan_macip_lens = 0, type_tucmd_mlhl = 0; 3380 u32 mss_l4len_idx = 0; 3381 u16 vtag = 0; 3382 int ctxd, ehdrlen, hdrlen, ip_hlen, tcp_hlen; 3383 struct ether_vlan_header *eh; 3384 struct ip *ip; 3385 struct tcphdr *th; 3386 3387 3388 /* 3389 * Determine where frame payload starts. 3390 * Jump over vlan headers if already present 3391 */ 3392 eh = mtod(mp, struct ether_vlan_header *); 3393 if (eh->evl_encap_proto == htons(ETHERTYPE_VLAN)) 3394 ehdrlen = ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN; 3395 else 3396 ehdrlen = ETHER_HDR_LEN; 3397 3398 /* Ensure we have at least the IP+TCP header in the first mbuf. */ 3399 if (mp->m_len < ehdrlen + sizeof(struct ip) + sizeof(struct tcphdr)) 3400 return FALSE; 3401 3402 ctxd = txr->next_avail_desc; 3403 tx_buffer = &txr->tx_buffers[ctxd]; 3404 TXD = (struct ixgbe_adv_tx_context_desc *) &txr->tx_base[ctxd]; 3405 3406 ip = (struct ip *)(mp->m_data + ehdrlen); 3407 if (ip->ip_p != IPPROTO_TCP) 3408 return FALSE; /* 0 */ 3409 ip->ip_sum = 0; 3410 ip_hlen = ip->ip_hl << 2; 3411 th = (struct tcphdr *)((char *)ip + ip_hlen); 3412 /* XXX Educated guess: FreeBSD's in_pseudo == NetBSD's in_cksum_phdr */ 3413 th->th_sum = in_cksum_phdr(ip->ip_src.s_addr, 3414 ip->ip_dst.s_addr, htons(IPPROTO_TCP)); 3415 tcp_hlen = th->th_off << 2; 3416 hdrlen = ehdrlen + ip_hlen + tcp_hlen; 3417 3418 /* This is used in the transmit desc in encap */ 3419 *paylen = mp->m_pkthdr.len - hdrlen; 3420 3421 /* VLAN MACLEN IPLEN */ 3422 if ((mtag = VLAN_OUTPUT_TAG(ec, mp)) != NULL) { 3423 vtag = htole16(VLAN_TAG_VALUE(mtag) & 0xffff); 3424 vlan_macip_lens |= (vtag << IXGBE_ADVTXD_VLAN_SHIFT); 3425 } 3426 3427 vlan_macip_lens |= ehdrlen << IXGBE_ADVTXD_MACLEN_SHIFT; 3428 vlan_macip_lens |= ip_hlen; 3429 TXD->vlan_macip_lens |= htole32(vlan_macip_lens); 3430 3431 /* ADV DTYPE TUCMD */ 3432 type_tucmd_mlhl |= IXGBE_ADVTXD_DCMD_DEXT | IXGBE_ADVTXD_DTYP_CTXT; 3433 type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_L4T_TCP; 3434 type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_IPV4; 3435 TXD->type_tucmd_mlhl |= htole32(type_tucmd_mlhl); 3436 3437 3438 /* MSS L4LEN IDX */ 3439 mss_l4len_idx |= (mp->m_pkthdr.segsz << IXGBE_ADVTXD_MSS_SHIFT); 3440 mss_l4len_idx |= (tcp_hlen << IXGBE_ADVTXD_L4LEN_SHIFT); 3441 TXD->mss_l4len_idx = htole32(mss_l4len_idx); 3442 3443 TXD->seqnum_seed = htole32(0); 3444 tx_buffer->m_head = NULL; 3445 tx_buffer->eop_index = -1; 3446 3447 if (++ctxd == adapter->num_tx_desc) 3448 ctxd = 0; 3449 3450 txr->tx_avail--; 3451 txr->next_avail_desc = ctxd; 3452 return TRUE; 3453 } 3454 3455 #ifdef IXGBE_FDIR 3456 /* 3457 ** This routine parses packet headers so that Flow 3458 ** Director can make a hashed filter table entry 3459 ** allowing traffic flows to be identified and kept 3460 ** on the same cpu. This would be a performance 3461 ** hit, but we only do it at IXGBE_FDIR_RATE of 3462 ** packets. 3463 */ 3464 static void 3465 ixgbe_atr(struct tx_ring *txr, struct mbuf *mp) 3466 { 3467 struct adapter *adapter = txr->adapter; 3468 struct ix_queue *que; 3469 struct ip *ip; 3470 struct tcphdr *th; 3471 struct udphdr *uh; 3472 struct ether_vlan_header *eh; 3473 union ixgbe_atr_hash_dword input = {.dword = 0}; 3474 union ixgbe_atr_hash_dword common = {.dword = 0}; 3475 int ehdrlen, ip_hlen; 3476 u16 etype; 3477 3478 eh = mtod(mp, struct ether_vlan_header *); 3479 if (eh->evl_encap_proto == htons(ETHERTYPE_VLAN)) { 3480 ehdrlen = ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN; 3481 etype = eh->evl_proto; 3482 } else { 3483 ehdrlen = ETHER_HDR_LEN; 3484 etype = eh->evl_encap_proto; 3485 } 3486 3487 /* Only handling IPv4 */ 3488 if (etype != htons(ETHERTYPE_IP)) 3489 return; 3490 3491 ip = (struct ip *)(mp->m_data + ehdrlen); 3492 ip_hlen = ip->ip_hl << 2; 3493 3494 /* check if we're UDP or TCP */ 3495 switch (ip->ip_p) { 3496 case IPPROTO_TCP: 3497 th = (struct tcphdr *)((char *)ip + ip_hlen); 3498 /* src and dst are inverted */ 3499 common.port.dst ^= th->th_sport; 3500 common.port.src ^= th->th_dport; 3501 input.formatted.flow_type ^= IXGBE_ATR_FLOW_TYPE_TCPV4; 3502 break; 3503 case IPPROTO_UDP: 3504 uh = (struct udphdr *)((char *)ip + ip_hlen); 3505 /* src and dst are inverted */ 3506 common.port.dst ^= uh->uh_sport; 3507 common.port.src ^= uh->uh_dport; 3508 input.formatted.flow_type ^= IXGBE_ATR_FLOW_TYPE_UDPV4; 3509 break; 3510 default: 3511 return; 3512 } 3513 3514 input.formatted.vlan_id = htobe16(mp->m_pkthdr.ether_vtag); 3515 if (mp->m_pkthdr.ether_vtag) 3516 common.flex_bytes ^= htons(ETHERTYPE_VLAN); 3517 else 3518 common.flex_bytes ^= etype; 3519 common.ip ^= ip->ip_src.s_addr ^ ip->ip_dst.s_addr; 3520 3521 que = &adapter->queues[txr->me]; 3522 /* 3523 ** This assumes the Rx queue and Tx 3524 ** queue are bound to the same CPU 3525 */ 3526 ixgbe_fdir_add_signature_filter_82599(&adapter->hw, 3527 input, common, que->msix); 3528 } 3529 #endif /* IXGBE_FDIR */ 3530 3531 /********************************************************************** 3532 * 3533 * Examine each tx_buffer in the used queue. If the hardware is done 3534 * processing the packet then free associated resources. The 3535 * tx_buffer is put back on the free queue. 3536 * 3537 **********************************************************************/ 3538 static bool 3539 ixgbe_txeof(struct tx_ring *txr) 3540 { 3541 struct adapter *adapter = txr->adapter; 3542 struct ifnet *ifp = adapter->ifp; 3543 u32 first, last, done, processed; 3544 struct ixgbe_tx_buf *tx_buffer; 3545 struct ixgbe_legacy_tx_desc *tx_desc, *eop_desc; 3546 struct timeval now, elapsed; 3547 3548 KASSERT(mutex_owned(&txr->tx_mtx)); 3549 3550 if (txr->tx_avail == adapter->num_tx_desc) { 3551 txr->queue_status = IXGBE_QUEUE_IDLE; 3552 return false; 3553 } 3554 3555 processed = 0; 3556 first = txr->next_to_clean; 3557 tx_buffer = &txr->tx_buffers[first]; 3558 /* For cleanup we just use legacy struct */ 3559 tx_desc = (struct ixgbe_legacy_tx_desc *)&txr->tx_base[first]; 3560 last = tx_buffer->eop_index; 3561 if (last == -1) 3562 return false; 3563 eop_desc = (struct ixgbe_legacy_tx_desc *)&txr->tx_base[last]; 3564 3565 /* 3566 ** Get the index of the first descriptor 3567 ** BEYOND the EOP and call that 'done'. 3568 ** I do this so the comparison in the 3569 ** inner while loop below can be simple 3570 */ 3571 if (++last == adapter->num_tx_desc) last = 0; 3572 done = last; 3573 3574 ixgbe_dmamap_sync(txr->txdma.dma_tag, txr->txdma.dma_map, 3575 BUS_DMASYNC_POSTREAD); 3576 /* 3577 ** Only the EOP descriptor of a packet now has the DD 3578 ** bit set, this is what we look for... 3579 */ 3580 while (eop_desc->upper.fields.status & IXGBE_TXD_STAT_DD) { 3581 /* We clean the range of the packet */ 3582 while (first != done) { 3583 tx_desc->upper.data = 0; 3584 tx_desc->lower.data = 0; 3585 tx_desc->buffer_addr = 0; 3586 ++txr->tx_avail; 3587 ++processed; 3588 3589 if (tx_buffer->m_head) { 3590 txr->bytes += 3591 tx_buffer->m_head->m_pkthdr.len; 3592 bus_dmamap_sync(txr->txtag->dt_dmat, 3593 tx_buffer->map, 3594 0, tx_buffer->m_head->m_pkthdr.len, 3595 BUS_DMASYNC_POSTWRITE); 3596 ixgbe_dmamap_unload(txr->txtag, tx_buffer->map); 3597 m_freem(tx_buffer->m_head); 3598 tx_buffer->m_head = NULL; 3599 } 3600 tx_buffer->eop_index = -1; 3601 getmicrotime(&txr->watchdog_time); 3602 3603 if (++first == adapter->num_tx_desc) 3604 first = 0; 3605 3606 tx_buffer = &txr->tx_buffers[first]; 3607 tx_desc = 3608 (struct ixgbe_legacy_tx_desc *)&txr->tx_base[first]; 3609 } 3610 ++txr->packets; 3611 ++ifp->if_opackets; 3612 /* See if there is more work now */ 3613 last = tx_buffer->eop_index; 3614 if (last != -1) { 3615 eop_desc = 3616 (struct ixgbe_legacy_tx_desc *)&txr->tx_base[last]; 3617 /* Get next done point */ 3618 if (++last == adapter->num_tx_desc) last = 0; 3619 done = last; 3620 } else 3621 break; 3622 } 3623 ixgbe_dmamap_sync(txr->txdma.dma_tag, txr->txdma.dma_map, 3624 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); 3625 3626 txr->next_to_clean = first; 3627 3628 /* 3629 ** Watchdog calculation, we know there's 3630 ** work outstanding or the first return 3631 ** would have been taken, so none processed 3632 ** for too long indicates a hang. 3633 */ 3634 getmicrotime(&now); 3635 timersub(&now, &txr->watchdog_time, &elapsed); 3636 if (!processed && tvtohz(&elapsed) > IXGBE_WATCHDOG) 3637 txr->queue_status = IXGBE_QUEUE_HUNG; 3638 3639 /* 3640 * If we have enough room, clear IFF_OACTIVE to tell the stack that 3641 * it is OK to send packets. If there are no pending descriptors, 3642 * clear the timeout. Otherwise, if some descriptors have been freed, 3643 * restart the timeout. 3644 */ 3645 if (txr->tx_avail > IXGBE_TX_CLEANUP_THRESHOLD) { 3646 ifp->if_flags &= ~IFF_OACTIVE; 3647 if (txr->tx_avail == adapter->num_tx_desc) { 3648 txr->queue_status = IXGBE_QUEUE_IDLE; 3649 return false; 3650 } 3651 } 3652 3653 return true; 3654 } 3655 3656 /********************************************************************* 3657 * 3658 * Refresh mbuf buffers for RX descriptor rings 3659 * - now keeps its own state so discards due to resource 3660 * exhaustion are unnecessary, if an mbuf cannot be obtained 3661 * it just returns, keeping its placeholder, thus it can simply 3662 * be recalled to try again. 3663 * 3664 **********************************************************************/ 3665 static void 3666 ixgbe_refresh_mbufs(struct rx_ring *rxr, int limit) 3667 { 3668 struct adapter *adapter = rxr->adapter; 3669 struct ixgbe_rx_buf *rxbuf; 3670 struct mbuf *mh, *mp; 3671 int i, j, error; 3672 bool refreshed = false; 3673 3674 i = j = rxr->next_to_refresh; 3675 /* Control the loop with one beyond */ 3676 if (++j == adapter->num_rx_desc) 3677 j = 0; 3678 3679 while (j != limit) { 3680 rxbuf = &rxr->rx_buffers[i]; 3681 if (rxr->hdr_split == FALSE) 3682 goto no_split; 3683 3684 if (rxbuf->m_head == NULL) { 3685 mh = m_gethdr(M_DONTWAIT, MT_DATA); 3686 if (mh == NULL) 3687 goto update; 3688 } else 3689 mh = rxbuf->m_head; 3690 3691 mh->m_pkthdr.len = mh->m_len = MHLEN; 3692 mh->m_len = MHLEN; 3693 mh->m_flags |= M_PKTHDR; 3694 /* Get the memory mapping */ 3695 error = bus_dmamap_load_mbuf(rxr->htag->dt_dmat, 3696 rxbuf->hmap, mh, BUS_DMA_NOWAIT); 3697 if (error != 0) { 3698 printf("Refresh mbufs: hdr dmamap load" 3699 " failure - %d\n", error); 3700 m_free(mh); 3701 rxbuf->m_head = NULL; 3702 goto update; 3703 } 3704 rxbuf->m_head = mh; 3705 ixgbe_dmamap_sync(rxr->htag, rxbuf->hmap, BUS_DMASYNC_PREREAD); 3706 rxr->rx_base[i].read.hdr_addr = 3707 htole64(rxbuf->hmap->dm_segs[0].ds_addr); 3708 3709 no_split: 3710 if (rxbuf->m_pack == NULL) { 3711 mp = ixgbe_getjcl(&adapter->jcl_head, M_DONTWAIT, 3712 MT_DATA, M_PKTHDR, adapter->rx_mbuf_sz); 3713 if (mp == NULL) { 3714 rxr->no_jmbuf.ev_count++; 3715 goto update; 3716 } 3717 } else 3718 mp = rxbuf->m_pack; 3719 3720 mp->m_pkthdr.len = mp->m_len = adapter->rx_mbuf_sz; 3721 /* Get the memory mapping */ 3722 error = bus_dmamap_load_mbuf(rxr->ptag->dt_dmat, 3723 rxbuf->pmap, mp, BUS_DMA_NOWAIT); 3724 if (error != 0) { 3725 printf("Refresh mbufs: payload dmamap load" 3726 " failure - %d\n", error); 3727 m_free(mp); 3728 rxbuf->m_pack = NULL; 3729 goto update; 3730 } 3731 rxbuf->m_pack = mp; 3732 bus_dmamap_sync(rxr->ptag->dt_dmat, rxbuf->pmap, 3733 0, mp->m_pkthdr.len, BUS_DMASYNC_PREREAD); 3734 rxr->rx_base[i].read.pkt_addr = 3735 htole64(rxbuf->pmap->dm_segs[0].ds_addr); 3736 3737 refreshed = true; 3738 /* Next is precalculated */ 3739 i = j; 3740 rxr->next_to_refresh = i; 3741 if (++j == adapter->num_rx_desc) 3742 j = 0; 3743 } 3744 update: 3745 if (refreshed) /* Update hardware tail index */ 3746 IXGBE_WRITE_REG(&adapter->hw, 3747 IXGBE_RDT(rxr->me), rxr->next_to_refresh); 3748 return; 3749 } 3750 3751 /********************************************************************* 3752 * 3753 * Allocate memory for rx_buffer structures. Since we use one 3754 * rx_buffer per received packet, the maximum number of rx_buffer's 3755 * that we'll need is equal to the number of receive descriptors 3756 * that we've allocated. 3757 * 3758 **********************************************************************/ 3759 static int 3760 ixgbe_allocate_receive_buffers(struct rx_ring *rxr) 3761 { 3762 struct adapter *adapter = rxr->adapter; 3763 device_t dev = adapter->dev; 3764 struct ixgbe_rx_buf *rxbuf; 3765 int i, bsize, error; 3766 3767 bsize = sizeof(struct ixgbe_rx_buf) * adapter->num_rx_desc; 3768 if (!(rxr->rx_buffers = 3769 (struct ixgbe_rx_buf *) malloc(bsize, 3770 M_DEVBUF, M_NOWAIT | M_ZERO))) { 3771 aprint_error_dev(dev, "Unable to allocate rx_buffer memory\n"); 3772 error = ENOMEM; 3773 goto fail; 3774 } 3775 3776 if ((error = ixgbe_dma_tag_create(adapter->osdep.dmat, /* parent */ 3777 1, 0, /* alignment, bounds */ 3778 MSIZE, /* maxsize */ 3779 1, /* nsegments */ 3780 MSIZE, /* maxsegsize */ 3781 0, /* flags */ 3782 &rxr->htag))) { 3783 aprint_error_dev(dev, "Unable to create RX DMA tag\n"); 3784 goto fail; 3785 } 3786 3787 if ((error = ixgbe_dma_tag_create(adapter->osdep.dmat, /* parent */ 3788 1, 0, /* alignment, bounds */ 3789 MJUM16BYTES, /* maxsize */ 3790 1, /* nsegments */ 3791 MJUM16BYTES, /* maxsegsize */ 3792 0, /* flags */ 3793 &rxr->ptag))) { 3794 aprint_error_dev(dev, "Unable to create RX DMA tag\n"); 3795 goto fail; 3796 } 3797 3798 for (i = 0; i < adapter->num_rx_desc; i++, rxbuf++) { 3799 rxbuf = &rxr->rx_buffers[i]; 3800 error = ixgbe_dmamap_create(rxr->htag, 3801 BUS_DMA_NOWAIT, &rxbuf->hmap); 3802 if (error) { 3803 aprint_error_dev(dev, "Unable to create RX head map\n"); 3804 goto fail; 3805 } 3806 error = ixgbe_dmamap_create(rxr->ptag, 3807 BUS_DMA_NOWAIT, &rxbuf->pmap); 3808 if (error) { 3809 aprint_error_dev(dev, "Unable to create RX pkt map\n"); 3810 goto fail; 3811 } 3812 } 3813 3814 return (0); 3815 3816 fail: 3817 /* Frees all, but can handle partial completion */ 3818 ixgbe_free_receive_structures(adapter); 3819 return (error); 3820 } 3821 3822 /* 3823 ** Used to detect a descriptor that has 3824 ** been merged by Hardware RSC. 3825 */ 3826 static inline u32 3827 ixgbe_rsc_count(union ixgbe_adv_rx_desc *rx) 3828 { 3829 return (le32toh(rx->wb.lower.lo_dword.data) & 3830 IXGBE_RXDADV_RSCCNT_MASK) >> IXGBE_RXDADV_RSCCNT_SHIFT; 3831 } 3832 3833 /********************************************************************* 3834 * 3835 * Initialize Hardware RSC (LRO) feature on 82599 3836 * for an RX ring, this is toggled by the LRO capability 3837 * even though it is transparent to the stack. 3838 * 3839 **********************************************************************/ 3840 static void 3841 ixgbe_setup_hw_rsc(struct rx_ring *rxr) 3842 { 3843 struct adapter *adapter = rxr->adapter; 3844 struct ixgbe_hw *hw = &adapter->hw; 3845 u32 rscctrl, rdrxctl; 3846 3847 rdrxctl = IXGBE_READ_REG(hw, IXGBE_RDRXCTL); 3848 rdrxctl &= ~IXGBE_RDRXCTL_RSCFRSTSIZE; 3849 rdrxctl |= IXGBE_RDRXCTL_CRCSTRIP; 3850 rdrxctl |= IXGBE_RDRXCTL_RSCACKC; 3851 IXGBE_WRITE_REG(hw, IXGBE_RDRXCTL, rdrxctl); 3852 3853 rscctrl = IXGBE_READ_REG(hw, IXGBE_RSCCTL(rxr->me)); 3854 rscctrl |= IXGBE_RSCCTL_RSCEN; 3855 /* 3856 ** Limit the total number of descriptors that 3857 ** can be combined, so it does not exceed 64K 3858 */ 3859 if (adapter->rx_mbuf_sz == MCLBYTES) 3860 rscctrl |= IXGBE_RSCCTL_MAXDESC_16; 3861 else if (adapter->rx_mbuf_sz == MJUMPAGESIZE) 3862 rscctrl |= IXGBE_RSCCTL_MAXDESC_8; 3863 else if (adapter->rx_mbuf_sz == MJUM9BYTES) 3864 rscctrl |= IXGBE_RSCCTL_MAXDESC_4; 3865 else /* Using 16K cluster */ 3866 rscctrl |= IXGBE_RSCCTL_MAXDESC_1; 3867 3868 IXGBE_WRITE_REG(hw, IXGBE_RSCCTL(rxr->me), rscctrl); 3869 3870 /* Enable TCP header recognition */ 3871 IXGBE_WRITE_REG(hw, IXGBE_PSRTYPE(0), 3872 (IXGBE_READ_REG(hw, IXGBE_PSRTYPE(0)) | 3873 IXGBE_PSRTYPE_TCPHDR)); 3874 3875 /* Disable RSC for ACK packets */ 3876 IXGBE_WRITE_REG(hw, IXGBE_RSCDBU, 3877 (IXGBE_RSCDBU_RSCACKDIS | IXGBE_READ_REG(hw, IXGBE_RSCDBU))); 3878 3879 rxr->hw_rsc = TRUE; 3880 } 3881 3882 3883 static void 3884 ixgbe_free_receive_ring(struct rx_ring *rxr) 3885 { 3886 struct adapter *adapter; 3887 struct ixgbe_rx_buf *rxbuf; 3888 int i; 3889 3890 adapter = rxr->adapter; 3891 for (i = 0; i < adapter->num_rx_desc; i++) { 3892 rxbuf = &rxr->rx_buffers[i]; 3893 if (rxbuf->m_head != NULL) { 3894 ixgbe_dmamap_sync(rxr->htag, rxbuf->hmap, 3895 BUS_DMASYNC_POSTREAD); 3896 ixgbe_dmamap_unload(rxr->htag, rxbuf->hmap); 3897 rxbuf->m_head->m_flags |= M_PKTHDR; 3898 m_freem(rxbuf->m_head); 3899 } 3900 if (rxbuf->m_pack != NULL) { 3901 bus_dmamap_sync(rxr->ptag->dt_dmat, rxbuf->pmap, 3902 0, rxbuf->m_pack->m_pkthdr.len, 3903 BUS_DMASYNC_POSTREAD); 3904 ixgbe_dmamap_unload(rxr->ptag, rxbuf->pmap); 3905 rxbuf->m_pack->m_flags |= M_PKTHDR; 3906 m_freem(rxbuf->m_pack); 3907 } 3908 rxbuf->m_head = NULL; 3909 rxbuf->m_pack = NULL; 3910 } 3911 } 3912 3913 3914 /********************************************************************* 3915 * 3916 * Initialize a receive ring and its buffers. 3917 * 3918 **********************************************************************/ 3919 static int 3920 ixgbe_setup_receive_ring(struct rx_ring *rxr) 3921 { 3922 struct adapter *adapter; 3923 struct ifnet *ifp; 3924 struct ixgbe_rx_buf *rxbuf; 3925 #ifdef LRO 3926 struct lro_ctrl *lro = &rxr->lro; 3927 #endif /* LRO */ 3928 int rsize, error = 0; 3929 3930 adapter = rxr->adapter; 3931 ifp = adapter->ifp; 3932 3933 /* Clear the ring contents */ 3934 IXGBE_RX_LOCK(rxr); 3935 rsize = roundup2(adapter->num_rx_desc * 3936 sizeof(union ixgbe_adv_rx_desc), DBA_ALIGN); 3937 bzero((void *)rxr->rx_base, rsize); 3938 3939 /* Free current RX buffer structs and their mbufs */ 3940 ixgbe_free_receive_ring(rxr); 3941 3942 IXGBE_RX_UNLOCK(rxr); 3943 3944 /* Now reinitialize our supply of jumbo mbufs. The number 3945 * or size of jumbo mbufs may have changed. 3946 */ 3947 ixgbe_jcl_reinit(&adapter->jcl_head, rxr->ptag->dt_dmat, 3948 2 * adapter->num_rx_desc, adapter->rx_mbuf_sz); 3949 3950 IXGBE_RX_LOCK(rxr); 3951 3952 /* Configure header split? */ 3953 if (ixgbe_header_split) 3954 rxr->hdr_split = TRUE; 3955 3956 /* Now replenish the mbufs */ 3957 for (int j = 0; j != adapter->num_rx_desc; ++j) { 3958 struct mbuf *mh, *mp; 3959 3960 rxbuf = &rxr->rx_buffers[j]; 3961 /* 3962 ** Don't allocate mbufs if not 3963 ** doing header split, its wasteful 3964 */ 3965 if (rxr->hdr_split == FALSE) 3966 goto skip_head; 3967 3968 /* First the header */ 3969 rxbuf->m_head = m_gethdr(M_DONTWAIT, MT_DATA); 3970 if (rxbuf->m_head == NULL) { 3971 error = ENOBUFS; 3972 goto fail; 3973 } 3974 m_adj(rxbuf->m_head, ETHER_ALIGN); 3975 mh = rxbuf->m_head; 3976 mh->m_len = mh->m_pkthdr.len = MHLEN; 3977 mh->m_flags |= M_PKTHDR; 3978 /* Get the memory mapping */ 3979 error = bus_dmamap_load_mbuf(rxr->htag->dt_dmat, 3980 rxbuf->hmap, rxbuf->m_head, BUS_DMA_NOWAIT); 3981 if (error != 0) /* Nothing elegant to do here */ 3982 goto fail; 3983 bus_dmamap_sync(rxr->htag->dt_dmat, rxbuf->hmap, 3984 0, mh->m_pkthdr.len, BUS_DMASYNC_PREREAD); 3985 /* Update descriptor */ 3986 rxr->rx_base[j].read.hdr_addr = 3987 htole64(rxbuf->hmap->dm_segs[0].ds_addr); 3988 3989 skip_head: 3990 /* Now the payload cluster */ 3991 rxbuf->m_pack = ixgbe_getjcl(&adapter->jcl_head, M_DONTWAIT, 3992 MT_DATA, M_PKTHDR, adapter->rx_mbuf_sz); 3993 if (rxbuf->m_pack == NULL) { 3994 error = ENOBUFS; 3995 goto fail; 3996 } 3997 mp = rxbuf->m_pack; 3998 mp->m_pkthdr.len = mp->m_len = adapter->rx_mbuf_sz; 3999 /* Get the memory mapping */ 4000 error = bus_dmamap_load_mbuf(rxr->ptag->dt_dmat, 4001 rxbuf->pmap, mp, BUS_DMA_NOWAIT); 4002 if (error != 0) 4003 goto fail; 4004 bus_dmamap_sync(rxr->ptag->dt_dmat, rxbuf->pmap, 4005 0, adapter->rx_mbuf_sz, BUS_DMASYNC_PREREAD); 4006 /* Update descriptor */ 4007 rxr->rx_base[j].read.pkt_addr = 4008 htole64(rxbuf->pmap->dm_segs[0].ds_addr); 4009 } 4010 4011 4012 /* Setup our descriptor indices */ 4013 rxr->next_to_check = 0; 4014 rxr->next_to_refresh = 0; 4015 rxr->lro_enabled = FALSE; 4016 rxr->rx_split_packets.ev_count = 0; 4017 rxr->rx_bytes.ev_count = 0; 4018 rxr->discard = FALSE; 4019 4020 ixgbe_dmamap_sync(rxr->rxdma.dma_tag, rxr->rxdma.dma_map, 4021 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); 4022 4023 /* 4024 ** Now set up the LRO interface: 4025 ** 82598 uses software LRO, the 4026 ** 82599 uses a hardware assist. 4027 */ 4028 if ((adapter->hw.mac.type != ixgbe_mac_82598EB) && 4029 (ifp->if_capenable & IFCAP_RXCSUM) && 4030 (ifp->if_capenable & IFCAP_LRO)) 4031 ixgbe_setup_hw_rsc(rxr); 4032 #ifdef LRO 4033 else if (ifp->if_capenable & IFCAP_LRO) { 4034 device_t dev = adapter->dev; 4035 int err = tcp_lro_init(lro); 4036 if (err) { 4037 device_printf(dev, "LRO Initialization failed!\n"); 4038 goto fail; 4039 } 4040 INIT_DEBUGOUT("RX Soft LRO Initialized\n"); 4041 rxr->lro_enabled = TRUE; 4042 lro->ifp = adapter->ifp; 4043 } 4044 #endif /* LRO */ 4045 4046 IXGBE_RX_UNLOCK(rxr); 4047 return (0); 4048 4049 fail: 4050 ixgbe_free_receive_ring(rxr); 4051 IXGBE_RX_UNLOCK(rxr); 4052 return (error); 4053 } 4054 4055 /********************************************************************* 4056 * 4057 * Initialize all receive rings. 4058 * 4059 **********************************************************************/ 4060 static int 4061 ixgbe_setup_receive_structures(struct adapter *adapter) 4062 { 4063 struct rx_ring *rxr = adapter->rx_rings; 4064 int j; 4065 4066 for (j = 0; j < adapter->num_queues; j++, rxr++) 4067 if (ixgbe_setup_receive_ring(rxr)) 4068 goto fail; 4069 4070 return (0); 4071 fail: 4072 /* 4073 * Free RX buffers allocated so far, we will only handle 4074 * the rings that completed, the failing case will have 4075 * cleaned up for itself. 'j' failed, so its the terminus. 4076 */ 4077 for (int i = 0; i < j; ++i) { 4078 rxr = &adapter->rx_rings[i]; 4079 ixgbe_free_receive_ring(rxr); 4080 } 4081 4082 return (ENOBUFS); 4083 } 4084 4085 /********************************************************************* 4086 * 4087 * Setup receive registers and features. 4088 * 4089 **********************************************************************/ 4090 #define IXGBE_SRRCTL_BSIZEHDRSIZE_SHIFT 2 4091 4092 static void 4093 ixgbe_initialize_receive_units(struct adapter *adapter) 4094 { 4095 int i; 4096 struct rx_ring *rxr = adapter->rx_rings; 4097 struct ixgbe_hw *hw = &adapter->hw; 4098 struct ifnet *ifp = adapter->ifp; 4099 u32 bufsz, rxctrl, fctrl, srrctl, rxcsum; 4100 u32 reta, mrqc = 0, hlreg, r[10]; 4101 4102 4103 /* 4104 * Make sure receives are disabled while 4105 * setting up the descriptor ring 4106 */ 4107 rxctrl = IXGBE_READ_REG(hw, IXGBE_RXCTRL); 4108 IXGBE_WRITE_REG(hw, IXGBE_RXCTRL, 4109 rxctrl & ~IXGBE_RXCTRL_RXEN); 4110 4111 /* Enable broadcasts */ 4112 fctrl = IXGBE_READ_REG(hw, IXGBE_FCTRL); 4113 fctrl |= IXGBE_FCTRL_BAM; 4114 fctrl |= IXGBE_FCTRL_DPF; 4115 fctrl |= IXGBE_FCTRL_PMCF; 4116 IXGBE_WRITE_REG(hw, IXGBE_FCTRL, fctrl); 4117 4118 /* Set for Jumbo Frames? */ 4119 hlreg = IXGBE_READ_REG(hw, IXGBE_HLREG0); 4120 if (ifp->if_mtu > ETHERMTU) 4121 hlreg |= IXGBE_HLREG0_JUMBOEN; 4122 else 4123 hlreg &= ~IXGBE_HLREG0_JUMBOEN; 4124 IXGBE_WRITE_REG(hw, IXGBE_HLREG0, hlreg); 4125 4126 bufsz = adapter->rx_mbuf_sz >> IXGBE_SRRCTL_BSIZEPKT_SHIFT; 4127 4128 for (i = 0; i < adapter->num_queues; i++, rxr++) { 4129 u64 rdba = rxr->rxdma.dma_paddr; 4130 4131 /* Setup the Base and Length of the Rx Descriptor Ring */ 4132 IXGBE_WRITE_REG(hw, IXGBE_RDBAL(i), 4133 (rdba & 0x00000000ffffffffULL)); 4134 IXGBE_WRITE_REG(hw, IXGBE_RDBAH(i), (rdba >> 32)); 4135 IXGBE_WRITE_REG(hw, IXGBE_RDLEN(i), 4136 adapter->num_rx_desc * sizeof(union ixgbe_adv_rx_desc)); 4137 4138 /* Set up the SRRCTL register */ 4139 srrctl = IXGBE_READ_REG(hw, IXGBE_SRRCTL(i)); 4140 srrctl &= ~IXGBE_SRRCTL_BSIZEHDR_MASK; 4141 srrctl &= ~IXGBE_SRRCTL_BSIZEPKT_MASK; 4142 srrctl |= bufsz; 4143 if (rxr->hdr_split) { 4144 /* Use a standard mbuf for the header */ 4145 srrctl |= ((IXGBE_RX_HDR << 4146 IXGBE_SRRCTL_BSIZEHDRSIZE_SHIFT) 4147 & IXGBE_SRRCTL_BSIZEHDR_MASK); 4148 srrctl |= IXGBE_SRRCTL_DESCTYPE_HDR_SPLIT_ALWAYS; 4149 } else 4150 srrctl |= IXGBE_SRRCTL_DESCTYPE_ADV_ONEBUF; 4151 IXGBE_WRITE_REG(hw, IXGBE_SRRCTL(i), srrctl); 4152 4153 /* Setup the HW Rx Head and Tail Descriptor Pointers */ 4154 IXGBE_WRITE_REG(hw, IXGBE_RDH(i), 0); 4155 IXGBE_WRITE_REG(hw, IXGBE_RDT(i), 0); 4156 } 4157 4158 if (adapter->hw.mac.type != ixgbe_mac_82598EB) { 4159 u32 psrtype = IXGBE_PSRTYPE_TCPHDR | 4160 IXGBE_PSRTYPE_UDPHDR | 4161 IXGBE_PSRTYPE_IPV4HDR | 4162 IXGBE_PSRTYPE_IPV6HDR; 4163 IXGBE_WRITE_REG(hw, IXGBE_PSRTYPE(0), psrtype); 4164 } 4165 4166 rxcsum = IXGBE_READ_REG(hw, IXGBE_RXCSUM); 4167 4168 /* Setup RSS */ 4169 if (adapter->num_queues > 1) { 4170 int j; 4171 reta = 0; 4172 4173 /* set up random bits */ 4174 cprng_fast(&r, sizeof(r)); 4175 4176 /* Set up the redirection table */ 4177 for (i = 0, j = 0; i < 128; i++, j++) { 4178 if (j == adapter->num_queues) j = 0; 4179 reta = (reta << 8) | (j * 0x11); 4180 if ((i & 3) == 3) 4181 IXGBE_WRITE_REG(hw, IXGBE_RETA(i >> 2), reta); 4182 } 4183 4184 /* Now fill our hash function seeds */ 4185 for (i = 0; i < 10; i++) 4186 IXGBE_WRITE_REG(hw, IXGBE_RSSRK(i), r[i]); 4187 4188 /* Perform hash on these packet types */ 4189 mrqc = IXGBE_MRQC_RSSEN 4190 | IXGBE_MRQC_RSS_FIELD_IPV4 4191 | IXGBE_MRQC_RSS_FIELD_IPV4_TCP 4192 | IXGBE_MRQC_RSS_FIELD_IPV4_UDP 4193 | IXGBE_MRQC_RSS_FIELD_IPV6_EX_TCP 4194 | IXGBE_MRQC_RSS_FIELD_IPV6_EX 4195 | IXGBE_MRQC_RSS_FIELD_IPV6 4196 | IXGBE_MRQC_RSS_FIELD_IPV6_TCP 4197 | IXGBE_MRQC_RSS_FIELD_IPV6_UDP 4198 | IXGBE_MRQC_RSS_FIELD_IPV6_EX_UDP; 4199 IXGBE_WRITE_REG(hw, IXGBE_MRQC, mrqc); 4200 4201 /* RSS and RX IPP Checksum are mutually exclusive */ 4202 rxcsum |= IXGBE_RXCSUM_PCSD; 4203 } 4204 4205 if (ifp->if_capenable & IFCAP_RXCSUM) 4206 rxcsum |= IXGBE_RXCSUM_PCSD; 4207 4208 if (!(rxcsum & IXGBE_RXCSUM_PCSD)) 4209 rxcsum |= IXGBE_RXCSUM_IPPCSE; 4210 4211 IXGBE_WRITE_REG(hw, IXGBE_RXCSUM, rxcsum); 4212 4213 return; 4214 } 4215 4216 /********************************************************************* 4217 * 4218 * Free all receive rings. 4219 * 4220 **********************************************************************/ 4221 static void 4222 ixgbe_free_receive_structures(struct adapter *adapter) 4223 { 4224 struct rx_ring *rxr = adapter->rx_rings; 4225 4226 for (int i = 0; i < adapter->num_queues; i++, rxr++) { 4227 #ifdef LRO 4228 struct lro_ctrl *lro = &rxr->lro; 4229 #endif /* LRO */ 4230 ixgbe_free_receive_buffers(rxr); 4231 #ifdef LRO 4232 /* Free LRO memory */ 4233 tcp_lro_free(lro); 4234 #endif /* LRO */ 4235 /* Free the ring memory as well */ 4236 ixgbe_dma_free(adapter, &rxr->rxdma); 4237 IXGBE_RX_LOCK_DESTROY(rxr); 4238 } 4239 4240 free(adapter->rx_rings, M_DEVBUF); 4241 } 4242 4243 4244 /********************************************************************* 4245 * 4246 * Free receive ring data structures 4247 * 4248 **********************************************************************/ 4249 static void 4250 ixgbe_free_receive_buffers(struct rx_ring *rxr) 4251 { 4252 struct adapter *adapter = rxr->adapter; 4253 struct ixgbe_rx_buf *rxbuf; 4254 4255 INIT_DEBUGOUT("free_receive_structures: begin"); 4256 4257 /* Cleanup any existing buffers */ 4258 if (rxr->rx_buffers != NULL) { 4259 for (int i = 0; i < adapter->num_rx_desc; i++) { 4260 rxbuf = &rxr->rx_buffers[i]; 4261 if (rxbuf->m_head != NULL) { 4262 ixgbe_dmamap_sync(rxr->htag, rxbuf->hmap, 4263 BUS_DMASYNC_POSTREAD); 4264 ixgbe_dmamap_unload(rxr->htag, rxbuf->hmap); 4265 rxbuf->m_head->m_flags |= M_PKTHDR; 4266 m_freem(rxbuf->m_head); 4267 } 4268 if (rxbuf->m_pack != NULL) { 4269 bus_dmamap_sync(rxr->ptag->dt_dmat, rxbuf->pmap, 4270 0, rxbuf->m_pack->m_pkthdr.len, 4271 BUS_DMASYNC_POSTREAD); 4272 ixgbe_dmamap_unload(rxr->ptag, rxbuf->pmap); 4273 rxbuf->m_pack->m_flags |= M_PKTHDR; 4274 m_freem(rxbuf->m_pack); 4275 } 4276 rxbuf->m_head = NULL; 4277 rxbuf->m_pack = NULL; 4278 if (rxbuf->hmap != NULL) { 4279 ixgbe_dmamap_destroy(rxr->htag, rxbuf->hmap); 4280 rxbuf->hmap = NULL; 4281 } 4282 if (rxbuf->pmap != NULL) { 4283 ixgbe_dmamap_destroy(rxr->ptag, rxbuf->pmap); 4284 rxbuf->pmap = NULL; 4285 } 4286 } 4287 if (rxr->rx_buffers != NULL) { 4288 free(rxr->rx_buffers, M_DEVBUF); 4289 rxr->rx_buffers = NULL; 4290 } 4291 } 4292 4293 if (rxr->htag != NULL) { 4294 ixgbe_dma_tag_destroy(rxr->htag); 4295 rxr->htag = NULL; 4296 } 4297 if (rxr->ptag != NULL) { 4298 ixgbe_dma_tag_destroy(rxr->ptag); 4299 rxr->ptag = NULL; 4300 } 4301 4302 return; 4303 } 4304 4305 static __inline void 4306 ixgbe_rx_input(struct rx_ring *rxr, struct ifnet *ifp, struct mbuf *m, u32 ptype) 4307 { 4308 int s; 4309 4310 #ifdef LRO 4311 struct adapter *adapter = ifp->if_softc; 4312 struct ethercom *ec = &adapter->osdep.ec; 4313 4314 /* 4315 * ATM LRO is only for IPv4/TCP packets and TCP checksum of the packet 4316 * should be computed by hardware. Also it should not have VLAN tag in 4317 * ethernet header. 4318 */ 4319 if (rxr->lro_enabled && 4320 (ec->ec_capenable & ETHERCAP_VLAN_HWTAGGING) != 0 && 4321 (ptype & IXGBE_RXDADV_PKTTYPE_ETQF) == 0 && 4322 (ptype & (IXGBE_RXDADV_PKTTYPE_IPV4 | IXGBE_RXDADV_PKTTYPE_TCP)) == 4323 (IXGBE_RXDADV_PKTTYPE_IPV4 | IXGBE_RXDADV_PKTTYPE_TCP) && 4324 (m->m_pkthdr.csum_flags & (CSUM_DATA_VALID | CSUM_PSEUDO_HDR)) == 4325 (CSUM_DATA_VALID | CSUM_PSEUDO_HDR)) { 4326 /* 4327 * Send to the stack if: 4328 ** - LRO not enabled, or 4329 ** - no LRO resources, or 4330 ** - lro enqueue fails 4331 */ 4332 if (rxr->lro.lro_cnt != 0) 4333 if (tcp_lro_rx(&rxr->lro, m, 0) == 0) 4334 return; 4335 } 4336 #endif /* LRO */ 4337 4338 IXGBE_RX_UNLOCK(rxr); 4339 4340 s = splnet(); 4341 /* Pass this up to any BPF listeners. */ 4342 bpf_mtap(ifp, m); 4343 (*ifp->if_input)(ifp, m); 4344 splx(s); 4345 4346 IXGBE_RX_LOCK(rxr); 4347 } 4348 4349 static __inline void 4350 ixgbe_rx_discard(struct rx_ring *rxr, int i) 4351 { 4352 struct ixgbe_rx_buf *rbuf; 4353 4354 rbuf = &rxr->rx_buffers[i]; 4355 4356 if (rbuf->fmp != NULL) {/* Partial chain ? */ 4357 rbuf->fmp->m_flags |= M_PKTHDR; 4358 m_freem(rbuf->fmp); 4359 rbuf->fmp = NULL; 4360 } 4361 4362 /* 4363 ** With advanced descriptors the writeback 4364 ** clobbers the buffer addrs, so its easier 4365 ** to just free the existing mbufs and take 4366 ** the normal refresh path to get new buffers 4367 ** and mapping. 4368 */ 4369 if (rbuf->m_head) { 4370 m_free(rbuf->m_head); 4371 rbuf->m_head = NULL; 4372 } 4373 4374 if (rbuf->m_pack) { 4375 m_free(rbuf->m_pack); 4376 rbuf->m_pack = NULL; 4377 } 4378 4379 return; 4380 } 4381 4382 4383 /********************************************************************* 4384 * 4385 * This routine executes in interrupt context. It replenishes 4386 * the mbufs in the descriptor and sends data which has been 4387 * dma'ed into host memory to upper layer. 4388 * 4389 * We loop at most count times if count is > 0, or until done if 4390 * count < 0. 4391 * 4392 * Return TRUE for more work, FALSE for all clean. 4393 *********************************************************************/ 4394 static bool 4395 ixgbe_rxeof(struct ix_queue *que, int count) 4396 { 4397 struct adapter *adapter = que->adapter; 4398 struct rx_ring *rxr = que->rxr; 4399 struct ifnet *ifp = adapter->ifp; 4400 #ifdef LRO 4401 struct lro_ctrl *lro = &rxr->lro; 4402 struct lro_entry *queued; 4403 #endif /* LRO */ 4404 int i, nextp, processed = 0; 4405 u32 staterr = 0; 4406 union ixgbe_adv_rx_desc *cur; 4407 struct ixgbe_rx_buf *rbuf, *nbuf; 4408 4409 IXGBE_RX_LOCK(rxr); 4410 4411 for (i = rxr->next_to_check; count != 0;) { 4412 struct mbuf *sendmp, *mh, *mp; 4413 u32 rsc, ptype; 4414 u16 hlen, plen, hdr, vtag; 4415 bool eop; 4416 4417 /* Sync the ring. */ 4418 ixgbe_dmamap_sync(rxr->rxdma.dma_tag, rxr->rxdma.dma_map, 4419 BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE); 4420 4421 cur = &rxr->rx_base[i]; 4422 staterr = le32toh(cur->wb.upper.status_error); 4423 4424 if ((staterr & IXGBE_RXD_STAT_DD) == 0) 4425 break; 4426 if ((ifp->if_flags & IFF_RUNNING) == 0) 4427 break; 4428 4429 count--; 4430 sendmp = NULL; 4431 nbuf = NULL; 4432 rsc = 0; 4433 cur->wb.upper.status_error = 0; 4434 rbuf = &rxr->rx_buffers[i]; 4435 mh = rbuf->m_head; 4436 mp = rbuf->m_pack; 4437 4438 plen = le16toh(cur->wb.upper.length); 4439 ptype = le32toh(cur->wb.lower.lo_dword.data) & 4440 IXGBE_RXDADV_PKTTYPE_MASK; 4441 hdr = le16toh(cur->wb.lower.lo_dword.hs_rss.hdr_info); 4442 vtag = le16toh(cur->wb.upper.vlan); 4443 eop = ((staterr & IXGBE_RXD_STAT_EOP) != 0); 4444 4445 /* Make sure bad packets are discarded */ 4446 if (((staterr & IXGBE_RXDADV_ERR_FRAME_ERR_MASK) != 0) || 4447 (rxr->discard)) { 4448 ifp->if_ierrors++; 4449 rxr->rx_discarded.ev_count++; 4450 if (eop) 4451 rxr->discard = FALSE; 4452 else 4453 rxr->discard = TRUE; 4454 ixgbe_rx_discard(rxr, i); 4455 goto next_desc; 4456 } 4457 4458 /* 4459 ** On 82599 which supports a hardware 4460 ** LRO (called HW RSC), packets need 4461 ** not be fragmented across sequential 4462 ** descriptors, rather the next descriptor 4463 ** is indicated in bits of the descriptor. 4464 ** This also means that we might proceses 4465 ** more than one packet at a time, something 4466 ** that has never been true before, it 4467 ** required eliminating global chain pointers 4468 ** in favor of what we are doing here. -jfv 4469 */ 4470 if (!eop) { 4471 /* 4472 ** Figure out the next descriptor 4473 ** of this frame. 4474 */ 4475 if (rxr->hw_rsc == TRUE) { 4476 rsc = ixgbe_rsc_count(cur); 4477 rxr->rsc_num += (rsc - 1); 4478 } 4479 if (rsc) { /* Get hardware index */ 4480 nextp = ((staterr & 4481 IXGBE_RXDADV_NEXTP_MASK) >> 4482 IXGBE_RXDADV_NEXTP_SHIFT); 4483 } else { /* Just sequential */ 4484 nextp = i + 1; 4485 if (nextp == adapter->num_rx_desc) 4486 nextp = 0; 4487 } 4488 nbuf = &rxr->rx_buffers[nextp]; 4489 prefetch(nbuf); 4490 } 4491 /* 4492 ** The header mbuf is ONLY used when header 4493 ** split is enabled, otherwise we get normal 4494 ** behavior, ie, both header and payload 4495 ** are DMA'd into the payload buffer. 4496 ** 4497 ** Rather than using the fmp/lmp global pointers 4498 ** we now keep the head of a packet chain in the 4499 ** buffer struct and pass this along from one 4500 ** descriptor to the next, until we get EOP. 4501 */ 4502 if (rxr->hdr_split && (rbuf->fmp == NULL)) { 4503 /* This must be an initial descriptor */ 4504 hlen = (hdr & IXGBE_RXDADV_HDRBUFLEN_MASK) >> 4505 IXGBE_RXDADV_HDRBUFLEN_SHIFT; 4506 if (hlen > IXGBE_RX_HDR) 4507 hlen = IXGBE_RX_HDR; 4508 mh->m_len = hlen; 4509 mh->m_flags |= M_PKTHDR; 4510 mh->m_next = NULL; 4511 mh->m_pkthdr.len = mh->m_len; 4512 /* Null buf pointer so it is refreshed */ 4513 rbuf->m_head = NULL; 4514 /* 4515 ** Check the payload length, this 4516 ** could be zero if its a small 4517 ** packet. 4518 */ 4519 if (plen > 0) { 4520 mp->m_len = plen; 4521 mp->m_next = NULL; 4522 mp->m_flags &= ~M_PKTHDR; 4523 mh->m_next = mp; 4524 mh->m_pkthdr.len += mp->m_len; 4525 /* Null buf pointer so it is refreshed */ 4526 rbuf->m_pack = NULL; 4527 rxr->rx_split_packets.ev_count++; 4528 } 4529 /* 4530 ** Now create the forward 4531 ** chain so when complete 4532 ** we wont have to. 4533 */ 4534 if (eop == 0) { 4535 /* stash the chain head */ 4536 nbuf->fmp = mh; 4537 /* Make forward chain */ 4538 if (plen) 4539 mp->m_next = nbuf->m_pack; 4540 else 4541 mh->m_next = nbuf->m_pack; 4542 } else { 4543 /* Singlet, prepare to send */ 4544 sendmp = mh; 4545 if (VLAN_ATTACHED(&adapter->osdep.ec) && 4546 (staterr & IXGBE_RXD_STAT_VP)) { 4547 /* XXX Do something reasonable on 4548 * error. 4549 */ 4550 #if 0 4551 printf("%s.%d: VLAN_INPUT_TAG\n", 4552 __func__, __LINE__); 4553 Debugger(); 4554 #endif 4555 VLAN_INPUT_TAG(ifp, sendmp, vtag, 4556 printf("%s: could not apply VLAN " 4557 "tag", __func__)); 4558 } 4559 } 4560 } else { 4561 /* 4562 ** Either no header split, or a 4563 ** secondary piece of a fragmented 4564 ** split packet. 4565 */ 4566 mp->m_len = plen; 4567 /* 4568 ** See if there is a stored head 4569 ** that determines what we are 4570 */ 4571 sendmp = rbuf->fmp; 4572 rbuf->m_pack = rbuf->fmp = NULL; 4573 4574 if (sendmp != NULL) /* secondary frag */ 4575 sendmp->m_pkthdr.len += mp->m_len; 4576 else { 4577 /* first desc of a non-ps chain */ 4578 sendmp = mp; 4579 sendmp->m_flags |= M_PKTHDR; 4580 sendmp->m_pkthdr.len = mp->m_len; 4581 if (staterr & IXGBE_RXD_STAT_VP) { 4582 /* XXX Do something reasonable on 4583 * error. 4584 */ 4585 #if 0 4586 printf("%s.%d: VLAN_INPUT_TAG\n", 4587 __func__, __LINE__); 4588 Debugger(); 4589 #endif 4590 VLAN_INPUT_TAG(ifp, sendmp, vtag, 4591 printf("%s: could not apply VLAN " 4592 "tag", __func__)); 4593 } 4594 } 4595 /* Pass the head pointer on */ 4596 if (eop == 0) { 4597 nbuf->fmp = sendmp; 4598 sendmp = NULL; 4599 mp->m_next = nbuf->m_pack; 4600 } 4601 } 4602 ++processed; 4603 /* Sending this frame? */ 4604 if (eop) { 4605 sendmp->m_pkthdr.rcvif = ifp; 4606 ifp->if_ipackets++; 4607 rxr->rx_packets.ev_count++; 4608 /* capture data for AIM */ 4609 rxr->bytes += sendmp->m_pkthdr.len; 4610 rxr->rx_bytes.ev_count += sendmp->m_pkthdr.len; 4611 if ((ifp->if_capenable & IFCAP_RXCSUM) != 0) { 4612 ixgbe_rx_checksum(staterr, sendmp, ptype, 4613 &adapter->stats); 4614 } 4615 #if __FreeBSD_version >= 800000 4616 sendmp->m_pkthdr.flowid = que->msix; 4617 sendmp->m_flags |= M_FLOWID; 4618 #endif 4619 } 4620 next_desc: 4621 ixgbe_dmamap_sync(rxr->rxdma.dma_tag, rxr->rxdma.dma_map, 4622 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); 4623 4624 /* Advance our pointers to the next descriptor. */ 4625 if (++i == adapter->num_rx_desc) 4626 i = 0; 4627 4628 /* Now send to the stack or do LRO */ 4629 if (sendmp != NULL) { 4630 rxr->next_to_check = i; 4631 ixgbe_rx_input(rxr, ifp, sendmp, ptype); 4632 i = rxr->next_to_check; 4633 } 4634 4635 /* Every 8 descriptors we go to refresh mbufs */ 4636 if (processed == 8) { 4637 ixgbe_refresh_mbufs(rxr, i); 4638 processed = 0; 4639 } 4640 } 4641 4642 /* Refresh any remaining buf structs */ 4643 if (ixgbe_rx_unrefreshed(rxr)) 4644 ixgbe_refresh_mbufs(rxr, i); 4645 4646 rxr->next_to_check = i; 4647 4648 #ifdef LRO 4649 /* 4650 * Flush any outstanding LRO work 4651 */ 4652 while ((queued = SLIST_FIRST(&lro->lro_active)) != NULL) { 4653 SLIST_REMOVE_HEAD(&lro->lro_active, next); 4654 tcp_lro_flush(lro, queued); 4655 } 4656 #endif /* LRO */ 4657 4658 IXGBE_RX_UNLOCK(rxr); 4659 4660 /* 4661 ** We still have cleaning to do? 4662 ** Schedule another interrupt if so. 4663 */ 4664 if ((staterr & IXGBE_RXD_STAT_DD) != 0) { 4665 ixgbe_rearm_queues(adapter, (u64)(1ULL << que->msix)); 4666 return true; 4667 } 4668 4669 return false; 4670 } 4671 4672 4673 /********************************************************************* 4674 * 4675 * Verify that the hardware indicated that the checksum is valid. 4676 * Inform the stack about the status of checksum so that stack 4677 * doesn't spend time verifying the checksum. 4678 * 4679 *********************************************************************/ 4680 static void 4681 ixgbe_rx_checksum(u32 staterr, struct mbuf * mp, u32 ptype, 4682 struct ixgbe_hw_stats *stats) 4683 { 4684 u16 status = (u16) staterr; 4685 u8 errors = (u8) (staterr >> 24); 4686 #if 0 4687 bool sctp = FALSE; 4688 4689 if ((ptype & IXGBE_RXDADV_PKTTYPE_ETQF) == 0 && 4690 (ptype & IXGBE_RXDADV_PKTTYPE_SCTP) != 0) 4691 sctp = TRUE; 4692 #endif 4693 4694 if (status & IXGBE_RXD_STAT_IPCS) { 4695 stats->ipcs.ev_count++; 4696 if (!(errors & IXGBE_RXD_ERR_IPE)) { 4697 /* IP Checksum Good */ 4698 mp->m_pkthdr.csum_flags = M_CSUM_IPv4; 4699 4700 } else { 4701 stats->ipcs_bad.ev_count++; 4702 mp->m_pkthdr.csum_flags = M_CSUM_IPv4|M_CSUM_IPv4_BAD; 4703 } 4704 } 4705 if (status & IXGBE_RXD_STAT_L4CS) { 4706 stats->l4cs.ev_count++; 4707 u16 type = M_CSUM_TCPv4|M_CSUM_TCPv6|M_CSUM_UDPv4|M_CSUM_UDPv6; 4708 if (!(errors & IXGBE_RXD_ERR_TCPE)) { 4709 mp->m_pkthdr.csum_flags |= type; 4710 } else { 4711 stats->l4cs_bad.ev_count++; 4712 mp->m_pkthdr.csum_flags |= type | M_CSUM_TCP_UDP_BAD; 4713 } 4714 } 4715 return; 4716 } 4717 4718 4719 #if 0 /* XXX Badly need to overhaul vlan(4) on NetBSD. */ 4720 /* 4721 ** This routine is run via an vlan config EVENT, 4722 ** it enables us to use the HW Filter table since 4723 ** we can get the vlan id. This just creates the 4724 ** entry in the soft version of the VFTA, init will 4725 ** repopulate the real table. 4726 */ 4727 static void 4728 ixgbe_register_vlan(void *arg, struct ifnet *ifp, u16 vtag) 4729 { 4730 struct adapter *adapter = ifp->if_softc; 4731 u16 index, bit; 4732 4733 if (ifp->if_softc != arg) /* Not our event */ 4734 return; 4735 4736 if ((vtag == 0) || (vtag > 4095)) /* Invalid */ 4737 return; 4738 4739 IXGBE_CORE_LOCK(adapter); 4740 index = (vtag >> 5) & 0x7F; 4741 bit = vtag & 0x1F; 4742 adapter->shadow_vfta[index] |= (1 << bit); 4743 ixgbe_init_locked(adapter); 4744 IXGBE_CORE_UNLOCK(adapter); 4745 } 4746 4747 /* 4748 ** This routine is run via an vlan 4749 ** unconfig EVENT, remove our entry 4750 ** in the soft vfta. 4751 */ 4752 static void 4753 ixgbe_unregister_vlan(void *arg, struct ifnet *ifp, u16 vtag) 4754 { 4755 struct adapter *adapter = ifp->if_softc; 4756 u16 index, bit; 4757 4758 if (ifp->if_softc != arg) 4759 return; 4760 4761 if ((vtag == 0) || (vtag > 4095)) /* Invalid */ 4762 return; 4763 4764 IXGBE_CORE_LOCK(adapter); 4765 index = (vtag >> 5) & 0x7F; 4766 bit = vtag & 0x1F; 4767 adapter->shadow_vfta[index] &= ~(1 << bit); 4768 /* Re-init to load the changes */ 4769 ixgbe_init_locked(adapter); 4770 IXGBE_CORE_UNLOCK(adapter); 4771 } 4772 #endif 4773 4774 static void 4775 ixgbe_setup_vlan_hw_support(struct adapter *adapter) 4776 { 4777 struct ethercom *ec = &adapter->osdep.ec; 4778 struct ixgbe_hw *hw = &adapter->hw; 4779 u32 ctrl; 4780 4781 /* 4782 ** We get here thru init_locked, meaning 4783 ** a soft reset, this has already cleared 4784 ** the VFTA and other state, so if there 4785 ** have been no vlan's registered do nothing. 4786 */ 4787 if (!VLAN_ATTACHED(&adapter->osdep.ec)) { 4788 return; 4789 } 4790 4791 /* 4792 ** A soft reset zero's out the VFTA, so 4793 ** we need to repopulate it now. 4794 */ 4795 for (int i = 0; i < IXGBE_VFTA_SIZE; i++) 4796 if (adapter->shadow_vfta[i] != 0) 4797 IXGBE_WRITE_REG(hw, IXGBE_VFTA(i), 4798 adapter->shadow_vfta[i]); 4799 4800 ctrl = IXGBE_READ_REG(hw, IXGBE_VLNCTRL); 4801 /* Enable the Filter Table if enabled */ 4802 if (ec->ec_capenable & ETHERCAP_VLAN_HWFILTER) { 4803 ctrl &= ~IXGBE_VLNCTRL_CFIEN; 4804 ctrl |= IXGBE_VLNCTRL_VFE; 4805 } 4806 if (hw->mac.type == ixgbe_mac_82598EB) 4807 ctrl |= IXGBE_VLNCTRL_VME; 4808 IXGBE_WRITE_REG(hw, IXGBE_VLNCTRL, ctrl); 4809 4810 /* On 82599 the VLAN enable is per/queue in RXDCTL */ 4811 if (hw->mac.type != ixgbe_mac_82598EB) 4812 for (int i = 0; i < adapter->num_queues; i++) { 4813 ctrl = IXGBE_READ_REG(hw, IXGBE_RXDCTL(i)); 4814 ctrl |= IXGBE_RXDCTL_VME; 4815 IXGBE_WRITE_REG(hw, IXGBE_RXDCTL(i), ctrl); 4816 } 4817 } 4818 4819 static void 4820 ixgbe_enable_intr(struct adapter *adapter) 4821 { 4822 struct ixgbe_hw *hw = &adapter->hw; 4823 struct ix_queue *que = adapter->queues; 4824 u32 mask = (IXGBE_EIMS_ENABLE_MASK & ~IXGBE_EIMS_RTX_QUEUE); 4825 4826 4827 /* Enable Fan Failure detection */ 4828 if (hw->device_id == IXGBE_DEV_ID_82598AT) 4829 mask |= IXGBE_EIMS_GPI_SDP1; 4830 else { 4831 mask |= IXGBE_EIMS_ECC; 4832 mask |= IXGBE_EIMS_GPI_SDP1; 4833 mask |= IXGBE_EIMS_GPI_SDP2; 4834 #ifdef IXGBE_FDIR 4835 mask |= IXGBE_EIMS_FLOW_DIR; 4836 #endif 4837 } 4838 4839 IXGBE_WRITE_REG(hw, IXGBE_EIMS, mask); 4840 4841 /* With RSS we use auto clear */ 4842 if (adapter->msix_mem) { 4843 mask = IXGBE_EIMS_ENABLE_MASK; 4844 /* Don't autoclear Link */ 4845 mask &= ~IXGBE_EIMS_OTHER; 4846 mask &= ~IXGBE_EIMS_LSC; 4847 IXGBE_WRITE_REG(hw, IXGBE_EIAC, mask); 4848 } 4849 4850 /* 4851 ** Now enable all queues, this is done separately to 4852 ** allow for handling the extended (beyond 32) MSIX 4853 ** vectors that can be used by 82599 4854 */ 4855 for (int i = 0; i < adapter->num_queues; i++, que++) 4856 ixgbe_enable_queue(adapter, que->msix); 4857 4858 IXGBE_WRITE_FLUSH(hw); 4859 4860 return; 4861 } 4862 4863 static void 4864 ixgbe_disable_intr(struct adapter *adapter) 4865 { 4866 if (adapter->msix_mem) 4867 IXGBE_WRITE_REG(&adapter->hw, IXGBE_EIAC, 0); 4868 if (adapter->hw.mac.type == ixgbe_mac_82598EB) { 4869 IXGBE_WRITE_REG(&adapter->hw, IXGBE_EIMC, ~0); 4870 } else { 4871 IXGBE_WRITE_REG(&adapter->hw, IXGBE_EIMC, 0xFFFF0000); 4872 IXGBE_WRITE_REG(&adapter->hw, IXGBE_EIMC_EX(0), ~0); 4873 IXGBE_WRITE_REG(&adapter->hw, IXGBE_EIMC_EX(1), ~0); 4874 } 4875 IXGBE_WRITE_FLUSH(&adapter->hw); 4876 return; 4877 } 4878 4879 u16 4880 ixgbe_read_pci_cfg(struct ixgbe_hw *hw, u32 reg) 4881 { 4882 switch (reg % 4) { 4883 case 0: 4884 return pci_conf_read(hw->back->pc, hw->back->tag, reg) & 4885 __BITS(15, 0); 4886 case 2: 4887 return __SHIFTOUT(pci_conf_read(hw->back->pc, hw->back->tag, 4888 reg - 2), __BITS(31, 16)); 4889 default: 4890 panic("%s: invalid register (%" PRIx32, __func__, reg); 4891 break; 4892 } 4893 } 4894 4895 void 4896 ixgbe_write_pci_cfg(struct ixgbe_hw *hw, u32 reg, u16 value) 4897 { 4898 pcireg_t old; 4899 4900 switch (reg % 4) { 4901 case 0: 4902 old = pci_conf_read(hw->back->pc, hw->back->tag, reg) & 4903 __BITS(31, 16); 4904 pci_conf_write(hw->back->pc, hw->back->tag, reg, value | old); 4905 break; 4906 case 2: 4907 old = pci_conf_read(hw->back->pc, hw->back->tag, reg - 2) & 4908 __BITS(15, 0); 4909 pci_conf_write(hw->back->pc, hw->back->tag, reg - 2, 4910 __SHIFTIN(value, __BITS(31, 16)) | old); 4911 break; 4912 default: 4913 panic("%s: invalid register (%" PRIx32, __func__, reg); 4914 break; 4915 } 4916 4917 return; 4918 } 4919 4920 /* 4921 ** Setup the correct IVAR register for a particular MSIX interrupt 4922 ** (yes this is all very magic and confusing :) 4923 ** - entry is the register array entry 4924 ** - vector is the MSIX vector for this queue 4925 ** - type is RX/TX/MISC 4926 */ 4927 static void 4928 ixgbe_set_ivar(struct adapter *adapter, u8 entry, u8 vector, s8 type) 4929 { 4930 struct ixgbe_hw *hw = &adapter->hw; 4931 u32 ivar, index; 4932 4933 vector |= IXGBE_IVAR_ALLOC_VAL; 4934 4935 switch (hw->mac.type) { 4936 4937 case ixgbe_mac_82598EB: 4938 if (type == -1) 4939 entry = IXGBE_IVAR_OTHER_CAUSES_INDEX; 4940 else 4941 entry += (type * 64); 4942 index = (entry >> 2) & 0x1F; 4943 ivar = IXGBE_READ_REG(hw, IXGBE_IVAR(index)); 4944 ivar &= ~(0xFF << (8 * (entry & 0x3))); 4945 ivar |= (vector << (8 * (entry & 0x3))); 4946 IXGBE_WRITE_REG(&adapter->hw, IXGBE_IVAR(index), ivar); 4947 break; 4948 4949 case ixgbe_mac_82599EB: 4950 if (type == -1) { /* MISC IVAR */ 4951 index = (entry & 1) * 8; 4952 ivar = IXGBE_READ_REG(hw, IXGBE_IVAR_MISC); 4953 ivar &= ~(0xFF << index); 4954 ivar |= (vector << index); 4955 IXGBE_WRITE_REG(hw, IXGBE_IVAR_MISC, ivar); 4956 } else { /* RX/TX IVARS */ 4957 index = (16 * (entry & 1)) + (8 * type); 4958 ivar = IXGBE_READ_REG(hw, IXGBE_IVAR(entry >> 1)); 4959 ivar &= ~(0xFF << index); 4960 ivar |= (vector << index); 4961 IXGBE_WRITE_REG(hw, IXGBE_IVAR(entry >> 1), ivar); 4962 } 4963 4964 default: 4965 break; 4966 } 4967 } 4968 4969 static void 4970 ixgbe_configure_ivars(struct adapter *adapter) 4971 { 4972 struct ix_queue *que = adapter->queues; 4973 u32 newitr; 4974 4975 if (ixgbe_max_interrupt_rate > 0) 4976 newitr = (8000000 / ixgbe_max_interrupt_rate) & 0x0FF8; 4977 else 4978 newitr = 0; 4979 4980 for (int i = 0; i < adapter->num_queues; i++, que++) { 4981 /* First the RX queue entry */ 4982 ixgbe_set_ivar(adapter, i, que->msix, 0); 4983 /* ... and the TX */ 4984 ixgbe_set_ivar(adapter, i, que->msix, 1); 4985 /* Set an Initial EITR value */ 4986 IXGBE_WRITE_REG(&adapter->hw, 4987 IXGBE_EITR(que->msix), newitr); 4988 } 4989 4990 /* For the Link interrupt */ 4991 ixgbe_set_ivar(adapter, 1, adapter->linkvec, -1); 4992 } 4993 4994 /* 4995 ** ixgbe_sfp_probe - called in the local timer to 4996 ** determine if a port had optics inserted. 4997 */ 4998 static bool ixgbe_sfp_probe(struct adapter *adapter) 4999 { 5000 struct ixgbe_hw *hw = &adapter->hw; 5001 device_t dev = adapter->dev; 5002 bool result = FALSE; 5003 5004 if ((hw->phy.type == ixgbe_phy_nl) && 5005 (hw->phy.sfp_type == ixgbe_sfp_type_not_present)) { 5006 s32 ret = hw->phy.ops.identify_sfp(hw); 5007 if (ret) 5008 goto out; 5009 ret = hw->phy.ops.reset(hw); 5010 if (ret == IXGBE_ERR_SFP_NOT_SUPPORTED) { 5011 device_printf(dev,"Unsupported SFP+ module detected!"); 5012 device_printf(dev, "Reload driver with supported module.\n"); 5013 adapter->sfp_probe = FALSE; 5014 goto out; 5015 } else 5016 device_printf(dev,"SFP+ module detected!\n"); 5017 /* We now have supported optics */ 5018 adapter->sfp_probe = FALSE; 5019 /* Set the optics type so system reports correctly */ 5020 ixgbe_setup_optics(adapter); 5021 result = TRUE; 5022 } 5023 out: 5024 return (result); 5025 } 5026 5027 /* 5028 ** Tasklet handler for MSIX Link interrupts 5029 ** - do outside interrupt since it might sleep 5030 */ 5031 static void 5032 ixgbe_handle_link(void *context) 5033 { 5034 struct adapter *adapter = context; 5035 5036 if (ixgbe_check_link(&adapter->hw, 5037 &adapter->link_speed, &adapter->link_up, 0) == 0) 5038 ixgbe_update_link_status(adapter); 5039 } 5040 5041 /* 5042 ** Tasklet for handling SFP module interrupts 5043 */ 5044 static void 5045 ixgbe_handle_mod(void *context) 5046 { 5047 struct adapter *adapter = context; 5048 struct ixgbe_hw *hw = &adapter->hw; 5049 device_t dev = adapter->dev; 5050 u32 err; 5051 5052 err = hw->phy.ops.identify_sfp(hw); 5053 if (err == IXGBE_ERR_SFP_NOT_SUPPORTED) { 5054 device_printf(dev, 5055 "Unsupported SFP+ module type was detected.\n"); 5056 return; 5057 } 5058 err = hw->mac.ops.setup_sfp(hw); 5059 if (err == IXGBE_ERR_SFP_NOT_SUPPORTED) { 5060 device_printf(dev, 5061 "Setup failure - unsupported SFP+ module type.\n"); 5062 return; 5063 } 5064 softint_schedule(adapter->msf_si); 5065 return; 5066 } 5067 5068 5069 /* 5070 ** Tasklet for handling MSF (multispeed fiber) interrupts 5071 */ 5072 static void 5073 ixgbe_handle_msf(void *context) 5074 { 5075 struct adapter *adapter = context; 5076 struct ixgbe_hw *hw = &adapter->hw; 5077 u32 autoneg; 5078 bool negotiate; 5079 5080 autoneg = hw->phy.autoneg_advertised; 5081 if ((!autoneg) && (hw->mac.ops.get_link_capabilities)) 5082 hw->mac.ops.get_link_capabilities(hw, &autoneg, &negotiate); 5083 else 5084 negotiate = 0; 5085 if (hw->mac.ops.setup_link) 5086 hw->mac.ops.setup_link(hw, autoneg, negotiate, TRUE); 5087 return; 5088 } 5089 5090 #ifdef IXGBE_FDIR 5091 /* 5092 ** Tasklet for reinitializing the Flow Director filter table 5093 */ 5094 static void 5095 ixgbe_reinit_fdir(void *context) 5096 { 5097 struct adapter *adapter = context; 5098 struct ifnet *ifp = adapter->ifp; 5099 5100 if (adapter->fdir_reinit != 1) /* Shouldn't happen */ 5101 return; 5102 ixgbe_reinit_fdir_tables_82599(&adapter->hw); 5103 adapter->fdir_reinit = 0; 5104 /* Restart the interface */ 5105 ifp->if_flags |= IFF_RUNNING; 5106 return; 5107 } 5108 #endif 5109 5110 /********************************************************************** 5111 * 5112 * Update the board statistics counters. 5113 * 5114 **********************************************************************/ 5115 static void 5116 ixgbe_update_stats_counters(struct adapter *adapter) 5117 { 5118 struct ifnet *ifp = adapter->ifp; 5119 struct ixgbe_hw *hw = &adapter->hw; 5120 u32 missed_rx = 0, bprc, lxon, lxoff, total; 5121 u64 total_missed_rx = 0; 5122 5123 adapter->stats.crcerrs.ev_count += IXGBE_READ_REG(hw, IXGBE_CRCERRS); 5124 adapter->stats.illerrc.ev_count += IXGBE_READ_REG(hw, IXGBE_ILLERRC); 5125 adapter->stats.errbc.ev_count += IXGBE_READ_REG(hw, IXGBE_ERRBC); 5126 adapter->stats.mspdc.ev_count += IXGBE_READ_REG(hw, IXGBE_MSPDC); 5127 5128 for (int i = 0; i < __arraycount(adapter->stats.mpc); i++) { 5129 int j = i % adapter->num_queues; 5130 u32 mp; 5131 mp = IXGBE_READ_REG(hw, IXGBE_MPC(i)); 5132 /* missed_rx tallies misses for the gprc workaround */ 5133 missed_rx += mp; 5134 /* global total per queue */ 5135 adapter->stats.mpc[j].ev_count += mp; 5136 /* Running comprehensive total for stats display */ 5137 total_missed_rx += adapter->stats.mpc[j].ev_count; 5138 if (hw->mac.type == ixgbe_mac_82598EB) 5139 adapter->stats.rnbc[j] += 5140 IXGBE_READ_REG(hw, IXGBE_RNBC(i)); 5141 adapter->stats.pxontxc[j].ev_count += 5142 IXGBE_READ_REG(hw, IXGBE_PXONTXC(i)); 5143 adapter->stats.pxonrxc[j].ev_count += 5144 IXGBE_READ_REG(hw, IXGBE_PXONRXC(i)); 5145 adapter->stats.pxofftxc[j].ev_count += 5146 IXGBE_READ_REG(hw, IXGBE_PXOFFTXC(i)); 5147 adapter->stats.pxoffrxc[j].ev_count += 5148 IXGBE_READ_REG(hw, IXGBE_PXOFFRXC(i)); 5149 adapter->stats.pxon2offc[j].ev_count += 5150 IXGBE_READ_REG(hw, IXGBE_PXON2OFFCNT(i)); 5151 } 5152 for (int i = 0; i < __arraycount(adapter->stats.qprc); i++) { 5153 int j = i % adapter->num_queues; 5154 adapter->stats.qprc[j].ev_count += IXGBE_READ_REG(hw, IXGBE_QPRC(i)); 5155 adapter->stats.qptc[j].ev_count += IXGBE_READ_REG(hw, IXGBE_QPTC(i)); 5156 adapter->stats.qbrc[j].ev_count += IXGBE_READ_REG(hw, IXGBE_QBRC(i)); 5157 adapter->stats.qbrc[j].ev_count += 5158 ((u64)IXGBE_READ_REG(hw, IXGBE_QBRC(i)) << 32); 5159 adapter->stats.qbtc[j].ev_count += IXGBE_READ_REG(hw, IXGBE_QBTC(i)); 5160 adapter->stats.qbtc[j].ev_count += 5161 ((u64)IXGBE_READ_REG(hw, IXGBE_QBTC(i)) << 32); 5162 adapter->stats.qprdc[j].ev_count += IXGBE_READ_REG(hw, IXGBE_QPRDC(i)); 5163 } 5164 adapter->stats.mlfc.ev_count += IXGBE_READ_REG(hw, IXGBE_MLFC); 5165 adapter->stats.mrfc.ev_count += IXGBE_READ_REG(hw, IXGBE_MRFC); 5166 adapter->stats.rlec.ev_count += IXGBE_READ_REG(hw, IXGBE_RLEC); 5167 5168 /* Hardware workaround, gprc counts missed packets */ 5169 adapter->stats.gprc.ev_count += IXGBE_READ_REG(hw, IXGBE_GPRC) - missed_rx; 5170 5171 lxon = IXGBE_READ_REG(hw, IXGBE_LXONTXC); 5172 adapter->stats.lxontxc.ev_count += lxon; 5173 lxoff = IXGBE_READ_REG(hw, IXGBE_LXOFFTXC); 5174 adapter->stats.lxofftxc.ev_count += lxoff; 5175 total = lxon + lxoff; 5176 5177 if (hw->mac.type != ixgbe_mac_82598EB) { 5178 adapter->stats.gorc.ev_count += IXGBE_READ_REG(hw, IXGBE_GORCL) + 5179 ((u64)IXGBE_READ_REG(hw, IXGBE_GORCH) << 32); 5180 adapter->stats.gotc.ev_count += IXGBE_READ_REG(hw, IXGBE_GOTCL) + 5181 ((u64)IXGBE_READ_REG(hw, IXGBE_GOTCH) << 32) - total * ETHER_MIN_LEN; 5182 adapter->stats.tor.ev_count += IXGBE_READ_REG(hw, IXGBE_TORL) + 5183 ((u64)IXGBE_READ_REG(hw, IXGBE_TORH) << 32); 5184 adapter->stats.lxonrxc.ev_count += IXGBE_READ_REG(hw, IXGBE_LXONRXCNT); 5185 adapter->stats.lxoffrxc.ev_count += IXGBE_READ_REG(hw, IXGBE_LXOFFRXCNT); 5186 } else { 5187 adapter->stats.lxonrxc.ev_count += IXGBE_READ_REG(hw, IXGBE_LXONRXC); 5188 adapter->stats.lxoffrxc.ev_count += IXGBE_READ_REG(hw, IXGBE_LXOFFRXC); 5189 /* 82598 only has a counter in the high register */ 5190 adapter->stats.gorc.ev_count += IXGBE_READ_REG(hw, IXGBE_GORCH); 5191 adapter->stats.gotc.ev_count += IXGBE_READ_REG(hw, IXGBE_GOTCH) - total * ETHER_MIN_LEN; 5192 adapter->stats.tor.ev_count += IXGBE_READ_REG(hw, IXGBE_TORH); 5193 } 5194 5195 /* 5196 * Workaround: mprc hardware is incorrectly counting 5197 * broadcasts, so for now we subtract those. 5198 */ 5199 bprc = IXGBE_READ_REG(hw, IXGBE_BPRC); 5200 adapter->stats.bprc.ev_count += bprc; 5201 adapter->stats.mprc.ev_count += IXGBE_READ_REG(hw, IXGBE_MPRC) - ((hw->mac.type == ixgbe_mac_82598EB) ? bprc : 0); 5202 5203 adapter->stats.prc64.ev_count += IXGBE_READ_REG(hw, IXGBE_PRC64); 5204 adapter->stats.prc127.ev_count += IXGBE_READ_REG(hw, IXGBE_PRC127); 5205 adapter->stats.prc255.ev_count += IXGBE_READ_REG(hw, IXGBE_PRC255); 5206 adapter->stats.prc511.ev_count += IXGBE_READ_REG(hw, IXGBE_PRC511); 5207 adapter->stats.prc1023.ev_count += IXGBE_READ_REG(hw, IXGBE_PRC1023); 5208 adapter->stats.prc1522.ev_count += IXGBE_READ_REG(hw, IXGBE_PRC1522); 5209 5210 adapter->stats.gptc.ev_count += IXGBE_READ_REG(hw, IXGBE_GPTC) - total; 5211 adapter->stats.mptc.ev_count += IXGBE_READ_REG(hw, IXGBE_MPTC) - total; 5212 adapter->stats.ptc64.ev_count += IXGBE_READ_REG(hw, IXGBE_PTC64) - total; 5213 5214 adapter->stats.ruc.ev_count += IXGBE_READ_REG(hw, IXGBE_RUC); 5215 adapter->stats.rfc.ev_count += IXGBE_READ_REG(hw, IXGBE_RFC); 5216 adapter->stats.roc.ev_count += IXGBE_READ_REG(hw, IXGBE_ROC); 5217 adapter->stats.rjc.ev_count += IXGBE_READ_REG(hw, IXGBE_RJC); 5218 adapter->stats.mngprc.ev_count += IXGBE_READ_REG(hw, IXGBE_MNGPRC); 5219 adapter->stats.mngpdc.ev_count += IXGBE_READ_REG(hw, IXGBE_MNGPDC); 5220 adapter->stats.mngptc.ev_count += IXGBE_READ_REG(hw, IXGBE_MNGPTC); 5221 adapter->stats.tpr.ev_count += IXGBE_READ_REG(hw, IXGBE_TPR); 5222 adapter->stats.tpt.ev_count += IXGBE_READ_REG(hw, IXGBE_TPT); 5223 adapter->stats.ptc127.ev_count += IXGBE_READ_REG(hw, IXGBE_PTC127); 5224 adapter->stats.ptc255.ev_count += IXGBE_READ_REG(hw, IXGBE_PTC255); 5225 adapter->stats.ptc511.ev_count += IXGBE_READ_REG(hw, IXGBE_PTC511); 5226 adapter->stats.ptc1023.ev_count += IXGBE_READ_REG(hw, IXGBE_PTC1023); 5227 adapter->stats.ptc1522.ev_count += IXGBE_READ_REG(hw, IXGBE_PTC1522); 5228 adapter->stats.bptc.ev_count += IXGBE_READ_REG(hw, IXGBE_BPTC); 5229 adapter->stats.xec.ev_count += IXGBE_READ_REG(hw, IXGBE_XEC); 5230 adapter->stats.fccrc.ev_count += IXGBE_READ_REG(hw, IXGBE_FCCRC); 5231 adapter->stats.fclast.ev_count += IXGBE_READ_REG(hw, IXGBE_FCLAST); 5232 5233 /* Only read FCOE on 82599 */ 5234 if (hw->mac.type != ixgbe_mac_82598EB) { 5235 adapter->stats.fcoerpdc.ev_count += 5236 IXGBE_READ_REG(hw, IXGBE_FCOERPDC); 5237 adapter->stats.fcoeprc.ev_count += 5238 IXGBE_READ_REG(hw, IXGBE_FCOEPRC); 5239 adapter->stats.fcoeptc.ev_count += 5240 IXGBE_READ_REG(hw, IXGBE_FCOEPTC); 5241 adapter->stats.fcoedwrc.ev_count += 5242 IXGBE_READ_REG(hw, IXGBE_FCOEDWRC); 5243 adapter->stats.fcoedwtc.ev_count += 5244 IXGBE_READ_REG(hw, IXGBE_FCOEDWTC); 5245 } 5246 5247 /* Fill out the OS statistics structure */ 5248 ifp->if_ipackets = adapter->stats.gprc.ev_count; 5249 ifp->if_opackets = adapter->stats.gptc.ev_count; 5250 ifp->if_ibytes = adapter->stats.gorc.ev_count; 5251 ifp->if_obytes = adapter->stats.gotc.ev_count; 5252 ifp->if_imcasts = adapter->stats.mprc.ev_count; 5253 ifp->if_collisions = 0; 5254 5255 /* Rx Errors */ 5256 ifp->if_ierrors = total_missed_rx + adapter->stats.crcerrs.ev_count + 5257 adapter->stats.rlec.ev_count; 5258 } 5259 5260 /** ixgbe_sysctl_tdh_handler - Handler function 5261 * Retrieves the TDH value from the hardware 5262 */ 5263 static int 5264 ixgbe_sysctl_tdh_handler(SYSCTLFN_ARGS) 5265 { 5266 struct sysctlnode node; 5267 uint32_t val; 5268 struct tx_ring *txr; 5269 5270 node = *rnode; 5271 txr = (struct tx_ring *)node.sysctl_data; 5272 if (txr == NULL) 5273 return 0; 5274 val = IXGBE_READ_REG(&txr->adapter->hw, IXGBE_TDH(txr->me)); 5275 node.sysctl_data = &val; 5276 return sysctl_lookup(SYSCTLFN_CALL(&node)); 5277 } 5278 5279 /** ixgbe_sysctl_tdt_handler - Handler function 5280 * Retrieves the TDT value from the hardware 5281 */ 5282 static int 5283 ixgbe_sysctl_tdt_handler(SYSCTLFN_ARGS) 5284 { 5285 struct sysctlnode node; 5286 uint32_t val; 5287 struct tx_ring *txr; 5288 5289 node = *rnode; 5290 txr = (struct tx_ring *)node.sysctl_data; 5291 if (txr == NULL) 5292 return 0; 5293 val = IXGBE_READ_REG(&txr->adapter->hw, IXGBE_TDT(txr->me)); 5294 node.sysctl_data = &val; 5295 return sysctl_lookup(SYSCTLFN_CALL(&node)); 5296 } 5297 5298 /** ixgbe_sysctl_rdh_handler - Handler function 5299 * Retrieves the RDH value from the hardware 5300 */ 5301 static int 5302 ixgbe_sysctl_rdh_handler(SYSCTLFN_ARGS) 5303 { 5304 struct sysctlnode node; 5305 uint32_t val; 5306 struct rx_ring *rxr; 5307 5308 node = *rnode; 5309 rxr = (struct rx_ring *)node.sysctl_data; 5310 if (rxr == NULL) 5311 return 0; 5312 val = IXGBE_READ_REG(&rxr->adapter->hw, IXGBE_RDH(rxr->me)); 5313 node.sysctl_data = &val; 5314 return sysctl_lookup(SYSCTLFN_CALL(&node)); 5315 } 5316 5317 /** ixgbe_sysctl_rdt_handler - Handler function 5318 * Retrieves the RDT value from the hardware 5319 */ 5320 static int 5321 ixgbe_sysctl_rdt_handler(SYSCTLFN_ARGS) 5322 { 5323 struct sysctlnode node; 5324 uint32_t val; 5325 struct rx_ring *rxr; 5326 5327 node = *rnode; 5328 rxr = (struct rx_ring *)node.sysctl_data; 5329 if (rxr == NULL) 5330 return 0; 5331 val = IXGBE_READ_REG(&rxr->adapter->hw, IXGBE_RDT(rxr->me)); 5332 node.sysctl_data = &val; 5333 return sysctl_lookup(SYSCTLFN_CALL(&node)); 5334 } 5335 5336 static int 5337 ixgbe_sysctl_interrupt_rate_handler(SYSCTLFN_ARGS) 5338 { 5339 struct sysctlnode node; 5340 struct ix_queue *que; 5341 uint32_t reg, usec, rate; 5342 5343 node = *rnode; 5344 que = (struct ix_queue *)node.sysctl_data; 5345 if (que == NULL) 5346 return 0; 5347 reg = IXGBE_READ_REG(&que->adapter->hw, IXGBE_EITR(que->msix)); 5348 usec = ((reg & 0x0FF8) >> 3); 5349 if (usec > 0) 5350 rate = 1000000 / usec; 5351 else 5352 rate = 0; 5353 node.sysctl_data = &rate; 5354 return sysctl_lookup(SYSCTLFN_CALL(&node)); 5355 } 5356 5357 const struct sysctlnode * 5358 ixgbe_sysctl_instance(struct adapter *adapter) 5359 { 5360 const char *dvname; 5361 struct sysctllog **log; 5362 int rc; 5363 const struct sysctlnode *rnode; 5364 5365 log = &adapter->sysctllog; 5366 dvname = device_xname(adapter->dev); 5367 5368 if ((rc = sysctl_createv(log, 0, NULL, &rnode, 5369 0, CTLTYPE_NODE, dvname, 5370 SYSCTL_DESCR("ixgbe information and settings"), 5371 NULL, 0, NULL, 0, CTL_HW, CTL_CREATE, CTL_EOL)) != 0) 5372 goto err; 5373 5374 return rnode; 5375 err: 5376 printf("%s: sysctl_createv failed, rc = %d\n", __func__, rc); 5377 return NULL; 5378 } 5379 5380 /* 5381 * Add sysctl variables, one per statistic, to the system. 5382 */ 5383 static void 5384 ixgbe_add_hw_stats(struct adapter *adapter) 5385 { 5386 device_t dev = adapter->dev; 5387 const struct sysctlnode *rnode, *cnode; 5388 struct sysctllog **log = &adapter->sysctllog; 5389 struct tx_ring *txr = adapter->tx_rings; 5390 struct rx_ring *rxr = adapter->rx_rings; 5391 struct ixgbe_hw *hw = &adapter->hw; 5392 5393 struct ixgbe_hw_stats *stats = &adapter->stats; 5394 5395 /* Driver Statistics */ 5396 #if 0 5397 /* These counters are not updated by the software */ 5398 SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "dropped", 5399 CTLFLAG_RD, &adapter->dropped_pkts, 5400 "Driver dropped packets"); 5401 SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "mbuf_header_failed", 5402 CTLFLAG_RD, &adapter->mbuf_header_failed, 5403 "???"); 5404 SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "mbuf_packet_failed", 5405 CTLFLAG_RD, &adapter->mbuf_packet_failed, 5406 "???"); 5407 SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "no_tx_map_avail", 5408 CTLFLAG_RD, &adapter->no_tx_map_avail, 5409 "???"); 5410 #endif 5411 evcnt_attach_dynamic(&adapter->handleq, EVCNT_TYPE_MISC, 5412 NULL, device_xname(dev), "Handled queue in softint"); 5413 evcnt_attach_dynamic(&adapter->req, EVCNT_TYPE_MISC, 5414 NULL, device_xname(dev), "Requeued in softint"); 5415 evcnt_attach_dynamic(&adapter->morerx, EVCNT_TYPE_MISC, 5416 NULL, device_xname(dev), "Interrupt handler more rx"); 5417 evcnt_attach_dynamic(&adapter->moretx, EVCNT_TYPE_MISC, 5418 NULL, device_xname(dev), "Interrupt handler more tx"); 5419 evcnt_attach_dynamic(&adapter->txloops, EVCNT_TYPE_MISC, 5420 NULL, device_xname(dev), "Interrupt handler tx loops"); 5421 evcnt_attach_dynamic(&adapter->efbig_tx_dma_setup, EVCNT_TYPE_MISC, 5422 NULL, device_xname(dev), "Driver tx dma soft fail EFBIG"); 5423 evcnt_attach_dynamic(&adapter->m_defrag_failed, EVCNT_TYPE_MISC, 5424 NULL, device_xname(dev), "m_defrag() failed"); 5425 evcnt_attach_dynamic(&adapter->efbig2_tx_dma_setup, EVCNT_TYPE_MISC, 5426 NULL, device_xname(dev), "Driver tx dma hard fail EFBIG"); 5427 evcnt_attach_dynamic(&adapter->einval_tx_dma_setup, EVCNT_TYPE_MISC, 5428 NULL, device_xname(dev), "Driver tx dma hard fail EINVAL"); 5429 evcnt_attach_dynamic(&adapter->other_tx_dma_setup, EVCNT_TYPE_MISC, 5430 NULL, device_xname(dev), "Driver tx dma hard fail other"); 5431 evcnt_attach_dynamic(&adapter->eagain_tx_dma_setup, EVCNT_TYPE_MISC, 5432 NULL, device_xname(dev), "Driver tx dma soft fail EAGAIN"); 5433 evcnt_attach_dynamic(&adapter->enomem_tx_dma_setup, EVCNT_TYPE_MISC, 5434 NULL, device_xname(dev), "Driver tx dma soft fail ENOMEM"); 5435 evcnt_attach_dynamic(&adapter->watchdog_events, EVCNT_TYPE_MISC, 5436 NULL, device_xname(dev), "Watchdog timeouts"); 5437 evcnt_attach_dynamic(&adapter->tso_err, EVCNT_TYPE_MISC, 5438 NULL, device_xname(dev), "TSO errors"); 5439 evcnt_attach_dynamic(&adapter->tso_tx, EVCNT_TYPE_MISC, 5440 NULL, device_xname(dev), "TSO"); 5441 evcnt_attach_dynamic(&adapter->link_irq, EVCNT_TYPE_MISC, 5442 NULL, device_xname(dev), "Link MSIX IRQ Handled"); 5443 5444 for (int i = 0; i < adapter->num_queues; i++, rxr++, txr++) { 5445 snprintf(adapter->queues[i].evnamebuf, 5446 sizeof(adapter->queues[i].evnamebuf), "%s queue%d", 5447 device_xname(dev), i); 5448 snprintf(adapter->queues[i].namebuf, 5449 sizeof(adapter->queues[i].namebuf), "queue%d", i); 5450 5451 if ((rnode = ixgbe_sysctl_instance(adapter)) == NULL) { 5452 aprint_error_dev(dev, "could not create sysctl root\n"); 5453 break; 5454 } 5455 5456 if (sysctl_createv(log, 0, &rnode, &rnode, 5457 0, CTLTYPE_NODE, 5458 adapter->queues[i].namebuf, SYSCTL_DESCR("Queue Name"), 5459 NULL, 0, NULL, 0, CTL_CREATE, CTL_EOL) != 0) 5460 break; 5461 5462 if (sysctl_createv(log, 0, &rnode, &cnode, 5463 CTLFLAG_READONLY, CTLTYPE_INT, 5464 "interrupt_rate", SYSCTL_DESCR("Interrupt Rate"), 5465 ixgbe_sysctl_interrupt_rate_handler, 0, 5466 (void *)&adapter->queues[i], 0, CTL_CREATE, CTL_EOL) != 0) 5467 break; 5468 5469 if (sysctl_createv(log, 0, &rnode, &cnode, 5470 CTLFLAG_READONLY, CTLTYPE_INT, 5471 "txd_head", SYSCTL_DESCR("Transmit Descriptor Head"), 5472 ixgbe_sysctl_tdh_handler, 0, (void *)txr, 5473 0, CTL_CREATE, CTL_EOL) != 0) 5474 break; 5475 5476 if (sysctl_createv(log, 0, &rnode, &cnode, 5477 CTLFLAG_READONLY, CTLTYPE_INT, 5478 "txd_tail", SYSCTL_DESCR("Transmit Descriptor Tail"), 5479 ixgbe_sysctl_tdt_handler, 0, (void *)txr, 5480 0, CTL_CREATE, CTL_EOL) != 0) 5481 break; 5482 5483 evcnt_attach_dynamic(&txr->no_desc_avail, EVCNT_TYPE_MISC, 5484 NULL, adapter->queues[i].evnamebuf, 5485 "Queue No Descriptor Available"); 5486 evcnt_attach_dynamic(&txr->total_packets, EVCNT_TYPE_MISC, 5487 NULL, adapter->queues[i].evnamebuf, 5488 "Queue Packets Transmitted"); 5489 5490 #ifdef LRO 5491 struct lro_ctrl *lro = &rxr->lro; 5492 #endif /* LRO */ 5493 5494 if (sysctl_createv(log, 0, &rnode, &cnode, 5495 CTLFLAG_READONLY, 5496 CTLTYPE_INT, 5497 "rxd_head", SYSCTL_DESCR("Receive Descriptor Head"), 5498 ixgbe_sysctl_rdh_handler, 0, (void *)rxr, 0, 5499 CTL_CREATE, CTL_EOL) != 0) 5500 break; 5501 5502 if (sysctl_createv(log, 0, &rnode, &cnode, 5503 CTLFLAG_READONLY, 5504 CTLTYPE_INT, 5505 "rxd_tail", SYSCTL_DESCR("Receive Descriptor Tail"), 5506 ixgbe_sysctl_rdt_handler, 0, (void *)rxr, 0, 5507 CTL_CREATE, CTL_EOL) != 0) 5508 break; 5509 5510 if (i < __arraycount(adapter->stats.mpc)) { 5511 evcnt_attach_dynamic(&adapter->stats.mpc[i], 5512 EVCNT_TYPE_MISC, NULL, adapter->queues[i].evnamebuf, 5513 "Missed Packet Count"); 5514 } 5515 if (i < __arraycount(adapter->stats.pxontxc)) { 5516 evcnt_attach_dynamic(&adapter->stats.pxontxc[i], 5517 EVCNT_TYPE_MISC, NULL, adapter->queues[i].evnamebuf, 5518 "pxontxc"); 5519 evcnt_attach_dynamic(&adapter->stats.pxonrxc[i], 5520 EVCNT_TYPE_MISC, NULL, adapter->queues[i].evnamebuf, 5521 "pxonrxc"); 5522 evcnt_attach_dynamic(&adapter->stats.pxofftxc[i], 5523 EVCNT_TYPE_MISC, NULL, adapter->queues[i].evnamebuf, 5524 "pxofftxc"); 5525 evcnt_attach_dynamic(&adapter->stats.pxoffrxc[i], 5526 EVCNT_TYPE_MISC, NULL, adapter->queues[i].evnamebuf, 5527 "pxoffrxc"); 5528 evcnt_attach_dynamic(&adapter->stats.pxon2offc[i], 5529 EVCNT_TYPE_MISC, NULL, adapter->queues[i].evnamebuf, 5530 "pxon2offc"); 5531 } 5532 if (i < __arraycount(adapter->stats.qprc)) { 5533 evcnt_attach_dynamic(&adapter->stats.qprc[i], 5534 EVCNT_TYPE_MISC, NULL, adapter->queues[i].evnamebuf, 5535 "qprc"); 5536 evcnt_attach_dynamic(&adapter->stats.qptc[i], 5537 EVCNT_TYPE_MISC, NULL, adapter->queues[i].evnamebuf, 5538 "qptc"); 5539 evcnt_attach_dynamic(&adapter->stats.qbrc[i], 5540 EVCNT_TYPE_MISC, NULL, adapter->queues[i].evnamebuf, 5541 "qbrc"); 5542 evcnt_attach_dynamic(&adapter->stats.qbtc[i], 5543 EVCNT_TYPE_MISC, NULL, adapter->queues[i].evnamebuf, 5544 "qbtc"); 5545 evcnt_attach_dynamic(&adapter->stats.qprdc[i], 5546 EVCNT_TYPE_MISC, NULL, adapter->queues[i].evnamebuf, 5547 "qprdc"); 5548 } 5549 5550 evcnt_attach_dynamic(&rxr->rx_packets, EVCNT_TYPE_MISC, 5551 NULL, adapter->queues[i].evnamebuf, "Queue Packets Received"); 5552 evcnt_attach_dynamic(&rxr->rx_bytes, EVCNT_TYPE_MISC, 5553 NULL, adapter->queues[i].evnamebuf, "Queue Bytes Received"); 5554 evcnt_attach_dynamic(&rxr->no_jmbuf, EVCNT_TYPE_MISC, 5555 NULL, adapter->queues[i].evnamebuf, "Rx no jumbo mbuf"); 5556 evcnt_attach_dynamic(&rxr->rx_discarded, EVCNT_TYPE_MISC, 5557 NULL, adapter->queues[i].evnamebuf, "Rx discarded"); 5558 evcnt_attach_dynamic(&rxr->rx_split_packets, EVCNT_TYPE_MISC, 5559 NULL, adapter->queues[i].evnamebuf, "Rx split packets"); 5560 evcnt_attach_dynamic(&rxr->rx_irq, EVCNT_TYPE_MISC, 5561 NULL, adapter->queues[i].evnamebuf, "Rx interrupts"); 5562 #ifdef LRO 5563 SYSCTL_ADD_INT(ctx, queue_list, OID_AUTO, "lro_queued", 5564 CTLFLAG_RD, &lro->lro_queued, 0, 5565 "LRO Queued"); 5566 SYSCTL_ADD_INT(ctx, queue_list, OID_AUTO, "lro_flushed", 5567 CTLFLAG_RD, &lro->lro_flushed, 0, 5568 "LRO Flushed"); 5569 #endif /* LRO */ 5570 } 5571 5572 /* MAC stats get the own sub node */ 5573 5574 5575 snprintf(stats->namebuf, 5576 sizeof(stats->namebuf), "%s MAC Statistics", device_xname(dev)); 5577 5578 evcnt_attach_dynamic(&stats->ipcs, EVCNT_TYPE_MISC, NULL, 5579 stats->namebuf, "rx csum offload - IP"); 5580 evcnt_attach_dynamic(&stats->l4cs, EVCNT_TYPE_MISC, NULL, 5581 stats->namebuf, "rx csum offload - L4"); 5582 evcnt_attach_dynamic(&stats->ipcs_bad, EVCNT_TYPE_MISC, NULL, 5583 stats->namebuf, "rx csum offload - IP bad"); 5584 evcnt_attach_dynamic(&stats->l4cs_bad, EVCNT_TYPE_MISC, NULL, 5585 stats->namebuf, "rx csum offload - L4 bad"); 5586 evcnt_attach_dynamic(&stats->intzero, EVCNT_TYPE_MISC, NULL, 5587 stats->namebuf, "Interrupt conditions zero"); 5588 evcnt_attach_dynamic(&stats->legint, EVCNT_TYPE_MISC, NULL, 5589 stats->namebuf, "Legacy interrupts"); 5590 evcnt_attach_dynamic(&stats->crcerrs, EVCNT_TYPE_MISC, NULL, 5591 stats->namebuf, "CRC Errors"); 5592 evcnt_attach_dynamic(&stats->illerrc, EVCNT_TYPE_MISC, NULL, 5593 stats->namebuf, "Illegal Byte Errors"); 5594 evcnt_attach_dynamic(&stats->errbc, EVCNT_TYPE_MISC, NULL, 5595 stats->namebuf, "Byte Errors"); 5596 evcnt_attach_dynamic(&stats->mspdc, EVCNT_TYPE_MISC, NULL, 5597 stats->namebuf, "MAC Short Packets Discarded"); 5598 evcnt_attach_dynamic(&stats->mlfc, EVCNT_TYPE_MISC, NULL, 5599 stats->namebuf, "MAC Local Faults"); 5600 evcnt_attach_dynamic(&stats->mrfc, EVCNT_TYPE_MISC, NULL, 5601 stats->namebuf, "MAC Remote Faults"); 5602 evcnt_attach_dynamic(&stats->rlec, EVCNT_TYPE_MISC, NULL, 5603 stats->namebuf, "Receive Length Errors"); 5604 evcnt_attach_dynamic(&stats->lxontxc, EVCNT_TYPE_MISC, NULL, 5605 stats->namebuf, "Link XON Transmitted"); 5606 evcnt_attach_dynamic(&stats->lxonrxc, EVCNT_TYPE_MISC, NULL, 5607 stats->namebuf, "Link XON Received"); 5608 evcnt_attach_dynamic(&stats->lxofftxc, EVCNT_TYPE_MISC, NULL, 5609 stats->namebuf, "Link XOFF Transmitted"); 5610 evcnt_attach_dynamic(&stats->lxoffrxc, EVCNT_TYPE_MISC, NULL, 5611 stats->namebuf, "Link XOFF Received"); 5612 5613 /* Packet Reception Stats */ 5614 evcnt_attach_dynamic(&stats->tor, EVCNT_TYPE_MISC, NULL, 5615 stats->namebuf, "Total Octets Received"); 5616 evcnt_attach_dynamic(&stats->gorc, EVCNT_TYPE_MISC, NULL, 5617 stats->namebuf, "Good Octets Received"); 5618 evcnt_attach_dynamic(&stats->tpr, EVCNT_TYPE_MISC, NULL, 5619 stats->namebuf, "Total Packets Received"); 5620 evcnt_attach_dynamic(&stats->gprc, EVCNT_TYPE_MISC, NULL, 5621 stats->namebuf, "Good Packets Received"); 5622 evcnt_attach_dynamic(&stats->mprc, EVCNT_TYPE_MISC, NULL, 5623 stats->namebuf, "Multicast Packets Received"); 5624 evcnt_attach_dynamic(&stats->bprc, EVCNT_TYPE_MISC, NULL, 5625 stats->namebuf, "Broadcast Packets Received"); 5626 evcnt_attach_dynamic(&stats->prc64, EVCNT_TYPE_MISC, NULL, 5627 stats->namebuf, "64 byte frames received "); 5628 evcnt_attach_dynamic(&stats->prc127, EVCNT_TYPE_MISC, NULL, 5629 stats->namebuf, "65-127 byte frames received"); 5630 evcnt_attach_dynamic(&stats->prc255, EVCNT_TYPE_MISC, NULL, 5631 stats->namebuf, "128-255 byte frames received"); 5632 evcnt_attach_dynamic(&stats->prc511, EVCNT_TYPE_MISC, NULL, 5633 stats->namebuf, "256-511 byte frames received"); 5634 evcnt_attach_dynamic(&stats->prc1023, EVCNT_TYPE_MISC, NULL, 5635 stats->namebuf, "512-1023 byte frames received"); 5636 evcnt_attach_dynamic(&stats->prc1522, EVCNT_TYPE_MISC, NULL, 5637 stats->namebuf, "1023-1522 byte frames received"); 5638 evcnt_attach_dynamic(&stats->ruc, EVCNT_TYPE_MISC, NULL, 5639 stats->namebuf, "Receive Undersized"); 5640 evcnt_attach_dynamic(&stats->rfc, EVCNT_TYPE_MISC, NULL, 5641 stats->namebuf, "Fragmented Packets Received "); 5642 evcnt_attach_dynamic(&stats->roc, EVCNT_TYPE_MISC, NULL, 5643 stats->namebuf, "Oversized Packets Received"); 5644 evcnt_attach_dynamic(&stats->rjc, EVCNT_TYPE_MISC, NULL, 5645 stats->namebuf, "Received Jabber"); 5646 evcnt_attach_dynamic(&stats->mngprc, EVCNT_TYPE_MISC, NULL, 5647 stats->namebuf, "Management Packets Received"); 5648 evcnt_attach_dynamic(&stats->xec, EVCNT_TYPE_MISC, NULL, 5649 stats->namebuf, "Checksum Errors"); 5650 5651 /* Packet Transmission Stats */ 5652 evcnt_attach_dynamic(&stats->gotc, EVCNT_TYPE_MISC, NULL, 5653 stats->namebuf, "Good Octets Transmitted"); 5654 evcnt_attach_dynamic(&stats->tpt, EVCNT_TYPE_MISC, NULL, 5655 stats->namebuf, "Total Packets Transmitted"); 5656 evcnt_attach_dynamic(&stats->gptc, EVCNT_TYPE_MISC, NULL, 5657 stats->namebuf, "Good Packets Transmitted"); 5658 evcnt_attach_dynamic(&stats->bptc, EVCNT_TYPE_MISC, NULL, 5659 stats->namebuf, "Broadcast Packets Transmitted"); 5660 evcnt_attach_dynamic(&stats->mptc, EVCNT_TYPE_MISC, NULL, 5661 stats->namebuf, "Multicast Packets Transmitted"); 5662 evcnt_attach_dynamic(&stats->mngptc, EVCNT_TYPE_MISC, NULL, 5663 stats->namebuf, "Management Packets Transmitted"); 5664 evcnt_attach_dynamic(&stats->ptc64, EVCNT_TYPE_MISC, NULL, 5665 stats->namebuf, "64 byte frames transmitted "); 5666 evcnt_attach_dynamic(&stats->ptc127, EVCNT_TYPE_MISC, NULL, 5667 stats->namebuf, "65-127 byte frames transmitted"); 5668 evcnt_attach_dynamic(&stats->ptc255, EVCNT_TYPE_MISC, NULL, 5669 stats->namebuf, "128-255 byte frames transmitted"); 5670 evcnt_attach_dynamic(&stats->ptc511, EVCNT_TYPE_MISC, NULL, 5671 stats->namebuf, "256-511 byte frames transmitted"); 5672 evcnt_attach_dynamic(&stats->ptc1023, EVCNT_TYPE_MISC, NULL, 5673 stats->namebuf, "512-1023 byte frames transmitted"); 5674 evcnt_attach_dynamic(&stats->ptc1522, EVCNT_TYPE_MISC, NULL, 5675 stats->namebuf, "1024-1522 byte frames transmitted"); 5676 5677 /* FC Stats */ 5678 evcnt_attach_dynamic(&stats->fccrc, EVCNT_TYPE_MISC, NULL, 5679 stats->namebuf, "FC CRC Errors"); 5680 evcnt_attach_dynamic(&stats->fclast, EVCNT_TYPE_MISC, NULL, 5681 stats->namebuf, "FC Last Error"); 5682 if (hw->mac.type != ixgbe_mac_82598EB) { 5683 evcnt_attach_dynamic(&stats->fcoerpdc, EVCNT_TYPE_MISC, NULL, 5684 stats->namebuf, "FCoE Packets Dropped"); 5685 evcnt_attach_dynamic(&stats->fcoeprc, EVCNT_TYPE_MISC, NULL, 5686 stats->namebuf, "FCoE Packets Received"); 5687 evcnt_attach_dynamic(&stats->fcoeptc, EVCNT_TYPE_MISC, NULL, 5688 stats->namebuf, "FCoE Packets Transmitted"); 5689 evcnt_attach_dynamic(&stats->fcoedwrc, EVCNT_TYPE_MISC, NULL, 5690 stats->namebuf, "FCoE DWords Received"); 5691 evcnt_attach_dynamic(&stats->fcoedwtc, EVCNT_TYPE_MISC, NULL, 5692 stats->namebuf, "FCoE DWords Transmitted"); 5693 } 5694 } 5695 5696 /* 5697 ** Set flow control using sysctl: 5698 ** Flow control values: 5699 ** 0 - off 5700 ** 1 - rx pause 5701 ** 2 - tx pause 5702 ** 3 - full 5703 */ 5704 static int 5705 ixgbe_set_flowcntl(SYSCTLFN_ARGS) 5706 { 5707 struct sysctlnode node; 5708 int error; 5709 int last = ixgbe_flow_control; 5710 struct adapter *adapter; 5711 5712 node = *rnode; 5713 adapter = (struct adapter *)node.sysctl_data; 5714 node.sysctl_data = &ixgbe_flow_control; 5715 error = sysctl_lookup(SYSCTLFN_CALL(&node)); 5716 if (error != 0 || newp == NULL) 5717 return error; 5718 5719 /* Don't bother if it's not changed */ 5720 if (ixgbe_flow_control == last) 5721 return (0); 5722 5723 switch (ixgbe_flow_control) { 5724 case ixgbe_fc_rx_pause: 5725 case ixgbe_fc_tx_pause: 5726 case ixgbe_fc_full: 5727 adapter->hw.fc.requested_mode = ixgbe_flow_control; 5728 break; 5729 case ixgbe_fc_none: 5730 default: 5731 adapter->hw.fc.requested_mode = ixgbe_fc_none; 5732 } 5733 5734 ixgbe_fc_enable(&adapter->hw, 0); 5735 return 0; 5736 } 5737 5738 static void 5739 ixgbe_add_rx_process_limit(struct adapter *adapter, const char *name, 5740 const char *description, int *limit, int value) 5741 { 5742 const struct sysctlnode *rnode, *cnode; 5743 struct sysctllog **log = &adapter->sysctllog; 5744 5745 *limit = value; 5746 5747 if ((rnode = ixgbe_sysctl_instance(adapter)) == NULL) 5748 aprint_error_dev(adapter->dev, 5749 "could not create sysctl root\n"); 5750 else if (sysctl_createv(log, 0, &rnode, &cnode, 5751 CTLFLAG_READWRITE, 5752 CTLTYPE_INT, 5753 name, SYSCTL_DESCR(description), 5754 NULL, 0, limit, 0, 5755 CTL_CREATE, CTL_EOL) != 0) { 5756 aprint_error_dev(adapter->dev, "%s: could not create sysctl", 5757 __func__); 5758 } 5759 } 5760 5761 /* 5762 ** Control link advertise speed: 5763 ** 0 - normal 5764 ** 1 - advertise only 1G 5765 */ 5766 static int 5767 ixgbe_set_advertise(SYSCTLFN_ARGS) 5768 { 5769 struct sysctlnode node; 5770 int t, error; 5771 struct adapter *adapter; 5772 struct ixgbe_hw *hw; 5773 ixgbe_link_speed speed, last; 5774 5775 node = *rnode; 5776 adapter = (struct adapter *)node.sysctl_data; 5777 t = adapter->advertise; 5778 node.sysctl_data = &t; 5779 error = sysctl_lookup(SYSCTLFN_CALL(&node)); 5780 if (error != 0 || newp == NULL) 5781 return error; 5782 5783 if (t == -1) 5784 return 0; 5785 5786 adapter->advertise = t; 5787 5788 hw = &adapter->hw; 5789 last = hw->phy.autoneg_advertised; 5790 5791 if (!((hw->phy.media_type == ixgbe_media_type_copper) || 5792 (hw->phy.multispeed_fiber))) 5793 return 0; 5794 5795 if (adapter->advertise == 1) 5796 speed = IXGBE_LINK_SPEED_1GB_FULL; 5797 else 5798 speed = IXGBE_LINK_SPEED_1GB_FULL | 5799 IXGBE_LINK_SPEED_10GB_FULL; 5800 5801 if (speed == last) /* no change */ 5802 return 0; 5803 5804 hw->mac.autotry_restart = TRUE; 5805 hw->mac.ops.setup_link(hw, speed, TRUE, TRUE); 5806 5807 return 0; 5808 } 5809
Indexes created Thu Oct 23 22:10:10 GMT 2025