ixgbe.c revision 1.1.2.3
1 /****************************************************************************** 2 3 Copyright (c) 2001-2011, Intel Corporation 4 All rights reserved. 5 6 Redistribution and use in source and binary forms, with or without 7 modification, are permitted provided that the following conditions are met: 8 9 1. Redistributions of source code must retain the above copyright notice, 10 this list of conditions and the following disclaimer. 11 12 2. Redistributions in binary form must reproduce the above copyright 13 notice, this list of conditions and the following disclaimer in the 14 documentation and/or other materials provided with the distribution. 15 16 3. Neither the name of the Intel Corporation nor the names of its 17 contributors may be used to endorse or promote products derived from 18 this software without specific prior written permission. 19 20 THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 21 AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 22 IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 23 ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE 24 LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 25 CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 26 SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 27 INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 28 CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 29 ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 30 POSSIBILITY OF SUCH DAMAGE. 31 32 ******************************************************************************/ 33 /* 34 * Copyright (c) 2011 The NetBSD Foundation, Inc. 35 * All rights reserved. 36 * 37 * This code is derived from software contributed to The NetBSD Foundation 38 * by Coyote Point Systems, Inc. 39 * 40 * Redistribution and use in source and binary forms, with or without 41 * modification, are permitted provided that the following conditions 42 * are met: 43 * 1. Redistributions of source code must retain the above copyright 44 * notice, this list of conditions and the following disclaimer. 45 * 2. Redistributions in binary form must reproduce the above copyright 46 * notice, this list of conditions and the following disclaimer in the 47 * documentation and/or other materials provided with the distribution. 48 * 49 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS 50 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED 51 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 52 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS 53 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 54 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 55 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 56 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 57 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 58 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 59 * POSSIBILITY OF SUCH DAMAGE. 60 */ 61 /*$FreeBSD: src/sys/dev/ixgbe/ixgbe.c,v 1.51 2011/04/25 23:34:21 jfv Exp $*/ 62 /*$NetBSD: ixgbe.c,v 1.1.2.3 2014/05/22 11:40:34 yamt Exp $*/ 63 64 #include "opt_inet.h" 65 66 #include "ixgbe.h" 67 68 /********************************************************************* 69 * Set this to one to display debug statistics 70 *********************************************************************/ 71 int ixgbe_display_debug_stats = 0; 72 73 /********************************************************************* 74 * Driver version 75 *********************************************************************/ 76 char ixgbe_driver_version[] = "2.3.10"; 77 78 /********************************************************************* 79 * PCI Device ID Table 80 * 81 * Used by probe to select devices to load on 82 * Last field stores an index into ixgbe_strings 83 * Last entry must be all 0s 84 * 85 * { Vendor ID, Device ID, SubVendor ID, SubDevice ID, String Index } 86 *********************************************************************/ 87 88 static ixgbe_vendor_info_t ixgbe_vendor_info_array[] = 89 { 90 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598AF_DUAL_PORT, 0, 0, 0}, 91 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598AF_SINGLE_PORT, 0, 0, 0}, 92 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598EB_CX4, 0, 0, 0}, 93 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598AT, 0, 0, 0}, 94 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598AT2, 0, 0, 0}, 95 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598, 0, 0, 0}, 96 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598_DA_DUAL_PORT, 0, 0, 0}, 97 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598_CX4_DUAL_PORT, 0, 0, 0}, 98 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598EB_XF_LR, 0, 0, 0}, 99 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598_SR_DUAL_PORT_EM, 0, 0, 0}, 100 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598EB_SFP_LOM, 0, 0, 0}, 101 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_KX4, 0, 0, 0}, 102 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_KX4_MEZZ, 0, 0, 0}, 103 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_SFP, 0, 0, 0}, 104 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_XAUI_LOM, 0, 0, 0}, 105 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_CX4, 0, 0, 0}, 106 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_T3_LOM, 0, 0, 0}, 107 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_COMBO_BACKPLANE, 0, 0, 0}, 108 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_BACKPLANE_FCOE, 0, 0, 0}, 109 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_SFP_FCOE, 0, 0, 0}, 110 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_SFP_DELL, 0, 0, 0}, 111 /* required last entry */ 112 {0, 0, 0, 0, 0} 113 }; 114 115 /********************************************************************* 116 * Table of branding strings 117 *********************************************************************/ 118 119 static const char *ixgbe_strings[] = { 120 "Intel(R) PRO/10GbE PCI-Express Network Driver" 121 }; 122 123 /********************************************************************* 124 * Function prototypes 125 *********************************************************************/ 126 static int ixgbe_probe(device_t, cfdata_t, void *); 127 static void ixgbe_attach(device_t, device_t, void *); 128 static int ixgbe_detach(device_t, int); 129 #if 0 130 static int ixgbe_shutdown(device_t); 131 #endif 132 static void ixgbe_start(struct ifnet *); 133 static void ixgbe_start_locked(struct tx_ring *, struct ifnet *); 134 #if __FreeBSD_version >= 800000 135 static int ixgbe_mq_start(struct ifnet *, struct mbuf *); 136 static int ixgbe_mq_start_locked(struct ifnet *, 137 struct tx_ring *, struct mbuf *); 138 static void ixgbe_qflush(struct ifnet *); 139 #endif 140 static int ixgbe_ioctl(struct ifnet *, u_long, void *); 141 static void ixgbe_ifstop(struct ifnet *, int); 142 static int ixgbe_init(struct ifnet *); 143 static void ixgbe_init_locked(struct adapter *); 144 static void ixgbe_stop(void *); 145 static void ixgbe_media_status(struct ifnet *, struct ifmediareq *); 146 static int ixgbe_media_change(struct ifnet *); 147 static void ixgbe_identify_hardware(struct adapter *); 148 static int ixgbe_allocate_pci_resources(struct adapter *, 149 const struct pci_attach_args *); 150 static int ixgbe_allocate_msix(struct adapter *, 151 const struct pci_attach_args *); 152 static int ixgbe_allocate_legacy(struct adapter *, 153 const struct pci_attach_args *); 154 static int ixgbe_allocate_queues(struct adapter *); 155 static int ixgbe_setup_msix(struct adapter *); 156 static void ixgbe_free_pci_resources(struct adapter *); 157 static void ixgbe_local_timer(void *); 158 static int ixgbe_setup_interface(device_t, struct adapter *); 159 static void ixgbe_config_link(struct adapter *); 160 161 static int ixgbe_allocate_transmit_buffers(struct tx_ring *); 162 static int ixgbe_setup_transmit_structures(struct adapter *); 163 static void ixgbe_setup_transmit_ring(struct tx_ring *); 164 static void ixgbe_initialize_transmit_units(struct adapter *); 165 static void ixgbe_free_transmit_structures(struct adapter *); 166 static void ixgbe_free_transmit_buffers(struct tx_ring *); 167 168 static int ixgbe_allocate_receive_buffers(struct rx_ring *); 169 static int ixgbe_setup_receive_structures(struct adapter *); 170 static int ixgbe_setup_receive_ring(struct rx_ring *); 171 static void ixgbe_initialize_receive_units(struct adapter *); 172 static void ixgbe_free_receive_structures(struct adapter *); 173 static void ixgbe_free_receive_buffers(struct rx_ring *); 174 static void ixgbe_setup_hw_rsc(struct rx_ring *); 175 176 static void ixgbe_enable_intr(struct adapter *); 177 static void ixgbe_disable_intr(struct adapter *); 178 static void ixgbe_update_stats_counters(struct adapter *); 179 static bool ixgbe_txeof(struct tx_ring *); 180 static bool ixgbe_rxeof(struct ix_queue *, int); 181 static void ixgbe_rx_checksum(u32, struct mbuf *, u32, 182 struct ixgbe_hw_stats *); 183 static void ixgbe_set_promisc(struct adapter *); 184 static void ixgbe_set_multi(struct adapter *); 185 static void ixgbe_update_link_status(struct adapter *); 186 static void ixgbe_refresh_mbufs(struct rx_ring *, int); 187 static int ixgbe_xmit(struct tx_ring *, struct mbuf *); 188 static int ixgbe_set_flowcntl(SYSCTLFN_PROTO); 189 static int ixgbe_set_advertise(SYSCTLFN_PROTO); 190 static int ixgbe_dma_malloc(struct adapter *, bus_size_t, 191 struct ixgbe_dma_alloc *, int); 192 static void ixgbe_dma_free(struct adapter *, struct ixgbe_dma_alloc *); 193 static void ixgbe_add_rx_process_limit(struct adapter *, const char *, 194 const char *, int *, int); 195 static u32 ixgbe_tx_ctx_setup(struct tx_ring *, struct mbuf *); 196 static bool ixgbe_tso_setup(struct tx_ring *, struct mbuf *, u32 *); 197 static void ixgbe_set_ivar(struct adapter *, u8, u8, s8); 198 static void ixgbe_configure_ivars(struct adapter *); 199 static u8 * ixgbe_mc_array_itr(struct ixgbe_hw *, u8 **, u32 *); 200 201 static void ixgbe_setup_vlan_hw_support(struct adapter *); 202 #if 0 203 static void ixgbe_register_vlan(void *, struct ifnet *, u16); 204 static void ixgbe_unregister_vlan(void *, struct ifnet *, u16); 205 #endif 206 207 static void ixgbe_add_hw_stats(struct adapter *adapter); 208 209 static __inline void ixgbe_rx_discard(struct rx_ring *, int); 210 static __inline void ixgbe_rx_input(struct rx_ring *, struct ifnet *, 211 struct mbuf *, u32); 212 213 /* Support for pluggable optic modules */ 214 static bool ixgbe_sfp_probe(struct adapter *); 215 static void ixgbe_setup_optics(struct adapter *); 216 217 /* Legacy (single vector interrupt handler */ 218 static int ixgbe_legacy_irq(void *); 219 220 #if defined(NETBSD_MSI_OR_MSIX) 221 /* The MSI/X Interrupt handlers */ 222 static void ixgbe_msix_que(void *); 223 static void ixgbe_msix_link(void *); 224 #endif 225 226 /* Software interrupts for deferred work */ 227 static void ixgbe_handle_que(void *); 228 static void ixgbe_handle_link(void *); 229 static void ixgbe_handle_msf(void *); 230 static void ixgbe_handle_mod(void *); 231 232 const struct sysctlnode *ixgbe_sysctl_instance(struct adapter *); 233 static ixgbe_vendor_info_t *ixgbe_lookup(const struct pci_attach_args *); 234 235 #ifdef IXGBE_FDIR 236 static void ixgbe_atr(struct tx_ring *, struct mbuf *); 237 static void ixgbe_reinit_fdir(void *, int); 238 #endif 239 240 /********************************************************************* 241 * FreeBSD Device Interface Entry Points 242 *********************************************************************/ 243 244 CFATTACH_DECL3_NEW(ixg, sizeof(struct adapter), 245 ixgbe_probe, ixgbe_attach, ixgbe_detach, NULL, NULL, NULL, 246 DVF_DETACH_SHUTDOWN); 247 248 #if 0 249 devclass_t ixgbe_devclass; 250 DRIVER_MODULE(ixgbe, pci, ixgbe_driver, ixgbe_devclass, 0, 0); 251 252 MODULE_DEPEND(ixgbe, pci, 1, 1, 1); 253 MODULE_DEPEND(ixgbe, ether, 1, 1, 1); 254 #endif 255 256 /* 257 ** TUNEABLE PARAMETERS: 258 */ 259 260 /* 261 ** AIM: Adaptive Interrupt Moderation 262 ** which means that the interrupt rate 263 ** is varied over time based on the 264 ** traffic for that interrupt vector 265 */ 266 static int ixgbe_enable_aim = TRUE; 267 #define TUNABLE_INT(__x, __y) 268 TUNABLE_INT("hw.ixgbe.enable_aim", &ixgbe_enable_aim); 269 270 static int ixgbe_max_interrupt_rate = (8000000 / IXGBE_LOW_LATENCY); 271 TUNABLE_INT("hw.ixgbe.max_interrupt_rate", &ixgbe_max_interrupt_rate); 272 273 /* How many packets rxeof tries to clean at a time */ 274 static int ixgbe_rx_process_limit = 256; 275 TUNABLE_INT("hw.ixgbe.rx_process_limit", &ixgbe_rx_process_limit); 276 277 /* Flow control setting, default to full */ 278 static int ixgbe_flow_control = ixgbe_fc_full; 279 TUNABLE_INT("hw.ixgbe.flow_control", &ixgbe_flow_control); 280 281 /* 282 ** Smart speed setting, default to on 283 ** this only works as a compile option 284 ** right now as its during attach, set 285 ** this to 'ixgbe_smart_speed_off' to 286 ** disable. 287 */ 288 static int ixgbe_smart_speed = ixgbe_smart_speed_on; 289 290 /* 291 * MSIX should be the default for best performance, 292 * but this allows it to be forced off for testing. 293 */ 294 static int ixgbe_enable_msix = 1; 295 TUNABLE_INT("hw.ixgbe.enable_msix", &ixgbe_enable_msix); 296 297 /* 298 * Header split: this causes the hardware to DMA 299 * the header into a separate mbuf from the payload, 300 * it can be a performance win in some workloads, but 301 * in others it actually hurts, its off by default. 302 */ 303 static bool ixgbe_header_split = FALSE; 304 TUNABLE_INT("hw.ixgbe.hdr_split", &ixgbe_header_split); 305 306 #if defined(NETBSD_MSI_OR_MSIX) 307 /* 308 * Number of Queues, can be set to 0, 309 * it then autoconfigures based on the 310 * number of cpus with a max of 8. This 311 * can be overriden manually here. 312 */ 313 static int ixgbe_num_queues = 0; 314 TUNABLE_INT("hw.ixgbe.num_queues", &ixgbe_num_queues); 315 #endif 316 317 /* 318 ** Number of TX descriptors per ring, 319 ** setting higher than RX as this seems 320 ** the better performing choice. 321 */ 322 static int ixgbe_txd = PERFORM_TXD; 323 TUNABLE_INT("hw.ixgbe.txd", &ixgbe_txd); 324 325 /* Number of RX descriptors per ring */ 326 static int ixgbe_rxd = PERFORM_RXD; 327 TUNABLE_INT("hw.ixgbe.rxd", &ixgbe_rxd); 328 329 /* Keep running tab on them for sanity check */ 330 static int ixgbe_total_ports; 331 332 #ifdef IXGBE_FDIR 333 /* 334 ** For Flow Director: this is the 335 ** number of TX packets we sample 336 ** for the filter pool, this means 337 ** every 20th packet will be probed. 338 ** 339 ** This feature can be disabled by 340 ** setting this to 0. 341 */ 342 static int atr_sample_rate = 20; 343 /* 344 ** Flow Director actually 'steals' 345 ** part of the packet buffer as its 346 ** filter pool, this variable controls 347 ** how much it uses: 348 ** 0 = 64K, 1 = 128K, 2 = 256K 349 */ 350 static int fdir_pballoc = 1; 351 #endif 352 353 /********************************************************************* 354 * Device identification routine 355 * 356 * ixgbe_probe determines if the driver should be loaded on 357 * adapter based on PCI vendor/device id of the adapter. 358 * 359 * return 1 on success, 0 on failure 360 *********************************************************************/ 361 362 static int 363 ixgbe_probe(device_t dev, cfdata_t cf, void *aux) 364 { 365 const struct pci_attach_args *pa = aux; 366 367 return (ixgbe_lookup(pa) != NULL) ? 1 : 0; 368 } 369 370 static ixgbe_vendor_info_t * 371 ixgbe_lookup(const struct pci_attach_args *pa) 372 { 373 pcireg_t subid; 374 ixgbe_vendor_info_t *ent; 375 376 INIT_DEBUGOUT("ixgbe_probe: begin"); 377 378 if (PCI_VENDOR(pa->pa_id) != IXGBE_INTEL_VENDOR_ID) 379 return NULL; 380 381 subid = pci_conf_read(pa->pa_pc, pa->pa_tag, PCI_SUBSYS_ID_REG); 382 383 for (ent = ixgbe_vendor_info_array; ent->vendor_id != 0; ent++) { 384 if (PCI_VENDOR(pa->pa_id) == ent->vendor_id && 385 PCI_PRODUCT(pa->pa_id) == ent->device_id && 386 387 (PCI_SUBSYS_VENDOR(subid) == ent->subvendor_id || 388 ent->subvendor_id == 0) && 389 390 (PCI_SUBSYS_ID(subid) == ent->subdevice_id || 391 ent->subdevice_id == 0)) { 392 ++ixgbe_total_ports; 393 return ent; 394 } 395 } 396 return NULL; 397 } 398 399 400 static void 401 ixgbe_sysctl_attach(struct adapter *adapter) 402 { 403 struct sysctllog **log; 404 const struct sysctlnode *rnode, *cnode; 405 device_t dev; 406 407 dev = adapter->dev; 408 log = &adapter->sysctllog; 409 410 if ((rnode = ixgbe_sysctl_instance(adapter)) == NULL) { 411 aprint_error_dev(dev, "could not create sysctl root\n"); 412 return; 413 } 414 415 if (sysctl_createv(log, 0, &rnode, &cnode, 416 CTLFLAG_READONLY, CTLTYPE_INT, 417 "num_rx_desc", SYSCTL_DESCR("Number of rx descriptors"), 418 NULL, 0, &adapter->num_rx_desc, 0, CTL_CREATE, CTL_EOL) != 0) 419 aprint_error_dev(dev, "could not create sysctl\n"); 420 421 if (sysctl_createv(log, 0, &rnode, &cnode, 422 CTLFLAG_READONLY, CTLTYPE_INT, 423 "num_queues", SYSCTL_DESCR("Number of queues"), 424 NULL, 0, &adapter->num_queues, 0, CTL_CREATE, CTL_EOL) != 0) 425 aprint_error_dev(dev, "could not create sysctl\n"); 426 427 if (sysctl_createv(log, 0, &rnode, &cnode, 428 CTLFLAG_READWRITE, CTLTYPE_INT, 429 "flow_control", SYSCTL_DESCR("Flow Control"), 430 ixgbe_set_flowcntl, 0, (void *)adapter, 0, CTL_CREATE, CTL_EOL) != 0) 431 aprint_error_dev(dev, "could not create sysctl\n"); 432 433 if (sysctl_createv(log, 0, &rnode, &cnode, 434 CTLFLAG_READWRITE, CTLTYPE_INT, 435 "advertise_gig", SYSCTL_DESCR("1G Link"), 436 ixgbe_set_advertise, 0, (void *)adapter, 0, CTL_CREATE, CTL_EOL) != 0) 437 aprint_error_dev(dev, "could not create sysctl\n"); 438 439 /* XXX This is an *instance* sysctl controlling a *global* variable. 440 * XXX It's that way in the FreeBSD driver that this derives from. 441 */ 442 if (sysctl_createv(log, 0, &rnode, &cnode, 443 CTLFLAG_READWRITE, CTLTYPE_INT, 444 "enable_aim", SYSCTL_DESCR("Interrupt Moderation"), 445 NULL, 0, &ixgbe_enable_aim, 0, CTL_CREATE, CTL_EOL) != 0) 446 aprint_error_dev(dev, "could not create sysctl\n"); 447 } 448 449 /********************************************************************* 450 * Device initialization routine 451 * 452 * The attach entry point is called when the driver is being loaded. 453 * This routine identifies the type of hardware, allocates all resources 454 * and initializes the hardware. 455 * 456 * return 0 on success, positive on failure 457 *********************************************************************/ 458 459 static void 460 ixgbe_attach(device_t parent, device_t dev, void *aux) 461 { 462 struct adapter *adapter; 463 struct ixgbe_hw *hw; 464 int error = 0; 465 u16 csum; 466 u32 ctrl_ext; 467 ixgbe_vendor_info_t *ent; 468 const struct pci_attach_args *pa = aux; 469 470 INIT_DEBUGOUT("ixgbe_attach: begin"); 471 472 /* Allocate, clear, and link in our adapter structure */ 473 adapter = device_private(dev); 474 adapter->dev = adapter->osdep.dev = dev; 475 hw = &adapter->hw; 476 adapter->osdep.pc = pa->pa_pc; 477 adapter->osdep.tag = pa->pa_tag; 478 adapter->osdep.dmat = pa->pa_dmat; 479 480 ent = ixgbe_lookup(pa); 481 482 KASSERT(ent != NULL); 483 484 aprint_normal(": %s, Version - %s\n", 485 ixgbe_strings[ent->index], ixgbe_driver_version); 486 487 /* Core Lock Init*/ 488 IXGBE_CORE_LOCK_INIT(adapter, device_xname(dev)); 489 490 /* SYSCTL APIs */ 491 492 ixgbe_sysctl_attach(adapter); 493 494 /* Set up the timer callout */ 495 callout_init(&adapter->timer, 0); 496 497 /* Determine hardware revision */ 498 ixgbe_identify_hardware(adapter); 499 500 /* Do base PCI setup - map BAR0 */ 501 if (ixgbe_allocate_pci_resources(adapter, pa)) { 502 aprint_error_dev(dev, "Allocation of PCI resources failed\n"); 503 error = ENXIO; 504 goto err_out; 505 } 506 507 /* Do descriptor calc and sanity checks */ 508 if (((ixgbe_txd * sizeof(union ixgbe_adv_tx_desc)) % DBA_ALIGN) != 0 || 509 ixgbe_txd < MIN_TXD || ixgbe_txd > MAX_TXD) { 510 aprint_error_dev(dev, "TXD config issue, using default!\n"); 511 adapter->num_tx_desc = DEFAULT_TXD; 512 } else 513 adapter->num_tx_desc = ixgbe_txd; 514 515 /* 516 ** With many RX rings it is easy to exceed the 517 ** system mbuf allocation. Tuning nmbclusters 518 ** can alleviate this. 519 */ 520 if (nmbclusters > 0 ) { 521 int s; 522 s = (ixgbe_rxd * adapter->num_queues) * ixgbe_total_ports; 523 if (s > nmbclusters) { 524 aprint_error_dev(dev, "RX Descriptors exceed " 525 "system mbuf max, using default instead!\n"); 526 ixgbe_rxd = DEFAULT_RXD; 527 } 528 } 529 530 if (((ixgbe_rxd * sizeof(union ixgbe_adv_rx_desc)) % DBA_ALIGN) != 0 || 531 ixgbe_rxd < MIN_TXD || ixgbe_rxd > MAX_TXD) { 532 aprint_error_dev(dev, "RXD config issue, using default!\n"); 533 adapter->num_rx_desc = DEFAULT_RXD; 534 } else 535 adapter->num_rx_desc = ixgbe_rxd; 536 537 /* Allocate our TX/RX Queues */ 538 if (ixgbe_allocate_queues(adapter)) { 539 error = ENOMEM; 540 goto err_out; 541 } 542 543 /* Allocate multicast array memory. */ 544 adapter->mta = malloc(sizeof(u8) * IXGBE_ETH_LENGTH_OF_ADDRESS * 545 MAX_NUM_MULTICAST_ADDRESSES, M_DEVBUF, M_NOWAIT); 546 if (adapter->mta == NULL) { 547 aprint_error_dev(dev, "Cannot allocate multicast setup array\n"); 548 error = ENOMEM; 549 goto err_late; 550 } 551 552 /* Initialize the shared code */ 553 error = ixgbe_init_shared_code(hw); 554 if (error == IXGBE_ERR_SFP_NOT_PRESENT) { 555 /* 556 ** No optics in this port, set up 557 ** so the timer routine will probe 558 ** for later insertion. 559 */ 560 adapter->sfp_probe = TRUE; 561 error = 0; 562 } else if (error == IXGBE_ERR_SFP_NOT_SUPPORTED) { 563 aprint_error_dev(dev,"Unsupported SFP+ module detected!\n"); 564 error = EIO; 565 goto err_late; 566 } else if (error) { 567 aprint_error_dev(dev,"Unable to initialize the shared code\n"); 568 error = EIO; 569 goto err_late; 570 } 571 572 /* Make sure we have a good EEPROM before we read from it */ 573 if (ixgbe_validate_eeprom_checksum(&adapter->hw, &csum) < 0) { 574 aprint_error_dev(dev,"The EEPROM Checksum Is Not Valid\n"); 575 error = EIO; 576 goto err_late; 577 } 578 579 /* Get Hardware Flow Control setting */ 580 hw->fc.requested_mode = ixgbe_fc_full; 581 hw->fc.pause_time = IXGBE_FC_PAUSE; 582 hw->fc.low_water = IXGBE_FC_LO; 583 hw->fc.high_water = IXGBE_FC_HI; 584 hw->fc.send_xon = TRUE; 585 586 error = ixgbe_init_hw(hw); 587 if (error == IXGBE_ERR_EEPROM_VERSION) { 588 aprint_error_dev(dev, "This device is a pre-production adapter/" 589 "LOM. Please be aware there may be issues associated " 590 "with your hardware.\n If you are experiencing problems " 591 "please contact your Intel or hardware representative " 592 "who provided you with this hardware.\n"); 593 } else if (error == IXGBE_ERR_SFP_NOT_SUPPORTED) 594 aprint_error_dev(dev,"Unsupported SFP+ Module\n"); 595 596 if (error) { 597 error = EIO; 598 aprint_error_dev(dev,"Hardware Initialization Failure\n"); 599 goto err_late; 600 } 601 602 /* Detect and set physical type */ 603 ixgbe_setup_optics(adapter); 604 605 if ((adapter->msix > 1) && (ixgbe_enable_msix)) 606 error = ixgbe_allocate_msix(adapter, pa); 607 else 608 error = ixgbe_allocate_legacy(adapter, pa); 609 if (error) 610 goto err_late; 611 612 /* Setup OS specific network interface */ 613 if (ixgbe_setup_interface(dev, adapter) != 0) 614 goto err_late; 615 616 /* Sysctl for limiting the amount of work done in software interrupts */ 617 ixgbe_add_rx_process_limit(adapter, "rx_processing_limit", 618 "max number of rx packets to process", &adapter->rx_process_limit, 619 ixgbe_rx_process_limit); 620 621 /* Initialize statistics */ 622 ixgbe_update_stats_counters(adapter); 623 624 /* Print PCIE bus type/speed/width info */ 625 ixgbe_get_bus_info(hw); 626 aprint_normal_dev(dev,"PCI Express Bus: Speed %s %s\n", 627 ((hw->bus.speed == ixgbe_bus_speed_5000) ? "5.0Gb/s": 628 (hw->bus.speed == ixgbe_bus_speed_2500) ? "2.5Gb/s":"Unknown"), 629 (hw->bus.width == ixgbe_bus_width_pcie_x8) ? "Width x8" : 630 (hw->bus.width == ixgbe_bus_width_pcie_x4) ? "Width x4" : 631 (hw->bus.width == ixgbe_bus_width_pcie_x1) ? "Width x1" : 632 ("Unknown")); 633 634 if ((hw->bus.width <= ixgbe_bus_width_pcie_x4) && 635 (hw->bus.speed == ixgbe_bus_speed_2500)) { 636 aprint_error_dev(dev, "PCI-Express bandwidth available" 637 " for this card\n is not sufficient for" 638 " optimal performance.\n"); 639 aprint_error_dev(dev, "For optimal performance a x8 " 640 "PCIE, or x4 PCIE 2 slot is required.\n"); 641 } 642 643 /* let hardware know driver is loaded */ 644 ctrl_ext = IXGBE_READ_REG(hw, IXGBE_CTRL_EXT); 645 ctrl_ext |= IXGBE_CTRL_EXT_DRV_LOAD; 646 IXGBE_WRITE_REG(hw, IXGBE_CTRL_EXT, ctrl_ext); 647 648 ixgbe_add_hw_stats(adapter); 649 650 INIT_DEBUGOUT("ixgbe_attach: end"); 651 return; 652 err_late: 653 ixgbe_free_transmit_structures(adapter); 654 ixgbe_free_receive_structures(adapter); 655 err_out: 656 if (adapter->ifp != NULL) 657 if_free(adapter->ifp); 658 ixgbe_free_pci_resources(adapter); 659 if (adapter->mta != NULL) 660 free(adapter->mta, M_DEVBUF); 661 return; 662 663 } 664 665 /********************************************************************* 666 * Device removal routine 667 * 668 * The detach entry point is called when the driver is being removed. 669 * This routine stops the adapter and deallocates all the resources 670 * that were allocated for driver operation. 671 * 672 * return 0 on success, positive on failure 673 *********************************************************************/ 674 675 static int 676 ixgbe_detach(device_t dev, int flags) 677 { 678 struct adapter *adapter = device_private(dev); 679 struct tx_ring *txr = adapter->tx_rings; 680 struct rx_ring *rxr = adapter->rx_rings; 681 struct ixgbe_hw_stats *stats = &adapter->stats; 682 struct ix_queue *que = adapter->queues; 683 u32 ctrl_ext; 684 685 INIT_DEBUGOUT("ixgbe_detach: begin"); 686 687 /* Make sure VLANs are not using driver */ 688 if (!VLAN_ATTACHED(&adapter->osdep.ec)) 689 ; /* nothing to do: no VLANs */ 690 else if ((flags & (DETACH_SHUTDOWN|DETACH_FORCE)) != 0) 691 vlan_ifdetach(adapter->ifp); 692 else { 693 aprint_error_dev(dev, "VLANs in use\n"); 694 return EBUSY; 695 } 696 697 IXGBE_CORE_LOCK(adapter); 698 ixgbe_stop(adapter); 699 IXGBE_CORE_UNLOCK(adapter); 700 701 for (int i = 0; i < adapter->num_queues; i++, que++) { 702 softint_disestablish(que->que_si); 703 } 704 705 /* Drain the Link queue */ 706 softint_disestablish(adapter->link_si); 707 softint_disestablish(adapter->mod_si); 708 softint_disestablish(adapter->msf_si); 709 #ifdef IXGBE_FDIR 710 softint_disestablish(adapter->fdir_si); 711 #endif 712 713 /* let hardware know driver is unloading */ 714 ctrl_ext = IXGBE_READ_REG(&adapter->hw, IXGBE_CTRL_EXT); 715 ctrl_ext &= ~IXGBE_CTRL_EXT_DRV_LOAD; 716 IXGBE_WRITE_REG(&adapter->hw, IXGBE_CTRL_EXT, ctrl_ext); 717 718 ether_ifdetach(adapter->ifp); 719 callout_halt(&adapter->timer, NULL); 720 ixgbe_free_pci_resources(adapter); 721 #if 0 /* XXX the NetBSD port is probably missing something here */ 722 bus_generic_detach(dev); 723 #endif 724 if_detach(adapter->ifp); 725 726 sysctl_teardown(&adapter->sysctllog); 727 evcnt_detach(&adapter->handleq); 728 evcnt_detach(&adapter->req); 729 evcnt_detach(&adapter->morerx); 730 evcnt_detach(&adapter->moretx); 731 evcnt_detach(&adapter->txloops); 732 evcnt_detach(&adapter->efbig_tx_dma_setup); 733 evcnt_detach(&adapter->m_defrag_failed); 734 evcnt_detach(&adapter->efbig2_tx_dma_setup); 735 evcnt_detach(&adapter->einval_tx_dma_setup); 736 evcnt_detach(&adapter->other_tx_dma_setup); 737 evcnt_detach(&adapter->eagain_tx_dma_setup); 738 evcnt_detach(&adapter->enomem_tx_dma_setup); 739 evcnt_detach(&adapter->watchdog_events); 740 evcnt_detach(&adapter->tso_err); 741 evcnt_detach(&adapter->tso_tx); 742 evcnt_detach(&adapter->link_irq); 743 for (int i = 0; i < adapter->num_queues; i++, rxr++, txr++) { 744 evcnt_detach(&txr->no_desc_avail); 745 evcnt_detach(&txr->total_packets); 746 747 if (i < __arraycount(adapter->stats.mpc)) { 748 evcnt_detach(&adapter->stats.mpc[i]); 749 } 750 if (i < __arraycount(adapter->stats.pxontxc)) { 751 evcnt_detach(&adapter->stats.pxontxc[i]); 752 evcnt_detach(&adapter->stats.pxonrxc[i]); 753 evcnt_detach(&adapter->stats.pxofftxc[i]); 754 evcnt_detach(&adapter->stats.pxoffrxc[i]); 755 evcnt_detach(&adapter->stats.pxon2offc[i]); 756 } 757 if (i < __arraycount(adapter->stats.qprc)) { 758 evcnt_detach(&adapter->stats.qprc[i]); 759 evcnt_detach(&adapter->stats.qptc[i]); 760 evcnt_detach(&adapter->stats.qbrc[i]); 761 evcnt_detach(&adapter->stats.qbtc[i]); 762 evcnt_detach(&adapter->stats.qprdc[i]); 763 } 764 765 evcnt_detach(&rxr->rx_packets); 766 evcnt_detach(&rxr->rx_bytes); 767 evcnt_detach(&rxr->no_jmbuf); 768 evcnt_detach(&rxr->rx_discarded); 769 evcnt_detach(&rxr->rx_split_packets); 770 evcnt_detach(&rxr->rx_irq); 771 } 772 evcnt_detach(&stats->ipcs); 773 evcnt_detach(&stats->l4cs); 774 evcnt_detach(&stats->ipcs_bad); 775 evcnt_detach(&stats->l4cs_bad); 776 evcnt_detach(&stats->intzero); 777 evcnt_detach(&stats->legint); 778 evcnt_detach(&stats->crcerrs); 779 evcnt_detach(&stats->illerrc); 780 evcnt_detach(&stats->errbc); 781 evcnt_detach(&stats->mspdc); 782 evcnt_detach(&stats->mlfc); 783 evcnt_detach(&stats->mrfc); 784 evcnt_detach(&stats->rlec); 785 evcnt_detach(&stats->lxontxc); 786 evcnt_detach(&stats->lxonrxc); 787 evcnt_detach(&stats->lxofftxc); 788 evcnt_detach(&stats->lxoffrxc); 789 790 /* Packet Reception Stats */ 791 evcnt_detach(&stats->tor); 792 evcnt_detach(&stats->gorc); 793 evcnt_detach(&stats->tpr); 794 evcnt_detach(&stats->gprc); 795 evcnt_detach(&stats->mprc); 796 evcnt_detach(&stats->bprc); 797 evcnt_detach(&stats->prc64); 798 evcnt_detach(&stats->prc127); 799 evcnt_detach(&stats->prc255); 800 evcnt_detach(&stats->prc511); 801 evcnt_detach(&stats->prc1023); 802 evcnt_detach(&stats->prc1522); 803 evcnt_detach(&stats->ruc); 804 evcnt_detach(&stats->rfc); 805 evcnt_detach(&stats->roc); 806 evcnt_detach(&stats->rjc); 807 evcnt_detach(&stats->mngprc); 808 evcnt_detach(&stats->xec); 809 810 /* Packet Transmission Stats */ 811 evcnt_detach(&stats->gotc); 812 evcnt_detach(&stats->tpt); 813 evcnt_detach(&stats->gptc); 814 evcnt_detach(&stats->bptc); 815 evcnt_detach(&stats->mptc); 816 evcnt_detach(&stats->mngptc); 817 evcnt_detach(&stats->ptc64); 818 evcnt_detach(&stats->ptc127); 819 evcnt_detach(&stats->ptc255); 820 evcnt_detach(&stats->ptc511); 821 evcnt_detach(&stats->ptc1023); 822 evcnt_detach(&stats->ptc1522); 823 824 /* FC Stats */ 825 evcnt_detach(&stats->fccrc); 826 evcnt_detach(&stats->fclast); 827 if (adapter->hw.mac.type != ixgbe_mac_82598EB) { 828 evcnt_detach(&stats->fcoerpdc); 829 evcnt_detach(&stats->fcoeprc); 830 evcnt_detach(&stats->fcoeptc); 831 evcnt_detach(&stats->fcoedwrc); 832 evcnt_detach(&stats->fcoedwtc); 833 } 834 835 ixgbe_free_transmit_structures(adapter); 836 ixgbe_free_receive_structures(adapter); 837 free(adapter->mta, M_DEVBUF); 838 839 IXGBE_CORE_LOCK_DESTROY(adapter); 840 return (0); 841 } 842 843 /********************************************************************* 844 * 845 * Shutdown entry point 846 * 847 **********************************************************************/ 848 849 #if 0 /* XXX NetBSD ought to register something like this through pmf(9) */ 850 static int 851 ixgbe_shutdown(device_t dev) 852 { 853 struct adapter *adapter = device_private(dev); 854 IXGBE_CORE_LOCK(adapter); 855 ixgbe_stop(adapter); 856 IXGBE_CORE_UNLOCK(adapter); 857 return (0); 858 } 859 #endif 860 861 862 /********************************************************************* 863 * Transmit entry point 864 * 865 * ixgbe_start is called by the stack to initiate a transmit. 866 * The driver will remain in this routine as long as there are 867 * packets to transmit and transmit resources are available. 868 * In case resources are not available stack is notified and 869 * the packet is requeued. 870 **********************************************************************/ 871 872 static void 873 ixgbe_start_locked(struct tx_ring *txr, struct ifnet * ifp) 874 { 875 int rc; 876 struct mbuf *m_head; 877 struct adapter *adapter = txr->adapter; 878 879 IXGBE_TX_LOCK_ASSERT(txr); 880 881 if ((ifp->if_flags & (IFF_RUNNING|IFF_OACTIVE)) != 882 IFF_RUNNING) 883 return; 884 if (!adapter->link_active) 885 return; 886 887 while (!IFQ_IS_EMPTY(&ifp->if_snd)) { 888 889 IFQ_POLL(&ifp->if_snd, m_head); 890 if (m_head == NULL) 891 break; 892 893 if ((rc = ixgbe_xmit(txr, m_head)) == EAGAIN) { 894 ifp->if_flags |= IFF_OACTIVE; 895 break; 896 } 897 IFQ_DEQUEUE(&ifp->if_snd, m_head); 898 if (rc == EFBIG) { 899 struct mbuf *mtmp; 900 901 if ((mtmp = m_defrag(m_head, M_DONTWAIT)) != NULL) { 902 m_head = mtmp; 903 rc = ixgbe_xmit(txr, m_head); 904 if (rc != 0) 905 adapter->efbig2_tx_dma_setup.ev_count++; 906 } else 907 adapter->m_defrag_failed.ev_count++; 908 } 909 if (rc != 0) { 910 m_freem(m_head); 911 continue; 912 } 913 914 /* Send a copy of the frame to the BPF listener */ 915 bpf_mtap(ifp, m_head); 916 917 /* Set watchdog on */ 918 getmicrotime(&txr->watchdog_time); 919 txr->queue_status = IXGBE_QUEUE_WORKING; 920 921 } 922 return; 923 } 924 925 /* 926 * Legacy TX start - called by the stack, this 927 * always uses the first tx ring, and should 928 * not be used with multiqueue tx enabled. 929 */ 930 static void 931 ixgbe_start(struct ifnet *ifp) 932 { 933 struct adapter *adapter = ifp->if_softc; 934 struct tx_ring *txr = adapter->tx_rings; 935 936 if (ifp->if_flags & IFF_RUNNING) { 937 IXGBE_TX_LOCK(txr); 938 ixgbe_start_locked(txr, ifp); 939 IXGBE_TX_UNLOCK(txr); 940 } 941 return; 942 } 943 944 #if __FreeBSD_version >= 800000 945 /* 946 ** Multiqueue Transmit driver 947 ** 948 */ 949 static int 950 ixgbe_mq_start(struct ifnet *ifp, struct mbuf *m) 951 { 952 struct adapter *adapter = ifp->if_softc; 953 struct ix_queue *que; 954 struct tx_ring *txr; 955 int i = 0, err = 0; 956 957 /* Which queue to use */ 958 if ((m->m_flags & M_FLOWID) != 0) 959 i = m->m_pkthdr.flowid % adapter->num_queues; 960 961 txr = &adapter->tx_rings[i]; 962 que = &adapter->queues[i]; 963 964 if (IXGBE_TX_TRYLOCK(txr)) { 965 err = ixgbe_mq_start_locked(ifp, txr, m); 966 IXGBE_TX_UNLOCK(txr); 967 } else { 968 err = drbr_enqueue(ifp, txr->br, m); 969 softint_schedule(que->que_si); 970 } 971 972 return (err); 973 } 974 975 static int 976 ixgbe_mq_start_locked(struct ifnet *ifp, struct tx_ring *txr, struct mbuf *m) 977 { 978 struct adapter *adapter = txr->adapter; 979 struct mbuf *next; 980 int enqueued, err = 0; 981 982 if ((ifp->if_flags & (IFF_RUNNING | IFF_OACTIVE)) != 983 IFF_RUNNING || adapter->link_active == 0) { 984 if (m != NULL) 985 err = drbr_enqueue(ifp, txr->br, m); 986 return (err); 987 } 988 989 enqueued = 0; 990 if (m == NULL) { 991 next = drbr_dequeue(ifp, txr->br); 992 } else if (drbr_needs_enqueue(ifp, txr->br)) { 993 if ((err = drbr_enqueue(ifp, txr->br, m)) != 0) 994 return (err); 995 next = drbr_dequeue(ifp, txr->br); 996 } else 997 next = m; 998 999 /* Process the queue */ 1000 while (next != NULL) { 1001 if ((err = ixgbe_xmit(txr, &next)) != 0) { 1002 if (next != NULL) 1003 err = drbr_enqueue(ifp, txr->br, next); 1004 break; 1005 } 1006 enqueued++; 1007 drbr_stats_update(ifp, next->m_pkthdr.len, next->m_flags); 1008 /* Send a copy of the frame to the BPF listener */ 1009 bpf_mtap(ifp, next); 1010 if ((ifp->if_flags & IFF_RUNNING) == 0) 1011 break; 1012 if (txr->tx_avail < IXGBE_TX_OP_THRESHOLD) 1013 ixgbe_txeof(txr); 1014 if (txr->tx_avail < IXGBE_TX_OP_THRESHOLD) { 1015 ifp->if_flags |= IFF_OACTIVE; 1016 break; 1017 } 1018 next = drbr_dequeue(ifp, txr->br); 1019 } 1020 1021 if (enqueued > 0) { 1022 /* Set watchdog on */ 1023 txr->queue_status = IXGBE_QUEUE_WORKING; 1024 getmicrotime(&txr->watchdog_time); 1025 } 1026 1027 return (err); 1028 } 1029 1030 /* 1031 ** Flush all ring buffers 1032 */ 1033 static void 1034 ixgbe_qflush(struct ifnet *ifp) 1035 { 1036 struct adapter *adapter = ifp->if_softc; 1037 struct tx_ring *txr = adapter->tx_rings; 1038 struct mbuf *m; 1039 1040 for (int i = 0; i < adapter->num_queues; i++, txr++) { 1041 IXGBE_TX_LOCK(txr); 1042 while ((m = buf_ring_dequeue_sc(txr->br)) != NULL) 1043 m_freem(m); 1044 IXGBE_TX_UNLOCK(txr); 1045 } 1046 if_qflush(ifp); 1047 } 1048 #endif /* __FreeBSD_version >= 800000 */ 1049 1050 static int 1051 ixgbe_ifflags_cb(struct ethercom *ec) 1052 { 1053 struct ifnet *ifp = &ec->ec_if; 1054 struct adapter *adapter = ifp->if_softc; 1055 int change = ifp->if_flags ^ adapter->if_flags, rc = 0; 1056 1057 IXGBE_CORE_LOCK(adapter); 1058 1059 if (change != 0) 1060 adapter->if_flags = ifp->if_flags; 1061 1062 if ((change & ~(IFF_CANTCHANGE|IFF_DEBUG)) != 0) 1063 rc = ENETRESET; 1064 else if ((change & (IFF_PROMISC | IFF_ALLMULTI)) != 0) 1065 ixgbe_set_promisc(adapter); 1066 1067 IXGBE_CORE_UNLOCK(adapter); 1068 1069 return rc; 1070 } 1071 1072 /********************************************************************* 1073 * Ioctl entry point 1074 * 1075 * ixgbe_ioctl is called when the user wants to configure the 1076 * interface. 1077 * 1078 * return 0 on success, positive on failure 1079 **********************************************************************/ 1080 1081 static int 1082 ixgbe_ioctl(struct ifnet * ifp, u_long command, void *data) 1083 { 1084 struct adapter *adapter = ifp->if_softc; 1085 struct ifcapreq *ifcr = data; 1086 struct ifreq *ifr = data; 1087 int error = 0; 1088 int l4csum_en; 1089 const int l4csum = IFCAP_CSUM_TCPv4_Rx|IFCAP_CSUM_UDPv4_Rx| 1090 IFCAP_CSUM_TCPv6_Rx|IFCAP_CSUM_UDPv6_Rx; 1091 1092 switch (command) { 1093 case SIOCSIFFLAGS: 1094 IOCTL_DEBUGOUT("ioctl: SIOCSIFFLAGS (Set Interface Flags)"); 1095 break; 1096 case SIOCADDMULTI: 1097 case SIOCDELMULTI: 1098 IOCTL_DEBUGOUT("ioctl: SIOC(ADD|DEL)MULTI"); 1099 break; 1100 case SIOCSIFMEDIA: 1101 case SIOCGIFMEDIA: 1102 IOCTL_DEBUGOUT("ioctl: SIOCxIFMEDIA (Get/Set Interface Media)"); 1103 break; 1104 case SIOCSIFCAP: 1105 IOCTL_DEBUGOUT("ioctl: SIOCSIFCAP (Set Capabilities)"); 1106 break; 1107 case SIOCSIFMTU: 1108 IOCTL_DEBUGOUT("ioctl: SIOCSIFMTU (Set Interface MTU)"); 1109 break; 1110 default: 1111 IOCTL_DEBUGOUT1("ioctl: UNKNOWN (0x%X)\n", (int)command); 1112 break; 1113 } 1114 1115 switch (command) { 1116 case SIOCSIFMEDIA: 1117 case SIOCGIFMEDIA: 1118 return ifmedia_ioctl(ifp, ifr, &adapter->media, command); 1119 case SIOCSIFCAP: 1120 /* Layer-4 Rx checksum offload has to be turned on and 1121 * off as a unit. 1122 */ 1123 l4csum_en = ifcr->ifcr_capenable & l4csum; 1124 if (l4csum_en != l4csum && l4csum_en != 0) 1125 return EINVAL; 1126 /*FALLTHROUGH*/ 1127 case SIOCADDMULTI: 1128 case SIOCDELMULTI: 1129 case SIOCSIFFLAGS: 1130 case SIOCSIFMTU: 1131 default: 1132 if ((error = ether_ioctl(ifp, command, data)) != ENETRESET) 1133 return error; 1134 if ((ifp->if_flags & IFF_RUNNING) == 0) 1135 ; 1136 else if (command == SIOCSIFCAP || command == SIOCSIFMTU) { 1137 IXGBE_CORE_LOCK(adapter); 1138 ixgbe_init_locked(adapter); 1139 IXGBE_CORE_UNLOCK(adapter); 1140 } else if (command == SIOCADDMULTI || command == SIOCDELMULTI) { 1141 /* 1142 * Multicast list has changed; set the hardware filter 1143 * accordingly. 1144 */ 1145 IXGBE_CORE_LOCK(adapter); 1146 ixgbe_disable_intr(adapter); 1147 ixgbe_set_multi(adapter); 1148 ixgbe_enable_intr(adapter); 1149 IXGBE_CORE_UNLOCK(adapter); 1150 } 1151 return 0; 1152 } 1153 } 1154 1155 /********************************************************************* 1156 * Init entry point 1157 * 1158 * This routine is used in two ways. It is used by the stack as 1159 * init entry point in network interface structure. It is also used 1160 * by the driver as a hw/sw initialization routine to get to a 1161 * consistent state. 1162 * 1163 * return 0 on success, positive on failure 1164 **********************************************************************/ 1165 #define IXGBE_MHADD_MFS_SHIFT 16 1166 1167 static void 1168 ixgbe_init_locked(struct adapter *adapter) 1169 { 1170 struct ifnet *ifp = adapter->ifp; 1171 device_t dev = adapter->dev; 1172 struct ixgbe_hw *hw = &adapter->hw; 1173 u32 k, txdctl, mhadd, gpie; 1174 u32 rxdctl, rxctrl; 1175 1176 /* XXX check IFF_UP and IFF_RUNNING, power-saving state! */ 1177 1178 KASSERT(mutex_owned(&adapter->core_mtx)); 1179 INIT_DEBUGOUT("ixgbe_init: begin"); 1180 hw->adapter_stopped = FALSE; 1181 ixgbe_stop_adapter(hw); 1182 callout_stop(&adapter->timer); 1183 1184 /* XXX I moved this here from the SIOCSIFMTU case in ixgbe_ioctl(). */ 1185 adapter->max_frame_size = 1186 ifp->if_mtu + ETHER_HDR_LEN + ETHER_CRC_LEN; 1187 1188 /* reprogram the RAR[0] in case user changed it. */ 1189 ixgbe_set_rar(hw, 0, adapter->hw.mac.addr, 0, IXGBE_RAH_AV); 1190 1191 /* Get the latest mac address, User can use a LAA */ 1192 memcpy(hw->mac.addr, CLLADDR(adapter->ifp->if_sadl), 1193 IXGBE_ETH_LENGTH_OF_ADDRESS); 1194 ixgbe_set_rar(hw, 0, hw->mac.addr, 0, 1); 1195 hw->addr_ctrl.rar_used_count = 1; 1196 1197 /* Prepare transmit descriptors and buffers */ 1198 if (ixgbe_setup_transmit_structures(adapter)) { 1199 device_printf(dev,"Could not setup transmit structures\n"); 1200 ixgbe_stop(adapter); 1201 return; 1202 } 1203 1204 ixgbe_init_hw(hw); 1205 ixgbe_initialize_transmit_units(adapter); 1206 1207 /* Setup Multicast table */ 1208 ixgbe_set_multi(adapter); 1209 1210 /* 1211 ** Determine the correct mbuf pool 1212 ** for doing jumbo/headersplit 1213 */ 1214 if (adapter->max_frame_size <= 2048) 1215 adapter->rx_mbuf_sz = MCLBYTES; 1216 else if (adapter->max_frame_size <= 4096) 1217 adapter->rx_mbuf_sz = MJUMPAGESIZE; 1218 else if (adapter->max_frame_size <= 9216) 1219 adapter->rx_mbuf_sz = MJUM9BYTES; 1220 else 1221 adapter->rx_mbuf_sz = MJUM16BYTES; 1222 1223 /* Prepare receive descriptors and buffers */ 1224 if (ixgbe_setup_receive_structures(adapter)) { 1225 device_printf(dev,"Could not setup receive structures\n"); 1226 ixgbe_stop(adapter); 1227 return; 1228 } 1229 1230 /* Configure RX settings */ 1231 ixgbe_initialize_receive_units(adapter); 1232 1233 gpie = IXGBE_READ_REG(&adapter->hw, IXGBE_GPIE); 1234 1235 /* Enable Fan Failure Interrupt */ 1236 gpie |= IXGBE_SDP1_GPIEN; 1237 1238 /* Add for Thermal detection */ 1239 if (hw->mac.type == ixgbe_mac_82599EB) 1240 gpie |= IXGBE_SDP2_GPIEN; 1241 1242 if (adapter->msix > 1) { 1243 /* Enable Enhanced MSIX mode */ 1244 gpie |= IXGBE_GPIE_MSIX_MODE; 1245 gpie |= IXGBE_GPIE_EIAME | IXGBE_GPIE_PBA_SUPPORT | 1246 IXGBE_GPIE_OCD; 1247 } 1248 IXGBE_WRITE_REG(hw, IXGBE_GPIE, gpie); 1249 1250 /* Set MTU size */ 1251 if (ifp->if_mtu > ETHERMTU) { 1252 mhadd = IXGBE_READ_REG(hw, IXGBE_MHADD); 1253 mhadd &= ~IXGBE_MHADD_MFS_MASK; 1254 mhadd |= adapter->max_frame_size << IXGBE_MHADD_MFS_SHIFT; 1255 IXGBE_WRITE_REG(hw, IXGBE_MHADD, mhadd); 1256 } 1257 1258 /* Now enable all the queues */ 1259 1260 for (int i = 0; i < adapter->num_queues; i++) { 1261 txdctl = IXGBE_READ_REG(hw, IXGBE_TXDCTL(i)); 1262 txdctl |= IXGBE_TXDCTL_ENABLE; 1263 /* Set WTHRESH to 8, burst writeback */ 1264 txdctl |= (8 << 16); 1265 IXGBE_WRITE_REG(hw, IXGBE_TXDCTL(i), txdctl); 1266 } 1267 1268 for (int i = 0; i < adapter->num_queues; i++) { 1269 rxdctl = IXGBE_READ_REG(hw, IXGBE_RXDCTL(i)); 1270 if (hw->mac.type == ixgbe_mac_82598EB) { 1271 /* 1272 ** PTHRESH = 21 1273 ** HTHRESH = 4 1274 ** WTHRESH = 8 1275 */ 1276 rxdctl &= ~0x3FFFFF; 1277 rxdctl |= 0x080420; 1278 } 1279 rxdctl |= IXGBE_RXDCTL_ENABLE; 1280 IXGBE_WRITE_REG(hw, IXGBE_RXDCTL(i), rxdctl); 1281 /* XXX I don't trust this loop, and I don't trust the 1282 * XXX memory barrier. What is this meant to do? --dyoung 1283 */ 1284 for (k = 0; k < 10; k++) { 1285 if (IXGBE_READ_REG(hw, IXGBE_RXDCTL(i)) & 1286 IXGBE_RXDCTL_ENABLE) 1287 break; 1288 else 1289 msec_delay(1); 1290 } 1291 wmb(); 1292 IXGBE_WRITE_REG(hw, IXGBE_RDT(i), adapter->num_rx_desc - 1); 1293 } 1294 1295 /* Set up VLAN support and filter */ 1296 ixgbe_setup_vlan_hw_support(adapter); 1297 1298 /* Enable Receive engine */ 1299 rxctrl = IXGBE_READ_REG(hw, IXGBE_RXCTRL); 1300 if (hw->mac.type == ixgbe_mac_82598EB) 1301 rxctrl |= IXGBE_RXCTRL_DMBYPS; 1302 rxctrl |= IXGBE_RXCTRL_RXEN; 1303 ixgbe_enable_rx_dma(hw, rxctrl); 1304 1305 callout_reset(&adapter->timer, hz, ixgbe_local_timer, adapter); 1306 1307 /* Set up MSI/X routing */ 1308 if (ixgbe_enable_msix) { 1309 ixgbe_configure_ivars(adapter); 1310 /* Set up auto-mask */ 1311 if (hw->mac.type == ixgbe_mac_82598EB) 1312 IXGBE_WRITE_REG(hw, IXGBE_EIAM, IXGBE_EICS_RTX_QUEUE); 1313 else { 1314 IXGBE_WRITE_REG(hw, IXGBE_EIAM_EX(0), 0xFFFFFFFF); 1315 IXGBE_WRITE_REG(hw, IXGBE_EIAM_EX(1), 0xFFFFFFFF); 1316 } 1317 } else { /* Simple settings for Legacy/MSI */ 1318 ixgbe_set_ivar(adapter, 0, 0, 0); 1319 ixgbe_set_ivar(adapter, 0, 0, 1); 1320 IXGBE_WRITE_REG(hw, IXGBE_EIAM, IXGBE_EICS_RTX_QUEUE); 1321 } 1322 1323 #ifdef IXGBE_FDIR 1324 /* Init Flow director */ 1325 if (hw->mac.type != ixgbe_mac_82598EB) 1326 ixgbe_init_fdir_signature_82599(&adapter->hw, fdir_pballoc); 1327 #endif 1328 1329 /* 1330 ** Check on any SFP devices that 1331 ** need to be kick-started 1332 */ 1333 if (hw->phy.type == ixgbe_phy_none) { 1334 int err = hw->phy.ops.identify(hw); 1335 if (err == IXGBE_ERR_SFP_NOT_SUPPORTED) { 1336 device_printf(dev, 1337 "Unsupported SFP+ module type was detected.\n"); 1338 return; 1339 } 1340 } 1341 1342 /* Set moderation on the Link interrupt */ 1343 IXGBE_WRITE_REG(hw, IXGBE_EITR(adapter->linkvec), IXGBE_LINK_ITR); 1344 1345 /* Config/Enable Link */ 1346 ixgbe_config_link(adapter); 1347 1348 /* And now turn on interrupts */ 1349 ixgbe_enable_intr(adapter); 1350 1351 /* Now inform the stack we're ready */ 1352 ifp->if_flags |= IFF_RUNNING; 1353 ifp->if_flags &= ~IFF_OACTIVE; 1354 1355 return; 1356 } 1357 1358 static int 1359 ixgbe_init(struct ifnet *ifp) 1360 { 1361 struct adapter *adapter = ifp->if_softc; 1362 1363 IXGBE_CORE_LOCK(adapter); 1364 ixgbe_init_locked(adapter); 1365 IXGBE_CORE_UNLOCK(adapter); 1366 return 0; /* XXX ixgbe_init_locked cannot fail? really? */ 1367 } 1368 1369 1370 /* 1371 ** 1372 ** MSIX Interrupt Handlers and Tasklets 1373 ** 1374 */ 1375 1376 static inline void 1377 ixgbe_enable_queue(struct adapter *adapter, u32 vector) 1378 { 1379 struct ixgbe_hw *hw = &adapter->hw; 1380 u64 queue = (u64)(1ULL << vector); 1381 u32 mask; 1382 1383 if (hw->mac.type == ixgbe_mac_82598EB) { 1384 mask = (IXGBE_EIMS_RTX_QUEUE & queue); 1385 IXGBE_WRITE_REG(hw, IXGBE_EIMS, mask); 1386 } else { 1387 mask = (queue & 0xFFFFFFFF); 1388 if (mask) 1389 IXGBE_WRITE_REG(hw, IXGBE_EIMS_EX(0), mask); 1390 mask = (queue >> 32); 1391 if (mask) 1392 IXGBE_WRITE_REG(hw, IXGBE_EIMS_EX(1), mask); 1393 } 1394 } 1395 1396 __unused static inline void 1397 ixgbe_disable_queue(struct adapter *adapter, u32 vector) 1398 { 1399 struct ixgbe_hw *hw = &adapter->hw; 1400 u64 queue = (u64)(1ULL << vector); 1401 u32 mask; 1402 1403 if (hw->mac.type == ixgbe_mac_82598EB) { 1404 mask = (IXGBE_EIMS_RTX_QUEUE & queue); 1405 IXGBE_WRITE_REG(hw, IXGBE_EIMC, mask); 1406 } else { 1407 mask = (queue & 0xFFFFFFFF); 1408 if (mask) 1409 IXGBE_WRITE_REG(hw, IXGBE_EIMC_EX(0), mask); 1410 mask = (queue >> 32); 1411 if (mask) 1412 IXGBE_WRITE_REG(hw, IXGBE_EIMC_EX(1), mask); 1413 } 1414 } 1415 1416 static inline void 1417 ixgbe_rearm_queues(struct adapter *adapter, u64 queues) 1418 { 1419 u32 mask; 1420 1421 if (adapter->hw.mac.type == ixgbe_mac_82598EB) { 1422 mask = (IXGBE_EIMS_RTX_QUEUE & queues); 1423 IXGBE_WRITE_REG(&adapter->hw, IXGBE_EICS, mask); 1424 } else { 1425 mask = (queues & 0xFFFFFFFF); 1426 IXGBE_WRITE_REG(&adapter->hw, IXGBE_EICS_EX(0), mask); 1427 mask = (queues >> 32); 1428 IXGBE_WRITE_REG(&adapter->hw, IXGBE_EICS_EX(1), mask); 1429 } 1430 } 1431 1432 1433 static void 1434 ixgbe_handle_que(void *context) 1435 { 1436 struct ix_queue *que = context; 1437 struct adapter *adapter = que->adapter; 1438 struct tx_ring *txr = que->txr; 1439 struct ifnet *ifp = adapter->ifp; 1440 bool more; 1441 1442 adapter->handleq.ev_count++; 1443 1444 if (ifp->if_flags & IFF_RUNNING) { 1445 more = ixgbe_rxeof(que, adapter->rx_process_limit); 1446 IXGBE_TX_LOCK(txr); 1447 ixgbe_txeof(txr); 1448 #if __FreeBSD_version >= 800000 1449 if (!drbr_empty(ifp, txr->br)) 1450 ixgbe_mq_start_locked(ifp, txr, NULL); 1451 #else 1452 if (!IFQ_IS_EMPTY(&ifp->if_snd)) 1453 ixgbe_start_locked(txr, ifp); 1454 #endif 1455 IXGBE_TX_UNLOCK(txr); 1456 if (more) { 1457 adapter->req.ev_count++; 1458 softint_schedule(que->que_si); 1459 return; 1460 } 1461 } 1462 1463 /* Reenable this interrupt */ 1464 ixgbe_enable_queue(adapter, que->msix); 1465 1466 return; 1467 } 1468 1469 1470 /********************************************************************* 1471 * 1472 * Legacy Interrupt Service routine 1473 * 1474 **********************************************************************/ 1475 1476 static int 1477 ixgbe_legacy_irq(void *arg) 1478 { 1479 struct ix_queue *que = arg; 1480 struct adapter *adapter = que->adapter; 1481 struct ixgbe_hw *hw = &adapter->hw; 1482 struct tx_ring *txr = adapter->tx_rings; 1483 bool more_tx, more_rx; 1484 u32 reg_eicr, loop = MAX_LOOP; 1485 1486 1487 reg_eicr = IXGBE_READ_REG(hw, IXGBE_EICR); 1488 1489 adapter->stats.legint.ev_count++; 1490 ++que->irqs; 1491 if (reg_eicr == 0) { 1492 adapter->stats.intzero.ev_count++; 1493 ixgbe_enable_intr(adapter); 1494 return 0; 1495 } 1496 1497 more_rx = ixgbe_rxeof(que, adapter->rx_process_limit); 1498 1499 IXGBE_TX_LOCK(txr); 1500 do { 1501 adapter->txloops.ev_count++; 1502 more_tx = ixgbe_txeof(txr); 1503 } while (loop-- && more_tx); 1504 IXGBE_TX_UNLOCK(txr); 1505 1506 if (more_rx || more_tx) { 1507 if (more_rx) 1508 adapter->morerx.ev_count++; 1509 if (more_tx) 1510 adapter->moretx.ev_count++; 1511 softint_schedule(que->que_si); 1512 } 1513 1514 /* Check for fan failure */ 1515 if ((hw->phy.media_type == ixgbe_media_type_copper) && 1516 (reg_eicr & IXGBE_EICR_GPI_SDP1)) { 1517 device_printf(adapter->dev, "\nCRITICAL: FAN FAILURE!! " 1518 "REPLACE IMMEDIATELY!!\n"); 1519 IXGBE_WRITE_REG(hw, IXGBE_EIMS, IXGBE_EICR_GPI_SDP1); 1520 } 1521 1522 /* Link status change */ 1523 if (reg_eicr & IXGBE_EICR_LSC) 1524 softint_schedule(adapter->link_si); 1525 1526 ixgbe_enable_intr(adapter); 1527 return 1; 1528 } 1529 1530 1531 #if defined(NETBSD_MSI_OR_MSIX) 1532 /********************************************************************* 1533 * 1534 * MSI Queue Interrupt Service routine 1535 * 1536 **********************************************************************/ 1537 void 1538 ixgbe_msix_que(void *arg) 1539 { 1540 struct ix_queue *que = arg; 1541 struct adapter *adapter = que->adapter; 1542 struct tx_ring *txr = que->txr; 1543 struct rx_ring *rxr = que->rxr; 1544 bool more_tx, more_rx; 1545 u32 newitr = 0; 1546 1547 ++que->irqs; 1548 1549 more_rx = ixgbe_rxeof(que, adapter->rx_process_limit); 1550 1551 IXGBE_TX_LOCK(txr); 1552 more_tx = ixgbe_txeof(txr); 1553 IXGBE_TX_UNLOCK(txr); 1554 1555 /* Do AIM now? */ 1556 1557 if (ixgbe_enable_aim == FALSE) 1558 goto no_calc; 1559 /* 1560 ** Do Adaptive Interrupt Moderation: 1561 ** - Write out last calculated setting 1562 ** - Calculate based on average size over 1563 ** the last interval. 1564 */ 1565 if (que->eitr_setting) 1566 IXGBE_WRITE_REG(&adapter->hw, 1567 IXGBE_EITR(que->msix), que->eitr_setting); 1568 1569 que->eitr_setting = 0; 1570 1571 /* Idle, do nothing */ 1572 if ((txr->bytes == 0) && (rxr->bytes == 0)) 1573 goto no_calc; 1574 1575 if ((txr->bytes) && (txr->packets)) 1576 newitr = txr->bytes/txr->packets; 1577 if ((rxr->bytes) && (rxr->packets)) 1578 newitr = max(newitr, 1579 (rxr->bytes / rxr->packets)); 1580 newitr += 24; /* account for hardware frame, crc */ 1581 1582 /* set an upper boundary */ 1583 newitr = min(newitr, 3000); 1584 1585 /* Be nice to the mid range */ 1586 if ((newitr > 300) && (newitr < 1200)) 1587 newitr = (newitr / 3); 1588 else 1589 newitr = (newitr / 2); 1590 1591 if (adapter->hw.mac.type == ixgbe_mac_82598EB) 1592 newitr |= newitr << 16; 1593 else 1594 newitr |= IXGBE_EITR_CNT_WDIS; 1595 1596 /* save for next interrupt */ 1597 que->eitr_setting = newitr; 1598 1599 /* Reset state */ 1600 txr->bytes = 0; 1601 txr->packets = 0; 1602 rxr->bytes = 0; 1603 rxr->packets = 0; 1604 1605 no_calc: 1606 if (more_tx || more_rx) 1607 softint_schedule(que->que_si); 1608 else /* Reenable this interrupt */ 1609 ixgbe_enable_queue(adapter, que->msix); 1610 return; 1611 } 1612 1613 1614 static void 1615 ixgbe_msix_link(void *arg) 1616 { 1617 struct adapter *adapter = arg; 1618 struct ixgbe_hw *hw = &adapter->hw; 1619 u32 reg_eicr; 1620 1621 ++adapter->link_irq.ev_count; 1622 1623 /* First get the cause */ 1624 reg_eicr = IXGBE_READ_REG(hw, IXGBE_EICS); 1625 /* Clear interrupt with write */ 1626 IXGBE_WRITE_REG(hw, IXGBE_EICR, reg_eicr); 1627 1628 /* Link status change */ 1629 if (reg_eicr & IXGBE_EICR_LSC) 1630 softint_schedule(adapter->link_si); 1631 1632 if (adapter->hw.mac.type != ixgbe_mac_82598EB) { 1633 #ifdef IXGBE_FDIR 1634 if (reg_eicr & IXGBE_EICR_FLOW_DIR) { 1635 /* This is probably overkill :) */ 1636 if (!atomic_cmpset_int(&adapter->fdir_reinit, 0, 1)) 1637 return; 1638 /* Clear the interrupt */ 1639 IXGBE_WRITE_REG(hw, IXGBE_EICR, IXGBE_EICR_FLOW_DIR); 1640 /* Turn off the interface */ 1641 adapter->ifp->if_flags &= ~IFF_RUNNING; 1642 softint_schedule(adapter->fdir_si); 1643 } else 1644 #endif 1645 if (reg_eicr & IXGBE_EICR_ECC) { 1646 device_printf(adapter->dev, "\nCRITICAL: ECC ERROR!! " 1647 "Please Reboot!!\n"); 1648 IXGBE_WRITE_REG(hw, IXGBE_EICR, IXGBE_EICR_ECC); 1649 } else 1650 1651 if (reg_eicr & IXGBE_EICR_GPI_SDP1) { 1652 /* Clear the interrupt */ 1653 IXGBE_WRITE_REG(hw, IXGBE_EICR, IXGBE_EICR_GPI_SDP1); 1654 softint_schedule(adapter->msf_si); 1655 } else if (reg_eicr & IXGBE_EICR_GPI_SDP2) { 1656 /* Clear the interrupt */ 1657 IXGBE_WRITE_REG(hw, IXGBE_EICR, IXGBE_EICR_GPI_SDP2); 1658 softint_schedule(adapter->mod_si); 1659 } 1660 } 1661 1662 /* Check for fan failure */ 1663 if ((hw->device_id == IXGBE_DEV_ID_82598AT) && 1664 (reg_eicr & IXGBE_EICR_GPI_SDP1)) { 1665 device_printf(adapter->dev, "\nCRITICAL: FAN FAILURE!! " 1666 "REPLACE IMMEDIATELY!!\n"); 1667 IXGBE_WRITE_REG(hw, IXGBE_EICR, IXGBE_EICR_GPI_SDP1); 1668 } 1669 1670 IXGBE_WRITE_REG(&adapter->hw, IXGBE_EIMS, IXGBE_EIMS_OTHER); 1671 return; 1672 } 1673 #endif 1674 1675 /********************************************************************* 1676 * 1677 * Media Ioctl callback 1678 * 1679 * This routine is called whenever the user queries the status of 1680 * the interface using ifconfig. 1681 * 1682 **********************************************************************/ 1683 static void 1684 ixgbe_media_status(struct ifnet * ifp, struct ifmediareq * ifmr) 1685 { 1686 struct adapter *adapter = ifp->if_softc; 1687 1688 INIT_DEBUGOUT("ixgbe_media_status: begin"); 1689 IXGBE_CORE_LOCK(adapter); 1690 ixgbe_update_link_status(adapter); 1691 1692 ifmr->ifm_status = IFM_AVALID; 1693 ifmr->ifm_active = IFM_ETHER; 1694 1695 if (!adapter->link_active) { 1696 IXGBE_CORE_UNLOCK(adapter); 1697 return; 1698 } 1699 1700 ifmr->ifm_status |= IFM_ACTIVE; 1701 1702 switch (adapter->link_speed) { 1703 case IXGBE_LINK_SPEED_1GB_FULL: 1704 ifmr->ifm_active |= IFM_1000_T | IFM_FDX; 1705 break; 1706 case IXGBE_LINK_SPEED_10GB_FULL: 1707 ifmr->ifm_active |= adapter->optics | IFM_FDX; 1708 break; 1709 } 1710 1711 IXGBE_CORE_UNLOCK(adapter); 1712 1713 return; 1714 } 1715 1716 /********************************************************************* 1717 * 1718 * Media Ioctl callback 1719 * 1720 * This routine is called when the user changes speed/duplex using 1721 * media/mediopt option with ifconfig. 1722 * 1723 **********************************************************************/ 1724 static int 1725 ixgbe_media_change(struct ifnet * ifp) 1726 { 1727 struct adapter *adapter = ifp->if_softc; 1728 struct ifmedia *ifm = &adapter->media; 1729 1730 INIT_DEBUGOUT("ixgbe_media_change: begin"); 1731 1732 if (IFM_TYPE(ifm->ifm_media) != IFM_ETHER) 1733 return (EINVAL); 1734 1735 switch (IFM_SUBTYPE(ifm->ifm_media)) { 1736 case IFM_AUTO: 1737 adapter->hw.phy.autoneg_advertised = 1738 IXGBE_LINK_SPEED_1GB_FULL | IXGBE_LINK_SPEED_10GB_FULL; 1739 break; 1740 default: 1741 device_printf(adapter->dev, "Only auto media type\n"); 1742 return (EINVAL); 1743 } 1744 1745 return (0); 1746 } 1747 1748 /********************************************************************* 1749 * 1750 * This routine maps the mbufs to tx descriptors, allowing the 1751 * TX engine to transmit the packets. 1752 * - return 0 on success, positive on failure 1753 * 1754 **********************************************************************/ 1755 1756 static int 1757 ixgbe_xmit(struct tx_ring *txr, struct mbuf *m_head) 1758 { 1759 struct m_tag *mtag; 1760 struct adapter *adapter = txr->adapter; 1761 struct ethercom *ec = &adapter->osdep.ec; 1762 u32 olinfo_status = 0, cmd_type_len; 1763 u32 paylen = 0; 1764 int i, j, error; 1765 int first, last = 0; 1766 bus_dmamap_t map; 1767 struct ixgbe_tx_buf *txbuf; 1768 union ixgbe_adv_tx_desc *txd = NULL; 1769 1770 /* Basic descriptor defines */ 1771 cmd_type_len = (IXGBE_ADVTXD_DTYP_DATA | 1772 IXGBE_ADVTXD_DCMD_IFCS | IXGBE_ADVTXD_DCMD_DEXT); 1773 1774 if ((mtag = VLAN_OUTPUT_TAG(ec, m_head)) != NULL) 1775 cmd_type_len |= IXGBE_ADVTXD_DCMD_VLE; 1776 1777 /* 1778 * Important to capture the first descriptor 1779 * used because it will contain the index of 1780 * the one we tell the hardware to report back 1781 */ 1782 first = txr->next_avail_desc; 1783 txbuf = &txr->tx_buffers[first]; 1784 map = txbuf->map; 1785 1786 /* 1787 * Map the packet for DMA. 1788 */ 1789 error = bus_dmamap_load_mbuf(txr->txtag->dt_dmat, map, 1790 m_head, BUS_DMA_NOWAIT); 1791 1792 switch (error) { 1793 case EAGAIN: 1794 adapter->eagain_tx_dma_setup.ev_count++; 1795 return EAGAIN; 1796 case ENOMEM: 1797 adapter->enomem_tx_dma_setup.ev_count++; 1798 return EAGAIN; 1799 case EFBIG: 1800 adapter->efbig_tx_dma_setup.ev_count++; 1801 return error; 1802 case EINVAL: 1803 adapter->einval_tx_dma_setup.ev_count++; 1804 return error; 1805 default: 1806 adapter->other_tx_dma_setup.ev_count++; 1807 return error; 1808 case 0: 1809 break; 1810 } 1811 1812 /* Make certain there are enough descriptors */ 1813 if (map->dm_nsegs > txr->tx_avail - 2) { 1814 txr->no_desc_avail.ev_count++; 1815 ixgbe_dmamap_unload(txr->txtag, txbuf->map); 1816 return EAGAIN; 1817 } 1818 1819 /* 1820 ** Set up the appropriate offload context 1821 ** this becomes the first descriptor of 1822 ** a packet. 1823 */ 1824 if (m_head->m_pkthdr.csum_flags & (M_CSUM_TSOv4|M_CSUM_TSOv6)) { 1825 if (ixgbe_tso_setup(txr, m_head, &paylen)) { 1826 cmd_type_len |= IXGBE_ADVTXD_DCMD_TSE; 1827 olinfo_status |= IXGBE_TXD_POPTS_IXSM << 8; 1828 olinfo_status |= IXGBE_TXD_POPTS_TXSM << 8; 1829 olinfo_status |= paylen << IXGBE_ADVTXD_PAYLEN_SHIFT; 1830 ++adapter->tso_tx.ev_count; 1831 } else { 1832 ++adapter->tso_err.ev_count; 1833 /* XXX unload DMA map! --dyoung */ 1834 return ENXIO; 1835 } 1836 } else 1837 olinfo_status |= ixgbe_tx_ctx_setup(txr, m_head); 1838 1839 #ifdef IXGBE_IEEE1588 1840 /* This is changing soon to an mtag detection */ 1841 if (we detect this mbuf has a TSTAMP mtag) 1842 cmd_type_len |= IXGBE_ADVTXD_MAC_TSTAMP; 1843 #endif 1844 1845 #ifdef IXGBE_FDIR 1846 /* Do the flow director magic */ 1847 if ((txr->atr_sample) && (!adapter->fdir_reinit)) { 1848 ++txr->atr_count; 1849 if (txr->atr_count >= atr_sample_rate) { 1850 ixgbe_atr(txr, m_head); 1851 txr->atr_count = 0; 1852 } 1853 } 1854 #endif 1855 /* Record payload length */ 1856 if (paylen == 0) 1857 olinfo_status |= m_head->m_pkthdr.len << 1858 IXGBE_ADVTXD_PAYLEN_SHIFT; 1859 1860 i = txr->next_avail_desc; 1861 for (j = 0; j < map->dm_nsegs; j++) { 1862 bus_size_t seglen; 1863 bus_addr_t segaddr; 1864 1865 txbuf = &txr->tx_buffers[i]; 1866 txd = &txr->tx_base[i]; 1867 seglen = map->dm_segs[j].ds_len; 1868 segaddr = htole64(map->dm_segs[j].ds_addr); 1869 1870 txd->read.buffer_addr = segaddr; 1871 txd->read.cmd_type_len = htole32(txr->txd_cmd | 1872 cmd_type_len |seglen); 1873 txd->read.olinfo_status = htole32(olinfo_status); 1874 last = i; /* descriptor that will get completion IRQ */ 1875 1876 if (++i == adapter->num_tx_desc) 1877 i = 0; 1878 1879 txbuf->m_head = NULL; 1880 txbuf->eop_index = -1; 1881 } 1882 1883 txd->read.cmd_type_len |= 1884 htole32(IXGBE_TXD_CMD_EOP | IXGBE_TXD_CMD_RS); 1885 txr->tx_avail -= map->dm_nsegs; 1886 txr->next_avail_desc = i; 1887 1888 txbuf->m_head = m_head; 1889 /* We exchange the maps instead of copying because otherwise 1890 * we end up with many pointers to the same map and we free 1891 * one map twice in ixgbe_free_transmit_structures(). Who 1892 * knows what other problems this caused. --dyoung 1893 */ 1894 txr->tx_buffers[first].map = txbuf->map; 1895 txbuf->map = map; 1896 bus_dmamap_sync(txr->txtag->dt_dmat, map, 0, m_head->m_pkthdr.len, 1897 BUS_DMASYNC_PREWRITE); 1898 1899 /* Set the index of the descriptor that will be marked done */ 1900 txbuf = &txr->tx_buffers[first]; 1901 txbuf->eop_index = last; 1902 1903 ixgbe_dmamap_sync(txr->txdma.dma_tag, txr->txdma.dma_map, 1904 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); 1905 /* 1906 * Advance the Transmit Descriptor Tail (Tdt), this tells the 1907 * hardware that this frame is available to transmit. 1908 */ 1909 ++txr->total_packets.ev_count; 1910 IXGBE_WRITE_REG(&adapter->hw, IXGBE_TDT(txr->me), i); 1911 1912 return 0; 1913 } 1914 1915 static void 1916 ixgbe_set_promisc(struct adapter *adapter) 1917 { 1918 u_int32_t reg_rctl; 1919 struct ifnet *ifp = adapter->ifp; 1920 1921 reg_rctl = IXGBE_READ_REG(&adapter->hw, IXGBE_FCTRL); 1922 reg_rctl &= (~IXGBE_FCTRL_UPE); 1923 reg_rctl &= (~IXGBE_FCTRL_MPE); 1924 IXGBE_WRITE_REG(&adapter->hw, IXGBE_FCTRL, reg_rctl); 1925 1926 if (ifp->if_flags & IFF_PROMISC) { 1927 reg_rctl |= (IXGBE_FCTRL_UPE | IXGBE_FCTRL_MPE); 1928 IXGBE_WRITE_REG(&adapter->hw, IXGBE_FCTRL, reg_rctl); 1929 } else if (ifp->if_flags & IFF_ALLMULTI) { 1930 reg_rctl |= IXGBE_FCTRL_MPE; 1931 reg_rctl &= ~IXGBE_FCTRL_UPE; 1932 IXGBE_WRITE_REG(&adapter->hw, IXGBE_FCTRL, reg_rctl); 1933 } 1934 return; 1935 } 1936 1937 1938 /********************************************************************* 1939 * Multicast Update 1940 * 1941 * This routine is called whenever multicast address list is updated. 1942 * 1943 **********************************************************************/ 1944 #define IXGBE_RAR_ENTRIES 16 1945 1946 static void 1947 ixgbe_set_multi(struct adapter *adapter) 1948 { 1949 struct ether_multi *enm; 1950 struct ether_multistep step; 1951 u32 fctrl; 1952 u8 *mta; 1953 u8 *update_ptr; 1954 int mcnt = 0; 1955 struct ethercom *ec = &adapter->osdep.ec; 1956 struct ifnet *ifp = adapter->ifp; 1957 1958 IOCTL_DEBUGOUT("ixgbe_set_multi: begin"); 1959 1960 mta = adapter->mta; 1961 bzero(mta, sizeof(u8) * IXGBE_ETH_LENGTH_OF_ADDRESS * 1962 MAX_NUM_MULTICAST_ADDRESSES); 1963 1964 fctrl = IXGBE_READ_REG(&adapter->hw, IXGBE_FCTRL); 1965 fctrl |= (IXGBE_FCTRL_UPE | IXGBE_FCTRL_MPE); 1966 if (ifp->if_flags & IFF_PROMISC) 1967 fctrl |= (IXGBE_FCTRL_UPE | IXGBE_FCTRL_MPE); 1968 else if (ifp->if_flags & IFF_ALLMULTI) { 1969 fctrl |= IXGBE_FCTRL_MPE; 1970 fctrl &= ~IXGBE_FCTRL_UPE; 1971 } else 1972 fctrl &= ~(IXGBE_FCTRL_UPE | IXGBE_FCTRL_MPE); 1973 1974 IXGBE_WRITE_REG(&adapter->hw, IXGBE_FCTRL, fctrl); 1975 1976 ETHER_FIRST_MULTI(step, ec, enm); 1977 while (enm != NULL) { 1978 if (memcmp(enm->enm_addrlo, enm->enm_addrhi, 1979 ETHER_ADDR_LEN) != 0) { 1980 fctrl |= IXGBE_FCTRL_MPE; 1981 IXGBE_WRITE_REG(&adapter->hw, IXGBE_FCTRL, fctrl); 1982 break; 1983 } 1984 bcopy(enm->enm_addrlo, 1985 &mta[mcnt * IXGBE_ETH_LENGTH_OF_ADDRESS], 1986 IXGBE_ETH_LENGTH_OF_ADDRESS); 1987 mcnt++; 1988 ETHER_NEXT_MULTI(step, enm); 1989 } 1990 1991 update_ptr = mta; 1992 ixgbe_update_mc_addr_list(&adapter->hw, 1993 update_ptr, mcnt, ixgbe_mc_array_itr); 1994 1995 return; 1996 } 1997 1998 /* 1999 * This is an iterator function now needed by the multicast 2000 * shared code. It simply feeds the shared code routine the 2001 * addresses in the array of ixgbe_set_multi() one by one. 2002 */ 2003 static u8 * 2004 ixgbe_mc_array_itr(struct ixgbe_hw *hw, u8 **update_ptr, u32 *vmdq) 2005 { 2006 u8 *addr = *update_ptr; 2007 u8 *newptr; 2008 *vmdq = 0; 2009 2010 newptr = addr + IXGBE_ETH_LENGTH_OF_ADDRESS; 2011 *update_ptr = newptr; 2012 return addr; 2013 } 2014 2015 2016 /********************************************************************* 2017 * Timer routine 2018 * 2019 * This routine checks for link status,updates statistics, 2020 * and runs the watchdog check. 2021 * 2022 **********************************************************************/ 2023 2024 static void 2025 ixgbe_local_timer1(void *arg) 2026 { 2027 struct adapter *adapter = arg; 2028 device_t dev = adapter->dev; 2029 struct tx_ring *txr = adapter->tx_rings; 2030 2031 KASSERT(mutex_owned(&adapter->core_mtx)); 2032 2033 /* Check for pluggable optics */ 2034 if (adapter->sfp_probe) 2035 if (!ixgbe_sfp_probe(adapter)) 2036 goto out; /* Nothing to do */ 2037 2038 ixgbe_update_link_status(adapter); 2039 ixgbe_update_stats_counters(adapter); 2040 2041 /* 2042 * If the interface has been paused 2043 * then don't do the watchdog check 2044 */ 2045 if (IXGBE_READ_REG(&adapter->hw, IXGBE_TFCS) & IXGBE_TFCS_TXOFF) 2046 goto out; 2047 2048 /* 2049 ** Check status on the TX queues for a hang 2050 */ 2051 for (int i = 0; i < adapter->num_queues; i++, txr++) 2052 if (txr->queue_status == IXGBE_QUEUE_HUNG) 2053 goto hung; 2054 2055 out: 2056 ixgbe_rearm_queues(adapter, adapter->que_mask); 2057 callout_reset(&adapter->timer, hz, ixgbe_local_timer, adapter); 2058 return; 2059 2060 hung: 2061 device_printf(adapter->dev, "Watchdog timeout -- resetting\n"); 2062 device_printf(dev,"Queue(%d) tdh = %d, hw tdt = %d\n", txr->me, 2063 IXGBE_READ_REG(&adapter->hw, IXGBE_TDH(txr->me)), 2064 IXGBE_READ_REG(&adapter->hw, IXGBE_TDT(txr->me))); 2065 device_printf(dev,"TX(%d) desc avail = %d," 2066 "Next TX to Clean = %d\n", 2067 txr->me, txr->tx_avail, txr->next_to_clean); 2068 adapter->ifp->if_flags &= ~IFF_RUNNING; 2069 adapter->watchdog_events.ev_count++; 2070 ixgbe_init_locked(adapter); 2071 } 2072 2073 static void 2074 ixgbe_local_timer(void *arg) 2075 { 2076 struct adapter *adapter = arg; 2077 2078 IXGBE_CORE_LOCK(adapter); 2079 ixgbe_local_timer1(adapter); 2080 IXGBE_CORE_UNLOCK(adapter); 2081 } 2082 2083 /* 2084 ** Note: this routine updates the OS on the link state 2085 ** the real check of the hardware only happens with 2086 ** a link interrupt. 2087 */ 2088 static void 2089 ixgbe_update_link_status(struct adapter *adapter) 2090 { 2091 struct ifnet *ifp = adapter->ifp; 2092 struct tx_ring *txr = adapter->tx_rings; 2093 device_t dev = adapter->dev; 2094 2095 2096 if (adapter->link_up){ 2097 if (adapter->link_active == FALSE) { 2098 if (bootverbose) 2099 device_printf(dev,"Link is up %d Gbps %s \n", 2100 ((adapter->link_speed == 128)? 10:1), 2101 "Full Duplex"); 2102 adapter->link_active = TRUE; 2103 if_link_state_change(ifp, LINK_STATE_UP); 2104 } 2105 } else { /* Link down */ 2106 if (adapter->link_active == TRUE) { 2107 if (bootverbose) 2108 device_printf(dev,"Link is Down\n"); 2109 if_link_state_change(ifp, LINK_STATE_DOWN); 2110 adapter->link_active = FALSE; 2111 for (int i = 0; i < adapter->num_queues; 2112 i++, txr++) 2113 txr->queue_status = IXGBE_QUEUE_IDLE; 2114 } 2115 } 2116 2117 return; 2118 } 2119 2120 2121 static void 2122 ixgbe_ifstop(struct ifnet *ifp, int disable) 2123 { 2124 struct adapter *adapter = ifp->if_softc; 2125 2126 IXGBE_CORE_LOCK(adapter); 2127 ixgbe_stop(adapter); 2128 IXGBE_CORE_UNLOCK(adapter); 2129 } 2130 2131 /********************************************************************* 2132 * 2133 * This routine disables all traffic on the adapter by issuing a 2134 * global reset on the MAC and deallocates TX/RX buffers. 2135 * 2136 **********************************************************************/ 2137 2138 static void 2139 ixgbe_stop(void *arg) 2140 { 2141 struct ifnet *ifp; 2142 struct adapter *adapter = arg; 2143 struct ixgbe_hw *hw = &adapter->hw; 2144 ifp = adapter->ifp; 2145 2146 KASSERT(mutex_owned(&adapter->core_mtx)); 2147 2148 INIT_DEBUGOUT("ixgbe_stop: begin\n"); 2149 ixgbe_disable_intr(adapter); 2150 2151 /* Tell the stack that the interface is no longer active */ 2152 ifp->if_flags &= ~(IFF_RUNNING | IFF_OACTIVE); 2153 2154 ixgbe_reset_hw(hw); 2155 hw->adapter_stopped = FALSE; 2156 ixgbe_stop_adapter(hw); 2157 /* Turn off the laser */ 2158 if (hw->phy.multispeed_fiber) 2159 ixgbe_disable_tx_laser(hw); 2160 callout_stop(&adapter->timer); 2161 2162 /* reprogram the RAR[0] in case user changed it. */ 2163 ixgbe_set_rar(&adapter->hw, 0, adapter->hw.mac.addr, 0, IXGBE_RAH_AV); 2164 2165 return; 2166 } 2167 2168 2169 /********************************************************************* 2170 * 2171 * Determine hardware revision. 2172 * 2173 **********************************************************************/ 2174 static void 2175 ixgbe_identify_hardware(struct adapter *adapter) 2176 { 2177 pcitag_t tag; 2178 pci_chipset_tag_t pc; 2179 pcireg_t subid, id; 2180 struct ixgbe_hw *hw = &adapter->hw; 2181 2182 pc = adapter->osdep.pc; 2183 tag = adapter->osdep.tag; 2184 2185 id = pci_conf_read(pc, tag, PCI_ID_REG); 2186 subid = pci_conf_read(pc, tag, PCI_SUBSYS_ID_REG); 2187 2188 /* Save off the information about this board */ 2189 hw->vendor_id = PCI_VENDOR(id); 2190 hw->device_id = PCI_PRODUCT(id); 2191 hw->revision_id = 2192 PCI_REVISION(pci_conf_read(pc, tag, PCI_CLASS_REG)); 2193 hw->subsystem_vendor_id = PCI_SUBSYS_VENDOR(subid); 2194 hw->subsystem_device_id = PCI_SUBSYS_ID(subid); 2195 2196 /* We need this here to set the num_segs below */ 2197 ixgbe_set_mac_type(hw); 2198 2199 /* Pick up the 82599 and VF settings */ 2200 if (hw->mac.type != ixgbe_mac_82598EB) { 2201 hw->phy.smart_speed = ixgbe_smart_speed; 2202 adapter->num_segs = IXGBE_82599_SCATTER; 2203 } else 2204 adapter->num_segs = IXGBE_82598_SCATTER; 2205 2206 return; 2207 } 2208 2209 /********************************************************************* 2210 * 2211 * Determine optic type 2212 * 2213 **********************************************************************/ 2214 static void 2215 ixgbe_setup_optics(struct adapter *adapter) 2216 { 2217 struct ixgbe_hw *hw = &adapter->hw; 2218 int layer; 2219 2220 layer = ixgbe_get_supported_physical_layer(hw); 2221 switch (layer) { 2222 case IXGBE_PHYSICAL_LAYER_10GBASE_T: 2223 adapter->optics = IFM_10G_T; 2224 break; 2225 case IXGBE_PHYSICAL_LAYER_1000BASE_T: 2226 adapter->optics = IFM_1000_T; 2227 break; 2228 case IXGBE_PHYSICAL_LAYER_10GBASE_LR: 2229 case IXGBE_PHYSICAL_LAYER_10GBASE_LRM: 2230 adapter->optics = IFM_10G_LR; 2231 break; 2232 case IXGBE_PHYSICAL_LAYER_10GBASE_SR: 2233 adapter->optics = IFM_10G_SR; 2234 break; 2235 case IXGBE_PHYSICAL_LAYER_10GBASE_KX4: 2236 case IXGBE_PHYSICAL_LAYER_10GBASE_CX4: 2237 adapter->optics = IFM_10G_CX4; 2238 break; 2239 case IXGBE_PHYSICAL_LAYER_SFP_PLUS_CU: 2240 adapter->optics = IFM_10G_TWINAX; 2241 break; 2242 case IXGBE_PHYSICAL_LAYER_1000BASE_KX: 2243 case IXGBE_PHYSICAL_LAYER_10GBASE_KR: 2244 case IXGBE_PHYSICAL_LAYER_10GBASE_XAUI: 2245 case IXGBE_PHYSICAL_LAYER_UNKNOWN: 2246 default: 2247 adapter->optics = IFM_ETHER | IFM_AUTO; 2248 break; 2249 } 2250 return; 2251 } 2252 2253 /********************************************************************* 2254 * 2255 * Setup the Legacy or MSI Interrupt handler 2256 * 2257 **********************************************************************/ 2258 static int 2259 ixgbe_allocate_legacy(struct adapter *adapter, const struct pci_attach_args *pa) 2260 { 2261 device_t dev = adapter->dev; 2262 struct ix_queue *que = adapter->queues; 2263 char intrbuf[PCI_INTRSTR_LEN]; 2264 #if 0 2265 int rid = 0; 2266 2267 /* MSI RID at 1 */ 2268 if (adapter->msix == 1) 2269 rid = 1; 2270 #endif 2271 2272 /* We allocate a single interrupt resource */ 2273 if (pci_intr_map(pa, &adapter->osdep.ih) != 0) { 2274 aprint_error_dev(dev, "unable to map interrupt\n"); 2275 return ENXIO; 2276 } else { 2277 aprint_normal_dev(dev, "interrupting at %s\n", 2278 pci_intr_string(adapter->osdep.pc, adapter->osdep.ih, 2279 intrbuf, sizeof(intrbuf))); 2280 } 2281 2282 /* 2283 * Try allocating a fast interrupt and the associated deferred 2284 * processing contexts. 2285 */ 2286 que->que_si = softint_establish(SOFTINT_NET, ixgbe_handle_que, que); 2287 2288 /* Tasklets for Link, SFP and Multispeed Fiber */ 2289 adapter->link_si = 2290 softint_establish(SOFTINT_NET, ixgbe_handle_link, adapter); 2291 adapter->mod_si = 2292 softint_establish(SOFTINT_NET, ixgbe_handle_mod, adapter); 2293 adapter->msf_si = 2294 softint_establish(SOFTINT_NET, ixgbe_handle_msf, adapter); 2295 2296 #ifdef IXGBE_FDIR 2297 adapter->fdir_si = 2298 softint_establish(SOFTINT_NET, ixgbe_reinit_fdir, adapter); 2299 #endif 2300 if (que->que_si == NULL || 2301 adapter->link_si == NULL || 2302 adapter->mod_si == NULL || 2303 #ifdef IXGBE_FDIR 2304 adapter->fdir_si == NULL || 2305 #endif 2306 adapter->msf_si == NULL) { 2307 aprint_error_dev(dev, 2308 "could not establish software interrupts\n"); 2309 return ENXIO; 2310 } 2311 2312 adapter->osdep.intr = pci_intr_establish(adapter->osdep.pc, 2313 adapter->osdep.ih, IPL_NET, ixgbe_legacy_irq, que); 2314 if (adapter->osdep.intr == NULL) { 2315 aprint_error_dev(dev, "failed to register interrupt handler\n"); 2316 softint_disestablish(que->que_si); 2317 softint_disestablish(adapter->link_si); 2318 softint_disestablish(adapter->mod_si); 2319 softint_disestablish(adapter->msf_si); 2320 #ifdef IXGBE_FDIR 2321 softint_disestablish(adapter->fdir_si); 2322 #endif 2323 return ENXIO; 2324 } 2325 /* For simplicity in the handlers */ 2326 adapter->que_mask = IXGBE_EIMS_ENABLE_MASK; 2327 2328 return (0); 2329 } 2330 2331 2332 /********************************************************************* 2333 * 2334 * Setup MSIX Interrupt resources and handlers 2335 * 2336 **********************************************************************/ 2337 static int 2338 ixgbe_allocate_msix(struct adapter *adapter, const struct pci_attach_args *pa) 2339 { 2340 #if !defined(NETBSD_MSI_OR_MSIX) 2341 return 0; 2342 #else 2343 device_t dev = adapter->dev; 2344 struct ix_queue *que = adapter->queues; 2345 int error, rid, vector = 0; 2346 2347 for (int i = 0; i < adapter->num_queues; i++, vector++, que++) { 2348 rid = vector + 1; 2349 que->res = bus_alloc_resource_any(dev, SYS_RES_IRQ, &rid, 2350 RF_SHAREABLE | RF_ACTIVE); 2351 if (que->res == NULL) { 2352 aprint_error_dev(dev,"Unable to allocate" 2353 " bus resource: que interrupt [%d]\n", vector); 2354 return (ENXIO); 2355 } 2356 /* Set the handler function */ 2357 error = bus_setup_intr(dev, que->res, 2358 INTR_TYPE_NET | INTR_MPSAFE, NULL, 2359 ixgbe_msix_que, que, &que->tag); 2360 if (error) { 2361 que->res = NULL; 2362 aprint_error_dev(dev, 2363 "Failed to register QUE handler\n"); 2364 return error; 2365 } 2366 #if __FreeBSD_version >= 800504 2367 bus_describe_intr(dev, que->res, que->tag, "que %d", i); 2368 #endif 2369 que->msix = vector; 2370 adapter->que_mask |= (u64)(1 << que->msix); 2371 /* 2372 ** Bind the msix vector, and thus the 2373 ** ring to the corresponding cpu. 2374 */ 2375 if (adapter->num_queues > 1) 2376 bus_bind_intr(dev, que->res, i); 2377 2378 que->que_si = softint_establish(ixgbe_handle_que, que); 2379 if (que->que_si == NULL) { 2380 aprint_error_dev(dev, 2381 "could not establish software interrupt\n"); 2382 } 2383 } 2384 2385 /* and Link */ 2386 rid = vector + 1; 2387 adapter->res = bus_alloc_resource_any(dev, 2388 SYS_RES_IRQ, &rid, RF_SHAREABLE | RF_ACTIVE); 2389 if (!adapter->res) { 2390 aprint_error_dev(dev,"Unable to allocate bus resource: " 2391 "Link interrupt [%d]\n", rid); 2392 return (ENXIO); 2393 } 2394 /* Set the link handler function */ 2395 error = bus_setup_intr(dev, adapter->res, 2396 INTR_TYPE_NET | INTR_MPSAFE, NULL, 2397 ixgbe_msix_link, adapter, &adapter->tag); 2398 if (error) { 2399 adapter->res = NULL; 2400 aprint_error_dev(dev, "Failed to register LINK handler\n"); 2401 return (error); 2402 } 2403 #if __FreeBSD_version >= 800504 2404 bus_describe_intr(dev, adapter->res, adapter->tag, "link"); 2405 #endif 2406 adapter->linkvec = vector; 2407 /* Tasklets for Link, SFP and Multispeed Fiber */ 2408 adapter->link_si = 2409 softint_establish(SOFTINT_NET, ixgbe_handle_link, adapter); 2410 adapter->mod_si = 2411 softint_establish(SOFTINT_NET, ixgbe_handle_mod, adapter); 2412 adapter->msf_si = 2413 softint_establish(SOFTINT_NET, ixgbe_handle_msf, adapter); 2414 #ifdef IXGBE_FDIR 2415 adapter->fdir_si = 2416 softint_establish(SOFTINT_NET, ixgbe_reinit_fdir, adapter); 2417 #endif 2418 2419 return (0); 2420 #endif 2421 } 2422 2423 /* 2424 * Setup Either MSI/X or MSI 2425 */ 2426 static int 2427 ixgbe_setup_msix(struct adapter *adapter) 2428 { 2429 #if !defined(NETBSD_MSI_OR_MSIX) 2430 return 0; 2431 #else 2432 device_t dev = adapter->dev; 2433 int rid, want, queues, msgs; 2434 2435 /* Override by tuneable */ 2436 if (ixgbe_enable_msix == 0) 2437 goto msi; 2438 2439 /* First try MSI/X */ 2440 rid = PCI_BAR(MSIX_82598_BAR); 2441 adapter->msix_mem = bus_alloc_resource_any(dev, 2442 SYS_RES_MEMORY, &rid, RF_ACTIVE); 2443 if (!adapter->msix_mem) { 2444 rid += 4; /* 82599 maps in higher BAR */ 2445 adapter->msix_mem = bus_alloc_resource_any(dev, 2446 SYS_RES_MEMORY, &rid, RF_ACTIVE); 2447 } 2448 if (!adapter->msix_mem) { 2449 /* May not be enabled */ 2450 device_printf(adapter->dev, 2451 "Unable to map MSIX table \n"); 2452 goto msi; 2453 } 2454 2455 msgs = pci_msix_count(dev); 2456 if (msgs == 0) { /* system has msix disabled */ 2457 bus_release_resource(dev, SYS_RES_MEMORY, 2458 rid, adapter->msix_mem); 2459 adapter->msix_mem = NULL; 2460 goto msi; 2461 } 2462 2463 /* Figure out a reasonable auto config value */ 2464 queues = (mp_ncpus > (msgs-1)) ? (msgs-1) : mp_ncpus; 2465 2466 if (ixgbe_num_queues != 0) 2467 queues = ixgbe_num_queues; 2468 /* Set max queues to 8 when autoconfiguring */ 2469 else if ((ixgbe_num_queues == 0) && (queues > 8)) 2470 queues = 8; 2471 2472 /* 2473 ** Want one vector (RX/TX pair) per queue 2474 ** plus an additional for Link. 2475 */ 2476 want = queues + 1; 2477 if (msgs >= want) 2478 msgs = want; 2479 else { 2480 device_printf(adapter->dev, 2481 "MSIX Configuration Problem, " 2482 "%d vectors but %d queues wanted!\n", 2483 msgs, want); 2484 return (0); /* Will go to Legacy setup */ 2485 } 2486 if ((msgs) && pci_alloc_msix(dev, &msgs) == 0) { 2487 device_printf(adapter->dev, 2488 "Using MSIX interrupts with %d vectors\n", msgs); 2489 adapter->num_queues = queues; 2490 return (msgs); 2491 } 2492 msi: 2493 msgs = pci_msi_count(dev); 2494 if (msgs == 1 && pci_alloc_msi(dev, &msgs) == 0) 2495 device_printf(adapter->dev,"Using MSI interrupt\n"); 2496 return (msgs); 2497 #endif 2498 } 2499 2500 2501 static int 2502 ixgbe_allocate_pci_resources(struct adapter *adapter, const struct pci_attach_args *pa) 2503 { 2504 pcireg_t memtype; 2505 device_t dev = adapter->dev; 2506 bus_addr_t addr; 2507 int flags; 2508 2509 memtype = pci_mapreg_type(pa->pa_pc, pa->pa_tag, PCI_BAR(0)); 2510 switch (memtype) { 2511 case PCI_MAPREG_TYPE_MEM | PCI_MAPREG_MEM_TYPE_32BIT: 2512 case PCI_MAPREG_TYPE_MEM | PCI_MAPREG_MEM_TYPE_64BIT: 2513 adapter->osdep.mem_bus_space_tag = pa->pa_memt; 2514 if (pci_mapreg_info(pa->pa_pc, pa->pa_tag, PCI_BAR(0), 2515 memtype, &addr, &adapter->osdep.mem_size, &flags) != 0) 2516 goto map_err; 2517 if ((flags & BUS_SPACE_MAP_PREFETCHABLE) != 0) { 2518 aprint_normal_dev(dev, "clearing prefetchable bit\n"); 2519 flags &= ~BUS_SPACE_MAP_PREFETCHABLE; 2520 } 2521 if (bus_space_map(adapter->osdep.mem_bus_space_tag, addr, 2522 adapter->osdep.mem_size, flags, 2523 &adapter->osdep.mem_bus_space_handle) != 0) { 2524 map_err: 2525 adapter->osdep.mem_size = 0; 2526 aprint_error_dev(dev, "unable to map BAR0\n"); 2527 return ENXIO; 2528 } 2529 break; 2530 default: 2531 aprint_error_dev(dev, "unexpected type on BAR0\n"); 2532 return ENXIO; 2533 } 2534 2535 /* Legacy defaults */ 2536 adapter->num_queues = 1; 2537 adapter->hw.back = &adapter->osdep; 2538 2539 /* 2540 ** Now setup MSI or MSI/X, should 2541 ** return us the number of supported 2542 ** vectors. (Will be 1 for MSI) 2543 */ 2544 adapter->msix = ixgbe_setup_msix(adapter); 2545 return (0); 2546 } 2547 2548 static void 2549 ixgbe_free_pci_resources(struct adapter * adapter) 2550 { 2551 #if defined(NETBSD_MSI_OR_MSIX) 2552 struct ix_queue *que = adapter->queues; 2553 device_t dev = adapter->dev; 2554 #endif 2555 int rid; 2556 2557 #if defined(NETBSD_MSI_OR_MSIX) 2558 int memrid; 2559 if (adapter->hw.mac.type == ixgbe_mac_82598EB) 2560 memrid = PCI_BAR(MSIX_82598_BAR); 2561 else 2562 memrid = PCI_BAR(MSIX_82599_BAR); 2563 2564 /* 2565 ** There is a slight possibility of a failure mode 2566 ** in attach that will result in entering this function 2567 ** before interrupt resources have been initialized, and 2568 ** in that case we do not want to execute the loops below 2569 ** We can detect this reliably by the state of the adapter 2570 ** res pointer. 2571 */ 2572 if (adapter->res == NULL) 2573 goto mem; 2574 2575 /* 2576 ** Release all msix queue resources: 2577 */ 2578 for (int i = 0; i < adapter->num_queues; i++, que++) { 2579 rid = que->msix + 1; 2580 if (que->tag != NULL) { 2581 bus_teardown_intr(dev, que->res, que->tag); 2582 que->tag = NULL; 2583 } 2584 if (que->res != NULL) 2585 bus_release_resource(dev, SYS_RES_IRQ, rid, que->res); 2586 } 2587 #endif 2588 2589 /* Clean the Legacy or Link interrupt last */ 2590 if (adapter->linkvec) /* we are doing MSIX */ 2591 rid = adapter->linkvec + 1; 2592 else 2593 (adapter->msix != 0) ? (rid = 1):(rid = 0); 2594 2595 pci_intr_disestablish(adapter->osdep.pc, adapter->osdep.intr); 2596 adapter->osdep.intr = NULL; 2597 2598 #if defined(NETBSD_MSI_OR_MSIX) 2599 mem: 2600 if (adapter->msix) 2601 pci_release_msi(dev); 2602 2603 if (adapter->msix_mem != NULL) 2604 bus_release_resource(dev, SYS_RES_MEMORY, 2605 memrid, adapter->msix_mem); 2606 #endif 2607 2608 if (adapter->osdep.mem_size != 0) { 2609 bus_space_unmap(adapter->osdep.mem_bus_space_tag, 2610 adapter->osdep.mem_bus_space_handle, 2611 adapter->osdep.mem_size); 2612 } 2613 2614 return; 2615 } 2616 2617 /********************************************************************* 2618 * 2619 * Setup networking device structure and register an interface. 2620 * 2621 **********************************************************************/ 2622 static int 2623 ixgbe_setup_interface(device_t dev, struct adapter *adapter) 2624 { 2625 struct ethercom *ec = &adapter->osdep.ec; 2626 struct ixgbe_hw *hw = &adapter->hw; 2627 struct ifnet *ifp; 2628 2629 INIT_DEBUGOUT("ixgbe_setup_interface: begin"); 2630 2631 ifp = adapter->ifp = &ec->ec_if; 2632 strlcpy(ifp->if_xname, device_xname(dev), IFNAMSIZ); 2633 ifp->if_mtu = ETHERMTU; 2634 ifp->if_baudrate = 1000000000; 2635 ifp->if_init = ixgbe_init; 2636 ifp->if_stop = ixgbe_ifstop; 2637 ifp->if_softc = adapter; 2638 ifp->if_flags = IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST; 2639 ifp->if_ioctl = ixgbe_ioctl; 2640 ifp->if_start = ixgbe_start; 2641 #if __FreeBSD_version >= 800000 2642 ifp->if_transmit = ixgbe_mq_start; 2643 ifp->if_qflush = ixgbe_qflush; 2644 #endif 2645 ifp->if_snd.ifq_maxlen = adapter->num_tx_desc - 2; 2646 2647 if_attach(ifp); 2648 ether_ifattach(ifp, adapter->hw.mac.addr); 2649 ether_set_ifflags_cb(ec, ixgbe_ifflags_cb); 2650 2651 adapter->max_frame_size = 2652 ifp->if_mtu + ETHER_HDR_LEN + ETHER_CRC_LEN; 2653 2654 /* 2655 * Tell the upper layer(s) we support long frames. 2656 */ 2657 ifp->if_hdrlen = sizeof(struct ether_vlan_header); 2658 2659 ifp->if_capabilities |= IFCAP_HWCSUM | IFCAP_TSOv4; 2660 ifp->if_capenable = 0; 2661 2662 ec->ec_capabilities |= ETHERCAP_VLAN_HWCSUM; 2663 ec->ec_capabilities |= ETHERCAP_VLAN_HWTAGGING | ETHERCAP_VLAN_MTU; 2664 ec->ec_capabilities |= ETHERCAP_JUMBO_MTU; 2665 ec->ec_capenable = ec->ec_capabilities; 2666 2667 /* Don't enable LRO by default */ 2668 ifp->if_capabilities |= IFCAP_LRO; 2669 2670 /* 2671 ** Dont turn this on by default, if vlans are 2672 ** created on another pseudo device (eg. lagg) 2673 ** then vlan events are not passed thru, breaking 2674 ** operation, but with HW FILTER off it works. If 2675 ** using vlans directly on the em driver you can 2676 ** enable this and get full hardware tag filtering. 2677 */ 2678 ec->ec_capabilities |= ETHERCAP_VLAN_HWFILTER; 2679 2680 /* 2681 * Specify the media types supported by this adapter and register 2682 * callbacks to update media and link information 2683 */ 2684 ifmedia_init(&adapter->media, IFM_IMASK, ixgbe_media_change, 2685 ixgbe_media_status); 2686 ifmedia_add(&adapter->media, IFM_ETHER | adapter->optics, 0, NULL); 2687 ifmedia_set(&adapter->media, IFM_ETHER | adapter->optics); 2688 if (hw->device_id == IXGBE_DEV_ID_82598AT) { 2689 ifmedia_add(&adapter->media, 2690 IFM_ETHER | IFM_1000_T | IFM_FDX, 0, NULL); 2691 ifmedia_add(&adapter->media, 2692 IFM_ETHER | IFM_1000_T, 0, NULL); 2693 } 2694 ifmedia_add(&adapter->media, IFM_ETHER | IFM_AUTO, 0, NULL); 2695 ifmedia_set(&adapter->media, IFM_ETHER | IFM_AUTO); 2696 2697 return (0); 2698 } 2699 2700 static void 2701 ixgbe_config_link(struct adapter *adapter) 2702 { 2703 struct ixgbe_hw *hw = &adapter->hw; 2704 u32 autoneg, err = 0; 2705 bool sfp, negotiate; 2706 2707 sfp = ixgbe_is_sfp(hw); 2708 2709 if (sfp) { 2710 if (hw->phy.multispeed_fiber) { 2711 hw->mac.ops.setup_sfp(hw); 2712 ixgbe_enable_tx_laser(hw); 2713 softint_schedule(adapter->msf_si); 2714 } else { 2715 softint_schedule(adapter->mod_si); 2716 } 2717 } else { 2718 if (hw->mac.ops.check_link) 2719 err = ixgbe_check_link(hw, &autoneg, 2720 &adapter->link_up, FALSE); 2721 if (err) 2722 goto out; 2723 autoneg = hw->phy.autoneg_advertised; 2724 if ((!autoneg) && (hw->mac.ops.get_link_capabilities)) 2725 err = hw->mac.ops.get_link_capabilities(hw, 2726 &autoneg, &negotiate); 2727 else 2728 negotiate = 0; 2729 if (err) 2730 goto out; 2731 if (hw->mac.ops.setup_link) 2732 err = hw->mac.ops.setup_link(hw, autoneg, 2733 negotiate, adapter->link_up); 2734 } 2735 out: 2736 return; 2737 } 2738 2739 /******************************************************************** 2740 * Manage DMA'able memory. 2741 *******************************************************************/ 2742 2743 static int 2744 ixgbe_dma_malloc(struct adapter *adapter, const bus_size_t size, 2745 struct ixgbe_dma_alloc *dma, const int mapflags) 2746 { 2747 device_t dev = adapter->dev; 2748 int r, rsegs; 2749 2750 r = ixgbe_dma_tag_create(adapter->osdep.dmat, /* parent */ 2751 DBA_ALIGN, 0, /* alignment, bounds */ 2752 size, /* maxsize */ 2753 1, /* nsegments */ 2754 size, /* maxsegsize */ 2755 BUS_DMA_ALLOCNOW, /* flags */ 2756 &dma->dma_tag); 2757 if (r != 0) { 2758 aprint_error_dev(dev, 2759 "%s: ixgbe_dma_tag_create failed; error %d\n", __func__, r); 2760 goto fail_0; 2761 } 2762 2763 r = bus_dmamem_alloc(dma->dma_tag->dt_dmat, 2764 size, 2765 dma->dma_tag->dt_alignment, 2766 dma->dma_tag->dt_boundary, 2767 &dma->dma_seg, 1, &rsegs, BUS_DMA_NOWAIT); 2768 if (r != 0) { 2769 aprint_error_dev(dev, 2770 "%s: bus_dmamem_alloc failed; error %d\n", __func__, r); 2771 goto fail_1; 2772 } 2773 2774 r = bus_dmamem_map(dma->dma_tag->dt_dmat, &dma->dma_seg, rsegs, 2775 size, &dma->dma_vaddr, BUS_DMA_NOWAIT); 2776 if (r != 0) { 2777 aprint_error_dev(dev, "%s: bus_dmamem_map failed; error %d\n", 2778 __func__, r); 2779 goto fail_2; 2780 } 2781 2782 r = ixgbe_dmamap_create(dma->dma_tag, 0, &dma->dma_map); 2783 if (r != 0) { 2784 aprint_error_dev(dev, "%s: bus_dmamem_map failed; error %d\n", 2785 __func__, r); 2786 goto fail_3; 2787 } 2788 2789 r = bus_dmamap_load(dma->dma_tag->dt_dmat, dma->dma_map, dma->dma_vaddr, 2790 size, 2791 NULL, 2792 mapflags | BUS_DMA_NOWAIT); 2793 if (r != 0) { 2794 aprint_error_dev(dev, "%s: bus_dmamap_load failed; error %d\n", 2795 __func__, r); 2796 goto fail_4; 2797 } 2798 dma->dma_paddr = dma->dma_map->dm_segs[0].ds_addr; 2799 dma->dma_size = size; 2800 return 0; 2801 fail_4: 2802 ixgbe_dmamap_destroy(dma->dma_tag, dma->dma_map); 2803 fail_3: 2804 bus_dmamem_unmap(dma->dma_tag->dt_dmat, dma->dma_vaddr, size); 2805 fail_2: 2806 bus_dmamem_free(dma->dma_tag->dt_dmat, &dma->dma_seg, rsegs); 2807 fail_1: 2808 ixgbe_dma_tag_destroy(dma->dma_tag); 2809 fail_0: 2810 return r; 2811 } 2812 2813 static void 2814 ixgbe_dma_free(struct adapter *adapter, struct ixgbe_dma_alloc *dma) 2815 { 2816 bus_dmamap_sync(dma->dma_tag->dt_dmat, dma->dma_map, 0, dma->dma_size, 2817 BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE); 2818 ixgbe_dmamap_unload(dma->dma_tag, dma->dma_map); 2819 bus_dmamem_free(dma->dma_tag->dt_dmat, &dma->dma_seg, 1); 2820 ixgbe_dma_tag_destroy(dma->dma_tag); 2821 } 2822 2823 2824 /********************************************************************* 2825 * 2826 * Allocate memory for the transmit and receive rings, and then 2827 * the descriptors associated with each, called only once at attach. 2828 * 2829 **********************************************************************/ 2830 static int 2831 ixgbe_allocate_queues(struct adapter *adapter) 2832 { 2833 device_t dev = adapter->dev; 2834 struct ix_queue *que; 2835 struct tx_ring *txr; 2836 struct rx_ring *rxr; 2837 int rsize, tsize, error = IXGBE_SUCCESS; 2838 int txconf = 0, rxconf = 0; 2839 2840 /* First allocate the top level queue structs */ 2841 if (!(adapter->queues = 2842 (struct ix_queue *) malloc(sizeof(struct ix_queue) * 2843 adapter->num_queues, M_DEVBUF, M_NOWAIT | M_ZERO))) { 2844 aprint_error_dev(dev, "Unable to allocate queue memory\n"); 2845 error = ENOMEM; 2846 goto fail; 2847 } 2848 2849 /* First allocate the TX ring struct memory */ 2850 if (!(adapter->tx_rings = 2851 (struct tx_ring *) malloc(sizeof(struct tx_ring) * 2852 adapter->num_queues, M_DEVBUF, M_NOWAIT | M_ZERO))) { 2853 aprint_error_dev(dev, "Unable to allocate TX ring memory\n"); 2854 error = ENOMEM; 2855 goto tx_fail; 2856 } 2857 2858 /* Next allocate the RX */ 2859 if (!(adapter->rx_rings = 2860 (struct rx_ring *) malloc(sizeof(struct rx_ring) * 2861 adapter->num_queues, M_DEVBUF, M_NOWAIT | M_ZERO))) { 2862 aprint_error_dev(dev, "Unable to allocate RX ring memory\n"); 2863 error = ENOMEM; 2864 goto rx_fail; 2865 } 2866 2867 /* For the ring itself */ 2868 tsize = roundup2(adapter->num_tx_desc * 2869 sizeof(union ixgbe_adv_tx_desc), DBA_ALIGN); 2870 2871 /* 2872 * Now set up the TX queues, txconf is needed to handle the 2873 * possibility that things fail midcourse and we need to 2874 * undo memory gracefully 2875 */ 2876 for (int i = 0; i < adapter->num_queues; i++, txconf++) { 2877 /* Set up some basics */ 2878 txr = &adapter->tx_rings[i]; 2879 txr->adapter = adapter; 2880 txr->me = i; 2881 2882 /* Initialize the TX side lock */ 2883 snprintf(txr->mtx_name, sizeof(txr->mtx_name), "%s:tx(%d)", 2884 device_xname(dev), txr->me); 2885 mutex_init(&txr->tx_mtx, MUTEX_DEFAULT, IPL_NET); 2886 2887 if (ixgbe_dma_malloc(adapter, tsize, 2888 &txr->txdma, BUS_DMA_NOWAIT)) { 2889 aprint_error_dev(dev, 2890 "Unable to allocate TX Descriptor memory\n"); 2891 error = ENOMEM; 2892 goto err_tx_desc; 2893 } 2894 txr->tx_base = (union ixgbe_adv_tx_desc *)txr->txdma.dma_vaddr; 2895 bzero((void *)txr->tx_base, tsize); 2896 2897 /* Now allocate transmit buffers for the ring */ 2898 if (ixgbe_allocate_transmit_buffers(txr)) { 2899 aprint_error_dev(dev, 2900 "Critical Failure setting up transmit buffers\n"); 2901 error = ENOMEM; 2902 goto err_tx_desc; 2903 } 2904 #if __FreeBSD_version >= 800000 2905 /* Allocate a buf ring */ 2906 txr->br = buf_ring_alloc(IXGBE_BR_SIZE, M_DEVBUF, 2907 M_WAITOK, &txr->tx_mtx); 2908 if (txr->br == NULL) { 2909 aprint_error_dev(dev, 2910 "Critical Failure setting up buf ring\n"); 2911 error = ENOMEM; 2912 goto err_tx_desc; 2913 } 2914 #endif 2915 } 2916 2917 /* 2918 * Next the RX queues... 2919 */ 2920 rsize = roundup2(adapter->num_rx_desc * 2921 sizeof(union ixgbe_adv_rx_desc), DBA_ALIGN); 2922 for (int i = 0; i < adapter->num_queues; i++, rxconf++) { 2923 rxr = &adapter->rx_rings[i]; 2924 /* Set up some basics */ 2925 rxr->adapter = adapter; 2926 rxr->me = i; 2927 2928 /* Initialize the RX side lock */ 2929 snprintf(rxr->mtx_name, sizeof(rxr->mtx_name), "%s:rx(%d)", 2930 device_xname(dev), rxr->me); 2931 mutex_init(&rxr->rx_mtx, MUTEX_DEFAULT, IPL_NET); 2932 2933 if (ixgbe_dma_malloc(adapter, rsize, 2934 &rxr->rxdma, BUS_DMA_NOWAIT)) { 2935 aprint_error_dev(dev, 2936 "Unable to allocate RxDescriptor memory\n"); 2937 error = ENOMEM; 2938 goto err_rx_desc; 2939 } 2940 rxr->rx_base = (union ixgbe_adv_rx_desc *)rxr->rxdma.dma_vaddr; 2941 bzero((void *)rxr->rx_base, rsize); 2942 2943 /* Allocate receive buffers for the ring*/ 2944 if (ixgbe_allocate_receive_buffers(rxr)) { 2945 aprint_error_dev(dev, 2946 "Critical Failure setting up receive buffers\n"); 2947 error = ENOMEM; 2948 goto err_rx_desc; 2949 } 2950 } 2951 2952 /* 2953 ** Finally set up the queue holding structs 2954 */ 2955 for (int i = 0; i < adapter->num_queues; i++) { 2956 que = &adapter->queues[i]; 2957 que->adapter = adapter; 2958 que->txr = &adapter->tx_rings[i]; 2959 que->rxr = &adapter->rx_rings[i]; 2960 } 2961 2962 return (0); 2963 2964 err_rx_desc: 2965 for (rxr = adapter->rx_rings; rxconf > 0; rxr++, rxconf--) 2966 ixgbe_dma_free(adapter, &rxr->rxdma); 2967 err_tx_desc: 2968 for (txr = adapter->tx_rings; txconf > 0; txr++, txconf--) 2969 ixgbe_dma_free(adapter, &txr->txdma); 2970 free(adapter->rx_rings, M_DEVBUF); 2971 rx_fail: 2972 free(adapter->tx_rings, M_DEVBUF); 2973 tx_fail: 2974 free(adapter->queues, M_DEVBUF); 2975 fail: 2976 return (error); 2977 } 2978 2979 /********************************************************************* 2980 * 2981 * Allocate memory for tx_buffer structures. The tx_buffer stores all 2982 * the information needed to transmit a packet on the wire. This is 2983 * called only once at attach, setup is done every reset. 2984 * 2985 **********************************************************************/ 2986 static int 2987 ixgbe_allocate_transmit_buffers(struct tx_ring *txr) 2988 { 2989 struct adapter *adapter = txr->adapter; 2990 device_t dev = adapter->dev; 2991 struct ixgbe_tx_buf *txbuf; 2992 int error, i; 2993 2994 /* 2995 * Setup DMA descriptor areas. 2996 */ 2997 if ((error = ixgbe_dma_tag_create(adapter->osdep.dmat, /* parent */ 2998 1, 0, /* alignment, bounds */ 2999 IXGBE_TSO_SIZE, /* maxsize */ 3000 adapter->num_segs, /* nsegments */ 3001 PAGE_SIZE, /* maxsegsize */ 3002 0, /* flags */ 3003 &txr->txtag))) { 3004 aprint_error_dev(dev,"Unable to allocate TX DMA tag\n"); 3005 goto fail; 3006 } 3007 3008 if (!(txr->tx_buffers = 3009 (struct ixgbe_tx_buf *) malloc(sizeof(struct ixgbe_tx_buf) * 3010 adapter->num_tx_desc, M_DEVBUF, M_NOWAIT | M_ZERO))) { 3011 aprint_error_dev(dev, "Unable to allocate tx_buffer memory\n"); 3012 error = ENOMEM; 3013 goto fail; 3014 } 3015 3016 /* Create the descriptor buffer dma maps */ 3017 txbuf = txr->tx_buffers; 3018 for (i = 0; i < adapter->num_tx_desc; i++, txbuf++) { 3019 error = ixgbe_dmamap_create(txr->txtag, 0, &txbuf->map); 3020 if (error != 0) { 3021 aprint_error_dev(dev, "Unable to create TX DMA map\n"); 3022 goto fail; 3023 } 3024 } 3025 3026 return 0; 3027 fail: 3028 /* We free all, it handles case where we are in the middle */ 3029 ixgbe_free_transmit_structures(adapter); 3030 return (error); 3031 } 3032 3033 /********************************************************************* 3034 * 3035 * Initialize a transmit ring. 3036 * 3037 **********************************************************************/ 3038 static void 3039 ixgbe_setup_transmit_ring(struct tx_ring *txr) 3040 { 3041 struct adapter *adapter = txr->adapter; 3042 struct ixgbe_tx_buf *txbuf; 3043 int i; 3044 3045 /* Clear the old ring contents */ 3046 IXGBE_TX_LOCK(txr); 3047 bzero((void *)txr->tx_base, 3048 (sizeof(union ixgbe_adv_tx_desc)) * adapter->num_tx_desc); 3049 /* Reset indices */ 3050 txr->next_avail_desc = 0; 3051 txr->next_to_clean = 0; 3052 3053 /* Free any existing tx buffers. */ 3054 txbuf = txr->tx_buffers; 3055 for (i = 0; i < adapter->num_tx_desc; i++, txbuf++) { 3056 if (txbuf->m_head != NULL) { 3057 bus_dmamap_sync(txr->txtag->dt_dmat, txbuf->map, 3058 0, txbuf->m_head->m_pkthdr.len, 3059 BUS_DMASYNC_POSTWRITE); 3060 ixgbe_dmamap_unload(txr->txtag, txbuf->map); 3061 m_freem(txbuf->m_head); 3062 txbuf->m_head = NULL; 3063 } 3064 /* Clear the EOP index */ 3065 txbuf->eop_index = -1; 3066 } 3067 3068 #ifdef IXGBE_FDIR 3069 /* Set the rate at which we sample packets */ 3070 if (adapter->hw.mac.type != ixgbe_mac_82598EB) 3071 txr->atr_sample = atr_sample_rate; 3072 #endif 3073 3074 /* Set number of descriptors available */ 3075 txr->tx_avail = adapter->num_tx_desc; 3076 3077 ixgbe_dmamap_sync(txr->txdma.dma_tag, txr->txdma.dma_map, 3078 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); 3079 IXGBE_TX_UNLOCK(txr); 3080 } 3081 3082 /********************************************************************* 3083 * 3084 * Initialize all transmit rings. 3085 * 3086 **********************************************************************/ 3087 static int 3088 ixgbe_setup_transmit_structures(struct adapter *adapter) 3089 { 3090 struct tx_ring *txr = adapter->tx_rings; 3091 3092 for (int i = 0; i < adapter->num_queues; i++, txr++) 3093 ixgbe_setup_transmit_ring(txr); 3094 3095 return (0); 3096 } 3097 3098 /********************************************************************* 3099 * 3100 * Enable transmit unit. 3101 * 3102 **********************************************************************/ 3103 static void 3104 ixgbe_initialize_transmit_units(struct adapter *adapter) 3105 { 3106 struct tx_ring *txr = adapter->tx_rings; 3107 struct ixgbe_hw *hw = &adapter->hw; 3108 3109 /* Setup the Base and Length of the Tx Descriptor Ring */ 3110 3111 for (int i = 0; i < adapter->num_queues; i++, txr++) { 3112 u64 tdba = txr->txdma.dma_paddr; 3113 u32 txctrl; 3114 3115 IXGBE_WRITE_REG(hw, IXGBE_TDBAL(i), 3116 (tdba & 0x00000000ffffffffULL)); 3117 IXGBE_WRITE_REG(hw, IXGBE_TDBAH(i), (tdba >> 32)); 3118 IXGBE_WRITE_REG(hw, IXGBE_TDLEN(i), 3119 adapter->num_tx_desc * sizeof(struct ixgbe_legacy_tx_desc)); 3120 3121 /* Setup the HW Tx Head and Tail descriptor pointers */ 3122 IXGBE_WRITE_REG(hw, IXGBE_TDH(i), 0); 3123 IXGBE_WRITE_REG(hw, IXGBE_TDT(i), 0); 3124 3125 /* Setup Transmit Descriptor Cmd Settings */ 3126 txr->txd_cmd = IXGBE_TXD_CMD_IFCS; 3127 txr->queue_status = IXGBE_QUEUE_IDLE; 3128 3129 /* Disable Head Writeback */ 3130 switch (hw->mac.type) { 3131 case ixgbe_mac_82598EB: 3132 txctrl = IXGBE_READ_REG(hw, IXGBE_DCA_TXCTRL(i)); 3133 break; 3134 case ixgbe_mac_82599EB: 3135 default: 3136 txctrl = IXGBE_READ_REG(hw, IXGBE_DCA_TXCTRL_82599(i)); 3137 break; 3138 } 3139 txctrl &= ~IXGBE_DCA_TXCTRL_TX_WB_RO_EN; 3140 switch (hw->mac.type) { 3141 case ixgbe_mac_82598EB: 3142 IXGBE_WRITE_REG(hw, IXGBE_DCA_TXCTRL(i), txctrl); 3143 break; 3144 case ixgbe_mac_82599EB: 3145 default: 3146 IXGBE_WRITE_REG(hw, IXGBE_DCA_TXCTRL_82599(i), txctrl); 3147 break; 3148 } 3149 3150 } 3151 3152 if (hw->mac.type != ixgbe_mac_82598EB) { 3153 u32 dmatxctl, rttdcs; 3154 dmatxctl = IXGBE_READ_REG(hw, IXGBE_DMATXCTL); 3155 dmatxctl |= IXGBE_DMATXCTL_TE; 3156 IXGBE_WRITE_REG(hw, IXGBE_DMATXCTL, dmatxctl); 3157 /* Disable arbiter to set MTQC */ 3158 rttdcs = IXGBE_READ_REG(hw, IXGBE_RTTDCS); 3159 rttdcs |= IXGBE_RTTDCS_ARBDIS; 3160 IXGBE_WRITE_REG(hw, IXGBE_RTTDCS, rttdcs); 3161 IXGBE_WRITE_REG(hw, IXGBE_MTQC, IXGBE_MTQC_64Q_1PB); 3162 rttdcs &= ~IXGBE_RTTDCS_ARBDIS; 3163 IXGBE_WRITE_REG(hw, IXGBE_RTTDCS, rttdcs); 3164 } 3165 3166 return; 3167 } 3168 3169 /********************************************************************* 3170 * 3171 * Free all transmit rings. 3172 * 3173 **********************************************************************/ 3174 static void 3175 ixgbe_free_transmit_structures(struct adapter *adapter) 3176 { 3177 struct tx_ring *txr = adapter->tx_rings; 3178 3179 for (int i = 0; i < adapter->num_queues; i++, txr++) { 3180 IXGBE_TX_LOCK(txr); 3181 ixgbe_free_transmit_buffers(txr); 3182 ixgbe_dma_free(adapter, &txr->txdma); 3183 IXGBE_TX_UNLOCK(txr); 3184 IXGBE_TX_LOCK_DESTROY(txr); 3185 } 3186 free(adapter->tx_rings, M_DEVBUF); 3187 } 3188 3189 /********************************************************************* 3190 * 3191 * Free transmit ring related data structures. 3192 * 3193 **********************************************************************/ 3194 static void 3195 ixgbe_free_transmit_buffers(struct tx_ring *txr) 3196 { 3197 struct adapter *adapter = txr->adapter; 3198 struct ixgbe_tx_buf *tx_buffer; 3199 int i; 3200 3201 INIT_DEBUGOUT("free_transmit_ring: begin"); 3202 3203 if (txr->tx_buffers == NULL) 3204 return; 3205 3206 tx_buffer = txr->tx_buffers; 3207 for (i = 0; i < adapter->num_tx_desc; i++, tx_buffer++) { 3208 if (tx_buffer->m_head != NULL) { 3209 bus_dmamap_sync(txr->txtag->dt_dmat, tx_buffer->map, 3210 0, tx_buffer->m_head->m_pkthdr.len, 3211 BUS_DMASYNC_POSTWRITE); 3212 ixgbe_dmamap_unload(txr->txtag, tx_buffer->map); 3213 m_freem(tx_buffer->m_head); 3214 tx_buffer->m_head = NULL; 3215 if (tx_buffer->map != NULL) { 3216 ixgbe_dmamap_destroy(txr->txtag, 3217 tx_buffer->map); 3218 tx_buffer->map = NULL; 3219 } 3220 } else if (tx_buffer->map != NULL) { 3221 ixgbe_dmamap_unload(txr->txtag, tx_buffer->map); 3222 ixgbe_dmamap_destroy(txr->txtag, tx_buffer->map); 3223 tx_buffer->map = NULL; 3224 } 3225 } 3226 #if __FreeBSD_version >= 800000 3227 if (txr->br != NULL) 3228 buf_ring_free(txr->br, M_DEVBUF); 3229 #endif 3230 if (txr->tx_buffers != NULL) { 3231 free(txr->tx_buffers, M_DEVBUF); 3232 txr->tx_buffers = NULL; 3233 } 3234 if (txr->txtag != NULL) { 3235 ixgbe_dma_tag_destroy(txr->txtag); 3236 txr->txtag = NULL; 3237 } 3238 return; 3239 } 3240 3241 /********************************************************************* 3242 * 3243 * Advanced Context Descriptor setup for VLAN or L4 CSUM 3244 * 3245 **********************************************************************/ 3246 3247 static u32 3248 ixgbe_tx_ctx_setup(struct tx_ring *txr, struct mbuf *mp) 3249 { 3250 struct m_tag *mtag; 3251 struct adapter *adapter = txr->adapter; 3252 struct ethercom *ec = &adapter->osdep.ec; 3253 struct ixgbe_adv_tx_context_desc *TXD; 3254 struct ixgbe_tx_buf *tx_buffer; 3255 u32 olinfo = 0, vlan_macip_lens = 0, type_tucmd_mlhl = 0; 3256 struct ether_vlan_header *eh; 3257 struct ip ip; 3258 struct ip6_hdr ip6; 3259 int ehdrlen, ip_hlen = 0; 3260 u16 etype; 3261 u8 ipproto __diagused = 0; 3262 bool offload; 3263 int ctxd = txr->next_avail_desc; 3264 u16 vtag = 0; 3265 3266 offload = ((mp->m_pkthdr.csum_flags & M_CSUM_OFFLOAD) != 0); 3267 3268 tx_buffer = &txr->tx_buffers[ctxd]; 3269 TXD = (struct ixgbe_adv_tx_context_desc *) &txr->tx_base[ctxd]; 3270 3271 /* 3272 ** In advanced descriptors the vlan tag must 3273 ** be placed into the descriptor itself. 3274 */ 3275 if ((mtag = VLAN_OUTPUT_TAG(ec, mp)) != NULL) { 3276 vtag = htole16(VLAN_TAG_VALUE(mtag) & 0xffff); 3277 vlan_macip_lens |= (vtag << IXGBE_ADVTXD_VLAN_SHIFT); 3278 } else if (!offload) 3279 return 0; 3280 3281 /* 3282 * Determine where frame payload starts. 3283 * Jump over vlan headers if already present, 3284 * helpful for QinQ too. 3285 */ 3286 KASSERT(mp->m_len >= offsetof(struct ether_vlan_header, evl_tag)); 3287 eh = mtod(mp, struct ether_vlan_header *); 3288 if (eh->evl_encap_proto == htons(ETHERTYPE_VLAN)) { 3289 KASSERT(mp->m_len >= sizeof(struct ether_vlan_header)); 3290 etype = ntohs(eh->evl_proto); 3291 ehdrlen = ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN; 3292 } else { 3293 etype = ntohs(eh->evl_encap_proto); 3294 ehdrlen = ETHER_HDR_LEN; 3295 } 3296 3297 /* Set the ether header length */ 3298 vlan_macip_lens |= ehdrlen << IXGBE_ADVTXD_MACLEN_SHIFT; 3299 3300 switch (etype) { 3301 case ETHERTYPE_IP: 3302 m_copydata(mp, ehdrlen, sizeof(ip), &ip); 3303 ip_hlen = ip.ip_hl << 2; 3304 ipproto = ip.ip_p; 3305 #if 0 3306 ip.ip_sum = 0; 3307 m_copyback(mp, ehdrlen, sizeof(ip), &ip); 3308 #else 3309 KASSERT((mp->m_pkthdr.csum_flags & M_CSUM_IPv4) == 0 || 3310 ip.ip_sum == 0); 3311 #endif 3312 type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_IPV4; 3313 break; 3314 case ETHERTYPE_IPV6: 3315 m_copydata(mp, ehdrlen, sizeof(ip6), &ip6); 3316 ip_hlen = sizeof(ip6); 3317 ipproto = ip6.ip6_nxt; 3318 type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_IPV6; 3319 break; 3320 default: 3321 break; 3322 } 3323 3324 if ((mp->m_pkthdr.csum_flags & M_CSUM_IPv4) != 0) 3325 olinfo |= IXGBE_TXD_POPTS_IXSM << 8; 3326 3327 vlan_macip_lens |= ip_hlen; 3328 type_tucmd_mlhl |= IXGBE_ADVTXD_DCMD_DEXT | IXGBE_ADVTXD_DTYP_CTXT; 3329 3330 if (mp->m_pkthdr.csum_flags & (M_CSUM_TCPv4|M_CSUM_TCPv6)) { 3331 type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_L4T_TCP; 3332 olinfo |= IXGBE_TXD_POPTS_TXSM << 8; 3333 KASSERT(ipproto == IPPROTO_TCP); 3334 } else if (mp->m_pkthdr.csum_flags & (M_CSUM_UDPv4|M_CSUM_UDPv6)) { 3335 type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_L4T_UDP; 3336 olinfo |= IXGBE_TXD_POPTS_TXSM << 8; 3337 KASSERT(ipproto == IPPROTO_UDP); 3338 } 3339 3340 /* Now copy bits into descriptor */ 3341 TXD->vlan_macip_lens |= htole32(vlan_macip_lens); 3342 TXD->type_tucmd_mlhl |= htole32(type_tucmd_mlhl); 3343 TXD->seqnum_seed = htole32(0); 3344 TXD->mss_l4len_idx = htole32(0); 3345 3346 tx_buffer->m_head = NULL; 3347 tx_buffer->eop_index = -1; 3348 3349 /* We've consumed the first desc, adjust counters */ 3350 if (++ctxd == adapter->num_tx_desc) 3351 ctxd = 0; 3352 txr->next_avail_desc = ctxd; 3353 --txr->tx_avail; 3354 3355 return olinfo; 3356 } 3357 3358 /********************************************************************** 3359 * 3360 * Setup work for hardware segmentation offload (TSO) on 3361 * adapters using advanced tx descriptors 3362 * 3363 **********************************************************************/ 3364 static bool 3365 ixgbe_tso_setup(struct tx_ring *txr, struct mbuf *mp, u32 *paylen) 3366 { 3367 struct m_tag *mtag; 3368 struct adapter *adapter = txr->adapter; 3369 struct ethercom *ec = &adapter->osdep.ec; 3370 struct ixgbe_adv_tx_context_desc *TXD; 3371 struct ixgbe_tx_buf *tx_buffer; 3372 u32 vlan_macip_lens = 0, type_tucmd_mlhl = 0; 3373 u32 mss_l4len_idx = 0; 3374 u16 vtag = 0; 3375 int ctxd, ehdrlen, hdrlen, ip_hlen, tcp_hlen; 3376 struct ether_vlan_header *eh; 3377 struct ip *ip; 3378 struct tcphdr *th; 3379 3380 3381 /* 3382 * Determine where frame payload starts. 3383 * Jump over vlan headers if already present 3384 */ 3385 eh = mtod(mp, struct ether_vlan_header *); 3386 if (eh->evl_encap_proto == htons(ETHERTYPE_VLAN)) 3387 ehdrlen = ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN; 3388 else 3389 ehdrlen = ETHER_HDR_LEN; 3390 3391 /* Ensure we have at least the IP+TCP header in the first mbuf. */ 3392 if (mp->m_len < ehdrlen + sizeof(struct ip) + sizeof(struct tcphdr)) 3393 return FALSE; 3394 3395 ctxd = txr->next_avail_desc; 3396 tx_buffer = &txr->tx_buffers[ctxd]; 3397 TXD = (struct ixgbe_adv_tx_context_desc *) &txr->tx_base[ctxd]; 3398 3399 ip = (struct ip *)(mp->m_data + ehdrlen); 3400 if (ip->ip_p != IPPROTO_TCP) 3401 return FALSE; /* 0 */ 3402 ip->ip_sum = 0; 3403 ip_hlen = ip->ip_hl << 2; 3404 th = (struct tcphdr *)((char *)ip + ip_hlen); 3405 /* XXX Educated guess: FreeBSD's in_pseudo == NetBSD's in_cksum_phdr */ 3406 th->th_sum = in_cksum_phdr(ip->ip_src.s_addr, 3407 ip->ip_dst.s_addr, htons(IPPROTO_TCP)); 3408 tcp_hlen = th->th_off << 2; 3409 hdrlen = ehdrlen + ip_hlen + tcp_hlen; 3410 3411 /* This is used in the transmit desc in encap */ 3412 *paylen = mp->m_pkthdr.len - hdrlen; 3413 3414 /* VLAN MACLEN IPLEN */ 3415 if ((mtag = VLAN_OUTPUT_TAG(ec, mp)) != NULL) { 3416 vtag = htole16(VLAN_TAG_VALUE(mtag) & 0xffff); 3417 vlan_macip_lens |= (vtag << IXGBE_ADVTXD_VLAN_SHIFT); 3418 } 3419 3420 vlan_macip_lens |= ehdrlen << IXGBE_ADVTXD_MACLEN_SHIFT; 3421 vlan_macip_lens |= ip_hlen; 3422 TXD->vlan_macip_lens |= htole32(vlan_macip_lens); 3423 3424 /* ADV DTYPE TUCMD */ 3425 type_tucmd_mlhl |= IXGBE_ADVTXD_DCMD_DEXT | IXGBE_ADVTXD_DTYP_CTXT; 3426 type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_L4T_TCP; 3427 type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_IPV4; 3428 TXD->type_tucmd_mlhl |= htole32(type_tucmd_mlhl); 3429 3430 3431 /* MSS L4LEN IDX */ 3432 mss_l4len_idx |= (mp->m_pkthdr.segsz << IXGBE_ADVTXD_MSS_SHIFT); 3433 mss_l4len_idx |= (tcp_hlen << IXGBE_ADVTXD_L4LEN_SHIFT); 3434 TXD->mss_l4len_idx = htole32(mss_l4len_idx); 3435 3436 TXD->seqnum_seed = htole32(0); 3437 tx_buffer->m_head = NULL; 3438 tx_buffer->eop_index = -1; 3439 3440 if (++ctxd == adapter->num_tx_desc) 3441 ctxd = 0; 3442 3443 txr->tx_avail--; 3444 txr->next_avail_desc = ctxd; 3445 return TRUE; 3446 } 3447 3448 #ifdef IXGBE_FDIR 3449 /* 3450 ** This routine parses packet headers so that Flow 3451 ** Director can make a hashed filter table entry 3452 ** allowing traffic flows to be identified and kept 3453 ** on the same cpu. This would be a performance 3454 ** hit, but we only do it at IXGBE_FDIR_RATE of 3455 ** packets. 3456 */ 3457 static void 3458 ixgbe_atr(struct tx_ring *txr, struct mbuf *mp) 3459 { 3460 struct adapter *adapter = txr->adapter; 3461 struct ix_queue *que; 3462 struct ip *ip; 3463 struct tcphdr *th; 3464 struct udphdr *uh; 3465 struct ether_vlan_header *eh; 3466 union ixgbe_atr_hash_dword input = {.dword = 0}; 3467 union ixgbe_atr_hash_dword common = {.dword = 0}; 3468 int ehdrlen, ip_hlen; 3469 u16 etype; 3470 3471 eh = mtod(mp, struct ether_vlan_header *); 3472 if (eh->evl_encap_proto == htons(ETHERTYPE_VLAN)) { 3473 ehdrlen = ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN; 3474 etype = eh->evl_proto; 3475 } else { 3476 ehdrlen = ETHER_HDR_LEN; 3477 etype = eh->evl_encap_proto; 3478 } 3479 3480 /* Only handling IPv4 */ 3481 if (etype != htons(ETHERTYPE_IP)) 3482 return; 3483 3484 ip = (struct ip *)(mp->m_data + ehdrlen); 3485 ip_hlen = ip->ip_hl << 2; 3486 3487 /* check if we're UDP or TCP */ 3488 switch (ip->ip_p) { 3489 case IPPROTO_TCP: 3490 th = (struct tcphdr *)((char *)ip + ip_hlen); 3491 /* src and dst are inverted */ 3492 common.port.dst ^= th->th_sport; 3493 common.port.src ^= th->th_dport; 3494 input.formatted.flow_type ^= IXGBE_ATR_FLOW_TYPE_TCPV4; 3495 break; 3496 case IPPROTO_UDP: 3497 uh = (struct udphdr *)((char *)ip + ip_hlen); 3498 /* src and dst are inverted */ 3499 common.port.dst ^= uh->uh_sport; 3500 common.port.src ^= uh->uh_dport; 3501 input.formatted.flow_type ^= IXGBE_ATR_FLOW_TYPE_UDPV4; 3502 break; 3503 default: 3504 return; 3505 } 3506 3507 input.formatted.vlan_id = htobe16(mp->m_pkthdr.ether_vtag); 3508 if (mp->m_pkthdr.ether_vtag) 3509 common.flex_bytes ^= htons(ETHERTYPE_VLAN); 3510 else 3511 common.flex_bytes ^= etype; 3512 common.ip ^= ip->ip_src.s_addr ^ ip->ip_dst.s_addr; 3513 3514 que = &adapter->queues[txr->me]; 3515 /* 3516 ** This assumes the Rx queue and Tx 3517 ** queue are bound to the same CPU 3518 */ 3519 ixgbe_fdir_add_signature_filter_82599(&adapter->hw, 3520 input, common, que->msix); 3521 } 3522 #endif /* IXGBE_FDIR */ 3523 3524 /********************************************************************** 3525 * 3526 * Examine each tx_buffer in the used queue. If the hardware is done 3527 * processing the packet then free associated resources. The 3528 * tx_buffer is put back on the free queue. 3529 * 3530 **********************************************************************/ 3531 static bool 3532 ixgbe_txeof(struct tx_ring *txr) 3533 { 3534 struct adapter *adapter = txr->adapter; 3535 struct ifnet *ifp = adapter->ifp; 3536 u32 first, last, done, processed; 3537 struct ixgbe_tx_buf *tx_buffer; 3538 struct ixgbe_legacy_tx_desc *tx_desc, *eop_desc; 3539 struct timeval now, elapsed; 3540 3541 KASSERT(mutex_owned(&txr->tx_mtx)); 3542 3543 if (txr->tx_avail == adapter->num_tx_desc) { 3544 txr->queue_status = IXGBE_QUEUE_IDLE; 3545 return false; 3546 } 3547 3548 processed = 0; 3549 first = txr->next_to_clean; 3550 tx_buffer = &txr->tx_buffers[first]; 3551 /* For cleanup we just use legacy struct */ 3552 tx_desc = (struct ixgbe_legacy_tx_desc *)&txr->tx_base[first]; 3553 last = tx_buffer->eop_index; 3554 if (last == -1) 3555 return false; 3556 eop_desc = (struct ixgbe_legacy_tx_desc *)&txr->tx_base[last]; 3557 3558 /* 3559 ** Get the index of the first descriptor 3560 ** BEYOND the EOP and call that 'done'. 3561 ** I do this so the comparison in the 3562 ** inner while loop below can be simple 3563 */ 3564 if (++last == adapter->num_tx_desc) last = 0; 3565 done = last; 3566 3567 ixgbe_dmamap_sync(txr->txdma.dma_tag, txr->txdma.dma_map, 3568 BUS_DMASYNC_POSTREAD); 3569 /* 3570 ** Only the EOP descriptor of a packet now has the DD 3571 ** bit set, this is what we look for... 3572 */ 3573 while (eop_desc->upper.fields.status & IXGBE_TXD_STAT_DD) { 3574 /* We clean the range of the packet */ 3575 while (first != done) { 3576 tx_desc->upper.data = 0; 3577 tx_desc->lower.data = 0; 3578 tx_desc->buffer_addr = 0; 3579 ++txr->tx_avail; 3580 ++processed; 3581 3582 if (tx_buffer->m_head) { 3583 txr->bytes += 3584 tx_buffer->m_head->m_pkthdr.len; 3585 bus_dmamap_sync(txr->txtag->dt_dmat, 3586 tx_buffer->map, 3587 0, tx_buffer->m_head->m_pkthdr.len, 3588 BUS_DMASYNC_POSTWRITE); 3589 ixgbe_dmamap_unload(txr->txtag, tx_buffer->map); 3590 m_freem(tx_buffer->m_head); 3591 tx_buffer->m_head = NULL; 3592 } 3593 tx_buffer->eop_index = -1; 3594 getmicrotime(&txr->watchdog_time); 3595 3596 if (++first == adapter->num_tx_desc) 3597 first = 0; 3598 3599 tx_buffer = &txr->tx_buffers[first]; 3600 tx_desc = 3601 (struct ixgbe_legacy_tx_desc *)&txr->tx_base[first]; 3602 } 3603 ++txr->packets; 3604 ++ifp->if_opackets; 3605 /* See if there is more work now */ 3606 last = tx_buffer->eop_index; 3607 if (last != -1) { 3608 eop_desc = 3609 (struct ixgbe_legacy_tx_desc *)&txr->tx_base[last]; 3610 /* Get next done point */ 3611 if (++last == adapter->num_tx_desc) last = 0; 3612 done = last; 3613 } else 3614 break; 3615 } 3616 ixgbe_dmamap_sync(txr->txdma.dma_tag, txr->txdma.dma_map, 3617 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); 3618 3619 txr->next_to_clean = first; 3620 3621 /* 3622 ** Watchdog calculation, we know there's 3623 ** work outstanding or the first return 3624 ** would have been taken, so none processed 3625 ** for too long indicates a hang. 3626 */ 3627 getmicrotime(&now); 3628 timersub(&now, &txr->watchdog_time, &elapsed); 3629 if (!processed && tvtohz(&elapsed) > IXGBE_WATCHDOG) 3630 txr->queue_status = IXGBE_QUEUE_HUNG; 3631 3632 /* 3633 * If we have enough room, clear IFF_OACTIVE to tell the stack that 3634 * it is OK to send packets. If there are no pending descriptors, 3635 * clear the timeout. Otherwise, if some descriptors have been freed, 3636 * restart the timeout. 3637 */ 3638 if (txr->tx_avail > IXGBE_TX_CLEANUP_THRESHOLD) { 3639 ifp->if_flags &= ~IFF_OACTIVE; 3640 if (txr->tx_avail == adapter->num_tx_desc) { 3641 txr->queue_status = IXGBE_QUEUE_IDLE; 3642 return false; 3643 } 3644 } 3645 3646 return true; 3647 } 3648 3649 /********************************************************************* 3650 * 3651 * Refresh mbuf buffers for RX descriptor rings 3652 * - now keeps its own state so discards due to resource 3653 * exhaustion are unnecessary, if an mbuf cannot be obtained 3654 * it just returns, keeping its placeholder, thus it can simply 3655 * be recalled to try again. 3656 * 3657 **********************************************************************/ 3658 static void 3659 ixgbe_refresh_mbufs(struct rx_ring *rxr, int limit) 3660 { 3661 struct adapter *adapter = rxr->adapter; 3662 struct ixgbe_rx_buf *rxbuf; 3663 struct mbuf *mh, *mp; 3664 int i, j, error; 3665 bool refreshed = false; 3666 3667 i = j = rxr->next_to_refresh; 3668 /* Control the loop with one beyond */ 3669 if (++j == adapter->num_rx_desc) 3670 j = 0; 3671 3672 while (j != limit) { 3673 rxbuf = &rxr->rx_buffers[i]; 3674 if (rxr->hdr_split == FALSE) 3675 goto no_split; 3676 3677 if (rxbuf->m_head == NULL) { 3678 mh = m_gethdr(M_DONTWAIT, MT_DATA); 3679 if (mh == NULL) 3680 goto update; 3681 } else 3682 mh = rxbuf->m_head; 3683 3684 mh->m_pkthdr.len = mh->m_len = MHLEN; 3685 mh->m_len = MHLEN; 3686 mh->m_flags |= M_PKTHDR; 3687 /* Get the memory mapping */ 3688 error = bus_dmamap_load_mbuf(rxr->htag->dt_dmat, 3689 rxbuf->hmap, mh, BUS_DMA_NOWAIT); 3690 if (error != 0) { 3691 printf("Refresh mbufs: hdr dmamap load" 3692 " failure - %d\n", error); 3693 m_free(mh); 3694 rxbuf->m_head = NULL; 3695 goto update; 3696 } 3697 rxbuf->m_head = mh; 3698 ixgbe_dmamap_sync(rxr->htag, rxbuf->hmap, BUS_DMASYNC_PREREAD); 3699 rxr->rx_base[i].read.hdr_addr = 3700 htole64(rxbuf->hmap->dm_segs[0].ds_addr); 3701 3702 no_split: 3703 if (rxbuf->m_pack == NULL) { 3704 mp = ixgbe_getjcl(&adapter->jcl_head, M_DONTWAIT, 3705 MT_DATA, M_PKTHDR, adapter->rx_mbuf_sz); 3706 if (mp == NULL) { 3707 rxr->no_jmbuf.ev_count++; 3708 goto update; 3709 } 3710 } else 3711 mp = rxbuf->m_pack; 3712 3713 mp->m_pkthdr.len = mp->m_len = adapter->rx_mbuf_sz; 3714 /* Get the memory mapping */ 3715 error = bus_dmamap_load_mbuf(rxr->ptag->dt_dmat, 3716 rxbuf->pmap, mp, BUS_DMA_NOWAIT); 3717 if (error != 0) { 3718 printf("Refresh mbufs: payload dmamap load" 3719 " failure - %d\n", error); 3720 m_free(mp); 3721 rxbuf->m_pack = NULL; 3722 goto update; 3723 } 3724 rxbuf->m_pack = mp; 3725 bus_dmamap_sync(rxr->ptag->dt_dmat, rxbuf->pmap, 3726 0, mp->m_pkthdr.len, BUS_DMASYNC_PREREAD); 3727 rxr->rx_base[i].read.pkt_addr = 3728 htole64(rxbuf->pmap->dm_segs[0].ds_addr); 3729 3730 refreshed = true; 3731 /* Next is precalculated */ 3732 i = j; 3733 rxr->next_to_refresh = i; 3734 if (++j == adapter->num_rx_desc) 3735 j = 0; 3736 } 3737 update: 3738 if (refreshed) /* Update hardware tail index */ 3739 IXGBE_WRITE_REG(&adapter->hw, 3740 IXGBE_RDT(rxr->me), rxr->next_to_refresh); 3741 return; 3742 } 3743 3744 /********************************************************************* 3745 * 3746 * Allocate memory for rx_buffer structures. Since we use one 3747 * rx_buffer per received packet, the maximum number of rx_buffer's 3748 * that we'll need is equal to the number of receive descriptors 3749 * that we've allocated. 3750 * 3751 **********************************************************************/ 3752 static int 3753 ixgbe_allocate_receive_buffers(struct rx_ring *rxr) 3754 { 3755 struct adapter *adapter = rxr->adapter; 3756 device_t dev = adapter->dev; 3757 struct ixgbe_rx_buf *rxbuf; 3758 int i, bsize, error; 3759 3760 bsize = sizeof(struct ixgbe_rx_buf) * adapter->num_rx_desc; 3761 if (!(rxr->rx_buffers = 3762 (struct ixgbe_rx_buf *) malloc(bsize, 3763 M_DEVBUF, M_NOWAIT | M_ZERO))) { 3764 aprint_error_dev(dev, "Unable to allocate rx_buffer memory\n"); 3765 error = ENOMEM; 3766 goto fail; 3767 } 3768 3769 if ((error = ixgbe_dma_tag_create(adapter->osdep.dmat, /* parent */ 3770 1, 0, /* alignment, bounds */ 3771 MSIZE, /* maxsize */ 3772 1, /* nsegments */ 3773 MSIZE, /* maxsegsize */ 3774 0, /* flags */ 3775 &rxr->htag))) { 3776 aprint_error_dev(dev, "Unable to create RX DMA tag\n"); 3777 goto fail; 3778 } 3779 3780 if ((error = ixgbe_dma_tag_create(adapter->osdep.dmat, /* parent */ 3781 1, 0, /* alignment, bounds */ 3782 MJUM16BYTES, /* maxsize */ 3783 1, /* nsegments */ 3784 MJUM16BYTES, /* maxsegsize */ 3785 0, /* flags */ 3786 &rxr->ptag))) { 3787 aprint_error_dev(dev, "Unable to create RX DMA tag\n"); 3788 goto fail; 3789 } 3790 3791 for (i = 0; i < adapter->num_rx_desc; i++, rxbuf++) { 3792 rxbuf = &rxr->rx_buffers[i]; 3793 error = ixgbe_dmamap_create(rxr->htag, 3794 BUS_DMA_NOWAIT, &rxbuf->hmap); 3795 if (error) { 3796 aprint_error_dev(dev, "Unable to create RX head map\n"); 3797 goto fail; 3798 } 3799 error = ixgbe_dmamap_create(rxr->ptag, 3800 BUS_DMA_NOWAIT, &rxbuf->pmap); 3801 if (error) { 3802 aprint_error_dev(dev, "Unable to create RX pkt map\n"); 3803 goto fail; 3804 } 3805 } 3806 3807 return (0); 3808 3809 fail: 3810 /* Frees all, but can handle partial completion */ 3811 ixgbe_free_receive_structures(adapter); 3812 return (error); 3813 } 3814 3815 /* 3816 ** Used to detect a descriptor that has 3817 ** been merged by Hardware RSC. 3818 */ 3819 static inline u32 3820 ixgbe_rsc_count(union ixgbe_adv_rx_desc *rx) 3821 { 3822 return (le32toh(rx->wb.lower.lo_dword.data) & 3823 IXGBE_RXDADV_RSCCNT_MASK) >> IXGBE_RXDADV_RSCCNT_SHIFT; 3824 } 3825 3826 /********************************************************************* 3827 * 3828 * Initialize Hardware RSC (LRO) feature on 82599 3829 * for an RX ring, this is toggled by the LRO capability 3830 * even though it is transparent to the stack. 3831 * 3832 **********************************************************************/ 3833 static void 3834 ixgbe_setup_hw_rsc(struct rx_ring *rxr) 3835 { 3836 struct adapter *adapter = rxr->adapter; 3837 struct ixgbe_hw *hw = &adapter->hw; 3838 u32 rscctrl, rdrxctl; 3839 3840 rdrxctl = IXGBE_READ_REG(hw, IXGBE_RDRXCTL); 3841 rdrxctl &= ~IXGBE_RDRXCTL_RSCFRSTSIZE; 3842 rdrxctl |= IXGBE_RDRXCTL_CRCSTRIP; 3843 rdrxctl |= IXGBE_RDRXCTL_RSCACKC; 3844 IXGBE_WRITE_REG(hw, IXGBE_RDRXCTL, rdrxctl); 3845 3846 rscctrl = IXGBE_READ_REG(hw, IXGBE_RSCCTL(rxr->me)); 3847 rscctrl |= IXGBE_RSCCTL_RSCEN; 3848 /* 3849 ** Limit the total number of descriptors that 3850 ** can be combined, so it does not exceed 64K 3851 */ 3852 if (adapter->rx_mbuf_sz == MCLBYTES) 3853 rscctrl |= IXGBE_RSCCTL_MAXDESC_16; 3854 else if (adapter->rx_mbuf_sz == MJUMPAGESIZE) 3855 rscctrl |= IXGBE_RSCCTL_MAXDESC_8; 3856 else if (adapter->rx_mbuf_sz == MJUM9BYTES) 3857 rscctrl |= IXGBE_RSCCTL_MAXDESC_4; 3858 else /* Using 16K cluster */ 3859 rscctrl |= IXGBE_RSCCTL_MAXDESC_1; 3860 3861 IXGBE_WRITE_REG(hw, IXGBE_RSCCTL(rxr->me), rscctrl); 3862 3863 /* Enable TCP header recognition */ 3864 IXGBE_WRITE_REG(hw, IXGBE_PSRTYPE(0), 3865 (IXGBE_READ_REG(hw, IXGBE_PSRTYPE(0)) | 3866 IXGBE_PSRTYPE_TCPHDR)); 3867 3868 /* Disable RSC for ACK packets */ 3869 IXGBE_WRITE_REG(hw, IXGBE_RSCDBU, 3870 (IXGBE_RSCDBU_RSCACKDIS | IXGBE_READ_REG(hw, IXGBE_RSCDBU))); 3871 3872 rxr->hw_rsc = TRUE; 3873 } 3874 3875 3876 static void 3877 ixgbe_free_receive_ring(struct rx_ring *rxr) 3878 { 3879 struct adapter *adapter; 3880 struct ixgbe_rx_buf *rxbuf; 3881 int i; 3882 3883 adapter = rxr->adapter; 3884 for (i = 0; i < adapter->num_rx_desc; i++) { 3885 rxbuf = &rxr->rx_buffers[i]; 3886 if (rxbuf->m_head != NULL) { 3887 ixgbe_dmamap_sync(rxr->htag, rxbuf->hmap, 3888 BUS_DMASYNC_POSTREAD); 3889 ixgbe_dmamap_unload(rxr->htag, rxbuf->hmap); 3890 rxbuf->m_head->m_flags |= M_PKTHDR; 3891 m_freem(rxbuf->m_head); 3892 } 3893 if (rxbuf->m_pack != NULL) { 3894 bus_dmamap_sync(rxr->ptag->dt_dmat, rxbuf->pmap, 3895 0, rxbuf->m_pack->m_pkthdr.len, 3896 BUS_DMASYNC_POSTREAD); 3897 ixgbe_dmamap_unload(rxr->ptag, rxbuf->pmap); 3898 rxbuf->m_pack->m_flags |= M_PKTHDR; 3899 m_freem(rxbuf->m_pack); 3900 } 3901 rxbuf->m_head = NULL; 3902 rxbuf->m_pack = NULL; 3903 } 3904 } 3905 3906 3907 /********************************************************************* 3908 * 3909 * Initialize a receive ring and its buffers. 3910 * 3911 **********************************************************************/ 3912 static int 3913 ixgbe_setup_receive_ring(struct rx_ring *rxr) 3914 { 3915 struct adapter *adapter; 3916 struct ifnet *ifp; 3917 struct ixgbe_rx_buf *rxbuf; 3918 #ifdef LRO 3919 struct lro_ctrl *lro = &rxr->lro; 3920 #endif /* LRO */ 3921 int rsize, error = 0; 3922 3923 adapter = rxr->adapter; 3924 ifp = adapter->ifp; 3925 3926 /* Clear the ring contents */ 3927 IXGBE_RX_LOCK(rxr); 3928 rsize = roundup2(adapter->num_rx_desc * 3929 sizeof(union ixgbe_adv_rx_desc), DBA_ALIGN); 3930 bzero((void *)rxr->rx_base, rsize); 3931 3932 /* Free current RX buffer structs and their mbufs */ 3933 ixgbe_free_receive_ring(rxr); 3934 3935 /* Now reinitialize our supply of jumbo mbufs. The number 3936 * or size of jumbo mbufs may have changed. 3937 */ 3938 ixgbe_jcl_reinit(&adapter->jcl_head, rxr->ptag->dt_dmat, 3939 2 * adapter->num_rx_desc, adapter->rx_mbuf_sz); 3940 3941 /* Configure header split? */ 3942 if (ixgbe_header_split) 3943 rxr->hdr_split = TRUE; 3944 3945 /* Now replenish the mbufs */ 3946 for (int j = 0; j != adapter->num_rx_desc; ++j) { 3947 struct mbuf *mh, *mp; 3948 3949 rxbuf = &rxr->rx_buffers[j]; 3950 /* 3951 ** Don't allocate mbufs if not 3952 ** doing header split, its wasteful 3953 */ 3954 if (rxr->hdr_split == FALSE) 3955 goto skip_head; 3956 3957 /* First the header */ 3958 rxbuf->m_head = m_gethdr(M_DONTWAIT, MT_DATA); 3959 if (rxbuf->m_head == NULL) { 3960 error = ENOBUFS; 3961 goto fail; 3962 } 3963 m_adj(rxbuf->m_head, ETHER_ALIGN); 3964 mh = rxbuf->m_head; 3965 mh->m_len = mh->m_pkthdr.len = MHLEN; 3966 mh->m_flags |= M_PKTHDR; 3967 /* Get the memory mapping */ 3968 error = bus_dmamap_load_mbuf(rxr->htag->dt_dmat, 3969 rxbuf->hmap, rxbuf->m_head, BUS_DMA_NOWAIT); 3970 if (error != 0) /* Nothing elegant to do here */ 3971 goto fail; 3972 bus_dmamap_sync(rxr->htag->dt_dmat, rxbuf->hmap, 3973 0, mh->m_pkthdr.len, BUS_DMASYNC_PREREAD); 3974 /* Update descriptor */ 3975 rxr->rx_base[j].read.hdr_addr = 3976 htole64(rxbuf->hmap->dm_segs[0].ds_addr); 3977 3978 skip_head: 3979 /* Now the payload cluster */ 3980 rxbuf->m_pack = ixgbe_getjcl(&adapter->jcl_head, M_DONTWAIT, 3981 MT_DATA, M_PKTHDR, adapter->rx_mbuf_sz); 3982 if (rxbuf->m_pack == NULL) { 3983 error = ENOBUFS; 3984 goto fail; 3985 } 3986 mp = rxbuf->m_pack; 3987 mp->m_pkthdr.len = mp->m_len = adapter->rx_mbuf_sz; 3988 /* Get the memory mapping */ 3989 error = bus_dmamap_load_mbuf(rxr->ptag->dt_dmat, 3990 rxbuf->pmap, mp, BUS_DMA_NOWAIT); 3991 if (error != 0) 3992 goto fail; 3993 bus_dmamap_sync(rxr->ptag->dt_dmat, rxbuf->pmap, 3994 0, adapter->rx_mbuf_sz, BUS_DMASYNC_PREREAD); 3995 /* Update descriptor */ 3996 rxr->rx_base[j].read.pkt_addr = 3997 htole64(rxbuf->pmap->dm_segs[0].ds_addr); 3998 } 3999 4000 4001 /* Setup our descriptor indices */ 4002 rxr->next_to_check = 0; 4003 rxr->next_to_refresh = 0; 4004 rxr->lro_enabled = FALSE; 4005 rxr->rx_split_packets.ev_count = 0; 4006 rxr->rx_bytes.ev_count = 0; 4007 rxr->discard = FALSE; 4008 4009 ixgbe_dmamap_sync(rxr->rxdma.dma_tag, rxr->rxdma.dma_map, 4010 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); 4011 4012 /* 4013 ** Now set up the LRO interface: 4014 ** 82598 uses software LRO, the 4015 ** 82599 uses a hardware assist. 4016 */ 4017 if ((adapter->hw.mac.type != ixgbe_mac_82598EB) && 4018 (ifp->if_capenable & IFCAP_RXCSUM) && 4019 (ifp->if_capenable & IFCAP_LRO)) 4020 ixgbe_setup_hw_rsc(rxr); 4021 #ifdef LRO 4022 else if (ifp->if_capenable & IFCAP_LRO) { 4023 device_t dev = adapter->dev; 4024 int err = tcp_lro_init(lro); 4025 if (err) { 4026 device_printf(dev, "LRO Initialization failed!\n"); 4027 goto fail; 4028 } 4029 INIT_DEBUGOUT("RX Soft LRO Initialized\n"); 4030 rxr->lro_enabled = TRUE; 4031 lro->ifp = adapter->ifp; 4032 } 4033 #endif /* LRO */ 4034 4035 IXGBE_RX_UNLOCK(rxr); 4036 return (0); 4037 4038 fail: 4039 ixgbe_free_receive_ring(rxr); 4040 IXGBE_RX_UNLOCK(rxr); 4041 return (error); 4042 } 4043 4044 /********************************************************************* 4045 * 4046 * Initialize all receive rings. 4047 * 4048 **********************************************************************/ 4049 static int 4050 ixgbe_setup_receive_structures(struct adapter *adapter) 4051 { 4052 struct rx_ring *rxr = adapter->rx_rings; 4053 int j; 4054 4055 for (j = 0; j < adapter->num_queues; j++, rxr++) 4056 if (ixgbe_setup_receive_ring(rxr)) 4057 goto fail; 4058 4059 return (0); 4060 fail: 4061 /* 4062 * Free RX buffers allocated so far, we will only handle 4063 * the rings that completed, the failing case will have 4064 * cleaned up for itself. 'j' failed, so its the terminus. 4065 */ 4066 for (int i = 0; i < j; ++i) { 4067 rxr = &adapter->rx_rings[i]; 4068 ixgbe_free_receive_ring(rxr); 4069 } 4070 4071 return (ENOBUFS); 4072 } 4073 4074 /********************************************************************* 4075 * 4076 * Setup receive registers and features. 4077 * 4078 **********************************************************************/ 4079 #define IXGBE_SRRCTL_BSIZEHDRSIZE_SHIFT 2 4080 4081 static void 4082 ixgbe_initialize_receive_units(struct adapter *adapter) 4083 { 4084 int i; 4085 struct rx_ring *rxr = adapter->rx_rings; 4086 struct ixgbe_hw *hw = &adapter->hw; 4087 struct ifnet *ifp = adapter->ifp; 4088 u32 bufsz, rxctrl, fctrl, srrctl, rxcsum; 4089 u32 reta, mrqc = 0, hlreg, r[10]; 4090 4091 4092 /* 4093 * Make sure receives are disabled while 4094 * setting up the descriptor ring 4095 */ 4096 rxctrl = IXGBE_READ_REG(hw, IXGBE_RXCTRL); 4097 IXGBE_WRITE_REG(hw, IXGBE_RXCTRL, 4098 rxctrl & ~IXGBE_RXCTRL_RXEN); 4099 4100 /* Enable broadcasts */ 4101 fctrl = IXGBE_READ_REG(hw, IXGBE_FCTRL); 4102 fctrl |= IXGBE_FCTRL_BAM; 4103 fctrl |= IXGBE_FCTRL_DPF; 4104 fctrl |= IXGBE_FCTRL_PMCF; 4105 IXGBE_WRITE_REG(hw, IXGBE_FCTRL, fctrl); 4106 4107 /* Set for Jumbo Frames? */ 4108 hlreg = IXGBE_READ_REG(hw, IXGBE_HLREG0); 4109 if (ifp->if_mtu > ETHERMTU) 4110 hlreg |= IXGBE_HLREG0_JUMBOEN; 4111 else 4112 hlreg &= ~IXGBE_HLREG0_JUMBOEN; 4113 IXGBE_WRITE_REG(hw, IXGBE_HLREG0, hlreg); 4114 4115 bufsz = adapter->rx_mbuf_sz >> IXGBE_SRRCTL_BSIZEPKT_SHIFT; 4116 4117 for (i = 0; i < adapter->num_queues; i++, rxr++) { 4118 u64 rdba = rxr->rxdma.dma_paddr; 4119 4120 /* Setup the Base and Length of the Rx Descriptor Ring */ 4121 IXGBE_WRITE_REG(hw, IXGBE_RDBAL(i), 4122 (rdba & 0x00000000ffffffffULL)); 4123 IXGBE_WRITE_REG(hw, IXGBE_RDBAH(i), (rdba >> 32)); 4124 IXGBE_WRITE_REG(hw, IXGBE_RDLEN(i), 4125 adapter->num_rx_desc * sizeof(union ixgbe_adv_rx_desc)); 4126 4127 /* Set up the SRRCTL register */ 4128 srrctl = IXGBE_READ_REG(hw, IXGBE_SRRCTL(i)); 4129 srrctl &= ~IXGBE_SRRCTL_BSIZEHDR_MASK; 4130 srrctl &= ~IXGBE_SRRCTL_BSIZEPKT_MASK; 4131 srrctl |= bufsz; 4132 if (rxr->hdr_split) { 4133 /* Use a standard mbuf for the header */ 4134 srrctl |= ((IXGBE_RX_HDR << 4135 IXGBE_SRRCTL_BSIZEHDRSIZE_SHIFT) 4136 & IXGBE_SRRCTL_BSIZEHDR_MASK); 4137 srrctl |= IXGBE_SRRCTL_DESCTYPE_HDR_SPLIT_ALWAYS; 4138 } else 4139 srrctl |= IXGBE_SRRCTL_DESCTYPE_ADV_ONEBUF; 4140 IXGBE_WRITE_REG(hw, IXGBE_SRRCTL(i), srrctl); 4141 4142 /* Setup the HW Rx Head and Tail Descriptor Pointers */ 4143 IXGBE_WRITE_REG(hw, IXGBE_RDH(i), 0); 4144 IXGBE_WRITE_REG(hw, IXGBE_RDT(i), 0); 4145 } 4146 4147 if (adapter->hw.mac.type != ixgbe_mac_82598EB) { 4148 u32 psrtype = IXGBE_PSRTYPE_TCPHDR | 4149 IXGBE_PSRTYPE_UDPHDR | 4150 IXGBE_PSRTYPE_IPV4HDR | 4151 IXGBE_PSRTYPE_IPV6HDR; 4152 IXGBE_WRITE_REG(hw, IXGBE_PSRTYPE(0), psrtype); 4153 } 4154 4155 rxcsum = IXGBE_READ_REG(hw, IXGBE_RXCSUM); 4156 4157 /* Setup RSS */ 4158 if (adapter->num_queues > 1) { 4159 int j; 4160 reta = 0; 4161 4162 /* set up random bits */ 4163 cprng_fast(&r, sizeof(r)); 4164 4165 /* Set up the redirection table */ 4166 for (i = 0, j = 0; i < 128; i++, j++) { 4167 if (j == adapter->num_queues) j = 0; 4168 reta = (reta << 8) | (j * 0x11); 4169 if ((i & 3) == 3) 4170 IXGBE_WRITE_REG(hw, IXGBE_RETA(i >> 2), reta); 4171 } 4172 4173 /* Now fill our hash function seeds */ 4174 for (i = 0; i < 10; i++) 4175 IXGBE_WRITE_REG(hw, IXGBE_RSSRK(i), r[i]); 4176 4177 /* Perform hash on these packet types */ 4178 mrqc = IXGBE_MRQC_RSSEN 4179 | IXGBE_MRQC_RSS_FIELD_IPV4 4180 | IXGBE_MRQC_RSS_FIELD_IPV4_TCP 4181 | IXGBE_MRQC_RSS_FIELD_IPV4_UDP 4182 | IXGBE_MRQC_RSS_FIELD_IPV6_EX_TCP 4183 | IXGBE_MRQC_RSS_FIELD_IPV6_EX 4184 | IXGBE_MRQC_RSS_FIELD_IPV6 4185 | IXGBE_MRQC_RSS_FIELD_IPV6_TCP 4186 | IXGBE_MRQC_RSS_FIELD_IPV6_UDP 4187 | IXGBE_MRQC_RSS_FIELD_IPV6_EX_UDP; 4188 IXGBE_WRITE_REG(hw, IXGBE_MRQC, mrqc); 4189 4190 /* RSS and RX IPP Checksum are mutually exclusive */ 4191 rxcsum |= IXGBE_RXCSUM_PCSD; 4192 } 4193 4194 if (ifp->if_capenable & IFCAP_RXCSUM) 4195 rxcsum |= IXGBE_RXCSUM_PCSD; 4196 4197 if (!(rxcsum & IXGBE_RXCSUM_PCSD)) 4198 rxcsum |= IXGBE_RXCSUM_IPPCSE; 4199 4200 IXGBE_WRITE_REG(hw, IXGBE_RXCSUM, rxcsum); 4201 4202 return; 4203 } 4204 4205 /********************************************************************* 4206 * 4207 * Free all receive rings. 4208 * 4209 **********************************************************************/ 4210 static void 4211 ixgbe_free_receive_structures(struct adapter *adapter) 4212 { 4213 struct rx_ring *rxr = adapter->rx_rings; 4214 4215 for (int i = 0; i < adapter->num_queues; i++, rxr++) { 4216 #ifdef LRO 4217 struct lro_ctrl *lro = &rxr->lro; 4218 #endif /* LRO */ 4219 ixgbe_free_receive_buffers(rxr); 4220 #ifdef LRO 4221 /* Free LRO memory */ 4222 tcp_lro_free(lro); 4223 #endif /* LRO */ 4224 /* Free the ring memory as well */ 4225 ixgbe_dma_free(adapter, &rxr->rxdma); 4226 } 4227 4228 free(adapter->rx_rings, M_DEVBUF); 4229 } 4230 4231 4232 /********************************************************************* 4233 * 4234 * Free receive ring data structures 4235 * 4236 **********************************************************************/ 4237 static void 4238 ixgbe_free_receive_buffers(struct rx_ring *rxr) 4239 { 4240 struct adapter *adapter = rxr->adapter; 4241 struct ixgbe_rx_buf *rxbuf; 4242 4243 INIT_DEBUGOUT("free_receive_structures: begin"); 4244 4245 /* Cleanup any existing buffers */ 4246 if (rxr->rx_buffers != NULL) { 4247 for (int i = 0; i < adapter->num_rx_desc; i++) { 4248 rxbuf = &rxr->rx_buffers[i]; 4249 if (rxbuf->m_head != NULL) { 4250 ixgbe_dmamap_sync(rxr->htag, rxbuf->hmap, 4251 BUS_DMASYNC_POSTREAD); 4252 ixgbe_dmamap_unload(rxr->htag, rxbuf->hmap); 4253 rxbuf->m_head->m_flags |= M_PKTHDR; 4254 m_freem(rxbuf->m_head); 4255 } 4256 if (rxbuf->m_pack != NULL) { 4257 bus_dmamap_sync(rxr->ptag->dt_dmat, rxbuf->pmap, 4258 0, rxbuf->m_pack->m_pkthdr.len, 4259 BUS_DMASYNC_POSTREAD); 4260 ixgbe_dmamap_unload(rxr->ptag, rxbuf->pmap); 4261 rxbuf->m_pack->m_flags |= M_PKTHDR; 4262 m_freem(rxbuf->m_pack); 4263 } 4264 rxbuf->m_head = NULL; 4265 rxbuf->m_pack = NULL; 4266 if (rxbuf->hmap != NULL) { 4267 ixgbe_dmamap_destroy(rxr->htag, rxbuf->hmap); 4268 rxbuf->hmap = NULL; 4269 } 4270 if (rxbuf->pmap != NULL) { 4271 ixgbe_dmamap_destroy(rxr->ptag, rxbuf->pmap); 4272 rxbuf->pmap = NULL; 4273 } 4274 } 4275 if (rxr->rx_buffers != NULL) { 4276 free(rxr->rx_buffers, M_DEVBUF); 4277 rxr->rx_buffers = NULL; 4278 } 4279 } 4280 4281 if (rxr->htag != NULL) { 4282 ixgbe_dma_tag_destroy(rxr->htag); 4283 rxr->htag = NULL; 4284 } 4285 if (rxr->ptag != NULL) { 4286 ixgbe_dma_tag_destroy(rxr->ptag); 4287 rxr->ptag = NULL; 4288 } 4289 4290 return; 4291 } 4292 4293 static __inline void 4294 ixgbe_rx_input(struct rx_ring *rxr, struct ifnet *ifp, struct mbuf *m, u32 ptype) 4295 { 4296 int s; 4297 4298 #ifdef LRO 4299 struct adapter *adapter = ifp->if_softc; 4300 struct ethercom *ec = &adapter->osdep.ec; 4301 4302 /* 4303 * ATM LRO is only for IPv4/TCP packets and TCP checksum of the packet 4304 * should be computed by hardware. Also it should not have VLAN tag in 4305 * ethernet header. 4306 */ 4307 if (rxr->lro_enabled && 4308 (ec->ec_capenable & ETHERCAP_VLAN_HWTAGGING) != 0 && 4309 (ptype & IXGBE_RXDADV_PKTTYPE_ETQF) == 0 && 4310 (ptype & (IXGBE_RXDADV_PKTTYPE_IPV4 | IXGBE_RXDADV_PKTTYPE_TCP)) == 4311 (IXGBE_RXDADV_PKTTYPE_IPV4 | IXGBE_RXDADV_PKTTYPE_TCP) && 4312 (m->m_pkthdr.csum_flags & (CSUM_DATA_VALID | CSUM_PSEUDO_HDR)) == 4313 (CSUM_DATA_VALID | CSUM_PSEUDO_HDR)) { 4314 /* 4315 * Send to the stack if: 4316 ** - LRO not enabled, or 4317 ** - no LRO resources, or 4318 ** - lro enqueue fails 4319 */ 4320 if (rxr->lro.lro_cnt != 0) 4321 if (tcp_lro_rx(&rxr->lro, m, 0) == 0) 4322 return; 4323 } 4324 #endif /* LRO */ 4325 4326 IXGBE_RX_UNLOCK(rxr); 4327 4328 s = splnet(); 4329 /* Pass this up to any BPF listeners. */ 4330 bpf_mtap(ifp, m); 4331 (*ifp->if_input)(ifp, m); 4332 splx(s); 4333 4334 IXGBE_RX_LOCK(rxr); 4335 } 4336 4337 static __inline void 4338 ixgbe_rx_discard(struct rx_ring *rxr, int i) 4339 { 4340 struct ixgbe_rx_buf *rbuf; 4341 4342 rbuf = &rxr->rx_buffers[i]; 4343 4344 if (rbuf->fmp != NULL) {/* Partial chain ? */ 4345 rbuf->fmp->m_flags |= M_PKTHDR; 4346 m_freem(rbuf->fmp); 4347 rbuf->fmp = NULL; 4348 } 4349 4350 /* 4351 ** With advanced descriptors the writeback 4352 ** clobbers the buffer addrs, so its easier 4353 ** to just free the existing mbufs and take 4354 ** the normal refresh path to get new buffers 4355 ** and mapping. 4356 */ 4357 if (rbuf->m_head) { 4358 m_free(rbuf->m_head); 4359 rbuf->m_head = NULL; 4360 } 4361 4362 if (rbuf->m_pack) { 4363 m_free(rbuf->m_pack); 4364 rbuf->m_pack = NULL; 4365 } 4366 4367 return; 4368 } 4369 4370 4371 /********************************************************************* 4372 * 4373 * This routine executes in interrupt context. It replenishes 4374 * the mbufs in the descriptor and sends data which has been 4375 * dma'ed into host memory to upper layer. 4376 * 4377 * We loop at most count times if count is > 0, or until done if 4378 * count < 0. 4379 * 4380 * Return TRUE for more work, FALSE for all clean. 4381 *********************************************************************/ 4382 static bool 4383 ixgbe_rxeof(struct ix_queue *que, int count) 4384 { 4385 struct adapter *adapter = que->adapter; 4386 struct rx_ring *rxr = que->rxr; 4387 struct ifnet *ifp = adapter->ifp; 4388 #ifdef LRO 4389 struct lro_ctrl *lro = &rxr->lro; 4390 struct lro_entry *queued; 4391 #endif /* LRO */ 4392 int i, nextp, processed = 0; 4393 u32 staterr = 0; 4394 union ixgbe_adv_rx_desc *cur; 4395 struct ixgbe_rx_buf *rbuf, *nbuf; 4396 4397 IXGBE_RX_LOCK(rxr); 4398 4399 for (i = rxr->next_to_check; count != 0;) { 4400 struct mbuf *sendmp, *mh, *mp; 4401 u32 rsc, ptype; 4402 u16 hlen, plen, hdr, vtag; 4403 bool eop; 4404 4405 /* Sync the ring. */ 4406 ixgbe_dmamap_sync(rxr->rxdma.dma_tag, rxr->rxdma.dma_map, 4407 BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE); 4408 4409 cur = &rxr->rx_base[i]; 4410 staterr = le32toh(cur->wb.upper.status_error); 4411 4412 if ((staterr & IXGBE_RXD_STAT_DD) == 0) 4413 break; 4414 if ((ifp->if_flags & IFF_RUNNING) == 0) 4415 break; 4416 4417 count--; 4418 sendmp = NULL; 4419 nbuf = NULL; 4420 rsc = 0; 4421 cur->wb.upper.status_error = 0; 4422 rbuf = &rxr->rx_buffers[i]; 4423 mh = rbuf->m_head; 4424 mp = rbuf->m_pack; 4425 4426 plen = le16toh(cur->wb.upper.length); 4427 ptype = le32toh(cur->wb.lower.lo_dword.data) & 4428 IXGBE_RXDADV_PKTTYPE_MASK; 4429 hdr = le16toh(cur->wb.lower.lo_dword.hs_rss.hdr_info); 4430 vtag = le16toh(cur->wb.upper.vlan); 4431 eop = ((staterr & IXGBE_RXD_STAT_EOP) != 0); 4432 4433 /* Make sure bad packets are discarded */ 4434 if (((staterr & IXGBE_RXDADV_ERR_FRAME_ERR_MASK) != 0) || 4435 (rxr->discard)) { 4436 ifp->if_ierrors++; 4437 rxr->rx_discarded.ev_count++; 4438 if (eop) 4439 rxr->discard = FALSE; 4440 else 4441 rxr->discard = TRUE; 4442 ixgbe_rx_discard(rxr, i); 4443 goto next_desc; 4444 } 4445 4446 /* 4447 ** On 82599 which supports a hardware 4448 ** LRO (called HW RSC), packets need 4449 ** not be fragmented across sequential 4450 ** descriptors, rather the next descriptor 4451 ** is indicated in bits of the descriptor. 4452 ** This also means that we might proceses 4453 ** more than one packet at a time, something 4454 ** that has never been true before, it 4455 ** required eliminating global chain pointers 4456 ** in favor of what we are doing here. -jfv 4457 */ 4458 if (!eop) { 4459 /* 4460 ** Figure out the next descriptor 4461 ** of this frame. 4462 */ 4463 if (rxr->hw_rsc == TRUE) { 4464 rsc = ixgbe_rsc_count(cur); 4465 rxr->rsc_num += (rsc - 1); 4466 } 4467 if (rsc) { /* Get hardware index */ 4468 nextp = ((staterr & 4469 IXGBE_RXDADV_NEXTP_MASK) >> 4470 IXGBE_RXDADV_NEXTP_SHIFT); 4471 } else { /* Just sequential */ 4472 nextp = i + 1; 4473 if (nextp == adapter->num_rx_desc) 4474 nextp = 0; 4475 } 4476 nbuf = &rxr->rx_buffers[nextp]; 4477 prefetch(nbuf); 4478 } 4479 /* 4480 ** The header mbuf is ONLY used when header 4481 ** split is enabled, otherwise we get normal 4482 ** behavior, ie, both header and payload 4483 ** are DMA'd into the payload buffer. 4484 ** 4485 ** Rather than using the fmp/lmp global pointers 4486 ** we now keep the head of a packet chain in the 4487 ** buffer struct and pass this along from one 4488 ** descriptor to the next, until we get EOP. 4489 */ 4490 if (rxr->hdr_split && (rbuf->fmp == NULL)) { 4491 /* This must be an initial descriptor */ 4492 hlen = (hdr & IXGBE_RXDADV_HDRBUFLEN_MASK) >> 4493 IXGBE_RXDADV_HDRBUFLEN_SHIFT; 4494 if (hlen > IXGBE_RX_HDR) 4495 hlen = IXGBE_RX_HDR; 4496 mh->m_len = hlen; 4497 mh->m_flags |= M_PKTHDR; 4498 mh->m_next = NULL; 4499 mh->m_pkthdr.len = mh->m_len; 4500 /* Null buf pointer so it is refreshed */ 4501 rbuf->m_head = NULL; 4502 /* 4503 ** Check the payload length, this 4504 ** could be zero if its a small 4505 ** packet. 4506 */ 4507 if (plen > 0) { 4508 mp->m_len = plen; 4509 mp->m_next = NULL; 4510 mp->m_flags &= ~M_PKTHDR; 4511 mh->m_next = mp; 4512 mh->m_pkthdr.len += mp->m_len; 4513 /* Null buf pointer so it is refreshed */ 4514 rbuf->m_pack = NULL; 4515 rxr->rx_split_packets.ev_count++; 4516 } 4517 /* 4518 ** Now create the forward 4519 ** chain so when complete 4520 ** we wont have to. 4521 */ 4522 if (eop == 0) { 4523 /* stash the chain head */ 4524 nbuf->fmp = mh; 4525 /* Make forward chain */ 4526 if (plen) 4527 mp->m_next = nbuf->m_pack; 4528 else 4529 mh->m_next = nbuf->m_pack; 4530 } else { 4531 /* Singlet, prepare to send */ 4532 sendmp = mh; 4533 if (VLAN_ATTACHED(&adapter->osdep.ec) && 4534 (staterr & IXGBE_RXD_STAT_VP)) { 4535 /* XXX Do something reasonable on 4536 * error. 4537 */ 4538 #if 0 4539 printf("%s.%d: VLAN_INPUT_TAG\n", 4540 __func__, __LINE__); 4541 Debugger(); 4542 #endif 4543 VLAN_INPUT_TAG(ifp, sendmp, vtag, 4544 printf("%s: could not apply VLAN " 4545 "tag", __func__)); 4546 } 4547 } 4548 } else { 4549 /* 4550 ** Either no header split, or a 4551 ** secondary piece of a fragmented 4552 ** split packet. 4553 */ 4554 mp->m_len = plen; 4555 /* 4556 ** See if there is a stored head 4557 ** that determines what we are 4558 */ 4559 sendmp = rbuf->fmp; 4560 rbuf->m_pack = rbuf->fmp = NULL; 4561 4562 if (sendmp != NULL) /* secondary frag */ 4563 sendmp->m_pkthdr.len += mp->m_len; 4564 else { 4565 /* first desc of a non-ps chain */ 4566 sendmp = mp; 4567 sendmp->m_flags |= M_PKTHDR; 4568 sendmp->m_pkthdr.len = mp->m_len; 4569 if (staterr & IXGBE_RXD_STAT_VP) { 4570 /* XXX Do something reasonable on 4571 * error. 4572 */ 4573 #if 0 4574 printf("%s.%d: VLAN_INPUT_TAG\n", 4575 __func__, __LINE__); 4576 Debugger(); 4577 #endif 4578 VLAN_INPUT_TAG(ifp, sendmp, vtag, 4579 printf("%s: could not apply VLAN " 4580 "tag", __func__)); 4581 } 4582 } 4583 /* Pass the head pointer on */ 4584 if (eop == 0) { 4585 nbuf->fmp = sendmp; 4586 sendmp = NULL; 4587 mp->m_next = nbuf->m_pack; 4588 } 4589 } 4590 ++processed; 4591 /* Sending this frame? */ 4592 if (eop) { 4593 sendmp->m_pkthdr.rcvif = ifp; 4594 ifp->if_ipackets++; 4595 rxr->rx_packets.ev_count++; 4596 /* capture data for AIM */ 4597 rxr->bytes += sendmp->m_pkthdr.len; 4598 rxr->rx_bytes.ev_count += sendmp->m_pkthdr.len; 4599 if ((ifp->if_capenable & IFCAP_RXCSUM) != 0) { 4600 ixgbe_rx_checksum(staterr, sendmp, ptype, 4601 &adapter->stats); 4602 } 4603 #if __FreeBSD_version >= 800000 4604 sendmp->m_pkthdr.flowid = que->msix; 4605 sendmp->m_flags |= M_FLOWID; 4606 #endif 4607 } 4608 next_desc: 4609 ixgbe_dmamap_sync(rxr->rxdma.dma_tag, rxr->rxdma.dma_map, 4610 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); 4611 4612 /* Advance our pointers to the next descriptor. */ 4613 if (++i == adapter->num_rx_desc) 4614 i = 0; 4615 4616 /* Now send to the stack or do LRO */ 4617 if (sendmp != NULL) { 4618 rxr->next_to_check = i; 4619 ixgbe_rx_input(rxr, ifp, sendmp, ptype); 4620 i = rxr->next_to_check; 4621 } 4622 4623 /* Every 8 descriptors we go to refresh mbufs */ 4624 if (processed == 8) { 4625 ixgbe_refresh_mbufs(rxr, i); 4626 processed = 0; 4627 } 4628 } 4629 4630 /* Refresh any remaining buf structs */ 4631 if (ixgbe_rx_unrefreshed(rxr)) 4632 ixgbe_refresh_mbufs(rxr, i); 4633 4634 rxr->next_to_check = i; 4635 4636 #ifdef LRO 4637 /* 4638 * Flush any outstanding LRO work 4639 */ 4640 while ((queued = SLIST_FIRST(&lro->lro_active)) != NULL) { 4641 SLIST_REMOVE_HEAD(&lro->lro_active, next); 4642 tcp_lro_flush(lro, queued); 4643 } 4644 #endif /* LRO */ 4645 4646 IXGBE_RX_UNLOCK(rxr); 4647 4648 /* 4649 ** We still have cleaning to do? 4650 ** Schedule another interrupt if so. 4651 */ 4652 if ((staterr & IXGBE_RXD_STAT_DD) != 0) { 4653 ixgbe_rearm_queues(adapter, (u64)(1ULL << que->msix)); 4654 return true; 4655 } 4656 4657 return false; 4658 } 4659 4660 4661 /********************************************************************* 4662 * 4663 * Verify that the hardware indicated that the checksum is valid. 4664 * Inform the stack about the status of checksum so that stack 4665 * doesn't spend time verifying the checksum. 4666 * 4667 *********************************************************************/ 4668 static void 4669 ixgbe_rx_checksum(u32 staterr, struct mbuf * mp, u32 ptype, 4670 struct ixgbe_hw_stats *stats) 4671 { 4672 u16 status = (u16) staterr; 4673 u8 errors = (u8) (staterr >> 24); 4674 #if 0 4675 bool sctp = FALSE; 4676 4677 if ((ptype & IXGBE_RXDADV_PKTTYPE_ETQF) == 0 && 4678 (ptype & IXGBE_RXDADV_PKTTYPE_SCTP) != 0) 4679 sctp = TRUE; 4680 #endif 4681 4682 if (status & IXGBE_RXD_STAT_IPCS) { 4683 stats->ipcs.ev_count++; 4684 if (!(errors & IXGBE_RXD_ERR_IPE)) { 4685 /* IP Checksum Good */ 4686 mp->m_pkthdr.csum_flags = M_CSUM_IPv4; 4687 4688 } else { 4689 stats->ipcs_bad.ev_count++; 4690 mp->m_pkthdr.csum_flags = M_CSUM_IPv4|M_CSUM_IPv4_BAD; 4691 } 4692 } 4693 if (status & IXGBE_RXD_STAT_L4CS) { 4694 stats->l4cs.ev_count++; 4695 u16 type = M_CSUM_TCPv4|M_CSUM_TCPv6|M_CSUM_UDPv4|M_CSUM_UDPv6; 4696 if (!(errors & IXGBE_RXD_ERR_TCPE)) { 4697 mp->m_pkthdr.csum_flags |= type; 4698 } else { 4699 stats->l4cs_bad.ev_count++; 4700 mp->m_pkthdr.csum_flags |= type | M_CSUM_TCP_UDP_BAD; 4701 } 4702 } 4703 return; 4704 } 4705 4706 4707 #if 0 /* XXX Badly need to overhaul vlan(4) on NetBSD. */ 4708 /* 4709 ** This routine is run via an vlan config EVENT, 4710 ** it enables us to use the HW Filter table since 4711 ** we can get the vlan id. This just creates the 4712 ** entry in the soft version of the VFTA, init will 4713 ** repopulate the real table. 4714 */ 4715 static void 4716 ixgbe_register_vlan(void *arg, struct ifnet *ifp, u16 vtag) 4717 { 4718 struct adapter *adapter = ifp->if_softc; 4719 u16 index, bit; 4720 4721 if (ifp->if_softc != arg) /* Not our event */ 4722 return; 4723 4724 if ((vtag == 0) || (vtag > 4095)) /* Invalid */ 4725 return; 4726 4727 IXGBE_CORE_LOCK(adapter); 4728 index = (vtag >> 5) & 0x7F; 4729 bit = vtag & 0x1F; 4730 adapter->shadow_vfta[index] |= (1 << bit); 4731 ixgbe_init_locked(adapter); 4732 IXGBE_CORE_UNLOCK(adapter); 4733 } 4734 4735 /* 4736 ** This routine is run via an vlan 4737 ** unconfig EVENT, remove our entry 4738 ** in the soft vfta. 4739 */ 4740 static void 4741 ixgbe_unregister_vlan(void *arg, struct ifnet *ifp, u16 vtag) 4742 { 4743 struct adapter *adapter = ifp->if_softc; 4744 u16 index, bit; 4745 4746 if (ifp->if_softc != arg) 4747 return; 4748 4749 if ((vtag == 0) || (vtag > 4095)) /* Invalid */ 4750 return; 4751 4752 IXGBE_CORE_LOCK(adapter); 4753 index = (vtag >> 5) & 0x7F; 4754 bit = vtag & 0x1F; 4755 adapter->shadow_vfta[index] &= ~(1 << bit); 4756 /* Re-init to load the changes */ 4757 ixgbe_init_locked(adapter); 4758 IXGBE_CORE_UNLOCK(adapter); 4759 } 4760 #endif 4761 4762 static void 4763 ixgbe_setup_vlan_hw_support(struct adapter *adapter) 4764 { 4765 struct ethercom *ec = &adapter->osdep.ec; 4766 struct ixgbe_hw *hw = &adapter->hw; 4767 u32 ctrl; 4768 4769 /* 4770 ** We get here thru init_locked, meaning 4771 ** a soft reset, this has already cleared 4772 ** the VFTA and other state, so if there 4773 ** have been no vlan's registered do nothing. 4774 */ 4775 if (!VLAN_ATTACHED(&adapter->osdep.ec)) { 4776 return; 4777 } 4778 4779 /* 4780 ** A soft reset zero's out the VFTA, so 4781 ** we need to repopulate it now. 4782 */ 4783 for (int i = 0; i < IXGBE_VFTA_SIZE; i++) 4784 if (adapter->shadow_vfta[i] != 0) 4785 IXGBE_WRITE_REG(hw, IXGBE_VFTA(i), 4786 adapter->shadow_vfta[i]); 4787 4788 ctrl = IXGBE_READ_REG(hw, IXGBE_VLNCTRL); 4789 /* Enable the Filter Table if enabled */ 4790 if (ec->ec_capenable & ETHERCAP_VLAN_HWFILTER) { 4791 ctrl &= ~IXGBE_VLNCTRL_CFIEN; 4792 ctrl |= IXGBE_VLNCTRL_VFE; 4793 } 4794 if (hw->mac.type == ixgbe_mac_82598EB) 4795 ctrl |= IXGBE_VLNCTRL_VME; 4796 IXGBE_WRITE_REG(hw, IXGBE_VLNCTRL, ctrl); 4797 4798 /* On 82599 the VLAN enable is per/queue in RXDCTL */ 4799 if (hw->mac.type != ixgbe_mac_82598EB) 4800 for (int i = 0; i < adapter->num_queues; i++) { 4801 ctrl = IXGBE_READ_REG(hw, IXGBE_RXDCTL(i)); 4802 ctrl |= IXGBE_RXDCTL_VME; 4803 IXGBE_WRITE_REG(hw, IXGBE_RXDCTL(i), ctrl); 4804 } 4805 } 4806 4807 static void 4808 ixgbe_enable_intr(struct adapter *adapter) 4809 { 4810 struct ixgbe_hw *hw = &adapter->hw; 4811 struct ix_queue *que = adapter->queues; 4812 u32 mask = (IXGBE_EIMS_ENABLE_MASK & ~IXGBE_EIMS_RTX_QUEUE); 4813 4814 4815 /* Enable Fan Failure detection */ 4816 if (hw->device_id == IXGBE_DEV_ID_82598AT) 4817 mask |= IXGBE_EIMS_GPI_SDP1; 4818 else { 4819 mask |= IXGBE_EIMS_ECC; 4820 mask |= IXGBE_EIMS_GPI_SDP1; 4821 mask |= IXGBE_EIMS_GPI_SDP2; 4822 #ifdef IXGBE_FDIR 4823 mask |= IXGBE_EIMS_FLOW_DIR; 4824 #endif 4825 } 4826 4827 IXGBE_WRITE_REG(hw, IXGBE_EIMS, mask); 4828 4829 /* With RSS we use auto clear */ 4830 if (adapter->msix_mem) { 4831 mask = IXGBE_EIMS_ENABLE_MASK; 4832 /* Don't autoclear Link */ 4833 mask &= ~IXGBE_EIMS_OTHER; 4834 mask &= ~IXGBE_EIMS_LSC; 4835 IXGBE_WRITE_REG(hw, IXGBE_EIAC, mask); 4836 } 4837 4838 /* 4839 ** Now enable all queues, this is done separately to 4840 ** allow for handling the extended (beyond 32) MSIX 4841 ** vectors that can be used by 82599 4842 */ 4843 for (int i = 0; i < adapter->num_queues; i++, que++) 4844 ixgbe_enable_queue(adapter, que->msix); 4845 4846 IXGBE_WRITE_FLUSH(hw); 4847 4848 return; 4849 } 4850 4851 static void 4852 ixgbe_disable_intr(struct adapter *adapter) 4853 { 4854 if (adapter->msix_mem) 4855 IXGBE_WRITE_REG(&adapter->hw, IXGBE_EIAC, 0); 4856 if (adapter->hw.mac.type == ixgbe_mac_82598EB) { 4857 IXGBE_WRITE_REG(&adapter->hw, IXGBE_EIMC, ~0); 4858 } else { 4859 IXGBE_WRITE_REG(&adapter->hw, IXGBE_EIMC, 0xFFFF0000); 4860 IXGBE_WRITE_REG(&adapter->hw, IXGBE_EIMC_EX(0), ~0); 4861 IXGBE_WRITE_REG(&adapter->hw, IXGBE_EIMC_EX(1), ~0); 4862 } 4863 IXGBE_WRITE_FLUSH(&adapter->hw); 4864 return; 4865 } 4866 4867 u16 4868 ixgbe_read_pci_cfg(struct ixgbe_hw *hw, u32 reg) 4869 { 4870 switch (reg % 4) { 4871 case 0: 4872 return pci_conf_read(hw->back->pc, hw->back->tag, reg) & 4873 __BITS(15, 0); 4874 case 2: 4875 return __SHIFTOUT(pci_conf_read(hw->back->pc, hw->back->tag, 4876 reg - 2), __BITS(31, 16)); 4877 default: 4878 panic("%s: invalid register (%" PRIx32, __func__, reg); 4879 break; 4880 } 4881 } 4882 4883 void 4884 ixgbe_write_pci_cfg(struct ixgbe_hw *hw, u32 reg, u16 value) 4885 { 4886 pcireg_t old; 4887 4888 switch (reg % 4) { 4889 case 0: 4890 old = pci_conf_read(hw->back->pc, hw->back->tag, reg) & 4891 __BITS(31, 16); 4892 pci_conf_write(hw->back->pc, hw->back->tag, reg, value | old); 4893 break; 4894 case 2: 4895 old = pci_conf_read(hw->back->pc, hw->back->tag, reg - 2) & 4896 __BITS(15, 0); 4897 pci_conf_write(hw->back->pc, hw->back->tag, reg - 2, 4898 __SHIFTIN(value, __BITS(31, 16)) | old); 4899 break; 4900 default: 4901 panic("%s: invalid register (%" PRIx32, __func__, reg); 4902 break; 4903 } 4904 4905 return; 4906 } 4907 4908 /* 4909 ** Setup the correct IVAR register for a particular MSIX interrupt 4910 ** (yes this is all very magic and confusing :) 4911 ** - entry is the register array entry 4912 ** - vector is the MSIX vector for this queue 4913 ** - type is RX/TX/MISC 4914 */ 4915 static void 4916 ixgbe_set_ivar(struct adapter *adapter, u8 entry, u8 vector, s8 type) 4917 { 4918 struct ixgbe_hw *hw = &adapter->hw; 4919 u32 ivar, index; 4920 4921 vector |= IXGBE_IVAR_ALLOC_VAL; 4922 4923 switch (hw->mac.type) { 4924 4925 case ixgbe_mac_82598EB: 4926 if (type == -1) 4927 entry = IXGBE_IVAR_OTHER_CAUSES_INDEX; 4928 else 4929 entry += (type * 64); 4930 index = (entry >> 2) & 0x1F; 4931 ivar = IXGBE_READ_REG(hw, IXGBE_IVAR(index)); 4932 ivar &= ~(0xFF << (8 * (entry & 0x3))); 4933 ivar |= (vector << (8 * (entry & 0x3))); 4934 IXGBE_WRITE_REG(&adapter->hw, IXGBE_IVAR(index), ivar); 4935 break; 4936 4937 case ixgbe_mac_82599EB: 4938 if (type == -1) { /* MISC IVAR */ 4939 index = (entry & 1) * 8; 4940 ivar = IXGBE_READ_REG(hw, IXGBE_IVAR_MISC); 4941 ivar &= ~(0xFF << index); 4942 ivar |= (vector << index); 4943 IXGBE_WRITE_REG(hw, IXGBE_IVAR_MISC, ivar); 4944 } else { /* RX/TX IVARS */ 4945 index = (16 * (entry & 1)) + (8 * type); 4946 ivar = IXGBE_READ_REG(hw, IXGBE_IVAR(entry >> 1)); 4947 ivar &= ~(0xFF << index); 4948 ivar |= (vector << index); 4949 IXGBE_WRITE_REG(hw, IXGBE_IVAR(entry >> 1), ivar); 4950 } 4951 4952 default: 4953 break; 4954 } 4955 } 4956 4957 static void 4958 ixgbe_configure_ivars(struct adapter *adapter) 4959 { 4960 struct ix_queue *que = adapter->queues; 4961 u32 newitr; 4962 4963 if (ixgbe_max_interrupt_rate > 0) 4964 newitr = (8000000 / ixgbe_max_interrupt_rate) & 0x0FF8; 4965 else 4966 newitr = 0; 4967 4968 for (int i = 0; i < adapter->num_queues; i++, que++) { 4969 /* First the RX queue entry */ 4970 ixgbe_set_ivar(adapter, i, que->msix, 0); 4971 /* ... and the TX */ 4972 ixgbe_set_ivar(adapter, i, que->msix, 1); 4973 /* Set an Initial EITR value */ 4974 IXGBE_WRITE_REG(&adapter->hw, 4975 IXGBE_EITR(que->msix), newitr); 4976 } 4977 4978 /* For the Link interrupt */ 4979 ixgbe_set_ivar(adapter, 1, adapter->linkvec, -1); 4980 } 4981 4982 /* 4983 ** ixgbe_sfp_probe - called in the local timer to 4984 ** determine if a port had optics inserted. 4985 */ 4986 static bool ixgbe_sfp_probe(struct adapter *adapter) 4987 { 4988 struct ixgbe_hw *hw = &adapter->hw; 4989 device_t dev = adapter->dev; 4990 bool result = FALSE; 4991 4992 if ((hw->phy.type == ixgbe_phy_nl) && 4993 (hw->phy.sfp_type == ixgbe_sfp_type_not_present)) { 4994 s32 ret = hw->phy.ops.identify_sfp(hw); 4995 if (ret) 4996 goto out; 4997 ret = hw->phy.ops.reset(hw); 4998 if (ret == IXGBE_ERR_SFP_NOT_SUPPORTED) { 4999 device_printf(dev,"Unsupported SFP+ module detected!"); 5000 device_printf(dev, "Reload driver with supported module.\n"); 5001 adapter->sfp_probe = FALSE; 5002 goto out; 5003 } else 5004 device_printf(dev,"SFP+ module detected!\n"); 5005 /* We now have supported optics */ 5006 adapter->sfp_probe = FALSE; 5007 /* Set the optics type so system reports correctly */ 5008 ixgbe_setup_optics(adapter); 5009 result = TRUE; 5010 } 5011 out: 5012 return (result); 5013 } 5014 5015 /* 5016 ** Tasklet handler for MSIX Link interrupts 5017 ** - do outside interrupt since it might sleep 5018 */ 5019 static void 5020 ixgbe_handle_link(void *context) 5021 { 5022 struct adapter *adapter = context; 5023 5024 if (ixgbe_check_link(&adapter->hw, 5025 &adapter->link_speed, &adapter->link_up, 0) == 0) 5026 ixgbe_update_link_status(adapter); 5027 } 5028 5029 /* 5030 ** Tasklet for handling SFP module interrupts 5031 */ 5032 static void 5033 ixgbe_handle_mod(void *context) 5034 { 5035 struct adapter *adapter = context; 5036 struct ixgbe_hw *hw = &adapter->hw; 5037 device_t dev = adapter->dev; 5038 u32 err; 5039 5040 err = hw->phy.ops.identify_sfp(hw); 5041 if (err == IXGBE_ERR_SFP_NOT_SUPPORTED) { 5042 device_printf(dev, 5043 "Unsupported SFP+ module type was detected.\n"); 5044 return; 5045 } 5046 err = hw->mac.ops.setup_sfp(hw); 5047 if (err == IXGBE_ERR_SFP_NOT_SUPPORTED) { 5048 device_printf(dev, 5049 "Setup failure - unsupported SFP+ module type.\n"); 5050 return; 5051 } 5052 softint_schedule(adapter->msf_si); 5053 return; 5054 } 5055 5056 5057 /* 5058 ** Tasklet for handling MSF (multispeed fiber) interrupts 5059 */ 5060 static void 5061 ixgbe_handle_msf(void *context) 5062 { 5063 struct adapter *adapter = context; 5064 struct ixgbe_hw *hw = &adapter->hw; 5065 u32 autoneg; 5066 bool negotiate; 5067 5068 autoneg = hw->phy.autoneg_advertised; 5069 if ((!autoneg) && (hw->mac.ops.get_link_capabilities)) 5070 hw->mac.ops.get_link_capabilities(hw, &autoneg, &negotiate); 5071 else 5072 negotiate = 0; 5073 if (hw->mac.ops.setup_link) 5074 hw->mac.ops.setup_link(hw, autoneg, negotiate, TRUE); 5075 return; 5076 } 5077 5078 #ifdef IXGBE_FDIR 5079 /* 5080 ** Tasklet for reinitializing the Flow Director filter table 5081 */ 5082 static void 5083 ixgbe_reinit_fdir(void *context) 5084 { 5085 struct adapter *adapter = context; 5086 struct ifnet *ifp = adapter->ifp; 5087 5088 if (adapter->fdir_reinit != 1) /* Shouldn't happen */ 5089 return; 5090 ixgbe_reinit_fdir_tables_82599(&adapter->hw); 5091 adapter->fdir_reinit = 0; 5092 /* Restart the interface */ 5093 ifp->if_flags |= IFF_RUNNING; 5094 return; 5095 } 5096 #endif 5097 5098 /********************************************************************** 5099 * 5100 * Update the board statistics counters. 5101 * 5102 **********************************************************************/ 5103 static void 5104 ixgbe_update_stats_counters(struct adapter *adapter) 5105 { 5106 struct ifnet *ifp = adapter->ifp; 5107 struct ixgbe_hw *hw = &adapter->hw; 5108 u32 missed_rx = 0, bprc, lxon, lxoff, total; 5109 u64 total_missed_rx = 0; 5110 5111 adapter->stats.crcerrs.ev_count += IXGBE_READ_REG(hw, IXGBE_CRCERRS); 5112 adapter->stats.illerrc.ev_count += IXGBE_READ_REG(hw, IXGBE_ILLERRC); 5113 adapter->stats.errbc.ev_count += IXGBE_READ_REG(hw, IXGBE_ERRBC); 5114 adapter->stats.mspdc.ev_count += IXGBE_READ_REG(hw, IXGBE_MSPDC); 5115 5116 for (int i = 0; i < __arraycount(adapter->stats.mpc); i++) { 5117 int j = i % adapter->num_queues; 5118 u32 mp; 5119 mp = IXGBE_READ_REG(hw, IXGBE_MPC(i)); 5120 /* missed_rx tallies misses for the gprc workaround */ 5121 missed_rx += mp; 5122 /* global total per queue */ 5123 adapter->stats.mpc[j].ev_count += mp; 5124 /* Running comprehensive total for stats display */ 5125 total_missed_rx += adapter->stats.mpc[j].ev_count; 5126 if (hw->mac.type == ixgbe_mac_82598EB) 5127 adapter->stats.rnbc[j] += 5128 IXGBE_READ_REG(hw, IXGBE_RNBC(i)); 5129 adapter->stats.pxontxc[j].ev_count += 5130 IXGBE_READ_REG(hw, IXGBE_PXONTXC(i)); 5131 adapter->stats.pxonrxc[j].ev_count += 5132 IXGBE_READ_REG(hw, IXGBE_PXONRXC(i)); 5133 adapter->stats.pxofftxc[j].ev_count += 5134 IXGBE_READ_REG(hw, IXGBE_PXOFFTXC(i)); 5135 adapter->stats.pxoffrxc[j].ev_count += 5136 IXGBE_READ_REG(hw, IXGBE_PXOFFRXC(i)); 5137 adapter->stats.pxon2offc[j].ev_count += 5138 IXGBE_READ_REG(hw, IXGBE_PXON2OFFCNT(i)); 5139 } 5140 for (int i = 0; i < __arraycount(adapter->stats.qprc); i++) { 5141 int j = i % adapter->num_queues; 5142 adapter->stats.qprc[j].ev_count += IXGBE_READ_REG(hw, IXGBE_QPRC(i)); 5143 adapter->stats.qptc[j].ev_count += IXGBE_READ_REG(hw, IXGBE_QPTC(i)); 5144 adapter->stats.qbrc[j].ev_count += IXGBE_READ_REG(hw, IXGBE_QBRC(i)); 5145 adapter->stats.qbrc[j].ev_count += 5146 ((u64)IXGBE_READ_REG(hw, IXGBE_QBRC(i)) << 32); 5147 adapter->stats.qbtc[j].ev_count += IXGBE_READ_REG(hw, IXGBE_QBTC(i)); 5148 adapter->stats.qbtc[j].ev_count += 5149 ((u64)IXGBE_READ_REG(hw, IXGBE_QBTC(i)) << 32); 5150 adapter->stats.qprdc[j].ev_count += IXGBE_READ_REG(hw, IXGBE_QPRDC(i)); 5151 } 5152 adapter->stats.mlfc.ev_count += IXGBE_READ_REG(hw, IXGBE_MLFC); 5153 adapter->stats.mrfc.ev_count += IXGBE_READ_REG(hw, IXGBE_MRFC); 5154 adapter->stats.rlec.ev_count += IXGBE_READ_REG(hw, IXGBE_RLEC); 5155 5156 /* Hardware workaround, gprc counts missed packets */ 5157 adapter->stats.gprc.ev_count += IXGBE_READ_REG(hw, IXGBE_GPRC) - missed_rx; 5158 5159 lxon = IXGBE_READ_REG(hw, IXGBE_LXONTXC); 5160 adapter->stats.lxontxc.ev_count += lxon; 5161 lxoff = IXGBE_READ_REG(hw, IXGBE_LXOFFTXC); 5162 adapter->stats.lxofftxc.ev_count += lxoff; 5163 total = lxon + lxoff; 5164 5165 if (hw->mac.type != ixgbe_mac_82598EB) { 5166 adapter->stats.gorc.ev_count += IXGBE_READ_REG(hw, IXGBE_GORCL) + 5167 ((u64)IXGBE_READ_REG(hw, IXGBE_GORCH) << 32); 5168 adapter->stats.gotc.ev_count += IXGBE_READ_REG(hw, IXGBE_GOTCL) + 5169 ((u64)IXGBE_READ_REG(hw, IXGBE_GOTCH) << 32) - total * ETHER_MIN_LEN; 5170 adapter->stats.tor.ev_count += IXGBE_READ_REG(hw, IXGBE_TORL) + 5171 ((u64)IXGBE_READ_REG(hw, IXGBE_TORH) << 32); 5172 adapter->stats.lxonrxc.ev_count += IXGBE_READ_REG(hw, IXGBE_LXONRXCNT); 5173 adapter->stats.lxoffrxc.ev_count += IXGBE_READ_REG(hw, IXGBE_LXOFFRXCNT); 5174 } else { 5175 adapter->stats.lxonrxc.ev_count += IXGBE_READ_REG(hw, IXGBE_LXONRXC); 5176 adapter->stats.lxoffrxc.ev_count += IXGBE_READ_REG(hw, IXGBE_LXOFFRXC); 5177 /* 82598 only has a counter in the high register */ 5178 adapter->stats.gorc.ev_count += IXGBE_READ_REG(hw, IXGBE_GORCH); 5179 adapter->stats.gotc.ev_count += IXGBE_READ_REG(hw, IXGBE_GOTCH) - total * ETHER_MIN_LEN; 5180 adapter->stats.tor.ev_count += IXGBE_READ_REG(hw, IXGBE_TORH); 5181 } 5182 5183 /* 5184 * Workaround: mprc hardware is incorrectly counting 5185 * broadcasts, so for now we subtract those. 5186 */ 5187 bprc = IXGBE_READ_REG(hw, IXGBE_BPRC); 5188 adapter->stats.bprc.ev_count += bprc; 5189 adapter->stats.mprc.ev_count += IXGBE_READ_REG(hw, IXGBE_MPRC) - ((hw->mac.type == ixgbe_mac_82598EB) ? bprc : 0); 5190 5191 adapter->stats.prc64.ev_count += IXGBE_READ_REG(hw, IXGBE_PRC64); 5192 adapter->stats.prc127.ev_count += IXGBE_READ_REG(hw, IXGBE_PRC127); 5193 adapter->stats.prc255.ev_count += IXGBE_READ_REG(hw, IXGBE_PRC255); 5194 adapter->stats.prc511.ev_count += IXGBE_READ_REG(hw, IXGBE_PRC511); 5195 adapter->stats.prc1023.ev_count += IXGBE_READ_REG(hw, IXGBE_PRC1023); 5196 adapter->stats.prc1522.ev_count += IXGBE_READ_REG(hw, IXGBE_PRC1522); 5197 5198 adapter->stats.gptc.ev_count += IXGBE_READ_REG(hw, IXGBE_GPTC) - total; 5199 adapter->stats.mptc.ev_count += IXGBE_READ_REG(hw, IXGBE_MPTC) - total; 5200 adapter->stats.ptc64.ev_count += IXGBE_READ_REG(hw, IXGBE_PTC64) - total; 5201 5202 adapter->stats.ruc.ev_count += IXGBE_READ_REG(hw, IXGBE_RUC); 5203 adapter->stats.rfc.ev_count += IXGBE_READ_REG(hw, IXGBE_RFC); 5204 adapter->stats.roc.ev_count += IXGBE_READ_REG(hw, IXGBE_ROC); 5205 adapter->stats.rjc.ev_count += IXGBE_READ_REG(hw, IXGBE_RJC); 5206 adapter->stats.mngprc.ev_count += IXGBE_READ_REG(hw, IXGBE_MNGPRC); 5207 adapter->stats.mngpdc.ev_count += IXGBE_READ_REG(hw, IXGBE_MNGPDC); 5208 adapter->stats.mngptc.ev_count += IXGBE_READ_REG(hw, IXGBE_MNGPTC); 5209 adapter->stats.tpr.ev_count += IXGBE_READ_REG(hw, IXGBE_TPR); 5210 adapter->stats.tpt.ev_count += IXGBE_READ_REG(hw, IXGBE_TPT); 5211 adapter->stats.ptc127.ev_count += IXGBE_READ_REG(hw, IXGBE_PTC127); 5212 adapter->stats.ptc255.ev_count += IXGBE_READ_REG(hw, IXGBE_PTC255); 5213 adapter->stats.ptc511.ev_count += IXGBE_READ_REG(hw, IXGBE_PTC511); 5214 adapter->stats.ptc1023.ev_count += IXGBE_READ_REG(hw, IXGBE_PTC1023); 5215 adapter->stats.ptc1522.ev_count += IXGBE_READ_REG(hw, IXGBE_PTC1522); 5216 adapter->stats.bptc.ev_count += IXGBE_READ_REG(hw, IXGBE_BPTC); 5217 adapter->stats.xec.ev_count += IXGBE_READ_REG(hw, IXGBE_XEC); 5218 adapter->stats.fccrc.ev_count += IXGBE_READ_REG(hw, IXGBE_FCCRC); 5219 adapter->stats.fclast.ev_count += IXGBE_READ_REG(hw, IXGBE_FCLAST); 5220 5221 /* Only read FCOE on 82599 */ 5222 if (hw->mac.type != ixgbe_mac_82598EB) { 5223 adapter->stats.fcoerpdc.ev_count += 5224 IXGBE_READ_REG(hw, IXGBE_FCOERPDC); 5225 adapter->stats.fcoeprc.ev_count += 5226 IXGBE_READ_REG(hw, IXGBE_FCOEPRC); 5227 adapter->stats.fcoeptc.ev_count += 5228 IXGBE_READ_REG(hw, IXGBE_FCOEPTC); 5229 adapter->stats.fcoedwrc.ev_count += 5230 IXGBE_READ_REG(hw, IXGBE_FCOEDWRC); 5231 adapter->stats.fcoedwtc.ev_count += 5232 IXGBE_READ_REG(hw, IXGBE_FCOEDWTC); 5233 } 5234 5235 /* Fill out the OS statistics structure */ 5236 ifp->if_ipackets = adapter->stats.gprc.ev_count; 5237 ifp->if_opackets = adapter->stats.gptc.ev_count; 5238 ifp->if_ibytes = adapter->stats.gorc.ev_count; 5239 ifp->if_obytes = adapter->stats.gotc.ev_count; 5240 ifp->if_imcasts = adapter->stats.mprc.ev_count; 5241 ifp->if_collisions = 0; 5242 5243 /* Rx Errors */ 5244 ifp->if_ierrors = total_missed_rx + adapter->stats.crcerrs.ev_count + 5245 adapter->stats.rlec.ev_count; 5246 } 5247 5248 /** ixgbe_sysctl_tdh_handler - Handler function 5249 * Retrieves the TDH value from the hardware 5250 */ 5251 static int 5252 ixgbe_sysctl_tdh_handler(SYSCTLFN_ARGS) 5253 { 5254 struct sysctlnode node; 5255 uint32_t val; 5256 struct tx_ring *txr; 5257 5258 node = *rnode; 5259 txr = (struct tx_ring *)node.sysctl_data; 5260 if (txr == NULL) 5261 return 0; 5262 val = IXGBE_READ_REG(&txr->adapter->hw, IXGBE_TDH(txr->me)); 5263 node.sysctl_data = &val; 5264 return sysctl_lookup(SYSCTLFN_CALL(&node)); 5265 } 5266 5267 /** ixgbe_sysctl_tdt_handler - Handler function 5268 * Retrieves the TDT value from the hardware 5269 */ 5270 static int 5271 ixgbe_sysctl_tdt_handler(SYSCTLFN_ARGS) 5272 { 5273 struct sysctlnode node; 5274 uint32_t val; 5275 struct tx_ring *txr; 5276 5277 node = *rnode; 5278 txr = (struct tx_ring *)node.sysctl_data; 5279 if (txr == NULL) 5280 return 0; 5281 val = IXGBE_READ_REG(&txr->adapter->hw, IXGBE_TDT(txr->me)); 5282 node.sysctl_data = &val; 5283 return sysctl_lookup(SYSCTLFN_CALL(&node)); 5284 } 5285 5286 /** ixgbe_sysctl_rdh_handler - Handler function 5287 * Retrieves the RDH value from the hardware 5288 */ 5289 static int 5290 ixgbe_sysctl_rdh_handler(SYSCTLFN_ARGS) 5291 { 5292 struct sysctlnode node; 5293 uint32_t val; 5294 struct rx_ring *rxr; 5295 5296 node = *rnode; 5297 rxr = (struct rx_ring *)node.sysctl_data; 5298 if (rxr == NULL) 5299 return 0; 5300 val = IXGBE_READ_REG(&rxr->adapter->hw, IXGBE_RDH(rxr->me)); 5301 node.sysctl_data = &val; 5302 return sysctl_lookup(SYSCTLFN_CALL(&node)); 5303 } 5304 5305 /** ixgbe_sysctl_rdt_handler - Handler function 5306 * Retrieves the RDT value from the hardware 5307 */ 5308 static int 5309 ixgbe_sysctl_rdt_handler(SYSCTLFN_ARGS) 5310 { 5311 struct sysctlnode node; 5312 uint32_t val; 5313 struct rx_ring *rxr; 5314 5315 node = *rnode; 5316 rxr = (struct rx_ring *)node.sysctl_data; 5317 if (rxr == NULL) 5318 return 0; 5319 val = IXGBE_READ_REG(&rxr->adapter->hw, IXGBE_RDT(rxr->me)); 5320 node.sysctl_data = &val; 5321 return sysctl_lookup(SYSCTLFN_CALL(&node)); 5322 } 5323 5324 static int 5325 ixgbe_sysctl_interrupt_rate_handler(SYSCTLFN_ARGS) 5326 { 5327 struct sysctlnode node; 5328 struct ix_queue *que; 5329 uint32_t reg, usec, rate; 5330 5331 node = *rnode; 5332 que = (struct ix_queue *)node.sysctl_data; 5333 if (que == NULL) 5334 return 0; 5335 reg = IXGBE_READ_REG(&que->adapter->hw, IXGBE_EITR(que->msix)); 5336 usec = ((reg & 0x0FF8) >> 3); 5337 if (usec > 0) 5338 rate = 1000000 / usec; 5339 else 5340 rate = 0; 5341 node.sysctl_data = &rate; 5342 return sysctl_lookup(SYSCTLFN_CALL(&node)); 5343 } 5344 5345 const struct sysctlnode * 5346 ixgbe_sysctl_instance(struct adapter *adapter) 5347 { 5348 const char *dvname; 5349 struct sysctllog **log; 5350 int rc; 5351 const struct sysctlnode *rnode; 5352 5353 log = &adapter->sysctllog; 5354 dvname = device_xname(adapter->dev); 5355 5356 if ((rc = sysctl_createv(log, 0, NULL, &rnode, 5357 0, CTLTYPE_NODE, dvname, 5358 SYSCTL_DESCR("ixgbe information and settings"), 5359 NULL, 0, NULL, 0, CTL_HW, CTL_CREATE, CTL_EOL)) != 0) 5360 goto err; 5361 5362 return rnode; 5363 err: 5364 printf("%s: sysctl_createv failed, rc = %d\n", __func__, rc); 5365 return NULL; 5366 } 5367 5368 /* 5369 * Add sysctl variables, one per statistic, to the system. 5370 */ 5371 static void 5372 ixgbe_add_hw_stats(struct adapter *adapter) 5373 { 5374 device_t dev = adapter->dev; 5375 const struct sysctlnode *rnode, *cnode; 5376 struct sysctllog **log = &adapter->sysctllog; 5377 struct tx_ring *txr = adapter->tx_rings; 5378 struct rx_ring *rxr = adapter->rx_rings; 5379 struct ixgbe_hw *hw = &adapter->hw; 5380 5381 struct ixgbe_hw_stats *stats = &adapter->stats; 5382 5383 /* Driver Statistics */ 5384 #if 0 5385 /* These counters are not updated by the software */ 5386 SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "dropped", 5387 CTLFLAG_RD, &adapter->dropped_pkts, 5388 "Driver dropped packets"); 5389 SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "mbuf_header_failed", 5390 CTLFLAG_RD, &adapter->mbuf_header_failed, 5391 "???"); 5392 SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "mbuf_packet_failed", 5393 CTLFLAG_RD, &adapter->mbuf_packet_failed, 5394 "???"); 5395 SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "no_tx_map_avail", 5396 CTLFLAG_RD, &adapter->no_tx_map_avail, 5397 "???"); 5398 #endif 5399 evcnt_attach_dynamic(&adapter->handleq, EVCNT_TYPE_MISC, 5400 NULL, device_xname(dev), "Handled queue in softint"); 5401 evcnt_attach_dynamic(&adapter->req, EVCNT_TYPE_MISC, 5402 NULL, device_xname(dev), "Requeued in softint"); 5403 evcnt_attach_dynamic(&adapter->morerx, EVCNT_TYPE_MISC, 5404 NULL, device_xname(dev), "Interrupt handler more rx"); 5405 evcnt_attach_dynamic(&adapter->moretx, EVCNT_TYPE_MISC, 5406 NULL, device_xname(dev), "Interrupt handler more tx"); 5407 evcnt_attach_dynamic(&adapter->txloops, EVCNT_TYPE_MISC, 5408 NULL, device_xname(dev), "Interrupt handler tx loops"); 5409 evcnt_attach_dynamic(&adapter->efbig_tx_dma_setup, EVCNT_TYPE_MISC, 5410 NULL, device_xname(dev), "Driver tx dma soft fail EFBIG"); 5411 evcnt_attach_dynamic(&adapter->m_defrag_failed, EVCNT_TYPE_MISC, 5412 NULL, device_xname(dev), "m_defrag() failed"); 5413 evcnt_attach_dynamic(&adapter->efbig2_tx_dma_setup, EVCNT_TYPE_MISC, 5414 NULL, device_xname(dev), "Driver tx dma hard fail EFBIG"); 5415 evcnt_attach_dynamic(&adapter->einval_tx_dma_setup, EVCNT_TYPE_MISC, 5416 NULL, device_xname(dev), "Driver tx dma hard fail EINVAL"); 5417 evcnt_attach_dynamic(&adapter->other_tx_dma_setup, EVCNT_TYPE_MISC, 5418 NULL, device_xname(dev), "Driver tx dma hard fail other"); 5419 evcnt_attach_dynamic(&adapter->eagain_tx_dma_setup, EVCNT_TYPE_MISC, 5420 NULL, device_xname(dev), "Driver tx dma soft fail EAGAIN"); 5421 evcnt_attach_dynamic(&adapter->enomem_tx_dma_setup, EVCNT_TYPE_MISC, 5422 NULL, device_xname(dev), "Driver tx dma soft fail ENOMEM"); 5423 evcnt_attach_dynamic(&adapter->watchdog_events, EVCNT_TYPE_MISC, 5424 NULL, device_xname(dev), "Watchdog timeouts"); 5425 evcnt_attach_dynamic(&adapter->tso_err, EVCNT_TYPE_MISC, 5426 NULL, device_xname(dev), "TSO errors"); 5427 evcnt_attach_dynamic(&adapter->tso_tx, EVCNT_TYPE_MISC, 5428 NULL, device_xname(dev), "TSO"); 5429 evcnt_attach_dynamic(&adapter->link_irq, EVCNT_TYPE_MISC, 5430 NULL, device_xname(dev), "Link MSIX IRQ Handled"); 5431 5432 for (int i = 0; i < adapter->num_queues; i++, rxr++, txr++) { 5433 snprintf(adapter->queues[i].evnamebuf, 5434 sizeof(adapter->queues[i].evnamebuf), "%s queue%d", 5435 device_xname(dev), i); 5436 snprintf(adapter->queues[i].namebuf, 5437 sizeof(adapter->queues[i].namebuf), "queue%d", i); 5438 5439 if ((rnode = ixgbe_sysctl_instance(adapter)) == NULL) { 5440 aprint_error_dev(dev, "could not create sysctl root\n"); 5441 break; 5442 } 5443 5444 if (sysctl_createv(log, 0, &rnode, &rnode, 5445 0, CTLTYPE_NODE, 5446 adapter->queues[i].namebuf, SYSCTL_DESCR("Queue Name"), 5447 NULL, 0, NULL, 0, CTL_CREATE, CTL_EOL) != 0) 5448 break; 5449 5450 if (sysctl_createv(log, 0, &rnode, &cnode, 5451 CTLFLAG_READONLY, CTLTYPE_INT, 5452 "interrupt_rate", SYSCTL_DESCR("Interrupt Rate"), 5453 ixgbe_sysctl_interrupt_rate_handler, 0, 5454 (void *)&adapter->queues[i], 0, CTL_CREATE, CTL_EOL) != 0) 5455 break; 5456 5457 if (sysctl_createv(log, 0, &rnode, &cnode, 5458 CTLFLAG_READONLY, CTLTYPE_INT, 5459 "txd_head", SYSCTL_DESCR("Transmit Descriptor Head"), 5460 ixgbe_sysctl_tdh_handler, 0, (void *)txr, 5461 0, CTL_CREATE, CTL_EOL) != 0) 5462 break; 5463 5464 if (sysctl_createv(log, 0, &rnode, &cnode, 5465 CTLFLAG_READONLY, CTLTYPE_INT, 5466 "txd_tail", SYSCTL_DESCR("Transmit Descriptor Tail"), 5467 ixgbe_sysctl_tdt_handler, 0, (void *)txr, 5468 0, CTL_CREATE, CTL_EOL) != 0) 5469 break; 5470 5471 evcnt_attach_dynamic(&txr->no_desc_avail, EVCNT_TYPE_MISC, 5472 NULL, adapter->queues[i].evnamebuf, 5473 "Queue No Descriptor Available"); 5474 evcnt_attach_dynamic(&txr->total_packets, EVCNT_TYPE_MISC, 5475 NULL, adapter->queues[i].evnamebuf, 5476 "Queue Packets Transmitted"); 5477 5478 #ifdef LRO 5479 struct lro_ctrl *lro = &rxr->lro; 5480 #endif /* LRO */ 5481 5482 if (sysctl_createv(log, 0, &rnode, &cnode, 5483 CTLFLAG_READONLY, 5484 CTLTYPE_INT, 5485 "rxd_head", SYSCTL_DESCR("Receive Descriptor Head"), 5486 ixgbe_sysctl_rdh_handler, 0, (void *)rxr, 0, 5487 CTL_CREATE, CTL_EOL) != 0) 5488 break; 5489 5490 if (sysctl_createv(log, 0, &rnode, &cnode, 5491 CTLFLAG_READONLY, 5492 CTLTYPE_INT, 5493 "rxd_tail", SYSCTL_DESCR("Receive Descriptor Tail"), 5494 ixgbe_sysctl_rdt_handler, 0, (void *)rxr, 0, 5495 CTL_CREATE, CTL_EOL) != 0) 5496 break; 5497 5498 if (i < __arraycount(adapter->stats.mpc)) { 5499 evcnt_attach_dynamic(&adapter->stats.mpc[i], 5500 EVCNT_TYPE_MISC, NULL, adapter->queues[i].evnamebuf, 5501 "Missed Packet Count"); 5502 } 5503 if (i < __arraycount(adapter->stats.pxontxc)) { 5504 evcnt_attach_dynamic(&adapter->stats.pxontxc[i], 5505 EVCNT_TYPE_MISC, NULL, adapter->queues[i].evnamebuf, 5506 "pxontxc"); 5507 evcnt_attach_dynamic(&adapter->stats.pxonrxc[i], 5508 EVCNT_TYPE_MISC, NULL, adapter->queues[i].evnamebuf, 5509 "pxonrxc"); 5510 evcnt_attach_dynamic(&adapter->stats.pxofftxc[i], 5511 EVCNT_TYPE_MISC, NULL, adapter->queues[i].evnamebuf, 5512 "pxofftxc"); 5513 evcnt_attach_dynamic(&adapter->stats.pxoffrxc[i], 5514 EVCNT_TYPE_MISC, NULL, adapter->queues[i].evnamebuf, 5515 "pxoffrxc"); 5516 evcnt_attach_dynamic(&adapter->stats.pxon2offc[i], 5517 EVCNT_TYPE_MISC, NULL, adapter->queues[i].evnamebuf, 5518 "pxon2offc"); 5519 } 5520 if (i < __arraycount(adapter->stats.qprc)) { 5521 evcnt_attach_dynamic(&adapter->stats.qprc[i], 5522 EVCNT_TYPE_MISC, NULL, adapter->queues[i].evnamebuf, 5523 "qprc"); 5524 evcnt_attach_dynamic(&adapter->stats.qptc[i], 5525 EVCNT_TYPE_MISC, NULL, adapter->queues[i].evnamebuf, 5526 "qptc"); 5527 evcnt_attach_dynamic(&adapter->stats.qbrc[i], 5528 EVCNT_TYPE_MISC, NULL, adapter->queues[i].evnamebuf, 5529 "qbrc"); 5530 evcnt_attach_dynamic(&adapter->stats.qbtc[i], 5531 EVCNT_TYPE_MISC, NULL, adapter->queues[i].evnamebuf, 5532 "qbtc"); 5533 evcnt_attach_dynamic(&adapter->stats.qprdc[i], 5534 EVCNT_TYPE_MISC, NULL, adapter->queues[i].evnamebuf, 5535 "qprdc"); 5536 } 5537 5538 evcnt_attach_dynamic(&rxr->rx_packets, EVCNT_TYPE_MISC, 5539 NULL, adapter->queues[i].evnamebuf, "Queue Packets Received"); 5540 evcnt_attach_dynamic(&rxr->rx_bytes, EVCNT_TYPE_MISC, 5541 NULL, adapter->queues[i].evnamebuf, "Queue Bytes Received"); 5542 evcnt_attach_dynamic(&rxr->no_jmbuf, EVCNT_TYPE_MISC, 5543 NULL, adapter->queues[i].evnamebuf, "Rx no jumbo mbuf"); 5544 evcnt_attach_dynamic(&rxr->rx_discarded, EVCNT_TYPE_MISC, 5545 NULL, adapter->queues[i].evnamebuf, "Rx discarded"); 5546 evcnt_attach_dynamic(&rxr->rx_split_packets, EVCNT_TYPE_MISC, 5547 NULL, adapter->queues[i].evnamebuf, "Rx split packets"); 5548 evcnt_attach_dynamic(&rxr->rx_irq, EVCNT_TYPE_MISC, 5549 NULL, adapter->queues[i].evnamebuf, "Rx interrupts"); 5550 #ifdef LRO 5551 SYSCTL_ADD_INT(ctx, queue_list, OID_AUTO, "lro_queued", 5552 CTLFLAG_RD, &lro->lro_queued, 0, 5553 "LRO Queued"); 5554 SYSCTL_ADD_INT(ctx, queue_list, OID_AUTO, "lro_flushed", 5555 CTLFLAG_RD, &lro->lro_flushed, 0, 5556 "LRO Flushed"); 5557 #endif /* LRO */ 5558 } 5559 5560 /* MAC stats get the own sub node */ 5561 5562 5563 snprintf(stats->namebuf, 5564 sizeof(stats->namebuf), "%s MAC Statistics", device_xname(dev)); 5565 5566 evcnt_attach_dynamic(&stats->ipcs, EVCNT_TYPE_MISC, NULL, 5567 stats->namebuf, "rx csum offload - IP"); 5568 evcnt_attach_dynamic(&stats->l4cs, EVCNT_TYPE_MISC, NULL, 5569 stats->namebuf, "rx csum offload - L4"); 5570 evcnt_attach_dynamic(&stats->ipcs_bad, EVCNT_TYPE_MISC, NULL, 5571 stats->namebuf, "rx csum offload - IP bad"); 5572 evcnt_attach_dynamic(&stats->l4cs_bad, EVCNT_TYPE_MISC, NULL, 5573 stats->namebuf, "rx csum offload - L4 bad"); 5574 evcnt_attach_dynamic(&stats->intzero, EVCNT_TYPE_MISC, NULL, 5575 stats->namebuf, "Interrupt conditions zero"); 5576 evcnt_attach_dynamic(&stats->legint, EVCNT_TYPE_MISC, NULL, 5577 stats->namebuf, "Legacy interrupts"); 5578 evcnt_attach_dynamic(&stats->crcerrs, EVCNT_TYPE_MISC, NULL, 5579 stats->namebuf, "CRC Errors"); 5580 evcnt_attach_dynamic(&stats->illerrc, EVCNT_TYPE_MISC, NULL, 5581 stats->namebuf, "Illegal Byte Errors"); 5582 evcnt_attach_dynamic(&stats->errbc, EVCNT_TYPE_MISC, NULL, 5583 stats->namebuf, "Byte Errors"); 5584 evcnt_attach_dynamic(&stats->mspdc, EVCNT_TYPE_MISC, NULL, 5585 stats->namebuf, "MAC Short Packets Discarded"); 5586 evcnt_attach_dynamic(&stats->mlfc, EVCNT_TYPE_MISC, NULL, 5587 stats->namebuf, "MAC Local Faults"); 5588 evcnt_attach_dynamic(&stats->mrfc, EVCNT_TYPE_MISC, NULL, 5589 stats->namebuf, "MAC Remote Faults"); 5590 evcnt_attach_dynamic(&stats->rlec, EVCNT_TYPE_MISC, NULL, 5591 stats->namebuf, "Receive Length Errors"); 5592 evcnt_attach_dynamic(&stats->lxontxc, EVCNT_TYPE_MISC, NULL, 5593 stats->namebuf, "Link XON Transmitted"); 5594 evcnt_attach_dynamic(&stats->lxonrxc, EVCNT_TYPE_MISC, NULL, 5595 stats->namebuf, "Link XON Received"); 5596 evcnt_attach_dynamic(&stats->lxofftxc, EVCNT_TYPE_MISC, NULL, 5597 stats->namebuf, "Link XOFF Transmitted"); 5598 evcnt_attach_dynamic(&stats->lxoffrxc, EVCNT_TYPE_MISC, NULL, 5599 stats->namebuf, "Link XOFF Received"); 5600 5601 /* Packet Reception Stats */ 5602 evcnt_attach_dynamic(&stats->tor, EVCNT_TYPE_MISC, NULL, 5603 stats->namebuf, "Total Octets Received"); 5604 evcnt_attach_dynamic(&stats->gorc, EVCNT_TYPE_MISC, NULL, 5605 stats->namebuf, "Good Octets Received"); 5606 evcnt_attach_dynamic(&stats->tpr, EVCNT_TYPE_MISC, NULL, 5607 stats->namebuf, "Total Packets Received"); 5608 evcnt_attach_dynamic(&stats->gprc, EVCNT_TYPE_MISC, NULL, 5609 stats->namebuf, "Good Packets Received"); 5610 evcnt_attach_dynamic(&stats->mprc, EVCNT_TYPE_MISC, NULL, 5611 stats->namebuf, "Multicast Packets Received"); 5612 evcnt_attach_dynamic(&stats->bprc, EVCNT_TYPE_MISC, NULL, 5613 stats->namebuf, "Broadcast Packets Received"); 5614 evcnt_attach_dynamic(&stats->prc64, EVCNT_TYPE_MISC, NULL, 5615 stats->namebuf, "64 byte frames received "); 5616 evcnt_attach_dynamic(&stats->prc127, EVCNT_TYPE_MISC, NULL, 5617 stats->namebuf, "65-127 byte frames received"); 5618 evcnt_attach_dynamic(&stats->prc255, EVCNT_TYPE_MISC, NULL, 5619 stats->namebuf, "128-255 byte frames received"); 5620 evcnt_attach_dynamic(&stats->prc511, EVCNT_TYPE_MISC, NULL, 5621 stats->namebuf, "256-511 byte frames received"); 5622 evcnt_attach_dynamic(&stats->prc1023, EVCNT_TYPE_MISC, NULL, 5623 stats->namebuf, "512-1023 byte frames received"); 5624 evcnt_attach_dynamic(&stats->prc1522, EVCNT_TYPE_MISC, NULL, 5625 stats->namebuf, "1023-1522 byte frames received"); 5626 evcnt_attach_dynamic(&stats->ruc, EVCNT_TYPE_MISC, NULL, 5627 stats->namebuf, "Receive Undersized"); 5628 evcnt_attach_dynamic(&stats->rfc, EVCNT_TYPE_MISC, NULL, 5629 stats->namebuf, "Fragmented Packets Received "); 5630 evcnt_attach_dynamic(&stats->roc, EVCNT_TYPE_MISC, NULL, 5631 stats->namebuf, "Oversized Packets Received"); 5632 evcnt_attach_dynamic(&stats->rjc, EVCNT_TYPE_MISC, NULL, 5633 stats->namebuf, "Received Jabber"); 5634 evcnt_attach_dynamic(&stats->mngprc, EVCNT_TYPE_MISC, NULL, 5635 stats->namebuf, "Management Packets Received"); 5636 evcnt_attach_dynamic(&stats->xec, EVCNT_TYPE_MISC, NULL, 5637 stats->namebuf, "Checksum Errors"); 5638 5639 /* Packet Transmission Stats */ 5640 evcnt_attach_dynamic(&stats->gotc, EVCNT_TYPE_MISC, NULL, 5641 stats->namebuf, "Good Octets Transmitted"); 5642 evcnt_attach_dynamic(&stats->tpt, EVCNT_TYPE_MISC, NULL, 5643 stats->namebuf, "Total Packets Transmitted"); 5644 evcnt_attach_dynamic(&stats->gptc, EVCNT_TYPE_MISC, NULL, 5645 stats->namebuf, "Good Packets Transmitted"); 5646 evcnt_attach_dynamic(&stats->bptc, EVCNT_TYPE_MISC, NULL, 5647 stats->namebuf, "Broadcast Packets Transmitted"); 5648 evcnt_attach_dynamic(&stats->mptc, EVCNT_TYPE_MISC, NULL, 5649 stats->namebuf, "Multicast Packets Transmitted"); 5650 evcnt_attach_dynamic(&stats->mngptc, EVCNT_TYPE_MISC, NULL, 5651 stats->namebuf, "Management Packets Transmitted"); 5652 evcnt_attach_dynamic(&stats->ptc64, EVCNT_TYPE_MISC, NULL, 5653 stats->namebuf, "64 byte frames transmitted "); 5654 evcnt_attach_dynamic(&stats->ptc127, EVCNT_TYPE_MISC, NULL, 5655 stats->namebuf, "65-127 byte frames transmitted"); 5656 evcnt_attach_dynamic(&stats->ptc255, EVCNT_TYPE_MISC, NULL, 5657 stats->namebuf, "128-255 byte frames transmitted"); 5658 evcnt_attach_dynamic(&stats->ptc511, EVCNT_TYPE_MISC, NULL, 5659 stats->namebuf, "256-511 byte frames transmitted"); 5660 evcnt_attach_dynamic(&stats->ptc1023, EVCNT_TYPE_MISC, NULL, 5661 stats->namebuf, "512-1023 byte frames transmitted"); 5662 evcnt_attach_dynamic(&stats->ptc1522, EVCNT_TYPE_MISC, NULL, 5663 stats->namebuf, "1024-1522 byte frames transmitted"); 5664 5665 /* FC Stats */ 5666 evcnt_attach_dynamic(&stats->fccrc, EVCNT_TYPE_MISC, NULL, 5667 stats->namebuf, "FC CRC Errors"); 5668 evcnt_attach_dynamic(&stats->fclast, EVCNT_TYPE_MISC, NULL, 5669 stats->namebuf, "FC Last Error"); 5670 if (hw->mac.type != ixgbe_mac_82598EB) { 5671 evcnt_attach_dynamic(&stats->fcoerpdc, EVCNT_TYPE_MISC, NULL, 5672 stats->namebuf, "FCoE Packets Dropped"); 5673 evcnt_attach_dynamic(&stats->fcoeprc, EVCNT_TYPE_MISC, NULL, 5674 stats->namebuf, "FCoE Packets Received"); 5675 evcnt_attach_dynamic(&stats->fcoeptc, EVCNT_TYPE_MISC, NULL, 5676 stats->namebuf, "FCoE Packets Transmitted"); 5677 evcnt_attach_dynamic(&stats->fcoedwrc, EVCNT_TYPE_MISC, NULL, 5678 stats->namebuf, "FCoE DWords Received"); 5679 evcnt_attach_dynamic(&stats->fcoedwtc, EVCNT_TYPE_MISC, NULL, 5680 stats->namebuf, "FCoE DWords Transmitted"); 5681 } 5682 } 5683 5684 /* 5685 ** Set flow control using sysctl: 5686 ** Flow control values: 5687 ** 0 - off 5688 ** 1 - rx pause 5689 ** 2 - tx pause 5690 ** 3 - full 5691 */ 5692 static int 5693 ixgbe_set_flowcntl(SYSCTLFN_ARGS) 5694 { 5695 struct sysctlnode node; 5696 int error; 5697 int last = ixgbe_flow_control; 5698 struct adapter *adapter; 5699 5700 node = *rnode; 5701 adapter = (struct adapter *)node.sysctl_data; 5702 node.sysctl_data = &ixgbe_flow_control; 5703 error = sysctl_lookup(SYSCTLFN_CALL(&node)); 5704 if (error != 0 || newp == NULL) 5705 return error; 5706 5707 /* Don't bother if it's not changed */ 5708 if (ixgbe_flow_control == last) 5709 return (0); 5710 5711 switch (ixgbe_flow_control) { 5712 case ixgbe_fc_rx_pause: 5713 case ixgbe_fc_tx_pause: 5714 case ixgbe_fc_full: 5715 adapter->hw.fc.requested_mode = ixgbe_flow_control; 5716 break; 5717 case ixgbe_fc_none: 5718 default: 5719 adapter->hw.fc.requested_mode = ixgbe_fc_none; 5720 } 5721 5722 ixgbe_fc_enable(&adapter->hw, 0); 5723 return 0; 5724 } 5725 5726 static void 5727 ixgbe_add_rx_process_limit(struct adapter *adapter, const char *name, 5728 const char *description, int *limit, int value) 5729 { 5730 const struct sysctlnode *rnode, *cnode; 5731 struct sysctllog **log = &adapter->sysctllog; 5732 5733 *limit = value; 5734 5735 if ((rnode = ixgbe_sysctl_instance(adapter)) == NULL) 5736 aprint_error_dev(adapter->dev, 5737 "could not create sysctl root\n"); 5738 else if (sysctl_createv(log, 0, &rnode, &cnode, 5739 CTLFLAG_READWRITE, 5740 CTLTYPE_INT, 5741 name, SYSCTL_DESCR(description), 5742 NULL, 0, limit, 0, 5743 CTL_CREATE, CTL_EOL) != 0) { 5744 aprint_error_dev(adapter->dev, "%s: could not create sysctl", 5745 __func__); 5746 } 5747 } 5748 5749 /* 5750 ** Control link advertise speed: 5751 ** 0 - normal 5752 ** 1 - advertise only 1G 5753 */ 5754 static int 5755 ixgbe_set_advertise(SYSCTLFN_ARGS) 5756 { 5757 struct sysctlnode node; 5758 int t, error; 5759 struct adapter *adapter; 5760 struct ixgbe_hw *hw; 5761 ixgbe_link_speed speed, last; 5762 5763 node = *rnode; 5764 adapter = (struct adapter *)node.sysctl_data; 5765 t = adapter->advertise; 5766 node.sysctl_data = &t; 5767 error = sysctl_lookup(SYSCTLFN_CALL(&node)); 5768 if (error != 0 || newp == NULL) 5769 return error; 5770 5771 if (t == -1) 5772 return 0; 5773 5774 adapter->advertise = t; 5775 5776 hw = &adapter->hw; 5777 last = hw->phy.autoneg_advertised; 5778 5779 if (!((hw->phy.media_type == ixgbe_media_type_copper) || 5780 (hw->phy.multispeed_fiber))) 5781 return 0; 5782 5783 if (adapter->advertise == 1) 5784 speed = IXGBE_LINK_SPEED_1GB_FULL; 5785 else 5786 speed = IXGBE_LINK_SPEED_1GB_FULL | 5787 IXGBE_LINK_SPEED_10GB_FULL; 5788 5789 if (speed == last) /* no change */ 5790 return 0; 5791 5792 hw->mac.autotry_restart = TRUE; 5793 hw->mac.ops.setup_link(hw, speed, TRUE, TRUE); 5794 5795 return 0; 5796 } 5797
Indexes created Thu Oct 23 22:10:10 GMT 2025