ixv.c revision 1.8
1 /****************************************************************************** 2 3 Copyright (c) 2001-2013, Intel Corporation 4 All rights reserved. 5 6 Redistribution and use in source and binary forms, with or without 7 modification, are permitted provided that the following conditions are met: 8 9 1. Redistributions of source code must retain the above copyright notice, 10 this list of conditions and the following disclaimer. 11 12 2. Redistributions in binary form must reproduce the above copyright 13 notice, this list of conditions and the following disclaimer in the 14 documentation and/or other materials provided with the distribution. 15 16 3. Neither the name of the Intel Corporation nor the names of its 17 contributors may be used to endorse or promote products derived from 18 this software without specific prior written permission. 19 20 THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 21 AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 22 IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 23 ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE 24 LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 25 CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 26 SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 27 INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 28 CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 29 ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 30 POSSIBILITY OF SUCH DAMAGE. 31 32 ******************************************************************************/ 33 /*$FreeBSD: head/sys/dev/ixgbe/ixv.c 247822 2013-03-04 23:07:40Z jfv $*/ 34 /*$NetBSD: ixv.c,v 1.8 2015/04/24 07:00:51 msaitoh Exp $*/ 35 36 #include "opt_inet.h" 37 #include "opt_inet6.h" 38 39 #include "ixv.h" 40 41 /********************************************************************* 42 * Driver version 43 *********************************************************************/ 44 char ixv_driver_version[] = "1.1.4"; 45 46 /********************************************************************* 47 * PCI Device ID Table 48 * 49 * Used by probe to select devices to load on 50 * Last field stores an index into ixv_strings 51 * Last entry must be all 0s 52 * 53 * { Vendor ID, Device ID, SubVendor ID, SubDevice ID, String Index } 54 *********************************************************************/ 55 56 static ixv_vendor_info_t ixv_vendor_info_array[] = 57 { 58 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_VF, 0, 0, 0}, 59 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_X540_VF, 0, 0, 0}, 60 /* required last entry */ 61 {0, 0, 0, 0, 0} 62 }; 63 64 /********************************************************************* 65 * Table of branding strings 66 *********************************************************************/ 67 68 static const char *ixv_strings[] = { 69 "Intel(R) PRO/10GbE Virtual Function Network Driver" 70 }; 71 72 /********************************************************************* 73 * Function prototypes 74 *********************************************************************/ 75 static int ixv_probe(device_t, cfdata_t, void *); 76 static void ixv_attach(device_t, device_t, void *); 77 static int ixv_detach(device_t, int); 78 #if 0 79 static int ixv_shutdown(device_t); 80 #endif 81 #if __FreeBSD_version < 800000 82 static void ixv_start(struct ifnet *); 83 static void ixv_start_locked(struct tx_ring *, struct ifnet *); 84 #else 85 static int ixv_mq_start(struct ifnet *, struct mbuf *); 86 static int ixv_mq_start_locked(struct ifnet *, 87 struct tx_ring *, struct mbuf *); 88 static void ixv_qflush(struct ifnet *); 89 #endif 90 static int ixv_ioctl(struct ifnet *, u_long, void *); 91 static int ixv_init(struct ifnet *); 92 static void ixv_init_locked(struct adapter *); 93 static void ixv_stop(void *); 94 static void ixv_media_status(struct ifnet *, struct ifmediareq *); 95 static int ixv_media_change(struct ifnet *); 96 static void ixv_identify_hardware(struct adapter *); 97 static int ixv_allocate_pci_resources(struct adapter *, 98 const struct pci_attach_args *); 99 static int ixv_allocate_msix(struct adapter *); 100 static int ixv_allocate_queues(struct adapter *); 101 static int ixv_setup_msix(struct adapter *); 102 static void ixv_free_pci_resources(struct adapter *); 103 static void ixv_local_timer(void *); 104 static void ixv_setup_interface(device_t, struct adapter *); 105 static void ixv_config_link(struct adapter *); 106 107 static int ixv_allocate_transmit_buffers(struct tx_ring *); 108 static int ixv_setup_transmit_structures(struct adapter *); 109 static void ixv_setup_transmit_ring(struct tx_ring *); 110 static void ixv_initialize_transmit_units(struct adapter *); 111 static void ixv_free_transmit_structures(struct adapter *); 112 static void ixv_free_transmit_buffers(struct tx_ring *); 113 114 static int ixv_allocate_receive_buffers(struct rx_ring *); 115 static int ixv_setup_receive_structures(struct adapter *); 116 static int ixv_setup_receive_ring(struct rx_ring *); 117 static void ixv_initialize_receive_units(struct adapter *); 118 static void ixv_free_receive_structures(struct adapter *); 119 static void ixv_free_receive_buffers(struct rx_ring *); 120 121 static void ixv_enable_intr(struct adapter *); 122 static void ixv_disable_intr(struct adapter *); 123 static bool ixv_txeof(struct tx_ring *); 124 static bool ixv_rxeof(struct ix_queue *, int); 125 static void ixv_rx_checksum(u32, struct mbuf *, u32, 126 struct ixgbevf_hw_stats *); 127 static void ixv_set_multi(struct adapter *); 128 static void ixv_update_link_status(struct adapter *); 129 static void ixv_refresh_mbufs(struct rx_ring *, int); 130 static int ixv_xmit(struct tx_ring *, struct mbuf *); 131 static int ixv_sysctl_stats(SYSCTLFN_PROTO); 132 static int ixv_sysctl_debug(SYSCTLFN_PROTO); 133 static int ixv_set_flowcntl(SYSCTLFN_PROTO); 134 static int ixv_dma_malloc(struct adapter *, bus_size_t, 135 struct ixv_dma_alloc *, int); 136 static void ixv_dma_free(struct adapter *, struct ixv_dma_alloc *); 137 static void ixv_add_rx_process_limit(struct adapter *, const char *, 138 const char *, int *, int); 139 static u32 ixv_tx_ctx_setup(struct tx_ring *, struct mbuf *); 140 static bool ixv_tso_setup(struct tx_ring *, struct mbuf *, u32 *); 141 static void ixv_set_ivar(struct adapter *, u8, u8, s8); 142 static void ixv_configure_ivars(struct adapter *); 143 static u8 * ixv_mc_array_itr(struct ixgbe_hw *, u8 **, u32 *); 144 145 static void ixv_setup_vlan_support(struct adapter *); 146 #if 0 147 static void ixv_register_vlan(void *, struct ifnet *, u16); 148 static void ixv_unregister_vlan(void *, struct ifnet *, u16); 149 #endif 150 151 static void ixv_save_stats(struct adapter *); 152 static void ixv_init_stats(struct adapter *); 153 static void ixv_update_stats(struct adapter *); 154 155 static __inline void ixv_rx_discard(struct rx_ring *, int); 156 static __inline void ixv_rx_input(struct rx_ring *, struct ifnet *, 157 struct mbuf *, u32); 158 159 /* The MSI/X Interrupt handlers */ 160 static void ixv_msix_que(void *); 161 static void ixv_msix_mbx(void *); 162 163 /* Deferred interrupt tasklets */ 164 static void ixv_handle_que(void *); 165 static void ixv_handle_mbx(void *); 166 167 const struct sysctlnode *ixv_sysctl_instance(struct adapter *); 168 static ixv_vendor_info_t *ixv_lookup(const struct pci_attach_args *); 169 170 /********************************************************************* 171 * FreeBSD Device Interface Entry Points 172 *********************************************************************/ 173 174 CFATTACH_DECL3_NEW(ixv, sizeof(struct adapter), 175 ixv_probe, ixv_attach, ixv_detach, NULL, NULL, NULL, 176 DVF_DETACH_SHUTDOWN); 177 178 # if 0 179 static device_method_t ixv_methods[] = { 180 /* Device interface */ 181 DEVMETHOD(device_probe, ixv_probe), 182 DEVMETHOD(device_attach, ixv_attach), 183 DEVMETHOD(device_detach, ixv_detach), 184 DEVMETHOD(device_shutdown, ixv_shutdown), 185 DEVMETHOD_END 186 }; 187 #endif 188 189 #if 0 190 static driver_t ixv_driver = { 191 "ix", ixv_methods, sizeof(struct adapter), 192 }; 193 194 extern devclass_t ixgbe_devclass; 195 DRIVER_MODULE(ixv, pci, ixv_driver, ixgbe_devclass, 0, 0); 196 MODULE_DEPEND(ixv, pci, 1, 1, 1); 197 MODULE_DEPEND(ixv, ether, 1, 1, 1); 198 #endif 199 200 /* 201 ** TUNEABLE PARAMETERS: 202 */ 203 204 /* 205 ** AIM: Adaptive Interrupt Moderation 206 ** which means that the interrupt rate 207 ** is varied over time based on the 208 ** traffic for that interrupt vector 209 */ 210 static int ixv_enable_aim = FALSE; 211 #define TUNABLE_INT(__x, __y) 212 TUNABLE_INT("hw.ixv.enable_aim", &ixv_enable_aim); 213 214 /* How many packets rxeof tries to clean at a time */ 215 static int ixv_rx_process_limit = 128; 216 TUNABLE_INT("hw.ixv.rx_process_limit", &ixv_rx_process_limit); 217 218 /* Flow control setting, default to full */ 219 static int ixv_flow_control = ixgbe_fc_full; 220 TUNABLE_INT("hw.ixv.flow_control", &ixv_flow_control); 221 222 /* 223 * Header split: this causes the hardware to DMA 224 * the header into a seperate mbuf from the payload, 225 * it can be a performance win in some workloads, but 226 * in others it actually hurts, its off by default. 227 */ 228 static int ixv_header_split = FALSE; 229 TUNABLE_INT("hw.ixv.hdr_split", &ixv_header_split); 230 231 /* 232 ** Number of TX descriptors per ring, 233 ** setting higher than RX as this seems 234 ** the better performing choice. 235 */ 236 static int ixv_txd = DEFAULT_TXD; 237 TUNABLE_INT("hw.ixv.txd", &ixv_txd); 238 239 /* Number of RX descriptors per ring */ 240 static int ixv_rxd = DEFAULT_RXD; 241 TUNABLE_INT("hw.ixv.rxd", &ixv_rxd); 242 243 /* 244 ** Shadow VFTA table, this is needed because 245 ** the real filter table gets cleared during 246 ** a soft reset and we need to repopulate it. 247 */ 248 static u32 ixv_shadow_vfta[VFTA_SIZE]; 249 250 /* Keep running tab on them for sanity check */ 251 static int ixv_total_ports; 252 253 /********************************************************************* 254 * Device identification routine 255 * 256 * ixv_probe determines if the driver should be loaded on 257 * adapter based on PCI vendor/device id of the adapter. 258 * 259 * return 1 on success, 0 on failure 260 *********************************************************************/ 261 262 static int 263 ixv_probe(device_t dev, cfdata_t cf, void *aux) 264 { 265 const struct pci_attach_args *pa = aux; 266 267 return (ixv_lookup(pa) != NULL) ? 1 : 0; 268 } 269 270 static ixv_vendor_info_t * 271 ixv_lookup(const struct pci_attach_args *pa) 272 { 273 pcireg_t subid; 274 ixv_vendor_info_t *ent; 275 276 INIT_DEBUGOUT("ixv_probe: begin"); 277 278 if (PCI_VENDOR(pa->pa_id) != IXGBE_INTEL_VENDOR_ID) 279 return NULL; 280 281 subid = pci_conf_read(pa->pa_pc, pa->pa_tag, PCI_SUBSYS_ID_REG); 282 283 for (ent = ixv_vendor_info_array; ent->vendor_id != 0; ent++) { 284 if ((PCI_VENDOR(pa->pa_id) == ent->vendor_id) && 285 (PCI_PRODUCT(pa->pa_id) == ent->device_id) && 286 287 ((PCI_SUBSYS_VENDOR(subid) == ent->subvendor_id) || 288 (ent->subvendor_id == 0)) && 289 290 ((PCI_SUBSYS_ID(subid) == ent->subdevice_id) || 291 (ent->subdevice_id == 0))) { 292 ++ixv_total_ports; 293 return ent; 294 } 295 } 296 return NULL; 297 } 298 299 300 static void 301 ixv_sysctl_attach(struct adapter *adapter) 302 { 303 struct sysctllog **log; 304 const struct sysctlnode *rnode, *cnode; 305 device_t dev; 306 307 dev = adapter->dev; 308 log = &adapter->sysctllog; 309 310 if ((rnode = ixv_sysctl_instance(adapter)) == NULL) { 311 aprint_error_dev(dev, "could not create sysctl root\n"); 312 return; 313 } 314 315 if (sysctl_createv(log, 0, &rnode, &cnode, 316 CTLFLAG_READWRITE, CTLTYPE_INT, 317 "stats", SYSCTL_DESCR("Statistics"), 318 ixv_sysctl_stats, 0, (void *)adapter, 0, CTL_CREATE, CTL_EOL) != 0) 319 aprint_error_dev(dev, "could not create sysctl\n"); 320 321 if (sysctl_createv(log, 0, &rnode, &cnode, 322 CTLFLAG_READWRITE, CTLTYPE_INT, 323 "debug", SYSCTL_DESCR("Debug Info"), 324 ixv_sysctl_debug, 0, (void *)adapter, 0, CTL_CREATE, CTL_EOL) != 0) 325 aprint_error_dev(dev, "could not create sysctl\n"); 326 327 if (sysctl_createv(log, 0, &rnode, &cnode, 328 CTLFLAG_READWRITE, CTLTYPE_INT, 329 "flow_control", SYSCTL_DESCR("Flow Control"), 330 ixv_set_flowcntl, 0, (void *)adapter, 0, CTL_CREATE, CTL_EOL) != 0) 331 aprint_error_dev(dev, "could not create sysctl\n"); 332 333 /* XXX This is an *instance* sysctl controlling a *global* variable. 334 * XXX It's that way in the FreeBSD driver that this derives from. 335 */ 336 if (sysctl_createv(log, 0, &rnode, &cnode, 337 CTLFLAG_READWRITE, CTLTYPE_INT, 338 "enable_aim", SYSCTL_DESCR("Interrupt Moderation"), 339 NULL, 0, &ixv_enable_aim, 0, CTL_CREATE, CTL_EOL) != 0) 340 aprint_error_dev(dev, "could not create sysctl\n"); 341 } 342 343 /********************************************************************* 344 * Device initialization routine 345 * 346 * The attach entry point is called when the driver is being loaded. 347 * This routine identifies the type of hardware, allocates all resources 348 * and initializes the hardware. 349 * 350 * return 0 on success, positive on failure 351 *********************************************************************/ 352 353 static void 354 ixv_attach(device_t parent, device_t dev, void *aux) 355 { 356 struct adapter *adapter; 357 struct ixgbe_hw *hw; 358 int error = 0; 359 ixv_vendor_info_t *ent; 360 const struct pci_attach_args *pa = aux; 361 362 INIT_DEBUGOUT("ixv_attach: begin"); 363 364 /* Allocate, clear, and link in our adapter structure */ 365 adapter = device_private(dev); 366 adapter->dev = adapter->osdep.dev = dev; 367 hw = &adapter->hw; 368 369 ent = ixv_lookup(pa); 370 371 KASSERT(ent != NULL); 372 373 aprint_normal(": %s, Version - %s\n", 374 ixv_strings[ent->index], ixv_driver_version); 375 376 /* Core Lock Init*/ 377 IXV_CORE_LOCK_INIT(adapter, device_xname(dev)); 378 379 /* SYSCTL APIs */ 380 ixv_sysctl_attach(adapter); 381 382 /* Set up the timer callout */ 383 callout_init(&adapter->timer, 0); 384 385 /* Determine hardware revision */ 386 ixv_identify_hardware(adapter); 387 388 /* Do base PCI setup - map BAR0 */ 389 if (ixv_allocate_pci_resources(adapter, pa)) { 390 aprint_error_dev(dev, "Allocation of PCI resources failed\n"); 391 error = ENXIO; 392 goto err_out; 393 } 394 395 /* Do descriptor calc and sanity checks */ 396 if (((ixv_txd * sizeof(union ixgbe_adv_tx_desc)) % DBA_ALIGN) != 0 || 397 ixv_txd < MIN_TXD || ixv_txd > MAX_TXD) { 398 aprint_error_dev(dev, "TXD config issue, using default!\n"); 399 adapter->num_tx_desc = DEFAULT_TXD; 400 } else 401 adapter->num_tx_desc = ixv_txd; 402 403 if (((ixv_rxd * sizeof(union ixgbe_adv_rx_desc)) % DBA_ALIGN) != 0 || 404 ixv_rxd < MIN_TXD || ixv_rxd > MAX_TXD) { 405 aprint_error_dev(dev, "RXD config issue, using default!\n"); 406 adapter->num_rx_desc = DEFAULT_RXD; 407 } else 408 adapter->num_rx_desc = ixv_rxd; 409 410 /* Allocate our TX/RX Queues */ 411 if (ixv_allocate_queues(adapter)) { 412 error = ENOMEM; 413 goto err_out; 414 } 415 416 /* 417 ** Initialize the shared code: its 418 ** at this point the mac type is set. 419 */ 420 error = ixgbe_init_shared_code(hw); 421 if (error) { 422 aprint_error_dev(dev,"Shared Code Initialization Failure\n"); 423 error = EIO; 424 goto err_late; 425 } 426 427 /* Setup the mailbox */ 428 ixgbe_init_mbx_params_vf(hw); 429 430 ixgbe_reset_hw(hw); 431 432 /* Get Hardware Flow Control setting */ 433 hw->fc.requested_mode = ixgbe_fc_full; 434 hw->fc.pause_time = IXV_FC_PAUSE; 435 hw->fc.low_water[0] = IXV_FC_LO; 436 hw->fc.high_water[0] = IXV_FC_HI; 437 hw->fc.send_xon = TRUE; 438 439 error = ixgbe_init_hw(hw); 440 if (error) { 441 aprint_error_dev(dev,"Hardware Initialization Failure\n"); 442 error = EIO; 443 goto err_late; 444 } 445 446 error = ixv_allocate_msix(adapter); 447 if (error) 448 goto err_late; 449 450 /* Setup OS specific network interface */ 451 ixv_setup_interface(dev, adapter); 452 453 /* Sysctl for limiting the amount of work done in the taskqueue */ 454 ixv_add_rx_process_limit(adapter, "rx_processing_limit", 455 "max number of rx packets to process", &adapter->rx_process_limit, 456 ixv_rx_process_limit); 457 458 /* Do the stats setup */ 459 ixv_save_stats(adapter); 460 ixv_init_stats(adapter); 461 462 /* Register for VLAN events */ 463 #if 0 /* XXX msaitoh delete after write? */ 464 adapter->vlan_attach = EVENTHANDLER_REGISTER(vlan_config, 465 ixv_register_vlan, adapter, EVENTHANDLER_PRI_FIRST); 466 adapter->vlan_detach = EVENTHANDLER_REGISTER(vlan_unconfig, 467 ixv_unregister_vlan, adapter, EVENTHANDLER_PRI_FIRST); 468 #endif 469 470 INIT_DEBUGOUT("ixv_attach: end"); 471 return; 472 473 err_late: 474 ixv_free_transmit_structures(adapter); 475 ixv_free_receive_structures(adapter); 476 err_out: 477 ixv_free_pci_resources(adapter); 478 return; 479 480 } 481 482 /********************************************************************* 483 * Device removal routine 484 * 485 * The detach entry point is called when the driver is being removed. 486 * This routine stops the adapter and deallocates all the resources 487 * that were allocated for driver operation. 488 * 489 * return 0 on success, positive on failure 490 *********************************************************************/ 491 492 static int 493 ixv_detach(device_t dev, int flags) 494 { 495 struct adapter *adapter = device_private(dev); 496 struct ix_queue *que = adapter->queues; 497 498 INIT_DEBUGOUT("ixv_detach: begin"); 499 500 /* Make sure VLANS are not using driver */ 501 if (!VLAN_ATTACHED(&adapter->osdep.ec)) 502 ; /* nothing to do: no VLANs */ 503 else if ((flags & (DETACH_SHUTDOWN|DETACH_FORCE)) != 0) 504 vlan_ifdetach(adapter->ifp); 505 else { 506 aprint_error_dev(dev, "VLANs in use\n"); 507 return EBUSY; 508 } 509 510 IXV_CORE_LOCK(adapter); 511 ixv_stop(adapter); 512 IXV_CORE_UNLOCK(adapter); 513 514 for (int i = 0; i < adapter->num_queues; i++, que++) { 515 softint_disestablish(que->que_si); 516 } 517 518 /* Drain the Link queue */ 519 softint_disestablish(adapter->mbx_si); 520 521 /* Unregister VLAN events */ 522 #if 0 /* XXX msaitoh delete after write? */ 523 if (adapter->vlan_attach != NULL) 524 EVENTHANDLER_DEREGISTER(vlan_config, adapter->vlan_attach); 525 if (adapter->vlan_detach != NULL) 526 EVENTHANDLER_DEREGISTER(vlan_unconfig, adapter->vlan_detach); 527 #endif 528 529 ether_ifdetach(adapter->ifp); 530 callout_halt(&adapter->timer, NULL); 531 ixv_free_pci_resources(adapter); 532 #if 0 /* XXX the NetBSD port is probably missing something here */ 533 bus_generic_detach(dev); 534 #endif 535 if_detach(adapter->ifp); 536 537 ixv_free_transmit_structures(adapter); 538 ixv_free_receive_structures(adapter); 539 540 IXV_CORE_LOCK_DESTROY(adapter); 541 return (0); 542 } 543 544 /********************************************************************* 545 * 546 * Shutdown entry point 547 * 548 **********************************************************************/ 549 #if 0 /* XXX NetBSD ought to register something like this through pmf(9) */ 550 static int 551 ixv_shutdown(device_t dev) 552 { 553 struct adapter *adapter = device_private(dev); 554 IXV_CORE_LOCK(adapter); 555 ixv_stop(adapter); 556 IXV_CORE_UNLOCK(adapter); 557 return (0); 558 } 559 #endif 560 561 #if __FreeBSD_version < 800000 562 /********************************************************************* 563 * Transmit entry point 564 * 565 * ixv_start is called by the stack to initiate a transmit. 566 * The driver will remain in this routine as long as there are 567 * packets to transmit and transmit resources are available. 568 * In case resources are not available stack is notified and 569 * the packet is requeued. 570 **********************************************************************/ 571 static void 572 ixv_start_locked(struct tx_ring *txr, struct ifnet * ifp) 573 { 574 int rc; 575 struct mbuf *m_head; 576 struct adapter *adapter = txr->adapter; 577 578 IXV_TX_LOCK_ASSERT(txr); 579 580 if ((ifp->if_flags & (IFF_RUNNING|IFF_OACTIVE)) != 581 IFF_RUNNING) 582 return; 583 if (!adapter->link_active) 584 return; 585 586 while (!IFQ_IS_EMPTY(&ifp->if_snd)) { 587 588 IFQ_POLL(&ifp->if_snd, m_head); 589 if (m_head == NULL) 590 break; 591 592 if (ixv_xmit(txr, m_head) == EAGAIN) { 593 ifp->if_flags |= IFF_OACTIVE; 594 break; 595 } 596 IFQ_DEQUEUE(&ifp->if_snd, m_head); 597 if (rc == EFBIG) { 598 struct mbuf *mtmp; 599 600 if ((mtmp = m_defrag(m_head, M_NOWAIT)) != NULL) { 601 m_head = mtmp; 602 rc = ixv_xmit(txr, m_head); 603 if (rc != 0) 604 adapter->efbig2_tx_dma_setup.ev_count++; 605 } else 606 adapter->m_defrag_failed.ev_count++; 607 } 608 if (rc != 0) { 609 m_freem(m_head); 610 continue; 611 } 612 /* Send a copy of the frame to the BPF listener */ 613 bpf_mtap(ifp, m_head); 614 615 /* Set watchdog on */ 616 txr->watchdog_check = TRUE; 617 getmicrotime(&txr->watchdog_time); 618 } 619 return; 620 } 621 622 /* 623 * Legacy TX start - called by the stack, this 624 * always uses the first tx ring, and should 625 * not be used with multiqueue tx enabled. 626 */ 627 static void 628 ixv_start(struct ifnet *ifp) 629 { 630 struct adapter *adapter = ifp->if_softc; 631 struct tx_ring *txr = adapter->tx_rings; 632 633 if (ifp->if_flags & IFF_RUNNING) { 634 IXV_TX_LOCK(txr); 635 ixv_start_locked(txr, ifp); 636 IXV_TX_UNLOCK(txr); 637 } 638 return; 639 } 640 641 #else 642 643 /* 644 ** Multiqueue Transmit driver 645 ** 646 */ 647 static int 648 ixv_mq_start(struct ifnet *ifp, struct mbuf *m) 649 { 650 struct adapter *adapter = ifp->if_softc; 651 struct ix_queue *que; 652 struct tx_ring *txr; 653 int i = 0, err = 0; 654 655 /* Which queue to use */ 656 if ((m->m_flags & M_FLOWID) != 0) 657 i = m->m_pkthdr.flowid % adapter->num_queues; 658 659 txr = &adapter->tx_rings[i]; 660 que = &adapter->queues[i]; 661 662 if (IXV_TX_TRYLOCK(txr)) { 663 err = ixv_mq_start_locked(ifp, txr, m); 664 IXV_TX_UNLOCK(txr); 665 } else { 666 err = drbr_enqueue(ifp, txr->br, m); 667 softint_schedule(que->que_si); 668 } 669 670 return (err); 671 } 672 673 static int 674 ixv_mq_start_locked(struct ifnet *ifp, struct tx_ring *txr, struct mbuf *m) 675 { 676 struct adapter *adapter = txr->adapter; 677 struct mbuf *next; 678 int enqueued, err = 0; 679 680 if ((ifp->if_flags & (IFF_RUNNING | IFF_OACTIVE)) != 681 IFF_RUNNING || adapter->link_active == 0) { 682 if (m != NULL) 683 err = drbr_enqueue(ifp, txr->br, m); 684 return (err); 685 } 686 687 /* Do a clean if descriptors are low */ 688 if (txr->tx_avail <= IXV_TX_CLEANUP_THRESHOLD) 689 ixv_txeof(txr); 690 691 enqueued = 0; 692 if (m == NULL) { 693 err = drbr_dequeue(ifp, txr->br, m); 694 if (err) { 695 return (err); 696 } 697 } 698 /* Process the queue */ 699 while ((next = drbr_peek(ifp, txr->br)) != NULL) { 700 if ((err = ixv_xmit(txr, next)) != 0) { 701 if (next != NULL) { 702 drbr_advance(ifp, txr->br); 703 } else { 704 drbr_putback(ifp, txr->br, next); 705 } 706 break; 707 } 708 drbr_advance(ifp, txr->br); 709 enqueued++; 710 ifp->if_obytes += next->m_pkthdr.len; 711 if (next->m_flags & M_MCAST) 712 ifp->if_omcasts++; 713 /* Send a copy of the frame to the BPF listener */ 714 ETHER_BPF_MTAP(ifp, next); 715 if ((ifp->if_flags & IFF_RUNNING) == 0) 716 break; 717 if (txr->tx_avail <= IXV_TX_OP_THRESHOLD) { 718 ifp->if_flags |= IFF_OACTIVE; 719 break; 720 } 721 } 722 723 if (enqueued > 0) { 724 /* Set watchdog on */ 725 txr->watchdog_check = TRUE; 726 getmicrotime(&txr->watchdog_time); 727 } 728 729 return (err); 730 } 731 732 /* 733 ** Flush all ring buffers 734 */ 735 static void 736 ixv_qflush(struct ifnet *ifp) 737 { 738 struct adapter *adapter = ifp->if_softc; 739 struct tx_ring *txr = adapter->tx_rings; 740 struct mbuf *m; 741 742 for (int i = 0; i < adapter->num_queues; i++, txr++) { 743 IXV_TX_LOCK(txr); 744 while ((m = buf_ring_dequeue_sc(txr->br)) != NULL) 745 m_freem(m); 746 IXV_TX_UNLOCK(txr); 747 } 748 if_qflush(ifp); 749 } 750 751 #endif 752 753 static int 754 ixv_ifflags_cb(struct ethercom *ec) 755 { 756 struct ifnet *ifp = &ec->ec_if; 757 struct adapter *adapter = ifp->if_softc; 758 int change = ifp->if_flags ^ adapter->if_flags, rc = 0; 759 760 IXV_CORE_LOCK(adapter); 761 762 if (change != 0) 763 adapter->if_flags = ifp->if_flags; 764 765 if ((change & ~(IFF_CANTCHANGE|IFF_DEBUG)) != 0) 766 rc = ENETRESET; 767 768 IXV_CORE_UNLOCK(adapter); 769 770 return rc; 771 } 772 773 /********************************************************************* 774 * Ioctl entry point 775 * 776 * ixv_ioctl is called when the user wants to configure the 777 * interface. 778 * 779 * return 0 on success, positive on failure 780 **********************************************************************/ 781 782 static int 783 ixv_ioctl(struct ifnet * ifp, u_long command, void *data) 784 { 785 struct adapter *adapter = ifp->if_softc; 786 struct ifcapreq *ifcr = data; 787 struct ifreq *ifr = (struct ifreq *) data; 788 int error = 0; 789 int l4csum_en; 790 const int l4csum = IFCAP_CSUM_TCPv4_Rx|IFCAP_CSUM_UDPv4_Rx| 791 IFCAP_CSUM_TCPv6_Rx|IFCAP_CSUM_UDPv6_Rx; 792 793 switch (command) { 794 case SIOCSIFFLAGS: 795 IOCTL_DEBUGOUT("ioctl: SIOCSIFFLAGS (Set Interface Flags)"); 796 break; 797 case SIOCADDMULTI: 798 case SIOCDELMULTI: 799 IOCTL_DEBUGOUT("ioctl: SIOC(ADD|DEL)MULTI"); 800 break; 801 case SIOCSIFMEDIA: 802 case SIOCGIFMEDIA: 803 IOCTL_DEBUGOUT("ioctl: SIOCxIFMEDIA (Get/Set Interface Media)"); 804 break; 805 case SIOCSIFCAP: 806 IOCTL_DEBUGOUT("ioctl: SIOCSIFCAP (Set Capabilities)"); 807 break; 808 case SIOCSIFMTU: 809 IOCTL_DEBUGOUT("ioctl: SIOCSIFMTU (Set Interface MTU)"); 810 break; 811 default: 812 IOCTL_DEBUGOUT1("ioctl: UNKNOWN (0x%X)\n", (int)command); 813 break; 814 } 815 816 switch (command) { 817 case SIOCSIFMEDIA: 818 case SIOCGIFMEDIA: 819 return ifmedia_ioctl(ifp, ifr, &adapter->media, command); 820 case SIOCSIFCAP: 821 /* Layer-4 Rx checksum offload has to be turned on and 822 * off as a unit. 823 */ 824 l4csum_en = ifcr->ifcr_capenable & l4csum; 825 if (l4csum_en != l4csum && l4csum_en != 0) 826 return EINVAL; 827 /*FALLTHROUGH*/ 828 case SIOCADDMULTI: 829 case SIOCDELMULTI: 830 case SIOCSIFFLAGS: 831 case SIOCSIFMTU: 832 default: 833 if ((error = ether_ioctl(ifp, command, data)) != ENETRESET) 834 return error; 835 if ((ifp->if_flags & IFF_RUNNING) == 0) 836 ; 837 else if (command == SIOCSIFCAP || command == SIOCSIFMTU) { 838 IXV_CORE_LOCK(adapter); 839 ixv_init_locked(adapter); 840 IXV_CORE_UNLOCK(adapter); 841 } else if (command == SIOCADDMULTI || command == SIOCDELMULTI) { 842 /* 843 * Multicast list has changed; set the hardware filter 844 * accordingly. 845 */ 846 IXV_CORE_LOCK(adapter); 847 ixv_disable_intr(adapter); 848 ixv_set_multi(adapter); 849 ixv_enable_intr(adapter); 850 IXV_CORE_UNLOCK(adapter); 851 } 852 return 0; 853 } 854 } 855 856 /********************************************************************* 857 * Init entry point 858 * 859 * This routine is used in two ways. It is used by the stack as 860 * init entry point in network interface structure. It is also used 861 * by the driver as a hw/sw initialization routine to get to a 862 * consistent state. 863 * 864 * return 0 on success, positive on failure 865 **********************************************************************/ 866 #define IXGBE_MHADD_MFS_SHIFT 16 867 868 static void 869 ixv_init_locked(struct adapter *adapter) 870 { 871 struct ifnet *ifp = adapter->ifp; 872 device_t dev = adapter->dev; 873 struct ixgbe_hw *hw = &adapter->hw; 874 u32 mhadd, gpie; 875 876 INIT_DEBUGOUT("ixv_init: begin"); 877 KASSERT(mutex_owned(&adapter->core_mtx)); 878 hw->adapter_stopped = FALSE; 879 ixgbe_stop_adapter(hw); 880 callout_stop(&adapter->timer); 881 882 /* reprogram the RAR[0] in case user changed it. */ 883 ixgbe_set_rar(hw, 0, hw->mac.addr, 0, IXGBE_RAH_AV); 884 885 /* Get the latest mac address, User can use a LAA */ 886 memcpy(hw->mac.addr, CLLADDR(adapter->ifp->if_sadl), 887 IXGBE_ETH_LENGTH_OF_ADDRESS); 888 ixgbe_set_rar(hw, 0, hw->mac.addr, 0, 1); 889 hw->addr_ctrl.rar_used_count = 1; 890 891 /* Prepare transmit descriptors and buffers */ 892 if (ixv_setup_transmit_structures(adapter)) { 893 aprint_error_dev(dev,"Could not setup transmit structures\n"); 894 ixv_stop(adapter); 895 return; 896 } 897 898 ixgbe_reset_hw(hw); 899 ixv_initialize_transmit_units(adapter); 900 901 /* Setup Multicast table */ 902 ixv_set_multi(adapter); 903 904 /* 905 ** Determine the correct mbuf pool 906 ** for doing jumbo/headersplit 907 */ 908 if (ifp->if_mtu > ETHERMTU) 909 adapter->rx_mbuf_sz = MJUMPAGESIZE; 910 else 911 adapter->rx_mbuf_sz = MCLBYTES; 912 913 /* Prepare receive descriptors and buffers */ 914 if (ixv_setup_receive_structures(adapter)) { 915 device_printf(dev,"Could not setup receive structures\n"); 916 ixv_stop(adapter); 917 return; 918 } 919 920 /* Configure RX settings */ 921 ixv_initialize_receive_units(adapter); 922 923 /* Enable Enhanced MSIX mode */ 924 gpie = IXGBE_READ_REG(&adapter->hw, IXGBE_GPIE); 925 gpie |= IXGBE_GPIE_MSIX_MODE | IXGBE_GPIE_EIAME; 926 gpie |= IXGBE_GPIE_PBA_SUPPORT | IXGBE_GPIE_OCD; 927 IXGBE_WRITE_REG(hw, IXGBE_GPIE, gpie); 928 929 #if 0 /* XXX isn't it required? -- msaitoh */ 930 /* Set the various hardware offload abilities */ 931 ifp->if_hwassist = 0; 932 if (ifp->if_capenable & IFCAP_TSO4) 933 ifp->if_hwassist |= CSUM_TSO; 934 if (ifp->if_capenable & IFCAP_TXCSUM) { 935 ifp->if_hwassist |= (CSUM_TCP | CSUM_UDP); 936 #if __FreeBSD_version >= 800000 937 ifp->if_hwassist |= CSUM_SCTP; 938 #endif 939 } 940 #endif 941 942 /* Set MTU size */ 943 if (ifp->if_mtu > ETHERMTU) { 944 mhadd = IXGBE_READ_REG(hw, IXGBE_MHADD); 945 mhadd &= ~IXGBE_MHADD_MFS_MASK; 946 mhadd |= adapter->max_frame_size << IXGBE_MHADD_MFS_SHIFT; 947 IXGBE_WRITE_REG(hw, IXGBE_MHADD, mhadd); 948 } 949 950 /* Set up VLAN offload and filter */ 951 ixv_setup_vlan_support(adapter); 952 953 callout_reset(&adapter->timer, hz, ixv_local_timer, adapter); 954 955 /* Set up MSI/X routing */ 956 ixv_configure_ivars(adapter); 957 958 /* Set up auto-mask */ 959 IXGBE_WRITE_REG(hw, IXGBE_VTEIAM, IXGBE_EICS_RTX_QUEUE); 960 961 /* Set moderation on the Link interrupt */ 962 IXGBE_WRITE_REG(hw, IXGBE_VTEITR(adapter->mbxvec), IXV_LINK_ITR); 963 964 /* Stats init */ 965 ixv_init_stats(adapter); 966 967 /* Config/Enable Link */ 968 ixv_config_link(adapter); 969 970 /* And now turn on interrupts */ 971 ixv_enable_intr(adapter); 972 973 /* Now inform the stack we're ready */ 974 ifp->if_flags |= IFF_RUNNING; 975 ifp->if_flags &= ~IFF_OACTIVE; 976 977 return; 978 } 979 980 static int 981 ixv_init(struct ifnet *ifp) 982 { 983 struct adapter *adapter = ifp->if_softc; 984 985 IXV_CORE_LOCK(adapter); 986 ixv_init_locked(adapter); 987 IXV_CORE_UNLOCK(adapter); 988 return 0; 989 } 990 991 992 /* 993 ** 994 ** MSIX Interrupt Handlers and Tasklets 995 ** 996 */ 997 998 static inline void 999 ixv_enable_queue(struct adapter *adapter, u32 vector) 1000 { 1001 struct ixgbe_hw *hw = &adapter->hw; 1002 u32 queue = 1 << vector; 1003 u32 mask; 1004 1005 mask = (IXGBE_EIMS_RTX_QUEUE & queue); 1006 IXGBE_WRITE_REG(hw, IXGBE_VTEIMS, mask); 1007 } 1008 1009 static inline void 1010 ixv_disable_queue(struct adapter *adapter, u32 vector) 1011 { 1012 struct ixgbe_hw *hw = &adapter->hw; 1013 u64 queue = (u64)(1 << vector); 1014 u32 mask; 1015 1016 mask = (IXGBE_EIMS_RTX_QUEUE & queue); 1017 IXGBE_WRITE_REG(hw, IXGBE_VTEIMC, mask); 1018 } 1019 1020 static inline void 1021 ixv_rearm_queues(struct adapter *adapter, u64 queues) 1022 { 1023 u32 mask = (IXGBE_EIMS_RTX_QUEUE & queues); 1024 IXGBE_WRITE_REG(&adapter->hw, IXGBE_VTEICS, mask); 1025 } 1026 1027 1028 static void 1029 ixv_handle_que(void *context) 1030 { 1031 struct ix_queue *que = context; 1032 struct adapter *adapter = que->adapter; 1033 struct tx_ring *txr = que->txr; 1034 struct ifnet *ifp = adapter->ifp; 1035 bool more; 1036 1037 if (ifp->if_flags & IFF_RUNNING) { 1038 more = ixv_rxeof(que, adapter->rx_process_limit); 1039 IXV_TX_LOCK(txr); 1040 ixv_txeof(txr); 1041 #if __FreeBSD_version >= 800000 1042 if (!drbr_empty(ifp, txr->br)) 1043 ixv_mq_start_locked(ifp, txr, NULL); 1044 #else 1045 if (!IFQ_IS_EMPTY(&ifp->if_snd)) 1046 ixv_start_locked(txr, ifp); 1047 #endif 1048 IXV_TX_UNLOCK(txr); 1049 if (more) { 1050 adapter->req.ev_count++; 1051 softint_schedule(que->que_si); 1052 return; 1053 } 1054 } 1055 1056 /* Reenable this interrupt */ 1057 ixv_enable_queue(adapter, que->msix); 1058 return; 1059 } 1060 1061 /********************************************************************* 1062 * 1063 * MSI Queue Interrupt Service routine 1064 * 1065 **********************************************************************/ 1066 void 1067 ixv_msix_que(void *arg) 1068 { 1069 struct ix_queue *que = arg; 1070 struct adapter *adapter = que->adapter; 1071 struct tx_ring *txr = que->txr; 1072 struct rx_ring *rxr = que->rxr; 1073 bool more_tx, more_rx; 1074 u32 newitr = 0; 1075 1076 ixv_disable_queue(adapter, que->msix); 1077 ++que->irqs; 1078 1079 more_rx = ixv_rxeof(que, adapter->rx_process_limit); 1080 1081 IXV_TX_LOCK(txr); 1082 more_tx = ixv_txeof(txr); 1083 /* 1084 ** Make certain that if the stack 1085 ** has anything queued the task gets 1086 ** scheduled to handle it. 1087 */ 1088 #if __FreeBSD_version < 800000 1089 if (!IFQ_IS_EMPTY(&adapter->ifp->if_snd)) 1090 #else 1091 if (!drbr_empty(adapter->ifp, txr->br)) 1092 #endif 1093 more_tx = 1; 1094 IXV_TX_UNLOCK(txr); 1095 1096 more_rx = ixv_rxeof(que, adapter->rx_process_limit); 1097 1098 /* Do AIM now? */ 1099 1100 if (ixv_enable_aim == FALSE) 1101 goto no_calc; 1102 /* 1103 ** Do Adaptive Interrupt Moderation: 1104 ** - Write out last calculated setting 1105 ** - Calculate based on average size over 1106 ** the last interval. 1107 */ 1108 if (que->eitr_setting) 1109 IXGBE_WRITE_REG(&adapter->hw, 1110 IXGBE_VTEITR(que->msix), 1111 que->eitr_setting); 1112 1113 que->eitr_setting = 0; 1114 1115 /* Idle, do nothing */ 1116 if ((txr->bytes == 0) && (rxr->bytes == 0)) 1117 goto no_calc; 1118 1119 if ((txr->bytes) && (txr->packets)) 1120 newitr = txr->bytes/txr->packets; 1121 if ((rxr->bytes) && (rxr->packets)) 1122 newitr = max(newitr, 1123 (rxr->bytes / rxr->packets)); 1124 newitr += 24; /* account for hardware frame, crc */ 1125 1126 /* set an upper boundary */ 1127 newitr = min(newitr, 3000); 1128 1129 /* Be nice to the mid range */ 1130 if ((newitr > 300) && (newitr < 1200)) 1131 newitr = (newitr / 3); 1132 else 1133 newitr = (newitr / 2); 1134 1135 newitr |= newitr << 16; 1136 1137 /* save for next interrupt */ 1138 que->eitr_setting = newitr; 1139 1140 /* Reset state */ 1141 txr->bytes = 0; 1142 txr->packets = 0; 1143 rxr->bytes = 0; 1144 rxr->packets = 0; 1145 1146 no_calc: 1147 if (more_tx || more_rx) 1148 softint_schedule(que->que_si); 1149 else /* Reenable this interrupt */ 1150 ixv_enable_queue(adapter, que->msix); 1151 return; 1152 } 1153 1154 static void 1155 ixv_msix_mbx(void *arg) 1156 { 1157 struct adapter *adapter = arg; 1158 struct ixgbe_hw *hw = &adapter->hw; 1159 u32 reg; 1160 1161 ++adapter->mbx_irq.ev_count; 1162 1163 /* First get the cause */ 1164 reg = IXGBE_READ_REG(hw, IXGBE_VTEICS); 1165 /* Clear interrupt with write */ 1166 IXGBE_WRITE_REG(hw, IXGBE_VTEICR, reg); 1167 1168 /* Link status change */ 1169 if (reg & IXGBE_EICR_LSC) 1170 softint_schedule(adapter->mbx_si); 1171 1172 IXGBE_WRITE_REG(hw, IXGBE_VTEIMS, IXGBE_EIMS_OTHER); 1173 return; 1174 } 1175 1176 /********************************************************************* 1177 * 1178 * Media Ioctl callback 1179 * 1180 * This routine is called whenever the user queries the status of 1181 * the interface using ifconfig. 1182 * 1183 **********************************************************************/ 1184 static void 1185 ixv_media_status(struct ifnet * ifp, struct ifmediareq * ifmr) 1186 { 1187 struct adapter *adapter = ifp->if_softc; 1188 1189 INIT_DEBUGOUT("ixv_media_status: begin"); 1190 IXV_CORE_LOCK(adapter); 1191 ixv_update_link_status(adapter); 1192 1193 ifmr->ifm_status = IFM_AVALID; 1194 ifmr->ifm_active = IFM_ETHER; 1195 1196 if (!adapter->link_active) { 1197 IXV_CORE_UNLOCK(adapter); 1198 return; 1199 } 1200 1201 ifmr->ifm_status |= IFM_ACTIVE; 1202 1203 switch (adapter->link_speed) { 1204 case IXGBE_LINK_SPEED_1GB_FULL: 1205 ifmr->ifm_active |= IFM_1000_T | IFM_FDX; 1206 break; 1207 case IXGBE_LINK_SPEED_10GB_FULL: 1208 ifmr->ifm_active |= IFM_FDX; 1209 break; 1210 } 1211 1212 IXV_CORE_UNLOCK(adapter); 1213 1214 return; 1215 } 1216 1217 /********************************************************************* 1218 * 1219 * Media Ioctl callback 1220 * 1221 * This routine is called when the user changes speed/duplex using 1222 * media/mediopt option with ifconfig. 1223 * 1224 **********************************************************************/ 1225 static int 1226 ixv_media_change(struct ifnet * ifp) 1227 { 1228 struct adapter *adapter = ifp->if_softc; 1229 struct ifmedia *ifm = &adapter->media; 1230 1231 INIT_DEBUGOUT("ixv_media_change: begin"); 1232 1233 if (IFM_TYPE(ifm->ifm_media) != IFM_ETHER) 1234 return (EINVAL); 1235 1236 switch (IFM_SUBTYPE(ifm->ifm_media)) { 1237 case IFM_AUTO: 1238 break; 1239 default: 1240 device_printf(adapter->dev, "Only auto media type\n"); 1241 return (EINVAL); 1242 } 1243 1244 return (0); 1245 } 1246 1247 /********************************************************************* 1248 * 1249 * This routine maps the mbufs to tx descriptors, allowing the 1250 * TX engine to transmit the packets. 1251 * - return 0 on success, positive on failure 1252 * 1253 **********************************************************************/ 1254 1255 static int 1256 ixv_xmit(struct tx_ring *txr, struct mbuf *m_head) 1257 { 1258 struct m_tag *mtag; 1259 struct adapter *adapter = txr->adapter; 1260 struct ethercom *ec = &adapter->osdep.ec; 1261 u32 olinfo_status = 0, cmd_type_len; 1262 u32 paylen = 0; 1263 int i, j, error, nsegs; 1264 int first, last = 0; 1265 bus_dmamap_t map; 1266 struct ixv_tx_buf *txbuf; 1267 union ixgbe_adv_tx_desc *txd = NULL; 1268 1269 /* Basic descriptor defines */ 1270 cmd_type_len = (IXGBE_ADVTXD_DTYP_DATA | 1271 IXGBE_ADVTXD_DCMD_IFCS | IXGBE_ADVTXD_DCMD_DEXT); 1272 1273 if ((mtag = VLAN_OUTPUT_TAG(ec, m_head)) != NULL) 1274 cmd_type_len |= IXGBE_ADVTXD_DCMD_VLE; 1275 1276 /* 1277 * Important to capture the first descriptor 1278 * used because it will contain the index of 1279 * the one we tell the hardware to report back 1280 */ 1281 first = txr->next_avail_desc; 1282 txbuf = &txr->tx_buffers[first]; 1283 map = txbuf->map; 1284 1285 /* 1286 * Map the packet for DMA. 1287 */ 1288 error = bus_dmamap_load_mbuf(txr->txtag->dt_dmat, map, 1289 m_head, BUS_DMA_NOWAIT); 1290 1291 switch (error) { 1292 case EAGAIN: 1293 adapter->eagain_tx_dma_setup.ev_count++; 1294 return EAGAIN; 1295 case ENOMEM: 1296 adapter->enomem_tx_dma_setup.ev_count++; 1297 return EAGAIN; 1298 case EFBIG: 1299 adapter->efbig_tx_dma_setup.ev_count++; 1300 return error; 1301 case EINVAL: 1302 adapter->einval_tx_dma_setup.ev_count++; 1303 return error; 1304 default: 1305 adapter->other_tx_dma_setup.ev_count++; 1306 return error; 1307 case 0: 1308 break; 1309 } 1310 1311 /* Make certain there are enough descriptors */ 1312 if (nsegs > txr->tx_avail - 2) { 1313 txr->no_desc_avail.ev_count++; 1314 /* XXX s/ixgbe/ixv/ */ 1315 ixgbe_dmamap_unload(txr->txtag, txbuf->map); 1316 return EAGAIN; 1317 } 1318 1319 /* 1320 ** Set up the appropriate offload context 1321 ** this becomes the first descriptor of 1322 ** a packet. 1323 */ 1324 if (m_head->m_pkthdr.csum_flags & (M_CSUM_TSOv4|M_CSUM_TSOv6)) { 1325 if (ixv_tso_setup(txr, m_head, &paylen)) { 1326 cmd_type_len |= IXGBE_ADVTXD_DCMD_TSE; 1327 olinfo_status |= IXGBE_TXD_POPTS_IXSM << 8; 1328 olinfo_status |= IXGBE_TXD_POPTS_TXSM << 8; 1329 olinfo_status |= paylen << IXGBE_ADVTXD_PAYLEN_SHIFT; 1330 ++adapter->tso_tx.ev_count; 1331 } else { 1332 ++adapter->tso_err.ev_count; 1333 /* XXX unload DMA map! --dyoung -> easy? --msaitoh */ 1334 return (ENXIO); 1335 } 1336 } else 1337 olinfo_status |= ixv_tx_ctx_setup(txr, m_head); 1338 1339 /* Record payload length */ 1340 if (paylen == 0) 1341 olinfo_status |= m_head->m_pkthdr.len << 1342 IXGBE_ADVTXD_PAYLEN_SHIFT; 1343 1344 i = txr->next_avail_desc; 1345 for (j = 0; j < map->dm_nsegs; j++) { 1346 bus_size_t seglen; 1347 bus_addr_t segaddr; 1348 1349 txbuf = &txr->tx_buffers[i]; 1350 txd = &txr->tx_base[i]; 1351 seglen = map->dm_segs[j].ds_len; 1352 segaddr = htole64(map->dm_segs[j].ds_addr); 1353 1354 txd->read.buffer_addr = segaddr; 1355 txd->read.cmd_type_len = htole32(txr->txd_cmd | 1356 cmd_type_len |seglen); 1357 txd->read.olinfo_status = htole32(olinfo_status); 1358 last = i; /* descriptor that will get completion IRQ */ 1359 1360 if (++i == adapter->num_tx_desc) 1361 i = 0; 1362 1363 txbuf->m_head = NULL; 1364 txbuf->eop_index = -1; 1365 } 1366 1367 txd->read.cmd_type_len |= 1368 htole32(IXGBE_TXD_CMD_EOP | IXGBE_TXD_CMD_RS); 1369 txr->tx_avail -= map->dm_nsegs; 1370 txr->next_avail_desc = i; 1371 1372 txbuf->m_head = m_head; 1373 /* Swap the dma map between the first and last descriptor */ 1374 txr->tx_buffers[first].map = txbuf->map; 1375 txbuf->map = map; 1376 bus_dmamap_sync(txr->txtag->dt_dmat, map, 0, m_head->m_pkthdr.len, 1377 BUS_DMASYNC_PREWRITE); 1378 1379 /* Set the index of the descriptor that will be marked done */ 1380 txbuf = &txr->tx_buffers[first]; 1381 txbuf->eop_index = last; 1382 1383 /* XXX s/ixgbe/ixg/ */ 1384 ixgbe_dmamap_sync(txr->txdma.dma_tag, txr->txdma.dma_map, 1385 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); 1386 /* 1387 * Advance the Transmit Descriptor Tail (Tdt), this tells the 1388 * hardware that this frame is available to transmit. 1389 */ 1390 ++txr->total_packets.ev_count; 1391 IXGBE_WRITE_REG(&adapter->hw, IXGBE_VFTDT(txr->me), i); 1392 1393 return 0; 1394 } 1395 1396 1397 /********************************************************************* 1398 * Multicast Update 1399 * 1400 * This routine is called whenever multicast address list is updated. 1401 * 1402 **********************************************************************/ 1403 #define IXGBE_RAR_ENTRIES 16 1404 1405 static void 1406 ixv_set_multi(struct adapter *adapter) 1407 { 1408 struct ether_multi *enm; 1409 struct ether_multistep step; 1410 u8 mta[MAX_NUM_MULTICAST_ADDRESSES * IXGBE_ETH_LENGTH_OF_ADDRESS]; 1411 u8 *update_ptr; 1412 int mcnt = 0; 1413 struct ethercom *ec = &adapter->osdep.ec; 1414 1415 IOCTL_DEBUGOUT("ixv_set_multi: begin"); 1416 1417 ETHER_FIRST_MULTI(step, ec, enm); 1418 while (enm != NULL) { 1419 bcopy(enm->enm_addrlo, 1420 &mta[mcnt * IXGBE_ETH_LENGTH_OF_ADDRESS], 1421 IXGBE_ETH_LENGTH_OF_ADDRESS); 1422 mcnt++; 1423 /* XXX This might be required --msaitoh */ 1424 if (mcnt >= MAX_NUM_MULTICAST_ADDRESSES) 1425 break; 1426 ETHER_NEXT_MULTI(step, enm); 1427 } 1428 1429 update_ptr = mta; 1430 1431 ixgbe_update_mc_addr_list(&adapter->hw, 1432 update_ptr, mcnt, ixv_mc_array_itr, TRUE); 1433 1434 return; 1435 } 1436 1437 /* 1438 * This is an iterator function now needed by the multicast 1439 * shared code. It simply feeds the shared code routine the 1440 * addresses in the array of ixv_set_multi() one by one. 1441 */ 1442 static u8 * 1443 ixv_mc_array_itr(struct ixgbe_hw *hw, u8 **update_ptr, u32 *vmdq) 1444 { 1445 u8 *addr = *update_ptr; 1446 u8 *newptr; 1447 *vmdq = 0; 1448 1449 newptr = addr + IXGBE_ETH_LENGTH_OF_ADDRESS; 1450 *update_ptr = newptr; 1451 return addr; 1452 } 1453 1454 /********************************************************************* 1455 * Timer routine 1456 * 1457 * This routine checks for link status,updates statistics, 1458 * and runs the watchdog check. 1459 * 1460 **********************************************************************/ 1461 1462 static void 1463 ixv_local_timer1(void *arg) 1464 { 1465 struct adapter *adapter = arg; 1466 device_t dev = adapter->dev; 1467 struct tx_ring *txr = adapter->tx_rings; 1468 int i; 1469 struct timeval now, elapsed; 1470 1471 KASSERT(mutex_owned(&adapter->core_mtx)); 1472 1473 ixv_update_link_status(adapter); 1474 1475 /* Stats Update */ 1476 ixv_update_stats(adapter); 1477 1478 /* 1479 * If the interface has been paused 1480 * then don't do the watchdog check 1481 */ 1482 if (IXGBE_READ_REG(&adapter->hw, IXGBE_TFCS) & IXGBE_TFCS_TXOFF) 1483 goto out; 1484 /* 1485 ** Check for time since any descriptor was cleaned 1486 */ 1487 for (i = 0; i < adapter->num_queues; i++, txr++) { 1488 IXV_TX_LOCK(txr); 1489 if (txr->watchdog_check == FALSE) { 1490 IXV_TX_UNLOCK(txr); 1491 continue; 1492 } 1493 getmicrotime(&now); 1494 timersub(&now, &txr->watchdog_time, &elapsed); 1495 if (tvtohz(&elapsed) > IXV_WATCHDOG) 1496 goto hung; 1497 IXV_TX_UNLOCK(txr); 1498 } 1499 out: 1500 ixv_rearm_queues(adapter, adapter->que_mask); 1501 callout_reset(&adapter->timer, hz, ixv_local_timer, adapter); 1502 return; 1503 1504 hung: 1505 device_printf(adapter->dev, "Watchdog timeout -- resetting\n"); 1506 device_printf(dev,"Queue(%d) tdh = %d, hw tdt = %d\n", txr->me, 1507 IXGBE_READ_REG(&adapter->hw, IXGBE_VFTDH(i)), 1508 IXGBE_READ_REG(&adapter->hw, IXGBE_VFTDT(i))); 1509 device_printf(dev,"TX(%d) desc avail = %d," 1510 "Next TX to Clean = %d\n", 1511 txr->me, txr->tx_avail, txr->next_to_clean); 1512 adapter->ifp->if_flags &= ~IFF_RUNNING; 1513 adapter->watchdog_events.ev_count++; 1514 IXV_TX_UNLOCK(txr); 1515 ixv_init_locked(adapter); 1516 } 1517 1518 static void 1519 ixv_local_timer(void *arg) 1520 { 1521 struct adapter *adapter = arg; 1522 1523 IXV_CORE_LOCK(adapter); 1524 ixv_local_timer1(adapter); 1525 IXV_CORE_UNLOCK(adapter); 1526 } 1527 1528 /* 1529 ** Note: this routine updates the OS on the link state 1530 ** the real check of the hardware only happens with 1531 ** a link interrupt. 1532 */ 1533 static void 1534 ixv_update_link_status(struct adapter *adapter) 1535 { 1536 struct ifnet *ifp = adapter->ifp; 1537 struct tx_ring *txr = adapter->tx_rings; 1538 device_t dev = adapter->dev; 1539 1540 1541 if (adapter->link_up){ 1542 if (adapter->link_active == FALSE) { 1543 if (bootverbose) 1544 device_printf(dev,"Link is up %d Gbps %s \n", 1545 ((adapter->link_speed == 128)? 10:1), 1546 "Full Duplex"); 1547 adapter->link_active = TRUE; 1548 if_link_state_change(ifp, LINK_STATE_UP); 1549 } 1550 } else { /* Link down */ 1551 if (adapter->link_active == TRUE) { 1552 if (bootverbose) 1553 device_printf(dev,"Link is Down\n"); 1554 if_link_state_change(ifp, LINK_STATE_DOWN); 1555 adapter->link_active = FALSE; 1556 for (int i = 0; i < adapter->num_queues; 1557 i++, txr++) 1558 txr->watchdog_check = FALSE; 1559 } 1560 } 1561 1562 return; 1563 } 1564 1565 1566 static void 1567 ixv_ifstop(struct ifnet *ifp, int disable) 1568 { 1569 struct adapter *adapter = ifp->if_softc; 1570 1571 IXV_CORE_LOCK(adapter); 1572 ixv_stop(adapter); 1573 IXV_CORE_UNLOCK(adapter); 1574 } 1575 1576 /********************************************************************* 1577 * 1578 * This routine disables all traffic on the adapter by issuing a 1579 * global reset on the MAC and deallocates TX/RX buffers. 1580 * 1581 **********************************************************************/ 1582 1583 static void 1584 ixv_stop(void *arg) 1585 { 1586 struct ifnet *ifp; 1587 struct adapter *adapter = arg; 1588 struct ixgbe_hw *hw = &adapter->hw; 1589 ifp = adapter->ifp; 1590 1591 KASSERT(mutex_owned(&adapter->core_mtx)); 1592 1593 INIT_DEBUGOUT("ixv_stop: begin\n"); 1594 ixv_disable_intr(adapter); 1595 1596 /* Tell the stack that the interface is no longer active */ 1597 ifp->if_flags &= ~(IFF_RUNNING | IFF_OACTIVE); 1598 1599 ixgbe_reset_hw(hw); 1600 adapter->hw.adapter_stopped = FALSE; 1601 ixgbe_stop_adapter(hw); 1602 callout_stop(&adapter->timer); 1603 1604 /* reprogram the RAR[0] in case user changed it. */ 1605 ixgbe_set_rar(hw, 0, hw->mac.addr, 0, IXGBE_RAH_AV); 1606 1607 return; 1608 } 1609 1610 1611 /********************************************************************* 1612 * 1613 * Determine hardware revision. 1614 * 1615 **********************************************************************/ 1616 static void 1617 ixv_identify_hardware(struct adapter *adapter) 1618 { 1619 u16 pci_cmd_word; 1620 pcitag_t tag; 1621 pci_chipset_tag_t pc; 1622 pcireg_t subid, id; 1623 struct ixgbe_hw *hw = &adapter->hw; 1624 1625 pc = adapter->osdep.pc; 1626 tag = adapter->osdep.tag; 1627 1628 /* 1629 ** Make sure BUSMASTER is set, on a VM under 1630 ** KVM it may not be and will break things. 1631 */ 1632 pci_cmd_word = pci_conf_read(pc, tag, PCI_COMMAND_STATUS_REG); 1633 if (!((pci_cmd_word & PCI_COMMAND_MASTER_ENABLE) && 1634 (pci_cmd_word & PCI_COMMAND_MEM_ENABLE))) { 1635 INIT_DEBUGOUT("Memory Access and/or Bus Master " 1636 "bits were not set!\n"); 1637 pci_cmd_word |= 1638 (PCI_COMMAND_MASTER_ENABLE | PCI_COMMAND_MEM_ENABLE); 1639 pci_conf_write(pc, tag, PCI_COMMAND_STATUS_REG, pci_cmd_word); 1640 } 1641 1642 id = pci_conf_read(pc, tag, PCI_ID_REG); 1643 subid = pci_conf_read(pc, tag, PCI_SUBSYS_ID_REG); 1644 1645 /* Save off the information about this board */ 1646 hw->vendor_id = PCI_VENDOR(id); 1647 hw->device_id = PCI_PRODUCT(id); 1648 hw->revision_id = PCI_REVISION(pci_conf_read(pc, tag, PCI_CLASS_REG)); 1649 hw->subsystem_vendor_id = PCI_SUBSYS_VENDOR(subid); 1650 hw->subsystem_device_id = PCI_SUBSYS_ID(subid); 1651 1652 return; 1653 } 1654 1655 /********************************************************************* 1656 * 1657 * Setup MSIX Interrupt resources and handlers 1658 * 1659 **********************************************************************/ 1660 static int 1661 ixv_allocate_msix(struct adapter *adapter) 1662 { 1663 #if !defined(NETBSD_MSI_OR_MSIX) 1664 return 0; 1665 #else 1666 device_t dev = adapter->dev; 1667 struct ix_queue *que = adapter->queues; 1668 int error, rid, vector = 0; 1669 pcitag_t tag; 1670 pci_chipset_tag_t pc; 1671 1672 pc = adapter->osdep.pc; 1673 tag = adapter->osdep.tag; 1674 1675 for (int i = 0; i < adapter->num_queues; i++, vector++, que++) { 1676 rid = vector + 1; 1677 que->res = bus_alloc_resource_any(dev, SYS_RES_IRQ, &rid, 1678 RF_SHAREABLE | RF_ACTIVE); 1679 if (que->res == NULL) { 1680 aprint_error_dev(dev,"Unable to allocate" 1681 " bus resource: que interrupt [%d]\n", vector); 1682 return (ENXIO); 1683 } 1684 /* Set the handler function */ 1685 error = bus_setup_intr(dev, que->res, 1686 INTR_TYPE_NET | INTR_MPSAFE, NULL, 1687 ixv_msix_que, que, &que->tag); 1688 if (error) { 1689 que->res = NULL; 1690 aprint_error_dev(dev, 1691 "Failed to register QUE handler"); 1692 return (error); 1693 } 1694 #if __FreeBSD_version >= 800504 1695 bus_describe_intr(dev, que->res, que->tag, "que %d", i); 1696 #endif 1697 que->msix = vector; 1698 adapter->que_mask |= (u64)(1 << que->msix); 1699 /* 1700 ** Bind the msix vector, and thus the 1701 ** ring to the corresponding cpu. 1702 */ 1703 if (adapter->num_queues > 1) 1704 bus_bind_intr(dev, que->res, i); 1705 1706 que->que_si = softint_establish(SOFTINT_NET, ixv_handle_que, 1707 que); 1708 } 1709 1710 /* and Mailbox */ 1711 rid = vector + 1; 1712 adapter->res = bus_alloc_resource_any(dev, 1713 SYS_RES_IRQ, &rid, RF_SHAREABLE | RF_ACTIVE); 1714 if (!adapter->res) { 1715 aprint_error_dev(dev,"Unable to allocate" 1716 " bus resource: MBX interrupt [%d]\n", rid); 1717 return (ENXIO); 1718 } 1719 /* Set the mbx handler function */ 1720 error = bus_setup_intr(dev, adapter->res, 1721 INTR_TYPE_NET | INTR_MPSAFE, NULL, 1722 ixv_msix_mbx, adapter, &adapter->tag); 1723 if (error) { 1724 adapter->res = NULL; 1725 aprint_error_dev(dev, "Failed to register LINK handler"); 1726 return (error); 1727 } 1728 #if __FreeBSD_version >= 800504 1729 bus_describe_intr(dev, adapter->res, adapter->tag, "mbx"); 1730 #endif 1731 adapter->mbxvec = vector; 1732 /* Tasklets for Mailbox */ 1733 adapter->mbx_si = softint_establish(SOFTINT_NET, ixv_handle_mbx, 1734 adapter); 1735 /* 1736 ** Due to a broken design QEMU will fail to properly 1737 ** enable the guest for MSIX unless the vectors in 1738 ** the table are all set up, so we must rewrite the 1739 ** ENABLE in the MSIX control register again at this 1740 ** point to cause it to successfully initialize us. 1741 */ 1742 if (adapter->hw.mac.type == ixgbe_mac_82599_vf) { 1743 int msix_ctrl; 1744 pci_get_capability(pc, tag, PCI_CAP_MSIX, &rid); 1745 rid += PCI_MSIX_CTL; 1746 msix_ctrl = pci_read_config(pc, tag, rid); 1747 msix_ctrl |= PCI_MSIX_CTL_ENABLE; 1748 pci_conf_write(pc, tag, msix_ctrl); 1749 } 1750 1751 return (0); 1752 #endif 1753 } 1754 1755 /* 1756 * Setup MSIX resources, note that the VF 1757 * device MUST use MSIX, there is no fallback. 1758 */ 1759 static int 1760 ixv_setup_msix(struct adapter *adapter) 1761 { 1762 #if !defined(NETBSD_MSI_OR_MSIX) 1763 return 0; 1764 #else 1765 device_t dev = adapter->dev; 1766 int rid, vectors, want = 2; 1767 1768 1769 /* First try MSI/X */ 1770 rid = PCIR_BAR(3); 1771 adapter->msix_mem = bus_alloc_resource_any(dev, 1772 SYS_RES_MEMORY, &rid, RF_ACTIVE); 1773 if (!adapter->msix_mem) { 1774 device_printf(adapter->dev, 1775 "Unable to map MSIX table \n"); 1776 goto out; 1777 } 1778 1779 vectors = pci_msix_count(dev); 1780 if (vectors < 2) { 1781 bus_release_resource(dev, SYS_RES_MEMORY, 1782 rid, adapter->msix_mem); 1783 adapter->msix_mem = NULL; 1784 goto out; 1785 } 1786 1787 /* 1788 ** Want two vectors: one for a queue, 1789 ** plus an additional for mailbox. 1790 */ 1791 if (pci_alloc_msix(dev, &want) == 0) { 1792 device_printf(adapter->dev, 1793 "Using MSIX interrupts with %d vectors\n", want); 1794 return (want); 1795 } 1796 out: 1797 device_printf(adapter->dev,"MSIX config error\n"); 1798 return (ENXIO); 1799 #endif 1800 } 1801 1802 1803 static int 1804 ixv_allocate_pci_resources(struct adapter *adapter, 1805 const struct pci_attach_args *pa) 1806 { 1807 pcireg_t memtype; 1808 device_t dev = adapter->dev; 1809 bus_addr_t addr; 1810 int flags; 1811 1812 memtype = pci_mapreg_type(pa->pa_pc, pa->pa_tag, PCI_BAR(0)); 1813 1814 switch (memtype) { 1815 case PCI_MAPREG_TYPE_MEM | PCI_MAPREG_MEM_TYPE_32BIT: 1816 case PCI_MAPREG_TYPE_MEM | PCI_MAPREG_MEM_TYPE_64BIT: 1817 adapter->osdep.mem_bus_space_tag = pa->pa_memt; 1818 if (pci_mapreg_info(pa->pa_pc, pa->pa_tag, PCI_BAR(0), 1819 memtype, &addr, &adapter->osdep.mem_size, &flags) != 0) 1820 goto map_err; 1821 if ((flags & BUS_SPACE_MAP_PREFETCHABLE) != 0) { 1822 aprint_normal_dev(dev, "clearing prefetchable bit\n"); 1823 flags &= ~BUS_SPACE_MAP_PREFETCHABLE; 1824 } 1825 if (bus_space_map(adapter->osdep.mem_bus_space_tag, addr, 1826 adapter->osdep.mem_size, flags, 1827 &adapter->osdep.mem_bus_space_handle) != 0) { 1828 map_err: 1829 adapter->osdep.mem_size = 0; 1830 aprint_error_dev(dev, "unable to map BAR0\n"); 1831 return ENXIO; 1832 } 1833 break; 1834 default: 1835 aprint_error_dev(dev, "unexpected type on BAR0\n"); 1836 return ENXIO; 1837 } 1838 1839 adapter->num_queues = 1; 1840 adapter->hw.back = &adapter->osdep; 1841 1842 /* 1843 ** Now setup MSI/X, should 1844 ** return us the number of 1845 ** configured vectors. 1846 */ 1847 adapter->msix = ixv_setup_msix(adapter); 1848 if (adapter->msix == ENXIO) 1849 return (ENXIO); 1850 else 1851 return (0); 1852 } 1853 1854 static void 1855 ixv_free_pci_resources(struct adapter * adapter) 1856 { 1857 #if defined(NETBSD_MSI_OR_MSIX) 1858 struct ix_queue *que = adapter->queues; 1859 device_t dev = adapter->dev; 1860 int rid, memrid; 1861 1862 memrid = PCI_BAR(MSIX_BAR); 1863 1864 /* 1865 ** There is a slight possibility of a failure mode 1866 ** in attach that will result in entering this function 1867 ** before interrupt resources have been initialized, and 1868 ** in that case we do not want to execute the loops below 1869 ** We can detect this reliably by the state of the adapter 1870 ** res pointer. 1871 */ 1872 if (adapter->res == NULL) 1873 goto mem; 1874 1875 /* 1876 ** Release all msix queue resources: 1877 */ 1878 for (int i = 0; i < adapter->num_queues; i++, que++) { 1879 rid = que->msix + 1; 1880 if (que->tag != NULL) { 1881 bus_teardown_intr(dev, que->res, que->tag); 1882 que->tag = NULL; 1883 } 1884 if (que->res != NULL) 1885 bus_release_resource(dev, SYS_RES_IRQ, rid, que->res); 1886 } 1887 1888 1889 /* Clean the Legacy or Link interrupt last */ 1890 if (adapter->mbxvec) /* we are doing MSIX */ 1891 rid = adapter->mbxvec + 1; 1892 else 1893 (adapter->msix != 0) ? (rid = 1):(rid = 0); 1894 1895 if (adapter->tag != NULL) { 1896 bus_teardown_intr(dev, adapter->res, adapter->tag); 1897 adapter->tag = NULL; 1898 } 1899 if (adapter->res != NULL) 1900 bus_release_resource(dev, SYS_RES_IRQ, rid, adapter->res); 1901 1902 mem: 1903 if (adapter->msix) 1904 pci_release_msi(dev); 1905 1906 if (adapter->msix_mem != NULL) 1907 bus_release_resource(dev, SYS_RES_MEMORY, 1908 memrid, adapter->msix_mem); 1909 1910 if (adapter->pci_mem != NULL) 1911 bus_release_resource(dev, SYS_RES_MEMORY, 1912 PCIR_BAR(0), adapter->pci_mem); 1913 1914 #endif 1915 return; 1916 } 1917 1918 /********************************************************************* 1919 * 1920 * Setup networking device structure and register an interface. 1921 * 1922 **********************************************************************/ 1923 static void 1924 ixv_setup_interface(device_t dev, struct adapter *adapter) 1925 { 1926 struct ethercom *ec = &adapter->osdep.ec; 1927 struct ifnet *ifp; 1928 1929 INIT_DEBUGOUT("ixv_setup_interface: begin"); 1930 1931 ifp = adapter->ifp = &ec->ec_if; 1932 strlcpy(ifp->if_xname, device_xname(dev), IFNAMSIZ); 1933 ifp->if_baudrate = 1000000000; 1934 ifp->if_init = ixv_init; 1935 ifp->if_stop = ixv_ifstop; 1936 ifp->if_softc = adapter; 1937 ifp->if_flags = IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST; 1938 ifp->if_ioctl = ixv_ioctl; 1939 #if __FreeBSD_version >= 800000 1940 ifp->if_transmit = ixv_mq_start; 1941 ifp->if_qflush = ixv_qflush; 1942 #else 1943 ifp->if_start = ixv_start; 1944 #endif 1945 ifp->if_snd.ifq_maxlen = adapter->num_tx_desc - 2; 1946 1947 if_attach(ifp); 1948 ether_ifattach(ifp, adapter->hw.mac.addr); 1949 ether_set_ifflags_cb(ec, ixv_ifflags_cb); 1950 1951 adapter->max_frame_size = 1952 ifp->if_mtu + ETHER_HDR_LEN + ETHER_CRC_LEN; 1953 1954 /* 1955 * Tell the upper layer(s) we support long frames. 1956 */ 1957 ifp->if_hdrlen = sizeof(struct ether_vlan_header); 1958 1959 ifp->if_capabilities |= IFCAP_HWCSUM | IFCAP_TSOv4; 1960 ifp->if_capenable = 0; 1961 1962 ec->ec_capabilities |= ETHERCAP_VLAN_HWCSUM; 1963 ec->ec_capabilities |= ETHERCAP_JUMBO_MTU; 1964 ec->ec_capabilities |= ETHERCAP_VLAN_HWTAGGING 1965 | ETHERCAP_VLAN_MTU; 1966 ec->ec_capenable = ec->ec_capabilities; 1967 1968 /* Don't enable LRO by default */ 1969 ifp->if_capabilities |= IFCAP_LRO; 1970 1971 /* 1972 ** Dont turn this on by default, if vlans are 1973 ** created on another pseudo device (eg. lagg) 1974 ** then vlan events are not passed thru, breaking 1975 ** operation, but with HW FILTER off it works. If 1976 ** using vlans directly on the em driver you can 1977 ** enable this and get full hardware tag filtering. 1978 */ 1979 ec->ec_capabilities |= ETHERCAP_VLAN_HWFILTER; 1980 1981 /* 1982 * Specify the media types supported by this adapter and register 1983 * callbacks to update media and link information 1984 */ 1985 ifmedia_init(&adapter->media, IFM_IMASK, ixv_media_change, 1986 ixv_media_status); 1987 ifmedia_add(&adapter->media, IFM_ETHER | IFM_FDX, 0, NULL); 1988 ifmedia_add(&adapter->media, IFM_ETHER | IFM_AUTO, 0, NULL); 1989 ifmedia_set(&adapter->media, IFM_ETHER | IFM_AUTO); 1990 1991 return; 1992 } 1993 1994 static void 1995 ixv_config_link(struct adapter *adapter) 1996 { 1997 struct ixgbe_hw *hw = &adapter->hw; 1998 u32 autoneg, err = 0; 1999 2000 if (hw->mac.ops.check_link) 2001 err = hw->mac.ops.check_link(hw, &autoneg, 2002 &adapter->link_up, FALSE); 2003 if (err) 2004 goto out; 2005 2006 if (hw->mac.ops.setup_link) 2007 err = hw->mac.ops.setup_link(hw, 2008 autoneg, adapter->link_up); 2009 out: 2010 return; 2011 } 2012 2013 /******************************************************************** 2014 * Manage DMA'able memory. 2015 *******************************************************************/ 2016 2017 static int 2018 ixv_dma_malloc(struct adapter *adapter, bus_size_t size, 2019 struct ixv_dma_alloc *dma, int mapflags) 2020 { 2021 device_t dev = adapter->dev; 2022 int r, rsegs; 2023 2024 r = ixgbe_dma_tag_create(adapter->osdep.dmat, /* parent */ 2025 DBA_ALIGN, 0, /* alignment, bounds */ 2026 size, /* maxsize */ 2027 1, /* nsegments */ 2028 size, /* maxsegsize */ 2029 BUS_DMA_ALLOCNOW, /* flags */ 2030 &dma->dma_tag); 2031 if (r != 0) { 2032 aprint_error_dev(dev, 2033 "ixv_dma_malloc: bus_dma_tag_create failed; error %u\n", r); 2034 goto fail_0; 2035 } 2036 r = bus_dmamem_alloc(dma->dma_tag->dt_dmat, 2037 size, 2038 dma->dma_tag->dt_alignment, 2039 dma->dma_tag->dt_boundary, 2040 &dma->dma_seg, 1, &rsegs, BUS_DMA_NOWAIT); 2041 if (r != 0) { 2042 aprint_error_dev(dev, 2043 "%s: bus_dmamem_alloc failed; error %u\n", __func__, r); 2044 goto fail_1; 2045 } 2046 2047 r = bus_dmamem_map(dma->dma_tag->dt_dmat, &dma->dma_seg, rsegs, 2048 size, &dma->dma_vaddr, BUS_DMA_NOWAIT); 2049 if (r != 0) { 2050 aprint_error_dev(dev, "%s: bus_dmamem_map failed; error %d\n", 2051 __func__, r); 2052 goto fail_2; 2053 } 2054 2055 r = ixgbe_dmamap_create(dma->dma_tag, 0, &dma->dma_map); 2056 if (r != 0) { 2057 aprint_error_dev(dev, "%s: bus_dmamem_map failed; error %d\n", 2058 __func__, r); 2059 goto fail_3; 2060 } 2061 2062 r = bus_dmamap_load(dma->dma_tag->dt_dmat, dma->dma_map, dma->dma_vaddr, 2063 size, 2064 NULL, 2065 mapflags | BUS_DMA_NOWAIT); 2066 if (r != 0) { 2067 aprint_error_dev(dev,"%s: bus_dmamap_load failed; error %u\n", 2068 __func__, r); 2069 goto fail_4; 2070 } 2071 dma->dma_paddr = dma->dma_map->dm_segs[0].ds_addr; 2072 dma->dma_size = size; 2073 return 0; 2074 fail_4: 2075 ixgbe_dmamap_destroy(dma->dma_tag, dma->dma_map); 2076 fail_3: 2077 bus_dmamem_unmap(dma->dma_tag->dt_dmat, dma->dma_vaddr, size); 2078 fail_2: 2079 bus_dmamem_free(dma->dma_tag->dt_dmat, &dma->dma_seg, rsegs); 2080 fail_1: 2081 ixgbe_dma_tag_destroy(dma->dma_tag); 2082 fail_0: 2083 dma->dma_map = NULL; 2084 dma->dma_tag = NULL; 2085 return (r); 2086 } 2087 2088 static void 2089 ixv_dma_free(struct adapter *adapter, struct ixv_dma_alloc *dma) 2090 { 2091 bus_dmamap_sync(dma->dma_tag->dt_dmat, dma->dma_map, 0, dma->dma_size, 2092 BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE); 2093 ixgbe_dmamap_unload(dma->dma_tag, dma->dma_map); 2094 bus_dmamem_free(dma->dma_tag->dt_dmat, &dma->dma_seg, 1); 2095 ixgbe_dma_tag_destroy(dma->dma_tag); 2096 } 2097 2098 2099 /********************************************************************* 2100 * 2101 * Allocate memory for the transmit and receive rings, and then 2102 * the descriptors associated with each, called only once at attach. 2103 * 2104 **********************************************************************/ 2105 static int 2106 ixv_allocate_queues(struct adapter *adapter) 2107 { 2108 device_t dev = adapter->dev; 2109 struct ix_queue *que; 2110 struct tx_ring *txr; 2111 struct rx_ring *rxr; 2112 int rsize, tsize, error = 0; 2113 int txconf = 0, rxconf = 0; 2114 2115 /* First allocate the top level queue structs */ 2116 if (!(adapter->queues = 2117 (struct ix_queue *) malloc(sizeof(struct ix_queue) * 2118 adapter->num_queues, M_DEVBUF, M_NOWAIT | M_ZERO))) { 2119 aprint_error_dev(dev, "Unable to allocate queue memory\n"); 2120 error = ENOMEM; 2121 goto fail; 2122 } 2123 2124 /* First allocate the TX ring struct memory */ 2125 if (!(adapter->tx_rings = 2126 (struct tx_ring *) malloc(sizeof(struct tx_ring) * 2127 adapter->num_queues, M_DEVBUF, M_NOWAIT | M_ZERO))) { 2128 aprint_error_dev(dev, "Unable to allocate TX ring memory\n"); 2129 error = ENOMEM; 2130 goto tx_fail; 2131 } 2132 2133 /* Next allocate the RX */ 2134 if (!(adapter->rx_rings = 2135 (struct rx_ring *) malloc(sizeof(struct rx_ring) * 2136 adapter->num_queues, M_DEVBUF, M_NOWAIT | M_ZERO))) { 2137 aprint_error_dev(dev, "Unable to allocate RX ring memory\n"); 2138 error = ENOMEM; 2139 goto rx_fail; 2140 } 2141 2142 /* For the ring itself */ 2143 tsize = roundup2(adapter->num_tx_desc * 2144 sizeof(union ixgbe_adv_tx_desc), DBA_ALIGN); 2145 2146 /* 2147 * Now set up the TX queues, txconf is needed to handle the 2148 * possibility that things fail midcourse and we need to 2149 * undo memory gracefully 2150 */ 2151 for (int i = 0; i < adapter->num_queues; i++, txconf++) { 2152 /* Set up some basics */ 2153 txr = &adapter->tx_rings[i]; 2154 txr->adapter = adapter; 2155 txr->me = i; 2156 2157 /* Initialize the TX side lock */ 2158 snprintf(txr->mtx_name, sizeof(txr->mtx_name), "%s:tx(%d)", 2159 device_xname(dev), txr->me); 2160 mutex_init(&txr->tx_mtx, MUTEX_DEFAULT, IPL_NET); 2161 2162 if (ixv_dma_malloc(adapter, tsize, 2163 &txr->txdma, BUS_DMA_NOWAIT)) { 2164 aprint_error_dev(dev, 2165 "Unable to allocate TX Descriptor memory\n"); 2166 error = ENOMEM; 2167 goto err_tx_desc; 2168 } 2169 txr->tx_base = (union ixgbe_adv_tx_desc *)txr->txdma.dma_vaddr; 2170 bzero((void *)txr->tx_base, tsize); 2171 2172 /* Now allocate transmit buffers for the ring */ 2173 if (ixv_allocate_transmit_buffers(txr)) { 2174 aprint_error_dev(dev, 2175 "Critical Failure setting up transmit buffers\n"); 2176 error = ENOMEM; 2177 goto err_tx_desc; 2178 } 2179 #if __FreeBSD_version >= 800000 2180 /* Allocate a buf ring */ 2181 txr->br = buf_ring_alloc(IXV_BR_SIZE, M_DEVBUF, 2182 M_WAITOK, &txr->tx_mtx); 2183 if (txr->br == NULL) { 2184 aprint_error_dev(dev, 2185 "Critical Failure setting up buf ring\n"); 2186 error = ENOMEM; 2187 goto err_tx_desc; 2188 } 2189 #endif 2190 } 2191 2192 /* 2193 * Next the RX queues... 2194 */ 2195 rsize = roundup2(adapter->num_rx_desc * 2196 sizeof(union ixgbe_adv_rx_desc), DBA_ALIGN); 2197 for (int i = 0; i < adapter->num_queues; i++, rxconf++) { 2198 rxr = &adapter->rx_rings[i]; 2199 /* Set up some basics */ 2200 rxr->adapter = adapter; 2201 rxr->me = i; 2202 2203 /* Initialize the RX side lock */ 2204 snprintf(rxr->mtx_name, sizeof(rxr->mtx_name), "%s:rx(%d)", 2205 device_xname(dev), rxr->me); 2206 mutex_init(&rxr->rx_mtx, MUTEX_DEFAULT, IPL_NET); 2207 2208 if (ixv_dma_malloc(adapter, rsize, 2209 &rxr->rxdma, BUS_DMA_NOWAIT)) { 2210 aprint_error_dev(dev, 2211 "Unable to allocate RxDescriptor memory\n"); 2212 error = ENOMEM; 2213 goto err_rx_desc; 2214 } 2215 rxr->rx_base = (union ixgbe_adv_rx_desc *)rxr->rxdma.dma_vaddr; 2216 bzero((void *)rxr->rx_base, rsize); 2217 2218 /* Allocate receive buffers for the ring*/ 2219 if (ixv_allocate_receive_buffers(rxr)) { 2220 aprint_error_dev(dev, 2221 "Critical Failure setting up receive buffers\n"); 2222 error = ENOMEM; 2223 goto err_rx_desc; 2224 } 2225 } 2226 2227 /* 2228 ** Finally set up the queue holding structs 2229 */ 2230 for (int i = 0; i < adapter->num_queues; i++) { 2231 que = &adapter->queues[i]; 2232 que->adapter = adapter; 2233 que->txr = &adapter->tx_rings[i]; 2234 que->rxr = &adapter->rx_rings[i]; 2235 } 2236 2237 return (0); 2238 2239 err_rx_desc: 2240 for (rxr = adapter->rx_rings; rxconf > 0; rxr++, rxconf--) 2241 ixv_dma_free(adapter, &rxr->rxdma); 2242 err_tx_desc: 2243 for (txr = adapter->tx_rings; txconf > 0; txr++, txconf--) 2244 ixv_dma_free(adapter, &txr->txdma); 2245 free(adapter->rx_rings, M_DEVBUF); 2246 rx_fail: 2247 free(adapter->tx_rings, M_DEVBUF); 2248 tx_fail: 2249 free(adapter->queues, M_DEVBUF); 2250 fail: 2251 return (error); 2252 } 2253 2254 2255 /********************************************************************* 2256 * 2257 * Allocate memory for tx_buffer structures. The tx_buffer stores all 2258 * the information needed to transmit a packet on the wire. This is 2259 * called only once at attach, setup is done every reset. 2260 * 2261 **********************************************************************/ 2262 static int 2263 ixv_allocate_transmit_buffers(struct tx_ring *txr) 2264 { 2265 struct adapter *adapter = txr->adapter; 2266 device_t dev = adapter->dev; 2267 struct ixv_tx_buf *txbuf; 2268 int error, i; 2269 2270 /* 2271 * Setup DMA descriptor areas. 2272 */ 2273 if ((error = ixgbe_dma_tag_create(adapter->osdep.dmat, /* parent */ 2274 1, 0, /* alignment, bounds */ 2275 IXV_TSO_SIZE, /* maxsize */ 2276 32, /* nsegments */ 2277 PAGE_SIZE, /* maxsegsize */ 2278 0, /* flags */ 2279 &txr->txtag))) { 2280 aprint_error_dev(dev,"Unable to allocate TX DMA tag\n"); 2281 goto fail; 2282 } 2283 2284 if (!(txr->tx_buffers = 2285 (struct ixv_tx_buf *) malloc(sizeof(struct ixv_tx_buf) * 2286 adapter->num_tx_desc, M_DEVBUF, M_NOWAIT | M_ZERO))) { 2287 aprint_error_dev(dev, "Unable to allocate tx_buffer memory\n"); 2288 error = ENOMEM; 2289 goto fail; 2290 } 2291 2292 /* Create the descriptor buffer dma maps */ 2293 txbuf = txr->tx_buffers; 2294 for (i = 0; i < adapter->num_tx_desc; i++, txbuf++) { 2295 error = ixgbe_dmamap_create(txr->txtag, 0, &txbuf->map); 2296 if (error != 0) { 2297 aprint_error_dev(dev, "Unable to create TX DMA map\n"); 2298 goto fail; 2299 } 2300 } 2301 2302 return 0; 2303 fail: 2304 /* We free all, it handles case where we are in the middle */ 2305 ixv_free_transmit_structures(adapter); 2306 return (error); 2307 } 2308 2309 /********************************************************************* 2310 * 2311 * Initialize a transmit ring. 2312 * 2313 **********************************************************************/ 2314 static void 2315 ixv_setup_transmit_ring(struct tx_ring *txr) 2316 { 2317 struct adapter *adapter = txr->adapter; 2318 struct ixv_tx_buf *txbuf; 2319 int i; 2320 2321 /* Clear the old ring contents */ 2322 IXV_TX_LOCK(txr); 2323 bzero((void *)txr->tx_base, 2324 (sizeof(union ixgbe_adv_tx_desc)) * adapter->num_tx_desc); 2325 /* Reset indices */ 2326 txr->next_avail_desc = 0; 2327 txr->next_to_clean = 0; 2328 2329 /* Free any existing tx buffers. */ 2330 txbuf = txr->tx_buffers; 2331 for (i = 0; i < adapter->num_tx_desc; i++, txbuf++) { 2332 if (txbuf->m_head != NULL) { 2333 bus_dmamap_sync(txr->txtag->dt_dmat, txbuf->map, 2334 0, txbuf->m_head->m_pkthdr.len, 2335 BUS_DMASYNC_POSTWRITE); 2336 ixgbe_dmamap_unload(txr->txtag, txbuf->map); 2337 m_freem(txbuf->m_head); 2338 txbuf->m_head = NULL; 2339 } 2340 /* Clear the EOP index */ 2341 txbuf->eop_index = -1; 2342 } 2343 2344 /* Set number of descriptors available */ 2345 txr->tx_avail = adapter->num_tx_desc; 2346 2347 ixgbe_dmamap_sync(txr->txdma.dma_tag, txr->txdma.dma_map, 2348 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); 2349 IXV_TX_UNLOCK(txr); 2350 } 2351 2352 /********************************************************************* 2353 * 2354 * Initialize all transmit rings. 2355 * 2356 **********************************************************************/ 2357 static int 2358 ixv_setup_transmit_structures(struct adapter *adapter) 2359 { 2360 struct tx_ring *txr = adapter->tx_rings; 2361 2362 for (int i = 0; i < adapter->num_queues; i++, txr++) 2363 ixv_setup_transmit_ring(txr); 2364 2365 return (0); 2366 } 2367 2368 /********************************************************************* 2369 * 2370 * Enable transmit unit. 2371 * 2372 **********************************************************************/ 2373 static void 2374 ixv_initialize_transmit_units(struct adapter *adapter) 2375 { 2376 struct tx_ring *txr = adapter->tx_rings; 2377 struct ixgbe_hw *hw = &adapter->hw; 2378 2379 2380 for (int i = 0; i < adapter->num_queues; i++, txr++) { 2381 u64 tdba = txr->txdma.dma_paddr; 2382 u32 txctrl, txdctl; 2383 2384 /* Set WTHRESH to 8, burst writeback */ 2385 txdctl = IXGBE_READ_REG(hw, IXGBE_VFTXDCTL(i)); 2386 txdctl |= (8 << 16); 2387 IXGBE_WRITE_REG(hw, IXGBE_VFTXDCTL(i), txdctl); 2388 /* Now enable */ 2389 txdctl = IXGBE_READ_REG(hw, IXGBE_VFTXDCTL(i)); 2390 txdctl |= IXGBE_TXDCTL_ENABLE; 2391 IXGBE_WRITE_REG(hw, IXGBE_VFTXDCTL(i), txdctl); 2392 2393 /* Set the HW Tx Head and Tail indices */ 2394 IXGBE_WRITE_REG(&adapter->hw, IXGBE_VFTDH(i), 0); 2395 IXGBE_WRITE_REG(&adapter->hw, IXGBE_VFTDT(i), 0); 2396 2397 /* Setup Transmit Descriptor Cmd Settings */ 2398 txr->txd_cmd = IXGBE_TXD_CMD_IFCS; 2399 txr->watchdog_check = FALSE; 2400 2401 /* Set Ring parameters */ 2402 IXGBE_WRITE_REG(hw, IXGBE_VFTDBAL(i), 2403 (tdba & 0x00000000ffffffffULL)); 2404 IXGBE_WRITE_REG(hw, IXGBE_VFTDBAH(i), (tdba >> 32)); 2405 IXGBE_WRITE_REG(hw, IXGBE_VFTDLEN(i), 2406 adapter->num_tx_desc * 2407 sizeof(struct ixgbe_legacy_tx_desc)); 2408 txctrl = IXGBE_READ_REG(hw, IXGBE_VFDCA_TXCTRL(i)); 2409 txctrl &= ~IXGBE_DCA_TXCTRL_DESC_WRO_EN; 2410 IXGBE_WRITE_REG(hw, IXGBE_VFDCA_TXCTRL(i), txctrl); 2411 break; 2412 } 2413 2414 return; 2415 } 2416 2417 /********************************************************************* 2418 * 2419 * Free all transmit rings. 2420 * 2421 **********************************************************************/ 2422 static void 2423 ixv_free_transmit_structures(struct adapter *adapter) 2424 { 2425 struct tx_ring *txr = adapter->tx_rings; 2426 2427 for (int i = 0; i < adapter->num_queues; i++, txr++) { 2428 ixv_free_transmit_buffers(txr); 2429 ixv_dma_free(adapter, &txr->txdma); 2430 IXV_TX_LOCK_DESTROY(txr); 2431 } 2432 free(adapter->tx_rings, M_DEVBUF); 2433 } 2434 2435 /********************************************************************* 2436 * 2437 * Free transmit ring related data structures. 2438 * 2439 **********************************************************************/ 2440 static void 2441 ixv_free_transmit_buffers(struct tx_ring *txr) 2442 { 2443 struct adapter *adapter = txr->adapter; 2444 struct ixv_tx_buf *tx_buffer; 2445 int i; 2446 2447 INIT_DEBUGOUT("free_transmit_ring: begin"); 2448 2449 if (txr->tx_buffers == NULL) 2450 return; 2451 2452 tx_buffer = txr->tx_buffers; 2453 for (i = 0; i < adapter->num_tx_desc; i++, tx_buffer++) { 2454 if (tx_buffer->m_head != NULL) { 2455 bus_dmamap_sync(txr->txtag->dt_dmat, tx_buffer->map, 2456 0, tx_buffer->m_head->m_pkthdr.len, 2457 BUS_DMASYNC_POSTWRITE); 2458 ixgbe_dmamap_unload(txr->txtag, tx_buffer->map); 2459 m_freem(tx_buffer->m_head); 2460 tx_buffer->m_head = NULL; 2461 if (tx_buffer->map != NULL) { 2462 ixgbe_dmamap_destroy(txr->txtag, 2463 tx_buffer->map); 2464 tx_buffer->map = NULL; 2465 } 2466 } else if (tx_buffer->map != NULL) { 2467 ixgbe_dmamap_unload(txr->txtag, tx_buffer->map); 2468 ixgbe_dmamap_destroy(txr->txtag, tx_buffer->map); 2469 tx_buffer->map = NULL; 2470 } 2471 } 2472 #if __FreeBSD_version >= 800000 2473 if (txr->br != NULL) 2474 buf_ring_free(txr->br, M_DEVBUF); 2475 #endif 2476 if (txr->tx_buffers != NULL) { 2477 free(txr->tx_buffers, M_DEVBUF); 2478 txr->tx_buffers = NULL; 2479 } 2480 if (txr->txtag != NULL) { 2481 ixgbe_dma_tag_destroy(txr->txtag); 2482 txr->txtag = NULL; 2483 } 2484 return; 2485 } 2486 2487 /********************************************************************* 2488 * 2489 * Advanced Context Descriptor setup for VLAN or L4 CSUM 2490 * 2491 **********************************************************************/ 2492 2493 static u32 2494 ixv_tx_ctx_setup(struct tx_ring *txr, struct mbuf *mp) 2495 { 2496 struct m_tag *mtag; 2497 struct adapter *adapter = txr->adapter; 2498 struct ethercom *ec = &adapter->osdep.ec; 2499 struct ixgbe_adv_tx_context_desc *TXD; 2500 struct ixv_tx_buf *tx_buffer; 2501 u32 olinfo = 0, vlan_macip_lens = 0, type_tucmd_mlhl = 0; 2502 struct ether_vlan_header *eh; 2503 struct ip ip; 2504 struct ip6_hdr ip6; 2505 int ehdrlen, ip_hlen = 0; 2506 u16 etype; 2507 u8 ipproto = 0; 2508 bool offload; 2509 int ctxd = txr->next_avail_desc; 2510 u16 vtag = 0; 2511 2512 2513 offload = ((mp->m_pkthdr.csum_flags & M_CSUM_OFFLOAD) != 0); 2514 2515 tx_buffer = &txr->tx_buffers[ctxd]; 2516 TXD = (struct ixgbe_adv_tx_context_desc *) &txr->tx_base[ctxd]; 2517 2518 /* 2519 ** In advanced descriptors the vlan tag must 2520 ** be placed into the descriptor itself. 2521 */ 2522 if ((mtag = VLAN_OUTPUT_TAG(ec, mp)) != NULL) { 2523 vtag = htole16(VLAN_TAG_VALUE(mtag) & 0xffff); 2524 vlan_macip_lens |= (vtag << IXGBE_ADVTXD_VLAN_SHIFT); 2525 } else if (!offload) 2526 return 0; 2527 2528 /* 2529 * Determine where frame payload starts. 2530 * Jump over vlan headers if already present, 2531 * helpful for QinQ too. 2532 */ 2533 KASSERT(mp->m_len >= offsetof(struct ether_vlan_header, evl_tag)); 2534 eh = mtod(mp, struct ether_vlan_header *); 2535 if (eh->evl_encap_proto == htons(ETHERTYPE_VLAN)) { 2536 KASSERT(mp->m_len >= sizeof(struct ether_vlan_header)); 2537 etype = ntohs(eh->evl_proto); 2538 ehdrlen = ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN; 2539 } else { 2540 etype = ntohs(eh->evl_encap_proto); 2541 ehdrlen = ETHER_HDR_LEN; 2542 } 2543 2544 /* Set the ether header length */ 2545 vlan_macip_lens |= ehdrlen << IXGBE_ADVTXD_MACLEN_SHIFT; 2546 2547 switch (etype) { 2548 case ETHERTYPE_IP: 2549 m_copydata(mp, ehdrlen, sizeof(ip), &ip); 2550 ip_hlen = ip.ip_hl << 2; 2551 ipproto = ip.ip_p; 2552 #if 0 2553 ip.ip_sum = 0; 2554 m_copyback(mp, ehdrlen, sizeof(ip), &ip); 2555 #else 2556 KASSERT((mp->m_pkthdr.csum_flags & M_CSUM_IPv4) == 0 || 2557 ip.ip_sum == 0); 2558 #endif 2559 type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_IPV4; 2560 break; 2561 case ETHERTYPE_IPV6: 2562 m_copydata(mp, ehdrlen, sizeof(ip6), &ip6); 2563 ip_hlen = sizeof(ip6); 2564 ipproto = ip6.ip6_nxt; 2565 type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_IPV6; 2566 break; 2567 default: 2568 break; 2569 } 2570 2571 if ((mp->m_pkthdr.csum_flags & M_CSUM_IPv4) != 0) 2572 olinfo |= IXGBE_TXD_POPTS_IXSM << 8; 2573 2574 vlan_macip_lens |= ip_hlen; 2575 type_tucmd_mlhl |= IXGBE_ADVTXD_DCMD_DEXT | IXGBE_ADVTXD_DTYP_CTXT; 2576 2577 if (mp->m_pkthdr.csum_flags & (M_CSUM_TCPv4|M_CSUM_TCPv6)) { 2578 type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_L4T_TCP; 2579 olinfo |= IXGBE_TXD_POPTS_TXSM << 8; 2580 KASSERT(ipproto == IPPROTO_TCP); 2581 } else if (mp->m_pkthdr.csum_flags & (M_CSUM_UDPv4|M_CSUM_UDPv6)) { 2582 type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_L4T_UDP; 2583 olinfo |= IXGBE_TXD_POPTS_TXSM << 8; 2584 KASSERT(ipproto == IPPROTO_UDP); 2585 } 2586 2587 /* Now copy bits into descriptor */ 2588 TXD->vlan_macip_lens |= htole32(vlan_macip_lens); 2589 TXD->type_tucmd_mlhl |= htole32(type_tucmd_mlhl); 2590 TXD->seqnum_seed = htole32(0); 2591 TXD->mss_l4len_idx = htole32(0); 2592 2593 tx_buffer->m_head = NULL; 2594 tx_buffer->eop_index = -1; 2595 2596 /* We've consumed the first desc, adjust counters */ 2597 if (++ctxd == adapter->num_tx_desc) 2598 ctxd = 0; 2599 txr->next_avail_desc = ctxd; 2600 --txr->tx_avail; 2601 2602 return olinfo; 2603 } 2604 2605 /********************************************************************** 2606 * 2607 * Setup work for hardware segmentation offload (TSO) on 2608 * adapters using advanced tx descriptors 2609 * 2610 **********************************************************************/ 2611 static bool 2612 ixv_tso_setup(struct tx_ring *txr, struct mbuf *mp, u32 *paylen) 2613 { 2614 struct m_tag *mtag; 2615 struct adapter *adapter = txr->adapter; 2616 struct ethercom *ec = &adapter->osdep.ec; 2617 struct ixgbe_adv_tx_context_desc *TXD; 2618 struct ixv_tx_buf *tx_buffer; 2619 u32 vlan_macip_lens = 0, type_tucmd_mlhl = 0; 2620 u32 mss_l4len_idx = 0; 2621 u16 vtag = 0; 2622 int ctxd, ehdrlen, hdrlen, ip_hlen, tcp_hlen; 2623 struct ether_vlan_header *eh; 2624 struct ip *ip; 2625 struct tcphdr *th; 2626 2627 2628 /* 2629 * Determine where frame payload starts. 2630 * Jump over vlan headers if already present 2631 */ 2632 eh = mtod(mp, struct ether_vlan_header *); 2633 if (eh->evl_encap_proto == htons(ETHERTYPE_VLAN)) 2634 ehdrlen = ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN; 2635 else 2636 ehdrlen = ETHER_HDR_LEN; 2637 2638 /* Ensure we have at least the IP+TCP header in the first mbuf. */ 2639 if (mp->m_len < ehdrlen + sizeof(struct ip) + sizeof(struct tcphdr)) 2640 return FALSE; 2641 2642 ctxd = txr->next_avail_desc; 2643 tx_buffer = &txr->tx_buffers[ctxd]; 2644 TXD = (struct ixgbe_adv_tx_context_desc *) &txr->tx_base[ctxd]; 2645 2646 ip = (struct ip *)(mp->m_data + ehdrlen); 2647 if (ip->ip_p != IPPROTO_TCP) 2648 return FALSE; /* 0 */ 2649 ip->ip_sum = 0; 2650 ip_hlen = ip->ip_hl << 2; 2651 th = (struct tcphdr *)((char *)ip + ip_hlen); 2652 /* XXX Educated guess: FreeBSD's in_pseudo == NetBSD's in_cksum_phdr */ 2653 th->th_sum = in_cksum_phdr(ip->ip_src.s_addr, 2654 ip->ip_dst.s_addr, htons(IPPROTO_TCP)); 2655 tcp_hlen = th->th_off << 2; 2656 hdrlen = ehdrlen + ip_hlen + tcp_hlen; 2657 2658 /* This is used in the transmit desc in encap */ 2659 *paylen = mp->m_pkthdr.len - hdrlen; 2660 2661 /* VLAN MACLEN IPLEN */ 2662 if ((mtag = VLAN_OUTPUT_TAG(ec, mp)) != NULL) { 2663 vtag = htole16(VLAN_TAG_VALUE(mtag) & 0xffff); 2664 vlan_macip_lens |= (vtag << IXGBE_ADVTXD_VLAN_SHIFT); 2665 } 2666 2667 vlan_macip_lens |= ehdrlen << IXGBE_ADVTXD_MACLEN_SHIFT; 2668 vlan_macip_lens |= ip_hlen; 2669 TXD->vlan_macip_lens |= htole32(vlan_macip_lens); 2670 2671 /* ADV DTYPE TUCMD */ 2672 type_tucmd_mlhl |= IXGBE_ADVTXD_DCMD_DEXT | IXGBE_ADVTXD_DTYP_CTXT; 2673 type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_L4T_TCP; 2674 type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_IPV4; 2675 TXD->type_tucmd_mlhl |= htole32(type_tucmd_mlhl); 2676 2677 2678 /* MSS L4LEN IDX */ 2679 mss_l4len_idx |= (mp->m_pkthdr.segsz << IXGBE_ADVTXD_MSS_SHIFT); 2680 mss_l4len_idx |= (tcp_hlen << IXGBE_ADVTXD_L4LEN_SHIFT); 2681 TXD->mss_l4len_idx = htole32(mss_l4len_idx); 2682 2683 TXD->seqnum_seed = htole32(0); 2684 tx_buffer->m_head = NULL; 2685 tx_buffer->eop_index = -1; 2686 2687 if (++ctxd == adapter->num_tx_desc) 2688 ctxd = 0; 2689 2690 txr->tx_avail--; 2691 txr->next_avail_desc = ctxd; 2692 return TRUE; 2693 } 2694 2695 2696 /********************************************************************** 2697 * 2698 * Examine each tx_buffer in the used queue. If the hardware is done 2699 * processing the packet then free associated resources. The 2700 * tx_buffer is put back on the free queue. 2701 * 2702 **********************************************************************/ 2703 static bool 2704 ixv_txeof(struct tx_ring *txr) 2705 { 2706 struct adapter *adapter = txr->adapter; 2707 struct ifnet *ifp = adapter->ifp; 2708 u32 first, last, done; 2709 struct ixv_tx_buf *tx_buffer; 2710 struct ixgbe_legacy_tx_desc *tx_desc, *eop_desc; 2711 2712 KASSERT(mutex_owned(&txr->tx_mtx)); 2713 2714 if (txr->tx_avail == adapter->num_tx_desc) 2715 return false; 2716 2717 first = txr->next_to_clean; 2718 tx_buffer = &txr->tx_buffers[first]; 2719 /* For cleanup we just use legacy struct */ 2720 tx_desc = (struct ixgbe_legacy_tx_desc *)&txr->tx_base[first]; 2721 last = tx_buffer->eop_index; 2722 if (last == -1) 2723 return false; 2724 eop_desc = (struct ixgbe_legacy_tx_desc *)&txr->tx_base[last]; 2725 2726 /* 2727 ** Get the index of the first descriptor 2728 ** BEYOND the EOP and call that 'done'. 2729 ** I do this so the comparison in the 2730 ** inner while loop below can be simple 2731 */ 2732 if (++last == adapter->num_tx_desc) last = 0; 2733 done = last; 2734 2735 ixgbe_dmamap_sync(txr->txdma.dma_tag, txr->txdma.dma_map, 2736 BUS_DMASYNC_POSTREAD); 2737 /* 2738 ** Only the EOP descriptor of a packet now has the DD 2739 ** bit set, this is what we look for... 2740 */ 2741 while (eop_desc->upper.fields.status & IXGBE_TXD_STAT_DD) { 2742 /* We clean the range of the packet */ 2743 while (first != done) { 2744 tx_desc->upper.data = 0; 2745 tx_desc->lower.data = 0; 2746 tx_desc->buffer_addr = 0; 2747 ++txr->tx_avail; 2748 2749 if (tx_buffer->m_head) { 2750 bus_dmamap_sync(txr->txtag->dt_dmat, 2751 tx_buffer->map, 2752 0, tx_buffer->m_head->m_pkthdr.len, 2753 BUS_DMASYNC_POSTWRITE); 2754 ixgbe_dmamap_unload(txr->txtag, tx_buffer->map); 2755 m_freem(tx_buffer->m_head); 2756 tx_buffer->m_head = NULL; 2757 tx_buffer->map = NULL; 2758 } 2759 tx_buffer->eop_index = -1; 2760 getmicrotime(&txr->watchdog_time); 2761 2762 if (++first == adapter->num_tx_desc) 2763 first = 0; 2764 2765 tx_buffer = &txr->tx_buffers[first]; 2766 tx_desc = 2767 (struct ixgbe_legacy_tx_desc *)&txr->tx_base[first]; 2768 } 2769 ++ifp->if_opackets; 2770 /* See if there is more work now */ 2771 last = tx_buffer->eop_index; 2772 if (last != -1) { 2773 eop_desc = 2774 (struct ixgbe_legacy_tx_desc *)&txr->tx_base[last]; 2775 /* Get next done point */ 2776 if (++last == adapter->num_tx_desc) last = 0; 2777 done = last; 2778 } else 2779 break; 2780 } 2781 ixgbe_dmamap_sync(txr->txdma.dma_tag, txr->txdma.dma_map, 2782 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); 2783 2784 txr->next_to_clean = first; 2785 2786 /* 2787 * If we have enough room, clear IFF_OACTIVE to tell the stack that 2788 * it is OK to send packets. If there are no pending descriptors, 2789 * clear the timeout. Otherwise, if some descriptors have been freed, 2790 * restart the timeout. 2791 */ 2792 if (txr->tx_avail > IXV_TX_CLEANUP_THRESHOLD) { 2793 ifp->if_flags &= ~IFF_OACTIVE; 2794 if (txr->tx_avail == adapter->num_tx_desc) { 2795 txr->watchdog_check = FALSE; 2796 return false; 2797 } 2798 } 2799 2800 return true; 2801 } 2802 2803 /********************************************************************* 2804 * 2805 * Refresh mbuf buffers for RX descriptor rings 2806 * - now keeps its own state so discards due to resource 2807 * exhaustion are unnecessary, if an mbuf cannot be obtained 2808 * it just returns, keeping its placeholder, thus it can simply 2809 * be recalled to try again. 2810 * 2811 **********************************************************************/ 2812 static void 2813 ixv_refresh_mbufs(struct rx_ring *rxr, int limit) 2814 { 2815 struct adapter *adapter = rxr->adapter; 2816 struct ixv_rx_buf *rxbuf; 2817 struct mbuf *mh, *mp; 2818 int i, j, error; 2819 bool refreshed = false; 2820 2821 i = j = rxr->next_to_refresh; 2822 /* Get the control variable, one beyond refresh point */ 2823 if (++j == adapter->num_rx_desc) 2824 j = 0; 2825 while (j != limit) { 2826 rxbuf = &rxr->rx_buffers[i]; 2827 if ((rxbuf->m_head == NULL) && (rxr->hdr_split)) { 2828 mh = m_gethdr(M_NOWAIT, MT_DATA); 2829 if (mh == NULL) 2830 goto update; 2831 mh->m_pkthdr.len = mh->m_len = MHLEN; 2832 mh->m_len = MHLEN; 2833 mh->m_flags |= M_PKTHDR; 2834 m_adj(mh, ETHER_ALIGN); 2835 /* Get the memory mapping */ 2836 error = bus_dmamap_load_mbuf(rxr->htag->dt_dmat, 2837 rxbuf->hmap, mh, BUS_DMA_NOWAIT); 2838 if (error != 0) { 2839 printf("GET BUF: dmamap load" 2840 " failure - %d\n", error); 2841 m_free(mh); 2842 goto update; 2843 } 2844 rxbuf->m_head = mh; 2845 ixgbe_dmamap_sync(rxr->htag, rxbuf->hmap, 2846 BUS_DMASYNC_PREREAD); 2847 rxr->rx_base[i].read.hdr_addr = 2848 htole64(rxbuf->hmap->dm_segs[0].ds_addr); 2849 } 2850 2851 if (rxbuf->m_pack == NULL) { 2852 mp = ixgbe_getjcl(&adapter->jcl_head, M_NOWAIT, 2853 MT_DATA, M_PKTHDR, adapter->rx_mbuf_sz); 2854 if (mp == NULL) { 2855 rxr->no_jmbuf.ev_count++; 2856 goto update; 2857 } else 2858 mp = rxbuf->m_pack; 2859 2860 mp->m_pkthdr.len = mp->m_len = adapter->rx_mbuf_sz; 2861 /* Get the memory mapping */ 2862 error = bus_dmamap_load_mbuf(rxr->ptag->dt_dmat, 2863 rxbuf->pmap, mp, BUS_DMA_NOWAIT); 2864 if (error != 0) { 2865 printf("GET BUF: dmamap load" 2866 " failure - %d\n", error); 2867 m_free(mp); 2868 rxbuf->m_pack = NULL; 2869 goto update; 2870 } 2871 rxbuf->m_pack = mp; 2872 bus_dmamap_sync(rxr->ptag->dt_dmat, rxbuf->pmap, 2873 0, mp->m_pkthdr.len, BUS_DMASYNC_PREREAD); 2874 rxr->rx_base[i].read.pkt_addr = 2875 htole64(rxbuf->pmap->dm_segs[0].ds_addr); 2876 } 2877 2878 refreshed = true; 2879 rxr->next_to_refresh = i = j; 2880 /* Calculate next index */ 2881 if (++j == adapter->num_rx_desc) 2882 j = 0; 2883 } 2884 update: 2885 if (refreshed) /* update tail index */ 2886 IXGBE_WRITE_REG(&adapter->hw, 2887 IXGBE_VFRDT(rxr->me), rxr->next_to_refresh); 2888 return; 2889 } 2890 2891 /********************************************************************* 2892 * 2893 * Allocate memory for rx_buffer structures. Since we use one 2894 * rx_buffer per received packet, the maximum number of rx_buffer's 2895 * that we'll need is equal to the number of receive descriptors 2896 * that we've allocated. 2897 * 2898 **********************************************************************/ 2899 static int 2900 ixv_allocate_receive_buffers(struct rx_ring *rxr) 2901 { 2902 struct adapter *adapter = rxr->adapter; 2903 device_t dev = adapter->dev; 2904 struct ixv_rx_buf *rxbuf; 2905 int i, bsize, error; 2906 2907 bsize = sizeof(struct ixv_rx_buf) * adapter->num_rx_desc; 2908 if (!(rxr->rx_buffers = 2909 (struct ixv_rx_buf *) malloc(bsize, 2910 M_DEVBUF, M_NOWAIT | M_ZERO))) { 2911 aprint_error_dev(dev, "Unable to allocate rx_buffer memory\n"); 2912 error = ENOMEM; 2913 goto fail; 2914 } 2915 2916 if ((error = ixgbe_dma_tag_create(adapter->osdep.dmat, /* parent */ 2917 1, 0, /* alignment, bounds */ 2918 MSIZE, /* maxsize */ 2919 1, /* nsegments */ 2920 MSIZE, /* maxsegsize */ 2921 0, /* flags */ 2922 &rxr->htag))) { 2923 aprint_error_dev(dev, "Unable to create RX DMA tag\n"); 2924 goto fail; 2925 } 2926 2927 if ((error = ixgbe_dma_tag_create(adapter->osdep.dmat, /* parent */ 2928 1, 0, /* alignment, bounds */ 2929 MJUMPAGESIZE, /* maxsize */ 2930 1, /* nsegments */ 2931 MJUMPAGESIZE, /* maxsegsize */ 2932 0, /* flags */ 2933 &rxr->ptag))) { 2934 aprint_error_dev(dev, "Unable to create RX DMA tag\n"); 2935 goto fail; 2936 } 2937 2938 for (i = 0; i < adapter->num_rx_desc; i++, rxbuf++) { 2939 rxbuf = &rxr->rx_buffers[i]; 2940 error = ixgbe_dmamap_create(rxr->htag, 2941 BUS_DMA_NOWAIT, &rxbuf->hmap); 2942 if (error) { 2943 aprint_error_dev(dev, "Unable to create RX head map\n"); 2944 goto fail; 2945 } 2946 error = ixgbe_dmamap_create(rxr->ptag, 2947 BUS_DMA_NOWAIT, &rxbuf->pmap); 2948 if (error) { 2949 aprint_error_dev(dev, "Unable to create RX pkt map\n"); 2950 goto fail; 2951 } 2952 } 2953 2954 return (0); 2955 2956 fail: 2957 /* Frees all, but can handle partial completion */ 2958 ixv_free_receive_structures(adapter); 2959 return (error); 2960 } 2961 2962 static void 2963 ixv_free_receive_ring(struct rx_ring *rxr) 2964 { 2965 struct adapter *adapter; 2966 struct ixv_rx_buf *rxbuf; 2967 int i; 2968 2969 adapter = rxr->adapter; 2970 for (i = 0; i < adapter->num_rx_desc; i++) { 2971 rxbuf = &rxr->rx_buffers[i]; 2972 if (rxbuf->m_head != NULL) { 2973 ixgbe_dmamap_sync(rxr->htag, rxbuf->hmap, 2974 BUS_DMASYNC_POSTREAD); 2975 ixgbe_dmamap_unload(rxr->htag, rxbuf->hmap); 2976 rxbuf->m_head->m_flags |= M_PKTHDR; 2977 m_freem(rxbuf->m_head); 2978 } 2979 if (rxbuf->m_pack != NULL) { 2980 /* XXX not ixgbe_ ? */ 2981 bus_dmamap_sync(rxr->ptag->dt_dmat, rxbuf->pmap, 2982 0, rxbuf->m_pack->m_pkthdr.len, 2983 BUS_DMASYNC_POSTREAD); 2984 ixgbe_dmamap_unload(rxr->ptag, rxbuf->pmap); 2985 rxbuf->m_pack->m_flags |= M_PKTHDR; 2986 m_freem(rxbuf->m_pack); 2987 } 2988 rxbuf->m_head = NULL; 2989 rxbuf->m_pack = NULL; 2990 } 2991 } 2992 2993 2994 /********************************************************************* 2995 * 2996 * Initialize a receive ring and its buffers. 2997 * 2998 **********************************************************************/ 2999 static int 3000 ixv_setup_receive_ring(struct rx_ring *rxr) 3001 { 3002 struct adapter *adapter; 3003 struct ixv_rx_buf *rxbuf; 3004 #ifdef LRO 3005 struct ifnet *ifp; 3006 struct lro_ctrl *lro = &rxr->lro; 3007 #endif /* LRO */ 3008 int rsize, error = 0; 3009 3010 adapter = rxr->adapter; 3011 #ifdef LRO 3012 ifp = adapter->ifp; 3013 #endif /* LRO */ 3014 3015 /* Clear the ring contents */ 3016 IXV_RX_LOCK(rxr); 3017 rsize = roundup2(adapter->num_rx_desc * 3018 sizeof(union ixgbe_adv_rx_desc), DBA_ALIGN); 3019 bzero((void *)rxr->rx_base, rsize); 3020 3021 /* Free current RX buffer structs and their mbufs */ 3022 ixv_free_receive_ring(rxr); 3023 3024 IXV_RX_UNLOCK(rxr); 3025 3026 /* Now reinitialize our supply of jumbo mbufs. The number 3027 * or size of jumbo mbufs may have changed. 3028 */ 3029 ixgbe_jcl_reinit(&adapter->jcl_head, rxr->ptag->dt_dmat, 3030 2 * adapter->num_rx_desc, adapter->rx_mbuf_sz); 3031 3032 IXV_RX_LOCK(rxr); 3033 3034 /* Configure header split? */ 3035 if (ixv_header_split) 3036 rxr->hdr_split = TRUE; 3037 3038 /* Now replenish the mbufs */ 3039 for (int j = 0; j != adapter->num_rx_desc; ++j) { 3040 struct mbuf *mh, *mp; 3041 3042 rxbuf = &rxr->rx_buffers[j]; 3043 /* 3044 ** Dont allocate mbufs if not 3045 ** doing header split, its wasteful 3046 */ 3047 if (rxr->hdr_split == FALSE) 3048 goto skip_head; 3049 3050 /* First the header */ 3051 rxbuf->m_head = m_gethdr(M_DONTWAIT, MT_DATA); 3052 if (rxbuf->m_head == NULL) { 3053 error = ENOBUFS; 3054 goto fail; 3055 } 3056 m_adj(rxbuf->m_head, ETHER_ALIGN); 3057 mh = rxbuf->m_head; 3058 mh->m_len = mh->m_pkthdr.len = MHLEN; 3059 mh->m_flags |= M_PKTHDR; 3060 /* Get the memory mapping */ 3061 error = bus_dmamap_load_mbuf(rxr->htag->dt_dmat, 3062 rxbuf->hmap, rxbuf->m_head, BUS_DMA_NOWAIT); 3063 if (error != 0) /* Nothing elegant to do here */ 3064 goto fail; 3065 bus_dmamap_sync(rxr->htag->dt_dmat, rxbuf->hmap, 3066 0, mh->m_pkthdr.len, BUS_DMASYNC_PREREAD); 3067 /* Update descriptor */ 3068 rxr->rx_base[j].read.hdr_addr = 3069 htole64(rxbuf->hmap->dm_segs[0].ds_addr); 3070 3071 skip_head: 3072 /* Now the payload cluster */ 3073 rxbuf->m_pack = ixgbe_getjcl(&adapter->jcl_head, M_DONTWAIT, 3074 MT_DATA, M_PKTHDR, adapter->rx_mbuf_sz); 3075 if (rxbuf->m_pack == NULL) { 3076 error = ENOBUFS; 3077 goto fail; 3078 } 3079 mp = rxbuf->m_pack; 3080 mp->m_pkthdr.len = mp->m_len = adapter->rx_mbuf_sz; 3081 /* Get the memory mapping */ 3082 error = bus_dmamap_load_mbuf(rxr->ptag->dt_dmat, 3083 rxbuf->pmap, mp, BUS_DMA_NOWAIT); 3084 if (error != 0) 3085 goto fail; 3086 bus_dmamap_sync(rxr->ptag->dt_dmat, rxbuf->pmap, 3087 0, adapter->rx_mbuf_sz, BUS_DMASYNC_PREREAD); 3088 /* Update descriptor */ 3089 rxr->rx_base[j].read.pkt_addr = 3090 htole64(rxbuf->pmap->dm_segs[0].ds_addr); 3091 } 3092 3093 3094 /* Setup our descriptor indices */ 3095 rxr->next_to_check = 0; 3096 rxr->next_to_refresh = 0; 3097 rxr->lro_enabled = FALSE; 3098 rxr->rx_split_packets.ev_count = 0; 3099 rxr->rx_bytes.ev_count = 0; 3100 rxr->discard = FALSE; 3101 3102 ixgbe_dmamap_sync(rxr->rxdma.dma_tag, rxr->rxdma.dma_map, 3103 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); 3104 3105 #ifdef LRO 3106 /* 3107 ** Now set up the LRO interface: 3108 */ 3109 if (ifp->if_capenable & IFCAP_LRO) { 3110 device_t dev = adapter->dev; 3111 int err = tcp_lro_init(lro); 3112 if (err) { 3113 device_printf(dev, "LRO Initialization failed!\n"); 3114 goto fail; 3115 } 3116 INIT_DEBUGOUT("RX Soft LRO Initialized\n"); 3117 rxr->lro_enabled = TRUE; 3118 lro->ifp = adapter->ifp; 3119 } 3120 #endif /* LRO */ 3121 3122 IXV_RX_UNLOCK(rxr); 3123 return (0); 3124 3125 fail: 3126 ixv_free_receive_ring(rxr); 3127 IXV_RX_UNLOCK(rxr); 3128 return (error); 3129 } 3130 3131 /********************************************************************* 3132 * 3133 * Initialize all receive rings. 3134 * 3135 **********************************************************************/ 3136 static int 3137 ixv_setup_receive_structures(struct adapter *adapter) 3138 { 3139 struct rx_ring *rxr = adapter->rx_rings; 3140 int j; 3141 3142 for (j = 0; j < adapter->num_queues; j++, rxr++) 3143 if (ixv_setup_receive_ring(rxr)) 3144 goto fail; 3145 3146 return (0); 3147 fail: 3148 /* 3149 * Free RX buffers allocated so far, we will only handle 3150 * the rings that completed, the failing case will have 3151 * cleaned up for itself. 'j' failed, so its the terminus. 3152 */ 3153 for (int i = 0; i < j; ++i) { 3154 rxr = &adapter->rx_rings[i]; 3155 ixv_free_receive_ring(rxr); 3156 } 3157 3158 return (ENOBUFS); 3159 } 3160 3161 /********************************************************************* 3162 * 3163 * Setup receive registers and features. 3164 * 3165 **********************************************************************/ 3166 #define IXGBE_SRRCTL_BSIZEHDRSIZE_SHIFT 2 3167 3168 static void 3169 ixv_initialize_receive_units(struct adapter *adapter) 3170 { 3171 int i; 3172 struct rx_ring *rxr = adapter->rx_rings; 3173 struct ixgbe_hw *hw = &adapter->hw; 3174 struct ifnet *ifp = adapter->ifp; 3175 u32 bufsz, fctrl, rxcsum, hlreg; 3176 3177 3178 /* Enable broadcasts */ 3179 fctrl = IXGBE_READ_REG(hw, IXGBE_FCTRL); 3180 fctrl |= IXGBE_FCTRL_BAM; 3181 fctrl |= IXGBE_FCTRL_DPF; 3182 fctrl |= IXGBE_FCTRL_PMCF; 3183 IXGBE_WRITE_REG(hw, IXGBE_FCTRL, fctrl); 3184 3185 /* Set for Jumbo Frames? */ 3186 hlreg = IXGBE_READ_REG(hw, IXGBE_HLREG0); 3187 if (ifp->if_mtu > ETHERMTU) { 3188 hlreg |= IXGBE_HLREG0_JUMBOEN; 3189 bufsz = 4096 >> IXGBE_SRRCTL_BSIZEPKT_SHIFT; 3190 } else { 3191 hlreg &= ~IXGBE_HLREG0_JUMBOEN; 3192 bufsz = 2048 >> IXGBE_SRRCTL_BSIZEPKT_SHIFT; 3193 } 3194 IXGBE_WRITE_REG(hw, IXGBE_HLREG0, hlreg); 3195 3196 for (i = 0; i < adapter->num_queues; i++, rxr++) { 3197 u64 rdba = rxr->rxdma.dma_paddr; 3198 u32 reg, rxdctl; 3199 3200 /* Do the queue enabling first */ 3201 rxdctl = IXGBE_READ_REG(hw, IXGBE_VFRXDCTL(i)); 3202 rxdctl |= IXGBE_RXDCTL_ENABLE; 3203 IXGBE_WRITE_REG(hw, IXGBE_VFRXDCTL(i), rxdctl); 3204 for (int k = 0; k < 10; k++) { 3205 if (IXGBE_READ_REG(hw, IXGBE_VFRXDCTL(i)) & 3206 IXGBE_RXDCTL_ENABLE) 3207 break; 3208 else 3209 msec_delay(1); 3210 } 3211 wmb(); 3212 3213 /* Setup the Base and Length of the Rx Descriptor Ring */ 3214 IXGBE_WRITE_REG(hw, IXGBE_VFRDBAL(i), 3215 (rdba & 0x00000000ffffffffULL)); 3216 IXGBE_WRITE_REG(hw, IXGBE_VFRDBAH(i), 3217 (rdba >> 32)); 3218 IXGBE_WRITE_REG(hw, IXGBE_VFRDLEN(i), 3219 adapter->num_rx_desc * sizeof(union ixgbe_adv_rx_desc)); 3220 3221 /* Set up the SRRCTL register */ 3222 reg = IXGBE_READ_REG(hw, IXGBE_VFSRRCTL(i)); 3223 reg &= ~IXGBE_SRRCTL_BSIZEHDR_MASK; 3224 reg &= ~IXGBE_SRRCTL_BSIZEPKT_MASK; 3225 reg |= bufsz; 3226 if (rxr->hdr_split) { 3227 /* Use a standard mbuf for the header */ 3228 reg |= ((IXV_RX_HDR << 3229 IXGBE_SRRCTL_BSIZEHDRSIZE_SHIFT) 3230 & IXGBE_SRRCTL_BSIZEHDR_MASK); 3231 reg |= IXGBE_SRRCTL_DESCTYPE_HDR_SPLIT_ALWAYS; 3232 } else 3233 reg |= IXGBE_SRRCTL_DESCTYPE_ADV_ONEBUF; 3234 IXGBE_WRITE_REG(hw, IXGBE_VFSRRCTL(i), reg); 3235 3236 /* Setup the HW Rx Head and Tail Descriptor Pointers */ 3237 IXGBE_WRITE_REG(hw, IXGBE_VFRDH(rxr->me), 0); 3238 IXGBE_WRITE_REG(hw, IXGBE_VFRDT(rxr->me), 3239 adapter->num_rx_desc - 1); 3240 } 3241 3242 rxcsum = IXGBE_READ_REG(hw, IXGBE_RXCSUM); 3243 3244 if (ifp->if_capenable & IFCAP_RXCSUM) 3245 rxcsum |= IXGBE_RXCSUM_PCSD; 3246 3247 if (!(rxcsum & IXGBE_RXCSUM_PCSD)) 3248 rxcsum |= IXGBE_RXCSUM_IPPCSE; 3249 3250 IXGBE_WRITE_REG(hw, IXGBE_RXCSUM, rxcsum); 3251 3252 return; 3253 } 3254 3255 /********************************************************************* 3256 * 3257 * Free all receive rings. 3258 * 3259 **********************************************************************/ 3260 static void 3261 ixv_free_receive_structures(struct adapter *adapter) 3262 { 3263 struct rx_ring *rxr = adapter->rx_rings; 3264 3265 for (int i = 0; i < adapter->num_queues; i++, rxr++) { 3266 #ifdef LRO 3267 struct lro_ctrl *lro = &rxr->lro; 3268 #endif /* LRO */ 3269 ixv_free_receive_buffers(rxr); 3270 #ifdef LRO 3271 /* Free LRO memory */ 3272 tcp_lro_free(lro); 3273 #endif /* LRO */ 3274 /* Free the ring memory as well */ 3275 ixv_dma_free(adapter, &rxr->rxdma); 3276 IXV_RX_LOCK_DESTROY(rxr); 3277 } 3278 3279 free(adapter->rx_rings, M_DEVBUF); 3280 } 3281 3282 3283 /********************************************************************* 3284 * 3285 * Free receive ring data structures 3286 * 3287 **********************************************************************/ 3288 static void 3289 ixv_free_receive_buffers(struct rx_ring *rxr) 3290 { 3291 struct adapter *adapter = rxr->adapter; 3292 struct ixv_rx_buf *rxbuf; 3293 3294 INIT_DEBUGOUT("free_receive_structures: begin"); 3295 3296 /* Cleanup any existing buffers */ 3297 if (rxr->rx_buffers != NULL) { 3298 for (int i = 0; i < adapter->num_rx_desc; i++) { 3299 rxbuf = &rxr->rx_buffers[i]; 3300 if (rxbuf->m_head != NULL) { 3301 ixgbe_dmamap_sync(rxr->htag, rxbuf->hmap, 3302 BUS_DMASYNC_POSTREAD); 3303 ixgbe_dmamap_unload(rxr->htag, rxbuf->hmap); 3304 rxbuf->m_head->m_flags |= M_PKTHDR; 3305 m_freem(rxbuf->m_head); 3306 } 3307 if (rxbuf->m_pack != NULL) { 3308 /* XXX not ixgbe_* ? */ 3309 bus_dmamap_sync(rxr->ptag->dt_dmat, rxbuf->pmap, 3310 0, rxbuf->m_pack->m_pkthdr.len, 3311 BUS_DMASYNC_POSTREAD); 3312 ixgbe_dmamap_unload(rxr->ptag, rxbuf->pmap); 3313 rxbuf->m_pack->m_flags |= M_PKTHDR; 3314 m_freem(rxbuf->m_pack); 3315 } 3316 rxbuf->m_head = NULL; 3317 rxbuf->m_pack = NULL; 3318 if (rxbuf->hmap != NULL) { 3319 ixgbe_dmamap_destroy(rxr->htag, rxbuf->hmap); 3320 rxbuf->hmap = NULL; 3321 } 3322 if (rxbuf->pmap != NULL) { 3323 ixgbe_dmamap_destroy(rxr->ptag, rxbuf->pmap); 3324 rxbuf->pmap = NULL; 3325 } 3326 } 3327 if (rxr->rx_buffers != NULL) { 3328 free(rxr->rx_buffers, M_DEVBUF); 3329 rxr->rx_buffers = NULL; 3330 } 3331 } 3332 3333 if (rxr->htag != NULL) { 3334 ixgbe_dma_tag_destroy(rxr->htag); 3335 rxr->htag = NULL; 3336 } 3337 if (rxr->ptag != NULL) { 3338 ixgbe_dma_tag_destroy(rxr->ptag); 3339 rxr->ptag = NULL; 3340 } 3341 3342 return; 3343 } 3344 3345 static __inline void 3346 ixv_rx_input(struct rx_ring *rxr, struct ifnet *ifp, struct mbuf *m, u32 ptype) 3347 { 3348 int s; 3349 3350 #ifdef LRO 3351 struct adapter *adapter = ifp->if_softc; 3352 struct ethercom *ec = &adapter->osdep.ec; 3353 3354 /* 3355 * ATM LRO is only for IPv4/TCP packets and TCP checksum of the packet 3356 * should be computed by hardware. Also it should not have VLAN tag in 3357 * ethernet header. 3358 */ 3359 if (rxr->lro_enabled && 3360 (ec->ec_capenable & ETHERCAP_VLAN_HWTAGGING) != 0 && 3361 (ptype & IXGBE_RXDADV_PKTTYPE_ETQF) == 0 && 3362 (ptype & (IXGBE_RXDADV_PKTTYPE_IPV4 | IXGBE_RXDADV_PKTTYPE_TCP)) == 3363 (IXGBE_RXDADV_PKTTYPE_IPV4 | IXGBE_RXDADV_PKTTYPE_TCP) && 3364 (m->m_pkthdr.csum_flags & (CSUM_DATA_VALID | CSUM_PSEUDO_HDR)) == 3365 (CSUM_DATA_VALID | CSUM_PSEUDO_HDR)) { 3366 /* 3367 * Send to the stack if: 3368 ** - LRO not enabled, or 3369 ** - no LRO resources, or 3370 ** - lro enqueue fails 3371 */ 3372 if (rxr->lro.lro_cnt != 0) 3373 if (tcp_lro_rx(&rxr->lro, m, 0) == 0) 3374 return; 3375 } 3376 #endif /* LRO */ 3377 3378 IXV_RX_UNLOCK(rxr); 3379 3380 s = splnet(); 3381 /* Pass this up to any BPF listeners. */ 3382 bpf_mtap(ifp, m); 3383 (*ifp->if_input)(ifp, m); 3384 splx(s); 3385 3386 IXV_RX_LOCK(rxr); 3387 } 3388 3389 static __inline void 3390 ixv_rx_discard(struct rx_ring *rxr, int i) 3391 { 3392 struct ixv_rx_buf *rbuf; 3393 3394 rbuf = &rxr->rx_buffers[i]; 3395 if (rbuf->fmp != NULL) {/* Partial chain ? */ 3396 rbuf->fmp->m_flags |= M_PKTHDR; 3397 m_freem(rbuf->fmp); 3398 rbuf->fmp = NULL; 3399 } 3400 3401 /* 3402 ** With advanced descriptors the writeback 3403 ** clobbers the buffer addrs, so its easier 3404 ** to just free the existing mbufs and take 3405 ** the normal refresh path to get new buffers 3406 ** and mapping. 3407 */ 3408 if (rbuf->m_head) { 3409 m_free(rbuf->m_head); 3410 rbuf->m_head = NULL; 3411 } 3412 3413 if (rbuf->m_pack) { 3414 m_free(rbuf->m_pack); 3415 rbuf->m_pack = NULL; 3416 } 3417 3418 return; 3419 } 3420 3421 3422 /********************************************************************* 3423 * 3424 * This routine executes in interrupt context. It replenishes 3425 * the mbufs in the descriptor and sends data which has been 3426 * dma'ed into host memory to upper layer. 3427 * 3428 * We loop at most count times if count is > 0, or until done if 3429 * count < 0. 3430 * 3431 * Return TRUE for more work, FALSE for all clean. 3432 *********************************************************************/ 3433 static bool 3434 ixv_rxeof(struct ix_queue *que, int count) 3435 { 3436 struct adapter *adapter = que->adapter; 3437 struct rx_ring *rxr = que->rxr; 3438 struct ifnet *ifp = adapter->ifp; 3439 #ifdef LRO 3440 struct lro_ctrl *lro = &rxr->lro; 3441 struct lro_entry *queued; 3442 #endif /* LRO */ 3443 int i, nextp, processed = 0; 3444 u32 staterr = 0; 3445 union ixgbe_adv_rx_desc *cur; 3446 struct ixv_rx_buf *rbuf, *nbuf; 3447 3448 IXV_RX_LOCK(rxr); 3449 3450 for (i = rxr->next_to_check; count != 0;) { 3451 struct mbuf *sendmp, *mh, *mp; 3452 u32 ptype; 3453 u16 hlen, plen, hdr, vtag; 3454 bool eop; 3455 3456 /* Sync the ring. */ 3457 ixgbe_dmamap_sync(rxr->rxdma.dma_tag, rxr->rxdma.dma_map, 3458 BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE); 3459 3460 cur = &rxr->rx_base[i]; 3461 staterr = le32toh(cur->wb.upper.status_error); 3462 3463 if ((staterr & IXGBE_RXD_STAT_DD) == 0) 3464 break; 3465 if ((ifp->if_flags & IFF_RUNNING) == 0) 3466 break; 3467 3468 count--; 3469 sendmp = NULL; 3470 nbuf = NULL; 3471 cur->wb.upper.status_error = 0; 3472 rbuf = &rxr->rx_buffers[i]; 3473 mh = rbuf->m_head; 3474 mp = rbuf->m_pack; 3475 3476 plen = le16toh(cur->wb.upper.length); 3477 ptype = le32toh(cur->wb.lower.lo_dword.data) & 3478 IXGBE_RXDADV_PKTTYPE_MASK; 3479 hdr = le16toh(cur->wb.lower.lo_dword.hs_rss.hdr_info); 3480 vtag = le16toh(cur->wb.upper.vlan); 3481 eop = ((staterr & IXGBE_RXD_STAT_EOP) != 0); 3482 3483 /* Make sure all parts of a bad packet are discarded */ 3484 if (((staterr & IXGBE_RXDADV_ERR_FRAME_ERR_MASK) != 0) || 3485 (rxr->discard)) { 3486 ifp->if_ierrors++; 3487 rxr->rx_discarded.ev_count++; 3488 if (!eop) 3489 rxr->discard = TRUE; 3490 else 3491 rxr->discard = FALSE; 3492 ixv_rx_discard(rxr, i); 3493 goto next_desc; 3494 } 3495 3496 if (!eop) { 3497 nextp = i + 1; 3498 if (nextp == adapter->num_rx_desc) 3499 nextp = 0; 3500 nbuf = &rxr->rx_buffers[nextp]; 3501 prefetch(nbuf); 3502 } 3503 /* 3504 ** The header mbuf is ONLY used when header 3505 ** split is enabled, otherwise we get normal 3506 ** behavior, ie, both header and payload 3507 ** are DMA'd into the payload buffer. 3508 ** 3509 ** Rather than using the fmp/lmp global pointers 3510 ** we now keep the head of a packet chain in the 3511 ** buffer struct and pass this along from one 3512 ** descriptor to the next, until we get EOP. 3513 */ 3514 if (rxr->hdr_split && (rbuf->fmp == NULL)) { 3515 /* This must be an initial descriptor */ 3516 hlen = (hdr & IXGBE_RXDADV_HDRBUFLEN_MASK) >> 3517 IXGBE_RXDADV_HDRBUFLEN_SHIFT; 3518 if (hlen > IXV_RX_HDR) 3519 hlen = IXV_RX_HDR; 3520 mh->m_len = hlen; 3521 mh->m_flags |= M_PKTHDR; 3522 mh->m_next = NULL; 3523 mh->m_pkthdr.len = mh->m_len; 3524 /* Null buf pointer so it is refreshed */ 3525 rbuf->m_head = NULL; 3526 /* 3527 ** Check the payload length, this 3528 ** could be zero if its a small 3529 ** packet. 3530 */ 3531 if (plen > 0) { 3532 mp->m_len = plen; 3533 mp->m_next = NULL; 3534 mp->m_flags &= ~M_PKTHDR; 3535 mh->m_next = mp; 3536 mh->m_pkthdr.len += mp->m_len; 3537 /* Null buf pointer so it is refreshed */ 3538 rbuf->m_pack = NULL; 3539 rxr->rx_split_packets.ev_count++; 3540 } 3541 /* 3542 ** Now create the forward 3543 ** chain so when complete 3544 ** we wont have to. 3545 */ 3546 if (eop == 0) { 3547 /* stash the chain head */ 3548 nbuf->fmp = mh; 3549 /* Make forward chain */ 3550 if (plen) 3551 mp->m_next = nbuf->m_pack; 3552 else 3553 mh->m_next = nbuf->m_pack; 3554 } else { 3555 /* Singlet, prepare to send */ 3556 sendmp = mh; 3557 if (VLAN_ATTACHED(&adapter->osdep.ec) && 3558 (staterr & IXGBE_RXD_STAT_VP)) { 3559 VLAN_INPUT_TAG(ifp, sendmp, vtag, 3560 printf("%s: could not apply VLAN " 3561 "tag", __func__)); 3562 } 3563 } 3564 } else { 3565 /* 3566 ** Either no header split, or a 3567 ** secondary piece of a fragmented 3568 ** split packet. 3569 */ 3570 mp->m_len = plen; 3571 /* 3572 ** See if there is a stored head 3573 ** that determines what we are 3574 */ 3575 sendmp = rbuf->fmp; 3576 rbuf->m_pack = rbuf->fmp = NULL; 3577 3578 if (sendmp != NULL) /* secondary frag */ 3579 sendmp->m_pkthdr.len += mp->m_len; 3580 else { 3581 /* first desc of a non-ps chain */ 3582 sendmp = mp; 3583 sendmp->m_flags |= M_PKTHDR; 3584 sendmp->m_pkthdr.len = mp->m_len; 3585 if (staterr & IXGBE_RXD_STAT_VP) { 3586 /* XXX Do something reasonable on 3587 * error. 3588 */ 3589 VLAN_INPUT_TAG(ifp, sendmp, vtag, 3590 printf("%s: could not apply VLAN " 3591 "tag", __func__)); 3592 } 3593 } 3594 /* Pass the head pointer on */ 3595 if (eop == 0) { 3596 nbuf->fmp = sendmp; 3597 sendmp = NULL; 3598 mp->m_next = nbuf->m_pack; 3599 } 3600 } 3601 ++processed; 3602 /* Sending this frame? */ 3603 if (eop) { 3604 sendmp->m_pkthdr.rcvif = ifp; 3605 ifp->if_ipackets++; 3606 rxr->rx_packets.ev_count++; 3607 /* capture data for AIM */ 3608 rxr->bytes += sendmp->m_pkthdr.len; 3609 rxr->rx_bytes.ev_count += sendmp->m_pkthdr.len; 3610 if ((ifp->if_capenable & IFCAP_RXCSUM) != 0) { 3611 ixv_rx_checksum(staterr, sendmp, ptype, 3612 &adapter->stats); 3613 } 3614 #if __FreeBSD_version >= 800000 3615 sendmp->m_pkthdr.flowid = que->msix; 3616 sendmp->m_flags |= M_FLOWID; 3617 #endif 3618 } 3619 next_desc: 3620 ixgbe_dmamap_sync(rxr->rxdma.dma_tag, rxr->rxdma.dma_map, 3621 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); 3622 3623 /* Advance our pointers to the next descriptor. */ 3624 if (++i == adapter->num_rx_desc) 3625 i = 0; 3626 3627 /* Now send to the stack or do LRO */ 3628 if (sendmp != NULL) 3629 ixv_rx_input(rxr, ifp, sendmp, ptype); 3630 3631 /* Every 8 descriptors we go to refresh mbufs */ 3632 if (processed == 8) { 3633 ixv_refresh_mbufs(rxr, i); 3634 processed = 0; 3635 } 3636 } 3637 3638 /* Refresh any remaining buf structs */ 3639 if (ixv_rx_unrefreshed(rxr)) 3640 ixv_refresh_mbufs(rxr, i); 3641 3642 rxr->next_to_check = i; 3643 3644 #ifdef LRO 3645 /* 3646 * Flush any outstanding LRO work 3647 */ 3648 while ((queued = SLIST_FIRST(&lro->lro_active)) != NULL) { 3649 SLIST_REMOVE_HEAD(&lro->lro_active, next); 3650 tcp_lro_flush(lro, queued); 3651 } 3652 #endif /* LRO */ 3653 3654 IXV_RX_UNLOCK(rxr); 3655 3656 /* 3657 ** We still have cleaning to do? 3658 ** Schedule another interrupt if so. 3659 */ 3660 if ((staterr & IXGBE_RXD_STAT_DD) != 0) { 3661 ixv_rearm_queues(adapter, (u64)(1ULL << que->msix)); 3662 return true; 3663 } 3664 3665 return false; 3666 } 3667 3668 3669 /********************************************************************* 3670 * 3671 * Verify that the hardware indicated that the checksum is valid. 3672 * Inform the stack about the status of checksum so that stack 3673 * doesn't spend time verifying the checksum. 3674 * 3675 *********************************************************************/ 3676 static void 3677 ixv_rx_checksum(u32 staterr, struct mbuf * mp, u32 ptype, 3678 struct ixgbevf_hw_stats *stats) 3679 { 3680 u16 status = (u16) staterr; 3681 u8 errors = (u8) (staterr >> 24); 3682 #if 0 3683 bool sctp = FALSE; 3684 if ((ptype & IXGBE_RXDADV_PKTTYPE_ETQF) == 0 && 3685 (ptype & IXGBE_RXDADV_PKTTYPE_SCTP) != 0) 3686 sctp = TRUE; 3687 #endif 3688 if (status & IXGBE_RXD_STAT_IPCS) { 3689 stats->ipcs.ev_count++; 3690 if (!(errors & IXGBE_RXD_ERR_IPE)) { 3691 /* IP Checksum Good */ 3692 mp->m_pkthdr.csum_flags |= M_CSUM_IPv4; 3693 3694 } else { 3695 stats->ipcs_bad.ev_count++; 3696 mp->m_pkthdr.csum_flags = M_CSUM_IPv4|M_CSUM_IPv4_BAD; 3697 } 3698 } 3699 if (status & IXGBE_RXD_STAT_L4CS) { 3700 stats->l4cs.ev_count++; 3701 u16 type = M_CSUM_TCPv4|M_CSUM_TCPv6|M_CSUM_UDPv4|M_CSUM_UDPv6; 3702 if (!(errors & IXGBE_RXD_ERR_TCPE)) { 3703 mp->m_pkthdr.csum_flags |= type; 3704 } else { 3705 stats->l4cs_bad.ev_count++; 3706 mp->m_pkthdr.csum_flags |= type | M_CSUM_TCP_UDP_BAD; 3707 } 3708 } 3709 return; 3710 } 3711 3712 static void 3713 ixv_setup_vlan_support(struct adapter *adapter) 3714 { 3715 struct ixgbe_hw *hw = &adapter->hw; 3716 u32 ctrl, vid, vfta, retry; 3717 3718 3719 /* 3720 ** We get here thru init_locked, meaning 3721 ** a soft reset, this has already cleared 3722 ** the VFTA and other state, so if there 3723 ** have been no vlan's registered do nothing. 3724 */ 3725 if (adapter->num_vlans == 0) 3726 return; 3727 3728 /* Enable the queues */ 3729 for (int i = 0; i < adapter->num_queues; i++) { 3730 ctrl = IXGBE_READ_REG(hw, IXGBE_VFRXDCTL(i)); 3731 ctrl |= IXGBE_RXDCTL_VME; 3732 IXGBE_WRITE_REG(hw, IXGBE_VFRXDCTL(i), ctrl); 3733 } 3734 3735 /* 3736 ** A soft reset zero's out the VFTA, so 3737 ** we need to repopulate it now. 3738 */ 3739 for (int i = 0; i < VFTA_SIZE; i++) { 3740 if (ixv_shadow_vfta[i] == 0) 3741 continue; 3742 vfta = ixv_shadow_vfta[i]; 3743 /* 3744 ** Reconstruct the vlan id's 3745 ** based on the bits set in each 3746 ** of the array ints. 3747 */ 3748 for ( int j = 0; j < 32; j++) { 3749 retry = 0; 3750 if ((vfta & (1 << j)) == 0) 3751 continue; 3752 vid = (i * 32) + j; 3753 /* Call the shared code mailbox routine */ 3754 while (ixgbe_set_vfta(hw, vid, 0, TRUE)) { 3755 if (++retry > 5) 3756 break; 3757 } 3758 } 3759 } 3760 } 3761 3762 #if 0 /* XXX Badly need to overhaul vlan(4) on NetBSD. */ 3763 /* 3764 ** This routine is run via an vlan config EVENT, 3765 ** it enables us to use the HW Filter table since 3766 ** we can get the vlan id. This just creates the 3767 ** entry in the soft version of the VFTA, init will 3768 ** repopulate the real table. 3769 */ 3770 static void 3771 ixv_register_vlan(void *arg, struct ifnet *ifp, u16 vtag) 3772 { 3773 struct adapter *adapter = ifp->if_softc; 3774 u16 index, bit; 3775 3776 if (ifp->if_softc != arg) /* Not our event */ 3777 return; 3778 3779 if ((vtag == 0) || (vtag > 4095)) /* Invalid */ 3780 return; 3781 3782 IXV_CORE_LOCK(adapter); 3783 index = (vtag >> 5) & 0x7F; 3784 bit = vtag & 0x1F; 3785 ixv_shadow_vfta[index] |= (1 << bit); 3786 /* Re-init to load the changes */ 3787 ixv_init_locked(adapter); 3788 IXV_CORE_UNLOCK(adapter); 3789 } 3790 3791 /* 3792 ** This routine is run via an vlan 3793 ** unconfig EVENT, remove our entry 3794 ** in the soft vfta. 3795 */ 3796 static void 3797 ixv_unregister_vlan(void *arg, struct ifnet *ifp, u16 vtag) 3798 { 3799 struct adapter *adapter = ifp->if_softc; 3800 u16 index, bit; 3801 3802 if (ifp->if_softc != arg) 3803 return; 3804 3805 if ((vtag == 0) || (vtag > 4095)) /* Invalid */ 3806 return; 3807 3808 IXV_CORE_LOCK(adapter); 3809 index = (vtag >> 5) & 0x7F; 3810 bit = vtag & 0x1F; 3811 ixv_shadow_vfta[index] &= ~(1 << bit); 3812 /* Re-init to load the changes */ 3813 ixv_init_locked(adapter); 3814 IXV_CORE_UNLOCK(adapter); 3815 } 3816 #endif 3817 3818 static void 3819 ixv_enable_intr(struct adapter *adapter) 3820 { 3821 struct ixgbe_hw *hw = &adapter->hw; 3822 struct ix_queue *que = adapter->queues; 3823 u32 mask = (IXGBE_EIMS_ENABLE_MASK & ~IXGBE_EIMS_RTX_QUEUE); 3824 3825 3826 IXGBE_WRITE_REG(hw, IXGBE_VTEIMS, mask); 3827 3828 mask = IXGBE_EIMS_ENABLE_MASK; 3829 mask &= ~(IXGBE_EIMS_OTHER | IXGBE_EIMS_LSC); 3830 IXGBE_WRITE_REG(hw, IXGBE_VTEIAC, mask); 3831 3832 for (int i = 0; i < adapter->num_queues; i++, que++) 3833 ixv_enable_queue(adapter, que->msix); 3834 3835 IXGBE_WRITE_FLUSH(hw); 3836 3837 return; 3838 } 3839 3840 static void 3841 ixv_disable_intr(struct adapter *adapter) 3842 { 3843 IXGBE_WRITE_REG(&adapter->hw, IXGBE_VTEIAC, 0); 3844 IXGBE_WRITE_REG(&adapter->hw, IXGBE_VTEIMC, ~0); 3845 IXGBE_WRITE_FLUSH(&adapter->hw); 3846 return; 3847 } 3848 3849 /* 3850 ** Setup the correct IVAR register for a particular MSIX interrupt 3851 ** - entry is the register array entry 3852 ** - vector is the MSIX vector for this queue 3853 ** - type is RX/TX/MISC 3854 */ 3855 static void 3856 ixv_set_ivar(struct adapter *adapter, u8 entry, u8 vector, s8 type) 3857 { 3858 struct ixgbe_hw *hw = &adapter->hw; 3859 u32 ivar, index; 3860 3861 vector |= IXGBE_IVAR_ALLOC_VAL; 3862 3863 if (type == -1) { /* MISC IVAR */ 3864 ivar = IXGBE_READ_REG(hw, IXGBE_VTIVAR_MISC); 3865 ivar &= ~0xFF; 3866 ivar |= vector; 3867 IXGBE_WRITE_REG(hw, IXGBE_VTIVAR_MISC, ivar); 3868 } else { /* RX/TX IVARS */ 3869 index = (16 * (entry & 1)) + (8 * type); 3870 ivar = IXGBE_READ_REG(hw, IXGBE_VTIVAR(entry >> 1)); 3871 ivar &= ~(0xFF << index); 3872 ivar |= (vector << index); 3873 IXGBE_WRITE_REG(hw, IXGBE_VTIVAR(entry >> 1), ivar); 3874 } 3875 } 3876 3877 static void 3878 ixv_configure_ivars(struct adapter *adapter) 3879 { 3880 struct ix_queue *que = adapter->queues; 3881 3882 for (int i = 0; i < adapter->num_queues; i++, que++) { 3883 /* First the RX queue entry */ 3884 ixv_set_ivar(adapter, i, que->msix, 0); 3885 /* ... and the TX */ 3886 ixv_set_ivar(adapter, i, que->msix, 1); 3887 /* Set an initial value in EITR */ 3888 IXGBE_WRITE_REG(&adapter->hw, 3889 IXGBE_VTEITR(que->msix), IXV_EITR_DEFAULT); 3890 } 3891 3892 /* For the Link interrupt */ 3893 ixv_set_ivar(adapter, 1, adapter->mbxvec, -1); 3894 } 3895 3896 3897 /* 3898 ** Tasklet handler for MSIX MBX interrupts 3899 ** - do outside interrupt since it might sleep 3900 */ 3901 static void 3902 ixv_handle_mbx(void *context) 3903 { 3904 struct adapter *adapter = context; 3905 3906 ixgbe_check_link(&adapter->hw, 3907 &adapter->link_speed, &adapter->link_up, 0); 3908 ixv_update_link_status(adapter); 3909 } 3910 3911 /* 3912 ** The VF stats registers never have a truely virgin 3913 ** starting point, so this routine tries to make an 3914 ** artificial one, marking ground zero on attach as 3915 ** it were. 3916 */ 3917 static void 3918 ixv_save_stats(struct adapter *adapter) 3919 { 3920 if (adapter->stats.vfgprc || adapter->stats.vfgptc) { 3921 adapter->stats.saved_reset_vfgprc += 3922 adapter->stats.vfgprc - adapter->stats.base_vfgprc; 3923 adapter->stats.saved_reset_vfgptc += 3924 adapter->stats.vfgptc - adapter->stats.base_vfgptc; 3925 adapter->stats.saved_reset_vfgorc += 3926 adapter->stats.vfgorc - adapter->stats.base_vfgorc; 3927 adapter->stats.saved_reset_vfgotc += 3928 adapter->stats.vfgotc - adapter->stats.base_vfgotc; 3929 adapter->stats.saved_reset_vfmprc += 3930 adapter->stats.vfmprc - adapter->stats.base_vfmprc; 3931 } 3932 } 3933 3934 static void 3935 ixv_init_stats(struct adapter *adapter) 3936 { 3937 struct ixgbe_hw *hw = &adapter->hw; 3938 3939 adapter->stats.last_vfgprc = IXGBE_READ_REG(hw, IXGBE_VFGPRC); 3940 adapter->stats.last_vfgorc = IXGBE_READ_REG(hw, IXGBE_VFGORC_LSB); 3941 adapter->stats.last_vfgorc |= 3942 (((u64)(IXGBE_READ_REG(hw, IXGBE_VFGORC_MSB))) << 32); 3943 3944 adapter->stats.last_vfgptc = IXGBE_READ_REG(hw, IXGBE_VFGPTC); 3945 adapter->stats.last_vfgotc = IXGBE_READ_REG(hw, IXGBE_VFGOTC_LSB); 3946 adapter->stats.last_vfgotc |= 3947 (((u64)(IXGBE_READ_REG(hw, IXGBE_VFGOTC_MSB))) << 32); 3948 3949 adapter->stats.last_vfmprc = IXGBE_READ_REG(hw, IXGBE_VFMPRC); 3950 3951 adapter->stats.base_vfgprc = adapter->stats.last_vfgprc; 3952 adapter->stats.base_vfgorc = adapter->stats.last_vfgorc; 3953 adapter->stats.base_vfgptc = adapter->stats.last_vfgptc; 3954 adapter->stats.base_vfgotc = adapter->stats.last_vfgotc; 3955 adapter->stats.base_vfmprc = adapter->stats.last_vfmprc; 3956 } 3957 3958 #define UPDATE_STAT_32(reg, last, count) \ 3959 { \ 3960 u32 current = IXGBE_READ_REG(hw, reg); \ 3961 if (current < last) \ 3962 count += 0x100000000LL; \ 3963 last = current; \ 3964 count &= 0xFFFFFFFF00000000LL; \ 3965 count |= current; \ 3966 } 3967 3968 #define UPDATE_STAT_36(lsb, msb, last, count) \ 3969 { \ 3970 u64 cur_lsb = IXGBE_READ_REG(hw, lsb); \ 3971 u64 cur_msb = IXGBE_READ_REG(hw, msb); \ 3972 u64 current = ((cur_msb << 32) | cur_lsb); \ 3973 if (current < last) \ 3974 count += 0x1000000000LL; \ 3975 last = current; \ 3976 count &= 0xFFFFFFF000000000LL; \ 3977 count |= current; \ 3978 } 3979 3980 /* 3981 ** ixv_update_stats - Update the board statistics counters. 3982 */ 3983 void 3984 ixv_update_stats(struct adapter *adapter) 3985 { 3986 struct ixgbe_hw *hw = &adapter->hw; 3987 3988 UPDATE_STAT_32(IXGBE_VFGPRC, adapter->stats.last_vfgprc, 3989 adapter->stats.vfgprc); 3990 UPDATE_STAT_32(IXGBE_VFGPTC, adapter->stats.last_vfgptc, 3991 adapter->stats.vfgptc); 3992 UPDATE_STAT_36(IXGBE_VFGORC_LSB, IXGBE_VFGORC_MSB, 3993 adapter->stats.last_vfgorc, adapter->stats.vfgorc); 3994 UPDATE_STAT_36(IXGBE_VFGOTC_LSB, IXGBE_VFGOTC_MSB, 3995 adapter->stats.last_vfgotc, adapter->stats.vfgotc); 3996 UPDATE_STAT_32(IXGBE_VFMPRC, adapter->stats.last_vfmprc, 3997 adapter->stats.vfmprc); 3998 } 3999 4000 /********************************************************************** 4001 * 4002 * This routine is called only when ixgbe_display_debug_stats is enabled. 4003 * This routine provides a way to take a look at important statistics 4004 * maintained by the driver and hardware. 4005 * 4006 **********************************************************************/ 4007 static void 4008 ixv_print_hw_stats(struct adapter * adapter) 4009 { 4010 device_t dev = adapter->dev; 4011 4012 device_printf(dev,"Std Mbuf Failed = %lu\n", 4013 adapter->mbuf_defrag_failed.ev_count); 4014 device_printf(dev,"Driver dropped packets = %lu\n", 4015 adapter->dropped_pkts.ev_count); 4016 device_printf(dev, "watchdog timeouts = %ld\n", 4017 adapter->watchdog_events.ev_count); 4018 4019 device_printf(dev,"Good Packets Rcvd = %llu\n", 4020 (long long)adapter->stats.vfgprc); 4021 device_printf(dev,"Good Packets Xmtd = %llu\n", 4022 (long long)adapter->stats.vfgptc); 4023 device_printf(dev,"TSO Transmissions = %lu\n", 4024 adapter->tso_tx.ev_count); 4025 4026 } 4027 4028 /********************************************************************** 4029 * 4030 * This routine is called only when em_display_debug_stats is enabled. 4031 * This routine provides a way to take a look at important statistics 4032 * maintained by the driver and hardware. 4033 * 4034 **********************************************************************/ 4035 static void 4036 ixv_print_debug_info(struct adapter *adapter) 4037 { 4038 device_t dev = adapter->dev; 4039 struct ixgbe_hw *hw = &adapter->hw; 4040 struct ix_queue *que = adapter->queues; 4041 struct rx_ring *rxr; 4042 struct tx_ring *txr; 4043 #ifdef LRO 4044 struct lro_ctrl *lro; 4045 #endif /* LRO */ 4046 4047 device_printf(dev,"Error Byte Count = %u \n", 4048 IXGBE_READ_REG(hw, IXGBE_ERRBC)); 4049 4050 for (int i = 0; i < adapter->num_queues; i++, que++) { 4051 txr = que->txr; 4052 rxr = que->rxr; 4053 #ifdef LRO 4054 lro = &rxr->lro; 4055 #endif /* LRO */ 4056 device_printf(dev,"QUE(%d) IRQs Handled: %lu\n", 4057 que->msix, (long)que->irqs); 4058 device_printf(dev,"RX(%d) Packets Received: %lld\n", 4059 rxr->me, (long long)rxr->rx_packets.ev_count); 4060 device_printf(dev,"RX(%d) Split RX Packets: %lld\n", 4061 rxr->me, (long long)rxr->rx_split_packets.ev_count); 4062 device_printf(dev,"RX(%d) Bytes Received: %lu\n", 4063 rxr->me, (long)rxr->rx_bytes.ev_count); 4064 #ifdef LRO 4065 device_printf(dev,"RX(%d) LRO Queued= %d\n", 4066 rxr->me, lro->lro_queued); 4067 device_printf(dev,"RX(%d) LRO Flushed= %d\n", 4068 rxr->me, lro->lro_flushed); 4069 #endif /* LRO */ 4070 device_printf(dev,"TX(%d) Packets Sent: %lu\n", 4071 txr->me, (long)txr->total_packets.ev_count); 4072 device_printf(dev,"TX(%d) NO Desc Avail: %lu\n", 4073 txr->me, (long)txr->no_desc_avail.ev_count); 4074 } 4075 4076 device_printf(dev,"MBX IRQ Handled: %lu\n", 4077 (long)adapter->mbx_irq.ev_count); 4078 return; 4079 } 4080 4081 static int 4082 ixv_sysctl_stats(SYSCTLFN_ARGS) 4083 { 4084 struct sysctlnode node; 4085 int error; 4086 int result; 4087 struct adapter *adapter; 4088 4089 node = *rnode; 4090 adapter = (struct adapter *)node.sysctl_data; 4091 node.sysctl_data = &result; 4092 error = sysctl_lookup(SYSCTLFN_CALL(&node)); 4093 if (error != 0) 4094 return error; 4095 4096 if (result == 1) 4097 ixv_print_hw_stats(adapter); 4098 4099 return 0; 4100 } 4101 4102 static int 4103 ixv_sysctl_debug(SYSCTLFN_ARGS) 4104 { 4105 struct sysctlnode node; 4106 int error, result; 4107 struct adapter *adapter; 4108 4109 node = *rnode; 4110 adapter = (struct adapter *)node.sysctl_data; 4111 node.sysctl_data = &result; 4112 error = sysctl_lookup(SYSCTLFN_CALL(&node)); 4113 4114 if (error) 4115 return error; 4116 4117 if (result == 1) 4118 ixv_print_debug_info(adapter); 4119 4120 return 0; 4121 } 4122 4123 /* 4124 ** Set flow control using sysctl: 4125 ** Flow control values: 4126 ** 0 - off 4127 ** 1 - rx pause 4128 ** 2 - tx pause 4129 ** 3 - full 4130 */ 4131 static int 4132 ixv_set_flowcntl(SYSCTLFN_ARGS) 4133 { 4134 struct sysctlnode node; 4135 int error; 4136 struct adapter *adapter; 4137 4138 node = *rnode; 4139 adapter = (struct adapter *)node.sysctl_data; 4140 node.sysctl_data = &ixv_flow_control; 4141 error = sysctl_lookup(SYSCTLFN_CALL(&node)); 4142 4143 if (error) 4144 return (error); 4145 4146 switch (ixv_flow_control) { 4147 case ixgbe_fc_rx_pause: 4148 case ixgbe_fc_tx_pause: 4149 case ixgbe_fc_full: 4150 adapter->hw.fc.requested_mode = ixv_flow_control; 4151 break; 4152 case ixgbe_fc_none: 4153 default: 4154 adapter->hw.fc.requested_mode = ixgbe_fc_none; 4155 } 4156 4157 ixgbe_fc_enable(&adapter->hw); 4158 return error; 4159 } 4160 4161 const struct sysctlnode * 4162 ixv_sysctl_instance(struct adapter *adapter) 4163 { 4164 const char *dvname; 4165 struct sysctllog **log; 4166 int rc; 4167 const struct sysctlnode *rnode; 4168 4169 log = &adapter->sysctllog; 4170 dvname = device_xname(adapter->dev); 4171 4172 if ((rc = sysctl_createv(log, 0, NULL, &rnode, 4173 0, CTLTYPE_NODE, dvname, 4174 SYSCTL_DESCR("ixv information and settings"), 4175 NULL, 0, NULL, 0, CTL_HW, CTL_CREATE, CTL_EOL)) != 0) 4176 goto err; 4177 4178 return rnode; 4179 err: 4180 printf("%s: sysctl_createv failed, rc = %d\n", __func__, rc); 4181 return NULL; 4182 } 4183 4184 static void 4185 ixv_add_rx_process_limit(struct adapter *adapter, const char *name, 4186 const char *description, int *limit, int value) 4187 { 4188 const struct sysctlnode *rnode, *cnode; 4189 struct sysctllog **log = &adapter->sysctllog; 4190 4191 *limit = value; 4192 4193 if ((rnode = ixv_sysctl_instance(adapter)) == NULL) 4194 aprint_error_dev(adapter->dev, 4195 "could not create sysctl root\n"); 4196 else if (sysctl_createv(log, 0, &rnode, &cnode, 4197 CTLFLAG_READWRITE, 4198 CTLTYPE_INT, 4199 name, SYSCTL_DESCR(description), 4200 NULL, 0, limit, 0, 4201 CTL_CREATE, CTL_EOL) != 0) { 4202 aprint_error_dev(adapter->dev, "%s: could not create sysctl", 4203 __func__); 4204 } 4205 } 4206 4207
Indexes created Tue Sep 30 20:09:53 GMT 2025