ixv.c revision 1.11
1 /****************************************************************************** 2 3 Copyright (c) 2001-2013, Intel Corporation 4 All rights reserved. 5 6 Redistribution and use in source and binary forms, with or without 7 modification, are permitted provided that the following conditions are met: 8 9 1. Redistributions of source code must retain the above copyright notice, 10 this list of conditions and the following disclaimer. 11 12 2. Redistributions in binary form must reproduce the above copyright 13 notice, this list of conditions and the following disclaimer in the 14 documentation and/or other materials provided with the distribution. 15 16 3. Neither the name of the Intel Corporation nor the names of its 17 contributors may be used to endorse or promote products derived from 18 this software without specific prior written permission. 19 20 THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 21 AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 22 IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 23 ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE 24 LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 25 CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 26 SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 27 INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 28 CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 29 ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 30 POSSIBILITY OF SUCH DAMAGE. 31 32 ******************************************************************************/ 33 /*$FreeBSD: head/sys/dev/ixgbe/ixv.c 275358 2014-12-01 11:45:24Z hselasky $*/ 34 /*$NetBSD: ixv.c,v 1.11 2015/08/13 04:56:43 msaitoh Exp $*/ 35 36 #include "opt_inet.h" 37 #include "opt_inet6.h" 38 39 #include "ixv.h" 40 41 /********************************************************************* 42 * Driver version 43 *********************************************************************/ 44 char ixv_driver_version[] = "1.1.4"; 45 46 /********************************************************************* 47 * PCI Device ID Table 48 * 49 * Used by probe to select devices to load on 50 * Last field stores an index into ixv_strings 51 * Last entry must be all 0s 52 * 53 * { Vendor ID, Device ID, SubVendor ID, SubDevice ID, String Index } 54 *********************************************************************/ 55 56 static ixv_vendor_info_t ixv_vendor_info_array[] = 57 { 58 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_VF, 0, 0, 0}, 59 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_X540_VF, 0, 0, 0}, 60 /* required last entry */ 61 {0, 0, 0, 0, 0} 62 }; 63 64 /********************************************************************* 65 * Table of branding strings 66 *********************************************************************/ 67 68 static const char *ixv_strings[] = { 69 "Intel(R) PRO/10GbE Virtual Function Network Driver" 70 }; 71 72 /********************************************************************* 73 * Function prototypes 74 *********************************************************************/ 75 static int ixv_probe(device_t, cfdata_t, void *); 76 static void ixv_attach(device_t, device_t, void *); 77 static int ixv_detach(device_t, int); 78 #if 0 79 static int ixv_shutdown(device_t); 80 #endif 81 #if __FreeBSD_version < 800000 82 static void ixv_start(struct ifnet *); 83 static void ixv_start_locked(struct tx_ring *, struct ifnet *); 84 #else 85 static int ixv_mq_start(struct ifnet *, struct mbuf *); 86 static int ixv_mq_start_locked(struct ifnet *, 87 struct tx_ring *, struct mbuf *); 88 static void ixv_qflush(struct ifnet *); 89 #endif 90 static int ixv_ioctl(struct ifnet *, u_long, void *); 91 static int ixv_init(struct ifnet *); 92 static void ixv_init_locked(struct adapter *); 93 static void ixv_stop(void *); 94 static void ixv_media_status(struct ifnet *, struct ifmediareq *); 95 static int ixv_media_change(struct ifnet *); 96 static void ixv_identify_hardware(struct adapter *); 97 static int ixv_allocate_pci_resources(struct adapter *, 98 const struct pci_attach_args *); 99 static int ixv_allocate_msix(struct adapter *, 100 const struct pci_attach_args *); 101 static int ixv_allocate_queues(struct adapter *); 102 static int ixv_setup_msix(struct adapter *); 103 static void ixv_free_pci_resources(struct adapter *); 104 static void ixv_local_timer(void *); 105 static void ixv_setup_interface(device_t, struct adapter *); 106 static void ixv_config_link(struct adapter *); 107 108 static int ixv_allocate_transmit_buffers(struct tx_ring *); 109 static int ixv_setup_transmit_structures(struct adapter *); 110 static void ixv_setup_transmit_ring(struct tx_ring *); 111 static void ixv_initialize_transmit_units(struct adapter *); 112 static void ixv_free_transmit_structures(struct adapter *); 113 static void ixv_free_transmit_buffers(struct tx_ring *); 114 115 static int ixv_allocate_receive_buffers(struct rx_ring *); 116 static int ixv_setup_receive_structures(struct adapter *); 117 static int ixv_setup_receive_ring(struct rx_ring *); 118 static void ixv_initialize_receive_units(struct adapter *); 119 static void ixv_free_receive_structures(struct adapter *); 120 static void ixv_free_receive_buffers(struct rx_ring *); 121 122 static void ixv_enable_intr(struct adapter *); 123 static void ixv_disable_intr(struct adapter *); 124 static bool ixv_txeof(struct tx_ring *); 125 static bool ixv_rxeof(struct ix_queue *, int); 126 static void ixv_rx_checksum(u32, struct mbuf *, u32, 127 struct ixgbevf_hw_stats *); 128 static void ixv_set_multi(struct adapter *); 129 static void ixv_update_link_status(struct adapter *); 130 static void ixv_refresh_mbufs(struct rx_ring *, int); 131 static int ixv_xmit(struct tx_ring *, struct mbuf *); 132 static int ixv_sysctl_stats(SYSCTLFN_PROTO); 133 static int ixv_sysctl_debug(SYSCTLFN_PROTO); 134 static int ixv_set_flowcntl(SYSCTLFN_PROTO); 135 static int ixv_dma_malloc(struct adapter *, bus_size_t, 136 struct ixv_dma_alloc *, int); 137 static void ixv_dma_free(struct adapter *, struct ixv_dma_alloc *); 138 static void ixv_add_rx_process_limit(struct adapter *, const char *, 139 const char *, int *, int); 140 static u32 ixv_tx_ctx_setup(struct tx_ring *, struct mbuf *); 141 static bool ixv_tso_setup(struct tx_ring *, struct mbuf *, u32 *); 142 static void ixv_set_ivar(struct adapter *, u8, u8, s8); 143 static void ixv_configure_ivars(struct adapter *); 144 static u8 * ixv_mc_array_itr(struct ixgbe_hw *, u8 **, u32 *); 145 146 static void ixv_setup_vlan_support(struct adapter *); 147 #if 0 148 static void ixv_register_vlan(void *, struct ifnet *, u16); 149 static void ixv_unregister_vlan(void *, struct ifnet *, u16); 150 #endif 151 152 static void ixv_save_stats(struct adapter *); 153 static void ixv_init_stats(struct adapter *); 154 static void ixv_update_stats(struct adapter *); 155 156 static __inline void ixv_rx_discard(struct rx_ring *, int); 157 static __inline void ixv_rx_input(struct rx_ring *, struct ifnet *, 158 struct mbuf *, u32); 159 160 /* The MSI/X Interrupt handlers */ 161 static int ixv_msix_que(void *); 162 static int ixv_msix_mbx(void *); 163 164 /* Deferred interrupt tasklets */ 165 static void ixv_handle_que(void *); 166 static void ixv_handle_mbx(void *); 167 168 const struct sysctlnode *ixv_sysctl_instance(struct adapter *); 169 static ixv_vendor_info_t *ixv_lookup(const struct pci_attach_args *); 170 171 /********************************************************************* 172 * FreeBSD Device Interface Entry Points 173 *********************************************************************/ 174 175 CFATTACH_DECL3_NEW(ixv, sizeof(struct adapter), 176 ixv_probe, ixv_attach, ixv_detach, NULL, NULL, NULL, 177 DVF_DETACH_SHUTDOWN); 178 179 # if 0 180 static device_method_t ixv_methods[] = { 181 /* Device interface */ 182 DEVMETHOD(device_probe, ixv_probe), 183 DEVMETHOD(device_attach, ixv_attach), 184 DEVMETHOD(device_detach, ixv_detach), 185 DEVMETHOD(device_shutdown, ixv_shutdown), 186 DEVMETHOD_END 187 }; 188 #endif 189 190 #if 0 191 static driver_t ixv_driver = { 192 "ix", ixv_methods, sizeof(struct adapter), 193 }; 194 195 extern devclass_t ixgbe_devclass; 196 DRIVER_MODULE(ixv, pci, ixv_driver, ixgbe_devclass, 0, 0); 197 MODULE_DEPEND(ixv, pci, 1, 1, 1); 198 MODULE_DEPEND(ixv, ether, 1, 1, 1); 199 #endif 200 201 /* 202 ** TUNEABLE PARAMETERS: 203 */ 204 205 /* 206 ** AIM: Adaptive Interrupt Moderation 207 ** which means that the interrupt rate 208 ** is varied over time based on the 209 ** traffic for that interrupt vector 210 */ 211 static int ixv_enable_aim = FALSE; 212 #define TUNABLE_INT(__x, __y) 213 TUNABLE_INT("hw.ixv.enable_aim", &ixv_enable_aim); 214 215 /* How many packets rxeof tries to clean at a time */ 216 static int ixv_rx_process_limit = 128; 217 TUNABLE_INT("hw.ixv.rx_process_limit", &ixv_rx_process_limit); 218 219 /* Flow control setting, default to full */ 220 static int ixv_flow_control = ixgbe_fc_full; 221 TUNABLE_INT("hw.ixv.flow_control", &ixv_flow_control); 222 223 /* 224 * Header split: this causes the hardware to DMA 225 * the header into a seperate mbuf from the payload, 226 * it can be a performance win in some workloads, but 227 * in others it actually hurts, its off by default. 228 */ 229 static int ixv_header_split = FALSE; 230 TUNABLE_INT("hw.ixv.hdr_split", &ixv_header_split); 231 232 /* 233 ** Number of TX descriptors per ring, 234 ** setting higher than RX as this seems 235 ** the better performing choice. 236 */ 237 static int ixv_txd = DEFAULT_TXD; 238 TUNABLE_INT("hw.ixv.txd", &ixv_txd); 239 240 /* Number of RX descriptors per ring */ 241 static int ixv_rxd = DEFAULT_RXD; 242 TUNABLE_INT("hw.ixv.rxd", &ixv_rxd); 243 244 /* 245 ** Shadow VFTA table, this is needed because 246 ** the real filter table gets cleared during 247 ** a soft reset and we need to repopulate it. 248 */ 249 static u32 ixv_shadow_vfta[VFTA_SIZE]; 250 251 /* Keep running tab on them for sanity check */ 252 static int ixv_total_ports; 253 254 /********************************************************************* 255 * Device identification routine 256 * 257 * ixv_probe determines if the driver should be loaded on 258 * adapter based on PCI vendor/device id of the adapter. 259 * 260 * return 1 on success, 0 on failure 261 *********************************************************************/ 262 263 static int 264 ixv_probe(device_t dev, cfdata_t cf, void *aux) 265 { 266 const struct pci_attach_args *pa = aux; 267 268 return (ixv_lookup(pa) != NULL) ? 1 : 0; 269 } 270 271 static ixv_vendor_info_t * 272 ixv_lookup(const struct pci_attach_args *pa) 273 { 274 pcireg_t subid; 275 ixv_vendor_info_t *ent; 276 277 INIT_DEBUGOUT("ixv_probe: begin"); 278 279 if (PCI_VENDOR(pa->pa_id) != IXGBE_INTEL_VENDOR_ID) 280 return NULL; 281 282 subid = pci_conf_read(pa->pa_pc, pa->pa_tag, PCI_SUBSYS_ID_REG); 283 284 for (ent = ixv_vendor_info_array; ent->vendor_id != 0; ent++) { 285 if ((PCI_VENDOR(pa->pa_id) == ent->vendor_id) && 286 (PCI_PRODUCT(pa->pa_id) == ent->device_id) && 287 288 ((PCI_SUBSYS_VENDOR(subid) == ent->subvendor_id) || 289 (ent->subvendor_id == 0)) && 290 291 ((PCI_SUBSYS_ID(subid) == ent->subdevice_id) || 292 (ent->subdevice_id == 0))) { 293 ++ixv_total_ports; 294 return ent; 295 } 296 } 297 return NULL; 298 } 299 300 301 static void 302 ixv_sysctl_attach(struct adapter *adapter) 303 { 304 struct sysctllog **log; 305 const struct sysctlnode *rnode, *cnode; 306 device_t dev; 307 308 dev = adapter->dev; 309 log = &adapter->sysctllog; 310 311 if ((rnode = ixv_sysctl_instance(adapter)) == NULL) { 312 aprint_error_dev(dev, "could not create sysctl root\n"); 313 return; 314 } 315 316 if (sysctl_createv(log, 0, &rnode, &cnode, 317 CTLFLAG_READWRITE, CTLTYPE_INT, 318 "stats", SYSCTL_DESCR("Statistics"), 319 ixv_sysctl_stats, 0, (void *)adapter, 0, CTL_CREATE, CTL_EOL) != 0) 320 aprint_error_dev(dev, "could not create sysctl\n"); 321 322 if (sysctl_createv(log, 0, &rnode, &cnode, 323 CTLFLAG_READWRITE, CTLTYPE_INT, 324 "debug", SYSCTL_DESCR("Debug Info"), 325 ixv_sysctl_debug, 0, (void *)adapter, 0, CTL_CREATE, CTL_EOL) != 0) 326 aprint_error_dev(dev, "could not create sysctl\n"); 327 328 if (sysctl_createv(log, 0, &rnode, &cnode, 329 CTLFLAG_READWRITE, CTLTYPE_INT, 330 "flow_control", SYSCTL_DESCR("Flow Control"), 331 ixv_set_flowcntl, 0, (void *)adapter, 0, CTL_CREATE, CTL_EOL) != 0) 332 aprint_error_dev(dev, "could not create sysctl\n"); 333 334 /* XXX This is an *instance* sysctl controlling a *global* variable. 335 * XXX It's that way in the FreeBSD driver that this derives from. 336 */ 337 if (sysctl_createv(log, 0, &rnode, &cnode, 338 CTLFLAG_READWRITE, CTLTYPE_INT, 339 "enable_aim", SYSCTL_DESCR("Interrupt Moderation"), 340 NULL, 0, &ixv_enable_aim, 0, CTL_CREATE, CTL_EOL) != 0) 341 aprint_error_dev(dev, "could not create sysctl\n"); 342 } 343 344 /********************************************************************* 345 * Device initialization routine 346 * 347 * The attach entry point is called when the driver is being loaded. 348 * This routine identifies the type of hardware, allocates all resources 349 * and initializes the hardware. 350 * 351 * return 0 on success, positive on failure 352 *********************************************************************/ 353 354 static void 355 ixv_attach(device_t parent, device_t dev, void *aux) 356 { 357 struct adapter *adapter; 358 struct ixgbe_hw *hw; 359 int error = 0; 360 ixv_vendor_info_t *ent; 361 const struct pci_attach_args *pa = aux; 362 363 INIT_DEBUGOUT("ixv_attach: begin"); 364 365 /* Allocate, clear, and link in our adapter structure */ 366 adapter = device_private(dev); 367 adapter->dev = adapter->osdep.dev = dev; 368 hw = &adapter->hw; 369 370 ent = ixv_lookup(pa); 371 372 KASSERT(ent != NULL); 373 374 aprint_normal(": %s, Version - %s\n", 375 ixv_strings[ent->index], ixv_driver_version); 376 377 /* Core Lock Init*/ 378 IXV_CORE_LOCK_INIT(adapter, device_xname(dev)); 379 380 /* SYSCTL APIs */ 381 ixv_sysctl_attach(adapter); 382 383 /* Set up the timer callout */ 384 callout_init(&adapter->timer, 0); 385 386 /* Determine hardware revision */ 387 ixv_identify_hardware(adapter); 388 389 /* Do base PCI setup - map BAR0 */ 390 if (ixv_allocate_pci_resources(adapter, pa)) { 391 aprint_error_dev(dev, "Allocation of PCI resources failed\n"); 392 error = ENXIO; 393 goto err_out; 394 } 395 396 /* Do descriptor calc and sanity checks */ 397 if (((ixv_txd * sizeof(union ixgbe_adv_tx_desc)) % DBA_ALIGN) != 0 || 398 ixv_txd < MIN_TXD || ixv_txd > MAX_TXD) { 399 aprint_error_dev(dev, "TXD config issue, using default!\n"); 400 adapter->num_tx_desc = DEFAULT_TXD; 401 } else 402 adapter->num_tx_desc = ixv_txd; 403 404 if (((ixv_rxd * sizeof(union ixgbe_adv_rx_desc)) % DBA_ALIGN) != 0 || 405 ixv_rxd < MIN_RXD || ixv_rxd > MAX_RXD) { 406 aprint_error_dev(dev, "RXD config issue, using default!\n"); 407 adapter->num_rx_desc = DEFAULT_RXD; 408 } else 409 adapter->num_rx_desc = ixv_rxd; 410 411 /* Allocate our TX/RX Queues */ 412 if (ixv_allocate_queues(adapter)) { 413 error = ENOMEM; 414 goto err_out; 415 } 416 417 /* 418 ** Initialize the shared code: its 419 ** at this point the mac type is set. 420 */ 421 error = ixgbe_init_shared_code(hw); 422 if (error) { 423 aprint_error_dev(dev,"Shared Code Initialization Failure\n"); 424 error = EIO; 425 goto err_late; 426 } 427 428 /* Setup the mailbox */ 429 ixgbe_init_mbx_params_vf(hw); 430 431 ixgbe_reset_hw(hw); 432 433 /* Get Hardware Flow Control setting */ 434 hw->fc.requested_mode = ixgbe_fc_full; 435 hw->fc.pause_time = IXV_FC_PAUSE; 436 hw->fc.low_water[0] = IXV_FC_LO; 437 hw->fc.high_water[0] = IXV_FC_HI; 438 hw->fc.send_xon = TRUE; 439 440 error = ixgbe_init_hw(hw); 441 if (error) { 442 aprint_error_dev(dev,"Hardware Initialization Failure\n"); 443 error = EIO; 444 goto err_late; 445 } 446 447 error = ixv_allocate_msix(adapter, pa); 448 if (error) 449 goto err_late; 450 451 /* Setup OS specific network interface */ 452 ixv_setup_interface(dev, adapter); 453 454 /* Sysctl for limiting the amount of work done in the taskqueue */ 455 ixv_add_rx_process_limit(adapter, "rx_processing_limit", 456 "max number of rx packets to process", &adapter->rx_process_limit, 457 ixv_rx_process_limit); 458 459 /* Do the stats setup */ 460 ixv_save_stats(adapter); 461 ixv_init_stats(adapter); 462 463 /* Register for VLAN events */ 464 #if 0 /* XXX msaitoh delete after write? */ 465 adapter->vlan_attach = EVENTHANDLER_REGISTER(vlan_config, 466 ixv_register_vlan, adapter, EVENTHANDLER_PRI_FIRST); 467 adapter->vlan_detach = EVENTHANDLER_REGISTER(vlan_unconfig, 468 ixv_unregister_vlan, adapter, EVENTHANDLER_PRI_FIRST); 469 #endif 470 471 INIT_DEBUGOUT("ixv_attach: end"); 472 return; 473 474 err_late: 475 ixv_free_transmit_structures(adapter); 476 ixv_free_receive_structures(adapter); 477 err_out: 478 ixv_free_pci_resources(adapter); 479 return; 480 481 } 482 483 /********************************************************************* 484 * Device removal routine 485 * 486 * The detach entry point is called when the driver is being removed. 487 * This routine stops the adapter and deallocates all the resources 488 * that were allocated for driver operation. 489 * 490 * return 0 on success, positive on failure 491 *********************************************************************/ 492 493 static int 494 ixv_detach(device_t dev, int flags) 495 { 496 struct adapter *adapter = device_private(dev); 497 struct ix_queue *que = adapter->queues; 498 499 INIT_DEBUGOUT("ixv_detach: begin"); 500 501 /* Make sure VLANS are not using driver */ 502 if (!VLAN_ATTACHED(&adapter->osdep.ec)) 503 ; /* nothing to do: no VLANs */ 504 else if ((flags & (DETACH_SHUTDOWN|DETACH_FORCE)) != 0) 505 vlan_ifdetach(adapter->ifp); 506 else { 507 aprint_error_dev(dev, "VLANs in use\n"); 508 return EBUSY; 509 } 510 511 IXV_CORE_LOCK(adapter); 512 ixv_stop(adapter); 513 IXV_CORE_UNLOCK(adapter); 514 515 for (int i = 0; i < adapter->num_queues; i++, que++) { 516 softint_disestablish(que->que_si); 517 } 518 519 /* Drain the Link queue */ 520 softint_disestablish(adapter->mbx_si); 521 522 /* Unregister VLAN events */ 523 #if 0 /* XXX msaitoh delete after write? */ 524 if (adapter->vlan_attach != NULL) 525 EVENTHANDLER_DEREGISTER(vlan_config, adapter->vlan_attach); 526 if (adapter->vlan_detach != NULL) 527 EVENTHANDLER_DEREGISTER(vlan_unconfig, adapter->vlan_detach); 528 #endif 529 530 ether_ifdetach(adapter->ifp); 531 callout_halt(&adapter->timer, NULL); 532 ixv_free_pci_resources(adapter); 533 #if 0 /* XXX the NetBSD port is probably missing something here */ 534 bus_generic_detach(dev); 535 #endif 536 if_detach(adapter->ifp); 537 538 ixv_free_transmit_structures(adapter); 539 ixv_free_receive_structures(adapter); 540 541 IXV_CORE_LOCK_DESTROY(adapter); 542 return (0); 543 } 544 545 /********************************************************************* 546 * 547 * Shutdown entry point 548 * 549 **********************************************************************/ 550 #if 0 /* XXX NetBSD ought to register something like this through pmf(9) */ 551 static int 552 ixv_shutdown(device_t dev) 553 { 554 struct adapter *adapter = device_private(dev); 555 IXV_CORE_LOCK(adapter); 556 ixv_stop(adapter); 557 IXV_CORE_UNLOCK(adapter); 558 return (0); 559 } 560 #endif 561 562 #if __FreeBSD_version < 800000 563 /********************************************************************* 564 * Transmit entry point 565 * 566 * ixv_start is called by the stack to initiate a transmit. 567 * The driver will remain in this routine as long as there are 568 * packets to transmit and transmit resources are available. 569 * In case resources are not available stack is notified and 570 * the packet is requeued. 571 **********************************************************************/ 572 static void 573 ixv_start_locked(struct tx_ring *txr, struct ifnet * ifp) 574 { 575 int rc; 576 struct mbuf *m_head; 577 struct adapter *adapter = txr->adapter; 578 579 IXV_TX_LOCK_ASSERT(txr); 580 581 if ((ifp->if_flags & (IFF_RUNNING|IFF_OACTIVE)) != 582 IFF_RUNNING) 583 return; 584 if (!adapter->link_active) 585 return; 586 587 while (!IFQ_IS_EMPTY(&ifp->if_snd)) { 588 589 IFQ_POLL(&ifp->if_snd, m_head); 590 if (m_head == NULL) 591 break; 592 593 if ((rc = ixv_xmit(txr, m_head)) == EAGAIN) { 594 ifp->if_flags |= IFF_OACTIVE; 595 break; 596 } 597 IFQ_DEQUEUE(&ifp->if_snd, m_head); 598 if (rc == EFBIG) { 599 struct mbuf *mtmp; 600 601 if ((mtmp = m_defrag(m_head, M_NOWAIT)) != NULL) { 602 m_head = mtmp; 603 rc = ixv_xmit(txr, m_head); 604 if (rc != 0) 605 adapter->efbig2_tx_dma_setup.ev_count++; 606 } else 607 adapter->m_defrag_failed.ev_count++; 608 } 609 if (rc != 0) { 610 m_freem(m_head); 611 continue; 612 } 613 /* Send a copy of the frame to the BPF listener */ 614 bpf_mtap(ifp, m_head); 615 616 /* Set watchdog on */ 617 txr->watchdog_check = TRUE; 618 getmicrotime(&txr->watchdog_time); 619 } 620 return; 621 } 622 623 /* 624 * Legacy TX start - called by the stack, this 625 * always uses the first tx ring, and should 626 * not be used with multiqueue tx enabled. 627 */ 628 static void 629 ixv_start(struct ifnet *ifp) 630 { 631 struct adapter *adapter = ifp->if_softc; 632 struct tx_ring *txr = adapter->tx_rings; 633 634 if (ifp->if_flags & IFF_RUNNING) { 635 IXV_TX_LOCK(txr); 636 ixv_start_locked(txr, ifp); 637 IXV_TX_UNLOCK(txr); 638 } 639 return; 640 } 641 642 #else 643 644 /* 645 ** Multiqueue Transmit driver 646 ** 647 */ 648 static int 649 ixv_mq_start(struct ifnet *ifp, struct mbuf *m) 650 { 651 struct adapter *adapter = ifp->if_softc; 652 struct ix_queue *que; 653 struct tx_ring *txr; 654 int i = 0, err = 0; 655 656 /* Which queue to use */ 657 if ((m->m_flags & M_FLOWID) != 0) 658 i = m->m_pkthdr.flowid % adapter->num_queues; 659 660 txr = &adapter->tx_rings[i]; 661 que = &adapter->queues[i]; 662 663 if (IXV_TX_TRYLOCK(txr)) { 664 err = ixv_mq_start_locked(ifp, txr, m); 665 IXV_TX_UNLOCK(txr); 666 } else { 667 err = drbr_enqueue(ifp, txr->br, m); 668 softint_schedule(que->que_si); 669 } 670 671 return (err); 672 } 673 674 static int 675 ixv_mq_start_locked(struct ifnet *ifp, struct tx_ring *txr, struct mbuf *m) 676 { 677 struct adapter *adapter = txr->adapter; 678 struct mbuf *next; 679 int enqueued, err = 0; 680 681 if ((ifp->if_flags & (IFF_RUNNING | IFF_OACTIVE)) != 682 IFF_RUNNING || adapter->link_active == 0) { 683 if (m != NULL) 684 err = drbr_enqueue(ifp, txr->br, m); 685 return (err); 686 } 687 688 /* Do a clean if descriptors are low */ 689 if (txr->tx_avail <= IXV_TX_CLEANUP_THRESHOLD) 690 ixv_txeof(txr); 691 692 enqueued = 0; 693 if (m == NULL) { 694 err = drbr_dequeue(ifp, txr->br, m); 695 if (err) { 696 return (err); 697 } 698 } 699 /* Process the queue */ 700 while ((next = drbr_peek(ifp, txr->br)) != NULL) { 701 if ((err = ixv_xmit(txr, next)) != 0) { 702 if (next != NULL) { 703 drbr_advance(ifp, txr->br); 704 } else { 705 drbr_putback(ifp, txr->br, next); 706 } 707 break; 708 } 709 drbr_advance(ifp, txr->br); 710 enqueued++; 711 ifp->if_obytes += next->m_pkthdr.len; 712 if (next->m_flags & M_MCAST) 713 ifp->if_omcasts++; 714 /* Send a copy of the frame to the BPF listener */ 715 ETHER_BPF_MTAP(ifp, next); 716 if ((ifp->if_flags & IFF_RUNNING) == 0) 717 break; 718 if (txr->tx_avail <= IXV_TX_OP_THRESHOLD) { 719 ifp->if_flags |= IFF_OACTIVE; 720 break; 721 } 722 } 723 724 if (enqueued > 0) { 725 /* Set watchdog on */ 726 txr->watchdog_check = TRUE; 727 getmicrotime(&txr->watchdog_time); 728 } 729 730 return (err); 731 } 732 733 /* 734 ** Flush all ring buffers 735 */ 736 static void 737 ixv_qflush(struct ifnet *ifp) 738 { 739 struct adapter *adapter = ifp->if_softc; 740 struct tx_ring *txr = adapter->tx_rings; 741 struct mbuf *m; 742 743 for (int i = 0; i < adapter->num_queues; i++, txr++) { 744 IXV_TX_LOCK(txr); 745 while ((m = buf_ring_dequeue_sc(txr->br)) != NULL) 746 m_freem(m); 747 IXV_TX_UNLOCK(txr); 748 } 749 if_qflush(ifp); 750 } 751 752 #endif 753 754 static int 755 ixv_ifflags_cb(struct ethercom *ec) 756 { 757 struct ifnet *ifp = &ec->ec_if; 758 struct adapter *adapter = ifp->if_softc; 759 int change = ifp->if_flags ^ adapter->if_flags, rc = 0; 760 761 IXV_CORE_LOCK(adapter); 762 763 if (change != 0) 764 adapter->if_flags = ifp->if_flags; 765 766 if ((change & ~(IFF_CANTCHANGE|IFF_DEBUG)) != 0) 767 rc = ENETRESET; 768 769 IXV_CORE_UNLOCK(adapter); 770 771 return rc; 772 } 773 774 /********************************************************************* 775 * Ioctl entry point 776 * 777 * ixv_ioctl is called when the user wants to configure the 778 * interface. 779 * 780 * return 0 on success, positive on failure 781 **********************************************************************/ 782 783 static int 784 ixv_ioctl(struct ifnet * ifp, u_long command, void *data) 785 { 786 struct adapter *adapter = ifp->if_softc; 787 struct ifcapreq *ifcr = data; 788 struct ifreq *ifr = (struct ifreq *) data; 789 int error = 0; 790 int l4csum_en; 791 const int l4csum = IFCAP_CSUM_TCPv4_Rx|IFCAP_CSUM_UDPv4_Rx| 792 IFCAP_CSUM_TCPv6_Rx|IFCAP_CSUM_UDPv6_Rx; 793 794 switch (command) { 795 case SIOCSIFFLAGS: 796 IOCTL_DEBUGOUT("ioctl: SIOCSIFFLAGS (Set Interface Flags)"); 797 break; 798 case SIOCADDMULTI: 799 case SIOCDELMULTI: 800 IOCTL_DEBUGOUT("ioctl: SIOC(ADD|DEL)MULTI"); 801 break; 802 case SIOCSIFMEDIA: 803 case SIOCGIFMEDIA: 804 IOCTL_DEBUGOUT("ioctl: SIOCxIFMEDIA (Get/Set Interface Media)"); 805 break; 806 case SIOCSIFCAP: 807 IOCTL_DEBUGOUT("ioctl: SIOCSIFCAP (Set Capabilities)"); 808 break; 809 case SIOCSIFMTU: 810 IOCTL_DEBUGOUT("ioctl: SIOCSIFMTU (Set Interface MTU)"); 811 break; 812 default: 813 IOCTL_DEBUGOUT1("ioctl: UNKNOWN (0x%X)\n", (int)command); 814 break; 815 } 816 817 switch (command) { 818 case SIOCSIFMEDIA: 819 case SIOCGIFMEDIA: 820 return ifmedia_ioctl(ifp, ifr, &adapter->media, command); 821 case SIOCSIFCAP: 822 /* Layer-4 Rx checksum offload has to be turned on and 823 * off as a unit. 824 */ 825 l4csum_en = ifcr->ifcr_capenable & l4csum; 826 if (l4csum_en != l4csum && l4csum_en != 0) 827 return EINVAL; 828 /*FALLTHROUGH*/ 829 case SIOCADDMULTI: 830 case SIOCDELMULTI: 831 case SIOCSIFFLAGS: 832 case SIOCSIFMTU: 833 default: 834 if ((error = ether_ioctl(ifp, command, data)) != ENETRESET) 835 return error; 836 if ((ifp->if_flags & IFF_RUNNING) == 0) 837 ; 838 else if (command == SIOCSIFCAP || command == SIOCSIFMTU) { 839 IXV_CORE_LOCK(adapter); 840 ixv_init_locked(adapter); 841 IXV_CORE_UNLOCK(adapter); 842 } else if (command == SIOCADDMULTI || command == SIOCDELMULTI) { 843 /* 844 * Multicast list has changed; set the hardware filter 845 * accordingly. 846 */ 847 IXV_CORE_LOCK(adapter); 848 ixv_disable_intr(adapter); 849 ixv_set_multi(adapter); 850 ixv_enable_intr(adapter); 851 IXV_CORE_UNLOCK(adapter); 852 } 853 return 0; 854 } 855 } 856 857 /********************************************************************* 858 * Init entry point 859 * 860 * This routine is used in two ways. It is used by the stack as 861 * init entry point in network interface structure. It is also used 862 * by the driver as a hw/sw initialization routine to get to a 863 * consistent state. 864 * 865 * return 0 on success, positive on failure 866 **********************************************************************/ 867 #define IXGBE_MHADD_MFS_SHIFT 16 868 869 static void 870 ixv_init_locked(struct adapter *adapter) 871 { 872 struct ifnet *ifp = adapter->ifp; 873 device_t dev = adapter->dev; 874 struct ixgbe_hw *hw = &adapter->hw; 875 u32 mhadd, gpie; 876 877 INIT_DEBUGOUT("ixv_init: begin"); 878 KASSERT(mutex_owned(&adapter->core_mtx)); 879 hw->adapter_stopped = FALSE; 880 ixgbe_stop_adapter(hw); 881 callout_stop(&adapter->timer); 882 883 /* reprogram the RAR[0] in case user changed it. */ 884 ixgbe_set_rar(hw, 0, hw->mac.addr, 0, IXGBE_RAH_AV); 885 886 /* Get the latest mac address, User can use a LAA */ 887 memcpy(hw->mac.addr, CLLADDR(adapter->ifp->if_sadl), 888 IXGBE_ETH_LENGTH_OF_ADDRESS); 889 ixgbe_set_rar(hw, 0, hw->mac.addr, 0, 1); 890 hw->addr_ctrl.rar_used_count = 1; 891 892 /* Prepare transmit descriptors and buffers */ 893 if (ixv_setup_transmit_structures(adapter)) { 894 aprint_error_dev(dev,"Could not setup transmit structures\n"); 895 ixv_stop(adapter); 896 return; 897 } 898 899 ixgbe_reset_hw(hw); 900 ixv_initialize_transmit_units(adapter); 901 902 /* Setup Multicast table */ 903 ixv_set_multi(adapter); 904 905 /* 906 ** Determine the correct mbuf pool 907 ** for doing jumbo/headersplit 908 */ 909 if (ifp->if_mtu > ETHERMTU) 910 adapter->rx_mbuf_sz = MJUMPAGESIZE; 911 else 912 adapter->rx_mbuf_sz = MCLBYTES; 913 914 /* Prepare receive descriptors and buffers */ 915 if (ixv_setup_receive_structures(adapter)) { 916 device_printf(dev,"Could not setup receive structures\n"); 917 ixv_stop(adapter); 918 return; 919 } 920 921 /* Configure RX settings */ 922 ixv_initialize_receive_units(adapter); 923 924 /* Enable Enhanced MSIX mode */ 925 gpie = IXGBE_READ_REG(&adapter->hw, IXGBE_GPIE); 926 gpie |= IXGBE_GPIE_MSIX_MODE | IXGBE_GPIE_EIAME; 927 gpie |= IXGBE_GPIE_PBA_SUPPORT | IXGBE_GPIE_OCD; 928 IXGBE_WRITE_REG(hw, IXGBE_GPIE, gpie); 929 930 #if 0 /* XXX isn't it required? -- msaitoh */ 931 /* Set the various hardware offload abilities */ 932 ifp->if_hwassist = 0; 933 if (ifp->if_capenable & IFCAP_TSO4) 934 ifp->if_hwassist |= CSUM_TSO; 935 if (ifp->if_capenable & IFCAP_TXCSUM) { 936 ifp->if_hwassist |= (CSUM_TCP | CSUM_UDP); 937 #if __FreeBSD_version >= 800000 938 ifp->if_hwassist |= CSUM_SCTP; 939 #endif 940 } 941 #endif 942 943 /* Set MTU size */ 944 if (ifp->if_mtu > ETHERMTU) { 945 mhadd = IXGBE_READ_REG(hw, IXGBE_MHADD); 946 mhadd &= ~IXGBE_MHADD_MFS_MASK; 947 mhadd |= adapter->max_frame_size << IXGBE_MHADD_MFS_SHIFT; 948 IXGBE_WRITE_REG(hw, IXGBE_MHADD, mhadd); 949 } 950 951 /* Set up VLAN offload and filter */ 952 ixv_setup_vlan_support(adapter); 953 954 callout_reset(&adapter->timer, hz, ixv_local_timer, adapter); 955 956 /* Set up MSI/X routing */ 957 ixv_configure_ivars(adapter); 958 959 /* Set up auto-mask */ 960 IXGBE_WRITE_REG(hw, IXGBE_VTEIAM, IXGBE_EICS_RTX_QUEUE); 961 962 /* Set moderation on the Link interrupt */ 963 IXGBE_WRITE_REG(hw, IXGBE_VTEITR(adapter->mbxvec), IXV_LINK_ITR); 964 965 /* Stats init */ 966 ixv_init_stats(adapter); 967 968 /* Config/Enable Link */ 969 ixv_config_link(adapter); 970 971 /* And now turn on interrupts */ 972 ixv_enable_intr(adapter); 973 974 /* Now inform the stack we're ready */ 975 ifp->if_flags |= IFF_RUNNING; 976 ifp->if_flags &= ~IFF_OACTIVE; 977 978 return; 979 } 980 981 static int 982 ixv_init(struct ifnet *ifp) 983 { 984 struct adapter *adapter = ifp->if_softc; 985 986 IXV_CORE_LOCK(adapter); 987 ixv_init_locked(adapter); 988 IXV_CORE_UNLOCK(adapter); 989 return 0; 990 } 991 992 993 /* 994 ** 995 ** MSIX Interrupt Handlers and Tasklets 996 ** 997 */ 998 999 static inline void 1000 ixv_enable_queue(struct adapter *adapter, u32 vector) 1001 { 1002 struct ixgbe_hw *hw = &adapter->hw; 1003 u32 queue = 1 << vector; 1004 u32 mask; 1005 1006 mask = (IXGBE_EIMS_RTX_QUEUE & queue); 1007 IXGBE_WRITE_REG(hw, IXGBE_VTEIMS, mask); 1008 } 1009 1010 static inline void 1011 ixv_disable_queue(struct adapter *adapter, u32 vector) 1012 { 1013 struct ixgbe_hw *hw = &adapter->hw; 1014 u64 queue = (u64)(1 << vector); 1015 u32 mask; 1016 1017 mask = (IXGBE_EIMS_RTX_QUEUE & queue); 1018 IXGBE_WRITE_REG(hw, IXGBE_VTEIMC, mask); 1019 } 1020 1021 static inline void 1022 ixv_rearm_queues(struct adapter *adapter, u64 queues) 1023 { 1024 u32 mask = (IXGBE_EIMS_RTX_QUEUE & queues); 1025 IXGBE_WRITE_REG(&adapter->hw, IXGBE_VTEICS, mask); 1026 } 1027 1028 1029 static void 1030 ixv_handle_que(void *context) 1031 { 1032 struct ix_queue *que = context; 1033 struct adapter *adapter = que->adapter; 1034 struct tx_ring *txr = que->txr; 1035 struct ifnet *ifp = adapter->ifp; 1036 bool more; 1037 1038 if (ifp->if_flags & IFF_RUNNING) { 1039 more = ixv_rxeof(que, adapter->rx_process_limit); 1040 IXV_TX_LOCK(txr); 1041 ixv_txeof(txr); 1042 #if __FreeBSD_version >= 800000 1043 if (!drbr_empty(ifp, txr->br)) 1044 ixv_mq_start_locked(ifp, txr, NULL); 1045 #else 1046 if (!IFQ_IS_EMPTY(&ifp->if_snd)) 1047 ixv_start_locked(txr, ifp); 1048 #endif 1049 IXV_TX_UNLOCK(txr); 1050 if (more) { 1051 adapter->req.ev_count++; 1052 softint_schedule(que->que_si); 1053 return; 1054 } 1055 } 1056 1057 /* Reenable this interrupt */ 1058 ixv_enable_queue(adapter, que->msix); 1059 return; 1060 } 1061 1062 /********************************************************************* 1063 * 1064 * MSI Queue Interrupt Service routine 1065 * 1066 **********************************************************************/ 1067 int 1068 ixv_msix_que(void *arg) 1069 { 1070 struct ix_queue *que = arg; 1071 struct adapter *adapter = que->adapter; 1072 struct tx_ring *txr = que->txr; 1073 struct rx_ring *rxr = que->rxr; 1074 bool more_tx, more_rx; 1075 u32 newitr = 0; 1076 1077 ixv_disable_queue(adapter, que->msix); 1078 ++que->irqs; 1079 1080 more_rx = ixv_rxeof(que, adapter->rx_process_limit); 1081 1082 IXV_TX_LOCK(txr); 1083 more_tx = ixv_txeof(txr); 1084 /* 1085 ** Make certain that if the stack 1086 ** has anything queued the task gets 1087 ** scheduled to handle it. 1088 */ 1089 #if __FreeBSD_version < 800000 1090 if (!IFQ_IS_EMPTY(&adapter->ifp->if_snd)) 1091 #else 1092 if (!drbr_empty(adapter->ifp, txr->br)) 1093 #endif 1094 more_tx = 1; 1095 IXV_TX_UNLOCK(txr); 1096 1097 more_rx = ixv_rxeof(que, adapter->rx_process_limit); 1098 1099 /* Do AIM now? */ 1100 1101 if (ixv_enable_aim == FALSE) 1102 goto no_calc; 1103 /* 1104 ** Do Adaptive Interrupt Moderation: 1105 ** - Write out last calculated setting 1106 ** - Calculate based on average size over 1107 ** the last interval. 1108 */ 1109 if (que->eitr_setting) 1110 IXGBE_WRITE_REG(&adapter->hw, 1111 IXGBE_VTEITR(que->msix), 1112 que->eitr_setting); 1113 1114 que->eitr_setting = 0; 1115 1116 /* Idle, do nothing */ 1117 if ((txr->bytes == 0) && (rxr->bytes == 0)) 1118 goto no_calc; 1119 1120 if ((txr->bytes) && (txr->packets)) 1121 newitr = txr->bytes/txr->packets; 1122 if ((rxr->bytes) && (rxr->packets)) 1123 newitr = max(newitr, 1124 (rxr->bytes / rxr->packets)); 1125 newitr += 24; /* account for hardware frame, crc */ 1126 1127 /* set an upper boundary */ 1128 newitr = min(newitr, 3000); 1129 1130 /* Be nice to the mid range */ 1131 if ((newitr > 300) && (newitr < 1200)) 1132 newitr = (newitr / 3); 1133 else 1134 newitr = (newitr / 2); 1135 1136 newitr |= newitr << 16; 1137 1138 /* save for next interrupt */ 1139 que->eitr_setting = newitr; 1140 1141 /* Reset state */ 1142 txr->bytes = 0; 1143 txr->packets = 0; 1144 rxr->bytes = 0; 1145 rxr->packets = 0; 1146 1147 no_calc: 1148 if (more_tx || more_rx) 1149 softint_schedule(que->que_si); 1150 else /* Reenable this interrupt */ 1151 ixv_enable_queue(adapter, que->msix); 1152 return 1; 1153 } 1154 1155 static int 1156 ixv_msix_mbx(void *arg) 1157 { 1158 struct adapter *adapter = arg; 1159 struct ixgbe_hw *hw = &adapter->hw; 1160 u32 reg; 1161 1162 ++adapter->mbx_irq.ev_count; 1163 1164 /* First get the cause */ 1165 reg = IXGBE_READ_REG(hw, IXGBE_VTEICS); 1166 /* Clear interrupt with write */ 1167 IXGBE_WRITE_REG(hw, IXGBE_VTEICR, reg); 1168 1169 /* Link status change */ 1170 if (reg & IXGBE_EICR_LSC) 1171 softint_schedule(adapter->mbx_si); 1172 1173 IXGBE_WRITE_REG(hw, IXGBE_VTEIMS, IXGBE_EIMS_OTHER); 1174 return 1; 1175 } 1176 1177 /********************************************************************* 1178 * 1179 * Media Ioctl callback 1180 * 1181 * This routine is called whenever the user queries the status of 1182 * the interface using ifconfig. 1183 * 1184 **********************************************************************/ 1185 static void 1186 ixv_media_status(struct ifnet * ifp, struct ifmediareq * ifmr) 1187 { 1188 struct adapter *adapter = ifp->if_softc; 1189 1190 INIT_DEBUGOUT("ixv_media_status: begin"); 1191 IXV_CORE_LOCK(adapter); 1192 ixv_update_link_status(adapter); 1193 1194 ifmr->ifm_status = IFM_AVALID; 1195 ifmr->ifm_active = IFM_ETHER; 1196 1197 if (!adapter->link_active) { 1198 IXV_CORE_UNLOCK(adapter); 1199 return; 1200 } 1201 1202 ifmr->ifm_status |= IFM_ACTIVE; 1203 1204 switch (adapter->link_speed) { 1205 case IXGBE_LINK_SPEED_1GB_FULL: 1206 ifmr->ifm_active |= IFM_1000_T | IFM_FDX; 1207 break; 1208 case IXGBE_LINK_SPEED_10GB_FULL: 1209 ifmr->ifm_active |= IFM_FDX; 1210 break; 1211 } 1212 1213 IXV_CORE_UNLOCK(adapter); 1214 1215 return; 1216 } 1217 1218 /********************************************************************* 1219 * 1220 * Media Ioctl callback 1221 * 1222 * This routine is called when the user changes speed/duplex using 1223 * media/mediopt option with ifconfig. 1224 * 1225 **********************************************************************/ 1226 static int 1227 ixv_media_change(struct ifnet * ifp) 1228 { 1229 struct adapter *adapter = ifp->if_softc; 1230 struct ifmedia *ifm = &adapter->media; 1231 1232 INIT_DEBUGOUT("ixv_media_change: begin"); 1233 1234 if (IFM_TYPE(ifm->ifm_media) != IFM_ETHER) 1235 return (EINVAL); 1236 1237 switch (IFM_SUBTYPE(ifm->ifm_media)) { 1238 case IFM_AUTO: 1239 break; 1240 default: 1241 device_printf(adapter->dev, "Only auto media type\n"); 1242 return (EINVAL); 1243 } 1244 1245 return (0); 1246 } 1247 1248 /********************************************************************* 1249 * 1250 * This routine maps the mbufs to tx descriptors, allowing the 1251 * TX engine to transmit the packets. 1252 * - return 0 on success, positive on failure 1253 * 1254 **********************************************************************/ 1255 1256 static int 1257 ixv_xmit(struct tx_ring *txr, struct mbuf *m_head) 1258 { 1259 struct m_tag *mtag; 1260 struct adapter *adapter = txr->adapter; 1261 struct ethercom *ec = &adapter->osdep.ec; 1262 u32 olinfo_status = 0, cmd_type_len; 1263 u32 paylen = 0; 1264 int i, j, error; 1265 int first, last = 0; 1266 bus_dmamap_t map; 1267 struct ixv_tx_buf *txbuf; 1268 union ixgbe_adv_tx_desc *txd = NULL; 1269 1270 /* Basic descriptor defines */ 1271 cmd_type_len = (IXGBE_ADVTXD_DTYP_DATA | 1272 IXGBE_ADVTXD_DCMD_IFCS | IXGBE_ADVTXD_DCMD_DEXT); 1273 1274 if ((mtag = VLAN_OUTPUT_TAG(ec, m_head)) != NULL) 1275 cmd_type_len |= IXGBE_ADVTXD_DCMD_VLE; 1276 1277 /* 1278 * Important to capture the first descriptor 1279 * used because it will contain the index of 1280 * the one we tell the hardware to report back 1281 */ 1282 first = txr->next_avail_desc; 1283 txbuf = &txr->tx_buffers[first]; 1284 map = txbuf->map; 1285 1286 /* 1287 * Map the packet for DMA. 1288 */ 1289 error = bus_dmamap_load_mbuf(txr->txtag->dt_dmat, map, 1290 m_head, BUS_DMA_NOWAIT); 1291 1292 switch (error) { 1293 case EAGAIN: 1294 adapter->eagain_tx_dma_setup.ev_count++; 1295 return EAGAIN; 1296 case ENOMEM: 1297 adapter->enomem_tx_dma_setup.ev_count++; 1298 return EAGAIN; 1299 case EFBIG: 1300 adapter->efbig_tx_dma_setup.ev_count++; 1301 return error; 1302 case EINVAL: 1303 adapter->einval_tx_dma_setup.ev_count++; 1304 return error; 1305 default: 1306 adapter->other_tx_dma_setup.ev_count++; 1307 return error; 1308 case 0: 1309 break; 1310 } 1311 1312 /* Make certain there are enough descriptors */ 1313 if (map->dm_nsegs > txr->tx_avail - 2) { 1314 txr->no_desc_avail.ev_count++; 1315 /* XXX s/ixgbe/ixv/ */ 1316 ixgbe_dmamap_unload(txr->txtag, txbuf->map); 1317 return EAGAIN; 1318 } 1319 1320 /* 1321 ** Set up the appropriate offload context 1322 ** this becomes the first descriptor of 1323 ** a packet. 1324 */ 1325 if (m_head->m_pkthdr.csum_flags & (M_CSUM_TSOv4|M_CSUM_TSOv6)) { 1326 if (ixv_tso_setup(txr, m_head, &paylen)) { 1327 cmd_type_len |= IXGBE_ADVTXD_DCMD_TSE; 1328 olinfo_status |= IXGBE_TXD_POPTS_IXSM << 8; 1329 olinfo_status |= IXGBE_TXD_POPTS_TXSM << 8; 1330 olinfo_status |= paylen << IXGBE_ADVTXD_PAYLEN_SHIFT; 1331 ++adapter->tso_tx.ev_count; 1332 } else { 1333 ++adapter->tso_err.ev_count; 1334 /* XXX unload DMA map! --dyoung -> easy? --msaitoh */ 1335 return (ENXIO); 1336 } 1337 } else 1338 olinfo_status |= ixv_tx_ctx_setup(txr, m_head); 1339 1340 /* Record payload length */ 1341 if (paylen == 0) 1342 olinfo_status |= m_head->m_pkthdr.len << 1343 IXGBE_ADVTXD_PAYLEN_SHIFT; 1344 1345 i = txr->next_avail_desc; 1346 for (j = 0; j < map->dm_nsegs; j++) { 1347 bus_size_t seglen; 1348 bus_addr_t segaddr; 1349 1350 txbuf = &txr->tx_buffers[i]; 1351 txd = &txr->tx_base[i]; 1352 seglen = map->dm_segs[j].ds_len; 1353 segaddr = htole64(map->dm_segs[j].ds_addr); 1354 1355 txd->read.buffer_addr = segaddr; 1356 txd->read.cmd_type_len = htole32(txr->txd_cmd | 1357 cmd_type_len |seglen); 1358 txd->read.olinfo_status = htole32(olinfo_status); 1359 last = i; /* descriptor that will get completion IRQ */ 1360 1361 if (++i == adapter->num_tx_desc) 1362 i = 0; 1363 1364 txbuf->m_head = NULL; 1365 txbuf->eop_index = -1; 1366 } 1367 1368 txd->read.cmd_type_len |= 1369 htole32(IXGBE_TXD_CMD_EOP | IXGBE_TXD_CMD_RS); 1370 txr->tx_avail -= map->dm_nsegs; 1371 txr->next_avail_desc = i; 1372 1373 txbuf->m_head = m_head; 1374 /* Swap the dma map between the first and last descriptor */ 1375 txr->tx_buffers[first].map = txbuf->map; 1376 txbuf->map = map; 1377 bus_dmamap_sync(txr->txtag->dt_dmat, map, 0, m_head->m_pkthdr.len, 1378 BUS_DMASYNC_PREWRITE); 1379 1380 /* Set the index of the descriptor that will be marked done */ 1381 txbuf = &txr->tx_buffers[first]; 1382 txbuf->eop_index = last; 1383 1384 /* XXX s/ixgbe/ixg/ */ 1385 ixgbe_dmamap_sync(txr->txdma.dma_tag, txr->txdma.dma_map, 1386 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); 1387 /* 1388 * Advance the Transmit Descriptor Tail (Tdt), this tells the 1389 * hardware that this frame is available to transmit. 1390 */ 1391 ++txr->total_packets.ev_count; 1392 IXGBE_WRITE_REG(&adapter->hw, IXGBE_VFTDT(txr->me), i); 1393 1394 return 0; 1395 } 1396 1397 1398 /********************************************************************* 1399 * Multicast Update 1400 * 1401 * This routine is called whenever multicast address list is updated. 1402 * 1403 **********************************************************************/ 1404 #define IXGBE_RAR_ENTRIES 16 1405 1406 static void 1407 ixv_set_multi(struct adapter *adapter) 1408 { 1409 struct ether_multi *enm; 1410 struct ether_multistep step; 1411 u8 mta[MAX_NUM_MULTICAST_ADDRESSES * IXGBE_ETH_LENGTH_OF_ADDRESS]; 1412 u8 *update_ptr; 1413 int mcnt = 0; 1414 struct ethercom *ec = &adapter->osdep.ec; 1415 1416 IOCTL_DEBUGOUT("ixv_set_multi: begin"); 1417 1418 ETHER_FIRST_MULTI(step, ec, enm); 1419 while (enm != NULL) { 1420 bcopy(enm->enm_addrlo, 1421 &mta[mcnt * IXGBE_ETH_LENGTH_OF_ADDRESS], 1422 IXGBE_ETH_LENGTH_OF_ADDRESS); 1423 mcnt++; 1424 /* XXX This might be required --msaitoh */ 1425 if (mcnt >= MAX_NUM_MULTICAST_ADDRESSES) 1426 break; 1427 ETHER_NEXT_MULTI(step, enm); 1428 } 1429 1430 update_ptr = mta; 1431 1432 ixgbe_update_mc_addr_list(&adapter->hw, 1433 update_ptr, mcnt, ixv_mc_array_itr, TRUE); 1434 1435 return; 1436 } 1437 1438 /* 1439 * This is an iterator function now needed by the multicast 1440 * shared code. It simply feeds the shared code routine the 1441 * addresses in the array of ixv_set_multi() one by one. 1442 */ 1443 static u8 * 1444 ixv_mc_array_itr(struct ixgbe_hw *hw, u8 **update_ptr, u32 *vmdq) 1445 { 1446 u8 *addr = *update_ptr; 1447 u8 *newptr; 1448 *vmdq = 0; 1449 1450 newptr = addr + IXGBE_ETH_LENGTH_OF_ADDRESS; 1451 *update_ptr = newptr; 1452 return addr; 1453 } 1454 1455 /********************************************************************* 1456 * Timer routine 1457 * 1458 * This routine checks for link status,updates statistics, 1459 * and runs the watchdog check. 1460 * 1461 **********************************************************************/ 1462 1463 static void 1464 ixv_local_timer1(void *arg) 1465 { 1466 struct adapter *adapter = arg; 1467 device_t dev = adapter->dev; 1468 struct tx_ring *txr = adapter->tx_rings; 1469 int i; 1470 struct timeval now, elapsed; 1471 1472 KASSERT(mutex_owned(&adapter->core_mtx)); 1473 1474 ixv_update_link_status(adapter); 1475 1476 /* Stats Update */ 1477 ixv_update_stats(adapter); 1478 1479 /* 1480 * If the interface has been paused 1481 * then don't do the watchdog check 1482 */ 1483 if (IXGBE_READ_REG(&adapter->hw, IXGBE_TFCS) & IXGBE_TFCS_TXOFF) 1484 goto out; 1485 /* 1486 ** Check for time since any descriptor was cleaned 1487 */ 1488 for (i = 0; i < adapter->num_queues; i++, txr++) { 1489 IXV_TX_LOCK(txr); 1490 if (txr->watchdog_check == FALSE) { 1491 IXV_TX_UNLOCK(txr); 1492 continue; 1493 } 1494 getmicrotime(&now); 1495 timersub(&now, &txr->watchdog_time, &elapsed); 1496 if (tvtohz(&elapsed) > IXV_WATCHDOG) 1497 goto hung; 1498 IXV_TX_UNLOCK(txr); 1499 } 1500 out: 1501 ixv_rearm_queues(adapter, adapter->que_mask); 1502 callout_reset(&adapter->timer, hz, ixv_local_timer, adapter); 1503 return; 1504 1505 hung: 1506 device_printf(adapter->dev, "Watchdog timeout -- resetting\n"); 1507 device_printf(dev,"Queue(%d) tdh = %d, hw tdt = %d\n", txr->me, 1508 IXGBE_READ_REG(&adapter->hw, IXGBE_VFTDH(i)), 1509 IXGBE_READ_REG(&adapter->hw, IXGBE_VFTDT(i))); 1510 device_printf(dev,"TX(%d) desc avail = %d," 1511 "Next TX to Clean = %d\n", 1512 txr->me, txr->tx_avail, txr->next_to_clean); 1513 adapter->ifp->if_flags &= ~IFF_RUNNING; 1514 adapter->watchdog_events.ev_count++; 1515 IXV_TX_UNLOCK(txr); 1516 ixv_init_locked(adapter); 1517 } 1518 1519 static void 1520 ixv_local_timer(void *arg) 1521 { 1522 struct adapter *adapter = arg; 1523 1524 IXV_CORE_LOCK(adapter); 1525 ixv_local_timer1(adapter); 1526 IXV_CORE_UNLOCK(adapter); 1527 } 1528 1529 /* 1530 ** Note: this routine updates the OS on the link state 1531 ** the real check of the hardware only happens with 1532 ** a link interrupt. 1533 */ 1534 static void 1535 ixv_update_link_status(struct adapter *adapter) 1536 { 1537 struct ifnet *ifp = adapter->ifp; 1538 struct tx_ring *txr = adapter->tx_rings; 1539 device_t dev = adapter->dev; 1540 1541 1542 if (adapter->link_up){ 1543 if (adapter->link_active == FALSE) { 1544 if (bootverbose) 1545 device_printf(dev,"Link is up %d Gbps %s \n", 1546 ((adapter->link_speed == 128)? 10:1), 1547 "Full Duplex"); 1548 adapter->link_active = TRUE; 1549 if_link_state_change(ifp, LINK_STATE_UP); 1550 } 1551 } else { /* Link down */ 1552 if (adapter->link_active == TRUE) { 1553 if (bootverbose) 1554 device_printf(dev,"Link is Down\n"); 1555 if_link_state_change(ifp, LINK_STATE_DOWN); 1556 adapter->link_active = FALSE; 1557 for (int i = 0; i < adapter->num_queues; 1558 i++, txr++) 1559 txr->watchdog_check = FALSE; 1560 } 1561 } 1562 1563 return; 1564 } 1565 1566 1567 static void 1568 ixv_ifstop(struct ifnet *ifp, int disable) 1569 { 1570 struct adapter *adapter = ifp->if_softc; 1571 1572 IXV_CORE_LOCK(adapter); 1573 ixv_stop(adapter); 1574 IXV_CORE_UNLOCK(adapter); 1575 } 1576 1577 /********************************************************************* 1578 * 1579 * This routine disables all traffic on the adapter by issuing a 1580 * global reset on the MAC and deallocates TX/RX buffers. 1581 * 1582 **********************************************************************/ 1583 1584 static void 1585 ixv_stop(void *arg) 1586 { 1587 struct ifnet *ifp; 1588 struct adapter *adapter = arg; 1589 struct ixgbe_hw *hw = &adapter->hw; 1590 ifp = adapter->ifp; 1591 1592 KASSERT(mutex_owned(&adapter->core_mtx)); 1593 1594 INIT_DEBUGOUT("ixv_stop: begin\n"); 1595 ixv_disable_intr(adapter); 1596 1597 /* Tell the stack that the interface is no longer active */ 1598 ifp->if_flags &= ~(IFF_RUNNING | IFF_OACTIVE); 1599 1600 ixgbe_reset_hw(hw); 1601 adapter->hw.adapter_stopped = FALSE; 1602 ixgbe_stop_adapter(hw); 1603 callout_stop(&adapter->timer); 1604 1605 /* reprogram the RAR[0] in case user changed it. */ 1606 ixgbe_set_rar(hw, 0, hw->mac.addr, 0, IXGBE_RAH_AV); 1607 1608 return; 1609 } 1610 1611 1612 /********************************************************************* 1613 * 1614 * Determine hardware revision. 1615 * 1616 **********************************************************************/ 1617 static void 1618 ixv_identify_hardware(struct adapter *adapter) 1619 { 1620 u16 pci_cmd_word; 1621 pcitag_t tag; 1622 pci_chipset_tag_t pc; 1623 pcireg_t subid, id; 1624 struct ixgbe_hw *hw = &adapter->hw; 1625 1626 pc = adapter->osdep.pc; 1627 tag = adapter->osdep.tag; 1628 1629 /* 1630 ** Make sure BUSMASTER is set, on a VM under 1631 ** KVM it may not be and will break things. 1632 */ 1633 pci_cmd_word = pci_conf_read(pc, tag, PCI_COMMAND_STATUS_REG); 1634 if (!(pci_cmd_word & PCI_COMMAND_MASTER_ENABLE)) { 1635 INIT_DEBUGOUT("Bus Master bit was not set!\n"); 1636 pci_cmd_word |= PCI_COMMAND_MASTER_ENABLE; 1637 pci_conf_write(pc, tag, PCI_COMMAND_STATUS_REG, pci_cmd_word); 1638 } 1639 1640 id = pci_conf_read(pc, tag, PCI_ID_REG); 1641 subid = pci_conf_read(pc, tag, PCI_SUBSYS_ID_REG); 1642 1643 /* Save off the information about this board */ 1644 hw->vendor_id = PCI_VENDOR(id); 1645 hw->device_id = PCI_PRODUCT(id); 1646 hw->revision_id = PCI_REVISION(pci_conf_read(pc, tag, PCI_CLASS_REG)); 1647 hw->subsystem_vendor_id = PCI_SUBSYS_VENDOR(subid); 1648 hw->subsystem_device_id = PCI_SUBSYS_ID(subid); 1649 1650 return; 1651 } 1652 1653 /********************************************************************* 1654 * 1655 * Setup MSIX Interrupt resources and handlers 1656 * 1657 **********************************************************************/ 1658 static int 1659 ixv_allocate_msix(struct adapter *adapter, const struct pci_attach_args *pa) 1660 { 1661 #if !defined(NETBSD_MSI_OR_MSIX) 1662 return 0; 1663 #else 1664 device_t dev = adapter->dev; 1665 struct ix_queue *que = adapter->queues; 1666 int error, rid, vector = 0; 1667 pci_chipset_tag_t pc; 1668 pcitag_t tag; 1669 char intrbuf[PCI_INTRSTR_LEN]; 1670 const char *intrstr = NULL; 1671 kcpuset_t *affinity; 1672 int cpu_id = 0; 1673 1674 pc = adapter->osdep.pc; 1675 tag = adapter->osdep.tag; 1676 1677 if (pci_msix_alloc_exact(pa, 1678 &adapter->osdep.intrs, IXG_MSIX_NINTR) != 0) 1679 return (ENXIO); 1680 1681 kcpuset_create(&affinity, false); 1682 for (int i = 0; i < adapter->num_queues; i++, vector++, que++) { 1683 intrstr = pci_intr_string(pc, adapter->osdep.intrs[i], intrbuf, 1684 sizeof(intrbuf)); 1685 #ifdef IXV_MPSAFE 1686 pci_intr_setattr(pc, adapter->osdep.intrs[i], PCI_INTR_MPSAFE, 1687 true); 1688 #endif 1689 /* Set the handler function */ 1690 adapter->osdep.ihs[i] = pci_intr_establish(pc, 1691 adapter->osdep.intrs[i], IPL_NET, ixv_msix_que, que); 1692 if (adapter->osdep.ihs[i] == NULL) { 1693 que->res = NULL; 1694 aprint_error_dev(dev, 1695 "Failed to register QUE handler"); 1696 kcpuset_destroy(affinity); 1697 return (ENXIO); 1698 } 1699 que->msix = vector; 1700 adapter->que_mask |= (u64)(1 << que->msix); 1701 1702 cpu_id = i; 1703 /* Round-robin affinity */ 1704 kcpuset_zero(affinity); 1705 kcpuset_set(affinity, cpu_id % ncpu); 1706 error = pci_intr_distribute(adapter->osdep.ihs[i], affinity, 1707 NULL); 1708 aprint_normal_dev(dev, "for TX/RX, interrupting at %s", 1709 intrstr); 1710 if (error == 0) 1711 aprint_normal(", bound queue %d to cpu %d\n", 1712 i, cpu_id); 1713 else 1714 aprint_normal("\n"); 1715 1716 que->que_si = softint_establish(SOFTINT_NET, ixv_handle_que, 1717 que); 1718 if (que->que_si == NULL) { 1719 aprint_error_dev(dev, 1720 "could not establish software interrupt\n"); 1721 } 1722 } 1723 1724 /* and Mailbox */ 1725 cpu_id++; 1726 intrstr = pci_intr_string(pc, adapter->osdep.intrs[vector], intrbuf, 1727 sizeof(intrbuf)); 1728 #ifdef IXG_MPSAFE 1729 pci_intr_setattr(pc, &adapter->osdep.intrs[vector], PCI_INTR_MPSAFE, true); 1730 #endif 1731 /* Set the mbx handler function */ 1732 adapter->osdep.ihs[vector] = pci_intr_establish(pc, 1733 adapter->osdep.intrs[vector], IPL_NET, ixv_msix_mbx, adapter); 1734 if (adapter->osdep.ihs[vector] == NULL) { 1735 adapter->res = NULL; 1736 aprint_error_dev(dev, "Failed to register LINK handler\n"); 1737 kcpuset_destroy(affinity); 1738 return (ENXIO); 1739 } 1740 /* Round-robin affinity */ 1741 kcpuset_zero(affinity); 1742 kcpuset_set(affinity, cpu_id % ncpu); 1743 error = pci_intr_distribute(adapter->osdep.ihs[vector], affinity,NULL); 1744 1745 aprint_normal_dev(dev, 1746 "for link, interrupting at %s, ", intrstr); 1747 if (error == 0) { 1748 aprint_normal("affinity to cpu %d\n", cpu_id); 1749 } 1750 adapter->mbxvec = vector; 1751 /* Tasklets for Mailbox */ 1752 adapter->mbx_si = softint_establish(SOFTINT_NET, ixv_handle_mbx, 1753 adapter); 1754 /* 1755 ** Due to a broken design QEMU will fail to properly 1756 ** enable the guest for MSIX unless the vectors in 1757 ** the table are all set up, so we must rewrite the 1758 ** ENABLE in the MSIX control register again at this 1759 ** point to cause it to successfully initialize us. 1760 */ 1761 if (adapter->hw.mac.type == ixgbe_mac_82599_vf) { 1762 int msix_ctrl; 1763 pci_get_capability(pc, tag, PCI_CAP_MSIX, &rid, NULL); 1764 rid += PCI_MSIX_CTL; 1765 msix_ctrl = pci_conf_read(pc, tag, rid); 1766 msix_ctrl |= PCI_MSIX_CTL_ENABLE; 1767 pci_conf_write(pc, tag, rid, msix_ctrl); 1768 } 1769 1770 return (0); 1771 #endif 1772 } 1773 1774 /* 1775 * Setup MSIX resources, note that the VF 1776 * device MUST use MSIX, there is no fallback. 1777 */ 1778 static int 1779 ixv_setup_msix(struct adapter *adapter) 1780 { 1781 #if !defined(NETBSD_MSI_OR_MSIX) 1782 return 0; 1783 #else 1784 device_t dev = adapter->dev; 1785 int want, msgs; 1786 1787 1788 /* 1789 ** Want two vectors: one for a queue, 1790 ** plus an additional for mailbox. 1791 */ 1792 msgs = pci_msix_count(adapter->osdep.pc, adapter->osdep.tag); 1793 if (msgs < IXG_MSIX_NINTR) { 1794 aprint_error_dev(dev,"MSIX config error\n"); 1795 return (ENXIO); 1796 } 1797 1798 adapter->msix_mem = (void *)1; /* XXX */ 1799 aprint_normal_dev(dev, 1800 "Using MSIX interrupts with %d vectors\n", msgs); 1801 return (want); 1802 #endif 1803 } 1804 1805 1806 static int 1807 ixv_allocate_pci_resources(struct adapter *adapter, 1808 const struct pci_attach_args *pa) 1809 { 1810 pcireg_t memtype; 1811 device_t dev = adapter->dev; 1812 bus_addr_t addr; 1813 int flags; 1814 1815 memtype = pci_mapreg_type(pa->pa_pc, pa->pa_tag, PCI_BAR(0)); 1816 1817 switch (memtype) { 1818 case PCI_MAPREG_TYPE_MEM | PCI_MAPREG_MEM_TYPE_32BIT: 1819 case PCI_MAPREG_TYPE_MEM | PCI_MAPREG_MEM_TYPE_64BIT: 1820 adapter->osdep.mem_bus_space_tag = pa->pa_memt; 1821 if (pci_mapreg_info(pa->pa_pc, pa->pa_tag, PCI_BAR(0), 1822 memtype, &addr, &adapter->osdep.mem_size, &flags) != 0) 1823 goto map_err; 1824 if ((flags & BUS_SPACE_MAP_PREFETCHABLE) != 0) { 1825 aprint_normal_dev(dev, "clearing prefetchable bit\n"); 1826 flags &= ~BUS_SPACE_MAP_PREFETCHABLE; 1827 } 1828 if (bus_space_map(adapter->osdep.mem_bus_space_tag, addr, 1829 adapter->osdep.mem_size, flags, 1830 &adapter->osdep.mem_bus_space_handle) != 0) { 1831 map_err: 1832 adapter->osdep.mem_size = 0; 1833 aprint_error_dev(dev, "unable to map BAR0\n"); 1834 return ENXIO; 1835 } 1836 break; 1837 default: 1838 aprint_error_dev(dev, "unexpected type on BAR0\n"); 1839 return ENXIO; 1840 } 1841 1842 adapter->num_queues = 1; 1843 adapter->hw.back = &adapter->osdep; 1844 1845 /* 1846 ** Now setup MSI/X, should 1847 ** return us the number of 1848 ** configured vectors. 1849 */ 1850 adapter->msix = ixv_setup_msix(adapter); 1851 if (adapter->msix == ENXIO) 1852 return (ENXIO); 1853 else 1854 return (0); 1855 } 1856 1857 static void 1858 ixv_free_pci_resources(struct adapter * adapter) 1859 { 1860 #if !defined(NETBSD_MSI_OR_MSIX) 1861 #else 1862 struct ix_queue *que = adapter->queues; 1863 int rid; 1864 1865 /* 1866 ** Release all msix queue resources: 1867 */ 1868 for (int i = 0; i < adapter->num_queues; i++, que++) { 1869 rid = que->msix + 1; 1870 if (que->res != NULL) 1871 pci_intr_disestablish(adapter->osdep.pc, 1872 adapter->osdep.ihs[i]); 1873 } 1874 1875 /* Clean the Legacy or Link interrupt last */ 1876 if (adapter->mbxvec) /* we are doing MSIX */ 1877 rid = adapter->mbxvec + 1; 1878 else 1879 (adapter->msix != 0) ? (rid = 1):(rid = 0); 1880 1881 if (adapter->osdep.ihs[rid] != NULL) 1882 pci_intr_disestablish(adapter->osdep.pc, 1883 adapter->osdep.ihs[rid]); 1884 adapter->osdep.ihs[rid] = NULL; 1885 1886 #if defined(NETBSD_MSI_OR_MSIX) 1887 pci_intr_release(adapter->osdep.pc, adapter->osdep.intrs, 1888 adapter->osdep.nintrs); 1889 #endif 1890 1891 if (adapter->osdep.mem_size != 0) { 1892 bus_space_unmap(adapter->osdep.mem_bus_space_tag, 1893 adapter->osdep.mem_bus_space_handle, 1894 adapter->osdep.mem_size); 1895 } 1896 1897 #endif 1898 return; 1899 } 1900 1901 /********************************************************************* 1902 * 1903 * Setup networking device structure and register an interface. 1904 * 1905 **********************************************************************/ 1906 static void 1907 ixv_setup_interface(device_t dev, struct adapter *adapter) 1908 { 1909 struct ethercom *ec = &adapter->osdep.ec; 1910 struct ifnet *ifp; 1911 1912 INIT_DEBUGOUT("ixv_setup_interface: begin"); 1913 1914 ifp = adapter->ifp = &ec->ec_if; 1915 strlcpy(ifp->if_xname, device_xname(dev), IFNAMSIZ); 1916 ifp->if_baudrate = 1000000000; 1917 ifp->if_init = ixv_init; 1918 ifp->if_stop = ixv_ifstop; 1919 ifp->if_softc = adapter; 1920 ifp->if_flags = IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST; 1921 ifp->if_ioctl = ixv_ioctl; 1922 #if __FreeBSD_version >= 800000 1923 ifp->if_transmit = ixv_mq_start; 1924 ifp->if_qflush = ixv_qflush; 1925 #else 1926 ifp->if_start = ixv_start; 1927 #endif 1928 ifp->if_snd.ifq_maxlen = adapter->num_tx_desc - 2; 1929 1930 if_attach(ifp); 1931 ether_ifattach(ifp, adapter->hw.mac.addr); 1932 ether_set_ifflags_cb(ec, ixv_ifflags_cb); 1933 1934 adapter->max_frame_size = 1935 ifp->if_mtu + ETHER_HDR_LEN + ETHER_CRC_LEN; 1936 1937 /* 1938 * Tell the upper layer(s) we support long frames. 1939 */ 1940 ifp->if_hdrlen = sizeof(struct ether_vlan_header); 1941 1942 ifp->if_capabilities |= IFCAP_HWCSUM | IFCAP_TSOv4; 1943 ifp->if_capenable = 0; 1944 1945 ec->ec_capabilities |= ETHERCAP_VLAN_HWCSUM; 1946 ec->ec_capabilities |= ETHERCAP_JUMBO_MTU; 1947 ec->ec_capabilities |= ETHERCAP_VLAN_HWTAGGING 1948 | ETHERCAP_VLAN_MTU; 1949 ec->ec_capenable = ec->ec_capabilities; 1950 1951 /* Don't enable LRO by default */ 1952 ifp->if_capabilities |= IFCAP_LRO; 1953 1954 /* 1955 ** Dont turn this on by default, if vlans are 1956 ** created on another pseudo device (eg. lagg) 1957 ** then vlan events are not passed thru, breaking 1958 ** operation, but with HW FILTER off it works. If 1959 ** using vlans directly on the em driver you can 1960 ** enable this and get full hardware tag filtering. 1961 */ 1962 ec->ec_capabilities |= ETHERCAP_VLAN_HWFILTER; 1963 1964 /* 1965 * Specify the media types supported by this adapter and register 1966 * callbacks to update media and link information 1967 */ 1968 ifmedia_init(&adapter->media, IFM_IMASK, ixv_media_change, 1969 ixv_media_status); 1970 ifmedia_add(&adapter->media, IFM_ETHER | IFM_FDX, 0, NULL); 1971 ifmedia_add(&adapter->media, IFM_ETHER | IFM_AUTO, 0, NULL); 1972 ifmedia_set(&adapter->media, IFM_ETHER | IFM_AUTO); 1973 1974 return; 1975 } 1976 1977 static void 1978 ixv_config_link(struct adapter *adapter) 1979 { 1980 struct ixgbe_hw *hw = &adapter->hw; 1981 u32 autoneg, err = 0; 1982 1983 if (hw->mac.ops.check_link) 1984 err = hw->mac.ops.check_link(hw, &autoneg, 1985 &adapter->link_up, FALSE); 1986 if (err) 1987 goto out; 1988 1989 if (hw->mac.ops.setup_link) 1990 err = hw->mac.ops.setup_link(hw, 1991 autoneg, adapter->link_up); 1992 out: 1993 return; 1994 } 1995 1996 /******************************************************************** 1997 * Manage DMA'able memory. 1998 *******************************************************************/ 1999 2000 static int 2001 ixv_dma_malloc(struct adapter *adapter, bus_size_t size, 2002 struct ixv_dma_alloc *dma, int mapflags) 2003 { 2004 device_t dev = adapter->dev; 2005 int r, rsegs; 2006 2007 r = ixgbe_dma_tag_create(adapter->osdep.dmat, /* parent */ 2008 DBA_ALIGN, 0, /* alignment, bounds */ 2009 size, /* maxsize */ 2010 1, /* nsegments */ 2011 size, /* maxsegsize */ 2012 BUS_DMA_ALLOCNOW, /* flags */ 2013 &dma->dma_tag); 2014 if (r != 0) { 2015 aprint_error_dev(dev, 2016 "ixv_dma_malloc: bus_dma_tag_create failed; error %u\n", r); 2017 goto fail_0; 2018 } 2019 r = bus_dmamem_alloc(dma->dma_tag->dt_dmat, 2020 size, 2021 dma->dma_tag->dt_alignment, 2022 dma->dma_tag->dt_boundary, 2023 &dma->dma_seg, 1, &rsegs, BUS_DMA_NOWAIT); 2024 if (r != 0) { 2025 aprint_error_dev(dev, 2026 "%s: bus_dmamem_alloc failed; error %u\n", __func__, r); 2027 goto fail_1; 2028 } 2029 2030 r = bus_dmamem_map(dma->dma_tag->dt_dmat, &dma->dma_seg, rsegs, 2031 size, &dma->dma_vaddr, BUS_DMA_NOWAIT); 2032 if (r != 0) { 2033 aprint_error_dev(dev, "%s: bus_dmamem_map failed; error %d\n", 2034 __func__, r); 2035 goto fail_2; 2036 } 2037 2038 r = ixgbe_dmamap_create(dma->dma_tag, 0, &dma->dma_map); 2039 if (r != 0) { 2040 aprint_error_dev(dev, "%s: bus_dmamem_map failed; error %d\n", 2041 __func__, r); 2042 goto fail_3; 2043 } 2044 2045 r = bus_dmamap_load(dma->dma_tag->dt_dmat, dma->dma_map, dma->dma_vaddr, 2046 size, 2047 NULL, 2048 mapflags | BUS_DMA_NOWAIT); 2049 if (r != 0) { 2050 aprint_error_dev(dev,"%s: bus_dmamap_load failed; error %u\n", 2051 __func__, r); 2052 goto fail_4; 2053 } 2054 dma->dma_paddr = dma->dma_map->dm_segs[0].ds_addr; 2055 dma->dma_size = size; 2056 return 0; 2057 fail_4: 2058 ixgbe_dmamap_destroy(dma->dma_tag, dma->dma_map); 2059 fail_3: 2060 bus_dmamem_unmap(dma->dma_tag->dt_dmat, dma->dma_vaddr, size); 2061 fail_2: 2062 bus_dmamem_free(dma->dma_tag->dt_dmat, &dma->dma_seg, rsegs); 2063 fail_1: 2064 ixgbe_dma_tag_destroy(dma->dma_tag); 2065 fail_0: 2066 dma->dma_tag = NULL; 2067 return (r); 2068 } 2069 2070 static void 2071 ixv_dma_free(struct adapter *adapter, struct ixv_dma_alloc *dma) 2072 { 2073 bus_dmamap_sync(dma->dma_tag->dt_dmat, dma->dma_map, 0, dma->dma_size, 2074 BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE); 2075 ixgbe_dmamap_unload(dma->dma_tag, dma->dma_map); 2076 bus_dmamem_free(dma->dma_tag->dt_dmat, &dma->dma_seg, 1); 2077 ixgbe_dma_tag_destroy(dma->dma_tag); 2078 } 2079 2080 2081 /********************************************************************* 2082 * 2083 * Allocate memory for the transmit and receive rings, and then 2084 * the descriptors associated with each, called only once at attach. 2085 * 2086 **********************************************************************/ 2087 static int 2088 ixv_allocate_queues(struct adapter *adapter) 2089 { 2090 device_t dev = adapter->dev; 2091 struct ix_queue *que; 2092 struct tx_ring *txr; 2093 struct rx_ring *rxr; 2094 int rsize, tsize, error = 0; 2095 int txconf = 0, rxconf = 0; 2096 2097 /* First allocate the top level queue structs */ 2098 if (!(adapter->queues = 2099 (struct ix_queue *) malloc(sizeof(struct ix_queue) * 2100 adapter->num_queues, M_DEVBUF, M_NOWAIT | M_ZERO))) { 2101 aprint_error_dev(dev, "Unable to allocate queue memory\n"); 2102 error = ENOMEM; 2103 goto fail; 2104 } 2105 2106 /* First allocate the TX ring struct memory */ 2107 if (!(adapter->tx_rings = 2108 (struct tx_ring *) malloc(sizeof(struct tx_ring) * 2109 adapter->num_queues, M_DEVBUF, M_NOWAIT | M_ZERO))) { 2110 aprint_error_dev(dev, "Unable to allocate TX ring memory\n"); 2111 error = ENOMEM; 2112 goto tx_fail; 2113 } 2114 2115 /* Next allocate the RX */ 2116 if (!(adapter->rx_rings = 2117 (struct rx_ring *) malloc(sizeof(struct rx_ring) * 2118 adapter->num_queues, M_DEVBUF, M_NOWAIT | M_ZERO))) { 2119 aprint_error_dev(dev, "Unable to allocate RX ring memory\n"); 2120 error = ENOMEM; 2121 goto rx_fail; 2122 } 2123 2124 /* For the ring itself */ 2125 tsize = roundup2(adapter->num_tx_desc * 2126 sizeof(union ixgbe_adv_tx_desc), DBA_ALIGN); 2127 2128 /* 2129 * Now set up the TX queues, txconf is needed to handle the 2130 * possibility that things fail midcourse and we need to 2131 * undo memory gracefully 2132 */ 2133 for (int i = 0; i < adapter->num_queues; i++, txconf++) { 2134 /* Set up some basics */ 2135 txr = &adapter->tx_rings[i]; 2136 txr->adapter = adapter; 2137 txr->me = i; 2138 2139 /* Initialize the TX side lock */ 2140 snprintf(txr->mtx_name, sizeof(txr->mtx_name), "%s:tx(%d)", 2141 device_xname(dev), txr->me); 2142 mutex_init(&txr->tx_mtx, MUTEX_DEFAULT, IPL_NET); 2143 2144 if (ixv_dma_malloc(adapter, tsize, 2145 &txr->txdma, BUS_DMA_NOWAIT)) { 2146 aprint_error_dev(dev, 2147 "Unable to allocate TX Descriptor memory\n"); 2148 error = ENOMEM; 2149 goto err_tx_desc; 2150 } 2151 txr->tx_base = (union ixgbe_adv_tx_desc *)txr->txdma.dma_vaddr; 2152 bzero((void *)txr->tx_base, tsize); 2153 2154 /* Now allocate transmit buffers for the ring */ 2155 if (ixv_allocate_transmit_buffers(txr)) { 2156 aprint_error_dev(dev, 2157 "Critical Failure setting up transmit buffers\n"); 2158 error = ENOMEM; 2159 goto err_tx_desc; 2160 } 2161 #if __FreeBSD_version >= 800000 2162 /* Allocate a buf ring */ 2163 txr->br = buf_ring_alloc(IXV_BR_SIZE, M_DEVBUF, 2164 M_WAITOK, &txr->tx_mtx); 2165 if (txr->br == NULL) { 2166 aprint_error_dev(dev, 2167 "Critical Failure setting up buf ring\n"); 2168 error = ENOMEM; 2169 goto err_tx_desc; 2170 } 2171 #endif 2172 } 2173 2174 /* 2175 * Next the RX queues... 2176 */ 2177 rsize = roundup2(adapter->num_rx_desc * 2178 sizeof(union ixgbe_adv_rx_desc), DBA_ALIGN); 2179 for (int i = 0; i < adapter->num_queues; i++, rxconf++) { 2180 rxr = &adapter->rx_rings[i]; 2181 /* Set up some basics */ 2182 rxr->adapter = adapter; 2183 rxr->me = i; 2184 2185 /* Initialize the RX side lock */ 2186 snprintf(rxr->mtx_name, sizeof(rxr->mtx_name), "%s:rx(%d)", 2187 device_xname(dev), rxr->me); 2188 mutex_init(&rxr->rx_mtx, MUTEX_DEFAULT, IPL_NET); 2189 2190 if (ixv_dma_malloc(adapter, rsize, 2191 &rxr->rxdma, BUS_DMA_NOWAIT)) { 2192 aprint_error_dev(dev, 2193 "Unable to allocate RxDescriptor memory\n"); 2194 error = ENOMEM; 2195 goto err_rx_desc; 2196 } 2197 rxr->rx_base = (union ixgbe_adv_rx_desc *)rxr->rxdma.dma_vaddr; 2198 bzero((void *)rxr->rx_base, rsize); 2199 2200 /* Allocate receive buffers for the ring*/ 2201 if (ixv_allocate_receive_buffers(rxr)) { 2202 aprint_error_dev(dev, 2203 "Critical Failure setting up receive buffers\n"); 2204 error = ENOMEM; 2205 goto err_rx_desc; 2206 } 2207 } 2208 2209 /* 2210 ** Finally set up the queue holding structs 2211 */ 2212 for (int i = 0; i < adapter->num_queues; i++) { 2213 que = &adapter->queues[i]; 2214 que->adapter = adapter; 2215 que->txr = &adapter->tx_rings[i]; 2216 que->rxr = &adapter->rx_rings[i]; 2217 } 2218 2219 return (0); 2220 2221 err_rx_desc: 2222 for (rxr = adapter->rx_rings; rxconf > 0; rxr++, rxconf--) 2223 ixv_dma_free(adapter, &rxr->rxdma); 2224 err_tx_desc: 2225 for (txr = adapter->tx_rings; txconf > 0; txr++, txconf--) 2226 ixv_dma_free(adapter, &txr->txdma); 2227 free(adapter->rx_rings, M_DEVBUF); 2228 rx_fail: 2229 free(adapter->tx_rings, M_DEVBUF); 2230 tx_fail: 2231 free(adapter->queues, M_DEVBUF); 2232 fail: 2233 return (error); 2234 } 2235 2236 2237 /********************************************************************* 2238 * 2239 * Allocate memory for tx_buffer structures. The tx_buffer stores all 2240 * the information needed to transmit a packet on the wire. This is 2241 * called only once at attach, setup is done every reset. 2242 * 2243 **********************************************************************/ 2244 static int 2245 ixv_allocate_transmit_buffers(struct tx_ring *txr) 2246 { 2247 struct adapter *adapter = txr->adapter; 2248 device_t dev = adapter->dev; 2249 struct ixv_tx_buf *txbuf; 2250 int error, i; 2251 2252 /* 2253 * Setup DMA descriptor areas. 2254 */ 2255 if ((error = ixgbe_dma_tag_create(adapter->osdep.dmat, /* parent */ 2256 1, 0, /* alignment, bounds */ 2257 IXV_TSO_SIZE, /* maxsize */ 2258 32, /* nsegments */ 2259 PAGE_SIZE, /* maxsegsize */ 2260 0, /* flags */ 2261 &txr->txtag))) { 2262 aprint_error_dev(dev,"Unable to allocate TX DMA tag\n"); 2263 goto fail; 2264 } 2265 2266 if (!(txr->tx_buffers = 2267 (struct ixv_tx_buf *) malloc(sizeof(struct ixv_tx_buf) * 2268 adapter->num_tx_desc, M_DEVBUF, M_NOWAIT | M_ZERO))) { 2269 aprint_error_dev(dev, "Unable to allocate tx_buffer memory\n"); 2270 error = ENOMEM; 2271 goto fail; 2272 } 2273 2274 /* Create the descriptor buffer dma maps */ 2275 txbuf = txr->tx_buffers; 2276 for (i = 0; i < adapter->num_tx_desc; i++, txbuf++) { 2277 error = ixgbe_dmamap_create(txr->txtag, 0, &txbuf->map); 2278 if (error != 0) { 2279 aprint_error_dev(dev, "Unable to create TX DMA map\n"); 2280 goto fail; 2281 } 2282 } 2283 2284 return 0; 2285 fail: 2286 /* We free all, it handles case where we are in the middle */ 2287 ixv_free_transmit_structures(adapter); 2288 return (error); 2289 } 2290 2291 /********************************************************************* 2292 * 2293 * Initialize a transmit ring. 2294 * 2295 **********************************************************************/ 2296 static void 2297 ixv_setup_transmit_ring(struct tx_ring *txr) 2298 { 2299 struct adapter *adapter = txr->adapter; 2300 struct ixv_tx_buf *txbuf; 2301 int i; 2302 2303 /* Clear the old ring contents */ 2304 IXV_TX_LOCK(txr); 2305 bzero((void *)txr->tx_base, 2306 (sizeof(union ixgbe_adv_tx_desc)) * adapter->num_tx_desc); 2307 /* Reset indices */ 2308 txr->next_avail_desc = 0; 2309 txr->next_to_clean = 0; 2310 2311 /* Free any existing tx buffers. */ 2312 txbuf = txr->tx_buffers; 2313 for (i = 0; i < adapter->num_tx_desc; i++, txbuf++) { 2314 if (txbuf->m_head != NULL) { 2315 bus_dmamap_sync(txr->txtag->dt_dmat, txbuf->map, 2316 0, txbuf->m_head->m_pkthdr.len, 2317 BUS_DMASYNC_POSTWRITE); 2318 ixgbe_dmamap_unload(txr->txtag, txbuf->map); 2319 m_freem(txbuf->m_head); 2320 txbuf->m_head = NULL; 2321 } 2322 /* Clear the EOP index */ 2323 txbuf->eop_index = -1; 2324 } 2325 2326 /* Set number of descriptors available */ 2327 txr->tx_avail = adapter->num_tx_desc; 2328 2329 ixgbe_dmamap_sync(txr->txdma.dma_tag, txr->txdma.dma_map, 2330 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); 2331 IXV_TX_UNLOCK(txr); 2332 } 2333 2334 /********************************************************************* 2335 * 2336 * Initialize all transmit rings. 2337 * 2338 **********************************************************************/ 2339 static int 2340 ixv_setup_transmit_structures(struct adapter *adapter) 2341 { 2342 struct tx_ring *txr = adapter->tx_rings; 2343 2344 for (int i = 0; i < adapter->num_queues; i++, txr++) 2345 ixv_setup_transmit_ring(txr); 2346 2347 return (0); 2348 } 2349 2350 /********************************************************************* 2351 * 2352 * Enable transmit unit. 2353 * 2354 **********************************************************************/ 2355 static void 2356 ixv_initialize_transmit_units(struct adapter *adapter) 2357 { 2358 struct tx_ring *txr = adapter->tx_rings; 2359 struct ixgbe_hw *hw = &adapter->hw; 2360 2361 2362 for (int i = 0; i < adapter->num_queues; i++, txr++) { 2363 u64 tdba = txr->txdma.dma_paddr; 2364 u32 txctrl, txdctl; 2365 2366 /* Set WTHRESH to 8, burst writeback */ 2367 txdctl = IXGBE_READ_REG(hw, IXGBE_VFTXDCTL(i)); 2368 txdctl |= (8 << 16); 2369 IXGBE_WRITE_REG(hw, IXGBE_VFTXDCTL(i), txdctl); 2370 /* Now enable */ 2371 txdctl = IXGBE_READ_REG(hw, IXGBE_VFTXDCTL(i)); 2372 txdctl |= IXGBE_TXDCTL_ENABLE; 2373 IXGBE_WRITE_REG(hw, IXGBE_VFTXDCTL(i), txdctl); 2374 2375 /* Set the HW Tx Head and Tail indices */ 2376 IXGBE_WRITE_REG(&adapter->hw, IXGBE_VFTDH(i), 0); 2377 IXGBE_WRITE_REG(&adapter->hw, IXGBE_VFTDT(i), 0); 2378 2379 /* Setup Transmit Descriptor Cmd Settings */ 2380 txr->txd_cmd = IXGBE_TXD_CMD_IFCS; 2381 txr->watchdog_check = FALSE; 2382 2383 /* Set Ring parameters */ 2384 IXGBE_WRITE_REG(hw, IXGBE_VFTDBAL(i), 2385 (tdba & 0x00000000ffffffffULL)); 2386 IXGBE_WRITE_REG(hw, IXGBE_VFTDBAH(i), (tdba >> 32)); 2387 IXGBE_WRITE_REG(hw, IXGBE_VFTDLEN(i), 2388 adapter->num_tx_desc * 2389 sizeof(struct ixgbe_legacy_tx_desc)); 2390 txctrl = IXGBE_READ_REG(hw, IXGBE_VFDCA_TXCTRL(i)); 2391 txctrl &= ~IXGBE_DCA_TXCTRL_DESC_WRO_EN; 2392 IXGBE_WRITE_REG(hw, IXGBE_VFDCA_TXCTRL(i), txctrl); 2393 break; 2394 } 2395 2396 return; 2397 } 2398 2399 /********************************************************************* 2400 * 2401 * Free all transmit rings. 2402 * 2403 **********************************************************************/ 2404 static void 2405 ixv_free_transmit_structures(struct adapter *adapter) 2406 { 2407 struct tx_ring *txr = adapter->tx_rings; 2408 2409 for (int i = 0; i < adapter->num_queues; i++, txr++) { 2410 ixv_free_transmit_buffers(txr); 2411 ixv_dma_free(adapter, &txr->txdma); 2412 IXV_TX_LOCK_DESTROY(txr); 2413 } 2414 free(adapter->tx_rings, M_DEVBUF); 2415 } 2416 2417 /********************************************************************* 2418 * 2419 * Free transmit ring related data structures. 2420 * 2421 **********************************************************************/ 2422 static void 2423 ixv_free_transmit_buffers(struct tx_ring *txr) 2424 { 2425 struct adapter *adapter = txr->adapter; 2426 struct ixv_tx_buf *tx_buffer; 2427 int i; 2428 2429 INIT_DEBUGOUT("free_transmit_ring: begin"); 2430 2431 if (txr->tx_buffers == NULL) 2432 return; 2433 2434 tx_buffer = txr->tx_buffers; 2435 for (i = 0; i < adapter->num_tx_desc; i++, tx_buffer++) { 2436 if (tx_buffer->m_head != NULL) { 2437 bus_dmamap_sync(txr->txtag->dt_dmat, tx_buffer->map, 2438 0, tx_buffer->m_head->m_pkthdr.len, 2439 BUS_DMASYNC_POSTWRITE); 2440 ixgbe_dmamap_unload(txr->txtag, tx_buffer->map); 2441 m_freem(tx_buffer->m_head); 2442 tx_buffer->m_head = NULL; 2443 if (tx_buffer->map != NULL) { 2444 ixgbe_dmamap_destroy(txr->txtag, 2445 tx_buffer->map); 2446 tx_buffer->map = NULL; 2447 } 2448 } else if (tx_buffer->map != NULL) { 2449 ixgbe_dmamap_unload(txr->txtag, tx_buffer->map); 2450 ixgbe_dmamap_destroy(txr->txtag, tx_buffer->map); 2451 tx_buffer->map = NULL; 2452 } 2453 } 2454 #if __FreeBSD_version >= 800000 2455 if (txr->br != NULL) 2456 buf_ring_free(txr->br, M_DEVBUF); 2457 #endif 2458 if (txr->tx_buffers != NULL) { 2459 free(txr->tx_buffers, M_DEVBUF); 2460 txr->tx_buffers = NULL; 2461 } 2462 if (txr->txtag != NULL) { 2463 ixgbe_dma_tag_destroy(txr->txtag); 2464 txr->txtag = NULL; 2465 } 2466 return; 2467 } 2468 2469 /********************************************************************* 2470 * 2471 * Advanced Context Descriptor setup for VLAN or L4 CSUM 2472 * 2473 **********************************************************************/ 2474 2475 static u32 2476 ixv_tx_ctx_setup(struct tx_ring *txr, struct mbuf *mp) 2477 { 2478 struct m_tag *mtag; 2479 struct adapter *adapter = txr->adapter; 2480 struct ethercom *ec = &adapter->osdep.ec; 2481 struct ixgbe_adv_tx_context_desc *TXD; 2482 struct ixv_tx_buf *tx_buffer; 2483 u32 olinfo = 0, vlan_macip_lens = 0, type_tucmd_mlhl = 0; 2484 struct ether_vlan_header *eh; 2485 struct ip ip; 2486 struct ip6_hdr ip6; 2487 int ehdrlen, ip_hlen = 0; 2488 u16 etype; 2489 u8 ipproto = 0; 2490 bool offload; 2491 int ctxd = txr->next_avail_desc; 2492 u16 vtag = 0; 2493 2494 2495 offload = ((mp->m_pkthdr.csum_flags & M_CSUM_OFFLOAD) != 0); 2496 2497 tx_buffer = &txr->tx_buffers[ctxd]; 2498 TXD = (struct ixgbe_adv_tx_context_desc *) &txr->tx_base[ctxd]; 2499 2500 /* 2501 ** In advanced descriptors the vlan tag must 2502 ** be placed into the descriptor itself. 2503 */ 2504 if ((mtag = VLAN_OUTPUT_TAG(ec, mp)) != NULL) { 2505 vtag = htole16(VLAN_TAG_VALUE(mtag) & 0xffff); 2506 vlan_macip_lens |= (vtag << IXGBE_ADVTXD_VLAN_SHIFT); 2507 } else if (!offload) 2508 return 0; 2509 2510 /* 2511 * Determine where frame payload starts. 2512 * Jump over vlan headers if already present, 2513 * helpful for QinQ too. 2514 */ 2515 KASSERT(mp->m_len >= offsetof(struct ether_vlan_header, evl_tag)); 2516 eh = mtod(mp, struct ether_vlan_header *); 2517 if (eh->evl_encap_proto == htons(ETHERTYPE_VLAN)) { 2518 KASSERT(mp->m_len >= sizeof(struct ether_vlan_header)); 2519 etype = ntohs(eh->evl_proto); 2520 ehdrlen = ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN; 2521 } else { 2522 etype = ntohs(eh->evl_encap_proto); 2523 ehdrlen = ETHER_HDR_LEN; 2524 } 2525 2526 /* Set the ether header length */ 2527 vlan_macip_lens |= ehdrlen << IXGBE_ADVTXD_MACLEN_SHIFT; 2528 2529 switch (etype) { 2530 case ETHERTYPE_IP: 2531 m_copydata(mp, ehdrlen, sizeof(ip), &ip); 2532 ip_hlen = ip.ip_hl << 2; 2533 ipproto = ip.ip_p; 2534 #if 0 2535 ip.ip_sum = 0; 2536 m_copyback(mp, ehdrlen, sizeof(ip), &ip); 2537 #else 2538 KASSERT((mp->m_pkthdr.csum_flags & M_CSUM_IPv4) == 0 || 2539 ip.ip_sum == 0); 2540 #endif 2541 type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_IPV4; 2542 break; 2543 case ETHERTYPE_IPV6: 2544 m_copydata(mp, ehdrlen, sizeof(ip6), &ip6); 2545 ip_hlen = sizeof(ip6); 2546 ipproto = ip6.ip6_nxt; 2547 type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_IPV6; 2548 break; 2549 default: 2550 break; 2551 } 2552 2553 if ((mp->m_pkthdr.csum_flags & M_CSUM_IPv4) != 0) 2554 olinfo |= IXGBE_TXD_POPTS_IXSM << 8; 2555 2556 vlan_macip_lens |= ip_hlen; 2557 type_tucmd_mlhl |= IXGBE_ADVTXD_DCMD_DEXT | IXGBE_ADVTXD_DTYP_CTXT; 2558 2559 if (mp->m_pkthdr.csum_flags & (M_CSUM_TCPv4|M_CSUM_TCPv6)) { 2560 type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_L4T_TCP; 2561 olinfo |= IXGBE_TXD_POPTS_TXSM << 8; 2562 KASSERT(ipproto == IPPROTO_TCP); 2563 } else if (mp->m_pkthdr.csum_flags & (M_CSUM_UDPv4|M_CSUM_UDPv6)) { 2564 type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_L4T_UDP; 2565 olinfo |= IXGBE_TXD_POPTS_TXSM << 8; 2566 KASSERT(ipproto == IPPROTO_UDP); 2567 } 2568 2569 /* Now copy bits into descriptor */ 2570 TXD->vlan_macip_lens |= htole32(vlan_macip_lens); 2571 TXD->type_tucmd_mlhl |= htole32(type_tucmd_mlhl); 2572 TXD->seqnum_seed = htole32(0); 2573 TXD->mss_l4len_idx = htole32(0); 2574 2575 tx_buffer->m_head = NULL; 2576 tx_buffer->eop_index = -1; 2577 2578 /* We've consumed the first desc, adjust counters */ 2579 if (++ctxd == adapter->num_tx_desc) 2580 ctxd = 0; 2581 txr->next_avail_desc = ctxd; 2582 --txr->tx_avail; 2583 2584 return olinfo; 2585 } 2586 2587 /********************************************************************** 2588 * 2589 * Setup work for hardware segmentation offload (TSO) on 2590 * adapters using advanced tx descriptors 2591 * 2592 **********************************************************************/ 2593 static bool 2594 ixv_tso_setup(struct tx_ring *txr, struct mbuf *mp, u32 *paylen) 2595 { 2596 struct m_tag *mtag; 2597 struct adapter *adapter = txr->adapter; 2598 struct ethercom *ec = &adapter->osdep.ec; 2599 struct ixgbe_adv_tx_context_desc *TXD; 2600 struct ixv_tx_buf *tx_buffer; 2601 u32 vlan_macip_lens = 0, type_tucmd_mlhl = 0; 2602 u32 mss_l4len_idx = 0; 2603 u16 vtag = 0; 2604 int ctxd, ehdrlen, hdrlen, ip_hlen, tcp_hlen; 2605 struct ether_vlan_header *eh; 2606 struct ip *ip; 2607 struct tcphdr *th; 2608 2609 2610 /* 2611 * Determine where frame payload starts. 2612 * Jump over vlan headers if already present 2613 */ 2614 eh = mtod(mp, struct ether_vlan_header *); 2615 if (eh->evl_encap_proto == htons(ETHERTYPE_VLAN)) 2616 ehdrlen = ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN; 2617 else 2618 ehdrlen = ETHER_HDR_LEN; 2619 2620 /* Ensure we have at least the IP+TCP header in the first mbuf. */ 2621 if (mp->m_len < ehdrlen + sizeof(struct ip) + sizeof(struct tcphdr)) 2622 return FALSE; 2623 2624 ctxd = txr->next_avail_desc; 2625 tx_buffer = &txr->tx_buffers[ctxd]; 2626 TXD = (struct ixgbe_adv_tx_context_desc *) &txr->tx_base[ctxd]; 2627 2628 ip = (struct ip *)(mp->m_data + ehdrlen); 2629 if (ip->ip_p != IPPROTO_TCP) 2630 return FALSE; /* 0 */ 2631 ip->ip_sum = 0; 2632 ip_hlen = ip->ip_hl << 2; 2633 th = (struct tcphdr *)((char *)ip + ip_hlen); 2634 /* XXX Educated guess: FreeBSD's in_pseudo == NetBSD's in_cksum_phdr */ 2635 th->th_sum = in_cksum_phdr(ip->ip_src.s_addr, 2636 ip->ip_dst.s_addr, htons(IPPROTO_TCP)); 2637 tcp_hlen = th->th_off << 2; 2638 hdrlen = ehdrlen + ip_hlen + tcp_hlen; 2639 2640 /* This is used in the transmit desc in encap */ 2641 *paylen = mp->m_pkthdr.len - hdrlen; 2642 2643 /* VLAN MACLEN IPLEN */ 2644 if ((mtag = VLAN_OUTPUT_TAG(ec, mp)) != NULL) { 2645 vtag = htole16(VLAN_TAG_VALUE(mtag) & 0xffff); 2646 vlan_macip_lens |= (vtag << IXGBE_ADVTXD_VLAN_SHIFT); 2647 } 2648 2649 vlan_macip_lens |= ehdrlen << IXGBE_ADVTXD_MACLEN_SHIFT; 2650 vlan_macip_lens |= ip_hlen; 2651 TXD->vlan_macip_lens |= htole32(vlan_macip_lens); 2652 2653 /* ADV DTYPE TUCMD */ 2654 type_tucmd_mlhl |= IXGBE_ADVTXD_DCMD_DEXT | IXGBE_ADVTXD_DTYP_CTXT; 2655 type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_L4T_TCP; 2656 type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_IPV4; 2657 TXD->type_tucmd_mlhl |= htole32(type_tucmd_mlhl); 2658 2659 2660 /* MSS L4LEN IDX */ 2661 mss_l4len_idx |= (mp->m_pkthdr.segsz << IXGBE_ADVTXD_MSS_SHIFT); 2662 mss_l4len_idx |= (tcp_hlen << IXGBE_ADVTXD_L4LEN_SHIFT); 2663 TXD->mss_l4len_idx = htole32(mss_l4len_idx); 2664 2665 TXD->seqnum_seed = htole32(0); 2666 tx_buffer->m_head = NULL; 2667 tx_buffer->eop_index = -1; 2668 2669 if (++ctxd == adapter->num_tx_desc) 2670 ctxd = 0; 2671 2672 txr->tx_avail--; 2673 txr->next_avail_desc = ctxd; 2674 return TRUE; 2675 } 2676 2677 2678 /********************************************************************** 2679 * 2680 * Examine each tx_buffer in the used queue. If the hardware is done 2681 * processing the packet then free associated resources. The 2682 * tx_buffer is put back on the free queue. 2683 * 2684 **********************************************************************/ 2685 static bool 2686 ixv_txeof(struct tx_ring *txr) 2687 { 2688 struct adapter *adapter = txr->adapter; 2689 struct ifnet *ifp = adapter->ifp; 2690 u32 first, last, done; 2691 struct ixv_tx_buf *tx_buffer; 2692 struct ixgbe_legacy_tx_desc *tx_desc, *eop_desc; 2693 2694 KASSERT(mutex_owned(&txr->tx_mtx)); 2695 2696 if (txr->tx_avail == adapter->num_tx_desc) 2697 return false; 2698 2699 first = txr->next_to_clean; 2700 tx_buffer = &txr->tx_buffers[first]; 2701 /* For cleanup we just use legacy struct */ 2702 tx_desc = (struct ixgbe_legacy_tx_desc *)&txr->tx_base[first]; 2703 last = tx_buffer->eop_index; 2704 if (last == -1) 2705 return false; 2706 eop_desc = (struct ixgbe_legacy_tx_desc *)&txr->tx_base[last]; 2707 2708 /* 2709 ** Get the index of the first descriptor 2710 ** BEYOND the EOP and call that 'done'. 2711 ** I do this so the comparison in the 2712 ** inner while loop below can be simple 2713 */ 2714 if (++last == adapter->num_tx_desc) last = 0; 2715 done = last; 2716 2717 ixgbe_dmamap_sync(txr->txdma.dma_tag, txr->txdma.dma_map, 2718 BUS_DMASYNC_POSTREAD); 2719 /* 2720 ** Only the EOP descriptor of a packet now has the DD 2721 ** bit set, this is what we look for... 2722 */ 2723 while (eop_desc->upper.fields.status & IXGBE_TXD_STAT_DD) { 2724 /* We clean the range of the packet */ 2725 while (first != done) { 2726 tx_desc->upper.data = 0; 2727 tx_desc->lower.data = 0; 2728 tx_desc->buffer_addr = 0; 2729 ++txr->tx_avail; 2730 2731 if (tx_buffer->m_head) { 2732 bus_dmamap_sync(txr->txtag->dt_dmat, 2733 tx_buffer->map, 2734 0, tx_buffer->m_head->m_pkthdr.len, 2735 BUS_DMASYNC_POSTWRITE); 2736 ixgbe_dmamap_unload(txr->txtag, tx_buffer->map); 2737 m_freem(tx_buffer->m_head); 2738 tx_buffer->m_head = NULL; 2739 tx_buffer->map = NULL; 2740 } 2741 tx_buffer->eop_index = -1; 2742 getmicrotime(&txr->watchdog_time); 2743 2744 if (++first == adapter->num_tx_desc) 2745 first = 0; 2746 2747 tx_buffer = &txr->tx_buffers[first]; 2748 tx_desc = 2749 (struct ixgbe_legacy_tx_desc *)&txr->tx_base[first]; 2750 } 2751 ++ifp->if_opackets; 2752 /* See if there is more work now */ 2753 last = tx_buffer->eop_index; 2754 if (last != -1) { 2755 eop_desc = 2756 (struct ixgbe_legacy_tx_desc *)&txr->tx_base[last]; 2757 /* Get next done point */ 2758 if (++last == adapter->num_tx_desc) last = 0; 2759 done = last; 2760 } else 2761 break; 2762 } 2763 ixgbe_dmamap_sync(txr->txdma.dma_tag, txr->txdma.dma_map, 2764 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); 2765 2766 txr->next_to_clean = first; 2767 2768 /* 2769 * If we have enough room, clear IFF_OACTIVE to tell the stack that 2770 * it is OK to send packets. If there are no pending descriptors, 2771 * clear the timeout. Otherwise, if some descriptors have been freed, 2772 * restart the timeout. 2773 */ 2774 if (txr->tx_avail > IXV_TX_CLEANUP_THRESHOLD) { 2775 ifp->if_flags &= ~IFF_OACTIVE; 2776 if (txr->tx_avail == adapter->num_tx_desc) { 2777 txr->watchdog_check = FALSE; 2778 return false; 2779 } 2780 } 2781 2782 return true; 2783 } 2784 2785 /********************************************************************* 2786 * 2787 * Refresh mbuf buffers for RX descriptor rings 2788 * - now keeps its own state so discards due to resource 2789 * exhaustion are unnecessary, if an mbuf cannot be obtained 2790 * it just returns, keeping its placeholder, thus it can simply 2791 * be recalled to try again. 2792 * 2793 **********************************************************************/ 2794 static void 2795 ixv_refresh_mbufs(struct rx_ring *rxr, int limit) 2796 { 2797 struct adapter *adapter = rxr->adapter; 2798 struct ixv_rx_buf *rxbuf; 2799 struct mbuf *mh, *mp; 2800 int i, j, error; 2801 bool refreshed = false; 2802 2803 i = j = rxr->next_to_refresh; 2804 /* Get the control variable, one beyond refresh point */ 2805 if (++j == adapter->num_rx_desc) 2806 j = 0; 2807 while (j != limit) { 2808 rxbuf = &rxr->rx_buffers[i]; 2809 if ((rxbuf->m_head == NULL) && (rxr->hdr_split)) { 2810 mh = m_gethdr(M_NOWAIT, MT_DATA); 2811 if (mh == NULL) 2812 goto update; 2813 mh->m_pkthdr.len = mh->m_len = MHLEN; 2814 mh->m_flags |= M_PKTHDR; 2815 m_adj(mh, ETHER_ALIGN); 2816 /* Get the memory mapping */ 2817 error = bus_dmamap_load_mbuf(rxr->htag->dt_dmat, 2818 rxbuf->hmap, mh, BUS_DMA_NOWAIT); 2819 if (error != 0) { 2820 printf("GET BUF: dmamap load" 2821 " failure - %d\n", error); 2822 m_free(mh); 2823 goto update; 2824 } 2825 rxbuf->m_head = mh; 2826 ixgbe_dmamap_sync(rxr->htag, rxbuf->hmap, 2827 BUS_DMASYNC_PREREAD); 2828 rxr->rx_base[i].read.hdr_addr = 2829 htole64(rxbuf->hmap->dm_segs[0].ds_addr); 2830 } 2831 2832 if (rxbuf->m_pack == NULL) { 2833 mp = ixgbe_getjcl(&adapter->jcl_head, M_NOWAIT, 2834 MT_DATA, M_PKTHDR, adapter->rx_mbuf_sz); 2835 if (mp == NULL) { 2836 rxr->no_jmbuf.ev_count++; 2837 goto update; 2838 } else 2839 mp = rxbuf->m_pack; 2840 2841 mp->m_pkthdr.len = mp->m_len = adapter->rx_mbuf_sz; 2842 /* Get the memory mapping */ 2843 error = bus_dmamap_load_mbuf(rxr->ptag->dt_dmat, 2844 rxbuf->pmap, mp, BUS_DMA_NOWAIT); 2845 if (error != 0) { 2846 printf("GET BUF: dmamap load" 2847 " failure - %d\n", error); 2848 m_free(mp); 2849 rxbuf->m_pack = NULL; 2850 goto update; 2851 } 2852 rxbuf->m_pack = mp; 2853 bus_dmamap_sync(rxr->ptag->dt_dmat, rxbuf->pmap, 2854 0, mp->m_pkthdr.len, BUS_DMASYNC_PREREAD); 2855 rxr->rx_base[i].read.pkt_addr = 2856 htole64(rxbuf->pmap->dm_segs[0].ds_addr); 2857 } 2858 2859 refreshed = true; 2860 rxr->next_to_refresh = i = j; 2861 /* Calculate next index */ 2862 if (++j == adapter->num_rx_desc) 2863 j = 0; 2864 } 2865 update: 2866 if (refreshed) /* update tail index */ 2867 IXGBE_WRITE_REG(&adapter->hw, 2868 IXGBE_VFRDT(rxr->me), rxr->next_to_refresh); 2869 return; 2870 } 2871 2872 /********************************************************************* 2873 * 2874 * Allocate memory for rx_buffer structures. Since we use one 2875 * rx_buffer per received packet, the maximum number of rx_buffer's 2876 * that we'll need is equal to the number of receive descriptors 2877 * that we've allocated. 2878 * 2879 **********************************************************************/ 2880 static int 2881 ixv_allocate_receive_buffers(struct rx_ring *rxr) 2882 { 2883 struct adapter *adapter = rxr->adapter; 2884 device_t dev = adapter->dev; 2885 struct ixv_rx_buf *rxbuf; 2886 int i, bsize, error; 2887 2888 bsize = sizeof(struct ixv_rx_buf) * adapter->num_rx_desc; 2889 if (!(rxr->rx_buffers = 2890 (struct ixv_rx_buf *) malloc(bsize, 2891 M_DEVBUF, M_NOWAIT | M_ZERO))) { 2892 aprint_error_dev(dev, "Unable to allocate rx_buffer memory\n"); 2893 error = ENOMEM; 2894 goto fail; 2895 } 2896 2897 if ((error = ixgbe_dma_tag_create(adapter->osdep.dmat, /* parent */ 2898 1, 0, /* alignment, bounds */ 2899 MSIZE, /* maxsize */ 2900 1, /* nsegments */ 2901 MSIZE, /* maxsegsize */ 2902 0, /* flags */ 2903 &rxr->htag))) { 2904 aprint_error_dev(dev, "Unable to create RX DMA tag\n"); 2905 goto fail; 2906 } 2907 2908 if ((error = ixgbe_dma_tag_create(adapter->osdep.dmat, /* parent */ 2909 1, 0, /* alignment, bounds */ 2910 MJUMPAGESIZE, /* maxsize */ 2911 1, /* nsegments */ 2912 MJUMPAGESIZE, /* maxsegsize */ 2913 0, /* flags */ 2914 &rxr->ptag))) { 2915 aprint_error_dev(dev, "Unable to create RX DMA tag\n"); 2916 goto fail; 2917 } 2918 2919 for (i = 0; i < adapter->num_rx_desc; i++, rxbuf++) { 2920 rxbuf = &rxr->rx_buffers[i]; 2921 error = ixgbe_dmamap_create(rxr->htag, 2922 BUS_DMA_NOWAIT, &rxbuf->hmap); 2923 if (error) { 2924 aprint_error_dev(dev, "Unable to create RX head map\n"); 2925 goto fail; 2926 } 2927 error = ixgbe_dmamap_create(rxr->ptag, 2928 BUS_DMA_NOWAIT, &rxbuf->pmap); 2929 if (error) { 2930 aprint_error_dev(dev, "Unable to create RX pkt map\n"); 2931 goto fail; 2932 } 2933 } 2934 2935 return (0); 2936 2937 fail: 2938 /* Frees all, but can handle partial completion */ 2939 ixv_free_receive_structures(adapter); 2940 return (error); 2941 } 2942 2943 static void 2944 ixv_free_receive_ring(struct rx_ring *rxr) 2945 { 2946 struct adapter *adapter; 2947 struct ixv_rx_buf *rxbuf; 2948 int i; 2949 2950 adapter = rxr->adapter; 2951 for (i = 0; i < adapter->num_rx_desc; i++) { 2952 rxbuf = &rxr->rx_buffers[i]; 2953 if (rxbuf->m_head != NULL) { 2954 ixgbe_dmamap_sync(rxr->htag, rxbuf->hmap, 2955 BUS_DMASYNC_POSTREAD); 2956 ixgbe_dmamap_unload(rxr->htag, rxbuf->hmap); 2957 rxbuf->m_head->m_flags |= M_PKTHDR; 2958 m_freem(rxbuf->m_head); 2959 } 2960 if (rxbuf->m_pack != NULL) { 2961 /* XXX not ixgbe_ ? */ 2962 bus_dmamap_sync(rxr->ptag->dt_dmat, rxbuf->pmap, 2963 0, rxbuf->m_pack->m_pkthdr.len, 2964 BUS_DMASYNC_POSTREAD); 2965 ixgbe_dmamap_unload(rxr->ptag, rxbuf->pmap); 2966 rxbuf->m_pack->m_flags |= M_PKTHDR; 2967 m_freem(rxbuf->m_pack); 2968 } 2969 rxbuf->m_head = NULL; 2970 rxbuf->m_pack = NULL; 2971 } 2972 } 2973 2974 2975 /********************************************************************* 2976 * 2977 * Initialize a receive ring and its buffers. 2978 * 2979 **********************************************************************/ 2980 static int 2981 ixv_setup_receive_ring(struct rx_ring *rxr) 2982 { 2983 struct adapter *adapter; 2984 struct ixv_rx_buf *rxbuf; 2985 #ifdef LRO 2986 struct ifnet *ifp; 2987 struct lro_ctrl *lro = &rxr->lro; 2988 #endif /* LRO */ 2989 int rsize, error = 0; 2990 2991 adapter = rxr->adapter; 2992 #ifdef LRO 2993 ifp = adapter->ifp; 2994 #endif /* LRO */ 2995 2996 /* Clear the ring contents */ 2997 IXV_RX_LOCK(rxr); 2998 rsize = roundup2(adapter->num_rx_desc * 2999 sizeof(union ixgbe_adv_rx_desc), DBA_ALIGN); 3000 bzero((void *)rxr->rx_base, rsize); 3001 3002 /* Free current RX buffer structs and their mbufs */ 3003 ixv_free_receive_ring(rxr); 3004 3005 IXV_RX_UNLOCK(rxr); 3006 3007 /* Now reinitialize our supply of jumbo mbufs. The number 3008 * or size of jumbo mbufs may have changed. 3009 */ 3010 ixgbe_jcl_reinit(&adapter->jcl_head, rxr->ptag->dt_dmat, 3011 2 * adapter->num_rx_desc, adapter->rx_mbuf_sz); 3012 3013 IXV_RX_LOCK(rxr); 3014 3015 /* Configure header split? */ 3016 if (ixv_header_split) 3017 rxr->hdr_split = TRUE; 3018 3019 /* Now replenish the mbufs */ 3020 for (int j = 0; j != adapter->num_rx_desc; ++j) { 3021 struct mbuf *mh, *mp; 3022 3023 rxbuf = &rxr->rx_buffers[j]; 3024 /* 3025 ** Dont allocate mbufs if not 3026 ** doing header split, its wasteful 3027 */ 3028 if (rxr->hdr_split == FALSE) 3029 goto skip_head; 3030 3031 /* First the header */ 3032 rxbuf->m_head = m_gethdr(M_DONTWAIT, MT_DATA); 3033 if (rxbuf->m_head == NULL) { 3034 error = ENOBUFS; 3035 goto fail; 3036 } 3037 m_adj(rxbuf->m_head, ETHER_ALIGN); 3038 mh = rxbuf->m_head; 3039 mh->m_len = mh->m_pkthdr.len = MHLEN; 3040 mh->m_flags |= M_PKTHDR; 3041 /* Get the memory mapping */ 3042 error = bus_dmamap_load_mbuf(rxr->htag->dt_dmat, 3043 rxbuf->hmap, rxbuf->m_head, BUS_DMA_NOWAIT); 3044 if (error != 0) /* Nothing elegant to do here */ 3045 goto fail; 3046 bus_dmamap_sync(rxr->htag->dt_dmat, rxbuf->hmap, 3047 0, mh->m_pkthdr.len, BUS_DMASYNC_PREREAD); 3048 /* Update descriptor */ 3049 rxr->rx_base[j].read.hdr_addr = 3050 htole64(rxbuf->hmap->dm_segs[0].ds_addr); 3051 3052 skip_head: 3053 /* Now the payload cluster */ 3054 rxbuf->m_pack = ixgbe_getjcl(&adapter->jcl_head, M_DONTWAIT, 3055 MT_DATA, M_PKTHDR, adapter->rx_mbuf_sz); 3056 if (rxbuf->m_pack == NULL) { 3057 error = ENOBUFS; 3058 goto fail; 3059 } 3060 mp = rxbuf->m_pack; 3061 mp->m_pkthdr.len = mp->m_len = adapter->rx_mbuf_sz; 3062 /* Get the memory mapping */ 3063 error = bus_dmamap_load_mbuf(rxr->ptag->dt_dmat, 3064 rxbuf->pmap, mp, BUS_DMA_NOWAIT); 3065 if (error != 0) 3066 goto fail; 3067 bus_dmamap_sync(rxr->ptag->dt_dmat, rxbuf->pmap, 3068 0, adapter->rx_mbuf_sz, BUS_DMASYNC_PREREAD); 3069 /* Update descriptor */ 3070 rxr->rx_base[j].read.pkt_addr = 3071 htole64(rxbuf->pmap->dm_segs[0].ds_addr); 3072 } 3073 3074 3075 /* Setup our descriptor indices */ 3076 rxr->next_to_check = 0; 3077 rxr->next_to_refresh = 0; 3078 rxr->lro_enabled = FALSE; 3079 rxr->rx_split_packets.ev_count = 0; 3080 rxr->rx_bytes.ev_count = 0; 3081 rxr->discard = FALSE; 3082 3083 ixgbe_dmamap_sync(rxr->rxdma.dma_tag, rxr->rxdma.dma_map, 3084 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); 3085 3086 #ifdef LRO 3087 /* 3088 ** Now set up the LRO interface: 3089 */ 3090 if (ifp->if_capenable & IFCAP_LRO) { 3091 device_t dev = adapter->dev; 3092 int err = tcp_lro_init(lro); 3093 if (err) { 3094 device_printf(dev, "LRO Initialization failed!\n"); 3095 goto fail; 3096 } 3097 INIT_DEBUGOUT("RX Soft LRO Initialized\n"); 3098 rxr->lro_enabled = TRUE; 3099 lro->ifp = adapter->ifp; 3100 } 3101 #endif /* LRO */ 3102 3103 IXV_RX_UNLOCK(rxr); 3104 return (0); 3105 3106 fail: 3107 ixv_free_receive_ring(rxr); 3108 IXV_RX_UNLOCK(rxr); 3109 return (error); 3110 } 3111 3112 /********************************************************************* 3113 * 3114 * Initialize all receive rings. 3115 * 3116 **********************************************************************/ 3117 static int 3118 ixv_setup_receive_structures(struct adapter *adapter) 3119 { 3120 struct rx_ring *rxr = adapter->rx_rings; 3121 int j; 3122 3123 for (j = 0; j < adapter->num_queues; j++, rxr++) 3124 if (ixv_setup_receive_ring(rxr)) 3125 goto fail; 3126 3127 return (0); 3128 fail: 3129 /* 3130 * Free RX buffers allocated so far, we will only handle 3131 * the rings that completed, the failing case will have 3132 * cleaned up for itself. 'j' failed, so its the terminus. 3133 */ 3134 for (int i = 0; i < j; ++i) { 3135 rxr = &adapter->rx_rings[i]; 3136 ixv_free_receive_ring(rxr); 3137 } 3138 3139 return (ENOBUFS); 3140 } 3141 3142 /********************************************************************* 3143 * 3144 * Setup receive registers and features. 3145 * 3146 **********************************************************************/ 3147 #define IXGBE_SRRCTL_BSIZEHDRSIZE_SHIFT 2 3148 3149 static void 3150 ixv_initialize_receive_units(struct adapter *adapter) 3151 { 3152 int i; 3153 struct rx_ring *rxr = adapter->rx_rings; 3154 struct ixgbe_hw *hw = &adapter->hw; 3155 struct ifnet *ifp = adapter->ifp; 3156 u32 bufsz, fctrl, rxcsum, hlreg; 3157 3158 3159 /* Enable broadcasts */ 3160 fctrl = IXGBE_READ_REG(hw, IXGBE_FCTRL); 3161 fctrl |= IXGBE_FCTRL_BAM; 3162 fctrl |= IXGBE_FCTRL_DPF; 3163 fctrl |= IXGBE_FCTRL_PMCF; 3164 IXGBE_WRITE_REG(hw, IXGBE_FCTRL, fctrl); 3165 3166 /* Set for Jumbo Frames? */ 3167 hlreg = IXGBE_READ_REG(hw, IXGBE_HLREG0); 3168 if (ifp->if_mtu > ETHERMTU) { 3169 hlreg |= IXGBE_HLREG0_JUMBOEN; 3170 bufsz = 4096 >> IXGBE_SRRCTL_BSIZEPKT_SHIFT; 3171 } else { 3172 hlreg &= ~IXGBE_HLREG0_JUMBOEN; 3173 bufsz = 2048 >> IXGBE_SRRCTL_BSIZEPKT_SHIFT; 3174 } 3175 IXGBE_WRITE_REG(hw, IXGBE_HLREG0, hlreg); 3176 3177 for (i = 0; i < adapter->num_queues; i++, rxr++) { 3178 u64 rdba = rxr->rxdma.dma_paddr; 3179 u32 reg, rxdctl; 3180 3181 /* Do the queue enabling first */ 3182 rxdctl = IXGBE_READ_REG(hw, IXGBE_VFRXDCTL(i)); 3183 rxdctl |= IXGBE_RXDCTL_ENABLE; 3184 IXGBE_WRITE_REG(hw, IXGBE_VFRXDCTL(i), rxdctl); 3185 for (int k = 0; k < 10; k++) { 3186 if (IXGBE_READ_REG(hw, IXGBE_VFRXDCTL(i)) & 3187 IXGBE_RXDCTL_ENABLE) 3188 break; 3189 else 3190 msec_delay(1); 3191 } 3192 wmb(); 3193 3194 /* Setup the Base and Length of the Rx Descriptor Ring */ 3195 IXGBE_WRITE_REG(hw, IXGBE_VFRDBAL(i), 3196 (rdba & 0x00000000ffffffffULL)); 3197 IXGBE_WRITE_REG(hw, IXGBE_VFRDBAH(i), 3198 (rdba >> 32)); 3199 IXGBE_WRITE_REG(hw, IXGBE_VFRDLEN(i), 3200 adapter->num_rx_desc * sizeof(union ixgbe_adv_rx_desc)); 3201 3202 /* Set up the SRRCTL register */ 3203 reg = IXGBE_READ_REG(hw, IXGBE_VFSRRCTL(i)); 3204 reg &= ~IXGBE_SRRCTL_BSIZEHDR_MASK; 3205 reg &= ~IXGBE_SRRCTL_BSIZEPKT_MASK; 3206 reg |= bufsz; 3207 if (rxr->hdr_split) { 3208 /* Use a standard mbuf for the header */ 3209 reg |= ((IXV_RX_HDR << 3210 IXGBE_SRRCTL_BSIZEHDRSIZE_SHIFT) 3211 & IXGBE_SRRCTL_BSIZEHDR_MASK); 3212 reg |= IXGBE_SRRCTL_DESCTYPE_HDR_SPLIT_ALWAYS; 3213 } else 3214 reg |= IXGBE_SRRCTL_DESCTYPE_ADV_ONEBUF; 3215 IXGBE_WRITE_REG(hw, IXGBE_VFSRRCTL(i), reg); 3216 3217 /* Setup the HW Rx Head and Tail Descriptor Pointers */ 3218 IXGBE_WRITE_REG(hw, IXGBE_VFRDH(rxr->me), 0); 3219 IXGBE_WRITE_REG(hw, IXGBE_VFRDT(rxr->me), 3220 adapter->num_rx_desc - 1); 3221 } 3222 3223 rxcsum = IXGBE_READ_REG(hw, IXGBE_RXCSUM); 3224 3225 if (ifp->if_capenable & IFCAP_RXCSUM) 3226 rxcsum |= IXGBE_RXCSUM_PCSD; 3227 3228 if (!(rxcsum & IXGBE_RXCSUM_PCSD)) 3229 rxcsum |= IXGBE_RXCSUM_IPPCSE; 3230 3231 IXGBE_WRITE_REG(hw, IXGBE_RXCSUM, rxcsum); 3232 3233 return; 3234 } 3235 3236 /********************************************************************* 3237 * 3238 * Free all receive rings. 3239 * 3240 **********************************************************************/ 3241 static void 3242 ixv_free_receive_structures(struct adapter *adapter) 3243 { 3244 struct rx_ring *rxr = adapter->rx_rings; 3245 3246 for (int i = 0; i < adapter->num_queues; i++, rxr++) { 3247 #ifdef LRO 3248 struct lro_ctrl *lro = &rxr->lro; 3249 #endif /* LRO */ 3250 ixv_free_receive_buffers(rxr); 3251 #ifdef LRO 3252 /* Free LRO memory */ 3253 tcp_lro_free(lro); 3254 #endif /* LRO */ 3255 /* Free the ring memory as well */ 3256 ixv_dma_free(adapter, &rxr->rxdma); 3257 IXV_RX_LOCK_DESTROY(rxr); 3258 } 3259 3260 free(adapter->rx_rings, M_DEVBUF); 3261 } 3262 3263 3264 /********************************************************************* 3265 * 3266 * Free receive ring data structures 3267 * 3268 **********************************************************************/ 3269 static void 3270 ixv_free_receive_buffers(struct rx_ring *rxr) 3271 { 3272 struct adapter *adapter = rxr->adapter; 3273 struct ixv_rx_buf *rxbuf; 3274 3275 INIT_DEBUGOUT("free_receive_structures: begin"); 3276 3277 /* Cleanup any existing buffers */ 3278 if (rxr->rx_buffers != NULL) { 3279 for (int i = 0; i < adapter->num_rx_desc; i++) { 3280 rxbuf = &rxr->rx_buffers[i]; 3281 if (rxbuf->m_head != NULL) { 3282 ixgbe_dmamap_sync(rxr->htag, rxbuf->hmap, 3283 BUS_DMASYNC_POSTREAD); 3284 ixgbe_dmamap_unload(rxr->htag, rxbuf->hmap); 3285 rxbuf->m_head->m_flags |= M_PKTHDR; 3286 m_freem(rxbuf->m_head); 3287 } 3288 if (rxbuf->m_pack != NULL) { 3289 /* XXX not ixgbe_* ? */ 3290 bus_dmamap_sync(rxr->ptag->dt_dmat, rxbuf->pmap, 3291 0, rxbuf->m_pack->m_pkthdr.len, 3292 BUS_DMASYNC_POSTREAD); 3293 ixgbe_dmamap_unload(rxr->ptag, rxbuf->pmap); 3294 rxbuf->m_pack->m_flags |= M_PKTHDR; 3295 m_freem(rxbuf->m_pack); 3296 } 3297 rxbuf->m_head = NULL; 3298 rxbuf->m_pack = NULL; 3299 if (rxbuf->hmap != NULL) { 3300 ixgbe_dmamap_destroy(rxr->htag, rxbuf->hmap); 3301 rxbuf->hmap = NULL; 3302 } 3303 if (rxbuf->pmap != NULL) { 3304 ixgbe_dmamap_destroy(rxr->ptag, rxbuf->pmap); 3305 rxbuf->pmap = NULL; 3306 } 3307 } 3308 if (rxr->rx_buffers != NULL) { 3309 free(rxr->rx_buffers, M_DEVBUF); 3310 rxr->rx_buffers = NULL; 3311 } 3312 } 3313 3314 if (rxr->htag != NULL) { 3315 ixgbe_dma_tag_destroy(rxr->htag); 3316 rxr->htag = NULL; 3317 } 3318 if (rxr->ptag != NULL) { 3319 ixgbe_dma_tag_destroy(rxr->ptag); 3320 rxr->ptag = NULL; 3321 } 3322 3323 return; 3324 } 3325 3326 static __inline void 3327 ixv_rx_input(struct rx_ring *rxr, struct ifnet *ifp, struct mbuf *m, u32 ptype) 3328 { 3329 int s; 3330 3331 #ifdef LRO 3332 struct adapter *adapter = ifp->if_softc; 3333 struct ethercom *ec = &adapter->osdep.ec; 3334 3335 /* 3336 * ATM LRO is only for IPv4/TCP packets and TCP checksum of the packet 3337 * should be computed by hardware. Also it should not have VLAN tag in 3338 * ethernet header. 3339 */ 3340 if (rxr->lro_enabled && 3341 (ec->ec_capenable & ETHERCAP_VLAN_HWTAGGING) != 0 && 3342 (ptype & IXGBE_RXDADV_PKTTYPE_ETQF) == 0 && 3343 (ptype & (IXGBE_RXDADV_PKTTYPE_IPV4 | IXGBE_RXDADV_PKTTYPE_TCP)) == 3344 (IXGBE_RXDADV_PKTTYPE_IPV4 | IXGBE_RXDADV_PKTTYPE_TCP) && 3345 (m->m_pkthdr.csum_flags & (CSUM_DATA_VALID | CSUM_PSEUDO_HDR)) == 3346 (CSUM_DATA_VALID | CSUM_PSEUDO_HDR)) { 3347 /* 3348 * Send to the stack if: 3349 ** - LRO not enabled, or 3350 ** - no LRO resources, or 3351 ** - lro enqueue fails 3352 */ 3353 if (rxr->lro.lro_cnt != 0) 3354 if (tcp_lro_rx(&rxr->lro, m, 0) == 0) 3355 return; 3356 } 3357 #endif /* LRO */ 3358 3359 IXV_RX_UNLOCK(rxr); 3360 3361 s = splnet(); 3362 /* Pass this up to any BPF listeners. */ 3363 bpf_mtap(ifp, m); 3364 (*ifp->if_input)(ifp, m); 3365 splx(s); 3366 3367 IXV_RX_LOCK(rxr); 3368 } 3369 3370 static __inline void 3371 ixv_rx_discard(struct rx_ring *rxr, int i) 3372 { 3373 struct ixv_rx_buf *rbuf; 3374 3375 rbuf = &rxr->rx_buffers[i]; 3376 if (rbuf->fmp != NULL) {/* Partial chain ? */ 3377 rbuf->fmp->m_flags |= M_PKTHDR; 3378 m_freem(rbuf->fmp); 3379 rbuf->fmp = NULL; 3380 } 3381 3382 /* 3383 ** With advanced descriptors the writeback 3384 ** clobbers the buffer addrs, so its easier 3385 ** to just free the existing mbufs and take 3386 ** the normal refresh path to get new buffers 3387 ** and mapping. 3388 */ 3389 if (rbuf->m_head) { 3390 m_free(rbuf->m_head); 3391 rbuf->m_head = NULL; 3392 } 3393 3394 if (rbuf->m_pack) { 3395 m_free(rbuf->m_pack); 3396 rbuf->m_pack = NULL; 3397 } 3398 3399 return; 3400 } 3401 3402 3403 /********************************************************************* 3404 * 3405 * This routine executes in interrupt context. It replenishes 3406 * the mbufs in the descriptor and sends data which has been 3407 * dma'ed into host memory to upper layer. 3408 * 3409 * We loop at most count times if count is > 0, or until done if 3410 * count < 0. 3411 * 3412 * Return TRUE for more work, FALSE for all clean. 3413 *********************************************************************/ 3414 static bool 3415 ixv_rxeof(struct ix_queue *que, int count) 3416 { 3417 struct adapter *adapter = que->adapter; 3418 struct rx_ring *rxr = que->rxr; 3419 struct ifnet *ifp = adapter->ifp; 3420 #ifdef LRO 3421 struct lro_ctrl *lro = &rxr->lro; 3422 struct lro_entry *queued; 3423 #endif /* LRO */ 3424 int i, nextp, processed = 0; 3425 u32 staterr = 0; 3426 union ixgbe_adv_rx_desc *cur; 3427 struct ixv_rx_buf *rbuf, *nbuf; 3428 3429 IXV_RX_LOCK(rxr); 3430 3431 for (i = rxr->next_to_check; count != 0;) { 3432 struct mbuf *sendmp, *mh, *mp; 3433 u32 ptype; 3434 u16 hlen, plen, hdr, vtag; 3435 bool eop; 3436 3437 /* Sync the ring. */ 3438 ixgbe_dmamap_sync(rxr->rxdma.dma_tag, rxr->rxdma.dma_map, 3439 BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE); 3440 3441 cur = &rxr->rx_base[i]; 3442 staterr = le32toh(cur->wb.upper.status_error); 3443 3444 if ((staterr & IXGBE_RXD_STAT_DD) == 0) 3445 break; 3446 if ((ifp->if_flags & IFF_RUNNING) == 0) 3447 break; 3448 3449 count--; 3450 sendmp = NULL; 3451 nbuf = NULL; 3452 cur->wb.upper.status_error = 0; 3453 rbuf = &rxr->rx_buffers[i]; 3454 mh = rbuf->m_head; 3455 mp = rbuf->m_pack; 3456 3457 plen = le16toh(cur->wb.upper.length); 3458 ptype = le32toh(cur->wb.lower.lo_dword.data) & 3459 IXGBE_RXDADV_PKTTYPE_MASK; 3460 hdr = le16toh(cur->wb.lower.lo_dword.hs_rss.hdr_info); 3461 vtag = le16toh(cur->wb.upper.vlan); 3462 eop = ((staterr & IXGBE_RXD_STAT_EOP) != 0); 3463 3464 /* Make sure all parts of a bad packet are discarded */ 3465 if (((staterr & IXGBE_RXDADV_ERR_FRAME_ERR_MASK) != 0) || 3466 (rxr->discard)) { 3467 ifp->if_ierrors++; 3468 rxr->rx_discarded.ev_count++; 3469 if (!eop) 3470 rxr->discard = TRUE; 3471 else 3472 rxr->discard = FALSE; 3473 ixv_rx_discard(rxr, i); 3474 goto next_desc; 3475 } 3476 3477 if (!eop) { 3478 nextp = i + 1; 3479 if (nextp == adapter->num_rx_desc) 3480 nextp = 0; 3481 nbuf = &rxr->rx_buffers[nextp]; 3482 prefetch(nbuf); 3483 } 3484 /* 3485 ** The header mbuf is ONLY used when header 3486 ** split is enabled, otherwise we get normal 3487 ** behavior, ie, both header and payload 3488 ** are DMA'd into the payload buffer. 3489 ** 3490 ** Rather than using the fmp/lmp global pointers 3491 ** we now keep the head of a packet chain in the 3492 ** buffer struct and pass this along from one 3493 ** descriptor to the next, until we get EOP. 3494 */ 3495 if (rxr->hdr_split && (rbuf->fmp == NULL)) { 3496 /* This must be an initial descriptor */ 3497 hlen = (hdr & IXGBE_RXDADV_HDRBUFLEN_MASK) >> 3498 IXGBE_RXDADV_HDRBUFLEN_SHIFT; 3499 if (hlen > IXV_RX_HDR) 3500 hlen = IXV_RX_HDR; 3501 mh->m_len = hlen; 3502 mh->m_flags |= M_PKTHDR; 3503 mh->m_next = NULL; 3504 mh->m_pkthdr.len = mh->m_len; 3505 /* Null buf pointer so it is refreshed */ 3506 rbuf->m_head = NULL; 3507 /* 3508 ** Check the payload length, this 3509 ** could be zero if its a small 3510 ** packet. 3511 */ 3512 if (plen > 0) { 3513 mp->m_len = plen; 3514 mp->m_next = NULL; 3515 mp->m_flags &= ~M_PKTHDR; 3516 mh->m_next = mp; 3517 mh->m_pkthdr.len += mp->m_len; 3518 /* Null buf pointer so it is refreshed */ 3519 rbuf->m_pack = NULL; 3520 rxr->rx_split_packets.ev_count++; 3521 } 3522 /* 3523 ** Now create the forward 3524 ** chain so when complete 3525 ** we wont have to. 3526 */ 3527 if (eop == 0) { 3528 /* stash the chain head */ 3529 nbuf->fmp = mh; 3530 /* Make forward chain */ 3531 if (plen) 3532 mp->m_next = nbuf->m_pack; 3533 else 3534 mh->m_next = nbuf->m_pack; 3535 } else { 3536 /* Singlet, prepare to send */ 3537 sendmp = mh; 3538 if (VLAN_ATTACHED(&adapter->osdep.ec) && 3539 (staterr & IXGBE_RXD_STAT_VP)) { 3540 VLAN_INPUT_TAG(ifp, sendmp, vtag, 3541 printf("%s: could not apply VLAN " 3542 "tag", __func__)); 3543 } 3544 } 3545 } else { 3546 /* 3547 ** Either no header split, or a 3548 ** secondary piece of a fragmented 3549 ** split packet. 3550 */ 3551 mp->m_len = plen; 3552 /* 3553 ** See if there is a stored head 3554 ** that determines what we are 3555 */ 3556 sendmp = rbuf->fmp; 3557 rbuf->m_pack = rbuf->fmp = NULL; 3558 3559 if (sendmp != NULL) /* secondary frag */ 3560 sendmp->m_pkthdr.len += mp->m_len; 3561 else { 3562 /* first desc of a non-ps chain */ 3563 sendmp = mp; 3564 sendmp->m_flags |= M_PKTHDR; 3565 sendmp->m_pkthdr.len = mp->m_len; 3566 if (staterr & IXGBE_RXD_STAT_VP) { 3567 /* XXX Do something reasonable on 3568 * error. 3569 */ 3570 VLAN_INPUT_TAG(ifp, sendmp, vtag, 3571 printf("%s: could not apply VLAN " 3572 "tag", __func__)); 3573 } 3574 } 3575 /* Pass the head pointer on */ 3576 if (eop == 0) { 3577 nbuf->fmp = sendmp; 3578 sendmp = NULL; 3579 mp->m_next = nbuf->m_pack; 3580 } 3581 } 3582 ++processed; 3583 /* Sending this frame? */ 3584 if (eop) { 3585 sendmp->m_pkthdr.rcvif = ifp; 3586 ifp->if_ipackets++; 3587 rxr->rx_packets.ev_count++; 3588 /* capture data for AIM */ 3589 rxr->bytes += sendmp->m_pkthdr.len; 3590 rxr->rx_bytes.ev_count += sendmp->m_pkthdr.len; 3591 if ((ifp->if_capenable & IFCAP_RXCSUM) != 0) { 3592 ixv_rx_checksum(staterr, sendmp, ptype, 3593 &adapter->stats); 3594 } 3595 #if __FreeBSD_version >= 800000 3596 sendmp->m_pkthdr.flowid = que->msix; 3597 sendmp->m_flags |= M_FLOWID; 3598 #endif 3599 } 3600 next_desc: 3601 ixgbe_dmamap_sync(rxr->rxdma.dma_tag, rxr->rxdma.dma_map, 3602 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); 3603 3604 /* Advance our pointers to the next descriptor. */ 3605 if (++i == adapter->num_rx_desc) 3606 i = 0; 3607 3608 /* Now send to the stack or do LRO */ 3609 if (sendmp != NULL) 3610 ixv_rx_input(rxr, ifp, sendmp, ptype); 3611 3612 /* Every 8 descriptors we go to refresh mbufs */ 3613 if (processed == 8) { 3614 ixv_refresh_mbufs(rxr, i); 3615 processed = 0; 3616 } 3617 } 3618 3619 /* Refresh any remaining buf structs */ 3620 if (ixv_rx_unrefreshed(rxr)) 3621 ixv_refresh_mbufs(rxr, i); 3622 3623 rxr->next_to_check = i; 3624 3625 #ifdef LRO 3626 /* 3627 * Flush any outstanding LRO work 3628 */ 3629 while ((queued = SLIST_FIRST(&lro->lro_active)) != NULL) { 3630 SLIST_REMOVE_HEAD(&lro->lro_active, next); 3631 tcp_lro_flush(lro, queued); 3632 } 3633 #endif /* LRO */ 3634 3635 IXV_RX_UNLOCK(rxr); 3636 3637 /* 3638 ** We still have cleaning to do? 3639 ** Schedule another interrupt if so. 3640 */ 3641 if ((staterr & IXGBE_RXD_STAT_DD) != 0) { 3642 ixv_rearm_queues(adapter, (u64)(1ULL << que->msix)); 3643 return true; 3644 } 3645 3646 return false; 3647 } 3648 3649 3650 /********************************************************************* 3651 * 3652 * Verify that the hardware indicated that the checksum is valid. 3653 * Inform the stack about the status of checksum so that stack 3654 * doesn't spend time verifying the checksum. 3655 * 3656 *********************************************************************/ 3657 static void 3658 ixv_rx_checksum(u32 staterr, struct mbuf * mp, u32 ptype, 3659 struct ixgbevf_hw_stats *stats) 3660 { 3661 u16 status = (u16) staterr; 3662 u8 errors = (u8) (staterr >> 24); 3663 #if 0 3664 bool sctp = FALSE; 3665 if ((ptype & IXGBE_RXDADV_PKTTYPE_ETQF) == 0 && 3666 (ptype & IXGBE_RXDADV_PKTTYPE_SCTP) != 0) 3667 sctp = TRUE; 3668 #endif 3669 if (status & IXGBE_RXD_STAT_IPCS) { 3670 stats->ipcs.ev_count++; 3671 if (!(errors & IXGBE_RXD_ERR_IPE)) { 3672 /* IP Checksum Good */ 3673 mp->m_pkthdr.csum_flags |= M_CSUM_IPv4; 3674 3675 } else { 3676 stats->ipcs_bad.ev_count++; 3677 mp->m_pkthdr.csum_flags = M_CSUM_IPv4|M_CSUM_IPv4_BAD; 3678 } 3679 } 3680 if (status & IXGBE_RXD_STAT_L4CS) { 3681 stats->l4cs.ev_count++; 3682 int type = M_CSUM_TCPv4|M_CSUM_TCPv6|M_CSUM_UDPv4|M_CSUM_UDPv6; 3683 if (!(errors & IXGBE_RXD_ERR_TCPE)) { 3684 mp->m_pkthdr.csum_flags |= type; 3685 } else { 3686 stats->l4cs_bad.ev_count++; 3687 mp->m_pkthdr.csum_flags |= type | M_CSUM_TCP_UDP_BAD; 3688 } 3689 } 3690 return; 3691 } 3692 3693 static void 3694 ixv_setup_vlan_support(struct adapter *adapter) 3695 { 3696 struct ixgbe_hw *hw = &adapter->hw; 3697 u32 ctrl, vid, vfta, retry; 3698 3699 3700 /* 3701 ** We get here thru init_locked, meaning 3702 ** a soft reset, this has already cleared 3703 ** the VFTA and other state, so if there 3704 ** have been no vlan's registered do nothing. 3705 */ 3706 if (adapter->num_vlans == 0) 3707 return; 3708 3709 /* Enable the queues */ 3710 for (int i = 0; i < adapter->num_queues; i++) { 3711 ctrl = IXGBE_READ_REG(hw, IXGBE_VFRXDCTL(i)); 3712 ctrl |= IXGBE_RXDCTL_VME; 3713 IXGBE_WRITE_REG(hw, IXGBE_VFRXDCTL(i), ctrl); 3714 } 3715 3716 /* 3717 ** A soft reset zero's out the VFTA, so 3718 ** we need to repopulate it now. 3719 */ 3720 for (int i = 0; i < VFTA_SIZE; i++) { 3721 if (ixv_shadow_vfta[i] == 0) 3722 continue; 3723 vfta = ixv_shadow_vfta[i]; 3724 /* 3725 ** Reconstruct the vlan id's 3726 ** based on the bits set in each 3727 ** of the array ints. 3728 */ 3729 for ( int j = 0; j < 32; j++) { 3730 retry = 0; 3731 if ((vfta & (1 << j)) == 0) 3732 continue; 3733 vid = (i * 32) + j; 3734 /* Call the shared code mailbox routine */ 3735 while (ixgbe_set_vfta(hw, vid, 0, TRUE)) { 3736 if (++retry > 5) 3737 break; 3738 } 3739 } 3740 } 3741 } 3742 3743 #if 0 /* XXX Badly need to overhaul vlan(4) on NetBSD. */ 3744 /* 3745 ** This routine is run via an vlan config EVENT, 3746 ** it enables us to use the HW Filter table since 3747 ** we can get the vlan id. This just creates the 3748 ** entry in the soft version of the VFTA, init will 3749 ** repopulate the real table. 3750 */ 3751 static void 3752 ixv_register_vlan(void *arg, struct ifnet *ifp, u16 vtag) 3753 { 3754 struct adapter *adapter = ifp->if_softc; 3755 u16 index, bit; 3756 3757 if (ifp->if_softc != arg) /* Not our event */ 3758 return; 3759 3760 if ((vtag == 0) || (vtag > 4095)) /* Invalid */ 3761 return; 3762 3763 IXV_CORE_LOCK(adapter); 3764 index = (vtag >> 5) & 0x7F; 3765 bit = vtag & 0x1F; 3766 ixv_shadow_vfta[index] |= (1 << bit); 3767 /* Re-init to load the changes */ 3768 ixv_init_locked(adapter); 3769 IXV_CORE_UNLOCK(adapter); 3770 } 3771 3772 /* 3773 ** This routine is run via an vlan 3774 ** unconfig EVENT, remove our entry 3775 ** in the soft vfta. 3776 */ 3777 static void 3778 ixv_unregister_vlan(void *arg, struct ifnet *ifp, u16 vtag) 3779 { 3780 struct adapter *adapter = ifp->if_softc; 3781 u16 index, bit; 3782 3783 if (ifp->if_softc != arg) 3784 return; 3785 3786 if ((vtag == 0) || (vtag > 4095)) /* Invalid */ 3787 return; 3788 3789 IXV_CORE_LOCK(adapter); 3790 index = (vtag >> 5) & 0x7F; 3791 bit = vtag & 0x1F; 3792 ixv_shadow_vfta[index] &= ~(1 << bit); 3793 /* Re-init to load the changes */ 3794 ixv_init_locked(adapter); 3795 IXV_CORE_UNLOCK(adapter); 3796 } 3797 #endif 3798 3799 static void 3800 ixv_enable_intr(struct adapter *adapter) 3801 { 3802 struct ixgbe_hw *hw = &adapter->hw; 3803 struct ix_queue *que = adapter->queues; 3804 u32 mask = (IXGBE_EIMS_ENABLE_MASK & ~IXGBE_EIMS_RTX_QUEUE); 3805 3806 3807 IXGBE_WRITE_REG(hw, IXGBE_VTEIMS, mask); 3808 3809 mask = IXGBE_EIMS_ENABLE_MASK; 3810 mask &= ~(IXGBE_EIMS_OTHER | IXGBE_EIMS_LSC); 3811 IXGBE_WRITE_REG(hw, IXGBE_VTEIAC, mask); 3812 3813 for (int i = 0; i < adapter->num_queues; i++, que++) 3814 ixv_enable_queue(adapter, que->msix); 3815 3816 IXGBE_WRITE_FLUSH(hw); 3817 3818 return; 3819 } 3820 3821 static void 3822 ixv_disable_intr(struct adapter *adapter) 3823 { 3824 IXGBE_WRITE_REG(&adapter->hw, IXGBE_VTEIAC, 0); 3825 IXGBE_WRITE_REG(&adapter->hw, IXGBE_VTEIMC, ~0); 3826 IXGBE_WRITE_FLUSH(&adapter->hw); 3827 return; 3828 } 3829 3830 /* 3831 ** Setup the correct IVAR register for a particular MSIX interrupt 3832 ** - entry is the register array entry 3833 ** - vector is the MSIX vector for this queue 3834 ** - type is RX/TX/MISC 3835 */ 3836 static void 3837 ixv_set_ivar(struct adapter *adapter, u8 entry, u8 vector, s8 type) 3838 { 3839 struct ixgbe_hw *hw = &adapter->hw; 3840 u32 ivar, index; 3841 3842 vector |= IXGBE_IVAR_ALLOC_VAL; 3843 3844 if (type == -1) { /* MISC IVAR */ 3845 ivar = IXGBE_READ_REG(hw, IXGBE_VTIVAR_MISC); 3846 ivar &= ~0xFF; 3847 ivar |= vector; 3848 IXGBE_WRITE_REG(hw, IXGBE_VTIVAR_MISC, ivar); 3849 } else { /* RX/TX IVARS */ 3850 index = (16 * (entry & 1)) + (8 * type); 3851 ivar = IXGBE_READ_REG(hw, IXGBE_VTIVAR(entry >> 1)); 3852 ivar &= ~(0xFF << index); 3853 ivar |= (vector << index); 3854 IXGBE_WRITE_REG(hw, IXGBE_VTIVAR(entry >> 1), ivar); 3855 } 3856 } 3857 3858 static void 3859 ixv_configure_ivars(struct adapter *adapter) 3860 { 3861 struct ix_queue *que = adapter->queues; 3862 3863 for (int i = 0; i < adapter->num_queues; i++, que++) { 3864 /* First the RX queue entry */ 3865 ixv_set_ivar(adapter, i, que->msix, 0); 3866 /* ... and the TX */ 3867 ixv_set_ivar(adapter, i, que->msix, 1); 3868 /* Set an initial value in EITR */ 3869 IXGBE_WRITE_REG(&adapter->hw, 3870 IXGBE_VTEITR(que->msix), IXV_EITR_DEFAULT); 3871 } 3872 3873 /* For the Link interrupt */ 3874 ixv_set_ivar(adapter, 1, adapter->mbxvec, -1); 3875 } 3876 3877 3878 /* 3879 ** Tasklet handler for MSIX MBX interrupts 3880 ** - do outside interrupt since it might sleep 3881 */ 3882 static void 3883 ixv_handle_mbx(void *context) 3884 { 3885 struct adapter *adapter = context; 3886 3887 ixgbe_check_link(&adapter->hw, 3888 &adapter->link_speed, &adapter->link_up, 0); 3889 ixv_update_link_status(adapter); 3890 } 3891 3892 /* 3893 ** The VF stats registers never have a truely virgin 3894 ** starting point, so this routine tries to make an 3895 ** artificial one, marking ground zero on attach as 3896 ** it were. 3897 */ 3898 static void 3899 ixv_save_stats(struct adapter *adapter) 3900 { 3901 if (adapter->stats.vfgprc || adapter->stats.vfgptc) { 3902 adapter->stats.saved_reset_vfgprc += 3903 adapter->stats.vfgprc - adapter->stats.base_vfgprc; 3904 adapter->stats.saved_reset_vfgptc += 3905 adapter->stats.vfgptc - adapter->stats.base_vfgptc; 3906 adapter->stats.saved_reset_vfgorc += 3907 adapter->stats.vfgorc - adapter->stats.base_vfgorc; 3908 adapter->stats.saved_reset_vfgotc += 3909 adapter->stats.vfgotc - adapter->stats.base_vfgotc; 3910 adapter->stats.saved_reset_vfmprc += 3911 adapter->stats.vfmprc - adapter->stats.base_vfmprc; 3912 } 3913 } 3914 3915 static void 3916 ixv_init_stats(struct adapter *adapter) 3917 { 3918 struct ixgbe_hw *hw = &adapter->hw; 3919 3920 adapter->stats.last_vfgprc = IXGBE_READ_REG(hw, IXGBE_VFGPRC); 3921 adapter->stats.last_vfgorc = IXGBE_READ_REG(hw, IXGBE_VFGORC_LSB); 3922 adapter->stats.last_vfgorc |= 3923 (((u64)(IXGBE_READ_REG(hw, IXGBE_VFGORC_MSB))) << 32); 3924 3925 adapter->stats.last_vfgptc = IXGBE_READ_REG(hw, IXGBE_VFGPTC); 3926 adapter->stats.last_vfgotc = IXGBE_READ_REG(hw, IXGBE_VFGOTC_LSB); 3927 adapter->stats.last_vfgotc |= 3928 (((u64)(IXGBE_READ_REG(hw, IXGBE_VFGOTC_MSB))) << 32); 3929 3930 adapter->stats.last_vfmprc = IXGBE_READ_REG(hw, IXGBE_VFMPRC); 3931 3932 adapter->stats.base_vfgprc = adapter->stats.last_vfgprc; 3933 adapter->stats.base_vfgorc = adapter->stats.last_vfgorc; 3934 adapter->stats.base_vfgptc = adapter->stats.last_vfgptc; 3935 adapter->stats.base_vfgotc = adapter->stats.last_vfgotc; 3936 adapter->stats.base_vfmprc = adapter->stats.last_vfmprc; 3937 } 3938 3939 #define UPDATE_STAT_32(reg, last, count) \ 3940 { \ 3941 u32 current = IXGBE_READ_REG(hw, reg); \ 3942 if (current < last) \ 3943 count += 0x100000000LL; \ 3944 last = current; \ 3945 count &= 0xFFFFFFFF00000000LL; \ 3946 count |= current; \ 3947 } 3948 3949 #define UPDATE_STAT_36(lsb, msb, last, count) \ 3950 { \ 3951 u64 cur_lsb = IXGBE_READ_REG(hw, lsb); \ 3952 u64 cur_msb = IXGBE_READ_REG(hw, msb); \ 3953 u64 current = ((cur_msb << 32) | cur_lsb); \ 3954 if (current < last) \ 3955 count += 0x1000000000LL; \ 3956 last = current; \ 3957 count &= 0xFFFFFFF000000000LL; \ 3958 count |= current; \ 3959 } 3960 3961 /* 3962 ** ixv_update_stats - Update the board statistics counters. 3963 */ 3964 void 3965 ixv_update_stats(struct adapter *adapter) 3966 { 3967 struct ixgbe_hw *hw = &adapter->hw; 3968 3969 UPDATE_STAT_32(IXGBE_VFGPRC, adapter->stats.last_vfgprc, 3970 adapter->stats.vfgprc); 3971 UPDATE_STAT_32(IXGBE_VFGPTC, adapter->stats.last_vfgptc, 3972 adapter->stats.vfgptc); 3973 UPDATE_STAT_36(IXGBE_VFGORC_LSB, IXGBE_VFGORC_MSB, 3974 adapter->stats.last_vfgorc, adapter->stats.vfgorc); 3975 UPDATE_STAT_36(IXGBE_VFGOTC_LSB, IXGBE_VFGOTC_MSB, 3976 adapter->stats.last_vfgotc, adapter->stats.vfgotc); 3977 UPDATE_STAT_32(IXGBE_VFMPRC, adapter->stats.last_vfmprc, 3978 adapter->stats.vfmprc); 3979 } 3980 3981 /********************************************************************** 3982 * 3983 * This routine is called only when ixgbe_display_debug_stats is enabled. 3984 * This routine provides a way to take a look at important statistics 3985 * maintained by the driver and hardware. 3986 * 3987 **********************************************************************/ 3988 static void 3989 ixv_print_hw_stats(struct adapter * adapter) 3990 { 3991 device_t dev = adapter->dev; 3992 3993 device_printf(dev,"Std Mbuf Failed = %"PRIu64"\n", 3994 adapter->mbuf_defrag_failed.ev_count); 3995 device_printf(dev,"Driver dropped packets = %"PRIu64"\n", 3996 adapter->dropped_pkts.ev_count); 3997 device_printf(dev, "watchdog timeouts = %"PRIu64"\n", 3998 adapter->watchdog_events.ev_count); 3999 4000 device_printf(dev,"Good Packets Rcvd = %lld\n", 4001 (long long)adapter->stats.vfgprc); 4002 device_printf(dev,"Good Packets Xmtd = %lld\n", 4003 (long long)adapter->stats.vfgptc); 4004 device_printf(dev,"TSO Transmissions = %"PRIu64"\n", 4005 adapter->tso_tx.ev_count); 4006 4007 } 4008 4009 /********************************************************************** 4010 * 4011 * This routine is called only when em_display_debug_stats is enabled. 4012 * This routine provides a way to take a look at important statistics 4013 * maintained by the driver and hardware. 4014 * 4015 **********************************************************************/ 4016 static void 4017 ixv_print_debug_info(struct adapter *adapter) 4018 { 4019 device_t dev = adapter->dev; 4020 struct ixgbe_hw *hw = &adapter->hw; 4021 struct ix_queue *que = adapter->queues; 4022 struct rx_ring *rxr; 4023 struct tx_ring *txr; 4024 #ifdef LRO 4025 struct lro_ctrl *lro; 4026 #endif /* LRO */ 4027 4028 device_printf(dev,"Error Byte Count = %u \n", 4029 IXGBE_READ_REG(hw, IXGBE_ERRBC)); 4030 4031 for (int i = 0; i < adapter->num_queues; i++, que++) { 4032 txr = que->txr; 4033 rxr = que->rxr; 4034 #ifdef LRO 4035 lro = &rxr->lro; 4036 #endif /* LRO */ 4037 device_printf(dev,"QUE(%d) IRQs Handled: %lu\n", 4038 que->msix, (long)que->irqs); 4039 device_printf(dev,"RX(%d) Packets Received: %lld\n", 4040 rxr->me, (long long)rxr->rx_packets.ev_count); 4041 device_printf(dev,"RX(%d) Split RX Packets: %lld\n", 4042 rxr->me, (long long)rxr->rx_split_packets.ev_count); 4043 device_printf(dev,"RX(%d) Bytes Received: %lu\n", 4044 rxr->me, (long)rxr->rx_bytes.ev_count); 4045 #ifdef LRO 4046 device_printf(dev,"RX(%d) LRO Queued= %d\n", 4047 rxr->me, lro->lro_queued); 4048 device_printf(dev,"RX(%d) LRO Flushed= %d\n", 4049 rxr->me, lro->lro_flushed); 4050 #endif /* LRO */ 4051 device_printf(dev,"TX(%d) Packets Sent: %lu\n", 4052 txr->me, (long)txr->total_packets.ev_count); 4053 device_printf(dev,"TX(%d) NO Desc Avail: %lu\n", 4054 txr->me, (long)txr->no_desc_avail.ev_count); 4055 } 4056 4057 device_printf(dev,"MBX IRQ Handled: %lu\n", 4058 (long)adapter->mbx_irq.ev_count); 4059 return; 4060 } 4061 4062 static int 4063 ixv_sysctl_stats(SYSCTLFN_ARGS) 4064 { 4065 struct sysctlnode node; 4066 int error; 4067 int result; 4068 struct adapter *adapter; 4069 4070 node = *rnode; 4071 adapter = (struct adapter *)node.sysctl_data; 4072 node.sysctl_data = &result; 4073 error = sysctl_lookup(SYSCTLFN_CALL(&node)); 4074 if (error != 0) 4075 return error; 4076 4077 if (result == 1) 4078 ixv_print_hw_stats(adapter); 4079 4080 return 0; 4081 } 4082 4083 static int 4084 ixv_sysctl_debug(SYSCTLFN_ARGS) 4085 { 4086 struct sysctlnode node; 4087 int error, result; 4088 struct adapter *adapter; 4089 4090 node = *rnode; 4091 adapter = (struct adapter *)node.sysctl_data; 4092 node.sysctl_data = &result; 4093 error = sysctl_lookup(SYSCTLFN_CALL(&node)); 4094 4095 if (error) 4096 return error; 4097 4098 if (result == 1) 4099 ixv_print_debug_info(adapter); 4100 4101 return 0; 4102 } 4103 4104 /* 4105 ** Set flow control using sysctl: 4106 ** Flow control values: 4107 ** 0 - off 4108 ** 1 - rx pause 4109 ** 2 - tx pause 4110 ** 3 - full 4111 */ 4112 static int 4113 ixv_set_flowcntl(SYSCTLFN_ARGS) 4114 { 4115 struct sysctlnode node; 4116 int error; 4117 struct adapter *adapter; 4118 4119 node = *rnode; 4120 adapter = (struct adapter *)node.sysctl_data; 4121 node.sysctl_data = &ixv_flow_control; 4122 error = sysctl_lookup(SYSCTLFN_CALL(&node)); 4123 4124 if (error) 4125 return (error); 4126 4127 switch (ixv_flow_control) { 4128 case ixgbe_fc_rx_pause: 4129 case ixgbe_fc_tx_pause: 4130 case ixgbe_fc_full: 4131 adapter->hw.fc.requested_mode = ixv_flow_control; 4132 break; 4133 case ixgbe_fc_none: 4134 default: 4135 adapter->hw.fc.requested_mode = ixgbe_fc_none; 4136 } 4137 4138 ixgbe_fc_enable(&adapter->hw); 4139 return error; 4140 } 4141 4142 const struct sysctlnode * 4143 ixv_sysctl_instance(struct adapter *adapter) 4144 { 4145 const char *dvname; 4146 struct sysctllog **log; 4147 int rc; 4148 const struct sysctlnode *rnode; 4149 4150 log = &adapter->sysctllog; 4151 dvname = device_xname(adapter->dev); 4152 4153 if ((rc = sysctl_createv(log, 0, NULL, &rnode, 4154 0, CTLTYPE_NODE, dvname, 4155 SYSCTL_DESCR("ixv information and settings"), 4156 NULL, 0, NULL, 0, CTL_HW, CTL_CREATE, CTL_EOL)) != 0) 4157 goto err; 4158 4159 return rnode; 4160 err: 4161 printf("%s: sysctl_createv failed, rc = %d\n", __func__, rc); 4162 return NULL; 4163 } 4164 4165 static void 4166 ixv_add_rx_process_limit(struct adapter *adapter, const char *name, 4167 const char *description, int *limit, int value) 4168 { 4169 const struct sysctlnode *rnode, *cnode; 4170 struct sysctllog **log = &adapter->sysctllog; 4171 4172 *limit = value; 4173 4174 if ((rnode = ixv_sysctl_instance(adapter)) == NULL) 4175 aprint_error_dev(adapter->dev, 4176 "could not create sysctl root\n"); 4177 else if (sysctl_createv(log, 0, &rnode, &cnode, 4178 CTLFLAG_READWRITE, 4179 CTLTYPE_INT, 4180 name, SYSCTL_DESCR(description), 4181 NULL, 0, limit, 0, 4182 CTL_CREATE, CTL_EOL) != 0) { 4183 aprint_error_dev(adapter->dev, "%s: could not create sysctl", 4184 __func__); 4185 } 4186 } 4187 4188
Indexes created Tue Sep 30 20:09:53 GMT 2025