ixv.c revision 1.90
1 /*$NetBSD: ixv.c,v 1.90 2018/03/30 03:58:20 knakahara Exp $*/ 2 3 /****************************************************************************** 4 5 Copyright (c) 2001-2017, Intel Corporation 6 All rights reserved. 7 8 Redistribution and use in source and binary forms, with or without 9 modification, are permitted provided that the following conditions are met: 10 11 1. Redistributions of source code must retain the above copyright notice, 12 this list of conditions and the following disclaimer. 13 14 2. Redistributions in binary form must reproduce the above copyright 15 notice, this list of conditions and the following disclaimer in the 16 documentation and/or other materials provided with the distribution. 17 18 3. Neither the name of the Intel Corporation nor the names of its 19 contributors may be used to endorse or promote products derived from 20 this software without specific prior written permission. 21 22 THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 23 AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 24 IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 25 ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE 26 LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 27 CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 28 SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 29 INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 30 CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 31 ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 32 POSSIBILITY OF SUCH DAMAGE. 33 34 ******************************************************************************/ 35 /*$FreeBSD: head/sys/dev/ixgbe/if_ixv.c 320688 2017-07-05 17:27:03Z erj $*/ 36 37 38 #ifdef _KERNEL_OPT 39 #include "opt_inet.h" 40 #include "opt_inet6.h" 41 #include "opt_net_mpsafe.h" 42 #endif 43 44 #include "ixgbe.h" 45 #include "vlan.h" 46 47 /************************************************************************ 48 * Driver version 49 ************************************************************************/ 50 char ixv_driver_version[] = "1.5.13-k"; 51 52 /************************************************************************ 53 * PCI Device ID Table 54 * 55 * Used by probe to select devices to load on 56 * Last field stores an index into ixv_strings 57 * Last entry must be all 0s 58 * 59 * { Vendor ID, Device ID, SubVendor ID, SubDevice ID, String Index } 60 ************************************************************************/ 61 static ixgbe_vendor_info_t ixv_vendor_info_array[] = 62 { 63 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_VF, 0, 0, 0}, 64 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_X540_VF, 0, 0, 0}, 65 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_X550_VF, 0, 0, 0}, 66 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_X550EM_X_VF, 0, 0, 0}, 67 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_X550EM_A_VF, 0, 0, 0}, 68 /* required last entry */ 69 {0, 0, 0, 0, 0} 70 }; 71 72 /************************************************************************ 73 * Table of branding strings 74 ************************************************************************/ 75 static const char *ixv_strings[] = { 76 "Intel(R) PRO/10GbE Virtual Function Network Driver" 77 }; 78 79 /********************************************************************* 80 * Function prototypes 81 *********************************************************************/ 82 static int ixv_probe(device_t, cfdata_t, void *); 83 static void ixv_attach(device_t, device_t, void *); 84 static int ixv_detach(device_t, int); 85 #if 0 86 static int ixv_shutdown(device_t); 87 #endif 88 static int ixv_ifflags_cb(struct ethercom *); 89 static int ixv_ioctl(struct ifnet *, u_long, void *); 90 static int ixv_init(struct ifnet *); 91 static void ixv_init_locked(struct adapter *); 92 static void ixv_ifstop(struct ifnet *, int); 93 static void ixv_stop(void *); 94 static void ixv_init_device_features(struct adapter *); 95 static void ixv_media_status(struct ifnet *, struct ifmediareq *); 96 static int ixv_media_change(struct ifnet *); 97 static int ixv_allocate_pci_resources(struct adapter *, 98 const struct pci_attach_args *); 99 static int ixv_allocate_msix(struct adapter *, 100 const struct pci_attach_args *); 101 static int ixv_configure_interrupts(struct adapter *); 102 static void ixv_free_pci_resources(struct adapter *); 103 static void ixv_local_timer(void *); 104 static void ixv_local_timer_locked(void *); 105 static int ixv_setup_interface(device_t, struct adapter *); 106 static int ixv_negotiate_api(struct adapter *); 107 108 static void ixv_initialize_transmit_units(struct adapter *); 109 static void ixv_initialize_receive_units(struct adapter *); 110 static void ixv_initialize_rss_mapping(struct adapter *); 111 static void ixv_check_link(struct adapter *); 112 113 static void ixv_enable_intr(struct adapter *); 114 static void ixv_disable_intr(struct adapter *); 115 static void ixv_set_multi(struct adapter *); 116 static void ixv_update_link_status(struct adapter *); 117 static int ixv_sysctl_debug(SYSCTLFN_PROTO); 118 static void ixv_set_ivar(struct adapter *, u8, u8, s8); 119 static void ixv_configure_ivars(struct adapter *); 120 static u8 * ixv_mc_array_itr(struct ixgbe_hw *, u8 **, u32 *); 121 static void ixv_eitr_write(struct ix_queue *, uint32_t); 122 123 static void ixv_setup_vlan_support(struct adapter *); 124 #if 0 125 static void ixv_register_vlan(void *, struct ifnet *, u16); 126 static void ixv_unregister_vlan(void *, struct ifnet *, u16); 127 #endif 128 129 static void ixv_add_device_sysctls(struct adapter *); 130 static void ixv_save_stats(struct adapter *); 131 static void ixv_init_stats(struct adapter *); 132 static void ixv_update_stats(struct adapter *); 133 static void ixv_add_stats_sysctls(struct adapter *); 134 135 136 /* Sysctl handlers */ 137 static void ixv_set_sysctl_value(struct adapter *, const char *, 138 const char *, int *, int); 139 static int ixv_sysctl_interrupt_rate_handler(SYSCTLFN_PROTO); 140 static int ixv_sysctl_rdh_handler(SYSCTLFN_PROTO); 141 static int ixv_sysctl_rdt_handler(SYSCTLFN_PROTO); 142 static int ixv_sysctl_tdt_handler(SYSCTLFN_PROTO); 143 static int ixv_sysctl_tdh_handler(SYSCTLFN_PROTO); 144 145 /* The MSI-X Interrupt handlers */ 146 static int ixv_msix_que(void *); 147 static int ixv_msix_mbx(void *); 148 149 /* Deferred interrupt tasklets */ 150 static void ixv_handle_que(void *); 151 static void ixv_handle_link(void *); 152 153 /* Workqueue handler for deferred work */ 154 static void ixv_handle_que_work(struct work *, void *); 155 156 const struct sysctlnode *ixv_sysctl_instance(struct adapter *); 157 static ixgbe_vendor_info_t *ixv_lookup(const struct pci_attach_args *); 158 159 /************************************************************************ 160 * FreeBSD Device Interface Entry Points 161 ************************************************************************/ 162 CFATTACH_DECL3_NEW(ixv, sizeof(struct adapter), 163 ixv_probe, ixv_attach, ixv_detach, NULL, NULL, NULL, 164 DVF_DETACH_SHUTDOWN); 165 166 #if 0 167 static driver_t ixv_driver = { 168 "ixv", ixv_methods, sizeof(struct adapter), 169 }; 170 171 devclass_t ixv_devclass; 172 DRIVER_MODULE(ixv, pci, ixv_driver, ixv_devclass, 0, 0); 173 MODULE_DEPEND(ixv, pci, 1, 1, 1); 174 MODULE_DEPEND(ixv, ether, 1, 1, 1); 175 #endif 176 177 /* 178 * TUNEABLE PARAMETERS: 179 */ 180 181 /* Number of Queues - do not exceed MSI-X vectors - 1 */ 182 static int ixv_num_queues = 0; 183 #define TUNABLE_INT(__x, __y) 184 TUNABLE_INT("hw.ixv.num_queues", &ixv_num_queues); 185 186 /* 187 * AIM: Adaptive Interrupt Moderation 188 * which means that the interrupt rate 189 * is varied over time based on the 190 * traffic for that interrupt vector 191 */ 192 static bool ixv_enable_aim = false; 193 TUNABLE_INT("hw.ixv.enable_aim", &ixv_enable_aim); 194 195 static int ixv_max_interrupt_rate = (4000000 / IXGBE_LOW_LATENCY); 196 TUNABLE_INT("hw.ixv.max_interrupt_rate", &ixv_max_interrupt_rate); 197 198 /* How many packets rxeof tries to clean at a time */ 199 static int ixv_rx_process_limit = 256; 200 TUNABLE_INT("hw.ixv.rx_process_limit", &ixv_rx_process_limit); 201 202 /* How many packets txeof tries to clean at a time */ 203 static int ixv_tx_process_limit = 256; 204 TUNABLE_INT("hw.ixv.tx_process_limit", &ixv_tx_process_limit); 205 206 /* Which pakcet processing uses workqueue or softint */ 207 static bool ixv_txrx_workqueue = false; 208 209 /* 210 * Number of TX descriptors per ring, 211 * setting higher than RX as this seems 212 * the better performing choice. 213 */ 214 static int ixv_txd = PERFORM_TXD; 215 TUNABLE_INT("hw.ixv.txd", &ixv_txd); 216 217 /* Number of RX descriptors per ring */ 218 static int ixv_rxd = PERFORM_RXD; 219 TUNABLE_INT("hw.ixv.rxd", &ixv_rxd); 220 221 /* Legacy Transmit (single queue) */ 222 static int ixv_enable_legacy_tx = 0; 223 TUNABLE_INT("hw.ixv.enable_legacy_tx", &ixv_enable_legacy_tx); 224 225 #ifdef NET_MPSAFE 226 #define IXGBE_MPSAFE 1 227 #define IXGBE_CALLOUT_FLAGS CALLOUT_MPSAFE 228 #define IXGBE_SOFTINFT_FLAGS SOFTINT_MPSAFE 229 #define IXGBE_WORKQUEUE_FLAGS WQ_PERCPU | WQ_MPSAFE 230 #else 231 #define IXGBE_CALLOUT_FLAGS 0 232 #define IXGBE_SOFTINFT_FLAGS 0 233 #define IXGBE_WORKQUEUE_FLAGS WQ_PERCPU 234 #endif 235 #define IXGBE_WORKQUEUE_PRI PRI_SOFTNET 236 237 #if 0 238 static int (*ixv_start_locked)(struct ifnet *, struct tx_ring *); 239 static int (*ixv_ring_empty)(struct ifnet *, struct buf_ring *); 240 #endif 241 242 /************************************************************************ 243 * ixv_probe - Device identification routine 244 * 245 * Determines if the driver should be loaded on 246 * adapter based on its PCI vendor/device ID. 247 * 248 * return BUS_PROBE_DEFAULT on success, positive on failure 249 ************************************************************************/ 250 static int 251 ixv_probe(device_t dev, cfdata_t cf, void *aux) 252 { 253 #ifdef __HAVE_PCI_MSI_MSIX 254 const struct pci_attach_args *pa = aux; 255 256 return (ixv_lookup(pa) != NULL) ? 1 : 0; 257 #else 258 return 0; 259 #endif 260 } /* ixv_probe */ 261 262 static ixgbe_vendor_info_t * 263 ixv_lookup(const struct pci_attach_args *pa) 264 { 265 ixgbe_vendor_info_t *ent; 266 pcireg_t subid; 267 268 INIT_DEBUGOUT("ixv_lookup: begin"); 269 270 if (PCI_VENDOR(pa->pa_id) != IXGBE_INTEL_VENDOR_ID) 271 return NULL; 272 273 subid = pci_conf_read(pa->pa_pc, pa->pa_tag, PCI_SUBSYS_ID_REG); 274 275 for (ent = ixv_vendor_info_array; ent->vendor_id != 0; ent++) { 276 if ((PCI_VENDOR(pa->pa_id) == ent->vendor_id) && 277 (PCI_PRODUCT(pa->pa_id) == ent->device_id) && 278 ((PCI_SUBSYS_VENDOR(subid) == ent->subvendor_id) || 279 (ent->subvendor_id == 0)) && 280 ((PCI_SUBSYS_ID(subid) == ent->subdevice_id) || 281 (ent->subdevice_id == 0))) { 282 return ent; 283 } 284 } 285 286 return NULL; 287 } 288 289 /************************************************************************ 290 * ixv_attach - Device initialization routine 291 * 292 * Called when the driver is being loaded. 293 * Identifies the type of hardware, allocates all resources 294 * and initializes the hardware. 295 * 296 * return 0 on success, positive on failure 297 ************************************************************************/ 298 static void 299 ixv_attach(device_t parent, device_t dev, void *aux) 300 { 301 struct adapter *adapter; 302 struct ixgbe_hw *hw; 303 int error = 0; 304 pcireg_t id, subid; 305 ixgbe_vendor_info_t *ent; 306 const struct pci_attach_args *pa = aux; 307 const char *apivstr; 308 const char *str; 309 char buf[256]; 310 311 INIT_DEBUGOUT("ixv_attach: begin"); 312 313 /* 314 * Make sure BUSMASTER is set, on a VM under 315 * KVM it may not be and will break things. 316 */ 317 ixgbe_pci_enable_busmaster(pa->pa_pc, pa->pa_tag); 318 319 /* Allocate, clear, and link in our adapter structure */ 320 adapter = device_private(dev); 321 adapter->dev = dev; 322 adapter->hw.back = adapter; 323 hw = &adapter->hw; 324 325 adapter->init_locked = ixv_init_locked; 326 adapter->stop_locked = ixv_stop; 327 328 adapter->osdep.pc = pa->pa_pc; 329 adapter->osdep.tag = pa->pa_tag; 330 if (pci_dma64_available(pa)) 331 adapter->osdep.dmat = pa->pa_dmat64; 332 else 333 adapter->osdep.dmat = pa->pa_dmat; 334 adapter->osdep.attached = false; 335 336 ent = ixv_lookup(pa); 337 338 KASSERT(ent != NULL); 339 340 aprint_normal(": %s, Version - %s\n", 341 ixv_strings[ent->index], ixv_driver_version); 342 343 /* Core Lock Init*/ 344 IXGBE_CORE_LOCK_INIT(adapter, device_xname(dev)); 345 346 /* Do base PCI setup - map BAR0 */ 347 if (ixv_allocate_pci_resources(adapter, pa)) { 348 aprint_error_dev(dev, "ixv_allocate_pci_resources() failed!\n"); 349 error = ENXIO; 350 goto err_out; 351 } 352 353 /* SYSCTL APIs */ 354 ixv_add_device_sysctls(adapter); 355 356 /* Set up the timer callout */ 357 callout_init(&adapter->timer, IXGBE_CALLOUT_FLAGS); 358 359 /* Save off the information about this board */ 360 id = pci_conf_read(pa->pa_pc, pa->pa_tag, PCI_ID_REG); 361 subid = pci_conf_read(pa->pa_pc, pa->pa_tag, PCI_SUBSYS_ID_REG); 362 hw->vendor_id = PCI_VENDOR(id); 363 hw->device_id = PCI_PRODUCT(id); 364 hw->revision_id = 365 PCI_REVISION(pci_conf_read(pa->pa_pc, pa->pa_tag, PCI_CLASS_REG)); 366 hw->subsystem_vendor_id = PCI_SUBSYS_VENDOR(subid); 367 hw->subsystem_device_id = PCI_SUBSYS_ID(subid); 368 369 /* A subset of set_mac_type */ 370 switch (hw->device_id) { 371 case IXGBE_DEV_ID_82599_VF: 372 hw->mac.type = ixgbe_mac_82599_vf; 373 str = "82599 VF"; 374 break; 375 case IXGBE_DEV_ID_X540_VF: 376 hw->mac.type = ixgbe_mac_X540_vf; 377 str = "X540 VF"; 378 break; 379 case IXGBE_DEV_ID_X550_VF: 380 hw->mac.type = ixgbe_mac_X550_vf; 381 str = "X550 VF"; 382 break; 383 case IXGBE_DEV_ID_X550EM_X_VF: 384 hw->mac.type = ixgbe_mac_X550EM_x_vf; 385 str = "X550EM X VF"; 386 break; 387 case IXGBE_DEV_ID_X550EM_A_VF: 388 hw->mac.type = ixgbe_mac_X550EM_a_vf; 389 str = "X550EM A VF"; 390 break; 391 default: 392 /* Shouldn't get here since probe succeeded */ 393 aprint_error_dev(dev, "Unknown device ID!\n"); 394 error = ENXIO; 395 goto err_out; 396 break; 397 } 398 aprint_normal_dev(dev, "device %s\n", str); 399 400 ixv_init_device_features(adapter); 401 402 /* Initialize the shared code */ 403 error = ixgbe_init_ops_vf(hw); 404 if (error) { 405 aprint_error_dev(dev, "ixgbe_init_ops_vf() failed!\n"); 406 error = EIO; 407 goto err_out; 408 } 409 410 /* Setup the mailbox */ 411 ixgbe_init_mbx_params_vf(hw); 412 413 /* Set the right number of segments */ 414 adapter->num_segs = IXGBE_82599_SCATTER; 415 416 /* Reset mbox api to 1.0 */ 417 error = hw->mac.ops.reset_hw(hw); 418 if (error == IXGBE_ERR_RESET_FAILED) 419 aprint_error_dev(dev, "...reset_hw() failure: Reset Failed!\n"); 420 else if (error) 421 aprint_error_dev(dev, "...reset_hw() failed with error %d\n", 422 error); 423 if (error) { 424 error = EIO; 425 goto err_out; 426 } 427 428 error = hw->mac.ops.init_hw(hw); 429 if (error) { 430 aprint_error_dev(dev, "...init_hw() failed!\n"); 431 error = EIO; 432 goto err_out; 433 } 434 435 /* Negotiate mailbox API version */ 436 error = ixv_negotiate_api(adapter); 437 if (error) 438 aprint_normal_dev(dev, 439 "MBX API negotiation failed during attach!\n"); 440 switch (hw->api_version) { 441 case ixgbe_mbox_api_10: 442 apivstr = "1.0"; 443 break; 444 case ixgbe_mbox_api_20: 445 apivstr = "2.0"; 446 break; 447 case ixgbe_mbox_api_11: 448 apivstr = "1.1"; 449 break; 450 case ixgbe_mbox_api_12: 451 apivstr = "1.2"; 452 break; 453 case ixgbe_mbox_api_13: 454 apivstr = "1.3"; 455 break; 456 default: 457 apivstr = "unknown"; 458 break; 459 } 460 aprint_normal_dev(dev, "Mailbox API %s\n", apivstr); 461 462 /* If no mac address was assigned, make a random one */ 463 if (!ixv_check_ether_addr(hw->mac.addr)) { 464 u8 addr[ETHER_ADDR_LEN]; 465 uint64_t rndval = cprng_strong64(); 466 467 memcpy(addr, &rndval, sizeof(addr)); 468 addr[0] &= 0xFE; 469 addr[0] |= 0x02; 470 bcopy(addr, hw->mac.addr, sizeof(addr)); 471 } 472 473 /* Register for VLAN events */ 474 #if 0 /* XXX delete after write? */ 475 adapter->vlan_attach = EVENTHANDLER_REGISTER(vlan_config, 476 ixv_register_vlan, adapter, EVENTHANDLER_PRI_FIRST); 477 adapter->vlan_detach = EVENTHANDLER_REGISTER(vlan_unconfig, 478 ixv_unregister_vlan, adapter, EVENTHANDLER_PRI_FIRST); 479 #endif 480 481 /* Sysctls for limiting the amount of work done in the taskqueues */ 482 ixv_set_sysctl_value(adapter, "rx_processing_limit", 483 "max number of rx packets to process", 484 &adapter->rx_process_limit, ixv_rx_process_limit); 485 486 ixv_set_sysctl_value(adapter, "tx_processing_limit", 487 "max number of tx packets to process", 488 &adapter->tx_process_limit, ixv_tx_process_limit); 489 490 /* Do descriptor calc and sanity checks */ 491 if (((ixv_txd * sizeof(union ixgbe_adv_tx_desc)) % DBA_ALIGN) != 0 || 492 ixv_txd < MIN_TXD || ixv_txd > MAX_TXD) { 493 aprint_error_dev(dev, "TXD config issue, using default!\n"); 494 adapter->num_tx_desc = DEFAULT_TXD; 495 } else 496 adapter->num_tx_desc = ixv_txd; 497 498 if (((ixv_rxd * sizeof(union ixgbe_adv_rx_desc)) % DBA_ALIGN) != 0 || 499 ixv_rxd < MIN_RXD || ixv_rxd > MAX_RXD) { 500 aprint_error_dev(dev, "RXD config issue, using default!\n"); 501 adapter->num_rx_desc = DEFAULT_RXD; 502 } else 503 adapter->num_rx_desc = ixv_rxd; 504 505 /* Setup MSI-X */ 506 error = ixv_configure_interrupts(adapter); 507 if (error) 508 goto err_out; 509 510 /* Allocate our TX/RX Queues */ 511 if (ixgbe_allocate_queues(adapter)) { 512 aprint_error_dev(dev, "ixgbe_allocate_queues() failed!\n"); 513 error = ENOMEM; 514 goto err_out; 515 } 516 517 /* hw.ix defaults init */ 518 adapter->enable_aim = ixv_enable_aim; 519 520 adapter->txrx_use_workqueue = ixv_txrx_workqueue; 521 522 error = ixv_allocate_msix(adapter, pa); 523 if (error) { 524 device_printf(dev, "ixv_allocate_msix() failed!\n"); 525 goto err_late; 526 } 527 528 /* Setup OS specific network interface */ 529 error = ixv_setup_interface(dev, adapter); 530 if (error != 0) { 531 aprint_error_dev(dev, "ixv_setup_interface() failed!\n"); 532 goto err_late; 533 } 534 535 /* Do the stats setup */ 536 ixv_save_stats(adapter); 537 ixv_init_stats(adapter); 538 ixv_add_stats_sysctls(adapter); 539 540 if (adapter->feat_en & IXGBE_FEATURE_NETMAP) 541 ixgbe_netmap_attach(adapter); 542 543 snprintb(buf, sizeof(buf), IXGBE_FEATURE_FLAGS, adapter->feat_cap); 544 aprint_verbose_dev(dev, "feature cap %s\n", buf); 545 snprintb(buf, sizeof(buf), IXGBE_FEATURE_FLAGS, adapter->feat_en); 546 aprint_verbose_dev(dev, "feature ena %s\n", buf); 547 548 INIT_DEBUGOUT("ixv_attach: end"); 549 adapter->osdep.attached = true; 550 551 return; 552 553 err_late: 554 ixgbe_free_transmit_structures(adapter); 555 ixgbe_free_receive_structures(adapter); 556 free(adapter->queues, M_DEVBUF); 557 err_out: 558 ixv_free_pci_resources(adapter); 559 IXGBE_CORE_LOCK_DESTROY(adapter); 560 561 return; 562 } /* ixv_attach */ 563 564 /************************************************************************ 565 * ixv_detach - Device removal routine 566 * 567 * Called when the driver is being removed. 568 * Stops the adapter and deallocates all the resources 569 * that were allocated for driver operation. 570 * 571 * return 0 on success, positive on failure 572 ************************************************************************/ 573 static int 574 ixv_detach(device_t dev, int flags) 575 { 576 struct adapter *adapter = device_private(dev); 577 struct ixgbe_hw *hw = &adapter->hw; 578 struct ix_queue *que = adapter->queues; 579 struct tx_ring *txr = adapter->tx_rings; 580 struct rx_ring *rxr = adapter->rx_rings; 581 struct ixgbevf_hw_stats *stats = &adapter->stats.vf; 582 583 INIT_DEBUGOUT("ixv_detach: begin"); 584 if (adapter->osdep.attached == false) 585 return 0; 586 587 /* Stop the interface. Callouts are stopped in it. */ 588 ixv_ifstop(adapter->ifp, 1); 589 590 #if NVLAN > 0 591 /* Make sure VLANs are not using driver */ 592 if (!VLAN_ATTACHED(&adapter->osdep.ec)) 593 ; /* nothing to do: no VLANs */ 594 else if ((flags & (DETACH_SHUTDOWN|DETACH_FORCE)) != 0) 595 vlan_ifdetach(adapter->ifp); 596 else { 597 aprint_error_dev(dev, "VLANs in use, detach first\n"); 598 return EBUSY; 599 } 600 #endif 601 602 IXGBE_CORE_LOCK(adapter); 603 ixv_stop(adapter); 604 IXGBE_CORE_UNLOCK(adapter); 605 606 for (int i = 0; i < adapter->num_queues; i++, que++, txr++) { 607 if (!(adapter->feat_en & IXGBE_FEATURE_LEGACY_TX)) 608 softint_disestablish(txr->txr_si); 609 softint_disestablish(que->que_si); 610 } 611 if (adapter->txr_wq != NULL) 612 workqueue_destroy(adapter->txr_wq); 613 if (adapter->txr_wq_enqueued != NULL) 614 percpu_free(adapter->txr_wq_enqueued, sizeof(u_int)); 615 if (adapter->que_wq != NULL) 616 workqueue_destroy(adapter->que_wq); 617 618 /* Drain the Mailbox(link) queue */ 619 softint_disestablish(adapter->link_si); 620 621 /* Unregister VLAN events */ 622 #if 0 /* XXX msaitoh delete after write? */ 623 if (adapter->vlan_attach != NULL) 624 EVENTHANDLER_DEREGISTER(vlan_config, adapter->vlan_attach); 625 if (adapter->vlan_detach != NULL) 626 EVENTHANDLER_DEREGISTER(vlan_unconfig, adapter->vlan_detach); 627 #endif 628 629 ether_ifdetach(adapter->ifp); 630 callout_halt(&adapter->timer, NULL); 631 632 if (adapter->feat_en & IXGBE_FEATURE_NETMAP) 633 netmap_detach(adapter->ifp); 634 635 ixv_free_pci_resources(adapter); 636 #if 0 /* XXX the NetBSD port is probably missing something here */ 637 bus_generic_detach(dev); 638 #endif 639 if_detach(adapter->ifp); 640 if_percpuq_destroy(adapter->ipq); 641 642 sysctl_teardown(&adapter->sysctllog); 643 evcnt_detach(&adapter->efbig_tx_dma_setup); 644 evcnt_detach(&adapter->mbuf_defrag_failed); 645 evcnt_detach(&adapter->efbig2_tx_dma_setup); 646 evcnt_detach(&adapter->einval_tx_dma_setup); 647 evcnt_detach(&adapter->other_tx_dma_setup); 648 evcnt_detach(&adapter->eagain_tx_dma_setup); 649 evcnt_detach(&adapter->enomem_tx_dma_setup); 650 evcnt_detach(&adapter->watchdog_events); 651 evcnt_detach(&adapter->tso_err); 652 evcnt_detach(&adapter->link_irq); 653 654 txr = adapter->tx_rings; 655 for (int i = 0; i < adapter->num_queues; i++, rxr++, txr++) { 656 evcnt_detach(&adapter->queues[i].irqs); 657 evcnt_detach(&adapter->queues[i].handleq); 658 evcnt_detach(&adapter->queues[i].req); 659 evcnt_detach(&txr->no_desc_avail); 660 evcnt_detach(&txr->total_packets); 661 evcnt_detach(&txr->tso_tx); 662 #ifndef IXGBE_LEGACY_TX 663 evcnt_detach(&txr->pcq_drops); 664 #endif 665 666 evcnt_detach(&rxr->rx_packets); 667 evcnt_detach(&rxr->rx_bytes); 668 evcnt_detach(&rxr->rx_copies); 669 evcnt_detach(&rxr->no_jmbuf); 670 evcnt_detach(&rxr->rx_discarded); 671 } 672 evcnt_detach(&stats->ipcs); 673 evcnt_detach(&stats->l4cs); 674 evcnt_detach(&stats->ipcs_bad); 675 evcnt_detach(&stats->l4cs_bad); 676 677 /* Packet Reception Stats */ 678 evcnt_detach(&stats->vfgorc); 679 evcnt_detach(&stats->vfgprc); 680 evcnt_detach(&stats->vfmprc); 681 682 /* Packet Transmission Stats */ 683 evcnt_detach(&stats->vfgotc); 684 evcnt_detach(&stats->vfgptc); 685 686 /* Mailbox Stats */ 687 evcnt_detach(&hw->mbx.stats.msgs_tx); 688 evcnt_detach(&hw->mbx.stats.msgs_rx); 689 evcnt_detach(&hw->mbx.stats.acks); 690 evcnt_detach(&hw->mbx.stats.reqs); 691 evcnt_detach(&hw->mbx.stats.rsts); 692 693 ixgbe_free_transmit_structures(adapter); 694 ixgbe_free_receive_structures(adapter); 695 for (int i = 0; i < adapter->num_queues; i++) { 696 struct ix_queue *lque = &adapter->queues[i]; 697 mutex_destroy(&lque->dc_mtx); 698 } 699 free(adapter->queues, M_DEVBUF); 700 701 IXGBE_CORE_LOCK_DESTROY(adapter); 702 703 return (0); 704 } /* ixv_detach */ 705 706 /************************************************************************ 707 * ixv_init_locked - Init entry point 708 * 709 * Used in two ways: It is used by the stack as an init entry 710 * point in network interface structure. It is also used 711 * by the driver as a hw/sw initialization routine to get 712 * to a consistent state. 713 * 714 * return 0 on success, positive on failure 715 ************************************************************************/ 716 static void 717 ixv_init_locked(struct adapter *adapter) 718 { 719 struct ifnet *ifp = adapter->ifp; 720 device_t dev = adapter->dev; 721 struct ixgbe_hw *hw = &adapter->hw; 722 struct ix_queue *que = adapter->queues; 723 int error = 0; 724 uint32_t mask; 725 int i; 726 727 INIT_DEBUGOUT("ixv_init_locked: begin"); 728 KASSERT(mutex_owned(&adapter->core_mtx)); 729 hw->adapter_stopped = FALSE; 730 hw->mac.ops.stop_adapter(hw); 731 callout_stop(&adapter->timer); 732 733 /* reprogram the RAR[0] in case user changed it. */ 734 hw->mac.ops.set_rar(hw, 0, hw->mac.addr, 0, IXGBE_RAH_AV); 735 736 /* Get the latest mac address, User can use a LAA */ 737 memcpy(hw->mac.addr, CLLADDR(adapter->ifp->if_sadl), 738 IXGBE_ETH_LENGTH_OF_ADDRESS); 739 hw->mac.ops.set_rar(hw, 0, hw->mac.addr, 0, 1); 740 741 /* Prepare transmit descriptors and buffers */ 742 if (ixgbe_setup_transmit_structures(adapter)) { 743 aprint_error_dev(dev, "Could not setup transmit structures\n"); 744 ixv_stop(adapter); 745 return; 746 } 747 748 /* Reset VF and renegotiate mailbox API version */ 749 hw->mac.ops.reset_hw(hw); 750 error = ixv_negotiate_api(adapter); 751 if (error) 752 device_printf(dev, 753 "Mailbox API negotiation failed in init_locked!\n"); 754 755 ixv_initialize_transmit_units(adapter); 756 757 /* Setup Multicast table */ 758 ixv_set_multi(adapter); 759 760 /* 761 * Determine the correct mbuf pool 762 * for doing jumbo/headersplit 763 */ 764 if (ifp->if_mtu > ETHERMTU) 765 adapter->rx_mbuf_sz = MJUMPAGESIZE; 766 else 767 adapter->rx_mbuf_sz = MCLBYTES; 768 769 /* Prepare receive descriptors and buffers */ 770 if (ixgbe_setup_receive_structures(adapter)) { 771 device_printf(dev, "Could not setup receive structures\n"); 772 ixv_stop(adapter); 773 return; 774 } 775 776 /* Configure RX settings */ 777 ixv_initialize_receive_units(adapter); 778 779 #if 0 /* XXX isn't it required? -- msaitoh */ 780 /* Set the various hardware offload abilities */ 781 ifp->if_hwassist = 0; 782 if (ifp->if_capenable & IFCAP_TSO4) 783 ifp->if_hwassist |= CSUM_TSO; 784 if (ifp->if_capenable & IFCAP_TXCSUM) { 785 ifp->if_hwassist |= (CSUM_TCP | CSUM_UDP); 786 #if __FreeBSD_version >= 800000 787 ifp->if_hwassist |= CSUM_SCTP; 788 #endif 789 } 790 #endif 791 792 /* Set up VLAN offload and filter */ 793 ixv_setup_vlan_support(adapter); 794 795 /* Set up MSI-X routing */ 796 ixv_configure_ivars(adapter); 797 798 /* Set up auto-mask */ 799 mask = (1 << adapter->vector); 800 for (i = 0; i < adapter->num_queues; i++, que++) 801 mask |= (1 << que->msix); 802 IXGBE_WRITE_REG(hw, IXGBE_VTEIAM, mask); 803 804 /* Set moderation on the Link interrupt */ 805 IXGBE_WRITE_REG(hw, IXGBE_VTEITR(adapter->vector), IXGBE_LINK_ITR); 806 807 /* Stats init */ 808 ixv_init_stats(adapter); 809 810 /* Config/Enable Link */ 811 hw->mac.get_link_status = TRUE; 812 hw->mac.ops.check_link(hw, &adapter->link_speed, &adapter->link_up, 813 FALSE); 814 815 /* Start watchdog */ 816 callout_reset(&adapter->timer, hz, ixv_local_timer, adapter); 817 818 /* And now turn on interrupts */ 819 ixv_enable_intr(adapter); 820 821 /* Update saved flags. See ixgbe_ifflags_cb() */ 822 adapter->if_flags = ifp->if_flags; 823 824 /* Now inform the stack we're ready */ 825 ifp->if_flags |= IFF_RUNNING; 826 ifp->if_flags &= ~IFF_OACTIVE; 827 828 return; 829 } /* ixv_init_locked */ 830 831 /************************************************************************ 832 * ixv_enable_queue 833 ************************************************************************/ 834 static inline void 835 ixv_enable_queue(struct adapter *adapter, u32 vector) 836 { 837 struct ixgbe_hw *hw = &adapter->hw; 838 struct ix_queue *que = &adapter->queues[vector]; 839 u32 queue = 1 << vector; 840 u32 mask; 841 842 mutex_enter(&que->dc_mtx); 843 if (que->disabled_count > 0 && --que->disabled_count > 0) 844 goto out; 845 846 mask = (IXGBE_EIMS_RTX_QUEUE & queue); 847 IXGBE_WRITE_REG(hw, IXGBE_VTEIMS, mask); 848 out: 849 mutex_exit(&que->dc_mtx); 850 } /* ixv_enable_queue */ 851 852 /************************************************************************ 853 * ixv_disable_queue 854 ************************************************************************/ 855 static inline void 856 ixv_disable_queue(struct adapter *adapter, u32 vector) 857 { 858 struct ixgbe_hw *hw = &adapter->hw; 859 struct ix_queue *que = &adapter->queues[vector]; 860 u64 queue = (u64)(1 << vector); 861 u32 mask; 862 863 mutex_enter(&que->dc_mtx); 864 if (que->disabled_count++ > 0) 865 goto out; 866 867 mask = (IXGBE_EIMS_RTX_QUEUE & queue); 868 IXGBE_WRITE_REG(hw, IXGBE_VTEIMC, mask); 869 out: 870 mutex_exit(&que->dc_mtx); 871 } /* ixv_disable_queue */ 872 873 static inline void 874 ixv_rearm_queues(struct adapter *adapter, u64 queues) 875 { 876 u32 mask = (IXGBE_EIMS_RTX_QUEUE & queues); 877 IXGBE_WRITE_REG(&adapter->hw, IXGBE_VTEICS, mask); 878 } /* ixv_rearm_queues */ 879 880 881 /************************************************************************ 882 * ixv_msix_que - MSI Queue Interrupt Service routine 883 ************************************************************************/ 884 static int 885 ixv_msix_que(void *arg) 886 { 887 struct ix_queue *que = arg; 888 struct adapter *adapter = que->adapter; 889 struct tx_ring *txr = que->txr; 890 struct rx_ring *rxr = que->rxr; 891 bool more; 892 u32 newitr = 0; 893 894 ixv_disable_queue(adapter, que->msix); 895 ++que->irqs.ev_count; 896 897 #ifdef __NetBSD__ 898 /* Don't run ixgbe_rxeof in interrupt context */ 899 more = true; 900 #else 901 more = ixgbe_rxeof(que); 902 #endif 903 904 IXGBE_TX_LOCK(txr); 905 ixgbe_txeof(txr); 906 IXGBE_TX_UNLOCK(txr); 907 908 /* Do AIM now? */ 909 910 if (adapter->enable_aim == false) 911 goto no_calc; 912 /* 913 * Do Adaptive Interrupt Moderation: 914 * - Write out last calculated setting 915 * - Calculate based on average size over 916 * the last interval. 917 */ 918 if (que->eitr_setting) 919 ixv_eitr_write(que, que->eitr_setting); 920 921 que->eitr_setting = 0; 922 923 /* Idle, do nothing */ 924 if ((txr->bytes == 0) && (rxr->bytes == 0)) 925 goto no_calc; 926 927 if ((txr->bytes) && (txr->packets)) 928 newitr = txr->bytes/txr->packets; 929 if ((rxr->bytes) && (rxr->packets)) 930 newitr = max(newitr, (rxr->bytes / rxr->packets)); 931 newitr += 24; /* account for hardware frame, crc */ 932 933 /* set an upper boundary */ 934 newitr = min(newitr, 3000); 935 936 /* Be nice to the mid range */ 937 if ((newitr > 300) && (newitr < 1200)) 938 newitr = (newitr / 3); 939 else 940 newitr = (newitr / 2); 941 942 /* 943 * When RSC is used, ITR interval must be larger than RSC_DELAY. 944 * Currently, we use 2us for RSC_DELAY. The minimum value is always 945 * greater than 2us on 100M (and 10M?(not documented)), but it's not 946 * on 1G and higher. 947 */ 948 if ((adapter->link_speed != IXGBE_LINK_SPEED_100_FULL) 949 && (adapter->link_speed != IXGBE_LINK_SPEED_10_FULL)) { 950 if (newitr < IXGBE_MIN_RSC_EITR_10G1G) 951 newitr = IXGBE_MIN_RSC_EITR_10G1G; 952 } 953 954 /* save for next interrupt */ 955 que->eitr_setting = newitr; 956 957 /* Reset state */ 958 txr->bytes = 0; 959 txr->packets = 0; 960 rxr->bytes = 0; 961 rxr->packets = 0; 962 963 no_calc: 964 if (more) 965 softint_schedule(que->que_si); 966 else /* Re-enable this interrupt */ 967 ixv_enable_queue(adapter, que->msix); 968 969 return 1; 970 } /* ixv_msix_que */ 971 972 /************************************************************************ 973 * ixv_msix_mbx 974 ************************************************************************/ 975 static int 976 ixv_msix_mbx(void *arg) 977 { 978 struct adapter *adapter = arg; 979 struct ixgbe_hw *hw = &adapter->hw; 980 981 ++adapter->link_irq.ev_count; 982 /* NetBSD: We use auto-clear, so it's not required to write VTEICR */ 983 984 /* Link status change */ 985 hw->mac.get_link_status = TRUE; 986 softint_schedule(adapter->link_si); 987 988 IXGBE_WRITE_REG(hw, IXGBE_VTEIMS, (1 << adapter->vector)); 989 990 return 1; 991 } /* ixv_msix_mbx */ 992 993 static void 994 ixv_eitr_write(struct ix_queue *que, uint32_t itr) 995 { 996 struct adapter *adapter = que->adapter; 997 998 /* 999 * Newer devices than 82598 have VF function, so this function is 1000 * simple. 1001 */ 1002 itr |= IXGBE_EITR_CNT_WDIS; 1003 1004 IXGBE_WRITE_REG(&adapter->hw, IXGBE_VTEITR(que->msix), itr); 1005 } 1006 1007 1008 /************************************************************************ 1009 * ixv_media_status - Media Ioctl callback 1010 * 1011 * Called whenever the user queries the status of 1012 * the interface using ifconfig. 1013 ************************************************************************/ 1014 static void 1015 ixv_media_status(struct ifnet *ifp, struct ifmediareq *ifmr) 1016 { 1017 struct adapter *adapter = ifp->if_softc; 1018 1019 INIT_DEBUGOUT("ixv_media_status: begin"); 1020 IXGBE_CORE_LOCK(adapter); 1021 ixv_update_link_status(adapter); 1022 1023 ifmr->ifm_status = IFM_AVALID; 1024 ifmr->ifm_active = IFM_ETHER; 1025 1026 if (!adapter->link_active) { 1027 ifmr->ifm_active |= IFM_NONE; 1028 IXGBE_CORE_UNLOCK(adapter); 1029 return; 1030 } 1031 1032 ifmr->ifm_status |= IFM_ACTIVE; 1033 1034 switch (adapter->link_speed) { 1035 case IXGBE_LINK_SPEED_10GB_FULL: 1036 ifmr->ifm_active |= IFM_10G_T | IFM_FDX; 1037 break; 1038 case IXGBE_LINK_SPEED_5GB_FULL: 1039 ifmr->ifm_active |= IFM_5000_T | IFM_FDX; 1040 break; 1041 case IXGBE_LINK_SPEED_2_5GB_FULL: 1042 ifmr->ifm_active |= IFM_2500_T | IFM_FDX; 1043 break; 1044 case IXGBE_LINK_SPEED_1GB_FULL: 1045 ifmr->ifm_active |= IFM_1000_T | IFM_FDX; 1046 break; 1047 case IXGBE_LINK_SPEED_100_FULL: 1048 ifmr->ifm_active |= IFM_100_TX | IFM_FDX; 1049 break; 1050 case IXGBE_LINK_SPEED_10_FULL: 1051 ifmr->ifm_active |= IFM_10_T | IFM_FDX; 1052 break; 1053 } 1054 1055 ifp->if_baudrate = ifmedia_baudrate(ifmr->ifm_active); 1056 1057 IXGBE_CORE_UNLOCK(adapter); 1058 } /* ixv_media_status */ 1059 1060 /************************************************************************ 1061 * ixv_media_change - Media Ioctl callback 1062 * 1063 * Called when the user changes speed/duplex using 1064 * media/mediopt option with ifconfig. 1065 ************************************************************************/ 1066 static int 1067 ixv_media_change(struct ifnet *ifp) 1068 { 1069 struct adapter *adapter = ifp->if_softc; 1070 struct ifmedia *ifm = &adapter->media; 1071 1072 INIT_DEBUGOUT("ixv_media_change: begin"); 1073 1074 if (IFM_TYPE(ifm->ifm_media) != IFM_ETHER) 1075 return (EINVAL); 1076 1077 switch (IFM_SUBTYPE(ifm->ifm_media)) { 1078 case IFM_AUTO: 1079 break; 1080 default: 1081 device_printf(adapter->dev, "Only auto media type\n"); 1082 return (EINVAL); 1083 } 1084 1085 return (0); 1086 } /* ixv_media_change */ 1087 1088 1089 /************************************************************************ 1090 * ixv_negotiate_api 1091 * 1092 * Negotiate the Mailbox API with the PF; 1093 * start with the most featured API first. 1094 ************************************************************************/ 1095 static int 1096 ixv_negotiate_api(struct adapter *adapter) 1097 { 1098 struct ixgbe_hw *hw = &adapter->hw; 1099 int mbx_api[] = { ixgbe_mbox_api_11, 1100 ixgbe_mbox_api_10, 1101 ixgbe_mbox_api_unknown }; 1102 int i = 0; 1103 1104 while (mbx_api[i] != ixgbe_mbox_api_unknown) { 1105 if (ixgbevf_negotiate_api_version(hw, mbx_api[i]) == 0) 1106 return (0); 1107 i++; 1108 } 1109 1110 return (EINVAL); 1111 } /* ixv_negotiate_api */ 1112 1113 1114 /************************************************************************ 1115 * ixv_set_multi - Multicast Update 1116 * 1117 * Called whenever multicast address list is updated. 1118 ************************************************************************/ 1119 static void 1120 ixv_set_multi(struct adapter *adapter) 1121 { 1122 struct ether_multi *enm; 1123 struct ether_multistep step; 1124 struct ethercom *ec = &adapter->osdep.ec; 1125 u8 mta[MAX_NUM_MULTICAST_ADDRESSES * IXGBE_ETH_LENGTH_OF_ADDRESS]; 1126 u8 *update_ptr; 1127 int mcnt = 0; 1128 1129 KASSERT(mutex_owned(&adapter->core_mtx)); 1130 IOCTL_DEBUGOUT("ixv_set_multi: begin"); 1131 1132 ETHER_LOCK(ec); 1133 ETHER_FIRST_MULTI(step, ec, enm); 1134 while (enm != NULL) { 1135 bcopy(enm->enm_addrlo, 1136 &mta[mcnt * IXGBE_ETH_LENGTH_OF_ADDRESS], 1137 IXGBE_ETH_LENGTH_OF_ADDRESS); 1138 mcnt++; 1139 /* XXX This might be required --msaitoh */ 1140 if (mcnt >= MAX_NUM_MULTICAST_ADDRESSES) 1141 break; 1142 ETHER_NEXT_MULTI(step, enm); 1143 } 1144 ETHER_UNLOCK(ec); 1145 1146 update_ptr = mta; 1147 1148 adapter->hw.mac.ops.update_mc_addr_list(&adapter->hw, update_ptr, mcnt, 1149 ixv_mc_array_itr, TRUE); 1150 } /* ixv_set_multi */ 1151 1152 /************************************************************************ 1153 * ixv_mc_array_itr 1154 * 1155 * An iterator function needed by the multicast shared code. 1156 * It feeds the shared code routine the addresses in the 1157 * array of ixv_set_multi() one by one. 1158 ************************************************************************/ 1159 static u8 * 1160 ixv_mc_array_itr(struct ixgbe_hw *hw, u8 **update_ptr, u32 *vmdq) 1161 { 1162 u8 *addr = *update_ptr; 1163 u8 *newptr; 1164 1165 *vmdq = 0; 1166 1167 newptr = addr + IXGBE_ETH_LENGTH_OF_ADDRESS; 1168 *update_ptr = newptr; 1169 1170 return addr; 1171 } /* ixv_mc_array_itr */ 1172 1173 /************************************************************************ 1174 * ixv_local_timer - Timer routine 1175 * 1176 * Checks for link status, updates statistics, 1177 * and runs the watchdog check. 1178 ************************************************************************/ 1179 static void 1180 ixv_local_timer(void *arg) 1181 { 1182 struct adapter *adapter = arg; 1183 1184 IXGBE_CORE_LOCK(adapter); 1185 ixv_local_timer_locked(adapter); 1186 IXGBE_CORE_UNLOCK(adapter); 1187 } 1188 1189 static void 1190 ixv_local_timer_locked(void *arg) 1191 { 1192 struct adapter *adapter = arg; 1193 device_t dev = adapter->dev; 1194 struct ix_queue *que = adapter->queues; 1195 u64 queues = 0; 1196 u64 v0, v1, v2, v3, v4, v5, v6, v7; 1197 int hung = 0; 1198 int i; 1199 1200 KASSERT(mutex_owned(&adapter->core_mtx)); 1201 1202 ixv_check_link(adapter); 1203 1204 /* Stats Update */ 1205 ixv_update_stats(adapter); 1206 1207 /* Update some event counters */ 1208 v0 = v1 = v2 = v3 = v4 = v5 = v6 = v7 = 0; 1209 que = adapter->queues; 1210 for (i = 0; i < adapter->num_queues; i++, que++) { 1211 struct tx_ring *txr = que->txr; 1212 1213 v0 += txr->q_efbig_tx_dma_setup; 1214 v1 += txr->q_mbuf_defrag_failed; 1215 v2 += txr->q_efbig2_tx_dma_setup; 1216 v3 += txr->q_einval_tx_dma_setup; 1217 v4 += txr->q_other_tx_dma_setup; 1218 v5 += txr->q_eagain_tx_dma_setup; 1219 v6 += txr->q_enomem_tx_dma_setup; 1220 v7 += txr->q_tso_err; 1221 } 1222 adapter->efbig_tx_dma_setup.ev_count = v0; 1223 adapter->mbuf_defrag_failed.ev_count = v1; 1224 adapter->efbig2_tx_dma_setup.ev_count = v2; 1225 adapter->einval_tx_dma_setup.ev_count = v3; 1226 adapter->other_tx_dma_setup.ev_count = v4; 1227 adapter->eagain_tx_dma_setup.ev_count = v5; 1228 adapter->enomem_tx_dma_setup.ev_count = v6; 1229 adapter->tso_err.ev_count = v7; 1230 1231 /* 1232 * Check the TX queues status 1233 * - mark hung queues so we don't schedule on them 1234 * - watchdog only if all queues show hung 1235 */ 1236 que = adapter->queues; 1237 for (i = 0; i < adapter->num_queues; i++, que++) { 1238 /* Keep track of queues with work for soft irq */ 1239 if (que->txr->busy) 1240 queues |= ((u64)1 << que->me); 1241 /* 1242 * Each time txeof runs without cleaning, but there 1243 * are uncleaned descriptors it increments busy. If 1244 * we get to the MAX we declare it hung. 1245 */ 1246 if (que->busy == IXGBE_QUEUE_HUNG) { 1247 ++hung; 1248 /* Mark the queue as inactive */ 1249 adapter->active_queues &= ~((u64)1 << que->me); 1250 continue; 1251 } else { 1252 /* Check if we've come back from hung */ 1253 if ((adapter->active_queues & ((u64)1 << que->me)) == 0) 1254 adapter->active_queues |= ((u64)1 << que->me); 1255 } 1256 if (que->busy >= IXGBE_MAX_TX_BUSY) { 1257 device_printf(dev, 1258 "Warning queue %d appears to be hung!\n", i); 1259 que->txr->busy = IXGBE_QUEUE_HUNG; 1260 ++hung; 1261 } 1262 } 1263 1264 /* Only truly watchdog if all queues show hung */ 1265 if (hung == adapter->num_queues) 1266 goto watchdog; 1267 else if (queues != 0) { /* Force an IRQ on queues with work */ 1268 ixv_rearm_queues(adapter, queues); 1269 } 1270 1271 callout_reset(&adapter->timer, hz, ixv_local_timer, adapter); 1272 1273 return; 1274 1275 watchdog: 1276 1277 device_printf(adapter->dev, "Watchdog timeout -- resetting\n"); 1278 adapter->ifp->if_flags &= ~IFF_RUNNING; 1279 adapter->watchdog_events.ev_count++; 1280 ixv_init_locked(adapter); 1281 } /* ixv_local_timer */ 1282 1283 /************************************************************************ 1284 * ixv_update_link_status - Update OS on link state 1285 * 1286 * Note: Only updates the OS on the cached link state. 1287 * The real check of the hardware only happens with 1288 * a link interrupt. 1289 ************************************************************************/ 1290 static void 1291 ixv_update_link_status(struct adapter *adapter) 1292 { 1293 struct ifnet *ifp = adapter->ifp; 1294 device_t dev = adapter->dev; 1295 1296 KASSERT(mutex_owned(&adapter->core_mtx)); 1297 1298 if (adapter->link_up) { 1299 if (adapter->link_active == FALSE) { 1300 if (bootverbose) { 1301 const char *bpsmsg; 1302 1303 switch (adapter->link_speed) { 1304 case IXGBE_LINK_SPEED_10GB_FULL: 1305 bpsmsg = "10 Gbps"; 1306 break; 1307 case IXGBE_LINK_SPEED_5GB_FULL: 1308 bpsmsg = "5 Gbps"; 1309 break; 1310 case IXGBE_LINK_SPEED_2_5GB_FULL: 1311 bpsmsg = "2.5 Gbps"; 1312 break; 1313 case IXGBE_LINK_SPEED_1GB_FULL: 1314 bpsmsg = "1 Gbps"; 1315 break; 1316 case IXGBE_LINK_SPEED_100_FULL: 1317 bpsmsg = "100 Mbps"; 1318 break; 1319 case IXGBE_LINK_SPEED_10_FULL: 1320 bpsmsg = "10 Mbps"; 1321 break; 1322 default: 1323 bpsmsg = "unknown speed"; 1324 break; 1325 } 1326 device_printf(dev, "Link is up %s %s \n", 1327 bpsmsg, "Full Duplex"); 1328 } 1329 adapter->link_active = TRUE; 1330 if_link_state_change(ifp, LINK_STATE_UP); 1331 } 1332 } else { /* Link down */ 1333 if (adapter->link_active == TRUE) { 1334 if (bootverbose) 1335 device_printf(dev, "Link is Down\n"); 1336 if_link_state_change(ifp, LINK_STATE_DOWN); 1337 adapter->link_active = FALSE; 1338 } 1339 } 1340 } /* ixv_update_link_status */ 1341 1342 1343 /************************************************************************ 1344 * ixv_stop - Stop the hardware 1345 * 1346 * Disables all traffic on the adapter by issuing a 1347 * global reset on the MAC and deallocates TX/RX buffers. 1348 ************************************************************************/ 1349 static void 1350 ixv_ifstop(struct ifnet *ifp, int disable) 1351 { 1352 struct adapter *adapter = ifp->if_softc; 1353 1354 IXGBE_CORE_LOCK(adapter); 1355 ixv_stop(adapter); 1356 IXGBE_CORE_UNLOCK(adapter); 1357 } 1358 1359 static void 1360 ixv_stop(void *arg) 1361 { 1362 struct ifnet *ifp; 1363 struct adapter *adapter = arg; 1364 struct ixgbe_hw *hw = &adapter->hw; 1365 1366 ifp = adapter->ifp; 1367 1368 KASSERT(mutex_owned(&adapter->core_mtx)); 1369 1370 INIT_DEBUGOUT("ixv_stop: begin\n"); 1371 ixv_disable_intr(adapter); 1372 1373 /* Tell the stack that the interface is no longer active */ 1374 ifp->if_flags &= ~(IFF_RUNNING | IFF_OACTIVE); 1375 1376 hw->mac.ops.reset_hw(hw); 1377 adapter->hw.adapter_stopped = FALSE; 1378 hw->mac.ops.stop_adapter(hw); 1379 callout_stop(&adapter->timer); 1380 1381 /* reprogram the RAR[0] in case user changed it. */ 1382 hw->mac.ops.set_rar(hw, 0, hw->mac.addr, 0, IXGBE_RAH_AV); 1383 1384 return; 1385 } /* ixv_stop */ 1386 1387 1388 /************************************************************************ 1389 * ixv_allocate_pci_resources 1390 ************************************************************************/ 1391 static int 1392 ixv_allocate_pci_resources(struct adapter *adapter, 1393 const struct pci_attach_args *pa) 1394 { 1395 pcireg_t memtype; 1396 device_t dev = adapter->dev; 1397 bus_addr_t addr; 1398 int flags; 1399 1400 memtype = pci_mapreg_type(pa->pa_pc, pa->pa_tag, PCI_BAR(0)); 1401 switch (memtype) { 1402 case PCI_MAPREG_TYPE_MEM | PCI_MAPREG_MEM_TYPE_32BIT: 1403 case PCI_MAPREG_TYPE_MEM | PCI_MAPREG_MEM_TYPE_64BIT: 1404 adapter->osdep.mem_bus_space_tag = pa->pa_memt; 1405 if (pci_mapreg_info(pa->pa_pc, pa->pa_tag, PCI_BAR(0), 1406 memtype, &addr, &adapter->osdep.mem_size, &flags) != 0) 1407 goto map_err; 1408 if ((flags & BUS_SPACE_MAP_PREFETCHABLE) != 0) { 1409 aprint_normal_dev(dev, "clearing prefetchable bit\n"); 1410 flags &= ~BUS_SPACE_MAP_PREFETCHABLE; 1411 } 1412 if (bus_space_map(adapter->osdep.mem_bus_space_tag, addr, 1413 adapter->osdep.mem_size, flags, 1414 &adapter->osdep.mem_bus_space_handle) != 0) { 1415 map_err: 1416 adapter->osdep.mem_size = 0; 1417 aprint_error_dev(dev, "unable to map BAR0\n"); 1418 return ENXIO; 1419 } 1420 break; 1421 default: 1422 aprint_error_dev(dev, "unexpected type on BAR0\n"); 1423 return ENXIO; 1424 } 1425 1426 /* Pick up the tuneable queues */ 1427 adapter->num_queues = ixv_num_queues; 1428 1429 return (0); 1430 } /* ixv_allocate_pci_resources */ 1431 1432 /************************************************************************ 1433 * ixv_free_pci_resources 1434 ************************************************************************/ 1435 static void 1436 ixv_free_pci_resources(struct adapter * adapter) 1437 { 1438 struct ix_queue *que = adapter->queues; 1439 int rid; 1440 1441 /* 1442 * Release all msix queue resources: 1443 */ 1444 for (int i = 0; i < adapter->num_queues; i++, que++) { 1445 if (que->res != NULL) 1446 pci_intr_disestablish(adapter->osdep.pc, 1447 adapter->osdep.ihs[i]); 1448 } 1449 1450 1451 /* Clean the Mailbox interrupt last */ 1452 rid = adapter->vector; 1453 1454 if (adapter->osdep.ihs[rid] != NULL) { 1455 pci_intr_disestablish(adapter->osdep.pc, 1456 adapter->osdep.ihs[rid]); 1457 adapter->osdep.ihs[rid] = NULL; 1458 } 1459 1460 pci_intr_release(adapter->osdep.pc, adapter->osdep.intrs, 1461 adapter->osdep.nintrs); 1462 1463 if (adapter->osdep.mem_size != 0) { 1464 bus_space_unmap(adapter->osdep.mem_bus_space_tag, 1465 adapter->osdep.mem_bus_space_handle, 1466 adapter->osdep.mem_size); 1467 } 1468 1469 return; 1470 } /* ixv_free_pci_resources */ 1471 1472 /************************************************************************ 1473 * ixv_setup_interface 1474 * 1475 * Setup networking device structure and register an interface. 1476 ************************************************************************/ 1477 static int 1478 ixv_setup_interface(device_t dev, struct adapter *adapter) 1479 { 1480 struct ethercom *ec = &adapter->osdep.ec; 1481 struct ifnet *ifp; 1482 int rv; 1483 1484 INIT_DEBUGOUT("ixv_setup_interface: begin"); 1485 1486 ifp = adapter->ifp = &ec->ec_if; 1487 strlcpy(ifp->if_xname, device_xname(dev), IFNAMSIZ); 1488 ifp->if_baudrate = IF_Gbps(10); 1489 ifp->if_init = ixv_init; 1490 ifp->if_stop = ixv_ifstop; 1491 ifp->if_softc = adapter; 1492 ifp->if_flags = IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST; 1493 #ifdef IXGBE_MPSAFE 1494 ifp->if_extflags = IFEF_MPSAFE; 1495 #endif 1496 ifp->if_ioctl = ixv_ioctl; 1497 if (adapter->feat_en & IXGBE_FEATURE_LEGACY_TX) { 1498 #if 0 1499 ixv_start_locked = ixgbe_legacy_start_locked; 1500 #endif 1501 } else { 1502 ifp->if_transmit = ixgbe_mq_start; 1503 #if 0 1504 ixv_start_locked = ixgbe_mq_start_locked; 1505 #endif 1506 } 1507 ifp->if_start = ixgbe_legacy_start; 1508 IFQ_SET_MAXLEN(&ifp->if_snd, adapter->num_tx_desc - 2); 1509 IFQ_SET_READY(&ifp->if_snd); 1510 1511 rv = if_initialize(ifp); 1512 if (rv != 0) { 1513 aprint_error_dev(dev, "if_initialize failed(%d)\n", rv); 1514 return rv; 1515 } 1516 adapter->ipq = if_percpuq_create(&adapter->osdep.ec.ec_if); 1517 ether_ifattach(ifp, adapter->hw.mac.addr); 1518 /* 1519 * We use per TX queue softint, so if_deferred_start_init() isn't 1520 * used. 1521 */ 1522 if_register(ifp); 1523 ether_set_ifflags_cb(ec, ixv_ifflags_cb); 1524 1525 adapter->max_frame_size = ifp->if_mtu + IXGBE_MTU_HDR; 1526 1527 /* 1528 * Tell the upper layer(s) we support long frames. 1529 */ 1530 ifp->if_hdrlen = sizeof(struct ether_vlan_header); 1531 1532 /* Set capability flags */ 1533 ifp->if_capabilities |= IFCAP_HWCSUM 1534 | IFCAP_TSOv4 1535 | IFCAP_TSOv6; 1536 ifp->if_capenable = 0; 1537 1538 ec->ec_capabilities |= ETHERCAP_VLAN_HWTAGGING 1539 | ETHERCAP_VLAN_HWCSUM 1540 | ETHERCAP_JUMBO_MTU 1541 | ETHERCAP_VLAN_MTU; 1542 1543 /* Enable the above capabilities by default */ 1544 ec->ec_capenable = ec->ec_capabilities; 1545 1546 /* Don't enable LRO by default */ 1547 ifp->if_capabilities |= IFCAP_LRO; 1548 #if 0 1549 ifp->if_capenable = ifp->if_capabilities; 1550 #endif 1551 1552 /* 1553 * Specify the media types supported by this adapter and register 1554 * callbacks to update media and link information 1555 */ 1556 ifmedia_init(&adapter->media, IFM_IMASK, ixv_media_change, 1557 ixv_media_status); 1558 ifmedia_add(&adapter->media, IFM_ETHER | IFM_AUTO, 0, NULL); 1559 ifmedia_set(&adapter->media, IFM_ETHER | IFM_AUTO); 1560 1561 return 0; 1562 } /* ixv_setup_interface */ 1563 1564 1565 /************************************************************************ 1566 * ixv_initialize_transmit_units - Enable transmit unit. 1567 ************************************************************************/ 1568 static void 1569 ixv_initialize_transmit_units(struct adapter *adapter) 1570 { 1571 struct tx_ring *txr = adapter->tx_rings; 1572 struct ixgbe_hw *hw = &adapter->hw; 1573 1574 1575 for (int i = 0; i < adapter->num_queues; i++, txr++) { 1576 u64 tdba = txr->txdma.dma_paddr; 1577 u32 txctrl, txdctl; 1578 1579 /* Set WTHRESH to 8, burst writeback */ 1580 txdctl = IXGBE_READ_REG(hw, IXGBE_VFTXDCTL(i)); 1581 txdctl |= (8 << 16); 1582 IXGBE_WRITE_REG(hw, IXGBE_VFTXDCTL(i), txdctl); 1583 1584 /* Set the HW Tx Head and Tail indices */ 1585 IXGBE_WRITE_REG(&adapter->hw, IXGBE_VFTDH(i), 0); 1586 IXGBE_WRITE_REG(&adapter->hw, IXGBE_VFTDT(i), 0); 1587 1588 /* Set Tx Tail register */ 1589 txr->tail = IXGBE_VFTDT(i); 1590 1591 /* Set Ring parameters */ 1592 IXGBE_WRITE_REG(hw, IXGBE_VFTDBAL(i), 1593 (tdba & 0x00000000ffffffffULL)); 1594 IXGBE_WRITE_REG(hw, IXGBE_VFTDBAH(i), (tdba >> 32)); 1595 IXGBE_WRITE_REG(hw, IXGBE_VFTDLEN(i), 1596 adapter->num_tx_desc * sizeof(struct ixgbe_legacy_tx_desc)); 1597 txctrl = IXGBE_READ_REG(hw, IXGBE_VFDCA_TXCTRL(i)); 1598 txctrl &= ~IXGBE_DCA_TXCTRL_DESC_WRO_EN; 1599 IXGBE_WRITE_REG(hw, IXGBE_VFDCA_TXCTRL(i), txctrl); 1600 1601 /* Now enable */ 1602 txdctl = IXGBE_READ_REG(hw, IXGBE_VFTXDCTL(i)); 1603 txdctl |= IXGBE_TXDCTL_ENABLE; 1604 IXGBE_WRITE_REG(hw, IXGBE_VFTXDCTL(i), txdctl); 1605 } 1606 1607 return; 1608 } /* ixv_initialize_transmit_units */ 1609 1610 1611 /************************************************************************ 1612 * ixv_initialize_rss_mapping 1613 ************************************************************************/ 1614 static void 1615 ixv_initialize_rss_mapping(struct adapter *adapter) 1616 { 1617 struct ixgbe_hw *hw = &adapter->hw; 1618 u32 reta = 0, mrqc, rss_key[10]; 1619 int queue_id; 1620 int i, j; 1621 u32 rss_hash_config; 1622 1623 /* force use default RSS key. */ 1624 #ifdef __NetBSD__ 1625 rss_getkey((uint8_t *) &rss_key); 1626 #else 1627 if (adapter->feat_en & IXGBE_FEATURE_RSS) { 1628 /* Fetch the configured RSS key */ 1629 rss_getkey((uint8_t *)&rss_key); 1630 } else { 1631 /* set up random bits */ 1632 cprng_fast(&rss_key, sizeof(rss_key)); 1633 } 1634 #endif 1635 1636 /* Now fill out hash function seeds */ 1637 for (i = 0; i < 10; i++) 1638 IXGBE_WRITE_REG(hw, IXGBE_VFRSSRK(i), rss_key[i]); 1639 1640 /* Set up the redirection table */ 1641 for (i = 0, j = 0; i < 64; i++, j++) { 1642 if (j == adapter->num_queues) 1643 j = 0; 1644 1645 if (adapter->feat_en & IXGBE_FEATURE_RSS) { 1646 /* 1647 * Fetch the RSS bucket id for the given indirection 1648 * entry. Cap it at the number of configured buckets 1649 * (which is num_queues.) 1650 */ 1651 queue_id = rss_get_indirection_to_bucket(i); 1652 queue_id = queue_id % adapter->num_queues; 1653 } else 1654 queue_id = j; 1655 1656 /* 1657 * The low 8 bits are for hash value (n+0); 1658 * The next 8 bits are for hash value (n+1), etc. 1659 */ 1660 reta >>= 8; 1661 reta |= ((uint32_t)queue_id) << 24; 1662 if ((i & 3) == 3) { 1663 IXGBE_WRITE_REG(hw, IXGBE_VFRETA(i >> 2), reta); 1664 reta = 0; 1665 } 1666 } 1667 1668 /* Perform hash on these packet types */ 1669 if (adapter->feat_en & IXGBE_FEATURE_RSS) 1670 rss_hash_config = rss_gethashconfig(); 1671 else { 1672 /* 1673 * Disable UDP - IP fragments aren't currently being handled 1674 * and so we end up with a mix of 2-tuple and 4-tuple 1675 * traffic. 1676 */ 1677 rss_hash_config = RSS_HASHTYPE_RSS_IPV4 1678 | RSS_HASHTYPE_RSS_TCP_IPV4 1679 | RSS_HASHTYPE_RSS_IPV6 1680 | RSS_HASHTYPE_RSS_TCP_IPV6; 1681 } 1682 1683 mrqc = IXGBE_MRQC_RSSEN; 1684 if (rss_hash_config & RSS_HASHTYPE_RSS_IPV4) 1685 mrqc |= IXGBE_MRQC_RSS_FIELD_IPV4; 1686 if (rss_hash_config & RSS_HASHTYPE_RSS_TCP_IPV4) 1687 mrqc |= IXGBE_MRQC_RSS_FIELD_IPV4_TCP; 1688 if (rss_hash_config & RSS_HASHTYPE_RSS_IPV6) 1689 mrqc |= IXGBE_MRQC_RSS_FIELD_IPV6; 1690 if (rss_hash_config & RSS_HASHTYPE_RSS_TCP_IPV6) 1691 mrqc |= IXGBE_MRQC_RSS_FIELD_IPV6_TCP; 1692 if (rss_hash_config & RSS_HASHTYPE_RSS_IPV6_EX) 1693 device_printf(adapter->dev, "%s: RSS_HASHTYPE_RSS_IPV6_EX defined, but not supported\n", 1694 __func__); 1695 if (rss_hash_config & RSS_HASHTYPE_RSS_TCP_IPV6_EX) 1696 device_printf(adapter->dev, "%s: RSS_HASHTYPE_RSS_TCP_IPV6_EX defined, but not supported\n", 1697 __func__); 1698 if (rss_hash_config & RSS_HASHTYPE_RSS_UDP_IPV4) 1699 mrqc |= IXGBE_MRQC_RSS_FIELD_IPV4_UDP; 1700 if (rss_hash_config & RSS_HASHTYPE_RSS_UDP_IPV6) 1701 mrqc |= IXGBE_MRQC_RSS_FIELD_IPV6_UDP; 1702 if (rss_hash_config & RSS_HASHTYPE_RSS_UDP_IPV6_EX) 1703 device_printf(adapter->dev, "%s: RSS_HASHTYPE_RSS_UDP_IPV6_EX defined, but not supported\n", 1704 __func__); 1705 IXGBE_WRITE_REG(hw, IXGBE_VFMRQC, mrqc); 1706 } /* ixv_initialize_rss_mapping */ 1707 1708 1709 /************************************************************************ 1710 * ixv_initialize_receive_units - Setup receive registers and features. 1711 ************************************************************************/ 1712 static void 1713 ixv_initialize_receive_units(struct adapter *adapter) 1714 { 1715 struct rx_ring *rxr = adapter->rx_rings; 1716 struct ixgbe_hw *hw = &adapter->hw; 1717 struct ifnet *ifp = adapter->ifp; 1718 u32 bufsz, rxcsum, psrtype; 1719 1720 if (ifp->if_mtu > ETHERMTU) 1721 bufsz = 4096 >> IXGBE_SRRCTL_BSIZEPKT_SHIFT; 1722 else 1723 bufsz = 2048 >> IXGBE_SRRCTL_BSIZEPKT_SHIFT; 1724 1725 psrtype = IXGBE_PSRTYPE_TCPHDR 1726 | IXGBE_PSRTYPE_UDPHDR 1727 | IXGBE_PSRTYPE_IPV4HDR 1728 | IXGBE_PSRTYPE_IPV6HDR 1729 | IXGBE_PSRTYPE_L2HDR; 1730 1731 if (adapter->num_queues > 1) 1732 psrtype |= 1 << 29; 1733 1734 IXGBE_WRITE_REG(hw, IXGBE_VFPSRTYPE, psrtype); 1735 1736 /* Tell PF our max_frame size */ 1737 if (ixgbevf_rlpml_set_vf(hw, adapter->max_frame_size) != 0) { 1738 device_printf(adapter->dev, "There is a problem with the PF setup. It is likely the receive unit for this VF will not function correctly.\n"); 1739 } 1740 1741 for (int i = 0; i < adapter->num_queues; i++, rxr++) { 1742 u64 rdba = rxr->rxdma.dma_paddr; 1743 u32 reg, rxdctl; 1744 1745 /* Disable the queue */ 1746 rxdctl = IXGBE_READ_REG(hw, IXGBE_VFRXDCTL(i)); 1747 rxdctl &= ~IXGBE_RXDCTL_ENABLE; 1748 IXGBE_WRITE_REG(hw, IXGBE_VFRXDCTL(i), rxdctl); 1749 for (int j = 0; j < 10; j++) { 1750 if (IXGBE_READ_REG(hw, IXGBE_VFRXDCTL(i)) & 1751 IXGBE_RXDCTL_ENABLE) 1752 msec_delay(1); 1753 else 1754 break; 1755 } 1756 wmb(); 1757 /* Setup the Base and Length of the Rx Descriptor Ring */ 1758 IXGBE_WRITE_REG(hw, IXGBE_VFRDBAL(i), 1759 (rdba & 0x00000000ffffffffULL)); 1760 IXGBE_WRITE_REG(hw, IXGBE_VFRDBAH(i), (rdba >> 32)); 1761 IXGBE_WRITE_REG(hw, IXGBE_VFRDLEN(i), 1762 adapter->num_rx_desc * sizeof(union ixgbe_adv_rx_desc)); 1763 1764 /* Reset the ring indices */ 1765 IXGBE_WRITE_REG(hw, IXGBE_VFRDH(rxr->me), 0); 1766 IXGBE_WRITE_REG(hw, IXGBE_VFRDT(rxr->me), 0); 1767 1768 /* Set up the SRRCTL register */ 1769 reg = IXGBE_READ_REG(hw, IXGBE_VFSRRCTL(i)); 1770 reg &= ~IXGBE_SRRCTL_BSIZEHDR_MASK; 1771 reg &= ~IXGBE_SRRCTL_BSIZEPKT_MASK; 1772 reg |= bufsz; 1773 reg |= IXGBE_SRRCTL_DESCTYPE_ADV_ONEBUF; 1774 IXGBE_WRITE_REG(hw, IXGBE_VFSRRCTL(i), reg); 1775 1776 /* Capture Rx Tail index */ 1777 rxr->tail = IXGBE_VFRDT(rxr->me); 1778 1779 /* Do the queue enabling last */ 1780 rxdctl |= IXGBE_RXDCTL_ENABLE | IXGBE_RXDCTL_VME; 1781 IXGBE_WRITE_REG(hw, IXGBE_VFRXDCTL(i), rxdctl); 1782 for (int k = 0; k < 10; k++) { 1783 if (IXGBE_READ_REG(hw, IXGBE_VFRXDCTL(i)) & 1784 IXGBE_RXDCTL_ENABLE) 1785 break; 1786 msec_delay(1); 1787 } 1788 wmb(); 1789 1790 /* Set the Tail Pointer */ 1791 #ifdef DEV_NETMAP 1792 /* 1793 * In netmap mode, we must preserve the buffers made 1794 * available to userspace before the if_init() 1795 * (this is true by default on the TX side, because 1796 * init makes all buffers available to userspace). 1797 * 1798 * netmap_reset() and the device specific routines 1799 * (e.g. ixgbe_setup_receive_rings()) map these 1800 * buffers at the end of the NIC ring, so here we 1801 * must set the RDT (tail) register to make sure 1802 * they are not overwritten. 1803 * 1804 * In this driver the NIC ring starts at RDH = 0, 1805 * RDT points to the last slot available for reception (?), 1806 * so RDT = num_rx_desc - 1 means the whole ring is available. 1807 */ 1808 if ((adapter->feat_en & IXGBE_FEATURE_NETMAP) && 1809 (ifp->if_capenable & IFCAP_NETMAP)) { 1810 struct netmap_adapter *na = NA(adapter->ifp); 1811 struct netmap_kring *kring = &na->rx_rings[i]; 1812 int t = na->num_rx_desc - 1 - nm_kr_rxspace(kring); 1813 1814 IXGBE_WRITE_REG(hw, IXGBE_VFRDT(rxr->me), t); 1815 } else 1816 #endif /* DEV_NETMAP */ 1817 IXGBE_WRITE_REG(hw, IXGBE_VFRDT(rxr->me), 1818 adapter->num_rx_desc - 1); 1819 } 1820 1821 rxcsum = IXGBE_READ_REG(hw, IXGBE_RXCSUM); 1822 1823 ixv_initialize_rss_mapping(adapter); 1824 1825 if (adapter->num_queues > 1) { 1826 /* RSS and RX IPP Checksum are mutually exclusive */ 1827 rxcsum |= IXGBE_RXCSUM_PCSD; 1828 } 1829 1830 if (ifp->if_capenable & IFCAP_RXCSUM) 1831 rxcsum |= IXGBE_RXCSUM_PCSD; 1832 1833 if (!(rxcsum & IXGBE_RXCSUM_PCSD)) 1834 rxcsum |= IXGBE_RXCSUM_IPPCSE; 1835 1836 IXGBE_WRITE_REG(hw, IXGBE_RXCSUM, rxcsum); 1837 } /* ixv_initialize_receive_units */ 1838 1839 /************************************************************************ 1840 * ixv_sysctl_tdh_handler - Transmit Descriptor Head handler function 1841 * 1842 * Retrieves the TDH value from the hardware 1843 ************************************************************************/ 1844 static int 1845 ixv_sysctl_tdh_handler(SYSCTLFN_ARGS) 1846 { 1847 struct sysctlnode node = *rnode; 1848 struct tx_ring *txr = (struct tx_ring *)node.sysctl_data; 1849 uint32_t val; 1850 1851 if (!txr) 1852 return (0); 1853 1854 val = IXGBE_READ_REG(&txr->adapter->hw, IXGBE_VFTDH(txr->me)); 1855 node.sysctl_data = &val; 1856 return sysctl_lookup(SYSCTLFN_CALL(&node)); 1857 } /* ixv_sysctl_tdh_handler */ 1858 1859 /************************************************************************ 1860 * ixgbe_sysctl_tdt_handler - Transmit Descriptor Tail handler function 1861 * 1862 * Retrieves the TDT value from the hardware 1863 ************************************************************************/ 1864 static int 1865 ixv_sysctl_tdt_handler(SYSCTLFN_ARGS) 1866 { 1867 struct sysctlnode node = *rnode; 1868 struct tx_ring *txr = (struct tx_ring *)node.sysctl_data; 1869 uint32_t val; 1870 1871 if (!txr) 1872 return (0); 1873 1874 val = IXGBE_READ_REG(&txr->adapter->hw, IXGBE_VFTDT(txr->me)); 1875 node.sysctl_data = &val; 1876 return sysctl_lookup(SYSCTLFN_CALL(&node)); 1877 } /* ixv_sysctl_tdt_handler */ 1878 1879 /************************************************************************ 1880 * ixv_sysctl_rdh_handler - Receive Descriptor Head handler function 1881 * 1882 * Retrieves the RDH value from the hardware 1883 ************************************************************************/ 1884 static int 1885 ixv_sysctl_rdh_handler(SYSCTLFN_ARGS) 1886 { 1887 struct sysctlnode node = *rnode; 1888 struct rx_ring *rxr = (struct rx_ring *)node.sysctl_data; 1889 uint32_t val; 1890 1891 if (!rxr) 1892 return (0); 1893 1894 val = IXGBE_READ_REG(&rxr->adapter->hw, IXGBE_VFRDH(rxr->me)); 1895 node.sysctl_data = &val; 1896 return sysctl_lookup(SYSCTLFN_CALL(&node)); 1897 } /* ixv_sysctl_rdh_handler */ 1898 1899 /************************************************************************ 1900 * ixv_sysctl_rdt_handler - Receive Descriptor Tail handler function 1901 * 1902 * Retrieves the RDT value from the hardware 1903 ************************************************************************/ 1904 static int 1905 ixv_sysctl_rdt_handler(SYSCTLFN_ARGS) 1906 { 1907 struct sysctlnode node = *rnode; 1908 struct rx_ring *rxr = (struct rx_ring *)node.sysctl_data; 1909 uint32_t val; 1910 1911 if (!rxr) 1912 return (0); 1913 1914 val = IXGBE_READ_REG(&rxr->adapter->hw, IXGBE_VFRDT(rxr->me)); 1915 node.sysctl_data = &val; 1916 return sysctl_lookup(SYSCTLFN_CALL(&node)); 1917 } /* ixv_sysctl_rdt_handler */ 1918 1919 /************************************************************************ 1920 * ixv_setup_vlan_support 1921 ************************************************************************/ 1922 static void 1923 ixv_setup_vlan_support(struct adapter *adapter) 1924 { 1925 struct ethercom *ec = &adapter->osdep.ec; 1926 struct ixgbe_hw *hw = &adapter->hw; 1927 struct rx_ring *rxr; 1928 u32 ctrl, vid, vfta, retry; 1929 1930 /* 1931 * We get here thru init_locked, meaning 1932 * a soft reset, this has already cleared 1933 * the VFTA and other state, so if there 1934 * have been no vlan's registered do nothing. 1935 */ 1936 if (!VLAN_ATTACHED(ec)) 1937 return; 1938 1939 /* Enable the queues */ 1940 for (int i = 0; i < adapter->num_queues; i++) { 1941 rxr = &adapter->rx_rings[i]; 1942 ctrl = IXGBE_READ_REG(hw, IXGBE_VFRXDCTL(rxr->me)); 1943 ctrl |= IXGBE_RXDCTL_VME; 1944 IXGBE_WRITE_REG(hw, IXGBE_VFRXDCTL(rxr->me), ctrl); 1945 /* 1946 * Let Rx path know that it needs to store VLAN tag 1947 * as part of extra mbuf info. 1948 */ 1949 rxr->vtag_strip = TRUE; 1950 } 1951 1952 #if 1 1953 /* XXX dirty hack. Enable all VIDs */ 1954 for (int i = 0; i < IXGBE_VFTA_SIZE; i++) 1955 adapter->shadow_vfta[i] = 0xffffffff; 1956 #endif 1957 /* 1958 * A soft reset zero's out the VFTA, so 1959 * we need to repopulate it now. 1960 */ 1961 for (int i = 0; i < IXGBE_VFTA_SIZE; i++) { 1962 if (adapter->shadow_vfta[i] == 0) 1963 continue; 1964 vfta = adapter->shadow_vfta[i]; 1965 /* 1966 * Reconstruct the vlan id's 1967 * based on the bits set in each 1968 * of the array ints. 1969 */ 1970 for (int j = 0; j < 32; j++) { 1971 retry = 0; 1972 if ((vfta & (1 << j)) == 0) 1973 continue; 1974 vid = (i * 32) + j; 1975 /* Call the shared code mailbox routine */ 1976 while (hw->mac.ops.set_vfta(hw, vid, 0, TRUE, FALSE)) { 1977 if (++retry > 5) 1978 break; 1979 } 1980 } 1981 } 1982 } /* ixv_setup_vlan_support */ 1983 1984 #if 0 /* XXX Badly need to overhaul vlan(4) on NetBSD. */ 1985 /************************************************************************ 1986 * ixv_register_vlan 1987 * 1988 * Run via a vlan config EVENT, it enables us to use the 1989 * HW Filter table since we can get the vlan id. This just 1990 * creates the entry in the soft version of the VFTA, init 1991 * will repopulate the real table. 1992 ************************************************************************/ 1993 static void 1994 ixv_register_vlan(void *arg, struct ifnet *ifp, u16 vtag) 1995 { 1996 struct adapter *adapter = ifp->if_softc; 1997 u16 index, bit; 1998 1999 if (ifp->if_softc != arg) /* Not our event */ 2000 return; 2001 2002 if ((vtag == 0) || (vtag > 4095)) /* Invalid */ 2003 return; 2004 2005 IXGBE_CORE_LOCK(adapter); 2006 index = (vtag >> 5) & 0x7F; 2007 bit = vtag & 0x1F; 2008 adapter->shadow_vfta[index] |= (1 << bit); 2009 /* Re-init to load the changes */ 2010 ixv_init_locked(adapter); 2011 IXGBE_CORE_UNLOCK(adapter); 2012 } /* ixv_register_vlan */ 2013 2014 /************************************************************************ 2015 * ixv_unregister_vlan 2016 * 2017 * Run via a vlan unconfig EVENT, remove our entry 2018 * in the soft vfta. 2019 ************************************************************************/ 2020 static void 2021 ixv_unregister_vlan(void *arg, struct ifnet *ifp, u16 vtag) 2022 { 2023 struct adapter *adapter = ifp->if_softc; 2024 u16 index, bit; 2025 2026 if (ifp->if_softc != arg) 2027 return; 2028 2029 if ((vtag == 0) || (vtag > 4095)) /* Invalid */ 2030 return; 2031 2032 IXGBE_CORE_LOCK(adapter); 2033 index = (vtag >> 5) & 0x7F; 2034 bit = vtag & 0x1F; 2035 adapter->shadow_vfta[index] &= ~(1 << bit); 2036 /* Re-init to load the changes */ 2037 ixv_init_locked(adapter); 2038 IXGBE_CORE_UNLOCK(adapter); 2039 } /* ixv_unregister_vlan */ 2040 #endif 2041 2042 /************************************************************************ 2043 * ixv_enable_intr 2044 ************************************************************************/ 2045 static void 2046 ixv_enable_intr(struct adapter *adapter) 2047 { 2048 struct ixgbe_hw *hw = &adapter->hw; 2049 struct ix_queue *que = adapter->queues; 2050 u32 mask; 2051 int i; 2052 2053 /* For VTEIAC */ 2054 mask = (1 << adapter->vector); 2055 for (i = 0; i < adapter->num_queues; i++, que++) 2056 mask |= (1 << que->msix); 2057 IXGBE_WRITE_REG(hw, IXGBE_VTEIAC, mask); 2058 2059 /* For VTEIMS */ 2060 IXGBE_WRITE_REG(hw, IXGBE_VTEIMS, (1 << adapter->vector)); 2061 que = adapter->queues; 2062 for (i = 0; i < adapter->num_queues; i++, que++) 2063 ixv_enable_queue(adapter, que->msix); 2064 2065 IXGBE_WRITE_FLUSH(hw); 2066 } /* ixv_enable_intr */ 2067 2068 /************************************************************************ 2069 * ixv_disable_intr 2070 ************************************************************************/ 2071 static void 2072 ixv_disable_intr(struct adapter *adapter) 2073 { 2074 struct ix_queue *que = adapter->queues; 2075 2076 IXGBE_WRITE_REG(&adapter->hw, IXGBE_VTEIAC, 0); 2077 2078 /* disable interrupts other than queues */ 2079 IXGBE_WRITE_REG(&adapter->hw, IXGBE_VTEIMC, adapter->vector); 2080 2081 for (int i = 0; i < adapter->num_queues; i++, que++) 2082 ixv_disable_queue(adapter, que->msix); 2083 2084 IXGBE_WRITE_FLUSH(&adapter->hw); 2085 } /* ixv_disable_intr */ 2086 2087 /************************************************************************ 2088 * ixv_set_ivar 2089 * 2090 * Setup the correct IVAR register for a particular MSI-X interrupt 2091 * - entry is the register array entry 2092 * - vector is the MSI-X vector for this queue 2093 * - type is RX/TX/MISC 2094 ************************************************************************/ 2095 static void 2096 ixv_set_ivar(struct adapter *adapter, u8 entry, u8 vector, s8 type) 2097 { 2098 struct ixgbe_hw *hw = &adapter->hw; 2099 u32 ivar, index; 2100 2101 vector |= IXGBE_IVAR_ALLOC_VAL; 2102 2103 if (type == -1) { /* MISC IVAR */ 2104 ivar = IXGBE_READ_REG(hw, IXGBE_VTIVAR_MISC); 2105 ivar &= ~0xFF; 2106 ivar |= vector; 2107 IXGBE_WRITE_REG(hw, IXGBE_VTIVAR_MISC, ivar); 2108 } else { /* RX/TX IVARS */ 2109 index = (16 * (entry & 1)) + (8 * type); 2110 ivar = IXGBE_READ_REG(hw, IXGBE_VTIVAR(entry >> 1)); 2111 ivar &= ~(0xFF << index); 2112 ivar |= (vector << index); 2113 IXGBE_WRITE_REG(hw, IXGBE_VTIVAR(entry >> 1), ivar); 2114 } 2115 } /* ixv_set_ivar */ 2116 2117 /************************************************************************ 2118 * ixv_configure_ivars 2119 ************************************************************************/ 2120 static void 2121 ixv_configure_ivars(struct adapter *adapter) 2122 { 2123 struct ix_queue *que = adapter->queues; 2124 2125 /* XXX We should sync EITR value calculation with ixgbe.c? */ 2126 2127 for (int i = 0; i < adapter->num_queues; i++, que++) { 2128 /* First the RX queue entry */ 2129 ixv_set_ivar(adapter, i, que->msix, 0); 2130 /* ... and the TX */ 2131 ixv_set_ivar(adapter, i, que->msix, 1); 2132 /* Set an initial value in EITR */ 2133 ixv_eitr_write(que, IXGBE_EITR_DEFAULT); 2134 } 2135 2136 /* For the mailbox interrupt */ 2137 ixv_set_ivar(adapter, 1, adapter->vector, -1); 2138 } /* ixv_configure_ivars */ 2139 2140 2141 /************************************************************************ 2142 * ixv_save_stats 2143 * 2144 * The VF stats registers never have a truly virgin 2145 * starting point, so this routine tries to make an 2146 * artificial one, marking ground zero on attach as 2147 * it were. 2148 ************************************************************************/ 2149 static void 2150 ixv_save_stats(struct adapter *adapter) 2151 { 2152 struct ixgbevf_hw_stats *stats = &adapter->stats.vf; 2153 2154 if (stats->vfgprc.ev_count || stats->vfgptc.ev_count) { 2155 stats->saved_reset_vfgprc += 2156 stats->vfgprc.ev_count - stats->base_vfgprc; 2157 stats->saved_reset_vfgptc += 2158 stats->vfgptc.ev_count - stats->base_vfgptc; 2159 stats->saved_reset_vfgorc += 2160 stats->vfgorc.ev_count - stats->base_vfgorc; 2161 stats->saved_reset_vfgotc += 2162 stats->vfgotc.ev_count - stats->base_vfgotc; 2163 stats->saved_reset_vfmprc += 2164 stats->vfmprc.ev_count - stats->base_vfmprc; 2165 } 2166 } /* ixv_save_stats */ 2167 2168 /************************************************************************ 2169 * ixv_init_stats 2170 ************************************************************************/ 2171 static void 2172 ixv_init_stats(struct adapter *adapter) 2173 { 2174 struct ixgbe_hw *hw = &adapter->hw; 2175 2176 adapter->stats.vf.last_vfgprc = IXGBE_READ_REG(hw, IXGBE_VFGPRC); 2177 adapter->stats.vf.last_vfgorc = IXGBE_READ_REG(hw, IXGBE_VFGORC_LSB); 2178 adapter->stats.vf.last_vfgorc |= 2179 (((u64)(IXGBE_READ_REG(hw, IXGBE_VFGORC_MSB))) << 32); 2180 2181 adapter->stats.vf.last_vfgptc = IXGBE_READ_REG(hw, IXGBE_VFGPTC); 2182 adapter->stats.vf.last_vfgotc = IXGBE_READ_REG(hw, IXGBE_VFGOTC_LSB); 2183 adapter->stats.vf.last_vfgotc |= 2184 (((u64)(IXGBE_READ_REG(hw, IXGBE_VFGOTC_MSB))) << 32); 2185 2186 adapter->stats.vf.last_vfmprc = IXGBE_READ_REG(hw, IXGBE_VFMPRC); 2187 2188 adapter->stats.vf.base_vfgprc = adapter->stats.vf.last_vfgprc; 2189 adapter->stats.vf.base_vfgorc = adapter->stats.vf.last_vfgorc; 2190 adapter->stats.vf.base_vfgptc = adapter->stats.vf.last_vfgptc; 2191 adapter->stats.vf.base_vfgotc = adapter->stats.vf.last_vfgotc; 2192 adapter->stats.vf.base_vfmprc = adapter->stats.vf.last_vfmprc; 2193 } /* ixv_init_stats */ 2194 2195 #define UPDATE_STAT_32(reg, last, count) \ 2196 { \ 2197 u32 current = IXGBE_READ_REG(hw, (reg)); \ 2198 if (current < (last)) \ 2199 count.ev_count += 0x100000000LL; \ 2200 (last) = current; \ 2201 count.ev_count &= 0xFFFFFFFF00000000LL; \ 2202 count.ev_count |= current; \ 2203 } 2204 2205 #define UPDATE_STAT_36(lsb, msb, last, count) \ 2206 { \ 2207 u64 cur_lsb = IXGBE_READ_REG(hw, (lsb)); \ 2208 u64 cur_msb = IXGBE_READ_REG(hw, (msb)); \ 2209 u64 current = ((cur_msb << 32) | cur_lsb); \ 2210 if (current < (last)) \ 2211 count.ev_count += 0x1000000000LL; \ 2212 (last) = current; \ 2213 count.ev_count &= 0xFFFFFFF000000000LL; \ 2214 count.ev_count |= current; \ 2215 } 2216 2217 /************************************************************************ 2218 * ixv_update_stats - Update the board statistics counters. 2219 ************************************************************************/ 2220 void 2221 ixv_update_stats(struct adapter *adapter) 2222 { 2223 struct ixgbe_hw *hw = &adapter->hw; 2224 struct ixgbevf_hw_stats *stats = &adapter->stats.vf; 2225 2226 UPDATE_STAT_32(IXGBE_VFGPRC, stats->last_vfgprc, stats->vfgprc); 2227 UPDATE_STAT_32(IXGBE_VFGPTC, stats->last_vfgptc, stats->vfgptc); 2228 UPDATE_STAT_36(IXGBE_VFGORC_LSB, IXGBE_VFGORC_MSB, stats->last_vfgorc, 2229 stats->vfgorc); 2230 UPDATE_STAT_36(IXGBE_VFGOTC_LSB, IXGBE_VFGOTC_MSB, stats->last_vfgotc, 2231 stats->vfgotc); 2232 UPDATE_STAT_32(IXGBE_VFMPRC, stats->last_vfmprc, stats->vfmprc); 2233 2234 /* Fill out the OS statistics structure */ 2235 /* 2236 * NetBSD: Don't override if_{i|o}{packets|bytes|mcasts} with 2237 * adapter->stats counters. It's required to make ifconfig -z 2238 * (SOICZIFDATA) work. 2239 */ 2240 } /* ixv_update_stats */ 2241 2242 /************************************************************************ 2243 * ixv_sysctl_interrupt_rate_handler 2244 ************************************************************************/ 2245 static int 2246 ixv_sysctl_interrupt_rate_handler(SYSCTLFN_ARGS) 2247 { 2248 struct sysctlnode node = *rnode; 2249 struct ix_queue *que = (struct ix_queue *)node.sysctl_data; 2250 struct adapter *adapter = que->adapter; 2251 uint32_t reg, usec, rate; 2252 int error; 2253 2254 if (que == NULL) 2255 return 0; 2256 reg = IXGBE_READ_REG(&que->adapter->hw, IXGBE_VTEITR(que->msix)); 2257 usec = ((reg & 0x0FF8) >> 3); 2258 if (usec > 0) 2259 rate = 500000 / usec; 2260 else 2261 rate = 0; 2262 node.sysctl_data = &rate; 2263 error = sysctl_lookup(SYSCTLFN_CALL(&node)); 2264 if (error || newp == NULL) 2265 return error; 2266 reg &= ~0xfff; /* default, no limitation */ 2267 if (rate > 0 && rate < 500000) { 2268 if (rate < 1000) 2269 rate = 1000; 2270 reg |= ((4000000/rate) & 0xff8); 2271 /* 2272 * When RSC is used, ITR interval must be larger than 2273 * RSC_DELAY. Currently, we use 2us for RSC_DELAY. 2274 * The minimum value is always greater than 2us on 100M 2275 * (and 10M?(not documented)), but it's not on 1G and higher. 2276 */ 2277 if ((adapter->link_speed != IXGBE_LINK_SPEED_100_FULL) 2278 && (adapter->link_speed != IXGBE_LINK_SPEED_10_FULL)) { 2279 if ((adapter->num_queues > 1) 2280 && (reg < IXGBE_MIN_RSC_EITR_10G1G)) 2281 return EINVAL; 2282 } 2283 ixv_max_interrupt_rate = rate; 2284 } else 2285 ixv_max_interrupt_rate = 0; 2286 ixv_eitr_write(que, reg); 2287 2288 return (0); 2289 } /* ixv_sysctl_interrupt_rate_handler */ 2290 2291 const struct sysctlnode * 2292 ixv_sysctl_instance(struct adapter *adapter) 2293 { 2294 const char *dvname; 2295 struct sysctllog **log; 2296 int rc; 2297 const struct sysctlnode *rnode; 2298 2299 log = &adapter->sysctllog; 2300 dvname = device_xname(adapter->dev); 2301 2302 if ((rc = sysctl_createv(log, 0, NULL, &rnode, 2303 0, CTLTYPE_NODE, dvname, 2304 SYSCTL_DESCR("ixv information and settings"), 2305 NULL, 0, NULL, 0, CTL_HW, CTL_CREATE, CTL_EOL)) != 0) 2306 goto err; 2307 2308 return rnode; 2309 err: 2310 printf("%s: sysctl_createv failed, rc = %d\n", __func__, rc); 2311 return NULL; 2312 } 2313 2314 static void 2315 ixv_add_device_sysctls(struct adapter *adapter) 2316 { 2317 struct sysctllog **log; 2318 const struct sysctlnode *rnode, *cnode; 2319 device_t dev; 2320 2321 dev = adapter->dev; 2322 log = &adapter->sysctllog; 2323 2324 if ((rnode = ixv_sysctl_instance(adapter)) == NULL) { 2325 aprint_error_dev(dev, "could not create sysctl root\n"); 2326 return; 2327 } 2328 2329 if (sysctl_createv(log, 0, &rnode, &cnode, 2330 CTLFLAG_READWRITE, CTLTYPE_INT, 2331 "debug", SYSCTL_DESCR("Debug Info"), 2332 ixv_sysctl_debug, 0, (void *)adapter, 0, CTL_CREATE, CTL_EOL) != 0) 2333 aprint_error_dev(dev, "could not create sysctl\n"); 2334 2335 if (sysctl_createv(log, 0, &rnode, &cnode, 2336 CTLFLAG_READWRITE, CTLTYPE_BOOL, 2337 "enable_aim", SYSCTL_DESCR("Interrupt Moderation"), 2338 NULL, 0, &adapter->enable_aim, 0, CTL_CREATE, CTL_EOL) != 0) 2339 aprint_error_dev(dev, "could not create sysctl\n"); 2340 2341 if (sysctl_createv(log, 0, &rnode, &cnode, 2342 CTLFLAG_READWRITE, CTLTYPE_BOOL, 2343 "txrx_workqueue", SYSCTL_DESCR("Use workqueue for packet processing"), 2344 NULL, 0, &adapter->txrx_use_workqueue, 0, CTL_CREATE, CTL_EOL) != 0) 2345 aprint_error_dev(dev, "could not create sysctl\n"); 2346 } 2347 2348 /************************************************************************ 2349 * ixv_add_stats_sysctls - Add statistic sysctls for the VF. 2350 ************************************************************************/ 2351 static void 2352 ixv_add_stats_sysctls(struct adapter *adapter) 2353 { 2354 device_t dev = adapter->dev; 2355 struct tx_ring *txr = adapter->tx_rings; 2356 struct rx_ring *rxr = adapter->rx_rings; 2357 struct ixgbevf_hw_stats *stats = &adapter->stats.vf; 2358 struct ixgbe_hw *hw = &adapter->hw; 2359 const struct sysctlnode *rnode, *cnode; 2360 struct sysctllog **log = &adapter->sysctllog; 2361 const char *xname = device_xname(dev); 2362 2363 /* Driver Statistics */ 2364 evcnt_attach_dynamic(&adapter->efbig_tx_dma_setup, EVCNT_TYPE_MISC, 2365 NULL, xname, "Driver tx dma soft fail EFBIG"); 2366 evcnt_attach_dynamic(&adapter->mbuf_defrag_failed, EVCNT_TYPE_MISC, 2367 NULL, xname, "m_defrag() failed"); 2368 evcnt_attach_dynamic(&adapter->efbig2_tx_dma_setup, EVCNT_TYPE_MISC, 2369 NULL, xname, "Driver tx dma hard fail EFBIG"); 2370 evcnt_attach_dynamic(&adapter->einval_tx_dma_setup, EVCNT_TYPE_MISC, 2371 NULL, xname, "Driver tx dma hard fail EINVAL"); 2372 evcnt_attach_dynamic(&adapter->other_tx_dma_setup, EVCNT_TYPE_MISC, 2373 NULL, xname, "Driver tx dma hard fail other"); 2374 evcnt_attach_dynamic(&adapter->eagain_tx_dma_setup, EVCNT_TYPE_MISC, 2375 NULL, xname, "Driver tx dma soft fail EAGAIN"); 2376 evcnt_attach_dynamic(&adapter->enomem_tx_dma_setup, EVCNT_TYPE_MISC, 2377 NULL, xname, "Driver tx dma soft fail ENOMEM"); 2378 evcnt_attach_dynamic(&adapter->watchdog_events, EVCNT_TYPE_MISC, 2379 NULL, xname, "Watchdog timeouts"); 2380 evcnt_attach_dynamic(&adapter->tso_err, EVCNT_TYPE_MISC, 2381 NULL, xname, "TSO errors"); 2382 evcnt_attach_dynamic(&adapter->link_irq, EVCNT_TYPE_INTR, 2383 NULL, xname, "Link MSI-X IRQ Handled"); 2384 2385 for (int i = 0; i < adapter->num_queues; i++, rxr++, txr++) { 2386 snprintf(adapter->queues[i].evnamebuf, 2387 sizeof(adapter->queues[i].evnamebuf), "%s q%d", 2388 xname, i); 2389 snprintf(adapter->queues[i].namebuf, 2390 sizeof(adapter->queues[i].namebuf), "q%d", i); 2391 2392 if ((rnode = ixv_sysctl_instance(adapter)) == NULL) { 2393 aprint_error_dev(dev, "could not create sysctl root\n"); 2394 break; 2395 } 2396 2397 if (sysctl_createv(log, 0, &rnode, &rnode, 2398 0, CTLTYPE_NODE, 2399 adapter->queues[i].namebuf, SYSCTL_DESCR("Queue Name"), 2400 NULL, 0, NULL, 0, CTL_CREATE, CTL_EOL) != 0) 2401 break; 2402 2403 if (sysctl_createv(log, 0, &rnode, &cnode, 2404 CTLFLAG_READWRITE, CTLTYPE_INT, 2405 "interrupt_rate", SYSCTL_DESCR("Interrupt Rate"), 2406 ixv_sysctl_interrupt_rate_handler, 0, 2407 (void *)&adapter->queues[i], 0, CTL_CREATE, CTL_EOL) != 0) 2408 break; 2409 2410 if (sysctl_createv(log, 0, &rnode, &cnode, 2411 CTLFLAG_READONLY, CTLTYPE_INT, 2412 "txd_head", SYSCTL_DESCR("Transmit Descriptor Head"), 2413 ixv_sysctl_tdh_handler, 0, (void *)txr, 2414 0, CTL_CREATE, CTL_EOL) != 0) 2415 break; 2416 2417 if (sysctl_createv(log, 0, &rnode, &cnode, 2418 CTLFLAG_READONLY, CTLTYPE_INT, 2419 "txd_tail", SYSCTL_DESCR("Transmit Descriptor Tail"), 2420 ixv_sysctl_tdt_handler, 0, (void *)txr, 2421 0, CTL_CREATE, CTL_EOL) != 0) 2422 break; 2423 2424 evcnt_attach_dynamic(&adapter->queues[i].irqs, EVCNT_TYPE_INTR, 2425 NULL, adapter->queues[i].evnamebuf, "IRQs on queue"); 2426 evcnt_attach_dynamic(&adapter->queues[i].handleq, 2427 EVCNT_TYPE_MISC, NULL, adapter->queues[i].evnamebuf, 2428 "Handled queue in softint"); 2429 evcnt_attach_dynamic(&adapter->queues[i].req, EVCNT_TYPE_MISC, 2430 NULL, adapter->queues[i].evnamebuf, "Requeued in softint"); 2431 evcnt_attach_dynamic(&txr->tso_tx, EVCNT_TYPE_MISC, 2432 NULL, adapter->queues[i].evnamebuf, "TSO"); 2433 evcnt_attach_dynamic(&txr->no_desc_avail, EVCNT_TYPE_MISC, 2434 NULL, adapter->queues[i].evnamebuf, 2435 "Queue No Descriptor Available"); 2436 evcnt_attach_dynamic(&txr->total_packets, EVCNT_TYPE_MISC, 2437 NULL, adapter->queues[i].evnamebuf, 2438 "Queue Packets Transmitted"); 2439 #ifndef IXGBE_LEGACY_TX 2440 evcnt_attach_dynamic(&txr->pcq_drops, EVCNT_TYPE_MISC, 2441 NULL, adapter->queues[i].evnamebuf, 2442 "Packets dropped in pcq"); 2443 #endif 2444 2445 #ifdef LRO 2446 struct lro_ctrl *lro = &rxr->lro; 2447 #endif /* LRO */ 2448 2449 if (sysctl_createv(log, 0, &rnode, &cnode, 2450 CTLFLAG_READONLY, 2451 CTLTYPE_INT, 2452 "rxd_head", SYSCTL_DESCR("Receive Descriptor Head"), 2453 ixv_sysctl_rdh_handler, 0, (void *)rxr, 0, 2454 CTL_CREATE, CTL_EOL) != 0) 2455 break; 2456 2457 if (sysctl_createv(log, 0, &rnode, &cnode, 2458 CTLFLAG_READONLY, 2459 CTLTYPE_INT, 2460 "rxd_tail", SYSCTL_DESCR("Receive Descriptor Tail"), 2461 ixv_sysctl_rdt_handler, 0, (void *)rxr, 0, 2462 CTL_CREATE, CTL_EOL) != 0) 2463 break; 2464 2465 evcnt_attach_dynamic(&rxr->rx_packets, EVCNT_TYPE_MISC, 2466 NULL, adapter->queues[i].evnamebuf, "Queue Packets Received"); 2467 evcnt_attach_dynamic(&rxr->rx_bytes, EVCNT_TYPE_MISC, 2468 NULL, adapter->queues[i].evnamebuf, "Queue Bytes Received"); 2469 evcnt_attach_dynamic(&rxr->rx_copies, EVCNT_TYPE_MISC, 2470 NULL, adapter->queues[i].evnamebuf, "Copied RX Frames"); 2471 evcnt_attach_dynamic(&rxr->no_jmbuf, EVCNT_TYPE_MISC, 2472 NULL, adapter->queues[i].evnamebuf, "Rx no jumbo mbuf"); 2473 evcnt_attach_dynamic(&rxr->rx_discarded, EVCNT_TYPE_MISC, 2474 NULL, adapter->queues[i].evnamebuf, "Rx discarded"); 2475 #ifdef LRO 2476 SYSCTL_ADD_INT(ctx, queue_list, OID_AUTO, "lro_queued", 2477 CTLFLAG_RD, &lro->lro_queued, 0, 2478 "LRO Queued"); 2479 SYSCTL_ADD_INT(ctx, queue_list, OID_AUTO, "lro_flushed", 2480 CTLFLAG_RD, &lro->lro_flushed, 0, 2481 "LRO Flushed"); 2482 #endif /* LRO */ 2483 } 2484 2485 /* MAC stats get their own sub node */ 2486 2487 snprintf(stats->namebuf, 2488 sizeof(stats->namebuf), "%s MAC Statistics", xname); 2489 2490 evcnt_attach_dynamic(&stats->ipcs, EVCNT_TYPE_MISC, NULL, 2491 stats->namebuf, "rx csum offload - IP"); 2492 evcnt_attach_dynamic(&stats->l4cs, EVCNT_TYPE_MISC, NULL, 2493 stats->namebuf, "rx csum offload - L4"); 2494 evcnt_attach_dynamic(&stats->ipcs_bad, EVCNT_TYPE_MISC, NULL, 2495 stats->namebuf, "rx csum offload - IP bad"); 2496 evcnt_attach_dynamic(&stats->l4cs_bad, EVCNT_TYPE_MISC, NULL, 2497 stats->namebuf, "rx csum offload - L4 bad"); 2498 2499 /* Packet Reception Stats */ 2500 evcnt_attach_dynamic(&stats->vfgprc, EVCNT_TYPE_MISC, NULL, 2501 xname, "Good Packets Received"); 2502 evcnt_attach_dynamic(&stats->vfgorc, EVCNT_TYPE_MISC, NULL, 2503 xname, "Good Octets Received"); 2504 evcnt_attach_dynamic(&stats->vfmprc, EVCNT_TYPE_MISC, NULL, 2505 xname, "Multicast Packets Received"); 2506 evcnt_attach_dynamic(&stats->vfgptc, EVCNT_TYPE_MISC, NULL, 2507 xname, "Good Packets Transmitted"); 2508 evcnt_attach_dynamic(&stats->vfgotc, EVCNT_TYPE_MISC, NULL, 2509 xname, "Good Octets Transmitted"); 2510 2511 /* Mailbox Stats */ 2512 evcnt_attach_dynamic(&hw->mbx.stats.msgs_tx, EVCNT_TYPE_MISC, NULL, 2513 xname, "message TXs"); 2514 evcnt_attach_dynamic(&hw->mbx.stats.msgs_rx, EVCNT_TYPE_MISC, NULL, 2515 xname, "message RXs"); 2516 evcnt_attach_dynamic(&hw->mbx.stats.acks, EVCNT_TYPE_MISC, NULL, 2517 xname, "ACKs"); 2518 evcnt_attach_dynamic(&hw->mbx.stats.reqs, EVCNT_TYPE_MISC, NULL, 2519 xname, "REQs"); 2520 evcnt_attach_dynamic(&hw->mbx.stats.rsts, EVCNT_TYPE_MISC, NULL, 2521 xname, "RSTs"); 2522 2523 } /* ixv_add_stats_sysctls */ 2524 2525 /************************************************************************ 2526 * ixv_set_sysctl_value 2527 ************************************************************************/ 2528 static void 2529 ixv_set_sysctl_value(struct adapter *adapter, const char *name, 2530 const char *description, int *limit, int value) 2531 { 2532 device_t dev = adapter->dev; 2533 struct sysctllog **log; 2534 const struct sysctlnode *rnode, *cnode; 2535 2536 log = &adapter->sysctllog; 2537 if ((rnode = ixv_sysctl_instance(adapter)) == NULL) { 2538 aprint_error_dev(dev, "could not create sysctl root\n"); 2539 return; 2540 } 2541 if (sysctl_createv(log, 0, &rnode, &cnode, 2542 CTLFLAG_READWRITE, CTLTYPE_INT, 2543 name, SYSCTL_DESCR(description), 2544 NULL, 0, limit, 0, CTL_CREATE, CTL_EOL) != 0) 2545 aprint_error_dev(dev, "could not create sysctl\n"); 2546 *limit = value; 2547 } /* ixv_set_sysctl_value */ 2548 2549 /************************************************************************ 2550 * ixv_print_debug_info 2551 * 2552 * Called only when em_display_debug_stats is enabled. 2553 * Provides a way to take a look at important statistics 2554 * maintained by the driver and hardware. 2555 ************************************************************************/ 2556 static void 2557 ixv_print_debug_info(struct adapter *adapter) 2558 { 2559 device_t dev = adapter->dev; 2560 struct ixgbe_hw *hw = &adapter->hw; 2561 struct ix_queue *que = adapter->queues; 2562 struct rx_ring *rxr; 2563 struct tx_ring *txr; 2564 #ifdef LRO 2565 struct lro_ctrl *lro; 2566 #endif /* LRO */ 2567 2568 device_printf(dev, "Error Byte Count = %u \n", 2569 IXGBE_READ_REG(hw, IXGBE_ERRBC)); 2570 2571 for (int i = 0; i < adapter->num_queues; i++, que++) { 2572 txr = que->txr; 2573 rxr = que->rxr; 2574 #ifdef LRO 2575 lro = &rxr->lro; 2576 #endif /* LRO */ 2577 device_printf(dev, "QUE(%d) IRQs Handled: %lu\n", 2578 que->msix, (long)que->irqs.ev_count); 2579 device_printf(dev, "RX(%d) Packets Received: %lld\n", 2580 rxr->me, (long long)rxr->rx_packets.ev_count); 2581 device_printf(dev, "RX(%d) Bytes Received: %lu\n", 2582 rxr->me, (long)rxr->rx_bytes.ev_count); 2583 #ifdef LRO 2584 device_printf(dev, "RX(%d) LRO Queued= %lld\n", 2585 rxr->me, (long long)lro->lro_queued); 2586 device_printf(dev, "RX(%d) LRO Flushed= %lld\n", 2587 rxr->me, (long long)lro->lro_flushed); 2588 #endif /* LRO */ 2589 device_printf(dev, "TX(%d) Packets Sent: %lu\n", 2590 txr->me, (long)txr->total_packets.ev_count); 2591 device_printf(dev, "TX(%d) NO Desc Avail: %lu\n", 2592 txr->me, (long)txr->no_desc_avail.ev_count); 2593 } 2594 2595 device_printf(dev, "MBX IRQ Handled: %lu\n", 2596 (long)adapter->link_irq.ev_count); 2597 } /* ixv_print_debug_info */ 2598 2599 /************************************************************************ 2600 * ixv_sysctl_debug 2601 ************************************************************************/ 2602 static int 2603 ixv_sysctl_debug(SYSCTLFN_ARGS) 2604 { 2605 struct sysctlnode node; 2606 struct adapter *adapter; 2607 int error, result; 2608 2609 node = *rnode; 2610 node.sysctl_data = &result; 2611 error = sysctl_lookup(SYSCTLFN_CALL(&node)); 2612 2613 if (error || newp == NULL) 2614 return error; 2615 2616 if (result == 1) { 2617 adapter = (struct adapter *)node.sysctl_data; 2618 ixv_print_debug_info(adapter); 2619 } 2620 2621 return 0; 2622 } /* ixv_sysctl_debug */ 2623 2624 /************************************************************************ 2625 * ixv_init_device_features 2626 ************************************************************************/ 2627 static void 2628 ixv_init_device_features(struct adapter *adapter) 2629 { 2630 adapter->feat_cap = IXGBE_FEATURE_NETMAP 2631 | IXGBE_FEATURE_VF 2632 | IXGBE_FEATURE_RSS 2633 | IXGBE_FEATURE_LEGACY_TX; 2634 2635 /* A tad short on feature flags for VFs, atm. */ 2636 switch (adapter->hw.mac.type) { 2637 case ixgbe_mac_82599_vf: 2638 break; 2639 case ixgbe_mac_X540_vf: 2640 break; 2641 case ixgbe_mac_X550_vf: 2642 case ixgbe_mac_X550EM_x_vf: 2643 case ixgbe_mac_X550EM_a_vf: 2644 adapter->feat_cap |= IXGBE_FEATURE_NEEDS_CTXD; 2645 break; 2646 default: 2647 break; 2648 } 2649 2650 /* Enabled by default... */ 2651 /* Is a virtual function (VF) */ 2652 if (adapter->feat_cap & IXGBE_FEATURE_VF) 2653 adapter->feat_en |= IXGBE_FEATURE_VF; 2654 /* Netmap */ 2655 if (adapter->feat_cap & IXGBE_FEATURE_NETMAP) 2656 adapter->feat_en |= IXGBE_FEATURE_NETMAP; 2657 /* Receive-Side Scaling (RSS) */ 2658 if (adapter->feat_cap & IXGBE_FEATURE_RSS) 2659 adapter->feat_en |= IXGBE_FEATURE_RSS; 2660 /* Needs advanced context descriptor regardless of offloads req'd */ 2661 if (adapter->feat_cap & IXGBE_FEATURE_NEEDS_CTXD) 2662 adapter->feat_en |= IXGBE_FEATURE_NEEDS_CTXD; 2663 2664 /* Enabled via sysctl... */ 2665 /* Legacy (single queue) transmit */ 2666 if ((adapter->feat_cap & IXGBE_FEATURE_LEGACY_TX) && 2667 ixv_enable_legacy_tx) 2668 adapter->feat_en |= IXGBE_FEATURE_LEGACY_TX; 2669 } /* ixv_init_device_features */ 2670 2671 /************************************************************************ 2672 * ixv_shutdown - Shutdown entry point 2673 ************************************************************************/ 2674 #if 0 /* XXX NetBSD ought to register something like this through pmf(9) */ 2675 static int 2676 ixv_shutdown(device_t dev) 2677 { 2678 struct adapter *adapter = device_private(dev); 2679 IXGBE_CORE_LOCK(adapter); 2680 ixv_stop(adapter); 2681 IXGBE_CORE_UNLOCK(adapter); 2682 2683 return (0); 2684 } /* ixv_shutdown */ 2685 #endif 2686 2687 static int 2688 ixv_ifflags_cb(struct ethercom *ec) 2689 { 2690 struct ifnet *ifp = &ec->ec_if; 2691 struct adapter *adapter = ifp->if_softc; 2692 int change = ifp->if_flags ^ adapter->if_flags, rc = 0; 2693 2694 IXGBE_CORE_LOCK(adapter); 2695 2696 if (change != 0) 2697 adapter->if_flags = ifp->if_flags; 2698 2699 if ((change & ~(IFF_CANTCHANGE | IFF_DEBUG)) != 0) 2700 rc = ENETRESET; 2701 2702 /* Set up VLAN support and filter */ 2703 ixv_setup_vlan_support(adapter); 2704 2705 IXGBE_CORE_UNLOCK(adapter); 2706 2707 return rc; 2708 } 2709 2710 2711 /************************************************************************ 2712 * ixv_ioctl - Ioctl entry point 2713 * 2714 * Called when the user wants to configure the interface. 2715 * 2716 * return 0 on success, positive on failure 2717 ************************************************************************/ 2718 static int 2719 ixv_ioctl(struct ifnet *ifp, u_long command, void *data) 2720 { 2721 struct adapter *adapter = ifp->if_softc; 2722 struct ifcapreq *ifcr = data; 2723 struct ifreq *ifr = data; 2724 int error = 0; 2725 int l4csum_en; 2726 const int l4csum = IFCAP_CSUM_TCPv4_Rx|IFCAP_CSUM_UDPv4_Rx| 2727 IFCAP_CSUM_TCPv6_Rx|IFCAP_CSUM_UDPv6_Rx; 2728 2729 switch (command) { 2730 case SIOCSIFFLAGS: 2731 IOCTL_DEBUGOUT("ioctl: SIOCSIFFLAGS (Set Interface Flags)"); 2732 break; 2733 case SIOCADDMULTI: 2734 case SIOCDELMULTI: 2735 IOCTL_DEBUGOUT("ioctl: SIOC(ADD|DEL)MULTI"); 2736 break; 2737 case SIOCSIFMEDIA: 2738 case SIOCGIFMEDIA: 2739 IOCTL_DEBUGOUT("ioctl: SIOCxIFMEDIA (Get/Set Interface Media)"); 2740 break; 2741 case SIOCSIFCAP: 2742 IOCTL_DEBUGOUT("ioctl: SIOCSIFCAP (Set Capabilities)"); 2743 break; 2744 case SIOCSIFMTU: 2745 IOCTL_DEBUGOUT("ioctl: SIOCSIFMTU (Set Interface MTU)"); 2746 break; 2747 default: 2748 IOCTL_DEBUGOUT1("ioctl: UNKNOWN (0x%X)", (int)command); 2749 break; 2750 } 2751 2752 switch (command) { 2753 case SIOCSIFMEDIA: 2754 case SIOCGIFMEDIA: 2755 return ifmedia_ioctl(ifp, ifr, &adapter->media, command); 2756 case SIOCSIFCAP: 2757 /* Layer-4 Rx checksum offload has to be turned on and 2758 * off as a unit. 2759 */ 2760 l4csum_en = ifcr->ifcr_capenable & l4csum; 2761 if (l4csum_en != l4csum && l4csum_en != 0) 2762 return EINVAL; 2763 /*FALLTHROUGH*/ 2764 case SIOCADDMULTI: 2765 case SIOCDELMULTI: 2766 case SIOCSIFFLAGS: 2767 case SIOCSIFMTU: 2768 default: 2769 if ((error = ether_ioctl(ifp, command, data)) != ENETRESET) 2770 return error; 2771 if ((ifp->if_flags & IFF_RUNNING) == 0) 2772 ; 2773 else if (command == SIOCSIFCAP || command == SIOCSIFMTU) { 2774 IXGBE_CORE_LOCK(adapter); 2775 ixv_init_locked(adapter); 2776 IXGBE_CORE_UNLOCK(adapter); 2777 } else if (command == SIOCADDMULTI || command == SIOCDELMULTI) { 2778 /* 2779 * Multicast list has changed; set the hardware filter 2780 * accordingly. 2781 */ 2782 IXGBE_CORE_LOCK(adapter); 2783 ixv_disable_intr(adapter); 2784 ixv_set_multi(adapter); 2785 ixv_enable_intr(adapter); 2786 IXGBE_CORE_UNLOCK(adapter); 2787 } 2788 return 0; 2789 } 2790 } /* ixv_ioctl */ 2791 2792 /************************************************************************ 2793 * ixv_init 2794 ************************************************************************/ 2795 static int 2796 ixv_init(struct ifnet *ifp) 2797 { 2798 struct adapter *adapter = ifp->if_softc; 2799 2800 IXGBE_CORE_LOCK(adapter); 2801 ixv_init_locked(adapter); 2802 IXGBE_CORE_UNLOCK(adapter); 2803 2804 return 0; 2805 } /* ixv_init */ 2806 2807 /************************************************************************ 2808 * ixv_handle_que 2809 ************************************************************************/ 2810 static void 2811 ixv_handle_que(void *context) 2812 { 2813 struct ix_queue *que = context; 2814 struct adapter *adapter = que->adapter; 2815 struct tx_ring *txr = que->txr; 2816 struct ifnet *ifp = adapter->ifp; 2817 bool more; 2818 2819 que->handleq.ev_count++; 2820 2821 if (ifp->if_flags & IFF_RUNNING) { 2822 more = ixgbe_rxeof(que); 2823 IXGBE_TX_LOCK(txr); 2824 more |= ixgbe_txeof(txr); 2825 if (!(adapter->feat_en & IXGBE_FEATURE_LEGACY_TX)) 2826 if (!ixgbe_mq_ring_empty(ifp, txr->txr_interq)) 2827 ixgbe_mq_start_locked(ifp, txr); 2828 /* Only for queue 0 */ 2829 /* NetBSD still needs this for CBQ */ 2830 if ((&adapter->queues[0] == que) 2831 && (!ixgbe_legacy_ring_empty(ifp, NULL))) 2832 ixgbe_legacy_start_locked(ifp, txr); 2833 IXGBE_TX_UNLOCK(txr); 2834 if (more) { 2835 que->req.ev_count++; 2836 if (adapter->txrx_use_workqueue) { 2837 /* 2838 * "enqueued flag" is not required here 2839 * the same as ixg(4). See ixgbe_msix_que(). 2840 */ 2841 workqueue_enqueue(adapter->que_wq, 2842 &que->wq_cookie, curcpu()); 2843 } else 2844 softint_schedule(que->que_si); 2845 return; 2846 } 2847 } 2848 2849 /* Re-enable this interrupt */ 2850 ixv_enable_queue(adapter, que->msix); 2851 2852 return; 2853 } /* ixv_handle_que */ 2854 2855 /************************************************************************ 2856 * ixv_handle_que_work 2857 ************************************************************************/ 2858 static void 2859 ixv_handle_que_work(struct work *wk, void *context) 2860 { 2861 struct ix_queue *que = container_of(wk, struct ix_queue, wq_cookie); 2862 2863 /* 2864 * "enqueued flag" is not required here the same as ixg(4). 2865 * See ixgbe_msix_que(). 2866 */ 2867 ixv_handle_que(que); 2868 } 2869 2870 /************************************************************************ 2871 * ixv_allocate_msix - Setup MSI-X Interrupt resources and handlers 2872 ************************************************************************/ 2873 static int 2874 ixv_allocate_msix(struct adapter *adapter, const struct pci_attach_args *pa) 2875 { 2876 device_t dev = adapter->dev; 2877 struct ix_queue *que = adapter->queues; 2878 struct tx_ring *txr = adapter->tx_rings; 2879 int error, msix_ctrl, rid, vector = 0; 2880 pci_chipset_tag_t pc; 2881 pcitag_t tag; 2882 char intrbuf[PCI_INTRSTR_LEN]; 2883 char wqname[MAXCOMLEN]; 2884 char intr_xname[32]; 2885 const char *intrstr = NULL; 2886 kcpuset_t *affinity; 2887 int cpu_id = 0; 2888 2889 pc = adapter->osdep.pc; 2890 tag = adapter->osdep.tag; 2891 2892 adapter->osdep.nintrs = adapter->num_queues + 1; 2893 if (pci_msix_alloc_exact(pa, &adapter->osdep.intrs, 2894 adapter->osdep.nintrs) != 0) { 2895 aprint_error_dev(dev, 2896 "failed to allocate MSI-X interrupt\n"); 2897 return (ENXIO); 2898 } 2899 2900 kcpuset_create(&affinity, false); 2901 for (int i = 0; i < adapter->num_queues; i++, vector++, que++, txr++) { 2902 snprintf(intr_xname, sizeof(intr_xname), "%s TXRX%d", 2903 device_xname(dev), i); 2904 intrstr = pci_intr_string(pc, adapter->osdep.intrs[i], intrbuf, 2905 sizeof(intrbuf)); 2906 #ifdef IXGBE_MPSAFE 2907 pci_intr_setattr(pc, &adapter->osdep.intrs[i], PCI_INTR_MPSAFE, 2908 true); 2909 #endif 2910 /* Set the handler function */ 2911 que->res = adapter->osdep.ihs[i] = pci_intr_establish_xname(pc, 2912 adapter->osdep.intrs[i], IPL_NET, ixv_msix_que, que, 2913 intr_xname); 2914 if (que->res == NULL) { 2915 pci_intr_release(pc, adapter->osdep.intrs, 2916 adapter->osdep.nintrs); 2917 aprint_error_dev(dev, 2918 "Failed to register QUE handler\n"); 2919 kcpuset_destroy(affinity); 2920 return (ENXIO); 2921 } 2922 que->msix = vector; 2923 adapter->active_queues |= (u64)(1 << que->msix); 2924 2925 cpu_id = i; 2926 /* Round-robin affinity */ 2927 kcpuset_zero(affinity); 2928 kcpuset_set(affinity, cpu_id % ncpu); 2929 error = interrupt_distribute(adapter->osdep.ihs[i], affinity, 2930 NULL); 2931 aprint_normal_dev(dev, "for TX/RX, interrupting at %s", 2932 intrstr); 2933 if (error == 0) 2934 aprint_normal(", bound queue %d to cpu %d\n", 2935 i, cpu_id % ncpu); 2936 else 2937 aprint_normal("\n"); 2938 2939 #ifndef IXGBE_LEGACY_TX 2940 txr->txr_si 2941 = softint_establish(SOFTINT_NET | IXGBE_SOFTINFT_FLAGS, 2942 ixgbe_deferred_mq_start, txr); 2943 #endif 2944 que->que_si 2945 = softint_establish(SOFTINT_NET | IXGBE_SOFTINFT_FLAGS, 2946 ixv_handle_que, que); 2947 if (que->que_si == NULL) { 2948 aprint_error_dev(dev, 2949 "could not establish software interrupt\n"); 2950 } 2951 } 2952 snprintf(wqname, sizeof(wqname), "%sdeferTx", device_xname(dev)); 2953 error = workqueue_create(&adapter->txr_wq, wqname, 2954 ixgbe_deferred_mq_start_work, adapter, IXGBE_WORKQUEUE_PRI, IPL_NET, 2955 IXGBE_WORKQUEUE_FLAGS); 2956 if (error) { 2957 aprint_error_dev(dev, "couldn't create workqueue for deferred Tx\n"); 2958 } 2959 adapter->txr_wq_enqueued = percpu_alloc(sizeof(u_int)); 2960 2961 snprintf(wqname, sizeof(wqname), "%sTxRx", device_xname(dev)); 2962 error = workqueue_create(&adapter->que_wq, wqname, 2963 ixv_handle_que_work, adapter, IXGBE_WORKQUEUE_PRI, IPL_NET, 2964 IXGBE_WORKQUEUE_FLAGS); 2965 if (error) { 2966 aprint_error_dev(dev, 2967 "couldn't create workqueue\n"); 2968 } 2969 2970 /* and Mailbox */ 2971 cpu_id++; 2972 snprintf(intr_xname, sizeof(intr_xname), "%s link", device_xname(dev)); 2973 adapter->vector = vector; 2974 intrstr = pci_intr_string(pc, adapter->osdep.intrs[vector], intrbuf, 2975 sizeof(intrbuf)); 2976 #ifdef IXGBE_MPSAFE 2977 pci_intr_setattr(pc, &adapter->osdep.intrs[vector], PCI_INTR_MPSAFE, 2978 true); 2979 #endif 2980 /* Set the mbx handler function */ 2981 adapter->osdep.ihs[vector] = pci_intr_establish_xname(pc, 2982 adapter->osdep.intrs[vector], IPL_NET, ixv_msix_mbx, adapter, 2983 intr_xname); 2984 if (adapter->osdep.ihs[vector] == NULL) { 2985 adapter->res = NULL; 2986 aprint_error_dev(dev, "Failed to register LINK handler\n"); 2987 kcpuset_destroy(affinity); 2988 return (ENXIO); 2989 } 2990 /* Round-robin affinity */ 2991 kcpuset_zero(affinity); 2992 kcpuset_set(affinity, cpu_id % ncpu); 2993 error = interrupt_distribute(adapter->osdep.ihs[vector], affinity,NULL); 2994 2995 aprint_normal_dev(dev, 2996 "for link, interrupting at %s", intrstr); 2997 if (error == 0) 2998 aprint_normal(", affinity to cpu %d\n", cpu_id % ncpu); 2999 else 3000 aprint_normal("\n"); 3001 3002 /* Tasklets for Mailbox */ 3003 adapter->link_si = softint_establish(SOFTINT_NET |IXGBE_SOFTINFT_FLAGS, 3004 ixv_handle_link, adapter); 3005 /* 3006 * Due to a broken design QEMU will fail to properly 3007 * enable the guest for MSI-X unless the vectors in 3008 * the table are all set up, so we must rewrite the 3009 * ENABLE in the MSI-X control register again at this 3010 * point to cause it to successfully initialize us. 3011 */ 3012 if (adapter->hw.mac.type == ixgbe_mac_82599_vf) { 3013 pci_get_capability(pc, tag, PCI_CAP_MSIX, &rid, NULL); 3014 rid += PCI_MSIX_CTL; 3015 msix_ctrl = pci_conf_read(pc, tag, rid); 3016 msix_ctrl |= PCI_MSIX_CTL_ENABLE; 3017 pci_conf_write(pc, tag, rid, msix_ctrl); 3018 } 3019 3020 kcpuset_destroy(affinity); 3021 return (0); 3022 } /* ixv_allocate_msix */ 3023 3024 /************************************************************************ 3025 * ixv_configure_interrupts - Setup MSI-X resources 3026 * 3027 * Note: The VF device MUST use MSI-X, there is no fallback. 3028 ************************************************************************/ 3029 static int 3030 ixv_configure_interrupts(struct adapter *adapter) 3031 { 3032 device_t dev = adapter->dev; 3033 int want, queues, msgs; 3034 3035 /* Must have at least 2 MSI-X vectors */ 3036 msgs = pci_msix_count(adapter->osdep.pc, adapter->osdep.tag); 3037 if (msgs < 2) { 3038 aprint_error_dev(dev, "MSIX config error\n"); 3039 return (ENXIO); 3040 } 3041 msgs = MIN(msgs, IXG_MAX_NINTR); 3042 3043 /* Figure out a reasonable auto config value */ 3044 queues = (ncpu > (msgs - 1)) ? (msgs - 1) : ncpu; 3045 3046 if (ixv_num_queues != 0) 3047 queues = ixv_num_queues; 3048 else if ((ixv_num_queues == 0) && (queues > IXGBE_VF_MAX_TX_QUEUES)) 3049 queues = IXGBE_VF_MAX_TX_QUEUES; 3050 3051 /* 3052 * Want vectors for the queues, 3053 * plus an additional for mailbox. 3054 */ 3055 want = queues + 1; 3056 if (msgs >= want) 3057 msgs = want; 3058 else { 3059 aprint_error_dev(dev, 3060 "MSI-X Configuration Problem, " 3061 "%d vectors but %d queues wanted!\n", 3062 msgs, want); 3063 return -1; 3064 } 3065 3066 adapter->msix_mem = (void *)1; /* XXX */ 3067 aprint_normal_dev(dev, 3068 "Using MSI-X interrupts with %d vectors\n", msgs); 3069 adapter->num_queues = queues; 3070 3071 return (0); 3072 } /* ixv_configure_interrupts */ 3073 3074 3075 /************************************************************************ 3076 * ixv_handle_link - Tasklet handler for MSI-X MBX interrupts 3077 * 3078 * Done outside of interrupt context since the driver might sleep 3079 ************************************************************************/ 3080 static void 3081 ixv_handle_link(void *context) 3082 { 3083 struct adapter *adapter = context; 3084 3085 IXGBE_CORE_LOCK(adapter); 3086 3087 adapter->hw.mac.ops.check_link(&adapter->hw, &adapter->link_speed, 3088 &adapter->link_up, FALSE); 3089 ixv_update_link_status(adapter); 3090 3091 IXGBE_CORE_UNLOCK(adapter); 3092 } /* ixv_handle_link */ 3093 3094 /************************************************************************ 3095 * ixv_check_link - Used in the local timer to poll for link changes 3096 ************************************************************************/ 3097 static void 3098 ixv_check_link(struct adapter *adapter) 3099 { 3100 3101 KASSERT(mutex_owned(&adapter->core_mtx)); 3102 3103 adapter->hw.mac.get_link_status = TRUE; 3104 3105 adapter->hw.mac.ops.check_link(&adapter->hw, &adapter->link_speed, 3106 &adapter->link_up, FALSE); 3107 ixv_update_link_status(adapter); 3108 } /* ixv_check_link */ 3109
Indexes created Tue Sep 30 20:09:53 GMT 2025