ixv.c revision 1.98
1 /*$NetBSD: ixv.c,v 1.98 2018/05/18 10:09:02 msaitoh Exp $*/ 2 3 /****************************************************************************** 4 5 Copyright (c) 2001-2017, Intel Corporation 6 All rights reserved. 7 8 Redistribution and use in source and binary forms, with or without 9 modification, are permitted provided that the following conditions are met: 10 11 1. Redistributions of source code must retain the above copyright notice, 12 this list of conditions and the following disclaimer. 13 14 2. Redistributions in binary form must reproduce the above copyright 15 notice, this list of conditions and the following disclaimer in the 16 documentation and/or other materials provided with the distribution. 17 18 3. Neither the name of the Intel Corporation nor the names of its 19 contributors may be used to endorse or promote products derived from 20 this software without specific prior written permission. 21 22 THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 23 AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 24 IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 25 ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE 26 LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 27 CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 28 SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 29 INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 30 CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 31 ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 32 POSSIBILITY OF SUCH DAMAGE. 33 34 ******************************************************************************/ 35 /*$FreeBSD: head/sys/dev/ixgbe/if_ixv.c 331224 2018-03-19 20:55:05Z erj $*/ 36 37 #ifdef _KERNEL_OPT 38 #include "opt_inet.h" 39 #include "opt_inet6.h" 40 #include "opt_net_mpsafe.h" 41 #endif 42 43 #include "ixgbe.h" 44 #include "vlan.h" 45 46 /************************************************************************ 47 * Driver version 48 ************************************************************************/ 49 char ixv_driver_version[] = "2.0.1-k"; 50 51 /************************************************************************ 52 * PCI Device ID Table 53 * 54 * Used by probe to select devices to load on 55 * Last field stores an index into ixv_strings 56 * Last entry must be all 0s 57 * 58 * { Vendor ID, Device ID, SubVendor ID, SubDevice ID, String Index } 59 ************************************************************************/ 60 static ixgbe_vendor_info_t ixv_vendor_info_array[] = 61 { 62 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_VF, 0, 0, 0}, 63 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_X540_VF, 0, 0, 0}, 64 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_X550_VF, 0, 0, 0}, 65 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_X550EM_X_VF, 0, 0, 0}, 66 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_X550EM_A_VF, 0, 0, 0}, 67 /* required last entry */ 68 {0, 0, 0, 0, 0} 69 }; 70 71 /************************************************************************ 72 * Table of branding strings 73 ************************************************************************/ 74 static const char *ixv_strings[] = { 75 "Intel(R) PRO/10GbE Virtual Function Network Driver" 76 }; 77 78 /********************************************************************* 79 * Function prototypes 80 *********************************************************************/ 81 static int ixv_probe(device_t, cfdata_t, void *); 82 static void ixv_attach(device_t, device_t, void *); 83 static int ixv_detach(device_t, int); 84 #if 0 85 static int ixv_shutdown(device_t); 86 #endif 87 static int ixv_ifflags_cb(struct ethercom *); 88 static int ixv_ioctl(struct ifnet *, u_long, void *); 89 static int ixv_init(struct ifnet *); 90 static void ixv_init_locked(struct adapter *); 91 static void ixv_ifstop(struct ifnet *, int); 92 static void ixv_stop(void *); 93 static void ixv_init_device_features(struct adapter *); 94 static void ixv_media_status(struct ifnet *, struct ifmediareq *); 95 static int ixv_media_change(struct ifnet *); 96 static int ixv_allocate_pci_resources(struct adapter *, 97 const struct pci_attach_args *); 98 static int ixv_allocate_msix(struct adapter *, 99 const struct pci_attach_args *); 100 static int ixv_configure_interrupts(struct adapter *); 101 static void ixv_free_pci_resources(struct adapter *); 102 static void ixv_local_timer(void *); 103 static void ixv_local_timer_locked(void *); 104 static int ixv_setup_interface(device_t, struct adapter *); 105 static int ixv_negotiate_api(struct adapter *); 106 107 static void ixv_initialize_transmit_units(struct adapter *); 108 static void ixv_initialize_receive_units(struct adapter *); 109 static void ixv_initialize_rss_mapping(struct adapter *); 110 static void ixv_check_link(struct adapter *); 111 112 static void ixv_enable_intr(struct adapter *); 113 static void ixv_disable_intr(struct adapter *); 114 static void ixv_set_multi(struct adapter *); 115 static void ixv_update_link_status(struct adapter *); 116 static int ixv_sysctl_debug(SYSCTLFN_PROTO); 117 static void ixv_set_ivar(struct adapter *, u8, u8, s8); 118 static void ixv_configure_ivars(struct adapter *); 119 static u8 * ixv_mc_array_itr(struct ixgbe_hw *, u8 **, u32 *); 120 static void ixv_eitr_write(struct adapter *, uint32_t, uint32_t); 121 122 static void ixv_setup_vlan_support(struct adapter *); 123 #if 0 124 static void ixv_register_vlan(void *, struct ifnet *, u16); 125 static void ixv_unregister_vlan(void *, struct ifnet *, u16); 126 #endif 127 128 static void ixv_add_device_sysctls(struct adapter *); 129 static void ixv_save_stats(struct adapter *); 130 static void ixv_init_stats(struct adapter *); 131 static void ixv_update_stats(struct adapter *); 132 static void ixv_add_stats_sysctls(struct adapter *); 133 134 135 /* Sysctl handlers */ 136 static void ixv_set_sysctl_value(struct adapter *, const char *, 137 const char *, int *, int); 138 static int ixv_sysctl_interrupt_rate_handler(SYSCTLFN_PROTO); 139 static int ixv_sysctl_rdh_handler(SYSCTLFN_PROTO); 140 static int ixv_sysctl_rdt_handler(SYSCTLFN_PROTO); 141 static int ixv_sysctl_tdt_handler(SYSCTLFN_PROTO); 142 static int ixv_sysctl_tdh_handler(SYSCTLFN_PROTO); 143 144 /* The MSI-X Interrupt handlers */ 145 static int ixv_msix_que(void *); 146 static int ixv_msix_mbx(void *); 147 148 /* Deferred interrupt tasklets */ 149 static void ixv_handle_que(void *); 150 static void ixv_handle_link(void *); 151 152 /* Workqueue handler for deferred work */ 153 static void ixv_handle_que_work(struct work *, void *); 154 155 const struct sysctlnode *ixv_sysctl_instance(struct adapter *); 156 static ixgbe_vendor_info_t *ixv_lookup(const struct pci_attach_args *); 157 158 /************************************************************************ 159 * FreeBSD Device Interface Entry Points 160 ************************************************************************/ 161 CFATTACH_DECL3_NEW(ixv, sizeof(struct adapter), 162 ixv_probe, ixv_attach, ixv_detach, NULL, NULL, NULL, 163 DVF_DETACH_SHUTDOWN); 164 165 #if 0 166 static driver_t ixv_driver = { 167 "ixv", ixv_methods, sizeof(struct adapter), 168 }; 169 170 devclass_t ixv_devclass; 171 DRIVER_MODULE(ixv, pci, ixv_driver, ixv_devclass, 0, 0); 172 MODULE_DEPEND(ixv, pci, 1, 1, 1); 173 MODULE_DEPEND(ixv, ether, 1, 1, 1); 174 #endif 175 176 /* 177 * TUNEABLE PARAMETERS: 178 */ 179 180 /* Number of Queues - do not exceed MSI-X vectors - 1 */ 181 static int ixv_num_queues = 0; 182 #define TUNABLE_INT(__x, __y) 183 TUNABLE_INT("hw.ixv.num_queues", &ixv_num_queues); 184 185 /* 186 * AIM: Adaptive Interrupt Moderation 187 * which means that the interrupt rate 188 * is varied over time based on the 189 * traffic for that interrupt vector 190 */ 191 static bool ixv_enable_aim = false; 192 TUNABLE_INT("hw.ixv.enable_aim", &ixv_enable_aim); 193 194 static int ixv_max_interrupt_rate = (4000000 / IXGBE_LOW_LATENCY); 195 TUNABLE_INT("hw.ixv.max_interrupt_rate", &ixv_max_interrupt_rate); 196 197 /* How many packets rxeof tries to clean at a time */ 198 static int ixv_rx_process_limit = 256; 199 TUNABLE_INT("hw.ixv.rx_process_limit", &ixv_rx_process_limit); 200 201 /* How many packets txeof tries to clean at a time */ 202 static int ixv_tx_process_limit = 256; 203 TUNABLE_INT("hw.ixv.tx_process_limit", &ixv_tx_process_limit); 204 205 /* Which pakcet processing uses workqueue or softint */ 206 static bool ixv_txrx_workqueue = false; 207 208 /* 209 * Number of TX descriptors per ring, 210 * setting higher than RX as this seems 211 * the better performing choice. 212 */ 213 static int ixv_txd = PERFORM_TXD; 214 TUNABLE_INT("hw.ixv.txd", &ixv_txd); 215 216 /* Number of RX descriptors per ring */ 217 static int ixv_rxd = PERFORM_RXD; 218 TUNABLE_INT("hw.ixv.rxd", &ixv_rxd); 219 220 /* Legacy Transmit (single queue) */ 221 static int ixv_enable_legacy_tx = 0; 222 TUNABLE_INT("hw.ixv.enable_legacy_tx", &ixv_enable_legacy_tx); 223 224 #ifdef NET_MPSAFE 225 #define IXGBE_MPSAFE 1 226 #define IXGBE_CALLOUT_FLAGS CALLOUT_MPSAFE 227 #define IXGBE_SOFTINFT_FLAGS SOFTINT_MPSAFE 228 #define IXGBE_WORKQUEUE_FLAGS WQ_PERCPU | WQ_MPSAFE 229 #else 230 #define IXGBE_CALLOUT_FLAGS 0 231 #define IXGBE_SOFTINFT_FLAGS 0 232 #define IXGBE_WORKQUEUE_FLAGS WQ_PERCPU 233 #endif 234 #define IXGBE_WORKQUEUE_PRI PRI_SOFTNET 235 236 #if 0 237 static int (*ixv_start_locked)(struct ifnet *, struct tx_ring *); 238 static int (*ixv_ring_empty)(struct ifnet *, struct buf_ring *); 239 #endif 240 241 /************************************************************************ 242 * ixv_probe - Device identification routine 243 * 244 * Determines if the driver should be loaded on 245 * adapter based on its PCI vendor/device ID. 246 * 247 * return BUS_PROBE_DEFAULT on success, positive on failure 248 ************************************************************************/ 249 static int 250 ixv_probe(device_t dev, cfdata_t cf, void *aux) 251 { 252 #ifdef __HAVE_PCI_MSI_MSIX 253 const struct pci_attach_args *pa = aux; 254 255 return (ixv_lookup(pa) != NULL) ? 1 : 0; 256 #else 257 return 0; 258 #endif 259 } /* ixv_probe */ 260 261 static ixgbe_vendor_info_t * 262 ixv_lookup(const struct pci_attach_args *pa) 263 { 264 ixgbe_vendor_info_t *ent; 265 pcireg_t subid; 266 267 INIT_DEBUGOUT("ixv_lookup: begin"); 268 269 if (PCI_VENDOR(pa->pa_id) != IXGBE_INTEL_VENDOR_ID) 270 return NULL; 271 272 subid = pci_conf_read(pa->pa_pc, pa->pa_tag, PCI_SUBSYS_ID_REG); 273 274 for (ent = ixv_vendor_info_array; ent->vendor_id != 0; ent++) { 275 if ((PCI_VENDOR(pa->pa_id) == ent->vendor_id) && 276 (PCI_PRODUCT(pa->pa_id) == ent->device_id) && 277 ((PCI_SUBSYS_VENDOR(subid) == ent->subvendor_id) || 278 (ent->subvendor_id == 0)) && 279 ((PCI_SUBSYS_ID(subid) == ent->subdevice_id) || 280 (ent->subdevice_id == 0))) { 281 return ent; 282 } 283 } 284 285 return NULL; 286 } 287 288 /************************************************************************ 289 * ixv_attach - Device initialization routine 290 * 291 * Called when the driver is being loaded. 292 * Identifies the type of hardware, allocates all resources 293 * and initializes the hardware. 294 * 295 * return 0 on success, positive on failure 296 ************************************************************************/ 297 static void 298 ixv_attach(device_t parent, device_t dev, void *aux) 299 { 300 struct adapter *adapter; 301 struct ixgbe_hw *hw; 302 int error = 0; 303 pcireg_t id, subid; 304 ixgbe_vendor_info_t *ent; 305 const struct pci_attach_args *pa = aux; 306 const char *apivstr; 307 const char *str; 308 char buf[256]; 309 310 INIT_DEBUGOUT("ixv_attach: begin"); 311 312 /* 313 * Make sure BUSMASTER is set, on a VM under 314 * KVM it may not be and will break things. 315 */ 316 ixgbe_pci_enable_busmaster(pa->pa_pc, pa->pa_tag); 317 318 /* Allocate, clear, and link in our adapter structure */ 319 adapter = device_private(dev); 320 adapter->dev = dev; 321 adapter->hw.back = adapter; 322 hw = &adapter->hw; 323 324 adapter->init_locked = ixv_init_locked; 325 adapter->stop_locked = ixv_stop; 326 327 adapter->osdep.pc = pa->pa_pc; 328 adapter->osdep.tag = pa->pa_tag; 329 if (pci_dma64_available(pa)) 330 adapter->osdep.dmat = pa->pa_dmat64; 331 else 332 adapter->osdep.dmat = pa->pa_dmat; 333 adapter->osdep.attached = false; 334 335 ent = ixv_lookup(pa); 336 337 KASSERT(ent != NULL); 338 339 aprint_normal(": %s, Version - %s\n", 340 ixv_strings[ent->index], ixv_driver_version); 341 342 /* Core Lock Init*/ 343 IXGBE_CORE_LOCK_INIT(adapter, device_xname(dev)); 344 345 /* Do base PCI setup - map BAR0 */ 346 if (ixv_allocate_pci_resources(adapter, pa)) { 347 aprint_error_dev(dev, "ixv_allocate_pci_resources() failed!\n"); 348 error = ENXIO; 349 goto err_out; 350 } 351 352 /* SYSCTL APIs */ 353 ixv_add_device_sysctls(adapter); 354 355 /* Set up the timer callout */ 356 callout_init(&adapter->timer, IXGBE_CALLOUT_FLAGS); 357 358 /* Save off the information about this board */ 359 id = pci_conf_read(pa->pa_pc, pa->pa_tag, PCI_ID_REG); 360 subid = pci_conf_read(pa->pa_pc, pa->pa_tag, PCI_SUBSYS_ID_REG); 361 hw->vendor_id = PCI_VENDOR(id); 362 hw->device_id = PCI_PRODUCT(id); 363 hw->revision_id = 364 PCI_REVISION(pci_conf_read(pa->pa_pc, pa->pa_tag, PCI_CLASS_REG)); 365 hw->subsystem_vendor_id = PCI_SUBSYS_VENDOR(subid); 366 hw->subsystem_device_id = PCI_SUBSYS_ID(subid); 367 368 /* A subset of set_mac_type */ 369 switch (hw->device_id) { 370 case IXGBE_DEV_ID_82599_VF: 371 hw->mac.type = ixgbe_mac_82599_vf; 372 str = "82599 VF"; 373 break; 374 case IXGBE_DEV_ID_X540_VF: 375 hw->mac.type = ixgbe_mac_X540_vf; 376 str = "X540 VF"; 377 break; 378 case IXGBE_DEV_ID_X550_VF: 379 hw->mac.type = ixgbe_mac_X550_vf; 380 str = "X550 VF"; 381 break; 382 case IXGBE_DEV_ID_X550EM_X_VF: 383 hw->mac.type = ixgbe_mac_X550EM_x_vf; 384 str = "X550EM X VF"; 385 break; 386 case IXGBE_DEV_ID_X550EM_A_VF: 387 hw->mac.type = ixgbe_mac_X550EM_a_vf; 388 str = "X550EM A VF"; 389 break; 390 default: 391 /* Shouldn't get here since probe succeeded */ 392 aprint_error_dev(dev, "Unknown device ID!\n"); 393 error = ENXIO; 394 goto err_out; 395 break; 396 } 397 aprint_normal_dev(dev, "device %s\n", str); 398 399 ixv_init_device_features(adapter); 400 401 /* Initialize the shared code */ 402 error = ixgbe_init_ops_vf(hw); 403 if (error) { 404 aprint_error_dev(dev, "ixgbe_init_ops_vf() failed!\n"); 405 error = EIO; 406 goto err_out; 407 } 408 409 /* Setup the mailbox */ 410 ixgbe_init_mbx_params_vf(hw); 411 412 /* Set the right number of segments */ 413 adapter->num_segs = IXGBE_82599_SCATTER; 414 415 /* Reset mbox api to 1.0 */ 416 error = hw->mac.ops.reset_hw(hw); 417 if (error == IXGBE_ERR_RESET_FAILED) 418 aprint_error_dev(dev, "...reset_hw() failure: Reset Failed!\n"); 419 else if (error) 420 aprint_error_dev(dev, "...reset_hw() failed with error %d\n", 421 error); 422 if (error) { 423 error = EIO; 424 goto err_out; 425 } 426 427 error = hw->mac.ops.init_hw(hw); 428 if (error) { 429 aprint_error_dev(dev, "...init_hw() failed!\n"); 430 error = EIO; 431 goto err_out; 432 } 433 434 /* Negotiate mailbox API version */ 435 error = ixv_negotiate_api(adapter); 436 if (error) 437 aprint_normal_dev(dev, 438 "MBX API negotiation failed during attach!\n"); 439 switch (hw->api_version) { 440 case ixgbe_mbox_api_10: 441 apivstr = "1.0"; 442 break; 443 case ixgbe_mbox_api_20: 444 apivstr = "2.0"; 445 break; 446 case ixgbe_mbox_api_11: 447 apivstr = "1.1"; 448 break; 449 case ixgbe_mbox_api_12: 450 apivstr = "1.2"; 451 break; 452 case ixgbe_mbox_api_13: 453 apivstr = "1.3"; 454 break; 455 default: 456 apivstr = "unknown"; 457 break; 458 } 459 aprint_normal_dev(dev, "Mailbox API %s\n", apivstr); 460 461 /* If no mac address was assigned, make a random one */ 462 if (!ixv_check_ether_addr(hw->mac.addr)) { 463 u8 addr[ETHER_ADDR_LEN]; 464 uint64_t rndval = cprng_strong64(); 465 466 memcpy(addr, &rndval, sizeof(addr)); 467 addr[0] &= 0xFE; 468 addr[0] |= 0x02; 469 bcopy(addr, hw->mac.addr, sizeof(addr)); 470 } 471 472 /* Register for VLAN events */ 473 #if 0 /* XXX delete after write? */ 474 adapter->vlan_attach = EVENTHANDLER_REGISTER(vlan_config, 475 ixv_register_vlan, adapter, EVENTHANDLER_PRI_FIRST); 476 adapter->vlan_detach = EVENTHANDLER_REGISTER(vlan_unconfig, 477 ixv_unregister_vlan, adapter, EVENTHANDLER_PRI_FIRST); 478 #endif 479 480 /* Sysctls for limiting the amount of work done in the taskqueues */ 481 ixv_set_sysctl_value(adapter, "rx_processing_limit", 482 "max number of rx packets to process", 483 &adapter->rx_process_limit, ixv_rx_process_limit); 484 485 ixv_set_sysctl_value(adapter, "tx_processing_limit", 486 "max number of tx packets to process", 487 &adapter->tx_process_limit, ixv_tx_process_limit); 488 489 /* Do descriptor calc and sanity checks */ 490 if (((ixv_txd * sizeof(union ixgbe_adv_tx_desc)) % DBA_ALIGN) != 0 || 491 ixv_txd < MIN_TXD || ixv_txd > MAX_TXD) { 492 aprint_error_dev(dev, "TXD config issue, using default!\n"); 493 adapter->num_tx_desc = DEFAULT_TXD; 494 } else 495 adapter->num_tx_desc = ixv_txd; 496 497 if (((ixv_rxd * sizeof(union ixgbe_adv_rx_desc)) % DBA_ALIGN) != 0 || 498 ixv_rxd < MIN_RXD || ixv_rxd > MAX_RXD) { 499 aprint_error_dev(dev, "RXD config issue, using default!\n"); 500 adapter->num_rx_desc = DEFAULT_RXD; 501 } else 502 adapter->num_rx_desc = ixv_rxd; 503 504 /* Setup MSI-X */ 505 error = ixv_configure_interrupts(adapter); 506 if (error) 507 goto err_out; 508 509 /* Allocate our TX/RX Queues */ 510 if (ixgbe_allocate_queues(adapter)) { 511 aprint_error_dev(dev, "ixgbe_allocate_queues() failed!\n"); 512 error = ENOMEM; 513 goto err_out; 514 } 515 516 /* hw.ix defaults init */ 517 adapter->enable_aim = ixv_enable_aim; 518 519 adapter->txrx_use_workqueue = ixv_txrx_workqueue; 520 521 error = ixv_allocate_msix(adapter, pa); 522 if (error) { 523 device_printf(dev, "ixv_allocate_msix() failed!\n"); 524 goto err_late; 525 } 526 527 /* Setup OS specific network interface */ 528 error = ixv_setup_interface(dev, adapter); 529 if (error != 0) { 530 aprint_error_dev(dev, "ixv_setup_interface() failed!\n"); 531 goto err_late; 532 } 533 534 /* Do the stats setup */ 535 ixv_save_stats(adapter); 536 ixv_init_stats(adapter); 537 ixv_add_stats_sysctls(adapter); 538 539 if (adapter->feat_en & IXGBE_FEATURE_NETMAP) 540 ixgbe_netmap_attach(adapter); 541 542 snprintb(buf, sizeof(buf), IXGBE_FEATURE_FLAGS, adapter->feat_cap); 543 aprint_verbose_dev(dev, "feature cap %s\n", buf); 544 snprintb(buf, sizeof(buf), IXGBE_FEATURE_FLAGS, adapter->feat_en); 545 aprint_verbose_dev(dev, "feature ena %s\n", buf); 546 547 INIT_DEBUGOUT("ixv_attach: end"); 548 adapter->osdep.attached = true; 549 550 return; 551 552 err_late: 553 ixgbe_free_transmit_structures(adapter); 554 ixgbe_free_receive_structures(adapter); 555 free(adapter->queues, M_DEVBUF); 556 err_out: 557 ixv_free_pci_resources(adapter); 558 IXGBE_CORE_LOCK_DESTROY(adapter); 559 560 return; 561 } /* ixv_attach */ 562 563 /************************************************************************ 564 * ixv_detach - Device removal routine 565 * 566 * Called when the driver is being removed. 567 * Stops the adapter and deallocates all the resources 568 * that were allocated for driver operation. 569 * 570 * return 0 on success, positive on failure 571 ************************************************************************/ 572 static int 573 ixv_detach(device_t dev, int flags) 574 { 575 struct adapter *adapter = device_private(dev); 576 struct ixgbe_hw *hw = &adapter->hw; 577 struct ix_queue *que = adapter->queues; 578 struct tx_ring *txr = adapter->tx_rings; 579 struct rx_ring *rxr = adapter->rx_rings; 580 struct ixgbevf_hw_stats *stats = &adapter->stats.vf; 581 582 INIT_DEBUGOUT("ixv_detach: begin"); 583 if (adapter->osdep.attached == false) 584 return 0; 585 586 /* Stop the interface. Callouts are stopped in it. */ 587 ixv_ifstop(adapter->ifp, 1); 588 589 #if NVLAN > 0 590 /* Make sure VLANs are not using driver */ 591 if (!VLAN_ATTACHED(&adapter->osdep.ec)) 592 ; /* nothing to do: no VLANs */ 593 else if ((flags & (DETACH_SHUTDOWN|DETACH_FORCE)) != 0) 594 vlan_ifdetach(adapter->ifp); 595 else { 596 aprint_error_dev(dev, "VLANs in use, detach first\n"); 597 return EBUSY; 598 } 599 #endif 600 601 IXGBE_CORE_LOCK(adapter); 602 ixv_stop(adapter); 603 IXGBE_CORE_UNLOCK(adapter); 604 605 for (int i = 0; i < adapter->num_queues; i++, que++, txr++) { 606 if (!(adapter->feat_en & IXGBE_FEATURE_LEGACY_TX)) 607 softint_disestablish(txr->txr_si); 608 softint_disestablish(que->que_si); 609 } 610 if (adapter->txr_wq != NULL) 611 workqueue_destroy(adapter->txr_wq); 612 if (adapter->txr_wq_enqueued != NULL) 613 percpu_free(adapter->txr_wq_enqueued, sizeof(u_int)); 614 if (adapter->que_wq != NULL) 615 workqueue_destroy(adapter->que_wq); 616 617 /* Drain the Mailbox(link) queue */ 618 softint_disestablish(adapter->link_si); 619 620 /* Unregister VLAN events */ 621 #if 0 /* XXX msaitoh delete after write? */ 622 if (adapter->vlan_attach != NULL) 623 EVENTHANDLER_DEREGISTER(vlan_config, adapter->vlan_attach); 624 if (adapter->vlan_detach != NULL) 625 EVENTHANDLER_DEREGISTER(vlan_unconfig, adapter->vlan_detach); 626 #endif 627 628 ether_ifdetach(adapter->ifp); 629 callout_halt(&adapter->timer, NULL); 630 631 if (adapter->feat_en & IXGBE_FEATURE_NETMAP) 632 netmap_detach(adapter->ifp); 633 634 ixv_free_pci_resources(adapter); 635 #if 0 /* XXX the NetBSD port is probably missing something here */ 636 bus_generic_detach(dev); 637 #endif 638 if_detach(adapter->ifp); 639 if_percpuq_destroy(adapter->ipq); 640 641 sysctl_teardown(&adapter->sysctllog); 642 evcnt_detach(&adapter->efbig_tx_dma_setup); 643 evcnt_detach(&adapter->mbuf_defrag_failed); 644 evcnt_detach(&adapter->efbig2_tx_dma_setup); 645 evcnt_detach(&adapter->einval_tx_dma_setup); 646 evcnt_detach(&adapter->other_tx_dma_setup); 647 evcnt_detach(&adapter->eagain_tx_dma_setup); 648 evcnt_detach(&adapter->enomem_tx_dma_setup); 649 evcnt_detach(&adapter->watchdog_events); 650 evcnt_detach(&adapter->tso_err); 651 evcnt_detach(&adapter->link_irq); 652 653 txr = adapter->tx_rings; 654 for (int i = 0; i < adapter->num_queues; i++, rxr++, txr++) { 655 evcnt_detach(&adapter->queues[i].irqs); 656 evcnt_detach(&adapter->queues[i].handleq); 657 evcnt_detach(&adapter->queues[i].req); 658 evcnt_detach(&txr->no_desc_avail); 659 evcnt_detach(&txr->total_packets); 660 evcnt_detach(&txr->tso_tx); 661 #ifndef IXGBE_LEGACY_TX 662 evcnt_detach(&txr->pcq_drops); 663 #endif 664 665 evcnt_detach(&rxr->rx_packets); 666 evcnt_detach(&rxr->rx_bytes); 667 evcnt_detach(&rxr->rx_copies); 668 evcnt_detach(&rxr->no_jmbuf); 669 evcnt_detach(&rxr->rx_discarded); 670 } 671 evcnt_detach(&stats->ipcs); 672 evcnt_detach(&stats->l4cs); 673 evcnt_detach(&stats->ipcs_bad); 674 evcnt_detach(&stats->l4cs_bad); 675 676 /* Packet Reception Stats */ 677 evcnt_detach(&stats->vfgorc); 678 evcnt_detach(&stats->vfgprc); 679 evcnt_detach(&stats->vfmprc); 680 681 /* Packet Transmission Stats */ 682 evcnt_detach(&stats->vfgotc); 683 evcnt_detach(&stats->vfgptc); 684 685 /* Mailbox Stats */ 686 evcnt_detach(&hw->mbx.stats.msgs_tx); 687 evcnt_detach(&hw->mbx.stats.msgs_rx); 688 evcnt_detach(&hw->mbx.stats.acks); 689 evcnt_detach(&hw->mbx.stats.reqs); 690 evcnt_detach(&hw->mbx.stats.rsts); 691 692 ixgbe_free_transmit_structures(adapter); 693 ixgbe_free_receive_structures(adapter); 694 for (int i = 0; i < adapter->num_queues; i++) { 695 struct ix_queue *lque = &adapter->queues[i]; 696 mutex_destroy(&lque->dc_mtx); 697 } 698 free(adapter->queues, M_DEVBUF); 699 700 IXGBE_CORE_LOCK_DESTROY(adapter); 701 702 return (0); 703 } /* ixv_detach */ 704 705 /************************************************************************ 706 * ixv_init_locked - Init entry point 707 * 708 * Used in two ways: It is used by the stack as an init entry 709 * point in network interface structure. It is also used 710 * by the driver as a hw/sw initialization routine to get 711 * to a consistent state. 712 * 713 * return 0 on success, positive on failure 714 ************************************************************************/ 715 static void 716 ixv_init_locked(struct adapter *adapter) 717 { 718 struct ifnet *ifp = adapter->ifp; 719 device_t dev = adapter->dev; 720 struct ixgbe_hw *hw = &adapter->hw; 721 struct ix_queue *que = adapter->queues; 722 int error = 0; 723 uint32_t mask; 724 int i; 725 726 INIT_DEBUGOUT("ixv_init_locked: begin"); 727 KASSERT(mutex_owned(&adapter->core_mtx)); 728 hw->adapter_stopped = FALSE; 729 hw->mac.ops.stop_adapter(hw); 730 callout_stop(&adapter->timer); 731 732 /* reprogram the RAR[0] in case user changed it. */ 733 hw->mac.ops.set_rar(hw, 0, hw->mac.addr, 0, IXGBE_RAH_AV); 734 735 /* Get the latest mac address, User can use a LAA */ 736 memcpy(hw->mac.addr, CLLADDR(ifp->if_sadl), 737 IXGBE_ETH_LENGTH_OF_ADDRESS); 738 hw->mac.ops.set_rar(hw, 0, hw->mac.addr, 0, 1); 739 740 /* Prepare transmit descriptors and buffers */ 741 if (ixgbe_setup_transmit_structures(adapter)) { 742 aprint_error_dev(dev, "Could not setup transmit structures\n"); 743 ixv_stop(adapter); 744 return; 745 } 746 747 /* Reset VF and renegotiate mailbox API version */ 748 hw->mac.ops.reset_hw(hw); 749 hw->mac.ops.start_hw(hw); 750 error = ixv_negotiate_api(adapter); 751 if (error) 752 device_printf(dev, 753 "Mailbox API negotiation failed in init_locked!\n"); 754 755 ixv_initialize_transmit_units(adapter); 756 757 /* Setup Multicast table */ 758 ixv_set_multi(adapter); 759 760 /* 761 * Determine the correct mbuf pool 762 * for doing jumbo/headersplit 763 */ 764 if (ifp->if_mtu > ETHERMTU) 765 adapter->rx_mbuf_sz = MJUMPAGESIZE; 766 else 767 adapter->rx_mbuf_sz = MCLBYTES; 768 769 /* Prepare receive descriptors and buffers */ 770 if (ixgbe_setup_receive_structures(adapter)) { 771 device_printf(dev, "Could not setup receive structures\n"); 772 ixv_stop(adapter); 773 return; 774 } 775 776 /* Configure RX settings */ 777 ixv_initialize_receive_units(adapter); 778 779 #if 0 /* XXX isn't it required? -- msaitoh */ 780 /* Set the various hardware offload abilities */ 781 ifp->if_hwassist = 0; 782 if (ifp->if_capenable & IFCAP_TSO4) 783 ifp->if_hwassist |= CSUM_TSO; 784 if (ifp->if_capenable & IFCAP_TXCSUM) { 785 ifp->if_hwassist |= (CSUM_TCP | CSUM_UDP); 786 #if __FreeBSD_version >= 800000 787 ifp->if_hwassist |= CSUM_SCTP; 788 #endif 789 } 790 #endif 791 792 /* Set up VLAN offload and filter */ 793 ixv_setup_vlan_support(adapter); 794 795 /* Set up MSI-X routing */ 796 ixv_configure_ivars(adapter); 797 798 /* Set up auto-mask */ 799 mask = (1 << adapter->vector); 800 for (i = 0; i < adapter->num_queues; i++, que++) 801 mask |= (1 << que->msix); 802 IXGBE_WRITE_REG(hw, IXGBE_VTEIAM, mask); 803 804 /* Set moderation on the Link interrupt */ 805 ixv_eitr_write(adapter, adapter->vector, IXGBE_LINK_ITR); 806 807 /* Stats init */ 808 ixv_init_stats(adapter); 809 810 /* Config/Enable Link */ 811 hw->mac.get_link_status = TRUE; 812 hw->mac.ops.check_link(hw, &adapter->link_speed, &adapter->link_up, 813 FALSE); 814 815 /* Start watchdog */ 816 callout_reset(&adapter->timer, hz, ixv_local_timer, adapter); 817 818 /* And now turn on interrupts */ 819 ixv_enable_intr(adapter); 820 821 /* Update saved flags. See ixgbe_ifflags_cb() */ 822 adapter->if_flags = ifp->if_flags; 823 824 /* Now inform the stack we're ready */ 825 ifp->if_flags |= IFF_RUNNING; 826 ifp->if_flags &= ~IFF_OACTIVE; 827 828 return; 829 } /* ixv_init_locked */ 830 831 /************************************************************************ 832 * ixv_enable_queue 833 ************************************************************************/ 834 static inline void 835 ixv_enable_queue(struct adapter *adapter, u32 vector) 836 { 837 struct ixgbe_hw *hw = &adapter->hw; 838 struct ix_queue *que = &adapter->queues[vector]; 839 u32 queue = 1 << vector; 840 u32 mask; 841 842 mutex_enter(&que->dc_mtx); 843 if (que->disabled_count > 0 && --que->disabled_count > 0) 844 goto out; 845 846 mask = (IXGBE_EIMS_RTX_QUEUE & queue); 847 IXGBE_WRITE_REG(hw, IXGBE_VTEIMS, mask); 848 out: 849 mutex_exit(&que->dc_mtx); 850 } /* ixv_enable_queue */ 851 852 /************************************************************************ 853 * ixv_disable_queue 854 ************************************************************************/ 855 static inline void 856 ixv_disable_queue(struct adapter *adapter, u32 vector) 857 { 858 struct ixgbe_hw *hw = &adapter->hw; 859 struct ix_queue *que = &adapter->queues[vector]; 860 u64 queue = (u64)(1 << vector); 861 u32 mask; 862 863 mutex_enter(&que->dc_mtx); 864 if (que->disabled_count++ > 0) 865 goto out; 866 867 mask = (IXGBE_EIMS_RTX_QUEUE & queue); 868 IXGBE_WRITE_REG(hw, IXGBE_VTEIMC, mask); 869 out: 870 mutex_exit(&que->dc_mtx); 871 } /* ixv_disable_queue */ 872 873 static inline void 874 ixv_rearm_queues(struct adapter *adapter, u64 queues) 875 { 876 u32 mask = (IXGBE_EIMS_RTX_QUEUE & queues); 877 IXGBE_WRITE_REG(&adapter->hw, IXGBE_VTEICS, mask); 878 } /* ixv_rearm_queues */ 879 880 881 /************************************************************************ 882 * ixv_msix_que - MSI-X Queue Interrupt Service routine 883 ************************************************************************/ 884 static int 885 ixv_msix_que(void *arg) 886 { 887 struct ix_queue *que = arg; 888 struct adapter *adapter = que->adapter; 889 struct tx_ring *txr = que->txr; 890 struct rx_ring *rxr = que->rxr; 891 bool more; 892 u32 newitr = 0; 893 894 ixv_disable_queue(adapter, que->msix); 895 ++que->irqs.ev_count; 896 897 #ifdef __NetBSD__ 898 /* Don't run ixgbe_rxeof in interrupt context */ 899 more = true; 900 #else 901 more = ixgbe_rxeof(que); 902 #endif 903 904 IXGBE_TX_LOCK(txr); 905 ixgbe_txeof(txr); 906 IXGBE_TX_UNLOCK(txr); 907 908 /* Do AIM now? */ 909 910 if (adapter->enable_aim == false) 911 goto no_calc; 912 /* 913 * Do Adaptive Interrupt Moderation: 914 * - Write out last calculated setting 915 * - Calculate based on average size over 916 * the last interval. 917 */ 918 if (que->eitr_setting) 919 ixv_eitr_write(adapter, que->msix, que->eitr_setting); 920 921 que->eitr_setting = 0; 922 923 /* Idle, do nothing */ 924 if ((txr->bytes == 0) && (rxr->bytes == 0)) 925 goto no_calc; 926 927 if ((txr->bytes) && (txr->packets)) 928 newitr = txr->bytes/txr->packets; 929 if ((rxr->bytes) && (rxr->packets)) 930 newitr = max(newitr, (rxr->bytes / rxr->packets)); 931 newitr += 24; /* account for hardware frame, crc */ 932 933 /* set an upper boundary */ 934 newitr = min(newitr, 3000); 935 936 /* Be nice to the mid range */ 937 if ((newitr > 300) && (newitr < 1200)) 938 newitr = (newitr / 3); 939 else 940 newitr = (newitr / 2); 941 942 /* 943 * When RSC is used, ITR interval must be larger than RSC_DELAY. 944 * Currently, we use 2us for RSC_DELAY. The minimum value is always 945 * greater than 2us on 100M (and 10M?(not documented)), but it's not 946 * on 1G and higher. 947 */ 948 if ((adapter->link_speed != IXGBE_LINK_SPEED_100_FULL) 949 && (adapter->link_speed != IXGBE_LINK_SPEED_10_FULL)) { 950 if (newitr < IXGBE_MIN_RSC_EITR_10G1G) 951 newitr = IXGBE_MIN_RSC_EITR_10G1G; 952 } 953 954 /* save for next interrupt */ 955 que->eitr_setting = newitr; 956 957 /* Reset state */ 958 txr->bytes = 0; 959 txr->packets = 0; 960 rxr->bytes = 0; 961 rxr->packets = 0; 962 963 no_calc: 964 if (more) 965 softint_schedule(que->que_si); 966 else /* Re-enable this interrupt */ 967 ixv_enable_queue(adapter, que->msix); 968 969 return 1; 970 } /* ixv_msix_que */ 971 972 /************************************************************************ 973 * ixv_msix_mbx 974 ************************************************************************/ 975 static int 976 ixv_msix_mbx(void *arg) 977 { 978 struct adapter *adapter = arg; 979 struct ixgbe_hw *hw = &adapter->hw; 980 981 ++adapter->link_irq.ev_count; 982 /* NetBSD: We use auto-clear, so it's not required to write VTEICR */ 983 984 /* Link status change */ 985 hw->mac.get_link_status = TRUE; 986 softint_schedule(adapter->link_si); 987 988 IXGBE_WRITE_REG(hw, IXGBE_VTEIMS, (1 << adapter->vector)); 989 990 return 1; 991 } /* ixv_msix_mbx */ 992 993 static void 994 ixv_eitr_write(struct adapter *adapter, uint32_t index, uint32_t itr) 995 { 996 997 /* 998 * Newer devices than 82598 have VF function, so this function is 999 * simple. 1000 */ 1001 itr |= IXGBE_EITR_CNT_WDIS; 1002 1003 IXGBE_WRITE_REG(&adapter->hw, IXGBE_VTEITR(index), itr); 1004 } 1005 1006 1007 /************************************************************************ 1008 * ixv_media_status - Media Ioctl callback 1009 * 1010 * Called whenever the user queries the status of 1011 * the interface using ifconfig. 1012 ************************************************************************/ 1013 static void 1014 ixv_media_status(struct ifnet *ifp, struct ifmediareq *ifmr) 1015 { 1016 struct adapter *adapter = ifp->if_softc; 1017 1018 INIT_DEBUGOUT("ixv_media_status: begin"); 1019 IXGBE_CORE_LOCK(adapter); 1020 ixv_update_link_status(adapter); 1021 1022 ifmr->ifm_status = IFM_AVALID; 1023 ifmr->ifm_active = IFM_ETHER; 1024 1025 if (!adapter->link_active) { 1026 ifmr->ifm_active |= IFM_NONE; 1027 IXGBE_CORE_UNLOCK(adapter); 1028 return; 1029 } 1030 1031 ifmr->ifm_status |= IFM_ACTIVE; 1032 1033 switch (adapter->link_speed) { 1034 case IXGBE_LINK_SPEED_10GB_FULL: 1035 ifmr->ifm_active |= IFM_10G_T | IFM_FDX; 1036 break; 1037 case IXGBE_LINK_SPEED_5GB_FULL: 1038 ifmr->ifm_active |= IFM_5000_T | IFM_FDX; 1039 break; 1040 case IXGBE_LINK_SPEED_2_5GB_FULL: 1041 ifmr->ifm_active |= IFM_2500_T | IFM_FDX; 1042 break; 1043 case IXGBE_LINK_SPEED_1GB_FULL: 1044 ifmr->ifm_active |= IFM_1000_T | IFM_FDX; 1045 break; 1046 case IXGBE_LINK_SPEED_100_FULL: 1047 ifmr->ifm_active |= IFM_100_TX | IFM_FDX; 1048 break; 1049 case IXGBE_LINK_SPEED_10_FULL: 1050 ifmr->ifm_active |= IFM_10_T | IFM_FDX; 1051 break; 1052 } 1053 1054 ifp->if_baudrate = ifmedia_baudrate(ifmr->ifm_active); 1055 1056 IXGBE_CORE_UNLOCK(adapter); 1057 } /* ixv_media_status */ 1058 1059 /************************************************************************ 1060 * ixv_media_change - Media Ioctl callback 1061 * 1062 * Called when the user changes speed/duplex using 1063 * media/mediopt option with ifconfig. 1064 ************************************************************************/ 1065 static int 1066 ixv_media_change(struct ifnet *ifp) 1067 { 1068 struct adapter *adapter = ifp->if_softc; 1069 struct ifmedia *ifm = &adapter->media; 1070 1071 INIT_DEBUGOUT("ixv_media_change: begin"); 1072 1073 if (IFM_TYPE(ifm->ifm_media) != IFM_ETHER) 1074 return (EINVAL); 1075 1076 switch (IFM_SUBTYPE(ifm->ifm_media)) { 1077 case IFM_AUTO: 1078 break; 1079 default: 1080 device_printf(adapter->dev, "Only auto media type\n"); 1081 return (EINVAL); 1082 } 1083 1084 return (0); 1085 } /* ixv_media_change */ 1086 1087 1088 /************************************************************************ 1089 * ixv_negotiate_api 1090 * 1091 * Negotiate the Mailbox API with the PF; 1092 * start with the most featured API first. 1093 ************************************************************************/ 1094 static int 1095 ixv_negotiate_api(struct adapter *adapter) 1096 { 1097 struct ixgbe_hw *hw = &adapter->hw; 1098 int mbx_api[] = { ixgbe_mbox_api_11, 1099 ixgbe_mbox_api_10, 1100 ixgbe_mbox_api_unknown }; 1101 int i = 0; 1102 1103 while (mbx_api[i] != ixgbe_mbox_api_unknown) { 1104 if (ixgbevf_negotiate_api_version(hw, mbx_api[i]) == 0) 1105 return (0); 1106 i++; 1107 } 1108 1109 return (EINVAL); 1110 } /* ixv_negotiate_api */ 1111 1112 1113 /************************************************************************ 1114 * ixv_set_multi - Multicast Update 1115 * 1116 * Called whenever multicast address list is updated. 1117 ************************************************************************/ 1118 static void 1119 ixv_set_multi(struct adapter *adapter) 1120 { 1121 struct ether_multi *enm; 1122 struct ether_multistep step; 1123 struct ethercom *ec = &adapter->osdep.ec; 1124 u8 mta[MAX_NUM_MULTICAST_ADDRESSES * IXGBE_ETH_LENGTH_OF_ADDRESS]; 1125 u8 *update_ptr; 1126 int mcnt = 0; 1127 1128 KASSERT(mutex_owned(&adapter->core_mtx)); 1129 IOCTL_DEBUGOUT("ixv_set_multi: begin"); 1130 1131 ETHER_LOCK(ec); 1132 ETHER_FIRST_MULTI(step, ec, enm); 1133 while (enm != NULL) { 1134 bcopy(enm->enm_addrlo, 1135 &mta[mcnt * IXGBE_ETH_LENGTH_OF_ADDRESS], 1136 IXGBE_ETH_LENGTH_OF_ADDRESS); 1137 mcnt++; 1138 /* XXX This might be required --msaitoh */ 1139 if (mcnt >= MAX_NUM_MULTICAST_ADDRESSES) 1140 break; 1141 ETHER_NEXT_MULTI(step, enm); 1142 } 1143 ETHER_UNLOCK(ec); 1144 1145 update_ptr = mta; 1146 1147 adapter->hw.mac.ops.update_mc_addr_list(&adapter->hw, update_ptr, mcnt, 1148 ixv_mc_array_itr, TRUE); 1149 } /* ixv_set_multi */ 1150 1151 /************************************************************************ 1152 * ixv_mc_array_itr 1153 * 1154 * An iterator function needed by the multicast shared code. 1155 * It feeds the shared code routine the addresses in the 1156 * array of ixv_set_multi() one by one. 1157 ************************************************************************/ 1158 static u8 * 1159 ixv_mc_array_itr(struct ixgbe_hw *hw, u8 **update_ptr, u32 *vmdq) 1160 { 1161 u8 *addr = *update_ptr; 1162 u8 *newptr; 1163 1164 *vmdq = 0; 1165 1166 newptr = addr + IXGBE_ETH_LENGTH_OF_ADDRESS; 1167 *update_ptr = newptr; 1168 1169 return addr; 1170 } /* ixv_mc_array_itr */ 1171 1172 /************************************************************************ 1173 * ixv_local_timer - Timer routine 1174 * 1175 * Checks for link status, updates statistics, 1176 * and runs the watchdog check. 1177 ************************************************************************/ 1178 static void 1179 ixv_local_timer(void *arg) 1180 { 1181 struct adapter *adapter = arg; 1182 1183 IXGBE_CORE_LOCK(adapter); 1184 ixv_local_timer_locked(adapter); 1185 IXGBE_CORE_UNLOCK(adapter); 1186 } 1187 1188 static void 1189 ixv_local_timer_locked(void *arg) 1190 { 1191 struct adapter *adapter = arg; 1192 device_t dev = adapter->dev; 1193 struct ix_queue *que = adapter->queues; 1194 u64 queues = 0; 1195 u64 v0, v1, v2, v3, v4, v5, v6, v7; 1196 int hung = 0; 1197 int i; 1198 1199 KASSERT(mutex_owned(&adapter->core_mtx)); 1200 1201 ixv_check_link(adapter); 1202 1203 /* Stats Update */ 1204 ixv_update_stats(adapter); 1205 1206 /* Update some event counters */ 1207 v0 = v1 = v2 = v3 = v4 = v5 = v6 = v7 = 0; 1208 que = adapter->queues; 1209 for (i = 0; i < adapter->num_queues; i++, que++) { 1210 struct tx_ring *txr = que->txr; 1211 1212 v0 += txr->q_efbig_tx_dma_setup; 1213 v1 += txr->q_mbuf_defrag_failed; 1214 v2 += txr->q_efbig2_tx_dma_setup; 1215 v3 += txr->q_einval_tx_dma_setup; 1216 v4 += txr->q_other_tx_dma_setup; 1217 v5 += txr->q_eagain_tx_dma_setup; 1218 v6 += txr->q_enomem_tx_dma_setup; 1219 v7 += txr->q_tso_err; 1220 } 1221 adapter->efbig_tx_dma_setup.ev_count = v0; 1222 adapter->mbuf_defrag_failed.ev_count = v1; 1223 adapter->efbig2_tx_dma_setup.ev_count = v2; 1224 adapter->einval_tx_dma_setup.ev_count = v3; 1225 adapter->other_tx_dma_setup.ev_count = v4; 1226 adapter->eagain_tx_dma_setup.ev_count = v5; 1227 adapter->enomem_tx_dma_setup.ev_count = v6; 1228 adapter->tso_err.ev_count = v7; 1229 1230 /* 1231 * Check the TX queues status 1232 * - mark hung queues so we don't schedule on them 1233 * - watchdog only if all queues show hung 1234 */ 1235 que = adapter->queues; 1236 for (i = 0; i < adapter->num_queues; i++, que++) { 1237 /* Keep track of queues with work for soft irq */ 1238 if (que->txr->busy) 1239 queues |= ((u64)1 << que->me); 1240 /* 1241 * Each time txeof runs without cleaning, but there 1242 * are uncleaned descriptors it increments busy. If 1243 * we get to the MAX we declare it hung. 1244 */ 1245 if (que->busy == IXGBE_QUEUE_HUNG) { 1246 ++hung; 1247 /* Mark the queue as inactive */ 1248 adapter->active_queues &= ~((u64)1 << que->me); 1249 continue; 1250 } else { 1251 /* Check if we've come back from hung */ 1252 if ((adapter->active_queues & ((u64)1 << que->me)) == 0) 1253 adapter->active_queues |= ((u64)1 << que->me); 1254 } 1255 if (que->busy >= IXGBE_MAX_TX_BUSY) { 1256 device_printf(dev, 1257 "Warning queue %d appears to be hung!\n", i); 1258 que->txr->busy = IXGBE_QUEUE_HUNG; 1259 ++hung; 1260 } 1261 } 1262 1263 /* Only truly watchdog if all queues show hung */ 1264 if (hung == adapter->num_queues) 1265 goto watchdog; 1266 else if (queues != 0) { /* Force an IRQ on queues with work */ 1267 ixv_rearm_queues(adapter, queues); 1268 } 1269 1270 callout_reset(&adapter->timer, hz, ixv_local_timer, adapter); 1271 1272 return; 1273 1274 watchdog: 1275 1276 device_printf(adapter->dev, "Watchdog timeout -- resetting\n"); 1277 adapter->ifp->if_flags &= ~IFF_RUNNING; 1278 adapter->watchdog_events.ev_count++; 1279 ixv_init_locked(adapter); 1280 } /* ixv_local_timer */ 1281 1282 /************************************************************************ 1283 * ixv_update_link_status - Update OS on link state 1284 * 1285 * Note: Only updates the OS on the cached link state. 1286 * The real check of the hardware only happens with 1287 * a link interrupt. 1288 ************************************************************************/ 1289 static void 1290 ixv_update_link_status(struct adapter *adapter) 1291 { 1292 struct ifnet *ifp = adapter->ifp; 1293 device_t dev = adapter->dev; 1294 1295 KASSERT(mutex_owned(&adapter->core_mtx)); 1296 1297 if (adapter->link_up) { 1298 if (adapter->link_active == FALSE) { 1299 if (bootverbose) { 1300 const char *bpsmsg; 1301 1302 switch (adapter->link_speed) { 1303 case IXGBE_LINK_SPEED_10GB_FULL: 1304 bpsmsg = "10 Gbps"; 1305 break; 1306 case IXGBE_LINK_SPEED_5GB_FULL: 1307 bpsmsg = "5 Gbps"; 1308 break; 1309 case IXGBE_LINK_SPEED_2_5GB_FULL: 1310 bpsmsg = "2.5 Gbps"; 1311 break; 1312 case IXGBE_LINK_SPEED_1GB_FULL: 1313 bpsmsg = "1 Gbps"; 1314 break; 1315 case IXGBE_LINK_SPEED_100_FULL: 1316 bpsmsg = "100 Mbps"; 1317 break; 1318 case IXGBE_LINK_SPEED_10_FULL: 1319 bpsmsg = "10 Mbps"; 1320 break; 1321 default: 1322 bpsmsg = "unknown speed"; 1323 break; 1324 } 1325 device_printf(dev, "Link is up %s %s \n", 1326 bpsmsg, "Full Duplex"); 1327 } 1328 adapter->link_active = TRUE; 1329 if_link_state_change(ifp, LINK_STATE_UP); 1330 } 1331 } else { /* Link down */ 1332 if (adapter->link_active == TRUE) { 1333 if (bootverbose) 1334 device_printf(dev, "Link is Down\n"); 1335 if_link_state_change(ifp, LINK_STATE_DOWN); 1336 adapter->link_active = FALSE; 1337 } 1338 } 1339 } /* ixv_update_link_status */ 1340 1341 1342 /************************************************************************ 1343 * ixv_stop - Stop the hardware 1344 * 1345 * Disables all traffic on the adapter by issuing a 1346 * global reset on the MAC and deallocates TX/RX buffers. 1347 ************************************************************************/ 1348 static void 1349 ixv_ifstop(struct ifnet *ifp, int disable) 1350 { 1351 struct adapter *adapter = ifp->if_softc; 1352 1353 IXGBE_CORE_LOCK(adapter); 1354 ixv_stop(adapter); 1355 IXGBE_CORE_UNLOCK(adapter); 1356 } 1357 1358 static void 1359 ixv_stop(void *arg) 1360 { 1361 struct ifnet *ifp; 1362 struct adapter *adapter = arg; 1363 struct ixgbe_hw *hw = &adapter->hw; 1364 1365 ifp = adapter->ifp; 1366 1367 KASSERT(mutex_owned(&adapter->core_mtx)); 1368 1369 INIT_DEBUGOUT("ixv_stop: begin\n"); 1370 ixv_disable_intr(adapter); 1371 1372 /* Tell the stack that the interface is no longer active */ 1373 ifp->if_flags &= ~(IFF_RUNNING | IFF_OACTIVE); 1374 1375 hw->mac.ops.reset_hw(hw); 1376 adapter->hw.adapter_stopped = FALSE; 1377 hw->mac.ops.stop_adapter(hw); 1378 callout_stop(&adapter->timer); 1379 1380 /* reprogram the RAR[0] in case user changed it. */ 1381 hw->mac.ops.set_rar(hw, 0, hw->mac.addr, 0, IXGBE_RAH_AV); 1382 1383 return; 1384 } /* ixv_stop */ 1385 1386 1387 /************************************************************************ 1388 * ixv_allocate_pci_resources 1389 ************************************************************************/ 1390 static int 1391 ixv_allocate_pci_resources(struct adapter *adapter, 1392 const struct pci_attach_args *pa) 1393 { 1394 pcireg_t memtype; 1395 device_t dev = adapter->dev; 1396 bus_addr_t addr; 1397 int flags; 1398 1399 memtype = pci_mapreg_type(pa->pa_pc, pa->pa_tag, PCI_BAR(0)); 1400 switch (memtype) { 1401 case PCI_MAPREG_TYPE_MEM | PCI_MAPREG_MEM_TYPE_32BIT: 1402 case PCI_MAPREG_TYPE_MEM | PCI_MAPREG_MEM_TYPE_64BIT: 1403 adapter->osdep.mem_bus_space_tag = pa->pa_memt; 1404 if (pci_mapreg_info(pa->pa_pc, pa->pa_tag, PCI_BAR(0), 1405 memtype, &addr, &adapter->osdep.mem_size, &flags) != 0) 1406 goto map_err; 1407 if ((flags & BUS_SPACE_MAP_PREFETCHABLE) != 0) { 1408 aprint_normal_dev(dev, "clearing prefetchable bit\n"); 1409 flags &= ~BUS_SPACE_MAP_PREFETCHABLE; 1410 } 1411 if (bus_space_map(adapter->osdep.mem_bus_space_tag, addr, 1412 adapter->osdep.mem_size, flags, 1413 &adapter->osdep.mem_bus_space_handle) != 0) { 1414 map_err: 1415 adapter->osdep.mem_size = 0; 1416 aprint_error_dev(dev, "unable to map BAR0\n"); 1417 return ENXIO; 1418 } 1419 break; 1420 default: 1421 aprint_error_dev(dev, "unexpected type on BAR0\n"); 1422 return ENXIO; 1423 } 1424 1425 /* Pick up the tuneable queues */ 1426 adapter->num_queues = ixv_num_queues; 1427 1428 return (0); 1429 } /* ixv_allocate_pci_resources */ 1430 1431 /************************************************************************ 1432 * ixv_free_pci_resources 1433 ************************************************************************/ 1434 static void 1435 ixv_free_pci_resources(struct adapter * adapter) 1436 { 1437 struct ix_queue *que = adapter->queues; 1438 int rid; 1439 1440 /* 1441 * Release all msix queue resources: 1442 */ 1443 for (int i = 0; i < adapter->num_queues; i++, que++) { 1444 if (que->res != NULL) 1445 pci_intr_disestablish(adapter->osdep.pc, 1446 adapter->osdep.ihs[i]); 1447 } 1448 1449 1450 /* Clean the Mailbox interrupt last */ 1451 rid = adapter->vector; 1452 1453 if (adapter->osdep.ihs[rid] != NULL) { 1454 pci_intr_disestablish(adapter->osdep.pc, 1455 adapter->osdep.ihs[rid]); 1456 adapter->osdep.ihs[rid] = NULL; 1457 } 1458 1459 pci_intr_release(adapter->osdep.pc, adapter->osdep.intrs, 1460 adapter->osdep.nintrs); 1461 1462 if (adapter->osdep.mem_size != 0) { 1463 bus_space_unmap(adapter->osdep.mem_bus_space_tag, 1464 adapter->osdep.mem_bus_space_handle, 1465 adapter->osdep.mem_size); 1466 } 1467 1468 return; 1469 } /* ixv_free_pci_resources */ 1470 1471 /************************************************************************ 1472 * ixv_setup_interface 1473 * 1474 * Setup networking device structure and register an interface. 1475 ************************************************************************/ 1476 static int 1477 ixv_setup_interface(device_t dev, struct adapter *adapter) 1478 { 1479 struct ethercom *ec = &adapter->osdep.ec; 1480 struct ifnet *ifp; 1481 int rv; 1482 1483 INIT_DEBUGOUT("ixv_setup_interface: begin"); 1484 1485 ifp = adapter->ifp = &ec->ec_if; 1486 strlcpy(ifp->if_xname, device_xname(dev), IFNAMSIZ); 1487 ifp->if_baudrate = IF_Gbps(10); 1488 ifp->if_init = ixv_init; 1489 ifp->if_stop = ixv_ifstop; 1490 ifp->if_softc = adapter; 1491 ifp->if_flags = IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST; 1492 #ifdef IXGBE_MPSAFE 1493 ifp->if_extflags = IFEF_MPSAFE; 1494 #endif 1495 ifp->if_ioctl = ixv_ioctl; 1496 if (adapter->feat_en & IXGBE_FEATURE_LEGACY_TX) { 1497 #if 0 1498 ixv_start_locked = ixgbe_legacy_start_locked; 1499 #endif 1500 } else { 1501 ifp->if_transmit = ixgbe_mq_start; 1502 #if 0 1503 ixv_start_locked = ixgbe_mq_start_locked; 1504 #endif 1505 } 1506 ifp->if_start = ixgbe_legacy_start; 1507 IFQ_SET_MAXLEN(&ifp->if_snd, adapter->num_tx_desc - 2); 1508 IFQ_SET_READY(&ifp->if_snd); 1509 1510 rv = if_initialize(ifp); 1511 if (rv != 0) { 1512 aprint_error_dev(dev, "if_initialize failed(%d)\n", rv); 1513 return rv; 1514 } 1515 adapter->ipq = if_percpuq_create(&adapter->osdep.ec.ec_if); 1516 ether_ifattach(ifp, adapter->hw.mac.addr); 1517 /* 1518 * We use per TX queue softint, so if_deferred_start_init() isn't 1519 * used. 1520 */ 1521 if_register(ifp); 1522 ether_set_ifflags_cb(ec, ixv_ifflags_cb); 1523 1524 adapter->max_frame_size = ifp->if_mtu + IXGBE_MTU_HDR; 1525 1526 /* 1527 * Tell the upper layer(s) we support long frames. 1528 */ 1529 ifp->if_hdrlen = sizeof(struct ether_vlan_header); 1530 1531 /* Set capability flags */ 1532 ifp->if_capabilities |= IFCAP_HWCSUM 1533 | IFCAP_TSOv4 1534 | IFCAP_TSOv6; 1535 ifp->if_capenable = 0; 1536 1537 ec->ec_capabilities |= ETHERCAP_VLAN_HWTAGGING 1538 | ETHERCAP_VLAN_HWCSUM 1539 | ETHERCAP_JUMBO_MTU 1540 | ETHERCAP_VLAN_MTU; 1541 1542 /* Enable the above capabilities by default */ 1543 ec->ec_capenable = ec->ec_capabilities; 1544 1545 /* Don't enable LRO by default */ 1546 ifp->if_capabilities |= IFCAP_LRO; 1547 #if 0 1548 ifp->if_capenable = ifp->if_capabilities; 1549 #endif 1550 1551 /* 1552 * Specify the media types supported by this adapter and register 1553 * callbacks to update media and link information 1554 */ 1555 ifmedia_init(&adapter->media, IFM_IMASK, ixv_media_change, 1556 ixv_media_status); 1557 ifmedia_add(&adapter->media, IFM_ETHER | IFM_AUTO, 0, NULL); 1558 ifmedia_set(&adapter->media, IFM_ETHER | IFM_AUTO); 1559 1560 return 0; 1561 } /* ixv_setup_interface */ 1562 1563 1564 /************************************************************************ 1565 * ixv_initialize_transmit_units - Enable transmit unit. 1566 ************************************************************************/ 1567 static void 1568 ixv_initialize_transmit_units(struct adapter *adapter) 1569 { 1570 struct tx_ring *txr = adapter->tx_rings; 1571 struct ixgbe_hw *hw = &adapter->hw; 1572 int i; 1573 1574 for (i = 0; i < adapter->num_queues; i++, txr++) { 1575 u64 tdba = txr->txdma.dma_paddr; 1576 u32 txctrl, txdctl; 1577 int j = txr->me; 1578 1579 /* Set WTHRESH to 8, burst writeback */ 1580 txdctl = IXGBE_READ_REG(hw, IXGBE_VFTXDCTL(j)); 1581 txdctl |= (8 << 16); 1582 IXGBE_WRITE_REG(hw, IXGBE_VFTXDCTL(j), txdctl); 1583 1584 /* Set the HW Tx Head and Tail indices */ 1585 IXGBE_WRITE_REG(&adapter->hw, IXGBE_VFTDH(j), 0); 1586 IXGBE_WRITE_REG(&adapter->hw, IXGBE_VFTDT(j), 0); 1587 1588 /* Set Tx Tail register */ 1589 txr->tail = IXGBE_VFTDT(j); 1590 1591 /* Set Ring parameters */ 1592 IXGBE_WRITE_REG(hw, IXGBE_VFTDBAL(j), 1593 (tdba & 0x00000000ffffffffULL)); 1594 IXGBE_WRITE_REG(hw, IXGBE_VFTDBAH(j), (tdba >> 32)); 1595 IXGBE_WRITE_REG(hw, IXGBE_VFTDLEN(j), 1596 adapter->num_tx_desc * sizeof(struct ixgbe_legacy_tx_desc)); 1597 txctrl = IXGBE_READ_REG(hw, IXGBE_VFDCA_TXCTRL(j)); 1598 txctrl &= ~IXGBE_DCA_TXCTRL_DESC_WRO_EN; 1599 IXGBE_WRITE_REG(hw, IXGBE_VFDCA_TXCTRL(j), txctrl); 1600 1601 /* Now enable */ 1602 txdctl = IXGBE_READ_REG(hw, IXGBE_VFTXDCTL(j)); 1603 txdctl |= IXGBE_TXDCTL_ENABLE; 1604 IXGBE_WRITE_REG(hw, IXGBE_VFTXDCTL(j), txdctl); 1605 } 1606 1607 return; 1608 } /* ixv_initialize_transmit_units */ 1609 1610 1611 /************************************************************************ 1612 * ixv_initialize_rss_mapping 1613 ************************************************************************/ 1614 static void 1615 ixv_initialize_rss_mapping(struct adapter *adapter) 1616 { 1617 struct ixgbe_hw *hw = &adapter->hw; 1618 u32 reta = 0, mrqc, rss_key[10]; 1619 int queue_id; 1620 int i, j; 1621 u32 rss_hash_config; 1622 1623 /* force use default RSS key. */ 1624 #ifdef __NetBSD__ 1625 rss_getkey((uint8_t *) &rss_key); 1626 #else 1627 if (adapter->feat_en & IXGBE_FEATURE_RSS) { 1628 /* Fetch the configured RSS key */ 1629 rss_getkey((uint8_t *)&rss_key); 1630 } else { 1631 /* set up random bits */ 1632 cprng_fast(&rss_key, sizeof(rss_key)); 1633 } 1634 #endif 1635 1636 /* Now fill out hash function seeds */ 1637 for (i = 0; i < 10; i++) 1638 IXGBE_WRITE_REG(hw, IXGBE_VFRSSRK(i), rss_key[i]); 1639 1640 /* Set up the redirection table */ 1641 for (i = 0, j = 0; i < 64; i++, j++) { 1642 if (j == adapter->num_queues) 1643 j = 0; 1644 1645 if (adapter->feat_en & IXGBE_FEATURE_RSS) { 1646 /* 1647 * Fetch the RSS bucket id for the given indirection 1648 * entry. Cap it at the number of configured buckets 1649 * (which is num_queues.) 1650 */ 1651 queue_id = rss_get_indirection_to_bucket(i); 1652 queue_id = queue_id % adapter->num_queues; 1653 } else 1654 queue_id = j; 1655 1656 /* 1657 * The low 8 bits are for hash value (n+0); 1658 * The next 8 bits are for hash value (n+1), etc. 1659 */ 1660 reta >>= 8; 1661 reta |= ((uint32_t)queue_id) << 24; 1662 if ((i & 3) == 3) { 1663 IXGBE_WRITE_REG(hw, IXGBE_VFRETA(i >> 2), reta); 1664 reta = 0; 1665 } 1666 } 1667 1668 /* Perform hash on these packet types */ 1669 if (adapter->feat_en & IXGBE_FEATURE_RSS) 1670 rss_hash_config = rss_gethashconfig(); 1671 else { 1672 /* 1673 * Disable UDP - IP fragments aren't currently being handled 1674 * and so we end up with a mix of 2-tuple and 4-tuple 1675 * traffic. 1676 */ 1677 rss_hash_config = RSS_HASHTYPE_RSS_IPV4 1678 | RSS_HASHTYPE_RSS_TCP_IPV4 1679 | RSS_HASHTYPE_RSS_IPV6 1680 | RSS_HASHTYPE_RSS_TCP_IPV6; 1681 } 1682 1683 mrqc = IXGBE_MRQC_RSSEN; 1684 if (rss_hash_config & RSS_HASHTYPE_RSS_IPV4) 1685 mrqc |= IXGBE_MRQC_RSS_FIELD_IPV4; 1686 if (rss_hash_config & RSS_HASHTYPE_RSS_TCP_IPV4) 1687 mrqc |= IXGBE_MRQC_RSS_FIELD_IPV4_TCP; 1688 if (rss_hash_config & RSS_HASHTYPE_RSS_IPV6) 1689 mrqc |= IXGBE_MRQC_RSS_FIELD_IPV6; 1690 if (rss_hash_config & RSS_HASHTYPE_RSS_TCP_IPV6) 1691 mrqc |= IXGBE_MRQC_RSS_FIELD_IPV6_TCP; 1692 if (rss_hash_config & RSS_HASHTYPE_RSS_IPV6_EX) 1693 device_printf(adapter->dev, "%s: RSS_HASHTYPE_RSS_IPV6_EX defined, but not supported\n", 1694 __func__); 1695 if (rss_hash_config & RSS_HASHTYPE_RSS_TCP_IPV6_EX) 1696 device_printf(adapter->dev, "%s: RSS_HASHTYPE_RSS_TCP_IPV6_EX defined, but not supported\n", 1697 __func__); 1698 if (rss_hash_config & RSS_HASHTYPE_RSS_UDP_IPV4) 1699 mrqc |= IXGBE_MRQC_RSS_FIELD_IPV4_UDP; 1700 if (rss_hash_config & RSS_HASHTYPE_RSS_UDP_IPV6) 1701 mrqc |= IXGBE_MRQC_RSS_FIELD_IPV6_UDP; 1702 if (rss_hash_config & RSS_HASHTYPE_RSS_UDP_IPV6_EX) 1703 device_printf(adapter->dev, "%s: RSS_HASHTYPE_RSS_UDP_IPV6_EX defined, but not supported\n", 1704 __func__); 1705 IXGBE_WRITE_REG(hw, IXGBE_VFMRQC, mrqc); 1706 } /* ixv_initialize_rss_mapping */ 1707 1708 1709 /************************************************************************ 1710 * ixv_initialize_receive_units - Setup receive registers and features. 1711 ************************************************************************/ 1712 static void 1713 ixv_initialize_receive_units(struct adapter *adapter) 1714 { 1715 struct rx_ring *rxr = adapter->rx_rings; 1716 struct ixgbe_hw *hw = &adapter->hw; 1717 struct ifnet *ifp = adapter->ifp; 1718 u32 bufsz, rxcsum, psrtype; 1719 1720 if (ifp->if_mtu > ETHERMTU) 1721 bufsz = 4096 >> IXGBE_SRRCTL_BSIZEPKT_SHIFT; 1722 else 1723 bufsz = 2048 >> IXGBE_SRRCTL_BSIZEPKT_SHIFT; 1724 1725 psrtype = IXGBE_PSRTYPE_TCPHDR 1726 | IXGBE_PSRTYPE_UDPHDR 1727 | IXGBE_PSRTYPE_IPV4HDR 1728 | IXGBE_PSRTYPE_IPV6HDR 1729 | IXGBE_PSRTYPE_L2HDR; 1730 1731 if (adapter->num_queues > 1) 1732 psrtype |= 1 << 29; 1733 1734 IXGBE_WRITE_REG(hw, IXGBE_VFPSRTYPE, psrtype); 1735 1736 /* Tell PF our max_frame size */ 1737 if (ixgbevf_rlpml_set_vf(hw, adapter->max_frame_size) != 0) { 1738 device_printf(adapter->dev, "There is a problem with the PF setup. It is likely the receive unit for this VF will not function correctly.\n"); 1739 } 1740 1741 for (int i = 0; i < adapter->num_queues; i++, rxr++) { 1742 u64 rdba = rxr->rxdma.dma_paddr; 1743 u32 reg, rxdctl; 1744 int j = rxr->me; 1745 1746 /* Disable the queue */ 1747 rxdctl = IXGBE_READ_REG(hw, IXGBE_VFRXDCTL(j)); 1748 rxdctl &= ~IXGBE_RXDCTL_ENABLE; 1749 IXGBE_WRITE_REG(hw, IXGBE_VFRXDCTL(j), rxdctl); 1750 for (int k = 0; k < 10; k++) { 1751 if (IXGBE_READ_REG(hw, IXGBE_VFRXDCTL(j)) & 1752 IXGBE_RXDCTL_ENABLE) 1753 msec_delay(1); 1754 else 1755 break; 1756 } 1757 wmb(); 1758 /* Setup the Base and Length of the Rx Descriptor Ring */ 1759 IXGBE_WRITE_REG(hw, IXGBE_VFRDBAL(j), 1760 (rdba & 0x00000000ffffffffULL)); 1761 IXGBE_WRITE_REG(hw, IXGBE_VFRDBAH(j), (rdba >> 32)); 1762 IXGBE_WRITE_REG(hw, IXGBE_VFRDLEN(j), 1763 adapter->num_rx_desc * sizeof(union ixgbe_adv_rx_desc)); 1764 1765 /* Reset the ring indices */ 1766 IXGBE_WRITE_REG(hw, IXGBE_VFRDH(rxr->me), 0); 1767 IXGBE_WRITE_REG(hw, IXGBE_VFRDT(rxr->me), 0); 1768 1769 /* Set up the SRRCTL register */ 1770 reg = IXGBE_READ_REG(hw, IXGBE_VFSRRCTL(j)); 1771 reg &= ~IXGBE_SRRCTL_BSIZEHDR_MASK; 1772 reg &= ~IXGBE_SRRCTL_BSIZEPKT_MASK; 1773 reg |= bufsz; 1774 reg |= IXGBE_SRRCTL_DESCTYPE_ADV_ONEBUF; 1775 IXGBE_WRITE_REG(hw, IXGBE_VFSRRCTL(j), reg); 1776 1777 /* Capture Rx Tail index */ 1778 rxr->tail = IXGBE_VFRDT(rxr->me); 1779 1780 /* Do the queue enabling last */ 1781 rxdctl |= IXGBE_RXDCTL_ENABLE | IXGBE_RXDCTL_VME; 1782 IXGBE_WRITE_REG(hw, IXGBE_VFRXDCTL(j), rxdctl); 1783 for (int k = 0; k < 10; k++) { 1784 if (IXGBE_READ_REG(hw, IXGBE_VFRXDCTL(j)) & 1785 IXGBE_RXDCTL_ENABLE) 1786 break; 1787 msec_delay(1); 1788 } 1789 wmb(); 1790 1791 /* Set the Tail Pointer */ 1792 #ifdef DEV_NETMAP 1793 /* 1794 * In netmap mode, we must preserve the buffers made 1795 * available to userspace before the if_init() 1796 * (this is true by default on the TX side, because 1797 * init makes all buffers available to userspace). 1798 * 1799 * netmap_reset() and the device specific routines 1800 * (e.g. ixgbe_setup_receive_rings()) map these 1801 * buffers at the end of the NIC ring, so here we 1802 * must set the RDT (tail) register to make sure 1803 * they are not overwritten. 1804 * 1805 * In this driver the NIC ring starts at RDH = 0, 1806 * RDT points to the last slot available for reception (?), 1807 * so RDT = num_rx_desc - 1 means the whole ring is available. 1808 */ 1809 if ((adapter->feat_en & IXGBE_FEATURE_NETMAP) && 1810 (ifp->if_capenable & IFCAP_NETMAP)) { 1811 struct netmap_adapter *na = NA(adapter->ifp); 1812 struct netmap_kring *kring = &na->rx_rings[i]; 1813 int t = na->num_rx_desc - 1 - nm_kr_rxspace(kring); 1814 1815 IXGBE_WRITE_REG(hw, IXGBE_VFRDT(rxr->me), t); 1816 } else 1817 #endif /* DEV_NETMAP */ 1818 IXGBE_WRITE_REG(hw, IXGBE_VFRDT(rxr->me), 1819 adapter->num_rx_desc - 1); 1820 } 1821 1822 rxcsum = IXGBE_READ_REG(hw, IXGBE_RXCSUM); 1823 1824 ixv_initialize_rss_mapping(adapter); 1825 1826 if (adapter->num_queues > 1) { 1827 /* RSS and RX IPP Checksum are mutually exclusive */ 1828 rxcsum |= IXGBE_RXCSUM_PCSD; 1829 } 1830 1831 if (ifp->if_capenable & IFCAP_RXCSUM) 1832 rxcsum |= IXGBE_RXCSUM_PCSD; 1833 1834 if (!(rxcsum & IXGBE_RXCSUM_PCSD)) 1835 rxcsum |= IXGBE_RXCSUM_IPPCSE; 1836 1837 IXGBE_WRITE_REG(hw, IXGBE_RXCSUM, rxcsum); 1838 } /* ixv_initialize_receive_units */ 1839 1840 /************************************************************************ 1841 * ixv_sysctl_tdh_handler - Transmit Descriptor Head handler function 1842 * 1843 * Retrieves the TDH value from the hardware 1844 ************************************************************************/ 1845 static int 1846 ixv_sysctl_tdh_handler(SYSCTLFN_ARGS) 1847 { 1848 struct sysctlnode node = *rnode; 1849 struct tx_ring *txr = (struct tx_ring *)node.sysctl_data; 1850 uint32_t val; 1851 1852 if (!txr) 1853 return (0); 1854 1855 val = IXGBE_READ_REG(&txr->adapter->hw, IXGBE_VFTDH(txr->me)); 1856 node.sysctl_data = &val; 1857 return sysctl_lookup(SYSCTLFN_CALL(&node)); 1858 } /* ixv_sysctl_tdh_handler */ 1859 1860 /************************************************************************ 1861 * ixgbe_sysctl_tdt_handler - Transmit Descriptor Tail handler function 1862 * 1863 * Retrieves the TDT value from the hardware 1864 ************************************************************************/ 1865 static int 1866 ixv_sysctl_tdt_handler(SYSCTLFN_ARGS) 1867 { 1868 struct sysctlnode node = *rnode; 1869 struct tx_ring *txr = (struct tx_ring *)node.sysctl_data; 1870 uint32_t val; 1871 1872 if (!txr) 1873 return (0); 1874 1875 val = IXGBE_READ_REG(&txr->adapter->hw, IXGBE_VFTDT(txr->me)); 1876 node.sysctl_data = &val; 1877 return sysctl_lookup(SYSCTLFN_CALL(&node)); 1878 } /* ixv_sysctl_tdt_handler */ 1879 1880 /************************************************************************ 1881 * ixv_sysctl_rdh_handler - Receive Descriptor Head handler function 1882 * 1883 * Retrieves the RDH value from the hardware 1884 ************************************************************************/ 1885 static int 1886 ixv_sysctl_rdh_handler(SYSCTLFN_ARGS) 1887 { 1888 struct sysctlnode node = *rnode; 1889 struct rx_ring *rxr = (struct rx_ring *)node.sysctl_data; 1890 uint32_t val; 1891 1892 if (!rxr) 1893 return (0); 1894 1895 val = IXGBE_READ_REG(&rxr->adapter->hw, IXGBE_VFRDH(rxr->me)); 1896 node.sysctl_data = &val; 1897 return sysctl_lookup(SYSCTLFN_CALL(&node)); 1898 } /* ixv_sysctl_rdh_handler */ 1899 1900 /************************************************************************ 1901 * ixv_sysctl_rdt_handler - Receive Descriptor Tail handler function 1902 * 1903 * Retrieves the RDT value from the hardware 1904 ************************************************************************/ 1905 static int 1906 ixv_sysctl_rdt_handler(SYSCTLFN_ARGS) 1907 { 1908 struct sysctlnode node = *rnode; 1909 struct rx_ring *rxr = (struct rx_ring *)node.sysctl_data; 1910 uint32_t val; 1911 1912 if (!rxr) 1913 return (0); 1914 1915 val = IXGBE_READ_REG(&rxr->adapter->hw, IXGBE_VFRDT(rxr->me)); 1916 node.sysctl_data = &val; 1917 return sysctl_lookup(SYSCTLFN_CALL(&node)); 1918 } /* ixv_sysctl_rdt_handler */ 1919 1920 /************************************************************************ 1921 * ixv_setup_vlan_support 1922 ************************************************************************/ 1923 static void 1924 ixv_setup_vlan_support(struct adapter *adapter) 1925 { 1926 struct ethercom *ec = &adapter->osdep.ec; 1927 struct ixgbe_hw *hw = &adapter->hw; 1928 struct rx_ring *rxr; 1929 u32 ctrl, vid, vfta, retry; 1930 1931 /* 1932 * We get here thru init_locked, meaning 1933 * a soft reset, this has already cleared 1934 * the VFTA and other state, so if there 1935 * have been no vlan's registered do nothing. 1936 */ 1937 if (!VLAN_ATTACHED(ec)) 1938 return; 1939 1940 /* Enable the queues */ 1941 for (int i = 0; i < adapter->num_queues; i++) { 1942 rxr = &adapter->rx_rings[i]; 1943 ctrl = IXGBE_READ_REG(hw, IXGBE_VFRXDCTL(rxr->me)); 1944 ctrl |= IXGBE_RXDCTL_VME; 1945 IXGBE_WRITE_REG(hw, IXGBE_VFRXDCTL(rxr->me), ctrl); 1946 /* 1947 * Let Rx path know that it needs to store VLAN tag 1948 * as part of extra mbuf info. 1949 */ 1950 rxr->vtag_strip = TRUE; 1951 } 1952 1953 #if 1 1954 /* XXX dirty hack. Enable all VIDs */ 1955 for (int i = 0; i < IXGBE_VFTA_SIZE; i++) 1956 adapter->shadow_vfta[i] = 0xffffffff; 1957 #endif 1958 /* 1959 * A soft reset zero's out the VFTA, so 1960 * we need to repopulate it now. 1961 */ 1962 for (int i = 0; i < IXGBE_VFTA_SIZE; i++) { 1963 if (adapter->shadow_vfta[i] == 0) 1964 continue; 1965 vfta = adapter->shadow_vfta[i]; 1966 /* 1967 * Reconstruct the vlan id's 1968 * based on the bits set in each 1969 * of the array ints. 1970 */ 1971 for (int j = 0; j < 32; j++) { 1972 retry = 0; 1973 if ((vfta & (1 << j)) == 0) 1974 continue; 1975 vid = (i * 32) + j; 1976 /* Call the shared code mailbox routine */ 1977 while (hw->mac.ops.set_vfta(hw, vid, 0, TRUE, FALSE)) { 1978 if (++retry > 5) 1979 break; 1980 } 1981 } 1982 } 1983 } /* ixv_setup_vlan_support */ 1984 1985 #if 0 /* XXX Badly need to overhaul vlan(4) on NetBSD. */ 1986 /************************************************************************ 1987 * ixv_register_vlan 1988 * 1989 * Run via a vlan config EVENT, it enables us to use the 1990 * HW Filter table since we can get the vlan id. This just 1991 * creates the entry in the soft version of the VFTA, init 1992 * will repopulate the real table. 1993 ************************************************************************/ 1994 static void 1995 ixv_register_vlan(void *arg, struct ifnet *ifp, u16 vtag) 1996 { 1997 struct adapter *adapter = ifp->if_softc; 1998 u16 index, bit; 1999 2000 if (ifp->if_softc != arg) /* Not our event */ 2001 return; 2002 2003 if ((vtag == 0) || (vtag > 4095)) /* Invalid */ 2004 return; 2005 2006 IXGBE_CORE_LOCK(adapter); 2007 index = (vtag >> 5) & 0x7F; 2008 bit = vtag & 0x1F; 2009 adapter->shadow_vfta[index] |= (1 << bit); 2010 /* Re-init to load the changes */ 2011 ixv_init_locked(adapter); 2012 IXGBE_CORE_UNLOCK(adapter); 2013 } /* ixv_register_vlan */ 2014 2015 /************************************************************************ 2016 * ixv_unregister_vlan 2017 * 2018 * Run via a vlan unconfig EVENT, remove our entry 2019 * in the soft vfta. 2020 ************************************************************************/ 2021 static void 2022 ixv_unregister_vlan(void *arg, struct ifnet *ifp, u16 vtag) 2023 { 2024 struct adapter *adapter = ifp->if_softc; 2025 u16 index, bit; 2026 2027 if (ifp->if_softc != arg) 2028 return; 2029 2030 if ((vtag == 0) || (vtag > 4095)) /* Invalid */ 2031 return; 2032 2033 IXGBE_CORE_LOCK(adapter); 2034 index = (vtag >> 5) & 0x7F; 2035 bit = vtag & 0x1F; 2036 adapter->shadow_vfta[index] &= ~(1 << bit); 2037 /* Re-init to load the changes */ 2038 ixv_init_locked(adapter); 2039 IXGBE_CORE_UNLOCK(adapter); 2040 } /* ixv_unregister_vlan */ 2041 #endif 2042 2043 /************************************************************************ 2044 * ixv_enable_intr 2045 ************************************************************************/ 2046 static void 2047 ixv_enable_intr(struct adapter *adapter) 2048 { 2049 struct ixgbe_hw *hw = &adapter->hw; 2050 struct ix_queue *que = adapter->queues; 2051 u32 mask; 2052 int i; 2053 2054 /* For VTEIAC */ 2055 mask = (1 << adapter->vector); 2056 for (i = 0; i < adapter->num_queues; i++, que++) 2057 mask |= (1 << que->msix); 2058 IXGBE_WRITE_REG(hw, IXGBE_VTEIAC, mask); 2059 2060 /* For VTEIMS */ 2061 IXGBE_WRITE_REG(hw, IXGBE_VTEIMS, (1 << adapter->vector)); 2062 que = adapter->queues; 2063 for (i = 0; i < adapter->num_queues; i++, que++) 2064 ixv_enable_queue(adapter, que->msix); 2065 2066 IXGBE_WRITE_FLUSH(hw); 2067 } /* ixv_enable_intr */ 2068 2069 /************************************************************************ 2070 * ixv_disable_intr 2071 ************************************************************************/ 2072 static void 2073 ixv_disable_intr(struct adapter *adapter) 2074 { 2075 struct ix_queue *que = adapter->queues; 2076 2077 IXGBE_WRITE_REG(&adapter->hw, IXGBE_VTEIAC, 0); 2078 2079 /* disable interrupts other than queues */ 2080 IXGBE_WRITE_REG(&adapter->hw, IXGBE_VTEIMC, adapter->vector); 2081 2082 for (int i = 0; i < adapter->num_queues; i++, que++) 2083 ixv_disable_queue(adapter, que->msix); 2084 2085 IXGBE_WRITE_FLUSH(&adapter->hw); 2086 } /* ixv_disable_intr */ 2087 2088 /************************************************************************ 2089 * ixv_set_ivar 2090 * 2091 * Setup the correct IVAR register for a particular MSI-X interrupt 2092 * - entry is the register array entry 2093 * - vector is the MSI-X vector for this queue 2094 * - type is RX/TX/MISC 2095 ************************************************************************/ 2096 static void 2097 ixv_set_ivar(struct adapter *adapter, u8 entry, u8 vector, s8 type) 2098 { 2099 struct ixgbe_hw *hw = &adapter->hw; 2100 u32 ivar, index; 2101 2102 vector |= IXGBE_IVAR_ALLOC_VAL; 2103 2104 if (type == -1) { /* MISC IVAR */ 2105 ivar = IXGBE_READ_REG(hw, IXGBE_VTIVAR_MISC); 2106 ivar &= ~0xFF; 2107 ivar |= vector; 2108 IXGBE_WRITE_REG(hw, IXGBE_VTIVAR_MISC, ivar); 2109 } else { /* RX/TX IVARS */ 2110 index = (16 * (entry & 1)) + (8 * type); 2111 ivar = IXGBE_READ_REG(hw, IXGBE_VTIVAR(entry >> 1)); 2112 ivar &= ~(0xFF << index); 2113 ivar |= (vector << index); 2114 IXGBE_WRITE_REG(hw, IXGBE_VTIVAR(entry >> 1), ivar); 2115 } 2116 } /* ixv_set_ivar */ 2117 2118 /************************************************************************ 2119 * ixv_configure_ivars 2120 ************************************************************************/ 2121 static void 2122 ixv_configure_ivars(struct adapter *adapter) 2123 { 2124 struct ix_queue *que = adapter->queues; 2125 2126 /* XXX We should sync EITR value calculation with ixgbe.c? */ 2127 2128 for (int i = 0; i < adapter->num_queues; i++, que++) { 2129 /* First the RX queue entry */ 2130 ixv_set_ivar(adapter, i, que->msix, 0); 2131 /* ... and the TX */ 2132 ixv_set_ivar(adapter, i, que->msix, 1); 2133 /* Set an initial value in EITR */ 2134 ixv_eitr_write(adapter, que->msix, IXGBE_EITR_DEFAULT); 2135 } 2136 2137 /* For the mailbox interrupt */ 2138 ixv_set_ivar(adapter, 1, adapter->vector, -1); 2139 } /* ixv_configure_ivars */ 2140 2141 2142 /************************************************************************ 2143 * ixv_save_stats 2144 * 2145 * The VF stats registers never have a truly virgin 2146 * starting point, so this routine tries to make an 2147 * artificial one, marking ground zero on attach as 2148 * it were. 2149 ************************************************************************/ 2150 static void 2151 ixv_save_stats(struct adapter *adapter) 2152 { 2153 struct ixgbevf_hw_stats *stats = &adapter->stats.vf; 2154 2155 if (stats->vfgprc.ev_count || stats->vfgptc.ev_count) { 2156 stats->saved_reset_vfgprc += 2157 stats->vfgprc.ev_count - stats->base_vfgprc; 2158 stats->saved_reset_vfgptc += 2159 stats->vfgptc.ev_count - stats->base_vfgptc; 2160 stats->saved_reset_vfgorc += 2161 stats->vfgorc.ev_count - stats->base_vfgorc; 2162 stats->saved_reset_vfgotc += 2163 stats->vfgotc.ev_count - stats->base_vfgotc; 2164 stats->saved_reset_vfmprc += 2165 stats->vfmprc.ev_count - stats->base_vfmprc; 2166 } 2167 } /* ixv_save_stats */ 2168 2169 /************************************************************************ 2170 * ixv_init_stats 2171 ************************************************************************/ 2172 static void 2173 ixv_init_stats(struct adapter *adapter) 2174 { 2175 struct ixgbe_hw *hw = &adapter->hw; 2176 2177 adapter->stats.vf.last_vfgprc = IXGBE_READ_REG(hw, IXGBE_VFGPRC); 2178 adapter->stats.vf.last_vfgorc = IXGBE_READ_REG(hw, IXGBE_VFGORC_LSB); 2179 adapter->stats.vf.last_vfgorc |= 2180 (((u64)(IXGBE_READ_REG(hw, IXGBE_VFGORC_MSB))) << 32); 2181 2182 adapter->stats.vf.last_vfgptc = IXGBE_READ_REG(hw, IXGBE_VFGPTC); 2183 adapter->stats.vf.last_vfgotc = IXGBE_READ_REG(hw, IXGBE_VFGOTC_LSB); 2184 adapter->stats.vf.last_vfgotc |= 2185 (((u64)(IXGBE_READ_REG(hw, IXGBE_VFGOTC_MSB))) << 32); 2186 2187 adapter->stats.vf.last_vfmprc = IXGBE_READ_REG(hw, IXGBE_VFMPRC); 2188 2189 adapter->stats.vf.base_vfgprc = adapter->stats.vf.last_vfgprc; 2190 adapter->stats.vf.base_vfgorc = adapter->stats.vf.last_vfgorc; 2191 adapter->stats.vf.base_vfgptc = adapter->stats.vf.last_vfgptc; 2192 adapter->stats.vf.base_vfgotc = adapter->stats.vf.last_vfgotc; 2193 adapter->stats.vf.base_vfmprc = adapter->stats.vf.last_vfmprc; 2194 } /* ixv_init_stats */ 2195 2196 #define UPDATE_STAT_32(reg, last, count) \ 2197 { \ 2198 u32 current = IXGBE_READ_REG(hw, (reg)); \ 2199 if (current < (last)) \ 2200 count.ev_count += 0x100000000LL; \ 2201 (last) = current; \ 2202 count.ev_count &= 0xFFFFFFFF00000000LL; \ 2203 count.ev_count |= current; \ 2204 } 2205 2206 #define UPDATE_STAT_36(lsb, msb, last, count) \ 2207 { \ 2208 u64 cur_lsb = IXGBE_READ_REG(hw, (lsb)); \ 2209 u64 cur_msb = IXGBE_READ_REG(hw, (msb)); \ 2210 u64 current = ((cur_msb << 32) | cur_lsb); \ 2211 if (current < (last)) \ 2212 count.ev_count += 0x1000000000LL; \ 2213 (last) = current; \ 2214 count.ev_count &= 0xFFFFFFF000000000LL; \ 2215 count.ev_count |= current; \ 2216 } 2217 2218 /************************************************************************ 2219 * ixv_update_stats - Update the board statistics counters. 2220 ************************************************************************/ 2221 void 2222 ixv_update_stats(struct adapter *adapter) 2223 { 2224 struct ixgbe_hw *hw = &adapter->hw; 2225 struct ixgbevf_hw_stats *stats = &adapter->stats.vf; 2226 2227 UPDATE_STAT_32(IXGBE_VFGPRC, stats->last_vfgprc, stats->vfgprc); 2228 UPDATE_STAT_32(IXGBE_VFGPTC, stats->last_vfgptc, stats->vfgptc); 2229 UPDATE_STAT_36(IXGBE_VFGORC_LSB, IXGBE_VFGORC_MSB, stats->last_vfgorc, 2230 stats->vfgorc); 2231 UPDATE_STAT_36(IXGBE_VFGOTC_LSB, IXGBE_VFGOTC_MSB, stats->last_vfgotc, 2232 stats->vfgotc); 2233 UPDATE_STAT_32(IXGBE_VFMPRC, stats->last_vfmprc, stats->vfmprc); 2234 2235 /* Fill out the OS statistics structure */ 2236 /* 2237 * NetBSD: Don't override if_{i|o}{packets|bytes|mcasts} with 2238 * adapter->stats counters. It's required to make ifconfig -z 2239 * (SOICZIFDATA) work. 2240 */ 2241 } /* ixv_update_stats */ 2242 2243 /************************************************************************ 2244 * ixv_sysctl_interrupt_rate_handler 2245 ************************************************************************/ 2246 static int 2247 ixv_sysctl_interrupt_rate_handler(SYSCTLFN_ARGS) 2248 { 2249 struct sysctlnode node = *rnode; 2250 struct ix_queue *que = (struct ix_queue *)node.sysctl_data; 2251 struct adapter *adapter = que->adapter; 2252 uint32_t reg, usec, rate; 2253 int error; 2254 2255 if (que == NULL) 2256 return 0; 2257 reg = IXGBE_READ_REG(&que->adapter->hw, IXGBE_VTEITR(que->msix)); 2258 usec = ((reg & 0x0FF8) >> 3); 2259 if (usec > 0) 2260 rate = 500000 / usec; 2261 else 2262 rate = 0; 2263 node.sysctl_data = &rate; 2264 error = sysctl_lookup(SYSCTLFN_CALL(&node)); 2265 if (error || newp == NULL) 2266 return error; 2267 reg &= ~0xfff; /* default, no limitation */ 2268 if (rate > 0 && rate < 500000) { 2269 if (rate < 1000) 2270 rate = 1000; 2271 reg |= ((4000000/rate) & 0xff8); 2272 /* 2273 * When RSC is used, ITR interval must be larger than 2274 * RSC_DELAY. Currently, we use 2us for RSC_DELAY. 2275 * The minimum value is always greater than 2us on 100M 2276 * (and 10M?(not documented)), but it's not on 1G and higher. 2277 */ 2278 if ((adapter->link_speed != IXGBE_LINK_SPEED_100_FULL) 2279 && (adapter->link_speed != IXGBE_LINK_SPEED_10_FULL)) { 2280 if ((adapter->num_queues > 1) 2281 && (reg < IXGBE_MIN_RSC_EITR_10G1G)) 2282 return EINVAL; 2283 } 2284 ixv_max_interrupt_rate = rate; 2285 } else 2286 ixv_max_interrupt_rate = 0; 2287 ixv_eitr_write(adapter, que->msix, reg); 2288 2289 return (0); 2290 } /* ixv_sysctl_interrupt_rate_handler */ 2291 2292 const struct sysctlnode * 2293 ixv_sysctl_instance(struct adapter *adapter) 2294 { 2295 const char *dvname; 2296 struct sysctllog **log; 2297 int rc; 2298 const struct sysctlnode *rnode; 2299 2300 log = &adapter->sysctllog; 2301 dvname = device_xname(adapter->dev); 2302 2303 if ((rc = sysctl_createv(log, 0, NULL, &rnode, 2304 0, CTLTYPE_NODE, dvname, 2305 SYSCTL_DESCR("ixv information and settings"), 2306 NULL, 0, NULL, 0, CTL_HW, CTL_CREATE, CTL_EOL)) != 0) 2307 goto err; 2308 2309 return rnode; 2310 err: 2311 printf("%s: sysctl_createv failed, rc = %d\n", __func__, rc); 2312 return NULL; 2313 } 2314 2315 static void 2316 ixv_add_device_sysctls(struct adapter *adapter) 2317 { 2318 struct sysctllog **log; 2319 const struct sysctlnode *rnode, *cnode; 2320 device_t dev; 2321 2322 dev = adapter->dev; 2323 log = &adapter->sysctllog; 2324 2325 if ((rnode = ixv_sysctl_instance(adapter)) == NULL) { 2326 aprint_error_dev(dev, "could not create sysctl root\n"); 2327 return; 2328 } 2329 2330 if (sysctl_createv(log, 0, &rnode, &cnode, 2331 CTLFLAG_READWRITE, CTLTYPE_INT, 2332 "debug", SYSCTL_DESCR("Debug Info"), 2333 ixv_sysctl_debug, 0, (void *)adapter, 0, CTL_CREATE, CTL_EOL) != 0) 2334 aprint_error_dev(dev, "could not create sysctl\n"); 2335 2336 if (sysctl_createv(log, 0, &rnode, &cnode, 2337 CTLFLAG_READWRITE, CTLTYPE_BOOL, 2338 "enable_aim", SYSCTL_DESCR("Interrupt Moderation"), 2339 NULL, 0, &adapter->enable_aim, 0, CTL_CREATE, CTL_EOL) != 0) 2340 aprint_error_dev(dev, "could not create sysctl\n"); 2341 2342 if (sysctl_createv(log, 0, &rnode, &cnode, 2343 CTLFLAG_READWRITE, CTLTYPE_BOOL, 2344 "txrx_workqueue", SYSCTL_DESCR("Use workqueue for packet processing"), 2345 NULL, 0, &adapter->txrx_use_workqueue, 0, CTL_CREATE, CTL_EOL) != 0) 2346 aprint_error_dev(dev, "could not create sysctl\n"); 2347 } 2348 2349 /************************************************************************ 2350 * ixv_add_stats_sysctls - Add statistic sysctls for the VF. 2351 ************************************************************************/ 2352 static void 2353 ixv_add_stats_sysctls(struct adapter *adapter) 2354 { 2355 device_t dev = adapter->dev; 2356 struct tx_ring *txr = adapter->tx_rings; 2357 struct rx_ring *rxr = adapter->rx_rings; 2358 struct ixgbevf_hw_stats *stats = &adapter->stats.vf; 2359 struct ixgbe_hw *hw = &adapter->hw; 2360 const struct sysctlnode *rnode, *cnode; 2361 struct sysctllog **log = &adapter->sysctllog; 2362 const char *xname = device_xname(dev); 2363 2364 /* Driver Statistics */ 2365 evcnt_attach_dynamic(&adapter->efbig_tx_dma_setup, EVCNT_TYPE_MISC, 2366 NULL, xname, "Driver tx dma soft fail EFBIG"); 2367 evcnt_attach_dynamic(&adapter->mbuf_defrag_failed, EVCNT_TYPE_MISC, 2368 NULL, xname, "m_defrag() failed"); 2369 evcnt_attach_dynamic(&adapter->efbig2_tx_dma_setup, EVCNT_TYPE_MISC, 2370 NULL, xname, "Driver tx dma hard fail EFBIG"); 2371 evcnt_attach_dynamic(&adapter->einval_tx_dma_setup, EVCNT_TYPE_MISC, 2372 NULL, xname, "Driver tx dma hard fail EINVAL"); 2373 evcnt_attach_dynamic(&adapter->other_tx_dma_setup, EVCNT_TYPE_MISC, 2374 NULL, xname, "Driver tx dma hard fail other"); 2375 evcnt_attach_dynamic(&adapter->eagain_tx_dma_setup, EVCNT_TYPE_MISC, 2376 NULL, xname, "Driver tx dma soft fail EAGAIN"); 2377 evcnt_attach_dynamic(&adapter->enomem_tx_dma_setup, EVCNT_TYPE_MISC, 2378 NULL, xname, "Driver tx dma soft fail ENOMEM"); 2379 evcnt_attach_dynamic(&adapter->watchdog_events, EVCNT_TYPE_MISC, 2380 NULL, xname, "Watchdog timeouts"); 2381 evcnt_attach_dynamic(&adapter->tso_err, EVCNT_TYPE_MISC, 2382 NULL, xname, "TSO errors"); 2383 evcnt_attach_dynamic(&adapter->link_irq, EVCNT_TYPE_INTR, 2384 NULL, xname, "Link MSI-X IRQ Handled"); 2385 2386 for (int i = 0; i < adapter->num_queues; i++, rxr++, txr++) { 2387 snprintf(adapter->queues[i].evnamebuf, 2388 sizeof(adapter->queues[i].evnamebuf), "%s q%d", 2389 xname, i); 2390 snprintf(adapter->queues[i].namebuf, 2391 sizeof(adapter->queues[i].namebuf), "q%d", i); 2392 2393 if ((rnode = ixv_sysctl_instance(adapter)) == NULL) { 2394 aprint_error_dev(dev, "could not create sysctl root\n"); 2395 break; 2396 } 2397 2398 if (sysctl_createv(log, 0, &rnode, &rnode, 2399 0, CTLTYPE_NODE, 2400 adapter->queues[i].namebuf, SYSCTL_DESCR("Queue Name"), 2401 NULL, 0, NULL, 0, CTL_CREATE, CTL_EOL) != 0) 2402 break; 2403 2404 if (sysctl_createv(log, 0, &rnode, &cnode, 2405 CTLFLAG_READWRITE, CTLTYPE_INT, 2406 "interrupt_rate", SYSCTL_DESCR("Interrupt Rate"), 2407 ixv_sysctl_interrupt_rate_handler, 0, 2408 (void *)&adapter->queues[i], 0, CTL_CREATE, CTL_EOL) != 0) 2409 break; 2410 2411 if (sysctl_createv(log, 0, &rnode, &cnode, 2412 CTLFLAG_READONLY, CTLTYPE_INT, 2413 "txd_head", SYSCTL_DESCR("Transmit Descriptor Head"), 2414 ixv_sysctl_tdh_handler, 0, (void *)txr, 2415 0, CTL_CREATE, CTL_EOL) != 0) 2416 break; 2417 2418 if (sysctl_createv(log, 0, &rnode, &cnode, 2419 CTLFLAG_READONLY, CTLTYPE_INT, 2420 "txd_tail", SYSCTL_DESCR("Transmit Descriptor Tail"), 2421 ixv_sysctl_tdt_handler, 0, (void *)txr, 2422 0, CTL_CREATE, CTL_EOL) != 0) 2423 break; 2424 2425 evcnt_attach_dynamic(&adapter->queues[i].irqs, EVCNT_TYPE_INTR, 2426 NULL, adapter->queues[i].evnamebuf, "IRQs on queue"); 2427 evcnt_attach_dynamic(&adapter->queues[i].handleq, 2428 EVCNT_TYPE_MISC, NULL, adapter->queues[i].evnamebuf, 2429 "Handled queue in softint"); 2430 evcnt_attach_dynamic(&adapter->queues[i].req, EVCNT_TYPE_MISC, 2431 NULL, adapter->queues[i].evnamebuf, "Requeued in softint"); 2432 evcnt_attach_dynamic(&txr->tso_tx, EVCNT_TYPE_MISC, 2433 NULL, adapter->queues[i].evnamebuf, "TSO"); 2434 evcnt_attach_dynamic(&txr->no_desc_avail, EVCNT_TYPE_MISC, 2435 NULL, adapter->queues[i].evnamebuf, 2436 "Queue No Descriptor Available"); 2437 evcnt_attach_dynamic(&txr->total_packets, EVCNT_TYPE_MISC, 2438 NULL, adapter->queues[i].evnamebuf, 2439 "Queue Packets Transmitted"); 2440 #ifndef IXGBE_LEGACY_TX 2441 evcnt_attach_dynamic(&txr->pcq_drops, EVCNT_TYPE_MISC, 2442 NULL, adapter->queues[i].evnamebuf, 2443 "Packets dropped in pcq"); 2444 #endif 2445 2446 #ifdef LRO 2447 struct lro_ctrl *lro = &rxr->lro; 2448 #endif /* LRO */ 2449 2450 if (sysctl_createv(log, 0, &rnode, &cnode, 2451 CTLFLAG_READONLY, 2452 CTLTYPE_INT, 2453 "rxd_head", SYSCTL_DESCR("Receive Descriptor Head"), 2454 ixv_sysctl_rdh_handler, 0, (void *)rxr, 0, 2455 CTL_CREATE, CTL_EOL) != 0) 2456 break; 2457 2458 if (sysctl_createv(log, 0, &rnode, &cnode, 2459 CTLFLAG_READONLY, 2460 CTLTYPE_INT, 2461 "rxd_tail", SYSCTL_DESCR("Receive Descriptor Tail"), 2462 ixv_sysctl_rdt_handler, 0, (void *)rxr, 0, 2463 CTL_CREATE, CTL_EOL) != 0) 2464 break; 2465 2466 evcnt_attach_dynamic(&rxr->rx_packets, EVCNT_TYPE_MISC, 2467 NULL, adapter->queues[i].evnamebuf, "Queue Packets Received"); 2468 evcnt_attach_dynamic(&rxr->rx_bytes, EVCNT_TYPE_MISC, 2469 NULL, adapter->queues[i].evnamebuf, "Queue Bytes Received"); 2470 evcnt_attach_dynamic(&rxr->rx_copies, EVCNT_TYPE_MISC, 2471 NULL, adapter->queues[i].evnamebuf, "Copied RX Frames"); 2472 evcnt_attach_dynamic(&rxr->no_jmbuf, EVCNT_TYPE_MISC, 2473 NULL, adapter->queues[i].evnamebuf, "Rx no jumbo mbuf"); 2474 evcnt_attach_dynamic(&rxr->rx_discarded, EVCNT_TYPE_MISC, 2475 NULL, adapter->queues[i].evnamebuf, "Rx discarded"); 2476 #ifdef LRO 2477 SYSCTL_ADD_INT(ctx, queue_list, OID_AUTO, "lro_queued", 2478 CTLFLAG_RD, &lro->lro_queued, 0, 2479 "LRO Queued"); 2480 SYSCTL_ADD_INT(ctx, queue_list, OID_AUTO, "lro_flushed", 2481 CTLFLAG_RD, &lro->lro_flushed, 0, 2482 "LRO Flushed"); 2483 #endif /* LRO */ 2484 } 2485 2486 /* MAC stats get their own sub node */ 2487 2488 snprintf(stats->namebuf, 2489 sizeof(stats->namebuf), "%s MAC Statistics", xname); 2490 2491 evcnt_attach_dynamic(&stats->ipcs, EVCNT_TYPE_MISC, NULL, 2492 stats->namebuf, "rx csum offload - IP"); 2493 evcnt_attach_dynamic(&stats->l4cs, EVCNT_TYPE_MISC, NULL, 2494 stats->namebuf, "rx csum offload - L4"); 2495 evcnt_attach_dynamic(&stats->ipcs_bad, EVCNT_TYPE_MISC, NULL, 2496 stats->namebuf, "rx csum offload - IP bad"); 2497 evcnt_attach_dynamic(&stats->l4cs_bad, EVCNT_TYPE_MISC, NULL, 2498 stats->namebuf, "rx csum offload - L4 bad"); 2499 2500 /* Packet Reception Stats */ 2501 evcnt_attach_dynamic(&stats->vfgprc, EVCNT_TYPE_MISC, NULL, 2502 xname, "Good Packets Received"); 2503 evcnt_attach_dynamic(&stats->vfgorc, EVCNT_TYPE_MISC, NULL, 2504 xname, "Good Octets Received"); 2505 evcnt_attach_dynamic(&stats->vfmprc, EVCNT_TYPE_MISC, NULL, 2506 xname, "Multicast Packets Received"); 2507 evcnt_attach_dynamic(&stats->vfgptc, EVCNT_TYPE_MISC, NULL, 2508 xname, "Good Packets Transmitted"); 2509 evcnt_attach_dynamic(&stats->vfgotc, EVCNT_TYPE_MISC, NULL, 2510 xname, "Good Octets Transmitted"); 2511 2512 /* Mailbox Stats */ 2513 evcnt_attach_dynamic(&hw->mbx.stats.msgs_tx, EVCNT_TYPE_MISC, NULL, 2514 xname, "message TXs"); 2515 evcnt_attach_dynamic(&hw->mbx.stats.msgs_rx, EVCNT_TYPE_MISC, NULL, 2516 xname, "message RXs"); 2517 evcnt_attach_dynamic(&hw->mbx.stats.acks, EVCNT_TYPE_MISC, NULL, 2518 xname, "ACKs"); 2519 evcnt_attach_dynamic(&hw->mbx.stats.reqs, EVCNT_TYPE_MISC, NULL, 2520 xname, "REQs"); 2521 evcnt_attach_dynamic(&hw->mbx.stats.rsts, EVCNT_TYPE_MISC, NULL, 2522 xname, "RSTs"); 2523 2524 } /* ixv_add_stats_sysctls */ 2525 2526 /************************************************************************ 2527 * ixv_set_sysctl_value 2528 ************************************************************************/ 2529 static void 2530 ixv_set_sysctl_value(struct adapter *adapter, const char *name, 2531 const char *description, int *limit, int value) 2532 { 2533 device_t dev = adapter->dev; 2534 struct sysctllog **log; 2535 const struct sysctlnode *rnode, *cnode; 2536 2537 log = &adapter->sysctllog; 2538 if ((rnode = ixv_sysctl_instance(adapter)) == NULL) { 2539 aprint_error_dev(dev, "could not create sysctl root\n"); 2540 return; 2541 } 2542 if (sysctl_createv(log, 0, &rnode, &cnode, 2543 CTLFLAG_READWRITE, CTLTYPE_INT, 2544 name, SYSCTL_DESCR(description), 2545 NULL, 0, limit, 0, CTL_CREATE, CTL_EOL) != 0) 2546 aprint_error_dev(dev, "could not create sysctl\n"); 2547 *limit = value; 2548 } /* ixv_set_sysctl_value */ 2549 2550 /************************************************************************ 2551 * ixv_print_debug_info 2552 * 2553 * Called only when em_display_debug_stats is enabled. 2554 * Provides a way to take a look at important statistics 2555 * maintained by the driver and hardware. 2556 ************************************************************************/ 2557 static void 2558 ixv_print_debug_info(struct adapter *adapter) 2559 { 2560 device_t dev = adapter->dev; 2561 struct ixgbe_hw *hw = &adapter->hw; 2562 struct ix_queue *que = adapter->queues; 2563 struct rx_ring *rxr; 2564 struct tx_ring *txr; 2565 #ifdef LRO 2566 struct lro_ctrl *lro; 2567 #endif /* LRO */ 2568 2569 device_printf(dev, "Error Byte Count = %u \n", 2570 IXGBE_READ_REG(hw, IXGBE_ERRBC)); 2571 2572 for (int i = 0; i < adapter->num_queues; i++, que++) { 2573 txr = que->txr; 2574 rxr = que->rxr; 2575 #ifdef LRO 2576 lro = &rxr->lro; 2577 #endif /* LRO */ 2578 device_printf(dev, "QUE(%d) IRQs Handled: %lu\n", 2579 que->msix, (long)que->irqs.ev_count); 2580 device_printf(dev, "RX(%d) Packets Received: %lld\n", 2581 rxr->me, (long long)rxr->rx_packets.ev_count); 2582 device_printf(dev, "RX(%d) Bytes Received: %lu\n", 2583 rxr->me, (long)rxr->rx_bytes.ev_count); 2584 #ifdef LRO 2585 device_printf(dev, "RX(%d) LRO Queued= %lld\n", 2586 rxr->me, (long long)lro->lro_queued); 2587 device_printf(dev, "RX(%d) LRO Flushed= %lld\n", 2588 rxr->me, (long long)lro->lro_flushed); 2589 #endif /* LRO */ 2590 device_printf(dev, "TX(%d) Packets Sent: %lu\n", 2591 txr->me, (long)txr->total_packets.ev_count); 2592 device_printf(dev, "TX(%d) NO Desc Avail: %lu\n", 2593 txr->me, (long)txr->no_desc_avail.ev_count); 2594 } 2595 2596 device_printf(dev, "MBX IRQ Handled: %lu\n", 2597 (long)adapter->link_irq.ev_count); 2598 } /* ixv_print_debug_info */ 2599 2600 /************************************************************************ 2601 * ixv_sysctl_debug 2602 ************************************************************************/ 2603 static int 2604 ixv_sysctl_debug(SYSCTLFN_ARGS) 2605 { 2606 struct sysctlnode node = *rnode; 2607 struct adapter *adapter = (struct adapter *)node.sysctl_data; 2608 int error, result; 2609 2610 node.sysctl_data = &result; 2611 error = sysctl_lookup(SYSCTLFN_CALL(&node)); 2612 2613 if (error || newp == NULL) 2614 return error; 2615 2616 if (result == 1) 2617 ixv_print_debug_info(adapter); 2618 2619 return 0; 2620 } /* ixv_sysctl_debug */ 2621 2622 /************************************************************************ 2623 * ixv_init_device_features 2624 ************************************************************************/ 2625 static void 2626 ixv_init_device_features(struct adapter *adapter) 2627 { 2628 adapter->feat_cap = IXGBE_FEATURE_NETMAP 2629 | IXGBE_FEATURE_VF 2630 | IXGBE_FEATURE_RSS 2631 | IXGBE_FEATURE_LEGACY_TX; 2632 2633 /* A tad short on feature flags for VFs, atm. */ 2634 switch (adapter->hw.mac.type) { 2635 case ixgbe_mac_82599_vf: 2636 break; 2637 case ixgbe_mac_X540_vf: 2638 break; 2639 case ixgbe_mac_X550_vf: 2640 case ixgbe_mac_X550EM_x_vf: 2641 case ixgbe_mac_X550EM_a_vf: 2642 adapter->feat_cap |= IXGBE_FEATURE_NEEDS_CTXD; 2643 break; 2644 default: 2645 break; 2646 } 2647 2648 /* Enabled by default... */ 2649 /* Is a virtual function (VF) */ 2650 if (adapter->feat_cap & IXGBE_FEATURE_VF) 2651 adapter->feat_en |= IXGBE_FEATURE_VF; 2652 /* Netmap */ 2653 if (adapter->feat_cap & IXGBE_FEATURE_NETMAP) 2654 adapter->feat_en |= IXGBE_FEATURE_NETMAP; 2655 /* Receive-Side Scaling (RSS) */ 2656 if (adapter->feat_cap & IXGBE_FEATURE_RSS) 2657 adapter->feat_en |= IXGBE_FEATURE_RSS; 2658 /* Needs advanced context descriptor regardless of offloads req'd */ 2659 if (adapter->feat_cap & IXGBE_FEATURE_NEEDS_CTXD) 2660 adapter->feat_en |= IXGBE_FEATURE_NEEDS_CTXD; 2661 2662 /* Enabled via sysctl... */ 2663 /* Legacy (single queue) transmit */ 2664 if ((adapter->feat_cap & IXGBE_FEATURE_LEGACY_TX) && 2665 ixv_enable_legacy_tx) 2666 adapter->feat_en |= IXGBE_FEATURE_LEGACY_TX; 2667 } /* ixv_init_device_features */ 2668 2669 /************************************************************************ 2670 * ixv_shutdown - Shutdown entry point 2671 ************************************************************************/ 2672 #if 0 /* XXX NetBSD ought to register something like this through pmf(9) */ 2673 static int 2674 ixv_shutdown(device_t dev) 2675 { 2676 struct adapter *adapter = device_private(dev); 2677 IXGBE_CORE_LOCK(adapter); 2678 ixv_stop(adapter); 2679 IXGBE_CORE_UNLOCK(adapter); 2680 2681 return (0); 2682 } /* ixv_shutdown */ 2683 #endif 2684 2685 static int 2686 ixv_ifflags_cb(struct ethercom *ec) 2687 { 2688 struct ifnet *ifp = &ec->ec_if; 2689 struct adapter *adapter = ifp->if_softc; 2690 int change, rc = 0; 2691 2692 IXGBE_CORE_LOCK(adapter); 2693 2694 change = ifp->if_flags ^ adapter->if_flags; 2695 if (change != 0) 2696 adapter->if_flags = ifp->if_flags; 2697 2698 if ((change & ~(IFF_CANTCHANGE | IFF_DEBUG)) != 0) 2699 rc = ENETRESET; 2700 2701 /* Set up VLAN support and filter */ 2702 ixv_setup_vlan_support(adapter); 2703 2704 IXGBE_CORE_UNLOCK(adapter); 2705 2706 return rc; 2707 } 2708 2709 2710 /************************************************************************ 2711 * ixv_ioctl - Ioctl entry point 2712 * 2713 * Called when the user wants to configure the interface. 2714 * 2715 * return 0 on success, positive on failure 2716 ************************************************************************/ 2717 static int 2718 ixv_ioctl(struct ifnet *ifp, u_long command, void *data) 2719 { 2720 struct adapter *adapter = ifp->if_softc; 2721 struct ifcapreq *ifcr = data; 2722 struct ifreq *ifr = data; 2723 int error = 0; 2724 int l4csum_en; 2725 const int l4csum = IFCAP_CSUM_TCPv4_Rx|IFCAP_CSUM_UDPv4_Rx| 2726 IFCAP_CSUM_TCPv6_Rx|IFCAP_CSUM_UDPv6_Rx; 2727 2728 switch (command) { 2729 case SIOCSIFFLAGS: 2730 IOCTL_DEBUGOUT("ioctl: SIOCSIFFLAGS (Set Interface Flags)"); 2731 break; 2732 case SIOCADDMULTI: 2733 case SIOCDELMULTI: 2734 IOCTL_DEBUGOUT("ioctl: SIOC(ADD|DEL)MULTI"); 2735 break; 2736 case SIOCSIFMEDIA: 2737 case SIOCGIFMEDIA: 2738 IOCTL_DEBUGOUT("ioctl: SIOCxIFMEDIA (Get/Set Interface Media)"); 2739 break; 2740 case SIOCSIFCAP: 2741 IOCTL_DEBUGOUT("ioctl: SIOCSIFCAP (Set Capabilities)"); 2742 break; 2743 case SIOCSIFMTU: 2744 IOCTL_DEBUGOUT("ioctl: SIOCSIFMTU (Set Interface MTU)"); 2745 break; 2746 default: 2747 IOCTL_DEBUGOUT1("ioctl: UNKNOWN (0x%X)", (int)command); 2748 break; 2749 } 2750 2751 switch (command) { 2752 case SIOCSIFMEDIA: 2753 case SIOCGIFMEDIA: 2754 return ifmedia_ioctl(ifp, ifr, &adapter->media, command); 2755 case SIOCSIFCAP: 2756 /* Layer-4 Rx checksum offload has to be turned on and 2757 * off as a unit. 2758 */ 2759 l4csum_en = ifcr->ifcr_capenable & l4csum; 2760 if (l4csum_en != l4csum && l4csum_en != 0) 2761 return EINVAL; 2762 /*FALLTHROUGH*/ 2763 case SIOCADDMULTI: 2764 case SIOCDELMULTI: 2765 case SIOCSIFFLAGS: 2766 case SIOCSIFMTU: 2767 default: 2768 if ((error = ether_ioctl(ifp, command, data)) != ENETRESET) 2769 return error; 2770 if ((ifp->if_flags & IFF_RUNNING) == 0) 2771 ; 2772 else if (command == SIOCSIFCAP || command == SIOCSIFMTU) { 2773 IXGBE_CORE_LOCK(adapter); 2774 ixv_init_locked(adapter); 2775 IXGBE_CORE_UNLOCK(adapter); 2776 } else if (command == SIOCADDMULTI || command == SIOCDELMULTI) { 2777 /* 2778 * Multicast list has changed; set the hardware filter 2779 * accordingly. 2780 */ 2781 IXGBE_CORE_LOCK(adapter); 2782 ixv_disable_intr(adapter); 2783 ixv_set_multi(adapter); 2784 ixv_enable_intr(adapter); 2785 IXGBE_CORE_UNLOCK(adapter); 2786 } 2787 return 0; 2788 } 2789 } /* ixv_ioctl */ 2790 2791 /************************************************************************ 2792 * ixv_init 2793 ************************************************************************/ 2794 static int 2795 ixv_init(struct ifnet *ifp) 2796 { 2797 struct adapter *adapter = ifp->if_softc; 2798 2799 IXGBE_CORE_LOCK(adapter); 2800 ixv_init_locked(adapter); 2801 IXGBE_CORE_UNLOCK(adapter); 2802 2803 return 0; 2804 } /* ixv_init */ 2805 2806 /************************************************************************ 2807 * ixv_handle_que 2808 ************************************************************************/ 2809 static void 2810 ixv_handle_que(void *context) 2811 { 2812 struct ix_queue *que = context; 2813 struct adapter *adapter = que->adapter; 2814 struct tx_ring *txr = que->txr; 2815 struct ifnet *ifp = adapter->ifp; 2816 bool more; 2817 2818 que->handleq.ev_count++; 2819 2820 if (ifp->if_flags & IFF_RUNNING) { 2821 more = ixgbe_rxeof(que); 2822 IXGBE_TX_LOCK(txr); 2823 more |= ixgbe_txeof(txr); 2824 if (!(adapter->feat_en & IXGBE_FEATURE_LEGACY_TX)) 2825 if (!ixgbe_mq_ring_empty(ifp, txr->txr_interq)) 2826 ixgbe_mq_start_locked(ifp, txr); 2827 /* Only for queue 0 */ 2828 /* NetBSD still needs this for CBQ */ 2829 if ((&adapter->queues[0] == que) 2830 && (!ixgbe_legacy_ring_empty(ifp, NULL))) 2831 ixgbe_legacy_start_locked(ifp, txr); 2832 IXGBE_TX_UNLOCK(txr); 2833 if (more) { 2834 que->req.ev_count++; 2835 if (adapter->txrx_use_workqueue) { 2836 /* 2837 * "enqueued flag" is not required here 2838 * the same as ixg(4). See ixgbe_msix_que(). 2839 */ 2840 workqueue_enqueue(adapter->que_wq, 2841 &que->wq_cookie, curcpu()); 2842 } else 2843 softint_schedule(que->que_si); 2844 return; 2845 } 2846 } 2847 2848 /* Re-enable this interrupt */ 2849 ixv_enable_queue(adapter, que->msix); 2850 2851 return; 2852 } /* ixv_handle_que */ 2853 2854 /************************************************************************ 2855 * ixv_handle_que_work 2856 ************************************************************************/ 2857 static void 2858 ixv_handle_que_work(struct work *wk, void *context) 2859 { 2860 struct ix_queue *que = container_of(wk, struct ix_queue, wq_cookie); 2861 2862 /* 2863 * "enqueued flag" is not required here the same as ixg(4). 2864 * See ixgbe_msix_que(). 2865 */ 2866 ixv_handle_que(que); 2867 } 2868 2869 /************************************************************************ 2870 * ixv_allocate_msix - Setup MSI-X Interrupt resources and handlers 2871 ************************************************************************/ 2872 static int 2873 ixv_allocate_msix(struct adapter *adapter, const struct pci_attach_args *pa) 2874 { 2875 device_t dev = adapter->dev; 2876 struct ix_queue *que = adapter->queues; 2877 struct tx_ring *txr = adapter->tx_rings; 2878 int error, msix_ctrl, rid, vector = 0; 2879 pci_chipset_tag_t pc; 2880 pcitag_t tag; 2881 char intrbuf[PCI_INTRSTR_LEN]; 2882 char wqname[MAXCOMLEN]; 2883 char intr_xname[32]; 2884 const char *intrstr = NULL; 2885 kcpuset_t *affinity; 2886 int cpu_id = 0; 2887 2888 pc = adapter->osdep.pc; 2889 tag = adapter->osdep.tag; 2890 2891 adapter->osdep.nintrs = adapter->num_queues + 1; 2892 if (pci_msix_alloc_exact(pa, &adapter->osdep.intrs, 2893 adapter->osdep.nintrs) != 0) { 2894 aprint_error_dev(dev, 2895 "failed to allocate MSI-X interrupt\n"); 2896 return (ENXIO); 2897 } 2898 2899 kcpuset_create(&affinity, false); 2900 for (int i = 0; i < adapter->num_queues; i++, vector++, que++, txr++) { 2901 snprintf(intr_xname, sizeof(intr_xname), "%s TXRX%d", 2902 device_xname(dev), i); 2903 intrstr = pci_intr_string(pc, adapter->osdep.intrs[i], intrbuf, 2904 sizeof(intrbuf)); 2905 #ifdef IXGBE_MPSAFE 2906 pci_intr_setattr(pc, &adapter->osdep.intrs[i], PCI_INTR_MPSAFE, 2907 true); 2908 #endif 2909 /* Set the handler function */ 2910 que->res = adapter->osdep.ihs[i] = pci_intr_establish_xname(pc, 2911 adapter->osdep.intrs[i], IPL_NET, ixv_msix_que, que, 2912 intr_xname); 2913 if (que->res == NULL) { 2914 pci_intr_release(pc, adapter->osdep.intrs, 2915 adapter->osdep.nintrs); 2916 aprint_error_dev(dev, 2917 "Failed to register QUE handler\n"); 2918 kcpuset_destroy(affinity); 2919 return (ENXIO); 2920 } 2921 que->msix = vector; 2922 adapter->active_queues |= (u64)(1 << que->msix); 2923 2924 cpu_id = i; 2925 /* Round-robin affinity */ 2926 kcpuset_zero(affinity); 2927 kcpuset_set(affinity, cpu_id % ncpu); 2928 error = interrupt_distribute(adapter->osdep.ihs[i], affinity, 2929 NULL); 2930 aprint_normal_dev(dev, "for TX/RX, interrupting at %s", 2931 intrstr); 2932 if (error == 0) 2933 aprint_normal(", bound queue %d to cpu %d\n", 2934 i, cpu_id % ncpu); 2935 else 2936 aprint_normal("\n"); 2937 2938 #ifndef IXGBE_LEGACY_TX 2939 txr->txr_si 2940 = softint_establish(SOFTINT_NET | IXGBE_SOFTINFT_FLAGS, 2941 ixgbe_deferred_mq_start, txr); 2942 #endif 2943 que->que_si 2944 = softint_establish(SOFTINT_NET | IXGBE_SOFTINFT_FLAGS, 2945 ixv_handle_que, que); 2946 if (que->que_si == NULL) { 2947 aprint_error_dev(dev, 2948 "could not establish software interrupt\n"); 2949 } 2950 } 2951 snprintf(wqname, sizeof(wqname), "%sdeferTx", device_xname(dev)); 2952 error = workqueue_create(&adapter->txr_wq, wqname, 2953 ixgbe_deferred_mq_start_work, adapter, IXGBE_WORKQUEUE_PRI, IPL_NET, 2954 IXGBE_WORKQUEUE_FLAGS); 2955 if (error) { 2956 aprint_error_dev(dev, "couldn't create workqueue for deferred Tx\n"); 2957 } 2958 adapter->txr_wq_enqueued = percpu_alloc(sizeof(u_int)); 2959 2960 snprintf(wqname, sizeof(wqname), "%sTxRx", device_xname(dev)); 2961 error = workqueue_create(&adapter->que_wq, wqname, 2962 ixv_handle_que_work, adapter, IXGBE_WORKQUEUE_PRI, IPL_NET, 2963 IXGBE_WORKQUEUE_FLAGS); 2964 if (error) { 2965 aprint_error_dev(dev, 2966 "couldn't create workqueue\n"); 2967 } 2968 2969 /* and Mailbox */ 2970 cpu_id++; 2971 snprintf(intr_xname, sizeof(intr_xname), "%s link", device_xname(dev)); 2972 adapter->vector = vector; 2973 intrstr = pci_intr_string(pc, adapter->osdep.intrs[vector], intrbuf, 2974 sizeof(intrbuf)); 2975 #ifdef IXGBE_MPSAFE 2976 pci_intr_setattr(pc, &adapter->osdep.intrs[vector], PCI_INTR_MPSAFE, 2977 true); 2978 #endif 2979 /* Set the mbx handler function */ 2980 adapter->osdep.ihs[vector] = pci_intr_establish_xname(pc, 2981 adapter->osdep.intrs[vector], IPL_NET, ixv_msix_mbx, adapter, 2982 intr_xname); 2983 if (adapter->osdep.ihs[vector] == NULL) { 2984 aprint_error_dev(dev, "Failed to register LINK handler\n"); 2985 kcpuset_destroy(affinity); 2986 return (ENXIO); 2987 } 2988 /* Round-robin affinity */ 2989 kcpuset_zero(affinity); 2990 kcpuset_set(affinity, cpu_id % ncpu); 2991 error = interrupt_distribute(adapter->osdep.ihs[vector], affinity,NULL); 2992 2993 aprint_normal_dev(dev, 2994 "for link, interrupting at %s", intrstr); 2995 if (error == 0) 2996 aprint_normal(", affinity to cpu %d\n", cpu_id % ncpu); 2997 else 2998 aprint_normal("\n"); 2999 3000 /* Tasklets for Mailbox */ 3001 adapter->link_si = softint_establish(SOFTINT_NET |IXGBE_SOFTINFT_FLAGS, 3002 ixv_handle_link, adapter); 3003 /* 3004 * Due to a broken design QEMU will fail to properly 3005 * enable the guest for MSI-X unless the vectors in 3006 * the table are all set up, so we must rewrite the 3007 * ENABLE in the MSI-X control register again at this 3008 * point to cause it to successfully initialize us. 3009 */ 3010 if (adapter->hw.mac.type == ixgbe_mac_82599_vf) { 3011 pci_get_capability(pc, tag, PCI_CAP_MSIX, &rid, NULL); 3012 rid += PCI_MSIX_CTL; 3013 msix_ctrl = pci_conf_read(pc, tag, rid); 3014 msix_ctrl |= PCI_MSIX_CTL_ENABLE; 3015 pci_conf_write(pc, tag, rid, msix_ctrl); 3016 } 3017 3018 kcpuset_destroy(affinity); 3019 return (0); 3020 } /* ixv_allocate_msix */ 3021 3022 /************************************************************************ 3023 * ixv_configure_interrupts - Setup MSI-X resources 3024 * 3025 * Note: The VF device MUST use MSI-X, there is no fallback. 3026 ************************************************************************/ 3027 static int 3028 ixv_configure_interrupts(struct adapter *adapter) 3029 { 3030 device_t dev = adapter->dev; 3031 int want, queues, msgs; 3032 3033 /* Must have at least 2 MSI-X vectors */ 3034 msgs = pci_msix_count(adapter->osdep.pc, adapter->osdep.tag); 3035 if (msgs < 2) { 3036 aprint_error_dev(dev, "MSIX config error\n"); 3037 return (ENXIO); 3038 } 3039 msgs = MIN(msgs, IXG_MAX_NINTR); 3040 3041 /* Figure out a reasonable auto config value */ 3042 queues = (ncpu > (msgs - 1)) ? (msgs - 1) : ncpu; 3043 3044 if (ixv_num_queues != 0) 3045 queues = ixv_num_queues; 3046 else if ((ixv_num_queues == 0) && (queues > IXGBE_VF_MAX_TX_QUEUES)) 3047 queues = IXGBE_VF_MAX_TX_QUEUES; 3048 3049 /* 3050 * Want vectors for the queues, 3051 * plus an additional for mailbox. 3052 */ 3053 want = queues + 1; 3054 if (msgs >= want) 3055 msgs = want; 3056 else { 3057 aprint_error_dev(dev, 3058 "MSI-X Configuration Problem, " 3059 "%d vectors but %d queues wanted!\n", 3060 msgs, want); 3061 return -1; 3062 } 3063 3064 adapter->msix_mem = (void *)1; /* XXX */ 3065 aprint_normal_dev(dev, 3066 "Using MSI-X interrupts with %d vectors\n", msgs); 3067 adapter->num_queues = queues; 3068 3069 return (0); 3070 } /* ixv_configure_interrupts */ 3071 3072 3073 /************************************************************************ 3074 * ixv_handle_link - Tasklet handler for MSI-X MBX interrupts 3075 * 3076 * Done outside of interrupt context since the driver might sleep 3077 ************************************************************************/ 3078 static void 3079 ixv_handle_link(void *context) 3080 { 3081 struct adapter *adapter = context; 3082 3083 IXGBE_CORE_LOCK(adapter); 3084 3085 adapter->hw.mac.ops.check_link(&adapter->hw, &adapter->link_speed, 3086 &adapter->link_up, FALSE); 3087 ixv_update_link_status(adapter); 3088 3089 IXGBE_CORE_UNLOCK(adapter); 3090 } /* ixv_handle_link */ 3091 3092 /************************************************************************ 3093 * ixv_check_link - Used in the local timer to poll for link changes 3094 ************************************************************************/ 3095 static void 3096 ixv_check_link(struct adapter *adapter) 3097 { 3098 3099 KASSERT(mutex_owned(&adapter->core_mtx)); 3100 3101 adapter->hw.mac.get_link_status = TRUE; 3102 3103 adapter->hw.mac.ops.check_link(&adapter->hw, &adapter->link_speed, 3104 &adapter->link_up, FALSE); 3105 ixv_update_link_status(adapter); 3106 } /* ixv_check_link */ 3107
Indexes created Tue Sep 30 20:09:53 GMT 2025