ixv.c revision 1.140
1 /*$NetBSD: ixv.c,v 1.140 2019/11/06 04:17:16 msaitoh Exp $*/ 2 3 /****************************************************************************** 4 5 Copyright (c) 2001-2017, Intel Corporation 6 All rights reserved. 7 8 Redistribution and use in source and binary forms, with or without 9 modification, are permitted provided that the following conditions are met: 10 11 1. Redistributions of source code must retain the above copyright notice, 12 this list of conditions and the following disclaimer. 13 14 2. Redistributions in binary form must reproduce the above copyright 15 notice, this list of conditions and the following disclaimer in the 16 documentation and/or other materials provided with the distribution. 17 18 3. Neither the name of the Intel Corporation nor the names of its 19 contributors may be used to endorse or promote products derived from 20 this software without specific prior written permission. 21 22 THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 23 AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 24 IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 25 ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE 26 LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 27 CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 28 SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 29 INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 30 CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 31 ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 32 POSSIBILITY OF SUCH DAMAGE. 33 34 ******************************************************************************/ 35 /*$FreeBSD: head/sys/dev/ixgbe/if_ixv.c 331224 2018-03-19 20:55:05Z erj $*/ 36 37 #ifdef _KERNEL_OPT 38 #include "opt_inet.h" 39 #include "opt_inet6.h" 40 #include "opt_net_mpsafe.h" 41 #endif 42 43 #include "ixgbe.h" 44 #include "vlan.h" 45 46 /************************************************************************ 47 * Driver version 48 ************************************************************************/ 49 static const char ixv_driver_version[] = "2.0.1-k"; 50 /* XXX NetBSD: + 1.5.17 */ 51 52 /************************************************************************ 53 * PCI Device ID Table 54 * 55 * Used by probe to select devices to load on 56 * Last field stores an index into ixv_strings 57 * Last entry must be all 0s 58 * 59 * { Vendor ID, Device ID, SubVendor ID, SubDevice ID, String Index } 60 ************************************************************************/ 61 static const ixgbe_vendor_info_t ixv_vendor_info_array[] = 62 { 63 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_VF, 0, 0, 0}, 64 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_X540_VF, 0, 0, 0}, 65 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_X550_VF, 0, 0, 0}, 66 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_X550EM_X_VF, 0, 0, 0}, 67 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_X550EM_A_VF, 0, 0, 0}, 68 /* required last entry */ 69 {0, 0, 0, 0, 0} 70 }; 71 72 /************************************************************************ 73 * Table of branding strings 74 ************************************************************************/ 75 static const char *ixv_strings[] = { 76 "Intel(R) PRO/10GbE Virtual Function Network Driver" 77 }; 78 79 /********************************************************************* 80 * Function prototypes 81 *********************************************************************/ 82 static int ixv_probe(device_t, cfdata_t, void *); 83 static void ixv_attach(device_t, device_t, void *); 84 static int ixv_detach(device_t, int); 85 #if 0 86 static int ixv_shutdown(device_t); 87 #endif 88 static int ixv_ifflags_cb(struct ethercom *); 89 static int ixv_ioctl(struct ifnet *, u_long, void *); 90 static int ixv_init(struct ifnet *); 91 static void ixv_init_locked(struct adapter *); 92 static void ixv_ifstop(struct ifnet *, int); 93 static void ixv_stop(void *); 94 static void ixv_init_device_features(struct adapter *); 95 static void ixv_media_status(struct ifnet *, struct ifmediareq *); 96 static int ixv_media_change(struct ifnet *); 97 static int ixv_allocate_pci_resources(struct adapter *, 98 const struct pci_attach_args *); 99 static int ixv_allocate_msix(struct adapter *, 100 const struct pci_attach_args *); 101 static int ixv_configure_interrupts(struct adapter *); 102 static void ixv_free_pci_resources(struct adapter *); 103 static void ixv_local_timer(void *); 104 static void ixv_local_timer_locked(void *); 105 static int ixv_setup_interface(device_t, struct adapter *); 106 static int ixv_negotiate_api(struct adapter *); 107 108 static void ixv_initialize_transmit_units(struct adapter *); 109 static void ixv_initialize_receive_units(struct adapter *); 110 static void ixv_initialize_rss_mapping(struct adapter *); 111 static s32 ixv_check_link(struct adapter *); 112 113 static void ixv_enable_intr(struct adapter *); 114 static void ixv_disable_intr(struct adapter *); 115 static int ixv_set_rxfilter(struct adapter *); 116 static void ixv_update_link_status(struct adapter *); 117 static int ixv_sysctl_debug(SYSCTLFN_PROTO); 118 static void ixv_set_ivar(struct adapter *, u8, u8, s8); 119 static void ixv_configure_ivars(struct adapter *); 120 static u8 * ixv_mc_array_itr(struct ixgbe_hw *, u8 **, u32 *); 121 static void ixv_eitr_write(struct adapter *, uint32_t, uint32_t); 122 123 static void ixv_setup_vlan_tagging(struct adapter *); 124 static int ixv_setup_vlan_support(struct adapter *); 125 static int ixv_vlan_cb(struct ethercom *, uint16_t, bool); 126 static int ixv_register_vlan(struct adapter *, u16); 127 static int ixv_unregister_vlan(struct adapter *, u16); 128 129 static void ixv_add_device_sysctls(struct adapter *); 130 static void ixv_save_stats(struct adapter *); 131 static void ixv_init_stats(struct adapter *); 132 static void ixv_update_stats(struct adapter *); 133 static void ixv_add_stats_sysctls(struct adapter *); 134 static void ixv_clear_evcnt(struct adapter *); 135 136 /* Sysctl handlers */ 137 static void ixv_set_sysctl_value(struct adapter *, const char *, 138 const char *, int *, int); 139 static int ixv_sysctl_interrupt_rate_handler(SYSCTLFN_PROTO); 140 static int ixv_sysctl_next_to_check_handler(SYSCTLFN_PROTO); 141 static int ixv_sysctl_rdh_handler(SYSCTLFN_PROTO); 142 static int ixv_sysctl_rdt_handler(SYSCTLFN_PROTO); 143 static int ixv_sysctl_tdt_handler(SYSCTLFN_PROTO); 144 static int ixv_sysctl_tdh_handler(SYSCTLFN_PROTO); 145 146 /* The MSI-X Interrupt handlers */ 147 static int ixv_msix_que(void *); 148 static int ixv_msix_mbx(void *); 149 150 /* Deferred interrupt tasklets */ 151 static void ixv_handle_que(void *); 152 static void ixv_handle_link(void *); 153 154 /* Workqueue handler for deferred work */ 155 static void ixv_handle_que_work(struct work *, void *); 156 157 const struct sysctlnode *ixv_sysctl_instance(struct adapter *); 158 static const ixgbe_vendor_info_t *ixv_lookup(const struct pci_attach_args *); 159 160 /************************************************************************ 161 * FreeBSD Device Interface Entry Points 162 ************************************************************************/ 163 CFATTACH_DECL3_NEW(ixv, sizeof(struct adapter), 164 ixv_probe, ixv_attach, ixv_detach, NULL, NULL, NULL, 165 DVF_DETACH_SHUTDOWN); 166 167 #if 0 168 static driver_t ixv_driver = { 169 "ixv", ixv_methods, sizeof(struct adapter), 170 }; 171 172 devclass_t ixv_devclass; 173 DRIVER_MODULE(ixv, pci, ixv_driver, ixv_devclass, 0, 0); 174 MODULE_DEPEND(ixv, pci, 1, 1, 1); 175 MODULE_DEPEND(ixv, ether, 1, 1, 1); 176 #endif 177 178 /* 179 * TUNEABLE PARAMETERS: 180 */ 181 182 /* Number of Queues - do not exceed MSI-X vectors - 1 */ 183 static int ixv_num_queues = 0; 184 #define TUNABLE_INT(__x, __y) 185 TUNABLE_INT("hw.ixv.num_queues", &ixv_num_queues); 186 187 /* 188 * AIM: Adaptive Interrupt Moderation 189 * which means that the interrupt rate 190 * is varied over time based on the 191 * traffic for that interrupt vector 192 */ 193 static bool ixv_enable_aim = false; 194 TUNABLE_INT("hw.ixv.enable_aim", &ixv_enable_aim); 195 196 static int ixv_max_interrupt_rate = (4000000 / IXGBE_LOW_LATENCY); 197 TUNABLE_INT("hw.ixv.max_interrupt_rate", &ixv_max_interrupt_rate); 198 199 /* How many packets rxeof tries to clean at a time */ 200 static int ixv_rx_process_limit = 256; 201 TUNABLE_INT("hw.ixv.rx_process_limit", &ixv_rx_process_limit); 202 203 /* How many packets txeof tries to clean at a time */ 204 static int ixv_tx_process_limit = 256; 205 TUNABLE_INT("hw.ixv.tx_process_limit", &ixv_tx_process_limit); 206 207 /* Which packet processing uses workqueue or softint */ 208 static bool ixv_txrx_workqueue = false; 209 210 /* 211 * Number of TX descriptors per ring, 212 * setting higher than RX as this seems 213 * the better performing choice. 214 */ 215 static int ixv_txd = PERFORM_TXD; 216 TUNABLE_INT("hw.ixv.txd", &ixv_txd); 217 218 /* Number of RX descriptors per ring */ 219 static int ixv_rxd = PERFORM_RXD; 220 TUNABLE_INT("hw.ixv.rxd", &ixv_rxd); 221 222 /* Legacy Transmit (single queue) */ 223 static int ixv_enable_legacy_tx = 0; 224 TUNABLE_INT("hw.ixv.enable_legacy_tx", &ixv_enable_legacy_tx); 225 226 #ifdef NET_MPSAFE 227 #define IXGBE_MPSAFE 1 228 #define IXGBE_CALLOUT_FLAGS CALLOUT_MPSAFE 229 #define IXGBE_SOFTINFT_FLAGS SOFTINT_MPSAFE 230 #define IXGBE_WORKQUEUE_FLAGS WQ_PERCPU | WQ_MPSAFE 231 #else 232 #define IXGBE_CALLOUT_FLAGS 0 233 #define IXGBE_SOFTINFT_FLAGS 0 234 #define IXGBE_WORKQUEUE_FLAGS WQ_PERCPU 235 #endif 236 #define IXGBE_WORKQUEUE_PRI PRI_SOFTNET 237 238 #if 0 239 static int (*ixv_start_locked)(struct ifnet *, struct tx_ring *); 240 static int (*ixv_ring_empty)(struct ifnet *, struct buf_ring *); 241 #endif 242 243 /************************************************************************ 244 * ixv_probe - Device identification routine 245 * 246 * Determines if the driver should be loaded on 247 * adapter based on its PCI vendor/device ID. 248 * 249 * return BUS_PROBE_DEFAULT on success, positive on failure 250 ************************************************************************/ 251 static int 252 ixv_probe(device_t dev, cfdata_t cf, void *aux) 253 { 254 #ifdef __HAVE_PCI_MSI_MSIX 255 const struct pci_attach_args *pa = aux; 256 257 return (ixv_lookup(pa) != NULL) ? 1 : 0; 258 #else 259 return 0; 260 #endif 261 } /* ixv_probe */ 262 263 static const ixgbe_vendor_info_t * 264 ixv_lookup(const struct pci_attach_args *pa) 265 { 266 const ixgbe_vendor_info_t *ent; 267 pcireg_t subid; 268 269 INIT_DEBUGOUT("ixv_lookup: begin"); 270 271 if (PCI_VENDOR(pa->pa_id) != IXGBE_INTEL_VENDOR_ID) 272 return NULL; 273 274 subid = pci_conf_read(pa->pa_pc, pa->pa_tag, PCI_SUBSYS_ID_REG); 275 276 for (ent = ixv_vendor_info_array; ent->vendor_id != 0; ent++) { 277 if ((PCI_VENDOR(pa->pa_id) == ent->vendor_id) && 278 (PCI_PRODUCT(pa->pa_id) == ent->device_id) && 279 ((PCI_SUBSYS_VENDOR(subid) == ent->subvendor_id) || 280 (ent->subvendor_id == 0)) && 281 ((PCI_SUBSYS_ID(subid) == ent->subdevice_id) || 282 (ent->subdevice_id == 0))) { 283 return ent; 284 } 285 } 286 287 return NULL; 288 } 289 290 /************************************************************************ 291 * ixv_attach - Device initialization routine 292 * 293 * Called when the driver is being loaded. 294 * Identifies the type of hardware, allocates all resources 295 * and initializes the hardware. 296 * 297 * return 0 on success, positive on failure 298 ************************************************************************/ 299 static void 300 ixv_attach(device_t parent, device_t dev, void *aux) 301 { 302 struct adapter *adapter; 303 struct ixgbe_hw *hw; 304 int error = 0; 305 pcireg_t id, subid; 306 const ixgbe_vendor_info_t *ent; 307 const struct pci_attach_args *pa = aux; 308 const char *apivstr; 309 const char *str; 310 char buf[256]; 311 312 INIT_DEBUGOUT("ixv_attach: begin"); 313 314 /* 315 * Make sure BUSMASTER is set, on a VM under 316 * KVM it may not be and will break things. 317 */ 318 ixgbe_pci_enable_busmaster(pa->pa_pc, pa->pa_tag); 319 320 /* Allocate, clear, and link in our adapter structure */ 321 adapter = device_private(dev); 322 adapter->dev = dev; 323 adapter->hw.back = adapter; 324 hw = &adapter->hw; 325 326 adapter->init_locked = ixv_init_locked; 327 adapter->stop_locked = ixv_stop; 328 329 adapter->osdep.pc = pa->pa_pc; 330 adapter->osdep.tag = pa->pa_tag; 331 if (pci_dma64_available(pa)) 332 adapter->osdep.dmat = pa->pa_dmat64; 333 else 334 adapter->osdep.dmat = pa->pa_dmat; 335 adapter->osdep.attached = false; 336 337 ent = ixv_lookup(pa); 338 339 KASSERT(ent != NULL); 340 341 aprint_normal(": %s, Version - %s\n", 342 ixv_strings[ent->index], ixv_driver_version); 343 344 /* Core Lock Init*/ 345 IXGBE_CORE_LOCK_INIT(adapter, device_xname(dev)); 346 347 /* Do base PCI setup - map BAR0 */ 348 if (ixv_allocate_pci_resources(adapter, pa)) { 349 aprint_error_dev(dev, "ixv_allocate_pci_resources() failed!\n"); 350 error = ENXIO; 351 goto err_out; 352 } 353 354 /* SYSCTL APIs */ 355 ixv_add_device_sysctls(adapter); 356 357 /* Set up the timer callout */ 358 callout_init(&adapter->timer, IXGBE_CALLOUT_FLAGS); 359 360 /* Save off the information about this board */ 361 id = pci_conf_read(pa->pa_pc, pa->pa_tag, PCI_ID_REG); 362 subid = pci_conf_read(pa->pa_pc, pa->pa_tag, PCI_SUBSYS_ID_REG); 363 hw->vendor_id = PCI_VENDOR(id); 364 hw->device_id = PCI_PRODUCT(id); 365 hw->revision_id = 366 PCI_REVISION(pci_conf_read(pa->pa_pc, pa->pa_tag, PCI_CLASS_REG)); 367 hw->subsystem_vendor_id = PCI_SUBSYS_VENDOR(subid); 368 hw->subsystem_device_id = PCI_SUBSYS_ID(subid); 369 370 /* A subset of set_mac_type */ 371 switch (hw->device_id) { 372 case IXGBE_DEV_ID_82599_VF: 373 hw->mac.type = ixgbe_mac_82599_vf; 374 str = "82599 VF"; 375 break; 376 case IXGBE_DEV_ID_X540_VF: 377 hw->mac.type = ixgbe_mac_X540_vf; 378 str = "X540 VF"; 379 break; 380 case IXGBE_DEV_ID_X550_VF: 381 hw->mac.type = ixgbe_mac_X550_vf; 382 str = "X550 VF"; 383 break; 384 case IXGBE_DEV_ID_X550EM_X_VF: 385 hw->mac.type = ixgbe_mac_X550EM_x_vf; 386 str = "X550EM X VF"; 387 break; 388 case IXGBE_DEV_ID_X550EM_A_VF: 389 hw->mac.type = ixgbe_mac_X550EM_a_vf; 390 str = "X550EM A VF"; 391 break; 392 default: 393 /* Shouldn't get here since probe succeeded */ 394 aprint_error_dev(dev, "Unknown device ID!\n"); 395 error = ENXIO; 396 goto err_out; 397 break; 398 } 399 aprint_normal_dev(dev, "device %s\n", str); 400 401 ixv_init_device_features(adapter); 402 403 /* Initialize the shared code */ 404 error = ixgbe_init_ops_vf(hw); 405 if (error) { 406 aprint_error_dev(dev, "ixgbe_init_ops_vf() failed!\n"); 407 error = EIO; 408 goto err_out; 409 } 410 411 /* Setup the mailbox */ 412 ixgbe_init_mbx_params_vf(hw); 413 414 /* Set the right number of segments */ 415 adapter->num_segs = IXGBE_82599_SCATTER; 416 417 /* Reset mbox api to 1.0 */ 418 error = hw->mac.ops.reset_hw(hw); 419 if (error == IXGBE_ERR_RESET_FAILED) 420 aprint_error_dev(dev, "...reset_hw() failure: Reset Failed!\n"); 421 else if (error) 422 aprint_error_dev(dev, "...reset_hw() failed with error %d\n", 423 error); 424 if (error) { 425 error = EIO; 426 goto err_out; 427 } 428 429 error = hw->mac.ops.init_hw(hw); 430 if (error) { 431 aprint_error_dev(dev, "...init_hw() failed!\n"); 432 error = EIO; 433 goto err_out; 434 } 435 436 /* Negotiate mailbox API version */ 437 error = ixv_negotiate_api(adapter); 438 if (error) 439 aprint_normal_dev(dev, 440 "MBX API negotiation failed during attach!\n"); 441 switch (hw->api_version) { 442 case ixgbe_mbox_api_10: 443 apivstr = "1.0"; 444 break; 445 case ixgbe_mbox_api_20: 446 apivstr = "2.0"; 447 break; 448 case ixgbe_mbox_api_11: 449 apivstr = "1.1"; 450 break; 451 case ixgbe_mbox_api_12: 452 apivstr = "1.2"; 453 break; 454 case ixgbe_mbox_api_13: 455 apivstr = "1.3"; 456 break; 457 default: 458 apivstr = "unknown"; 459 break; 460 } 461 aprint_normal_dev(dev, "Mailbox API %s\n", apivstr); 462 463 /* If no mac address was assigned, make a random one */ 464 if (!ixv_check_ether_addr(hw->mac.addr)) { 465 u8 addr[ETHER_ADDR_LEN]; 466 uint64_t rndval = cprng_strong64(); 467 468 memcpy(addr, &rndval, sizeof(addr)); 469 addr[0] &= 0xFE; 470 addr[0] |= 0x02; 471 bcopy(addr, hw->mac.addr, sizeof(addr)); 472 } 473 474 /* Register for VLAN events */ 475 ether_set_vlan_cb(&adapter->osdep.ec, ixv_vlan_cb); 476 477 /* Sysctls for limiting the amount of work done in the taskqueues */ 478 ixv_set_sysctl_value(adapter, "rx_processing_limit", 479 "max number of rx packets to process", 480 &adapter->rx_process_limit, ixv_rx_process_limit); 481 482 ixv_set_sysctl_value(adapter, "tx_processing_limit", 483 "max number of tx packets to process", 484 &adapter->tx_process_limit, ixv_tx_process_limit); 485 486 /* Do descriptor calc and sanity checks */ 487 if (((ixv_txd * sizeof(union ixgbe_adv_tx_desc)) % DBA_ALIGN) != 0 || 488 ixv_txd < MIN_TXD || ixv_txd > MAX_TXD) { 489 aprint_error_dev(dev, "TXD config issue, using default!\n"); 490 adapter->num_tx_desc = DEFAULT_TXD; 491 } else 492 adapter->num_tx_desc = ixv_txd; 493 494 if (((ixv_rxd * sizeof(union ixgbe_adv_rx_desc)) % DBA_ALIGN) != 0 || 495 ixv_rxd < MIN_RXD || ixv_rxd > MAX_RXD) { 496 aprint_error_dev(dev, "RXD config issue, using default!\n"); 497 adapter->num_rx_desc = DEFAULT_RXD; 498 } else 499 adapter->num_rx_desc = ixv_rxd; 500 501 /* Setup MSI-X */ 502 error = ixv_configure_interrupts(adapter); 503 if (error) 504 goto err_out; 505 506 /* Allocate our TX/RX Queues */ 507 if (ixgbe_allocate_queues(adapter)) { 508 aprint_error_dev(dev, "ixgbe_allocate_queues() failed!\n"); 509 error = ENOMEM; 510 goto err_out; 511 } 512 513 /* hw.ix defaults init */ 514 adapter->enable_aim = ixv_enable_aim; 515 516 adapter->txrx_use_workqueue = ixv_txrx_workqueue; 517 518 error = ixv_allocate_msix(adapter, pa); 519 if (error) { 520 aprint_error_dev(dev, "ixv_allocate_msix() failed!\n"); 521 goto err_late; 522 } 523 524 /* Setup OS specific network interface */ 525 error = ixv_setup_interface(dev, adapter); 526 if (error != 0) { 527 aprint_error_dev(dev, "ixv_setup_interface() failed!\n"); 528 goto err_late; 529 } 530 531 /* Do the stats setup */ 532 ixv_save_stats(adapter); 533 ixv_init_stats(adapter); 534 ixv_add_stats_sysctls(adapter); 535 536 if (adapter->feat_en & IXGBE_FEATURE_NETMAP) 537 ixgbe_netmap_attach(adapter); 538 539 snprintb(buf, sizeof(buf), IXGBE_FEATURE_FLAGS, adapter->feat_cap); 540 aprint_verbose_dev(dev, "feature cap %s\n", buf); 541 snprintb(buf, sizeof(buf), IXGBE_FEATURE_FLAGS, adapter->feat_en); 542 aprint_verbose_dev(dev, "feature ena %s\n", buf); 543 544 INIT_DEBUGOUT("ixv_attach: end"); 545 adapter->osdep.attached = true; 546 547 return; 548 549 err_late: 550 ixgbe_free_transmit_structures(adapter); 551 ixgbe_free_receive_structures(adapter); 552 free(adapter->queues, M_DEVBUF); 553 err_out: 554 ixv_free_pci_resources(adapter); 555 IXGBE_CORE_LOCK_DESTROY(adapter); 556 557 return; 558 } /* ixv_attach */ 559 560 /************************************************************************ 561 * ixv_detach - Device removal routine 562 * 563 * Called when the driver is being removed. 564 * Stops the adapter and deallocates all the resources 565 * that were allocated for driver operation. 566 * 567 * return 0 on success, positive on failure 568 ************************************************************************/ 569 static int 570 ixv_detach(device_t dev, int flags) 571 { 572 struct adapter *adapter = device_private(dev); 573 struct ixgbe_hw *hw = &adapter->hw; 574 struct ix_queue *que = adapter->queues; 575 struct tx_ring *txr = adapter->tx_rings; 576 struct rx_ring *rxr = adapter->rx_rings; 577 struct ixgbevf_hw_stats *stats = &adapter->stats.vf; 578 579 INIT_DEBUGOUT("ixv_detach: begin"); 580 if (adapter->osdep.attached == false) 581 return 0; 582 583 /* Stop the interface. Callouts are stopped in it. */ 584 ixv_ifstop(adapter->ifp, 1); 585 586 #if NVLAN > 0 587 /* Make sure VLANs are not using driver */ 588 if (!VLAN_ATTACHED(&adapter->osdep.ec)) 589 ; /* nothing to do: no VLANs */ 590 else if ((flags & (DETACH_SHUTDOWN | DETACH_FORCE)) != 0) 591 vlan_ifdetach(adapter->ifp); 592 else { 593 aprint_error_dev(dev, "VLANs in use, detach first\n"); 594 return EBUSY; 595 } 596 #endif 597 598 for (int i = 0; i < adapter->num_queues; i++, que++, txr++) { 599 if (!(adapter->feat_en & IXGBE_FEATURE_LEGACY_TX)) 600 softint_disestablish(txr->txr_si); 601 softint_disestablish(que->que_si); 602 } 603 if (adapter->txr_wq != NULL) 604 workqueue_destroy(adapter->txr_wq); 605 if (adapter->txr_wq_enqueued != NULL) 606 percpu_free(adapter->txr_wq_enqueued, sizeof(u_int)); 607 if (adapter->que_wq != NULL) 608 workqueue_destroy(adapter->que_wq); 609 610 /* Drain the Mailbox(link) queue */ 611 softint_disestablish(adapter->link_si); 612 613 ether_ifdetach(adapter->ifp); 614 callout_halt(&adapter->timer, NULL); 615 616 if (adapter->feat_en & IXGBE_FEATURE_NETMAP) 617 netmap_detach(adapter->ifp); 618 619 ixv_free_pci_resources(adapter); 620 #if 0 /* XXX the NetBSD port is probably missing something here */ 621 bus_generic_detach(dev); 622 #endif 623 if_detach(adapter->ifp); 624 if_percpuq_destroy(adapter->ipq); 625 626 sysctl_teardown(&adapter->sysctllog); 627 evcnt_detach(&adapter->efbig_tx_dma_setup); 628 evcnt_detach(&adapter->mbuf_defrag_failed); 629 evcnt_detach(&adapter->efbig2_tx_dma_setup); 630 evcnt_detach(&adapter->einval_tx_dma_setup); 631 evcnt_detach(&adapter->other_tx_dma_setup); 632 evcnt_detach(&adapter->eagain_tx_dma_setup); 633 evcnt_detach(&adapter->enomem_tx_dma_setup); 634 evcnt_detach(&adapter->watchdog_events); 635 evcnt_detach(&adapter->tso_err); 636 evcnt_detach(&adapter->link_irq); 637 638 txr = adapter->tx_rings; 639 for (int i = 0; i < adapter->num_queues; i++, rxr++, txr++) { 640 evcnt_detach(&adapter->queues[i].irqs); 641 evcnt_detach(&adapter->queues[i].handleq); 642 evcnt_detach(&adapter->queues[i].req); 643 evcnt_detach(&txr->no_desc_avail); 644 evcnt_detach(&txr->total_packets); 645 evcnt_detach(&txr->tso_tx); 646 #ifndef IXGBE_LEGACY_TX 647 evcnt_detach(&txr->pcq_drops); 648 #endif 649 650 evcnt_detach(&rxr->rx_packets); 651 evcnt_detach(&rxr->rx_bytes); 652 evcnt_detach(&rxr->rx_copies); 653 evcnt_detach(&rxr->no_jmbuf); 654 evcnt_detach(&rxr->rx_discarded); 655 } 656 evcnt_detach(&stats->ipcs); 657 evcnt_detach(&stats->l4cs); 658 evcnt_detach(&stats->ipcs_bad); 659 evcnt_detach(&stats->l4cs_bad); 660 661 /* Packet Reception Stats */ 662 evcnt_detach(&stats->vfgorc); 663 evcnt_detach(&stats->vfgprc); 664 evcnt_detach(&stats->vfmprc); 665 666 /* Packet Transmission Stats */ 667 evcnt_detach(&stats->vfgotc); 668 evcnt_detach(&stats->vfgptc); 669 670 /* Mailbox Stats */ 671 evcnt_detach(&hw->mbx.stats.msgs_tx); 672 evcnt_detach(&hw->mbx.stats.msgs_rx); 673 evcnt_detach(&hw->mbx.stats.acks); 674 evcnt_detach(&hw->mbx.stats.reqs); 675 evcnt_detach(&hw->mbx.stats.rsts); 676 677 ixgbe_free_transmit_structures(adapter); 678 ixgbe_free_receive_structures(adapter); 679 for (int i = 0; i < adapter->num_queues; i++) { 680 struct ix_queue *lque = &adapter->queues[i]; 681 mutex_destroy(&lque->dc_mtx); 682 } 683 free(adapter->queues, M_DEVBUF); 684 685 IXGBE_CORE_LOCK_DESTROY(adapter); 686 687 return (0); 688 } /* ixv_detach */ 689 690 /************************************************************************ 691 * ixv_init_locked - Init entry point 692 * 693 * Used in two ways: It is used by the stack as an init entry 694 * point in network interface structure. It is also used 695 * by the driver as a hw/sw initialization routine to get 696 * to a consistent state. 697 * 698 * return 0 on success, positive on failure 699 ************************************************************************/ 700 static void 701 ixv_init_locked(struct adapter *adapter) 702 { 703 struct ifnet *ifp = adapter->ifp; 704 device_t dev = adapter->dev; 705 struct ixgbe_hw *hw = &adapter->hw; 706 struct ix_queue *que; 707 int error = 0; 708 uint32_t mask; 709 int i; 710 711 INIT_DEBUGOUT("ixv_init_locked: begin"); 712 KASSERT(mutex_owned(&adapter->core_mtx)); 713 hw->adapter_stopped = FALSE; 714 hw->mac.ops.stop_adapter(hw); 715 callout_stop(&adapter->timer); 716 for (i = 0, que = adapter->queues; i < adapter->num_queues; i++, que++) 717 que->disabled_count = 0; 718 719 adapter->max_frame_size = 720 ifp->if_mtu + ETHER_HDR_LEN + ETHER_CRC_LEN; 721 722 /* reprogram the RAR[0] in case user changed it. */ 723 hw->mac.ops.set_rar(hw, 0, hw->mac.addr, 0, IXGBE_RAH_AV); 724 725 /* Get the latest mac address, User can use a LAA */ 726 memcpy(hw->mac.addr, CLLADDR(ifp->if_sadl), 727 IXGBE_ETH_LENGTH_OF_ADDRESS); 728 hw->mac.ops.set_rar(hw, 0, hw->mac.addr, 0, 1); 729 730 /* Prepare transmit descriptors and buffers */ 731 if (ixgbe_setup_transmit_structures(adapter)) { 732 aprint_error_dev(dev, "Could not setup transmit structures\n"); 733 ixv_stop(adapter); 734 return; 735 } 736 737 /* Reset VF and renegotiate mailbox API version */ 738 hw->mac.ops.reset_hw(hw); 739 hw->mac.ops.start_hw(hw); 740 error = ixv_negotiate_api(adapter); 741 if (error) 742 device_printf(dev, 743 "Mailbox API negotiation failed in init_locked!\n"); 744 745 ixv_initialize_transmit_units(adapter); 746 747 /* Setup Multicast table */ 748 ixv_set_rxfilter(adapter); 749 750 /* 751 * Determine the correct mbuf pool 752 * for doing jumbo/headersplit 753 */ 754 if (adapter->max_frame_size <= MCLBYTES) 755 adapter->rx_mbuf_sz = MCLBYTES; 756 else 757 adapter->rx_mbuf_sz = MJUMPAGESIZE; 758 759 /* Prepare receive descriptors and buffers */ 760 if (ixgbe_setup_receive_structures(adapter)) { 761 device_printf(dev, "Could not setup receive structures\n"); 762 ixv_stop(adapter); 763 return; 764 } 765 766 /* Configure RX settings */ 767 ixv_initialize_receive_units(adapter); 768 769 #if 0 /* XXX isn't it required? -- msaitoh */ 770 /* Set the various hardware offload abilities */ 771 ifp->if_hwassist = 0; 772 if (ifp->if_capenable & IFCAP_TSO4) 773 ifp->if_hwassist |= CSUM_TSO; 774 if (ifp->if_capenable & IFCAP_TXCSUM) { 775 ifp->if_hwassist |= (CSUM_TCP | CSUM_UDP); 776 #if __FreeBSD_version >= 800000 777 ifp->if_hwassist |= CSUM_SCTP; 778 #endif 779 } 780 #endif 781 782 /* Set up VLAN offload and filter */ 783 ixv_setup_vlan_support(adapter); 784 785 /* Set up MSI-X routing */ 786 ixv_configure_ivars(adapter); 787 788 /* Set up auto-mask */ 789 mask = (1 << adapter->vector); 790 for (i = 0, que = adapter->queues; i < adapter->num_queues; i++, que++) 791 mask |= (1 << que->msix); 792 IXGBE_WRITE_REG(hw, IXGBE_VTEIAM, mask); 793 794 /* Set moderation on the Link interrupt */ 795 ixv_eitr_write(adapter, adapter->vector, IXGBE_LINK_ITR); 796 797 /* Stats init */ 798 ixv_init_stats(adapter); 799 800 /* Config/Enable Link */ 801 hw->mac.get_link_status = TRUE; 802 hw->mac.ops.check_link(hw, &adapter->link_speed, &adapter->link_up, 803 FALSE); 804 805 /* Start watchdog */ 806 callout_reset(&adapter->timer, hz, ixv_local_timer, adapter); 807 808 /* And now turn on interrupts */ 809 ixv_enable_intr(adapter); 810 811 /* Update saved flags. See ixgbe_ifflags_cb() */ 812 adapter->if_flags = ifp->if_flags; 813 adapter->ec_capenable = adapter->osdep.ec.ec_capenable; 814 815 /* Now inform the stack we're ready */ 816 ifp->if_flags |= IFF_RUNNING; 817 ifp->if_flags &= ~IFF_OACTIVE; 818 819 return; 820 } /* ixv_init_locked */ 821 822 /************************************************************************ 823 * ixv_enable_queue 824 ************************************************************************/ 825 static inline void 826 ixv_enable_queue(struct adapter *adapter, u32 vector) 827 { 828 struct ixgbe_hw *hw = &adapter->hw; 829 struct ix_queue *que = &adapter->queues[vector]; 830 u32 queue = 1UL << vector; 831 u32 mask; 832 833 mutex_enter(&que->dc_mtx); 834 if (que->disabled_count > 0 && --que->disabled_count > 0) 835 goto out; 836 837 mask = (IXGBE_EIMS_RTX_QUEUE & queue); 838 IXGBE_WRITE_REG(hw, IXGBE_VTEIMS, mask); 839 out: 840 mutex_exit(&que->dc_mtx); 841 } /* ixv_enable_queue */ 842 843 /************************************************************************ 844 * ixv_disable_queue 845 ************************************************************************/ 846 static inline void 847 ixv_disable_queue(struct adapter *adapter, u32 vector) 848 { 849 struct ixgbe_hw *hw = &adapter->hw; 850 struct ix_queue *que = &adapter->queues[vector]; 851 u32 queue = 1UL << vector; 852 u32 mask; 853 854 mutex_enter(&que->dc_mtx); 855 if (que->disabled_count++ > 0) 856 goto out; 857 858 mask = (IXGBE_EIMS_RTX_QUEUE & queue); 859 IXGBE_WRITE_REG(hw, IXGBE_VTEIMC, mask); 860 out: 861 mutex_exit(&que->dc_mtx); 862 } /* ixv_disable_queue */ 863 864 #if 0 865 static inline void 866 ixv_rearm_queues(struct adapter *adapter, u64 queues) 867 { 868 u32 mask = (IXGBE_EIMS_RTX_QUEUE & queues); 869 IXGBE_WRITE_REG(&adapter->hw, IXGBE_VTEICS, mask); 870 } /* ixv_rearm_queues */ 871 #endif 872 873 874 /************************************************************************ 875 * ixv_msix_que - MSI-X Queue Interrupt Service routine 876 ************************************************************************/ 877 static int 878 ixv_msix_que(void *arg) 879 { 880 struct ix_queue *que = arg; 881 struct adapter *adapter = que->adapter; 882 struct tx_ring *txr = que->txr; 883 struct rx_ring *rxr = que->rxr; 884 bool more; 885 u32 newitr = 0; 886 887 ixv_disable_queue(adapter, que->msix); 888 ++que->irqs.ev_count; 889 890 #ifdef __NetBSD__ 891 /* Don't run ixgbe_rxeof in interrupt context */ 892 more = true; 893 #else 894 more = ixgbe_rxeof(que); 895 #endif 896 897 IXGBE_TX_LOCK(txr); 898 ixgbe_txeof(txr); 899 IXGBE_TX_UNLOCK(txr); 900 901 /* Do AIM now? */ 902 903 if (adapter->enable_aim == false) 904 goto no_calc; 905 /* 906 * Do Adaptive Interrupt Moderation: 907 * - Write out last calculated setting 908 * - Calculate based on average size over 909 * the last interval. 910 */ 911 if (que->eitr_setting) 912 ixv_eitr_write(adapter, que->msix, que->eitr_setting); 913 914 que->eitr_setting = 0; 915 916 /* Idle, do nothing */ 917 if ((txr->bytes == 0) && (rxr->bytes == 0)) 918 goto no_calc; 919 920 if ((txr->bytes) && (txr->packets)) 921 newitr = txr->bytes/txr->packets; 922 if ((rxr->bytes) && (rxr->packets)) 923 newitr = uimax(newitr, (rxr->bytes / rxr->packets)); 924 newitr += 24; /* account for hardware frame, crc */ 925 926 /* set an upper boundary */ 927 newitr = uimin(newitr, 3000); 928 929 /* Be nice to the mid range */ 930 if ((newitr > 300) && (newitr < 1200)) 931 newitr = (newitr / 3); 932 else 933 newitr = (newitr / 2); 934 935 /* 936 * When RSC is used, ITR interval must be larger than RSC_DELAY. 937 * Currently, we use 2us for RSC_DELAY. The minimum value is always 938 * greater than 2us on 100M (and 10M?(not documented)), but it's not 939 * on 1G and higher. 940 */ 941 if ((adapter->link_speed != IXGBE_LINK_SPEED_100_FULL) 942 && (adapter->link_speed != IXGBE_LINK_SPEED_10_FULL)) { 943 if (newitr < IXGBE_MIN_RSC_EITR_10G1G) 944 newitr = IXGBE_MIN_RSC_EITR_10G1G; 945 } 946 947 /* save for next interrupt */ 948 que->eitr_setting = newitr; 949 950 /* Reset state */ 951 txr->bytes = 0; 952 txr->packets = 0; 953 rxr->bytes = 0; 954 rxr->packets = 0; 955 956 no_calc: 957 if (more) 958 softint_schedule(que->que_si); 959 else /* Re-enable this interrupt */ 960 ixv_enable_queue(adapter, que->msix); 961 962 return 1; 963 } /* ixv_msix_que */ 964 965 /************************************************************************ 966 * ixv_msix_mbx 967 ************************************************************************/ 968 static int 969 ixv_msix_mbx(void *arg) 970 { 971 struct adapter *adapter = arg; 972 struct ixgbe_hw *hw = &adapter->hw; 973 974 ++adapter->link_irq.ev_count; 975 /* NetBSD: We use auto-clear, so it's not required to write VTEICR */ 976 977 /* Link status change */ 978 hw->mac.get_link_status = TRUE; 979 softint_schedule(adapter->link_si); 980 981 IXGBE_WRITE_REG(hw, IXGBE_VTEIMS, (1 << adapter->vector)); 982 983 return 1; 984 } /* ixv_msix_mbx */ 985 986 static void 987 ixv_eitr_write(struct adapter *adapter, uint32_t index, uint32_t itr) 988 { 989 990 /* 991 * Newer devices than 82598 have VF function, so this function is 992 * simple. 993 */ 994 itr |= IXGBE_EITR_CNT_WDIS; 995 996 IXGBE_WRITE_REG(&adapter->hw, IXGBE_VTEITR(index), itr); 997 } 998 999 1000 /************************************************************************ 1001 * ixv_media_status - Media Ioctl callback 1002 * 1003 * Called whenever the user queries the status of 1004 * the interface using ifconfig. 1005 ************************************************************************/ 1006 static void 1007 ixv_media_status(struct ifnet *ifp, struct ifmediareq *ifmr) 1008 { 1009 struct adapter *adapter = ifp->if_softc; 1010 1011 INIT_DEBUGOUT("ixv_media_status: begin"); 1012 IXGBE_CORE_LOCK(adapter); 1013 ixv_update_link_status(adapter); 1014 1015 ifmr->ifm_status = IFM_AVALID; 1016 ifmr->ifm_active = IFM_ETHER; 1017 1018 if (adapter->link_active != LINK_STATE_UP) { 1019 ifmr->ifm_active |= IFM_NONE; 1020 IXGBE_CORE_UNLOCK(adapter); 1021 return; 1022 } 1023 1024 ifmr->ifm_status |= IFM_ACTIVE; 1025 1026 switch (adapter->link_speed) { 1027 case IXGBE_LINK_SPEED_10GB_FULL: 1028 ifmr->ifm_active |= IFM_10G_T | IFM_FDX; 1029 break; 1030 case IXGBE_LINK_SPEED_5GB_FULL: 1031 ifmr->ifm_active |= IFM_5000_T | IFM_FDX; 1032 break; 1033 case IXGBE_LINK_SPEED_2_5GB_FULL: 1034 ifmr->ifm_active |= IFM_2500_T | IFM_FDX; 1035 break; 1036 case IXGBE_LINK_SPEED_1GB_FULL: 1037 ifmr->ifm_active |= IFM_1000_T | IFM_FDX; 1038 break; 1039 case IXGBE_LINK_SPEED_100_FULL: 1040 ifmr->ifm_active |= IFM_100_TX | IFM_FDX; 1041 break; 1042 case IXGBE_LINK_SPEED_10_FULL: 1043 ifmr->ifm_active |= IFM_10_T | IFM_FDX; 1044 break; 1045 } 1046 1047 ifp->if_baudrate = ifmedia_baudrate(ifmr->ifm_active); 1048 1049 IXGBE_CORE_UNLOCK(adapter); 1050 } /* ixv_media_status */ 1051 1052 /************************************************************************ 1053 * ixv_media_change - Media Ioctl callback 1054 * 1055 * Called when the user changes speed/duplex using 1056 * media/mediopt option with ifconfig. 1057 ************************************************************************/ 1058 static int 1059 ixv_media_change(struct ifnet *ifp) 1060 { 1061 struct adapter *adapter = ifp->if_softc; 1062 struct ifmedia *ifm = &adapter->media; 1063 1064 INIT_DEBUGOUT("ixv_media_change: begin"); 1065 1066 if (IFM_TYPE(ifm->ifm_media) != IFM_ETHER) 1067 return (EINVAL); 1068 1069 switch (IFM_SUBTYPE(ifm->ifm_media)) { 1070 case IFM_AUTO: 1071 break; 1072 default: 1073 device_printf(adapter->dev, "Only auto media type\n"); 1074 return (EINVAL); 1075 } 1076 1077 return (0); 1078 } /* ixv_media_change */ 1079 1080 /************************************************************************ 1081 * ixv_negotiate_api 1082 * 1083 * Negotiate the Mailbox API with the PF; 1084 * start with the most featured API first. 1085 ************************************************************************/ 1086 static int 1087 ixv_negotiate_api(struct adapter *adapter) 1088 { 1089 struct ixgbe_hw *hw = &adapter->hw; 1090 int mbx_api[] = { ixgbe_mbox_api_13, 1091 ixgbe_mbox_api_12, 1092 ixgbe_mbox_api_11, 1093 ixgbe_mbox_api_10, 1094 ixgbe_mbox_api_unknown }; 1095 int i = 0; 1096 1097 while (mbx_api[i] != ixgbe_mbox_api_unknown) { 1098 if (ixgbevf_negotiate_api_version(hw, mbx_api[i]) == 0) 1099 return (0); 1100 i++; 1101 } 1102 1103 return (EINVAL); 1104 } /* ixv_negotiate_api */ 1105 1106 1107 /************************************************************************ 1108 * ixv_set_multi - Multicast Update 1109 * 1110 * Called whenever multicast address list is updated. 1111 ************************************************************************/ 1112 static int 1113 ixv_set_rxfilter(struct adapter *adapter) 1114 { 1115 u8 mta[IXGBE_MAX_VF_MC * IXGBE_ETH_LENGTH_OF_ADDRESS]; 1116 struct ifnet *ifp = adapter->ifp; 1117 struct ixgbe_hw *hw = &adapter->hw; 1118 u8 *update_ptr; 1119 int mcnt = 0; 1120 struct ethercom *ec = &adapter->osdep.ec; 1121 struct ether_multi *enm; 1122 struct ether_multistep step; 1123 bool overflow = false; 1124 int error, rc = 0; 1125 1126 KASSERT(mutex_owned(&adapter->core_mtx)); 1127 IOCTL_DEBUGOUT("ixv_set_rxfilter: begin"); 1128 1129 /* 1: For PROMISC */ 1130 if (ifp->if_flags & IFF_PROMISC) { 1131 error = hw->mac.ops.update_xcast_mode(hw, 1132 IXGBEVF_XCAST_MODE_PROMISC); 1133 if (error == IXGBE_ERR_NOT_TRUSTED) { 1134 device_printf(adapter->dev, 1135 "this interface is not trusted\n"); 1136 error = EPERM; 1137 } else if (error == IXGBE_ERR_FEATURE_NOT_SUPPORTED) { 1138 device_printf(adapter->dev, 1139 "the PF doesn't support promisc mode\n"); 1140 error = EOPNOTSUPP; 1141 } else if (error == IXGBE_ERR_NOT_IN_PROMISC) { 1142 device_printf(adapter->dev, 1143 "the PF may not in promisc mode\n"); 1144 error = EINVAL; 1145 } else if (error) { 1146 device_printf(adapter->dev, 1147 "failed to set promisc mode. error = %d\n", 1148 error); 1149 error = EIO; 1150 } else 1151 return 0; 1152 rc = error; 1153 } 1154 1155 /* 2: For ALLMULTI or normal */ 1156 ETHER_LOCK(ec); 1157 ETHER_FIRST_MULTI(step, ec, enm); 1158 while (enm != NULL) { 1159 if ((mcnt >= IXGBE_MAX_VF_MC) || 1160 (memcmp(enm->enm_addrlo, enm->enm_addrhi, 1161 ETHER_ADDR_LEN) != 0)) { 1162 overflow = true; 1163 break; 1164 } 1165 bcopy(enm->enm_addrlo, 1166 &mta[mcnt * IXGBE_ETH_LENGTH_OF_ADDRESS], 1167 IXGBE_ETH_LENGTH_OF_ADDRESS); 1168 mcnt++; 1169 ETHER_NEXT_MULTI(step, enm); 1170 } 1171 ETHER_UNLOCK(ec); 1172 1173 /* 3: For ALLMULTI */ 1174 if (overflow) { 1175 error = hw->mac.ops.update_xcast_mode(hw, 1176 IXGBEVF_XCAST_MODE_ALLMULTI); 1177 if (error == IXGBE_ERR_NOT_TRUSTED) { 1178 device_printf(adapter->dev, 1179 "this interface is not trusted\n"); 1180 error = EPERM; 1181 } else if (error == IXGBE_ERR_FEATURE_NOT_SUPPORTED) { 1182 device_printf(adapter->dev, 1183 "the PF doesn't support allmulti mode\n"); 1184 error = EOPNOTSUPP; 1185 } else if (error) { 1186 device_printf(adapter->dev, 1187 "number of Ethernet multicast addresses " 1188 "exceeds the limit (%d). error = %d\n", 1189 IXGBE_MAX_VF_MC, error); 1190 error = ENOSPC; 1191 } else { 1192 ETHER_LOCK(ec); 1193 ec->ec_flags |= ETHER_F_ALLMULTI; 1194 ETHER_UNLOCK(ec); 1195 return rc; /* Promisc might failed */ 1196 } 1197 1198 if (rc == 0) 1199 rc = error; 1200 1201 /* Continue to update the multicast table as many as we can */ 1202 } 1203 1204 /* 4: For normal operation */ 1205 error = hw->mac.ops.update_xcast_mode(hw, IXGBEVF_XCAST_MODE_MULTI); 1206 if ((error == IXGBE_ERR_FEATURE_NOT_SUPPORTED) || (error == 0)) { 1207 /* Normal operation */ 1208 ETHER_LOCK(ec); 1209 ec->ec_flags &= ~ETHER_F_ALLMULTI; 1210 ETHER_UNLOCK(ec); 1211 error = 0; 1212 } else if (error) { 1213 device_printf(adapter->dev, 1214 "failed to set Ethernet multicast address " 1215 "operation to normal. error = %d\n", error); 1216 } 1217 1218 update_ptr = mta; 1219 1220 error = adapter->hw.mac.ops.update_mc_addr_list(&adapter->hw, 1221 update_ptr, mcnt, ixv_mc_array_itr, TRUE); 1222 if (rc == 0) 1223 rc = error; 1224 1225 return rc; 1226 } /* ixv_set_rxfilter */ 1227 1228 /************************************************************************ 1229 * ixv_mc_array_itr 1230 * 1231 * An iterator function needed by the multicast shared code. 1232 * It feeds the shared code routine the addresses in the 1233 * array of ixv_set_rxfilter() one by one. 1234 ************************************************************************/ 1235 static u8 * 1236 ixv_mc_array_itr(struct ixgbe_hw *hw, u8 **update_ptr, u32 *vmdq) 1237 { 1238 u8 *addr = *update_ptr; 1239 u8 *newptr; 1240 1241 *vmdq = 0; 1242 1243 newptr = addr + IXGBE_ETH_LENGTH_OF_ADDRESS; 1244 *update_ptr = newptr; 1245 1246 return addr; 1247 } /* ixv_mc_array_itr */ 1248 1249 /************************************************************************ 1250 * ixv_local_timer - Timer routine 1251 * 1252 * Checks for link status, updates statistics, 1253 * and runs the watchdog check. 1254 ************************************************************************/ 1255 static void 1256 ixv_local_timer(void *arg) 1257 { 1258 struct adapter *adapter = arg; 1259 1260 IXGBE_CORE_LOCK(adapter); 1261 ixv_local_timer_locked(adapter); 1262 IXGBE_CORE_UNLOCK(adapter); 1263 } 1264 1265 static void 1266 ixv_local_timer_locked(void *arg) 1267 { 1268 struct adapter *adapter = arg; 1269 device_t dev = adapter->dev; 1270 struct ix_queue *que = adapter->queues; 1271 u64 queues = 0; 1272 u64 v0, v1, v2, v3, v4, v5, v6, v7; 1273 int hung = 0; 1274 int i; 1275 1276 KASSERT(mutex_owned(&adapter->core_mtx)); 1277 1278 if (ixv_check_link(adapter)) { 1279 ixv_init_locked(adapter); 1280 return; 1281 } 1282 1283 /* Stats Update */ 1284 ixv_update_stats(adapter); 1285 1286 /* Update some event counters */ 1287 v0 = v1 = v2 = v3 = v4 = v5 = v6 = v7 = 0; 1288 que = adapter->queues; 1289 for (i = 0; i < adapter->num_queues; i++, que++) { 1290 struct tx_ring *txr = que->txr; 1291 1292 v0 += txr->q_efbig_tx_dma_setup; 1293 v1 += txr->q_mbuf_defrag_failed; 1294 v2 += txr->q_efbig2_tx_dma_setup; 1295 v3 += txr->q_einval_tx_dma_setup; 1296 v4 += txr->q_other_tx_dma_setup; 1297 v5 += txr->q_eagain_tx_dma_setup; 1298 v6 += txr->q_enomem_tx_dma_setup; 1299 v7 += txr->q_tso_err; 1300 } 1301 adapter->efbig_tx_dma_setup.ev_count = v0; 1302 adapter->mbuf_defrag_failed.ev_count = v1; 1303 adapter->efbig2_tx_dma_setup.ev_count = v2; 1304 adapter->einval_tx_dma_setup.ev_count = v3; 1305 adapter->other_tx_dma_setup.ev_count = v4; 1306 adapter->eagain_tx_dma_setup.ev_count = v5; 1307 adapter->enomem_tx_dma_setup.ev_count = v6; 1308 adapter->tso_err.ev_count = v7; 1309 1310 /* 1311 * Check the TX queues status 1312 * - mark hung queues so we don't schedule on them 1313 * - watchdog only if all queues show hung 1314 */ 1315 que = adapter->queues; 1316 for (i = 0; i < adapter->num_queues; i++, que++) { 1317 /* Keep track of queues with work for soft irq */ 1318 if (que->txr->busy) 1319 queues |= ((u64)1 << que->me); 1320 /* 1321 * Each time txeof runs without cleaning, but there 1322 * are uncleaned descriptors it increments busy. If 1323 * we get to the MAX we declare it hung. 1324 */ 1325 if (que->busy == IXGBE_QUEUE_HUNG) { 1326 ++hung; 1327 /* Mark the queue as inactive */ 1328 adapter->active_queues &= ~((u64)1 << que->me); 1329 continue; 1330 } else { 1331 /* Check if we've come back from hung */ 1332 if ((adapter->active_queues & ((u64)1 << que->me)) == 0) 1333 adapter->active_queues |= ((u64)1 << que->me); 1334 } 1335 if (que->busy >= IXGBE_MAX_TX_BUSY) { 1336 device_printf(dev, 1337 "Warning queue %d appears to be hung!\n", i); 1338 que->txr->busy = IXGBE_QUEUE_HUNG; 1339 ++hung; 1340 } 1341 } 1342 1343 /* Only truly watchdog if all queues show hung */ 1344 if (hung == adapter->num_queues) 1345 goto watchdog; 1346 #if 0 1347 else if (queues != 0) { /* Force an IRQ on queues with work */ 1348 ixv_rearm_queues(adapter, queues); 1349 } 1350 #endif 1351 1352 callout_reset(&adapter->timer, hz, ixv_local_timer, adapter); 1353 1354 return; 1355 1356 watchdog: 1357 1358 device_printf(adapter->dev, "Watchdog timeout -- resetting\n"); 1359 adapter->ifp->if_flags &= ~IFF_RUNNING; 1360 adapter->watchdog_events.ev_count++; 1361 ixv_init_locked(adapter); 1362 } /* ixv_local_timer */ 1363 1364 /************************************************************************ 1365 * ixv_update_link_status - Update OS on link state 1366 * 1367 * Note: Only updates the OS on the cached link state. 1368 * The real check of the hardware only happens with 1369 * a link interrupt. 1370 ************************************************************************/ 1371 static void 1372 ixv_update_link_status(struct adapter *adapter) 1373 { 1374 struct ifnet *ifp = adapter->ifp; 1375 device_t dev = adapter->dev; 1376 1377 KASSERT(mutex_owned(&adapter->core_mtx)); 1378 1379 if (adapter->link_up) { 1380 if (adapter->link_active != LINK_STATE_UP) { 1381 if (bootverbose) { 1382 const char *bpsmsg; 1383 1384 switch (adapter->link_speed) { 1385 case IXGBE_LINK_SPEED_10GB_FULL: 1386 bpsmsg = "10 Gbps"; 1387 break; 1388 case IXGBE_LINK_SPEED_5GB_FULL: 1389 bpsmsg = "5 Gbps"; 1390 break; 1391 case IXGBE_LINK_SPEED_2_5GB_FULL: 1392 bpsmsg = "2.5 Gbps"; 1393 break; 1394 case IXGBE_LINK_SPEED_1GB_FULL: 1395 bpsmsg = "1 Gbps"; 1396 break; 1397 case IXGBE_LINK_SPEED_100_FULL: 1398 bpsmsg = "100 Mbps"; 1399 break; 1400 case IXGBE_LINK_SPEED_10_FULL: 1401 bpsmsg = "10 Mbps"; 1402 break; 1403 default: 1404 bpsmsg = "unknown speed"; 1405 break; 1406 } 1407 device_printf(dev, "Link is up %s %s \n", 1408 bpsmsg, "Full Duplex"); 1409 } 1410 adapter->link_active = LINK_STATE_UP; 1411 if_link_state_change(ifp, LINK_STATE_UP); 1412 } 1413 } else { 1414 /* 1415 * Do it when link active changes to DOWN. i.e. 1416 * a) LINK_STATE_UNKNOWN -> LINK_STATE_DOWN 1417 * b) LINK_STATE_UP -> LINK_STATE_DOWN 1418 */ 1419 if (adapter->link_active != LINK_STATE_DOWN) { 1420 if (bootverbose) 1421 device_printf(dev, "Link is Down\n"); 1422 if_link_state_change(ifp, LINK_STATE_DOWN); 1423 adapter->link_active = LINK_STATE_DOWN; 1424 } 1425 } 1426 } /* ixv_update_link_status */ 1427 1428 1429 /************************************************************************ 1430 * ixv_stop - Stop the hardware 1431 * 1432 * Disables all traffic on the adapter by issuing a 1433 * global reset on the MAC and deallocates TX/RX buffers. 1434 ************************************************************************/ 1435 static void 1436 ixv_ifstop(struct ifnet *ifp, int disable) 1437 { 1438 struct adapter *adapter = ifp->if_softc; 1439 1440 IXGBE_CORE_LOCK(adapter); 1441 ixv_stop(adapter); 1442 IXGBE_CORE_UNLOCK(adapter); 1443 } 1444 1445 static void 1446 ixv_stop(void *arg) 1447 { 1448 struct ifnet *ifp; 1449 struct adapter *adapter = arg; 1450 struct ixgbe_hw *hw = &adapter->hw; 1451 1452 ifp = adapter->ifp; 1453 1454 KASSERT(mutex_owned(&adapter->core_mtx)); 1455 1456 INIT_DEBUGOUT("ixv_stop: begin\n"); 1457 ixv_disable_intr(adapter); 1458 1459 /* Tell the stack that the interface is no longer active */ 1460 ifp->if_flags &= ~(IFF_RUNNING | IFF_OACTIVE); 1461 1462 hw->mac.ops.reset_hw(hw); 1463 adapter->hw.adapter_stopped = FALSE; 1464 hw->mac.ops.stop_adapter(hw); 1465 callout_stop(&adapter->timer); 1466 1467 /* reprogram the RAR[0] in case user changed it. */ 1468 hw->mac.ops.set_rar(hw, 0, hw->mac.addr, 0, IXGBE_RAH_AV); 1469 1470 return; 1471 } /* ixv_stop */ 1472 1473 1474 /************************************************************************ 1475 * ixv_allocate_pci_resources 1476 ************************************************************************/ 1477 static int 1478 ixv_allocate_pci_resources(struct adapter *adapter, 1479 const struct pci_attach_args *pa) 1480 { 1481 pcireg_t memtype, csr; 1482 device_t dev = adapter->dev; 1483 bus_addr_t addr; 1484 int flags; 1485 1486 memtype = pci_mapreg_type(pa->pa_pc, pa->pa_tag, PCI_BAR(0)); 1487 switch (memtype) { 1488 case PCI_MAPREG_TYPE_MEM | PCI_MAPREG_MEM_TYPE_32BIT: 1489 case PCI_MAPREG_TYPE_MEM | PCI_MAPREG_MEM_TYPE_64BIT: 1490 adapter->osdep.mem_bus_space_tag = pa->pa_memt; 1491 if (pci_mapreg_info(pa->pa_pc, pa->pa_tag, PCI_BAR(0), 1492 memtype, &addr, &adapter->osdep.mem_size, &flags) != 0) 1493 goto map_err; 1494 if ((flags & BUS_SPACE_MAP_PREFETCHABLE) != 0) { 1495 aprint_normal_dev(dev, "clearing prefetchable bit\n"); 1496 flags &= ~BUS_SPACE_MAP_PREFETCHABLE; 1497 } 1498 if (bus_space_map(adapter->osdep.mem_bus_space_tag, addr, 1499 adapter->osdep.mem_size, flags, 1500 &adapter->osdep.mem_bus_space_handle) != 0) { 1501 map_err: 1502 adapter->osdep.mem_size = 0; 1503 aprint_error_dev(dev, "unable to map BAR0\n"); 1504 return ENXIO; 1505 } 1506 /* 1507 * Enable address decoding for memory range in case it's not 1508 * set. 1509 */ 1510 csr = pci_conf_read(pa->pa_pc, pa->pa_tag, 1511 PCI_COMMAND_STATUS_REG); 1512 csr |= PCI_COMMAND_MEM_ENABLE; 1513 pci_conf_write(pa->pa_pc, pa->pa_tag, PCI_COMMAND_STATUS_REG, 1514 csr); 1515 break; 1516 default: 1517 aprint_error_dev(dev, "unexpected type on BAR0\n"); 1518 return ENXIO; 1519 } 1520 1521 /* Pick up the tuneable queues */ 1522 adapter->num_queues = ixv_num_queues; 1523 1524 return (0); 1525 } /* ixv_allocate_pci_resources */ 1526 1527 /************************************************************************ 1528 * ixv_free_pci_resources 1529 ************************************************************************/ 1530 static void 1531 ixv_free_pci_resources(struct adapter * adapter) 1532 { 1533 struct ix_queue *que = adapter->queues; 1534 int rid; 1535 1536 /* 1537 * Release all msix queue resources: 1538 */ 1539 for (int i = 0; i < adapter->num_queues; i++, que++) { 1540 if (que->res != NULL) 1541 pci_intr_disestablish(adapter->osdep.pc, 1542 adapter->osdep.ihs[i]); 1543 } 1544 1545 1546 /* Clean the Mailbox interrupt last */ 1547 rid = adapter->vector; 1548 1549 if (adapter->osdep.ihs[rid] != NULL) { 1550 pci_intr_disestablish(adapter->osdep.pc, 1551 adapter->osdep.ihs[rid]); 1552 adapter->osdep.ihs[rid] = NULL; 1553 } 1554 1555 pci_intr_release(adapter->osdep.pc, adapter->osdep.intrs, 1556 adapter->osdep.nintrs); 1557 1558 if (adapter->osdep.mem_size != 0) { 1559 bus_space_unmap(adapter->osdep.mem_bus_space_tag, 1560 adapter->osdep.mem_bus_space_handle, 1561 adapter->osdep.mem_size); 1562 } 1563 1564 return; 1565 } /* ixv_free_pci_resources */ 1566 1567 /************************************************************************ 1568 * ixv_setup_interface 1569 * 1570 * Setup networking device structure and register an interface. 1571 ************************************************************************/ 1572 static int 1573 ixv_setup_interface(device_t dev, struct adapter *adapter) 1574 { 1575 struct ethercom *ec = &adapter->osdep.ec; 1576 struct ifnet *ifp; 1577 int rv; 1578 1579 INIT_DEBUGOUT("ixv_setup_interface: begin"); 1580 1581 ifp = adapter->ifp = &ec->ec_if; 1582 strlcpy(ifp->if_xname, device_xname(dev), IFNAMSIZ); 1583 ifp->if_baudrate = IF_Gbps(10); 1584 ifp->if_init = ixv_init; 1585 ifp->if_stop = ixv_ifstop; 1586 ifp->if_softc = adapter; 1587 ifp->if_flags = IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST; 1588 #ifdef IXGBE_MPSAFE 1589 ifp->if_extflags = IFEF_MPSAFE; 1590 #endif 1591 ifp->if_ioctl = ixv_ioctl; 1592 if (adapter->feat_en & IXGBE_FEATURE_LEGACY_TX) { 1593 #if 0 1594 ixv_start_locked = ixgbe_legacy_start_locked; 1595 #endif 1596 } else { 1597 ifp->if_transmit = ixgbe_mq_start; 1598 #if 0 1599 ixv_start_locked = ixgbe_mq_start_locked; 1600 #endif 1601 } 1602 ifp->if_start = ixgbe_legacy_start; 1603 IFQ_SET_MAXLEN(&ifp->if_snd, adapter->num_tx_desc - 2); 1604 IFQ_SET_READY(&ifp->if_snd); 1605 1606 rv = if_initialize(ifp); 1607 if (rv != 0) { 1608 aprint_error_dev(dev, "if_initialize failed(%d)\n", rv); 1609 return rv; 1610 } 1611 adapter->ipq = if_percpuq_create(&adapter->osdep.ec.ec_if); 1612 ether_ifattach(ifp, adapter->hw.mac.addr); 1613 /* 1614 * We use per TX queue softint, so if_deferred_start_init() isn't 1615 * used. 1616 */ 1617 ether_set_ifflags_cb(ec, ixv_ifflags_cb); 1618 1619 adapter->max_frame_size = ifp->if_mtu + IXGBE_MTU_HDR; 1620 1621 /* 1622 * Tell the upper layer(s) we support long frames. 1623 */ 1624 ifp->if_hdrlen = sizeof(struct ether_vlan_header); 1625 1626 /* Set capability flags */ 1627 ifp->if_capabilities |= IFCAP_HWCSUM 1628 | IFCAP_TSOv4 1629 | IFCAP_TSOv6; 1630 ifp->if_capenable = 0; 1631 1632 ec->ec_capabilities |= ETHERCAP_VLAN_HWFILTER 1633 | ETHERCAP_VLAN_HWTAGGING 1634 | ETHERCAP_VLAN_HWCSUM 1635 | ETHERCAP_JUMBO_MTU 1636 | ETHERCAP_VLAN_MTU; 1637 1638 /* Enable the above capabilities by default */ 1639 ec->ec_capenable = ec->ec_capabilities; 1640 1641 /* Don't enable LRO by default */ 1642 #if 0 1643 /* NetBSD doesn't support LRO yet */ 1644 ifp->if_capabilities |= IFCAP_LRO; 1645 #endif 1646 1647 /* 1648 * Specify the media types supported by this adapter and register 1649 * callbacks to update media and link information 1650 */ 1651 ec->ec_ifmedia = &adapter->media; 1652 ifmedia_init(&adapter->media, IFM_IMASK, ixv_media_change, 1653 ixv_media_status); 1654 ifmedia_add(&adapter->media, IFM_ETHER | IFM_AUTO, 0, NULL); 1655 ifmedia_set(&adapter->media, IFM_ETHER | IFM_AUTO); 1656 1657 if_register(ifp); 1658 1659 return 0; 1660 } /* ixv_setup_interface */ 1661 1662 1663 /************************************************************************ 1664 * ixv_initialize_transmit_units - Enable transmit unit. 1665 ************************************************************************/ 1666 static void 1667 ixv_initialize_transmit_units(struct adapter *adapter) 1668 { 1669 struct tx_ring *txr = adapter->tx_rings; 1670 struct ixgbe_hw *hw = &adapter->hw; 1671 int i; 1672 1673 for (i = 0; i < adapter->num_queues; i++, txr++) { 1674 u64 tdba = txr->txdma.dma_paddr; 1675 u32 txctrl, txdctl; 1676 int j = txr->me; 1677 1678 /* Set WTHRESH to 8, burst writeback */ 1679 txdctl = IXGBE_READ_REG(hw, IXGBE_VFTXDCTL(j)); 1680 txdctl |= (8 << 16); 1681 IXGBE_WRITE_REG(hw, IXGBE_VFTXDCTL(j), txdctl); 1682 1683 /* Set the HW Tx Head and Tail indices */ 1684 IXGBE_WRITE_REG(&adapter->hw, IXGBE_VFTDH(j), 0); 1685 IXGBE_WRITE_REG(&adapter->hw, IXGBE_VFTDT(j), 0); 1686 1687 /* Set Tx Tail register */ 1688 txr->tail = IXGBE_VFTDT(j); 1689 1690 txr->txr_no_space = false; 1691 1692 /* Set Ring parameters */ 1693 IXGBE_WRITE_REG(hw, IXGBE_VFTDBAL(j), 1694 (tdba & 0x00000000ffffffffULL)); 1695 IXGBE_WRITE_REG(hw, IXGBE_VFTDBAH(j), (tdba >> 32)); 1696 IXGBE_WRITE_REG(hw, IXGBE_VFTDLEN(j), 1697 adapter->num_tx_desc * sizeof(struct ixgbe_legacy_tx_desc)); 1698 txctrl = IXGBE_READ_REG(hw, IXGBE_VFDCA_TXCTRL(j)); 1699 txctrl &= ~IXGBE_DCA_TXCTRL_DESC_WRO_EN; 1700 IXGBE_WRITE_REG(hw, IXGBE_VFDCA_TXCTRL(j), txctrl); 1701 1702 /* Now enable */ 1703 txdctl = IXGBE_READ_REG(hw, IXGBE_VFTXDCTL(j)); 1704 txdctl |= IXGBE_TXDCTL_ENABLE; 1705 IXGBE_WRITE_REG(hw, IXGBE_VFTXDCTL(j), txdctl); 1706 } 1707 1708 return; 1709 } /* ixv_initialize_transmit_units */ 1710 1711 1712 /************************************************************************ 1713 * ixv_initialize_rss_mapping 1714 ************************************************************************/ 1715 static void 1716 ixv_initialize_rss_mapping(struct adapter *adapter) 1717 { 1718 struct ixgbe_hw *hw = &adapter->hw; 1719 u32 reta = 0, mrqc, rss_key[10]; 1720 int queue_id; 1721 int i, j; 1722 u32 rss_hash_config; 1723 1724 /* force use default RSS key. */ 1725 #ifdef __NetBSD__ 1726 rss_getkey((uint8_t *) &rss_key); 1727 #else 1728 if (adapter->feat_en & IXGBE_FEATURE_RSS) { 1729 /* Fetch the configured RSS key */ 1730 rss_getkey((uint8_t *)&rss_key); 1731 } else { 1732 /* set up random bits */ 1733 cprng_fast(&rss_key, sizeof(rss_key)); 1734 } 1735 #endif 1736 1737 /* Now fill out hash function seeds */ 1738 for (i = 0; i < 10; i++) 1739 IXGBE_WRITE_REG(hw, IXGBE_VFRSSRK(i), rss_key[i]); 1740 1741 /* Set up the redirection table */ 1742 for (i = 0, j = 0; i < 64; i++, j++) { 1743 if (j == adapter->num_queues) 1744 j = 0; 1745 1746 if (adapter->feat_en & IXGBE_FEATURE_RSS) { 1747 /* 1748 * Fetch the RSS bucket id for the given indirection 1749 * entry. Cap it at the number of configured buckets 1750 * (which is num_queues.) 1751 */ 1752 queue_id = rss_get_indirection_to_bucket(i); 1753 queue_id = queue_id % adapter->num_queues; 1754 } else 1755 queue_id = j; 1756 1757 /* 1758 * The low 8 bits are for hash value (n+0); 1759 * The next 8 bits are for hash value (n+1), etc. 1760 */ 1761 reta >>= 8; 1762 reta |= ((uint32_t)queue_id) << 24; 1763 if ((i & 3) == 3) { 1764 IXGBE_WRITE_REG(hw, IXGBE_VFRETA(i >> 2), reta); 1765 reta = 0; 1766 } 1767 } 1768 1769 /* Perform hash on these packet types */ 1770 if (adapter->feat_en & IXGBE_FEATURE_RSS) 1771 rss_hash_config = rss_gethashconfig(); 1772 else { 1773 /* 1774 * Disable UDP - IP fragments aren't currently being handled 1775 * and so we end up with a mix of 2-tuple and 4-tuple 1776 * traffic. 1777 */ 1778 rss_hash_config = RSS_HASHTYPE_RSS_IPV4 1779 | RSS_HASHTYPE_RSS_TCP_IPV4 1780 | RSS_HASHTYPE_RSS_IPV6 1781 | RSS_HASHTYPE_RSS_TCP_IPV6; 1782 } 1783 1784 mrqc = IXGBE_MRQC_RSSEN; 1785 if (rss_hash_config & RSS_HASHTYPE_RSS_IPV4) 1786 mrqc |= IXGBE_MRQC_RSS_FIELD_IPV4; 1787 if (rss_hash_config & RSS_HASHTYPE_RSS_TCP_IPV4) 1788 mrqc |= IXGBE_MRQC_RSS_FIELD_IPV4_TCP; 1789 if (rss_hash_config & RSS_HASHTYPE_RSS_IPV6) 1790 mrqc |= IXGBE_MRQC_RSS_FIELD_IPV6; 1791 if (rss_hash_config & RSS_HASHTYPE_RSS_TCP_IPV6) 1792 mrqc |= IXGBE_MRQC_RSS_FIELD_IPV6_TCP; 1793 if (rss_hash_config & RSS_HASHTYPE_RSS_IPV6_EX) 1794 device_printf(adapter->dev, "%s: RSS_HASHTYPE_RSS_IPV6_EX defined, but not supported\n", 1795 __func__); 1796 if (rss_hash_config & RSS_HASHTYPE_RSS_TCP_IPV6_EX) 1797 device_printf(adapter->dev, "%s: RSS_HASHTYPE_RSS_TCP_IPV6_EX defined, but not supported\n", 1798 __func__); 1799 if (rss_hash_config & RSS_HASHTYPE_RSS_UDP_IPV4) 1800 mrqc |= IXGBE_MRQC_RSS_FIELD_IPV4_UDP; 1801 if (rss_hash_config & RSS_HASHTYPE_RSS_UDP_IPV6) 1802 mrqc |= IXGBE_MRQC_RSS_FIELD_IPV6_UDP; 1803 if (rss_hash_config & RSS_HASHTYPE_RSS_UDP_IPV6_EX) 1804 device_printf(adapter->dev, "%s: RSS_HASHTYPE_RSS_UDP_IPV6_EX defined, but not supported\n", 1805 __func__); 1806 IXGBE_WRITE_REG(hw, IXGBE_VFMRQC, mrqc); 1807 } /* ixv_initialize_rss_mapping */ 1808 1809 1810 /************************************************************************ 1811 * ixv_initialize_receive_units - Setup receive registers and features. 1812 ************************************************************************/ 1813 static void 1814 ixv_initialize_receive_units(struct adapter *adapter) 1815 { 1816 struct rx_ring *rxr = adapter->rx_rings; 1817 struct ixgbe_hw *hw = &adapter->hw; 1818 struct ifnet *ifp = adapter->ifp; 1819 u32 bufsz, psrtype; 1820 1821 if (ifp->if_mtu > ETHERMTU) 1822 bufsz = 4096 >> IXGBE_SRRCTL_BSIZEPKT_SHIFT; 1823 else 1824 bufsz = 2048 >> IXGBE_SRRCTL_BSIZEPKT_SHIFT; 1825 1826 psrtype = IXGBE_PSRTYPE_TCPHDR 1827 | IXGBE_PSRTYPE_UDPHDR 1828 | IXGBE_PSRTYPE_IPV4HDR 1829 | IXGBE_PSRTYPE_IPV6HDR 1830 | IXGBE_PSRTYPE_L2HDR; 1831 1832 if (adapter->num_queues > 1) 1833 psrtype |= 1 << 29; 1834 1835 IXGBE_WRITE_REG(hw, IXGBE_VFPSRTYPE, psrtype); 1836 1837 /* Tell PF our max_frame size */ 1838 if (ixgbevf_rlpml_set_vf(hw, adapter->max_frame_size) != 0) { 1839 device_printf(adapter->dev, "There is a problem with the PF setup. It is likely the receive unit for this VF will not function correctly.\n"); 1840 } 1841 1842 for (int i = 0; i < adapter->num_queues; i++, rxr++) { 1843 u64 rdba = rxr->rxdma.dma_paddr; 1844 u32 reg, rxdctl; 1845 int j = rxr->me; 1846 1847 /* Disable the queue */ 1848 rxdctl = IXGBE_READ_REG(hw, IXGBE_VFRXDCTL(j)); 1849 rxdctl &= ~IXGBE_RXDCTL_ENABLE; 1850 IXGBE_WRITE_REG(hw, IXGBE_VFRXDCTL(j), rxdctl); 1851 for (int k = 0; k < 10; k++) { 1852 if (IXGBE_READ_REG(hw, IXGBE_VFRXDCTL(j)) & 1853 IXGBE_RXDCTL_ENABLE) 1854 msec_delay(1); 1855 else 1856 break; 1857 } 1858 wmb(); 1859 /* Setup the Base and Length of the Rx Descriptor Ring */ 1860 IXGBE_WRITE_REG(hw, IXGBE_VFRDBAL(j), 1861 (rdba & 0x00000000ffffffffULL)); 1862 IXGBE_WRITE_REG(hw, IXGBE_VFRDBAH(j), (rdba >> 32)); 1863 IXGBE_WRITE_REG(hw, IXGBE_VFRDLEN(j), 1864 adapter->num_rx_desc * sizeof(union ixgbe_adv_rx_desc)); 1865 1866 /* Reset the ring indices */ 1867 IXGBE_WRITE_REG(hw, IXGBE_VFRDH(rxr->me), 0); 1868 IXGBE_WRITE_REG(hw, IXGBE_VFRDT(rxr->me), 0); 1869 1870 /* Set up the SRRCTL register */ 1871 reg = IXGBE_READ_REG(hw, IXGBE_VFSRRCTL(j)); 1872 reg &= ~IXGBE_SRRCTL_BSIZEHDR_MASK; 1873 reg &= ~IXGBE_SRRCTL_BSIZEPKT_MASK; 1874 reg |= bufsz; 1875 reg |= IXGBE_SRRCTL_DESCTYPE_ADV_ONEBUF; 1876 IXGBE_WRITE_REG(hw, IXGBE_VFSRRCTL(j), reg); 1877 1878 /* Capture Rx Tail index */ 1879 rxr->tail = IXGBE_VFRDT(rxr->me); 1880 1881 /* Do the queue enabling last */ 1882 rxdctl |= IXGBE_RXDCTL_ENABLE | IXGBE_RXDCTL_VME; 1883 IXGBE_WRITE_REG(hw, IXGBE_VFRXDCTL(j), rxdctl); 1884 for (int k = 0; k < 10; k++) { 1885 if (IXGBE_READ_REG(hw, IXGBE_VFRXDCTL(j)) & 1886 IXGBE_RXDCTL_ENABLE) 1887 break; 1888 msec_delay(1); 1889 } 1890 wmb(); 1891 1892 /* Set the Tail Pointer */ 1893 #ifdef DEV_NETMAP 1894 /* 1895 * In netmap mode, we must preserve the buffers made 1896 * available to userspace before the if_init() 1897 * (this is true by default on the TX side, because 1898 * init makes all buffers available to userspace). 1899 * 1900 * netmap_reset() and the device specific routines 1901 * (e.g. ixgbe_setup_receive_rings()) map these 1902 * buffers at the end of the NIC ring, so here we 1903 * must set the RDT (tail) register to make sure 1904 * they are not overwritten. 1905 * 1906 * In this driver the NIC ring starts at RDH = 0, 1907 * RDT points to the last slot available for reception (?), 1908 * so RDT = num_rx_desc - 1 means the whole ring is available. 1909 */ 1910 if ((adapter->feat_en & IXGBE_FEATURE_NETMAP) && 1911 (ifp->if_capenable & IFCAP_NETMAP)) { 1912 struct netmap_adapter *na = NA(adapter->ifp); 1913 struct netmap_kring *kring = na->rx_rings[i]; 1914 int t = na->num_rx_desc - 1 - nm_kr_rxspace(kring); 1915 1916 IXGBE_WRITE_REG(hw, IXGBE_VFRDT(rxr->me), t); 1917 } else 1918 #endif /* DEV_NETMAP */ 1919 IXGBE_WRITE_REG(hw, IXGBE_VFRDT(rxr->me), 1920 adapter->num_rx_desc - 1); 1921 } 1922 1923 if (adapter->hw.mac.type >= ixgbe_mac_X550_vf) 1924 ixv_initialize_rss_mapping(adapter); 1925 } /* ixv_initialize_receive_units */ 1926 1927 /************************************************************************ 1928 * ixv_sysctl_tdh_handler - Transmit Descriptor Head handler function 1929 * 1930 * Retrieves the TDH value from the hardware 1931 ************************************************************************/ 1932 static int 1933 ixv_sysctl_tdh_handler(SYSCTLFN_ARGS) 1934 { 1935 struct sysctlnode node = *rnode; 1936 struct tx_ring *txr = (struct tx_ring *)node.sysctl_data; 1937 uint32_t val; 1938 1939 if (!txr) 1940 return (0); 1941 1942 val = IXGBE_READ_REG(&txr->adapter->hw, IXGBE_VFTDH(txr->me)); 1943 node.sysctl_data = &val; 1944 return sysctl_lookup(SYSCTLFN_CALL(&node)); 1945 } /* ixv_sysctl_tdh_handler */ 1946 1947 /************************************************************************ 1948 * ixgbe_sysctl_tdt_handler - Transmit Descriptor Tail handler function 1949 * 1950 * Retrieves the TDT value from the hardware 1951 ************************************************************************/ 1952 static int 1953 ixv_sysctl_tdt_handler(SYSCTLFN_ARGS) 1954 { 1955 struct sysctlnode node = *rnode; 1956 struct tx_ring *txr = (struct tx_ring *)node.sysctl_data; 1957 uint32_t val; 1958 1959 if (!txr) 1960 return (0); 1961 1962 val = IXGBE_READ_REG(&txr->adapter->hw, IXGBE_VFTDT(txr->me)); 1963 node.sysctl_data = &val; 1964 return sysctl_lookup(SYSCTLFN_CALL(&node)); 1965 } /* ixv_sysctl_tdt_handler */ 1966 1967 /************************************************************************ 1968 * ixv_sysctl_next_to_check_handler - Receive Descriptor next to check 1969 * handler function 1970 * 1971 * Retrieves the next_to_check value 1972 ************************************************************************/ 1973 static int 1974 ixv_sysctl_next_to_check_handler(SYSCTLFN_ARGS) 1975 { 1976 struct sysctlnode node = *rnode; 1977 struct rx_ring *rxr = (struct rx_ring *)node.sysctl_data; 1978 uint32_t val; 1979 1980 if (!rxr) 1981 return (0); 1982 1983 val = rxr->next_to_check; 1984 node.sysctl_data = &val; 1985 return sysctl_lookup(SYSCTLFN_CALL(&node)); 1986 } /* ixv_sysctl_next_to_check_handler */ 1987 1988 /************************************************************************ 1989 * ixv_sysctl_rdh_handler - Receive Descriptor Head handler function 1990 * 1991 * Retrieves the RDH value from the hardware 1992 ************************************************************************/ 1993 static int 1994 ixv_sysctl_rdh_handler(SYSCTLFN_ARGS) 1995 { 1996 struct sysctlnode node = *rnode; 1997 struct rx_ring *rxr = (struct rx_ring *)node.sysctl_data; 1998 uint32_t val; 1999 2000 if (!rxr) 2001 return (0); 2002 2003 val = IXGBE_READ_REG(&rxr->adapter->hw, IXGBE_VFRDH(rxr->me)); 2004 node.sysctl_data = &val; 2005 return sysctl_lookup(SYSCTLFN_CALL(&node)); 2006 } /* ixv_sysctl_rdh_handler */ 2007 2008 /************************************************************************ 2009 * ixv_sysctl_rdt_handler - Receive Descriptor Tail handler function 2010 * 2011 * Retrieves the RDT value from the hardware 2012 ************************************************************************/ 2013 static int 2014 ixv_sysctl_rdt_handler(SYSCTLFN_ARGS) 2015 { 2016 struct sysctlnode node = *rnode; 2017 struct rx_ring *rxr = (struct rx_ring *)node.sysctl_data; 2018 uint32_t val; 2019 2020 if (!rxr) 2021 return (0); 2022 2023 val = IXGBE_READ_REG(&rxr->adapter->hw, IXGBE_VFRDT(rxr->me)); 2024 node.sysctl_data = &val; 2025 return sysctl_lookup(SYSCTLFN_CALL(&node)); 2026 } /* ixv_sysctl_rdt_handler */ 2027 2028 static void 2029 ixv_setup_vlan_tagging(struct adapter *adapter) 2030 { 2031 struct ethercom *ec = &adapter->osdep.ec; 2032 struct ixgbe_hw *hw = &adapter->hw; 2033 struct rx_ring *rxr; 2034 u32 ctrl; 2035 int i; 2036 bool hwtagging; 2037 2038 /* Enable HW tagging only if any vlan is attached */ 2039 hwtagging = (ec->ec_capenable & ETHERCAP_VLAN_HWTAGGING) 2040 && VLAN_ATTACHED(ec); 2041 2042 /* Enable the queues */ 2043 for (i = 0; i < adapter->num_queues; i++) { 2044 rxr = &adapter->rx_rings[i]; 2045 ctrl = IXGBE_READ_REG(hw, IXGBE_VFRXDCTL(rxr->me)); 2046 if (hwtagging) 2047 ctrl |= IXGBE_RXDCTL_VME; 2048 else 2049 ctrl &= ~IXGBE_RXDCTL_VME; 2050 IXGBE_WRITE_REG(hw, IXGBE_VFRXDCTL(rxr->me), ctrl); 2051 /* 2052 * Let Rx path know that it needs to store VLAN tag 2053 * as part of extra mbuf info. 2054 */ 2055 rxr->vtag_strip = hwtagging ? TRUE : FALSE; 2056 } 2057 } /* ixv_setup_vlan_tagging */ 2058 2059 /************************************************************************ 2060 * ixv_setup_vlan_support 2061 ************************************************************************/ 2062 static int 2063 ixv_setup_vlan_support(struct adapter *adapter) 2064 { 2065 struct ethercom *ec = &adapter->osdep.ec; 2066 struct ixgbe_hw *hw = &adapter->hw; 2067 u32 vid, vfta, retry; 2068 struct vlanid_list *vlanidp; 2069 int rv, error = 0; 2070 2071 /* 2072 * This function is called from both if_init and ifflags_cb() 2073 * on NetBSD. 2074 */ 2075 2076 /* 2077 * Part 1: 2078 * Setup VLAN HW tagging 2079 */ 2080 ixv_setup_vlan_tagging(adapter); 2081 2082 if (!VLAN_ATTACHED(ec)) 2083 return 0; 2084 2085 /* 2086 * Part 2: 2087 * Setup VLAN HW filter 2088 */ 2089 /* Cleanup shadow_vfta */ 2090 for (int i = 0; i < IXGBE_VFTA_SIZE; i++) 2091 adapter->shadow_vfta[i] = 0; 2092 /* Generate shadow_vfta from ec_vids */ 2093 ETHER_LOCK(ec); 2094 SIMPLEQ_FOREACH(vlanidp, &ec->ec_vids, vid_list) { 2095 uint32_t idx; 2096 2097 idx = vlanidp->vid / 32; 2098 KASSERT(idx < IXGBE_VFTA_SIZE); 2099 adapter->shadow_vfta[idx] |= (u32)1 << (vlanidp->vid % 32); 2100 } 2101 ETHER_UNLOCK(ec); 2102 2103 /* 2104 * A soft reset zero's out the VFTA, so 2105 * we need to repopulate it now. 2106 */ 2107 for (int i = 0; i < IXGBE_VFTA_SIZE; i++) { 2108 if (adapter->shadow_vfta[i] == 0) 2109 continue; 2110 vfta = adapter->shadow_vfta[i]; 2111 /* 2112 * Reconstruct the vlan id's 2113 * based on the bits set in each 2114 * of the array ints. 2115 */ 2116 for (int j = 0; j < 32; j++) { 2117 retry = 0; 2118 if ((vfta & ((u32)1 << j)) == 0) 2119 continue; 2120 vid = (i * 32) + j; 2121 2122 /* Call the shared code mailbox routine */ 2123 while ((rv = hw->mac.ops.set_vfta(hw, vid, 0, TRUE, 2124 FALSE)) != 0) { 2125 if (++retry > 5) { 2126 device_printf(adapter->dev, 2127 "%s: max retry exceeded\n", 2128 __func__); 2129 break; 2130 } 2131 } 2132 if (rv != 0) { 2133 device_printf(adapter->dev, 2134 "failed to set vlan %d\n", vid); 2135 error = EACCES; 2136 } 2137 } 2138 } 2139 return error; 2140 } /* ixv_setup_vlan_support */ 2141 2142 static int 2143 ixv_vlan_cb(struct ethercom *ec, uint16_t vid, bool set) 2144 { 2145 struct ifnet *ifp = &ec->ec_if; 2146 struct adapter *adapter = ifp->if_softc; 2147 int rv; 2148 2149 if (set) 2150 rv = ixv_register_vlan(adapter, vid); 2151 else 2152 rv = ixv_unregister_vlan(adapter, vid); 2153 2154 if (rv != 0) 2155 return rv; 2156 2157 /* 2158 * Control VLAN HW tagging when ec_nvlan is changed from 1 to 0 2159 * or 0 to 1. 2160 */ 2161 if ((set && (ec->ec_nvlans == 1)) || (!set && (ec->ec_nvlans == 0))) 2162 ixv_setup_vlan_tagging(adapter); 2163 2164 return rv; 2165 } 2166 2167 /************************************************************************ 2168 * ixv_register_vlan 2169 * 2170 * Run via a vlan config EVENT, it enables us to use the 2171 * HW Filter table since we can get the vlan id. This just 2172 * creates the entry in the soft version of the VFTA, init 2173 * will repopulate the real table. 2174 ************************************************************************/ 2175 static int 2176 ixv_register_vlan(struct adapter *adapter, u16 vtag) 2177 { 2178 struct ixgbe_hw *hw = &adapter->hw; 2179 u16 index, bit; 2180 int error; 2181 2182 if ((vtag == 0) || (vtag > 4095)) /* Invalid */ 2183 return EINVAL; 2184 IXGBE_CORE_LOCK(adapter); 2185 index = (vtag >> 5) & 0x7F; 2186 bit = vtag & 0x1F; 2187 adapter->shadow_vfta[index] |= ((u32)1 << bit); 2188 error = hw->mac.ops.set_vfta(hw, vtag, 0, true, false); 2189 IXGBE_CORE_UNLOCK(adapter); 2190 2191 if (error != 0) { 2192 device_printf(adapter->dev, "failed to register vlan %hu\n", 2193 vtag); 2194 error = EACCES; 2195 } 2196 return error; 2197 } /* ixv_register_vlan */ 2198 2199 /************************************************************************ 2200 * ixv_unregister_vlan 2201 * 2202 * Run via a vlan unconfig EVENT, remove our entry 2203 * in the soft vfta. 2204 ************************************************************************/ 2205 static int 2206 ixv_unregister_vlan(struct adapter *adapter, u16 vtag) 2207 { 2208 struct ixgbe_hw *hw = &adapter->hw; 2209 u16 index, bit; 2210 int error; 2211 2212 if ((vtag == 0) || (vtag > 4095)) /* Invalid */ 2213 return EINVAL; 2214 2215 IXGBE_CORE_LOCK(adapter); 2216 index = (vtag >> 5) & 0x7F; 2217 bit = vtag & 0x1F; 2218 adapter->shadow_vfta[index] &= ~((u32)1 << bit); 2219 error = hw->mac.ops.set_vfta(hw, vtag, 0, false, false); 2220 IXGBE_CORE_UNLOCK(adapter); 2221 2222 if (error != 0) { 2223 device_printf(adapter->dev, "failed to unregister vlan %hu\n", 2224 vtag); 2225 error = EIO; 2226 } 2227 return error; 2228 } /* ixv_unregister_vlan */ 2229 2230 /************************************************************************ 2231 * ixv_enable_intr 2232 ************************************************************************/ 2233 static void 2234 ixv_enable_intr(struct adapter *adapter) 2235 { 2236 struct ixgbe_hw *hw = &adapter->hw; 2237 struct ix_queue *que = adapter->queues; 2238 u32 mask; 2239 int i; 2240 2241 /* For VTEIAC */ 2242 mask = (1 << adapter->vector); 2243 for (i = 0; i < adapter->num_queues; i++, que++) 2244 mask |= (1 << que->msix); 2245 IXGBE_WRITE_REG(hw, IXGBE_VTEIAC, mask); 2246 2247 /* For VTEIMS */ 2248 IXGBE_WRITE_REG(hw, IXGBE_VTEIMS, (1 << adapter->vector)); 2249 que = adapter->queues; 2250 for (i = 0; i < adapter->num_queues; i++, que++) 2251 ixv_enable_queue(adapter, que->msix); 2252 2253 IXGBE_WRITE_FLUSH(hw); 2254 } /* ixv_enable_intr */ 2255 2256 /************************************************************************ 2257 * ixv_disable_intr 2258 ************************************************************************/ 2259 static void 2260 ixv_disable_intr(struct adapter *adapter) 2261 { 2262 struct ix_queue *que = adapter->queues; 2263 2264 IXGBE_WRITE_REG(&adapter->hw, IXGBE_VTEIAC, 0); 2265 2266 /* disable interrupts other than queues */ 2267 IXGBE_WRITE_REG(&adapter->hw, IXGBE_VTEIMC, adapter->vector); 2268 2269 for (int i = 0; i < adapter->num_queues; i++, que++) 2270 ixv_disable_queue(adapter, que->msix); 2271 2272 IXGBE_WRITE_FLUSH(&adapter->hw); 2273 } /* ixv_disable_intr */ 2274 2275 /************************************************************************ 2276 * ixv_set_ivar 2277 * 2278 * Setup the correct IVAR register for a particular MSI-X interrupt 2279 * - entry is the register array entry 2280 * - vector is the MSI-X vector for this queue 2281 * - type is RX/TX/MISC 2282 ************************************************************************/ 2283 static void 2284 ixv_set_ivar(struct adapter *adapter, u8 entry, u8 vector, s8 type) 2285 { 2286 struct ixgbe_hw *hw = &adapter->hw; 2287 u32 ivar, index; 2288 2289 vector |= IXGBE_IVAR_ALLOC_VAL; 2290 2291 if (type == -1) { /* MISC IVAR */ 2292 ivar = IXGBE_READ_REG(hw, IXGBE_VTIVAR_MISC); 2293 ivar &= ~0xFF; 2294 ivar |= vector; 2295 IXGBE_WRITE_REG(hw, IXGBE_VTIVAR_MISC, ivar); 2296 } else { /* RX/TX IVARS */ 2297 index = (16 * (entry & 1)) + (8 * type); 2298 ivar = IXGBE_READ_REG(hw, IXGBE_VTIVAR(entry >> 1)); 2299 ivar &= ~(0xffUL << index); 2300 ivar |= ((u32)vector << index); 2301 IXGBE_WRITE_REG(hw, IXGBE_VTIVAR(entry >> 1), ivar); 2302 } 2303 } /* ixv_set_ivar */ 2304 2305 /************************************************************************ 2306 * ixv_configure_ivars 2307 ************************************************************************/ 2308 static void 2309 ixv_configure_ivars(struct adapter *adapter) 2310 { 2311 struct ix_queue *que = adapter->queues; 2312 2313 /* XXX We should sync EITR value calculation with ixgbe.c? */ 2314 2315 for (int i = 0; i < adapter->num_queues; i++, que++) { 2316 /* First the RX queue entry */ 2317 ixv_set_ivar(adapter, i, que->msix, 0); 2318 /* ... and the TX */ 2319 ixv_set_ivar(adapter, i, que->msix, 1); 2320 /* Set an initial value in EITR */ 2321 ixv_eitr_write(adapter, que->msix, IXGBE_EITR_DEFAULT); 2322 } 2323 2324 /* For the mailbox interrupt */ 2325 ixv_set_ivar(adapter, 1, adapter->vector, -1); 2326 } /* ixv_configure_ivars */ 2327 2328 2329 /************************************************************************ 2330 * ixv_save_stats 2331 * 2332 * The VF stats registers never have a truly virgin 2333 * starting point, so this routine tries to make an 2334 * artificial one, marking ground zero on attach as 2335 * it were. 2336 ************************************************************************/ 2337 static void 2338 ixv_save_stats(struct adapter *adapter) 2339 { 2340 struct ixgbevf_hw_stats *stats = &adapter->stats.vf; 2341 2342 if (stats->vfgprc.ev_count || stats->vfgptc.ev_count) { 2343 stats->saved_reset_vfgprc += 2344 stats->vfgprc.ev_count - stats->base_vfgprc; 2345 stats->saved_reset_vfgptc += 2346 stats->vfgptc.ev_count - stats->base_vfgptc; 2347 stats->saved_reset_vfgorc += 2348 stats->vfgorc.ev_count - stats->base_vfgorc; 2349 stats->saved_reset_vfgotc += 2350 stats->vfgotc.ev_count - stats->base_vfgotc; 2351 stats->saved_reset_vfmprc += 2352 stats->vfmprc.ev_count - stats->base_vfmprc; 2353 } 2354 } /* ixv_save_stats */ 2355 2356 /************************************************************************ 2357 * ixv_init_stats 2358 ************************************************************************/ 2359 static void 2360 ixv_init_stats(struct adapter *adapter) 2361 { 2362 struct ixgbe_hw *hw = &adapter->hw; 2363 2364 adapter->stats.vf.last_vfgprc = IXGBE_READ_REG(hw, IXGBE_VFGPRC); 2365 adapter->stats.vf.last_vfgorc = IXGBE_READ_REG(hw, IXGBE_VFGORC_LSB); 2366 adapter->stats.vf.last_vfgorc |= 2367 (((u64)(IXGBE_READ_REG(hw, IXGBE_VFGORC_MSB))) << 32); 2368 2369 adapter->stats.vf.last_vfgptc = IXGBE_READ_REG(hw, IXGBE_VFGPTC); 2370 adapter->stats.vf.last_vfgotc = IXGBE_READ_REG(hw, IXGBE_VFGOTC_LSB); 2371 adapter->stats.vf.last_vfgotc |= 2372 (((u64)(IXGBE_READ_REG(hw, IXGBE_VFGOTC_MSB))) << 32); 2373 2374 adapter->stats.vf.last_vfmprc = IXGBE_READ_REG(hw, IXGBE_VFMPRC); 2375 2376 adapter->stats.vf.base_vfgprc = adapter->stats.vf.last_vfgprc; 2377 adapter->stats.vf.base_vfgorc = adapter->stats.vf.last_vfgorc; 2378 adapter->stats.vf.base_vfgptc = adapter->stats.vf.last_vfgptc; 2379 adapter->stats.vf.base_vfgotc = adapter->stats.vf.last_vfgotc; 2380 adapter->stats.vf.base_vfmprc = adapter->stats.vf.last_vfmprc; 2381 } /* ixv_init_stats */ 2382 2383 #define UPDATE_STAT_32(reg, last, count) \ 2384 { \ 2385 u32 current = IXGBE_READ_REG(hw, (reg)); \ 2386 if (current < (last)) \ 2387 count.ev_count += 0x100000000LL; \ 2388 (last) = current; \ 2389 count.ev_count &= 0xFFFFFFFF00000000LL; \ 2390 count.ev_count |= current; \ 2391 } 2392 2393 #define UPDATE_STAT_36(lsb, msb, last, count) \ 2394 { \ 2395 u64 cur_lsb = IXGBE_READ_REG(hw, (lsb)); \ 2396 u64 cur_msb = IXGBE_READ_REG(hw, (msb)); \ 2397 u64 current = ((cur_msb << 32) | cur_lsb); \ 2398 if (current < (last)) \ 2399 count.ev_count += 0x1000000000LL; \ 2400 (last) = current; \ 2401 count.ev_count &= 0xFFFFFFF000000000LL; \ 2402 count.ev_count |= current; \ 2403 } 2404 2405 /************************************************************************ 2406 * ixv_update_stats - Update the board statistics counters. 2407 ************************************************************************/ 2408 void 2409 ixv_update_stats(struct adapter *adapter) 2410 { 2411 struct ixgbe_hw *hw = &adapter->hw; 2412 struct ixgbevf_hw_stats *stats = &adapter->stats.vf; 2413 2414 UPDATE_STAT_32(IXGBE_VFGPRC, stats->last_vfgprc, stats->vfgprc); 2415 UPDATE_STAT_32(IXGBE_VFGPTC, stats->last_vfgptc, stats->vfgptc); 2416 UPDATE_STAT_36(IXGBE_VFGORC_LSB, IXGBE_VFGORC_MSB, stats->last_vfgorc, 2417 stats->vfgorc); 2418 UPDATE_STAT_36(IXGBE_VFGOTC_LSB, IXGBE_VFGOTC_MSB, stats->last_vfgotc, 2419 stats->vfgotc); 2420 UPDATE_STAT_32(IXGBE_VFMPRC, stats->last_vfmprc, stats->vfmprc); 2421 2422 /* Fill out the OS statistics structure */ 2423 /* 2424 * NetBSD: Don't override if_{i|o}{packets|bytes|mcasts} with 2425 * adapter->stats counters. It's required to make ifconfig -z 2426 * (SOICZIFDATA) work. 2427 */ 2428 } /* ixv_update_stats */ 2429 2430 /************************************************************************ 2431 * ixv_sysctl_interrupt_rate_handler 2432 ************************************************************************/ 2433 static int 2434 ixv_sysctl_interrupt_rate_handler(SYSCTLFN_ARGS) 2435 { 2436 struct sysctlnode node = *rnode; 2437 struct ix_queue *que = (struct ix_queue *)node.sysctl_data; 2438 struct adapter *adapter = que->adapter; 2439 uint32_t reg, usec, rate; 2440 int error; 2441 2442 if (que == NULL) 2443 return 0; 2444 reg = IXGBE_READ_REG(&que->adapter->hw, IXGBE_VTEITR(que->msix)); 2445 usec = ((reg & 0x0FF8) >> 3); 2446 if (usec > 0) 2447 rate = 500000 / usec; 2448 else 2449 rate = 0; 2450 node.sysctl_data = &rate; 2451 error = sysctl_lookup(SYSCTLFN_CALL(&node)); 2452 if (error || newp == NULL) 2453 return error; 2454 reg &= ~0xfff; /* default, no limitation */ 2455 if (rate > 0 && rate < 500000) { 2456 if (rate < 1000) 2457 rate = 1000; 2458 reg |= ((4000000/rate) & 0xff8); 2459 /* 2460 * When RSC is used, ITR interval must be larger than 2461 * RSC_DELAY. Currently, we use 2us for RSC_DELAY. 2462 * The minimum value is always greater than 2us on 100M 2463 * (and 10M?(not documented)), but it's not on 1G and higher. 2464 */ 2465 if ((adapter->link_speed != IXGBE_LINK_SPEED_100_FULL) 2466 && (adapter->link_speed != IXGBE_LINK_SPEED_10_FULL)) { 2467 if ((adapter->num_queues > 1) 2468 && (reg < IXGBE_MIN_RSC_EITR_10G1G)) 2469 return EINVAL; 2470 } 2471 ixv_max_interrupt_rate = rate; 2472 } else 2473 ixv_max_interrupt_rate = 0; 2474 ixv_eitr_write(adapter, que->msix, reg); 2475 2476 return (0); 2477 } /* ixv_sysctl_interrupt_rate_handler */ 2478 2479 const struct sysctlnode * 2480 ixv_sysctl_instance(struct adapter *adapter) 2481 { 2482 const char *dvname; 2483 struct sysctllog **log; 2484 int rc; 2485 const struct sysctlnode *rnode; 2486 2487 log = &adapter->sysctllog; 2488 dvname = device_xname(adapter->dev); 2489 2490 if ((rc = sysctl_createv(log, 0, NULL, &rnode, 2491 0, CTLTYPE_NODE, dvname, 2492 SYSCTL_DESCR("ixv information and settings"), 2493 NULL, 0, NULL, 0, CTL_HW, CTL_CREATE, CTL_EOL)) != 0) 2494 goto err; 2495 2496 return rnode; 2497 err: 2498 device_printf(adapter->dev, 2499 "%s: sysctl_createv failed, rc = %d\n", __func__, rc); 2500 return NULL; 2501 } 2502 2503 static void 2504 ixv_add_device_sysctls(struct adapter *adapter) 2505 { 2506 struct sysctllog **log; 2507 const struct sysctlnode *rnode, *cnode; 2508 device_t dev; 2509 2510 dev = adapter->dev; 2511 log = &adapter->sysctllog; 2512 2513 if ((rnode = ixv_sysctl_instance(adapter)) == NULL) { 2514 aprint_error_dev(dev, "could not create sysctl root\n"); 2515 return; 2516 } 2517 2518 if (sysctl_createv(log, 0, &rnode, &cnode, 2519 CTLFLAG_READWRITE, CTLTYPE_INT, 2520 "debug", SYSCTL_DESCR("Debug Info"), 2521 ixv_sysctl_debug, 0, (void *)adapter, 0, CTL_CREATE, CTL_EOL) != 0) 2522 aprint_error_dev(dev, "could not create sysctl\n"); 2523 2524 if (sysctl_createv(log, 0, &rnode, &cnode, 2525 CTLFLAG_READWRITE, CTLTYPE_BOOL, 2526 "enable_aim", SYSCTL_DESCR("Interrupt Moderation"), 2527 NULL, 0, &adapter->enable_aim, 0, CTL_CREATE, CTL_EOL) != 0) 2528 aprint_error_dev(dev, "could not create sysctl\n"); 2529 2530 if (sysctl_createv(log, 0, &rnode, &cnode, 2531 CTLFLAG_READWRITE, CTLTYPE_BOOL, 2532 "txrx_workqueue", SYSCTL_DESCR("Use workqueue for packet processing"), 2533 NULL, 0, &adapter->txrx_use_workqueue, 0, CTL_CREATE, CTL_EOL) != 0) 2534 aprint_error_dev(dev, "could not create sysctl\n"); 2535 } 2536 2537 /************************************************************************ 2538 * ixv_add_stats_sysctls - Add statistic sysctls for the VF. 2539 ************************************************************************/ 2540 static void 2541 ixv_add_stats_sysctls(struct adapter *adapter) 2542 { 2543 device_t dev = adapter->dev; 2544 struct tx_ring *txr = adapter->tx_rings; 2545 struct rx_ring *rxr = adapter->rx_rings; 2546 struct ixgbevf_hw_stats *stats = &adapter->stats.vf; 2547 struct ixgbe_hw *hw = &adapter->hw; 2548 const struct sysctlnode *rnode, *cnode; 2549 struct sysctllog **log = &adapter->sysctllog; 2550 const char *xname = device_xname(dev); 2551 2552 /* Driver Statistics */ 2553 evcnt_attach_dynamic(&adapter->efbig_tx_dma_setup, EVCNT_TYPE_MISC, 2554 NULL, xname, "Driver tx dma soft fail EFBIG"); 2555 evcnt_attach_dynamic(&adapter->mbuf_defrag_failed, EVCNT_TYPE_MISC, 2556 NULL, xname, "m_defrag() failed"); 2557 evcnt_attach_dynamic(&adapter->efbig2_tx_dma_setup, EVCNT_TYPE_MISC, 2558 NULL, xname, "Driver tx dma hard fail EFBIG"); 2559 evcnt_attach_dynamic(&adapter->einval_tx_dma_setup, EVCNT_TYPE_MISC, 2560 NULL, xname, "Driver tx dma hard fail EINVAL"); 2561 evcnt_attach_dynamic(&adapter->other_tx_dma_setup, EVCNT_TYPE_MISC, 2562 NULL, xname, "Driver tx dma hard fail other"); 2563 evcnt_attach_dynamic(&adapter->eagain_tx_dma_setup, EVCNT_TYPE_MISC, 2564 NULL, xname, "Driver tx dma soft fail EAGAIN"); 2565 evcnt_attach_dynamic(&adapter->enomem_tx_dma_setup, EVCNT_TYPE_MISC, 2566 NULL, xname, "Driver tx dma soft fail ENOMEM"); 2567 evcnt_attach_dynamic(&adapter->watchdog_events, EVCNT_TYPE_MISC, 2568 NULL, xname, "Watchdog timeouts"); 2569 evcnt_attach_dynamic(&adapter->tso_err, EVCNT_TYPE_MISC, 2570 NULL, xname, "TSO errors"); 2571 evcnt_attach_dynamic(&adapter->link_irq, EVCNT_TYPE_INTR, 2572 NULL, xname, "Link MSI-X IRQ Handled"); 2573 2574 for (int i = 0; i < adapter->num_queues; i++, rxr++, txr++) { 2575 snprintf(adapter->queues[i].evnamebuf, 2576 sizeof(adapter->queues[i].evnamebuf), "%s q%d", 2577 xname, i); 2578 snprintf(adapter->queues[i].namebuf, 2579 sizeof(adapter->queues[i].namebuf), "q%d", i); 2580 2581 if ((rnode = ixv_sysctl_instance(adapter)) == NULL) { 2582 aprint_error_dev(dev, "could not create sysctl root\n"); 2583 break; 2584 } 2585 2586 if (sysctl_createv(log, 0, &rnode, &rnode, 2587 0, CTLTYPE_NODE, 2588 adapter->queues[i].namebuf, SYSCTL_DESCR("Queue Name"), 2589 NULL, 0, NULL, 0, CTL_CREATE, CTL_EOL) != 0) 2590 break; 2591 2592 if (sysctl_createv(log, 0, &rnode, &cnode, 2593 CTLFLAG_READWRITE, CTLTYPE_INT, 2594 "interrupt_rate", SYSCTL_DESCR("Interrupt Rate"), 2595 ixv_sysctl_interrupt_rate_handler, 0, 2596 (void *)&adapter->queues[i], 0, CTL_CREATE, CTL_EOL) != 0) 2597 break; 2598 2599 if (sysctl_createv(log, 0, &rnode, &cnode, 2600 CTLFLAG_READONLY, CTLTYPE_INT, 2601 "txd_head", SYSCTL_DESCR("Transmit Descriptor Head"), 2602 ixv_sysctl_tdh_handler, 0, (void *)txr, 2603 0, CTL_CREATE, CTL_EOL) != 0) 2604 break; 2605 2606 if (sysctl_createv(log, 0, &rnode, &cnode, 2607 CTLFLAG_READONLY, CTLTYPE_INT, 2608 "txd_tail", SYSCTL_DESCR("Transmit Descriptor Tail"), 2609 ixv_sysctl_tdt_handler, 0, (void *)txr, 2610 0, CTL_CREATE, CTL_EOL) != 0) 2611 break; 2612 2613 evcnt_attach_dynamic(&adapter->queues[i].irqs, EVCNT_TYPE_INTR, 2614 NULL, adapter->queues[i].evnamebuf, "IRQs on queue"); 2615 evcnt_attach_dynamic(&adapter->queues[i].handleq, 2616 EVCNT_TYPE_MISC, NULL, adapter->queues[i].evnamebuf, 2617 "Handled queue in softint"); 2618 evcnt_attach_dynamic(&adapter->queues[i].req, EVCNT_TYPE_MISC, 2619 NULL, adapter->queues[i].evnamebuf, "Requeued in softint"); 2620 evcnt_attach_dynamic(&txr->tso_tx, EVCNT_TYPE_MISC, 2621 NULL, adapter->queues[i].evnamebuf, "TSO"); 2622 evcnt_attach_dynamic(&txr->no_desc_avail, EVCNT_TYPE_MISC, 2623 NULL, adapter->queues[i].evnamebuf, 2624 "Queue No Descriptor Available"); 2625 evcnt_attach_dynamic(&txr->total_packets, EVCNT_TYPE_MISC, 2626 NULL, adapter->queues[i].evnamebuf, 2627 "Queue Packets Transmitted"); 2628 #ifndef IXGBE_LEGACY_TX 2629 evcnt_attach_dynamic(&txr->pcq_drops, EVCNT_TYPE_MISC, 2630 NULL, adapter->queues[i].evnamebuf, 2631 "Packets dropped in pcq"); 2632 #endif 2633 2634 #ifdef LRO 2635 struct lro_ctrl *lro = &rxr->lro; 2636 #endif /* LRO */ 2637 2638 if (sysctl_createv(log, 0, &rnode, &cnode, 2639 CTLFLAG_READONLY, 2640 CTLTYPE_INT, 2641 "rxd_nxck", SYSCTL_DESCR("Receive Descriptor next to check"), 2642 ixv_sysctl_next_to_check_handler, 0, (void *)rxr, 0, 2643 CTL_CREATE, CTL_EOL) != 0) 2644 break; 2645 2646 if (sysctl_createv(log, 0, &rnode, &cnode, 2647 CTLFLAG_READONLY, 2648 CTLTYPE_INT, 2649 "rxd_head", SYSCTL_DESCR("Receive Descriptor Head"), 2650 ixv_sysctl_rdh_handler, 0, (void *)rxr, 0, 2651 CTL_CREATE, CTL_EOL) != 0) 2652 break; 2653 2654 if (sysctl_createv(log, 0, &rnode, &cnode, 2655 CTLFLAG_READONLY, 2656 CTLTYPE_INT, 2657 "rxd_tail", SYSCTL_DESCR("Receive Descriptor Tail"), 2658 ixv_sysctl_rdt_handler, 0, (void *)rxr, 0, 2659 CTL_CREATE, CTL_EOL) != 0) 2660 break; 2661 2662 evcnt_attach_dynamic(&rxr->rx_packets, EVCNT_TYPE_MISC, 2663 NULL, adapter->queues[i].evnamebuf, "Queue Packets Received"); 2664 evcnt_attach_dynamic(&rxr->rx_bytes, EVCNT_TYPE_MISC, 2665 NULL, adapter->queues[i].evnamebuf, "Queue Bytes Received"); 2666 evcnt_attach_dynamic(&rxr->rx_copies, EVCNT_TYPE_MISC, 2667 NULL, adapter->queues[i].evnamebuf, "Copied RX Frames"); 2668 evcnt_attach_dynamic(&rxr->no_jmbuf, EVCNT_TYPE_MISC, 2669 NULL, adapter->queues[i].evnamebuf, "Rx no jumbo mbuf"); 2670 evcnt_attach_dynamic(&rxr->rx_discarded, EVCNT_TYPE_MISC, 2671 NULL, adapter->queues[i].evnamebuf, "Rx discarded"); 2672 #ifdef LRO 2673 SYSCTL_ADD_INT(ctx, queue_list, OID_AUTO, "lro_queued", 2674 CTLFLAG_RD, &lro->lro_queued, 0, 2675 "LRO Queued"); 2676 SYSCTL_ADD_INT(ctx, queue_list, OID_AUTO, "lro_flushed", 2677 CTLFLAG_RD, &lro->lro_flushed, 0, 2678 "LRO Flushed"); 2679 #endif /* LRO */ 2680 } 2681 2682 /* MAC stats get their own sub node */ 2683 2684 snprintf(stats->namebuf, 2685 sizeof(stats->namebuf), "%s MAC Statistics", xname); 2686 2687 evcnt_attach_dynamic(&stats->ipcs, EVCNT_TYPE_MISC, NULL, 2688 stats->namebuf, "rx csum offload - IP"); 2689 evcnt_attach_dynamic(&stats->l4cs, EVCNT_TYPE_MISC, NULL, 2690 stats->namebuf, "rx csum offload - L4"); 2691 evcnt_attach_dynamic(&stats->ipcs_bad, EVCNT_TYPE_MISC, NULL, 2692 stats->namebuf, "rx csum offload - IP bad"); 2693 evcnt_attach_dynamic(&stats->l4cs_bad, EVCNT_TYPE_MISC, NULL, 2694 stats->namebuf, "rx csum offload - L4 bad"); 2695 2696 /* Packet Reception Stats */ 2697 evcnt_attach_dynamic(&stats->vfgprc, EVCNT_TYPE_MISC, NULL, 2698 xname, "Good Packets Received"); 2699 evcnt_attach_dynamic(&stats->vfgorc, EVCNT_TYPE_MISC, NULL, 2700 xname, "Good Octets Received"); 2701 evcnt_attach_dynamic(&stats->vfmprc, EVCNT_TYPE_MISC, NULL, 2702 xname, "Multicast Packets Received"); 2703 evcnt_attach_dynamic(&stats->vfgptc, EVCNT_TYPE_MISC, NULL, 2704 xname, "Good Packets Transmitted"); 2705 evcnt_attach_dynamic(&stats->vfgotc, EVCNT_TYPE_MISC, NULL, 2706 xname, "Good Octets Transmitted"); 2707 2708 /* Mailbox Stats */ 2709 evcnt_attach_dynamic(&hw->mbx.stats.msgs_tx, EVCNT_TYPE_MISC, NULL, 2710 xname, "message TXs"); 2711 evcnt_attach_dynamic(&hw->mbx.stats.msgs_rx, EVCNT_TYPE_MISC, NULL, 2712 xname, "message RXs"); 2713 evcnt_attach_dynamic(&hw->mbx.stats.acks, EVCNT_TYPE_MISC, NULL, 2714 xname, "ACKs"); 2715 evcnt_attach_dynamic(&hw->mbx.stats.reqs, EVCNT_TYPE_MISC, NULL, 2716 xname, "REQs"); 2717 evcnt_attach_dynamic(&hw->mbx.stats.rsts, EVCNT_TYPE_MISC, NULL, 2718 xname, "RSTs"); 2719 2720 } /* ixv_add_stats_sysctls */ 2721 2722 static void 2723 ixv_clear_evcnt(struct adapter *adapter) 2724 { 2725 struct tx_ring *txr = adapter->tx_rings; 2726 struct rx_ring *rxr = adapter->rx_rings; 2727 struct ixgbevf_hw_stats *stats = &adapter->stats.vf; 2728 struct ixgbe_hw *hw = &adapter->hw; 2729 int i; 2730 2731 /* Driver Statistics */ 2732 adapter->efbig_tx_dma_setup.ev_count = 0; 2733 adapter->mbuf_defrag_failed.ev_count = 0; 2734 adapter->efbig2_tx_dma_setup.ev_count = 0; 2735 adapter->einval_tx_dma_setup.ev_count = 0; 2736 adapter->other_tx_dma_setup.ev_count = 0; 2737 adapter->eagain_tx_dma_setup.ev_count = 0; 2738 adapter->enomem_tx_dma_setup.ev_count = 0; 2739 adapter->watchdog_events.ev_count = 0; 2740 adapter->tso_err.ev_count = 0; 2741 adapter->link_irq.ev_count = 0; 2742 2743 for (i = 0; i < adapter->num_queues; i++, rxr++, txr++) { 2744 adapter->queues[i].irqs.ev_count = 0; 2745 adapter->queues[i].handleq.ev_count = 0; 2746 adapter->queues[i].req.ev_count = 0; 2747 txr->tso_tx.ev_count = 0; 2748 txr->no_desc_avail.ev_count = 0; 2749 txr->total_packets.ev_count = 0; 2750 #ifndef IXGBE_LEGACY_TX 2751 txr->pcq_drops.ev_count = 0; 2752 #endif 2753 txr->q_efbig_tx_dma_setup = 0; 2754 txr->q_mbuf_defrag_failed = 0; 2755 txr->q_efbig2_tx_dma_setup = 0; 2756 txr->q_einval_tx_dma_setup = 0; 2757 txr->q_other_tx_dma_setup = 0; 2758 txr->q_eagain_tx_dma_setup = 0; 2759 txr->q_enomem_tx_dma_setup = 0; 2760 txr->q_tso_err = 0; 2761 2762 rxr->rx_packets.ev_count = 0; 2763 rxr->rx_bytes.ev_count = 0; 2764 rxr->rx_copies.ev_count = 0; 2765 rxr->no_jmbuf.ev_count = 0; 2766 rxr->rx_discarded.ev_count = 0; 2767 } 2768 2769 /* MAC stats get their own sub node */ 2770 2771 stats->ipcs.ev_count = 0; 2772 stats->l4cs.ev_count = 0; 2773 stats->ipcs_bad.ev_count = 0; 2774 stats->l4cs_bad.ev_count = 0; 2775 2776 /* Packet Reception Stats */ 2777 stats->vfgprc.ev_count = 0; 2778 stats->vfgorc.ev_count = 0; 2779 stats->vfmprc.ev_count = 0; 2780 stats->vfgptc.ev_count = 0; 2781 stats->vfgotc.ev_count = 0; 2782 2783 /* Mailbox Stats */ 2784 hw->mbx.stats.msgs_tx.ev_count = 0; 2785 hw->mbx.stats.msgs_rx.ev_count = 0; 2786 hw->mbx.stats.acks.ev_count = 0; 2787 hw->mbx.stats.reqs.ev_count = 0; 2788 hw->mbx.stats.rsts.ev_count = 0; 2789 2790 } /* ixv_clear_evcnt */ 2791 2792 /************************************************************************ 2793 * ixv_set_sysctl_value 2794 ************************************************************************/ 2795 static void 2796 ixv_set_sysctl_value(struct adapter *adapter, const char *name, 2797 const char *description, int *limit, int value) 2798 { 2799 device_t dev = adapter->dev; 2800 struct sysctllog **log; 2801 const struct sysctlnode *rnode, *cnode; 2802 2803 log = &adapter->sysctllog; 2804 if ((rnode = ixv_sysctl_instance(adapter)) == NULL) { 2805 aprint_error_dev(dev, "could not create sysctl root\n"); 2806 return; 2807 } 2808 if (sysctl_createv(log, 0, &rnode, &cnode, 2809 CTLFLAG_READWRITE, CTLTYPE_INT, 2810 name, SYSCTL_DESCR(description), 2811 NULL, 0, limit, 0, CTL_CREATE, CTL_EOL) != 0) 2812 aprint_error_dev(dev, "could not create sysctl\n"); 2813 *limit = value; 2814 } /* ixv_set_sysctl_value */ 2815 2816 /************************************************************************ 2817 * ixv_print_debug_info 2818 * 2819 * Called only when em_display_debug_stats is enabled. 2820 * Provides a way to take a look at important statistics 2821 * maintained by the driver and hardware. 2822 ************************************************************************/ 2823 static void 2824 ixv_print_debug_info(struct adapter *adapter) 2825 { 2826 device_t dev = adapter->dev; 2827 struct ix_queue *que = adapter->queues; 2828 struct rx_ring *rxr; 2829 struct tx_ring *txr; 2830 #ifdef LRO 2831 struct lro_ctrl *lro; 2832 #endif /* LRO */ 2833 2834 for (int i = 0; i < adapter->num_queues; i++, que++) { 2835 txr = que->txr; 2836 rxr = que->rxr; 2837 #ifdef LRO 2838 lro = &rxr->lro; 2839 #endif /* LRO */ 2840 device_printf(dev, "QUE(%d) IRQs Handled: %lu\n", 2841 que->msix, (long)que->irqs.ev_count); 2842 device_printf(dev, "RX(%d) Packets Received: %lld\n", 2843 rxr->me, (long long)rxr->rx_packets.ev_count); 2844 device_printf(dev, "RX(%d) Bytes Received: %lu\n", 2845 rxr->me, (long)rxr->rx_bytes.ev_count); 2846 #ifdef LRO 2847 device_printf(dev, "RX(%d) LRO Queued= %ju\n", 2848 rxr->me, (uintmax_t)lro->lro_queued); 2849 device_printf(dev, "RX(%d) LRO Flushed= %ju\n", 2850 rxr->me, (uintmax_t)lro->lro_flushed); 2851 #endif /* LRO */ 2852 device_printf(dev, "TX(%d) Packets Sent: %lu\n", 2853 txr->me, (long)txr->total_packets.ev_count); 2854 device_printf(dev, "TX(%d) NO Desc Avail: %lu\n", 2855 txr->me, (long)txr->no_desc_avail.ev_count); 2856 } 2857 2858 device_printf(dev, "MBX IRQ Handled: %lu\n", 2859 (long)adapter->link_irq.ev_count); 2860 } /* ixv_print_debug_info */ 2861 2862 /************************************************************************ 2863 * ixv_sysctl_debug 2864 ************************************************************************/ 2865 static int 2866 ixv_sysctl_debug(SYSCTLFN_ARGS) 2867 { 2868 struct sysctlnode node = *rnode; 2869 struct adapter *adapter = (struct adapter *)node.sysctl_data; 2870 int error, result; 2871 2872 node.sysctl_data = &result; 2873 error = sysctl_lookup(SYSCTLFN_CALL(&node)); 2874 2875 if (error || newp == NULL) 2876 return error; 2877 2878 if (result == 1) 2879 ixv_print_debug_info(adapter); 2880 2881 return 0; 2882 } /* ixv_sysctl_debug */ 2883 2884 /************************************************************************ 2885 * ixv_init_device_features 2886 ************************************************************************/ 2887 static void 2888 ixv_init_device_features(struct adapter *adapter) 2889 { 2890 adapter->feat_cap = IXGBE_FEATURE_NETMAP 2891 | IXGBE_FEATURE_VF 2892 | IXGBE_FEATURE_RSS 2893 | IXGBE_FEATURE_LEGACY_TX; 2894 2895 /* A tad short on feature flags for VFs, atm. */ 2896 switch (adapter->hw.mac.type) { 2897 case ixgbe_mac_82599_vf: 2898 break; 2899 case ixgbe_mac_X540_vf: 2900 break; 2901 case ixgbe_mac_X550_vf: 2902 case ixgbe_mac_X550EM_x_vf: 2903 case ixgbe_mac_X550EM_a_vf: 2904 adapter->feat_cap |= IXGBE_FEATURE_NEEDS_CTXD; 2905 break; 2906 default: 2907 break; 2908 } 2909 2910 /* Enabled by default... */ 2911 /* Is a virtual function (VF) */ 2912 if (adapter->feat_cap & IXGBE_FEATURE_VF) 2913 adapter->feat_en |= IXGBE_FEATURE_VF; 2914 /* Netmap */ 2915 if (adapter->feat_cap & IXGBE_FEATURE_NETMAP) 2916 adapter->feat_en |= IXGBE_FEATURE_NETMAP; 2917 /* Receive-Side Scaling (RSS) */ 2918 if (adapter->feat_cap & IXGBE_FEATURE_RSS) 2919 adapter->feat_en |= IXGBE_FEATURE_RSS; 2920 /* Needs advanced context descriptor regardless of offloads req'd */ 2921 if (adapter->feat_cap & IXGBE_FEATURE_NEEDS_CTXD) 2922 adapter->feat_en |= IXGBE_FEATURE_NEEDS_CTXD; 2923 2924 /* Enabled via sysctl... */ 2925 /* Legacy (single queue) transmit */ 2926 if ((adapter->feat_cap & IXGBE_FEATURE_LEGACY_TX) && 2927 ixv_enable_legacy_tx) 2928 adapter->feat_en |= IXGBE_FEATURE_LEGACY_TX; 2929 } /* ixv_init_device_features */ 2930 2931 /************************************************************************ 2932 * ixv_shutdown - Shutdown entry point 2933 ************************************************************************/ 2934 #if 0 /* XXX NetBSD ought to register something like this through pmf(9) */ 2935 static int 2936 ixv_shutdown(device_t dev) 2937 { 2938 struct adapter *adapter = device_private(dev); 2939 IXGBE_CORE_LOCK(adapter); 2940 ixv_stop(adapter); 2941 IXGBE_CORE_UNLOCK(adapter); 2942 2943 return (0); 2944 } /* ixv_shutdown */ 2945 #endif 2946 2947 static int 2948 ixv_ifflags_cb(struct ethercom *ec) 2949 { 2950 struct ifnet *ifp = &ec->ec_if; 2951 struct adapter *adapter = ifp->if_softc; 2952 u_short saved_flags; 2953 u_short change; 2954 int rv = 0; 2955 2956 IXGBE_CORE_LOCK(adapter); 2957 2958 saved_flags = adapter->if_flags; 2959 change = ifp->if_flags ^ adapter->if_flags; 2960 if (change != 0) 2961 adapter->if_flags = ifp->if_flags; 2962 2963 if ((change & ~(IFF_CANTCHANGE | IFF_DEBUG)) != 0) { 2964 rv = ENETRESET; 2965 goto out; 2966 } else if ((change & IFF_PROMISC) != 0) { 2967 rv = ixv_set_rxfilter(adapter); 2968 if (rv != 0) { 2969 /* Restore previous */ 2970 adapter->if_flags = saved_flags; 2971 goto out; 2972 } 2973 } 2974 2975 /* Check for ec_capenable. */ 2976 change = ec->ec_capenable ^ adapter->ec_capenable; 2977 adapter->ec_capenable = ec->ec_capenable; 2978 if ((change & ~(ETHERCAP_VLAN_MTU | ETHERCAP_VLAN_HWTAGGING 2979 | ETHERCAP_VLAN_HWFILTER)) != 0) { 2980 rv = ENETRESET; 2981 goto out; 2982 } 2983 2984 /* 2985 * Special handling is not required for ETHERCAP_VLAN_MTU. 2986 * PF's MAXFRS(MHADD) does not include the 4bytes of the VLAN header. 2987 */ 2988 2989 /* Set up VLAN support and filter */ 2990 if ((change & (ETHERCAP_VLAN_HWTAGGING | ETHERCAP_VLAN_HWFILTER)) != 0) 2991 rv = ixv_setup_vlan_support(adapter); 2992 2993 out: 2994 IXGBE_CORE_UNLOCK(adapter); 2995 2996 return rv; 2997 } 2998 2999 3000 /************************************************************************ 3001 * ixv_ioctl - Ioctl entry point 3002 * 3003 * Called when the user wants to configure the interface. 3004 * 3005 * return 0 on success, positive on failure 3006 ************************************************************************/ 3007 static int 3008 ixv_ioctl(struct ifnet *ifp, u_long command, void *data) 3009 { 3010 struct adapter *adapter = ifp->if_softc; 3011 struct ixgbe_hw *hw = &adapter->hw; 3012 struct ifcapreq *ifcr = data; 3013 int error; 3014 int l4csum_en; 3015 const int l4csum = IFCAP_CSUM_TCPv4_Rx | IFCAP_CSUM_UDPv4_Rx | 3016 IFCAP_CSUM_TCPv6_Rx | IFCAP_CSUM_UDPv6_Rx; 3017 3018 switch (command) { 3019 case SIOCSIFFLAGS: 3020 IOCTL_DEBUGOUT("ioctl: SIOCSIFFLAGS (Set Interface Flags)"); 3021 break; 3022 case SIOCADDMULTI: { 3023 struct ether_multi *enm; 3024 struct ether_multistep step; 3025 struct ethercom *ec = &adapter->osdep.ec; 3026 bool overflow = false; 3027 int mcnt = 0; 3028 3029 /* 3030 * Check the number of multicast address. If it exceeds, 3031 * return ENOSPC. 3032 * Update this code when we support API 1.3. 3033 */ 3034 ETHER_LOCK(ec); 3035 ETHER_FIRST_MULTI(step, ec, enm); 3036 while (enm != NULL) { 3037 mcnt++; 3038 3039 /* 3040 * This code is before adding, so one room is required 3041 * at least. 3042 */ 3043 if (mcnt > (IXGBE_MAX_VF_MC - 1)) { 3044 overflow = true; 3045 break; 3046 } 3047 ETHER_NEXT_MULTI(step, enm); 3048 } 3049 ETHER_UNLOCK(ec); 3050 error = 0; 3051 if (overflow && ((ec->ec_flags & ETHER_F_ALLMULTI) == 0)) { 3052 error = hw->mac.ops.update_xcast_mode(hw, 3053 IXGBEVF_XCAST_MODE_ALLMULTI); 3054 if (error == IXGBE_ERR_NOT_TRUSTED) { 3055 device_printf(adapter->dev, 3056 "this interface is not trusted\n"); 3057 error = EPERM; 3058 } else if (error == IXGBE_ERR_FEATURE_NOT_SUPPORTED) { 3059 device_printf(adapter->dev, 3060 "the PF doesn't support allmulti mode\n"); 3061 error = EOPNOTSUPP; 3062 } else if (error) { 3063 device_printf(adapter->dev, 3064 "number of Ethernet multicast addresses " 3065 "exceeds the limit (%d). error = %d\n", 3066 IXGBE_MAX_VF_MC, error); 3067 error = ENOSPC; 3068 } else 3069 ec->ec_flags |= ETHER_F_ALLMULTI; 3070 } 3071 if (error) 3072 return error; 3073 } 3074 /*FALLTHROUGH*/ 3075 case SIOCDELMULTI: 3076 IOCTL_DEBUGOUT("ioctl: SIOC(ADD|DEL)MULTI"); 3077 break; 3078 case SIOCSIFMEDIA: 3079 case SIOCGIFMEDIA: 3080 IOCTL_DEBUGOUT("ioctl: SIOCxIFMEDIA (Get/Set Interface Media)"); 3081 break; 3082 case SIOCSIFCAP: 3083 IOCTL_DEBUGOUT("ioctl: SIOCSIFCAP (Set Capabilities)"); 3084 break; 3085 case SIOCSIFMTU: 3086 IOCTL_DEBUGOUT("ioctl: SIOCSIFMTU (Set Interface MTU)"); 3087 break; 3088 case SIOCZIFDATA: 3089 IOCTL_DEBUGOUT("ioctl: SIOCZIFDATA (Zero counter)"); 3090 ixv_update_stats(adapter); 3091 ixv_clear_evcnt(adapter); 3092 break; 3093 default: 3094 IOCTL_DEBUGOUT1("ioctl: UNKNOWN (0x%X)", (int)command); 3095 break; 3096 } 3097 3098 switch (command) { 3099 case SIOCSIFCAP: 3100 /* Layer-4 Rx checksum offload has to be turned on and 3101 * off as a unit. 3102 */ 3103 l4csum_en = ifcr->ifcr_capenable & l4csum; 3104 if (l4csum_en != l4csum && l4csum_en != 0) 3105 return EINVAL; 3106 /*FALLTHROUGH*/ 3107 case SIOCADDMULTI: 3108 case SIOCDELMULTI: 3109 case SIOCSIFFLAGS: 3110 case SIOCSIFMTU: 3111 default: 3112 if ((error = ether_ioctl(ifp, command, data)) != ENETRESET) 3113 return error; 3114 if ((ifp->if_flags & IFF_RUNNING) == 0) 3115 ; 3116 else if (command == SIOCSIFCAP || command == SIOCSIFMTU) { 3117 IXGBE_CORE_LOCK(adapter); 3118 ixv_init_locked(adapter); 3119 IXGBE_CORE_UNLOCK(adapter); 3120 } else if (command == SIOCADDMULTI || command == SIOCDELMULTI) { 3121 /* 3122 * Multicast list has changed; set the hardware filter 3123 * accordingly. 3124 */ 3125 IXGBE_CORE_LOCK(adapter); 3126 ixv_disable_intr(adapter); 3127 ixv_set_rxfilter(adapter); 3128 ixv_enable_intr(adapter); 3129 IXGBE_CORE_UNLOCK(adapter); 3130 } 3131 return 0; 3132 } 3133 } /* ixv_ioctl */ 3134 3135 /************************************************************************ 3136 * ixv_init 3137 ************************************************************************/ 3138 static int 3139 ixv_init(struct ifnet *ifp) 3140 { 3141 struct adapter *adapter = ifp->if_softc; 3142 3143 IXGBE_CORE_LOCK(adapter); 3144 ixv_init_locked(adapter); 3145 IXGBE_CORE_UNLOCK(adapter); 3146 3147 return 0; 3148 } /* ixv_init */ 3149 3150 /************************************************************************ 3151 * ixv_handle_que 3152 ************************************************************************/ 3153 static void 3154 ixv_handle_que(void *context) 3155 { 3156 struct ix_queue *que = context; 3157 struct adapter *adapter = que->adapter; 3158 struct tx_ring *txr = que->txr; 3159 struct ifnet *ifp = adapter->ifp; 3160 bool more; 3161 3162 que->handleq.ev_count++; 3163 3164 if (ifp->if_flags & IFF_RUNNING) { 3165 more = ixgbe_rxeof(que); 3166 IXGBE_TX_LOCK(txr); 3167 more |= ixgbe_txeof(txr); 3168 if (!(adapter->feat_en & IXGBE_FEATURE_LEGACY_TX)) 3169 if (!ixgbe_mq_ring_empty(ifp, txr->txr_interq)) 3170 ixgbe_mq_start_locked(ifp, txr); 3171 /* Only for queue 0 */ 3172 /* NetBSD still needs this for CBQ */ 3173 if ((&adapter->queues[0] == que) 3174 && (!ixgbe_legacy_ring_empty(ifp, NULL))) 3175 ixgbe_legacy_start_locked(ifp, txr); 3176 IXGBE_TX_UNLOCK(txr); 3177 if (more) { 3178 que->req.ev_count++; 3179 if (adapter->txrx_use_workqueue) { 3180 /* 3181 * "enqueued flag" is not required here 3182 * the same as ixg(4). See ixgbe_msix_que(). 3183 */ 3184 workqueue_enqueue(adapter->que_wq, 3185 &que->wq_cookie, curcpu()); 3186 } else 3187 softint_schedule(que->que_si); 3188 return; 3189 } 3190 } 3191 3192 /* Re-enable this interrupt */ 3193 ixv_enable_queue(adapter, que->msix); 3194 3195 return; 3196 } /* ixv_handle_que */ 3197 3198 /************************************************************************ 3199 * ixv_handle_que_work 3200 ************************************************************************/ 3201 static void 3202 ixv_handle_que_work(struct work *wk, void *context) 3203 { 3204 struct ix_queue *que = container_of(wk, struct ix_queue, wq_cookie); 3205 3206 /* 3207 * "enqueued flag" is not required here the same as ixg(4). 3208 * See ixgbe_msix_que(). 3209 */ 3210 ixv_handle_que(que); 3211 } 3212 3213 /************************************************************************ 3214 * ixv_allocate_msix - Setup MSI-X Interrupt resources and handlers 3215 ************************************************************************/ 3216 static int 3217 ixv_allocate_msix(struct adapter *adapter, const struct pci_attach_args *pa) 3218 { 3219 device_t dev = adapter->dev; 3220 struct ix_queue *que = adapter->queues; 3221 struct tx_ring *txr = adapter->tx_rings; 3222 int error, msix_ctrl, rid, vector = 0; 3223 pci_chipset_tag_t pc; 3224 pcitag_t tag; 3225 char intrbuf[PCI_INTRSTR_LEN]; 3226 char wqname[MAXCOMLEN]; 3227 char intr_xname[32]; 3228 const char *intrstr = NULL; 3229 kcpuset_t *affinity; 3230 int cpu_id = 0; 3231 3232 pc = adapter->osdep.pc; 3233 tag = adapter->osdep.tag; 3234 3235 adapter->osdep.nintrs = adapter->num_queues + 1; 3236 if (pci_msix_alloc_exact(pa, &adapter->osdep.intrs, 3237 adapter->osdep.nintrs) != 0) { 3238 aprint_error_dev(dev, 3239 "failed to allocate MSI-X interrupt\n"); 3240 return (ENXIO); 3241 } 3242 3243 kcpuset_create(&affinity, false); 3244 for (int i = 0; i < adapter->num_queues; i++, vector++, que++, txr++) { 3245 snprintf(intr_xname, sizeof(intr_xname), "%s TXRX%d", 3246 device_xname(dev), i); 3247 intrstr = pci_intr_string(pc, adapter->osdep.intrs[i], intrbuf, 3248 sizeof(intrbuf)); 3249 #ifdef IXGBE_MPSAFE 3250 pci_intr_setattr(pc, &adapter->osdep.intrs[i], PCI_INTR_MPSAFE, 3251 true); 3252 #endif 3253 /* Set the handler function */ 3254 que->res = adapter->osdep.ihs[i] = pci_intr_establish_xname(pc, 3255 adapter->osdep.intrs[i], IPL_NET, ixv_msix_que, que, 3256 intr_xname); 3257 if (que->res == NULL) { 3258 pci_intr_release(pc, adapter->osdep.intrs, 3259 adapter->osdep.nintrs); 3260 aprint_error_dev(dev, 3261 "Failed to register QUE handler\n"); 3262 kcpuset_destroy(affinity); 3263 return (ENXIO); 3264 } 3265 que->msix = vector; 3266 adapter->active_queues |= (u64)(1 << que->msix); 3267 3268 cpu_id = i; 3269 /* Round-robin affinity */ 3270 kcpuset_zero(affinity); 3271 kcpuset_set(affinity, cpu_id % ncpu); 3272 error = interrupt_distribute(adapter->osdep.ihs[i], affinity, 3273 NULL); 3274 aprint_normal_dev(dev, "for TX/RX, interrupting at %s", 3275 intrstr); 3276 if (error == 0) 3277 aprint_normal(", bound queue %d to cpu %d\n", 3278 i, cpu_id % ncpu); 3279 else 3280 aprint_normal("\n"); 3281 3282 #ifndef IXGBE_LEGACY_TX 3283 txr->txr_si 3284 = softint_establish(SOFTINT_NET | IXGBE_SOFTINFT_FLAGS, 3285 ixgbe_deferred_mq_start, txr); 3286 #endif 3287 que->que_si 3288 = softint_establish(SOFTINT_NET | IXGBE_SOFTINFT_FLAGS, 3289 ixv_handle_que, que); 3290 if (que->que_si == NULL) { 3291 aprint_error_dev(dev, 3292 "could not establish software interrupt\n"); 3293 } 3294 } 3295 snprintf(wqname, sizeof(wqname), "%sdeferTx", device_xname(dev)); 3296 error = workqueue_create(&adapter->txr_wq, wqname, 3297 ixgbe_deferred_mq_start_work, adapter, IXGBE_WORKQUEUE_PRI, IPL_NET, 3298 IXGBE_WORKQUEUE_FLAGS); 3299 if (error) { 3300 aprint_error_dev(dev, "couldn't create workqueue for deferred Tx\n"); 3301 } 3302 adapter->txr_wq_enqueued = percpu_alloc(sizeof(u_int)); 3303 3304 snprintf(wqname, sizeof(wqname), "%sTxRx", device_xname(dev)); 3305 error = workqueue_create(&adapter->que_wq, wqname, 3306 ixv_handle_que_work, adapter, IXGBE_WORKQUEUE_PRI, IPL_NET, 3307 IXGBE_WORKQUEUE_FLAGS); 3308 if (error) { 3309 aprint_error_dev(dev, 3310 "couldn't create workqueue\n"); 3311 } 3312 3313 /* and Mailbox */ 3314 cpu_id++; 3315 snprintf(intr_xname, sizeof(intr_xname), "%s link", device_xname(dev)); 3316 adapter->vector = vector; 3317 intrstr = pci_intr_string(pc, adapter->osdep.intrs[vector], intrbuf, 3318 sizeof(intrbuf)); 3319 #ifdef IXGBE_MPSAFE 3320 pci_intr_setattr(pc, &adapter->osdep.intrs[vector], PCI_INTR_MPSAFE, 3321 true); 3322 #endif 3323 /* Set the mbx handler function */ 3324 adapter->osdep.ihs[vector] = pci_intr_establish_xname(pc, 3325 adapter->osdep.intrs[vector], IPL_NET, ixv_msix_mbx, adapter, 3326 intr_xname); 3327 if (adapter->osdep.ihs[vector] == NULL) { 3328 aprint_error_dev(dev, "Failed to register LINK handler\n"); 3329 kcpuset_destroy(affinity); 3330 return (ENXIO); 3331 } 3332 /* Round-robin affinity */ 3333 kcpuset_zero(affinity); 3334 kcpuset_set(affinity, cpu_id % ncpu); 3335 error = interrupt_distribute(adapter->osdep.ihs[vector], 3336 affinity, NULL); 3337 3338 aprint_normal_dev(dev, 3339 "for link, interrupting at %s", intrstr); 3340 if (error == 0) 3341 aprint_normal(", affinity to cpu %d\n", cpu_id % ncpu); 3342 else 3343 aprint_normal("\n"); 3344 3345 /* Tasklets for Mailbox */ 3346 adapter->link_si = softint_establish(SOFTINT_NET |IXGBE_SOFTINFT_FLAGS, 3347 ixv_handle_link, adapter); 3348 /* 3349 * Due to a broken design QEMU will fail to properly 3350 * enable the guest for MSI-X unless the vectors in 3351 * the table are all set up, so we must rewrite the 3352 * ENABLE in the MSI-X control register again at this 3353 * point to cause it to successfully initialize us. 3354 */ 3355 if (adapter->hw.mac.type == ixgbe_mac_82599_vf) { 3356 pci_get_capability(pc, tag, PCI_CAP_MSIX, &rid, NULL); 3357 rid += PCI_MSIX_CTL; 3358 msix_ctrl = pci_conf_read(pc, tag, rid); 3359 msix_ctrl |= PCI_MSIX_CTL_ENABLE; 3360 pci_conf_write(pc, tag, rid, msix_ctrl); 3361 } 3362 3363 kcpuset_destroy(affinity); 3364 return (0); 3365 } /* ixv_allocate_msix */ 3366 3367 /************************************************************************ 3368 * ixv_configure_interrupts - Setup MSI-X resources 3369 * 3370 * Note: The VF device MUST use MSI-X, there is no fallback. 3371 ************************************************************************/ 3372 static int 3373 ixv_configure_interrupts(struct adapter *adapter) 3374 { 3375 device_t dev = adapter->dev; 3376 int want, queues, msgs; 3377 3378 /* Must have at least 2 MSI-X vectors */ 3379 msgs = pci_msix_count(adapter->osdep.pc, adapter->osdep.tag); 3380 if (msgs < 2) { 3381 aprint_error_dev(dev, "MSIX config error\n"); 3382 return (ENXIO); 3383 } 3384 msgs = MIN(msgs, IXG_MAX_NINTR); 3385 3386 /* Figure out a reasonable auto config value */ 3387 queues = (ncpu > (msgs - 1)) ? (msgs - 1) : ncpu; 3388 3389 if (ixv_num_queues != 0) 3390 queues = ixv_num_queues; 3391 else if ((ixv_num_queues == 0) && (queues > IXGBE_VF_MAX_TX_QUEUES)) 3392 queues = IXGBE_VF_MAX_TX_QUEUES; 3393 3394 /* 3395 * Want vectors for the queues, 3396 * plus an additional for mailbox. 3397 */ 3398 want = queues + 1; 3399 if (msgs >= want) 3400 msgs = want; 3401 else { 3402 aprint_error_dev(dev, 3403 "MSI-X Configuration Problem, " 3404 "%d vectors but %d queues wanted!\n", 3405 msgs, want); 3406 return -1; 3407 } 3408 3409 adapter->msix_mem = (void *)1; /* XXX */ 3410 aprint_normal_dev(dev, 3411 "Using MSI-X interrupts with %d vectors\n", msgs); 3412 adapter->num_queues = queues; 3413 3414 return (0); 3415 } /* ixv_configure_interrupts */ 3416 3417 3418 /************************************************************************ 3419 * ixv_handle_link - Tasklet handler for MSI-X MBX interrupts 3420 * 3421 * Done outside of interrupt context since the driver might sleep 3422 ************************************************************************/ 3423 static void 3424 ixv_handle_link(void *context) 3425 { 3426 struct adapter *adapter = context; 3427 3428 IXGBE_CORE_LOCK(adapter); 3429 3430 adapter->hw.mac.ops.check_link(&adapter->hw, &adapter->link_speed, 3431 &adapter->link_up, FALSE); 3432 ixv_update_link_status(adapter); 3433 3434 IXGBE_CORE_UNLOCK(adapter); 3435 } /* ixv_handle_link */ 3436 3437 /************************************************************************ 3438 * ixv_check_link - Used in the local timer to poll for link changes 3439 ************************************************************************/ 3440 static s32 3441 ixv_check_link(struct adapter *adapter) 3442 { 3443 s32 error; 3444 3445 KASSERT(mutex_owned(&adapter->core_mtx)); 3446 3447 adapter->hw.mac.get_link_status = TRUE; 3448 3449 error = adapter->hw.mac.ops.check_link(&adapter->hw, 3450 &adapter->link_speed, &adapter->link_up, FALSE); 3451 ixv_update_link_status(adapter); 3452 3453 return error; 3454 } /* ixv_check_link */ 3455
Indexes created Tue Sep 30 20:09:53 GMT 2025