ixv.c revision 1.149
1 /*$NetBSD: ixv.c,v 1.149 2020/06/11 09:16:05 msaitoh Exp $*/ 2 3 /****************************************************************************** 4 5 Copyright (c) 2001-2017, Intel Corporation 6 All rights reserved. 7 8 Redistribution and use in source and binary forms, with or without 9 modification, are permitted provided that the following conditions are met: 10 11 1. Redistributions of source code must retain the above copyright notice, 12 this list of conditions and the following disclaimer. 13 14 2. Redistributions in binary form must reproduce the above copyright 15 notice, this list of conditions and the following disclaimer in the 16 documentation and/or other materials provided with the distribution. 17 18 3. Neither the name of the Intel Corporation nor the names of its 19 contributors may be used to endorse or promote products derived from 20 this software without specific prior written permission. 21 22 THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 23 AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 24 IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 25 ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE 26 LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 27 CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 28 SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 29 INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 30 CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 31 ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 32 POSSIBILITY OF SUCH DAMAGE. 33 34 ******************************************************************************/ 35 /*$FreeBSD: head/sys/dev/ixgbe/if_ixv.c 331224 2018-03-19 20:55:05Z erj $*/ 36 37 #ifdef _KERNEL_OPT 38 #include "opt_inet.h" 39 #include "opt_inet6.h" 40 #include "opt_net_mpsafe.h" 41 #endif 42 43 #include "ixgbe.h" 44 #include "vlan.h" 45 46 /************************************************************************ 47 * Driver version 48 ************************************************************************/ 49 static const char ixv_driver_version[] = "2.0.1-k"; 50 /* XXX NetBSD: + 1.5.17 */ 51 52 /************************************************************************ 53 * PCI Device ID Table 54 * 55 * Used by probe to select devices to load on 56 * Last field stores an index into ixv_strings 57 * Last entry must be all 0s 58 * 59 * { Vendor ID, Device ID, SubVendor ID, SubDevice ID, String Index } 60 ************************************************************************/ 61 static const ixgbe_vendor_info_t ixv_vendor_info_array[] = 62 { 63 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_VF, 0, 0, 0}, 64 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_X540_VF, 0, 0, 0}, 65 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_X550_VF, 0, 0, 0}, 66 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_X550EM_X_VF, 0, 0, 0}, 67 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_X550EM_A_VF, 0, 0, 0}, 68 /* required last entry */ 69 {0, 0, 0, 0, 0} 70 }; 71 72 /************************************************************************ 73 * Table of branding strings 74 ************************************************************************/ 75 static const char *ixv_strings[] = { 76 "Intel(R) PRO/10GbE Virtual Function Network Driver" 77 }; 78 79 /********************************************************************* 80 * Function prototypes 81 *********************************************************************/ 82 static int ixv_probe(device_t, cfdata_t, void *); 83 static void ixv_attach(device_t, device_t, void *); 84 static int ixv_detach(device_t, int); 85 #if 0 86 static int ixv_shutdown(device_t); 87 #endif 88 static int ixv_ifflags_cb(struct ethercom *); 89 static int ixv_ioctl(struct ifnet *, u_long, void *); 90 static int ixv_init(struct ifnet *); 91 static void ixv_init_locked(struct adapter *); 92 static void ixv_ifstop(struct ifnet *, int); 93 static void ixv_stop(void *); 94 static void ixv_init_device_features(struct adapter *); 95 static void ixv_media_status(struct ifnet *, struct ifmediareq *); 96 static int ixv_media_change(struct ifnet *); 97 static int ixv_allocate_pci_resources(struct adapter *, 98 const struct pci_attach_args *); 99 static int ixv_allocate_msix(struct adapter *, 100 const struct pci_attach_args *); 101 static int ixv_configure_interrupts(struct adapter *); 102 static void ixv_free_pci_resources(struct adapter *); 103 static void ixv_local_timer(void *); 104 static void ixv_local_timer_locked(void *); 105 static int ixv_setup_interface(device_t, struct adapter *); 106 static int ixv_negotiate_api(struct adapter *); 107 108 static void ixv_initialize_transmit_units(struct adapter *); 109 static void ixv_initialize_receive_units(struct adapter *); 110 static void ixv_initialize_rss_mapping(struct adapter *); 111 static s32 ixv_check_link(struct adapter *); 112 113 static void ixv_enable_intr(struct adapter *); 114 static void ixv_disable_intr(struct adapter *); 115 static int ixv_set_rxfilter(struct adapter *); 116 static void ixv_update_link_status(struct adapter *); 117 static int ixv_sysctl_debug(SYSCTLFN_PROTO); 118 static void ixv_set_ivar(struct adapter *, u8, u8, s8); 119 static void ixv_configure_ivars(struct adapter *); 120 static u8 * ixv_mc_array_itr(struct ixgbe_hw *, u8 **, u32 *); 121 static void ixv_eitr_write(struct adapter *, uint32_t, uint32_t); 122 123 static void ixv_setup_vlan_tagging(struct adapter *); 124 static int ixv_setup_vlan_support(struct adapter *); 125 static int ixv_vlan_cb(struct ethercom *, uint16_t, bool); 126 static int ixv_register_vlan(struct adapter *, u16); 127 static int ixv_unregister_vlan(struct adapter *, u16); 128 129 static void ixv_add_device_sysctls(struct adapter *); 130 static void ixv_save_stats(struct adapter *); 131 static void ixv_init_stats(struct adapter *); 132 static void ixv_update_stats(struct adapter *); 133 static void ixv_add_stats_sysctls(struct adapter *); 134 static void ixv_clear_evcnt(struct adapter *); 135 136 /* Sysctl handlers */ 137 static void ixv_set_sysctl_value(struct adapter *, const char *, 138 const char *, int *, int); 139 static int ixv_sysctl_interrupt_rate_handler(SYSCTLFN_PROTO); 140 static int ixv_sysctl_next_to_check_handler(SYSCTLFN_PROTO); 141 static int ixv_sysctl_rdh_handler(SYSCTLFN_PROTO); 142 static int ixv_sysctl_rdt_handler(SYSCTLFN_PROTO); 143 static int ixv_sysctl_tdt_handler(SYSCTLFN_PROTO); 144 static int ixv_sysctl_tdh_handler(SYSCTLFN_PROTO); 145 146 /* The MSI-X Interrupt handlers */ 147 static int ixv_msix_que(void *); 148 static int ixv_msix_mbx(void *); 149 150 /* Deferred interrupt tasklets */ 151 static void ixv_handle_que(void *); 152 static void ixv_handle_link(void *); 153 154 /* Workqueue handler for deferred work */ 155 static void ixv_handle_que_work(struct work *, void *); 156 157 const struct sysctlnode *ixv_sysctl_instance(struct adapter *); 158 static const ixgbe_vendor_info_t *ixv_lookup(const struct pci_attach_args *); 159 160 /************************************************************************ 161 * FreeBSD Device Interface Entry Points 162 ************************************************************************/ 163 CFATTACH_DECL3_NEW(ixv, sizeof(struct adapter), 164 ixv_probe, ixv_attach, ixv_detach, NULL, NULL, NULL, 165 DVF_DETACH_SHUTDOWN); 166 167 #if 0 168 static driver_t ixv_driver = { 169 "ixv", ixv_methods, sizeof(struct adapter), 170 }; 171 172 devclass_t ixv_devclass; 173 DRIVER_MODULE(ixv, pci, ixv_driver, ixv_devclass, 0, 0); 174 MODULE_DEPEND(ixv, pci, 1, 1, 1); 175 MODULE_DEPEND(ixv, ether, 1, 1, 1); 176 #endif 177 178 /* 179 * TUNEABLE PARAMETERS: 180 */ 181 182 /* Number of Queues - do not exceed MSI-X vectors - 1 */ 183 static int ixv_num_queues = 0; 184 #define TUNABLE_INT(__x, __y) 185 TUNABLE_INT("hw.ixv.num_queues", &ixv_num_queues); 186 187 /* 188 * AIM: Adaptive Interrupt Moderation 189 * which means that the interrupt rate 190 * is varied over time based on the 191 * traffic for that interrupt vector 192 */ 193 static bool ixv_enable_aim = false; 194 TUNABLE_INT("hw.ixv.enable_aim", &ixv_enable_aim); 195 196 static int ixv_max_interrupt_rate = (4000000 / IXGBE_LOW_LATENCY); 197 TUNABLE_INT("hw.ixv.max_interrupt_rate", &ixv_max_interrupt_rate); 198 199 /* How many packets rxeof tries to clean at a time */ 200 static int ixv_rx_process_limit = 256; 201 TUNABLE_INT("hw.ixv.rx_process_limit", &ixv_rx_process_limit); 202 203 /* How many packets txeof tries to clean at a time */ 204 static int ixv_tx_process_limit = 256; 205 TUNABLE_INT("hw.ixv.tx_process_limit", &ixv_tx_process_limit); 206 207 /* Which packet processing uses workqueue or softint */ 208 static bool ixv_txrx_workqueue = false; 209 210 /* 211 * Number of TX descriptors per ring, 212 * setting higher than RX as this seems 213 * the better performing choice. 214 */ 215 static int ixv_txd = PERFORM_TXD; 216 TUNABLE_INT("hw.ixv.txd", &ixv_txd); 217 218 /* Number of RX descriptors per ring */ 219 static int ixv_rxd = PERFORM_RXD; 220 TUNABLE_INT("hw.ixv.rxd", &ixv_rxd); 221 222 /* Legacy Transmit (single queue) */ 223 static int ixv_enable_legacy_tx = 0; 224 TUNABLE_INT("hw.ixv.enable_legacy_tx", &ixv_enable_legacy_tx); 225 226 #ifdef NET_MPSAFE 227 #define IXGBE_MPSAFE 1 228 #define IXGBE_CALLOUT_FLAGS CALLOUT_MPSAFE 229 #define IXGBE_SOFTINT_FLAGS SOFTINT_MPSAFE 230 #define IXGBE_WORKQUEUE_FLAGS WQ_PERCPU | WQ_MPSAFE 231 #else 232 #define IXGBE_CALLOUT_FLAGS 0 233 #define IXGBE_SOFTINT_FLAGS 0 234 #define IXGBE_WORKQUEUE_FLAGS WQ_PERCPU 235 #endif 236 #define IXGBE_WORKQUEUE_PRI PRI_SOFTNET 237 238 #if 0 239 static int (*ixv_start_locked)(struct ifnet *, struct tx_ring *); 240 static int (*ixv_ring_empty)(struct ifnet *, struct buf_ring *); 241 #endif 242 243 /************************************************************************ 244 * ixv_probe - Device identification routine 245 * 246 * Determines if the driver should be loaded on 247 * adapter based on its PCI vendor/device ID. 248 * 249 * return BUS_PROBE_DEFAULT on success, positive on failure 250 ************************************************************************/ 251 static int 252 ixv_probe(device_t dev, cfdata_t cf, void *aux) 253 { 254 #ifdef __HAVE_PCI_MSI_MSIX 255 const struct pci_attach_args *pa = aux; 256 257 return (ixv_lookup(pa) != NULL) ? 1 : 0; 258 #else 259 return 0; 260 #endif 261 } /* ixv_probe */ 262 263 static const ixgbe_vendor_info_t * 264 ixv_lookup(const struct pci_attach_args *pa) 265 { 266 const ixgbe_vendor_info_t *ent; 267 pcireg_t subid; 268 269 INIT_DEBUGOUT("ixv_lookup: begin"); 270 271 if (PCI_VENDOR(pa->pa_id) != IXGBE_INTEL_VENDOR_ID) 272 return NULL; 273 274 subid = pci_conf_read(pa->pa_pc, pa->pa_tag, PCI_SUBSYS_ID_REG); 275 276 for (ent = ixv_vendor_info_array; ent->vendor_id != 0; ent++) { 277 if ((PCI_VENDOR(pa->pa_id) == ent->vendor_id) && 278 (PCI_PRODUCT(pa->pa_id) == ent->device_id) && 279 ((PCI_SUBSYS_VENDOR(subid) == ent->subvendor_id) || 280 (ent->subvendor_id == 0)) && 281 ((PCI_SUBSYS_ID(subid) == ent->subdevice_id) || 282 (ent->subdevice_id == 0))) { 283 return ent; 284 } 285 } 286 287 return NULL; 288 } 289 290 /************************************************************************ 291 * ixv_attach - Device initialization routine 292 * 293 * Called when the driver is being loaded. 294 * Identifies the type of hardware, allocates all resources 295 * and initializes the hardware. 296 * 297 * return 0 on success, positive on failure 298 ************************************************************************/ 299 static void 300 ixv_attach(device_t parent, device_t dev, void *aux) 301 { 302 struct adapter *adapter; 303 struct ixgbe_hw *hw; 304 int error = 0; 305 pcireg_t id, subid; 306 const ixgbe_vendor_info_t *ent; 307 const struct pci_attach_args *pa = aux; 308 const char *apivstr; 309 const char *str; 310 char buf[256]; 311 312 INIT_DEBUGOUT("ixv_attach: begin"); 313 314 /* 315 * Make sure BUSMASTER is set, on a VM under 316 * KVM it may not be and will break things. 317 */ 318 ixgbe_pci_enable_busmaster(pa->pa_pc, pa->pa_tag); 319 320 /* Allocate, clear, and link in our adapter structure */ 321 adapter = device_private(dev); 322 adapter->dev = dev; 323 adapter->hw.back = adapter; 324 hw = &adapter->hw; 325 326 adapter->init_locked = ixv_init_locked; 327 adapter->stop_locked = ixv_stop; 328 329 adapter->osdep.pc = pa->pa_pc; 330 adapter->osdep.tag = pa->pa_tag; 331 if (pci_dma64_available(pa)) 332 adapter->osdep.dmat = pa->pa_dmat64; 333 else 334 adapter->osdep.dmat = pa->pa_dmat; 335 adapter->osdep.attached = false; 336 337 ent = ixv_lookup(pa); 338 339 KASSERT(ent != NULL); 340 341 aprint_normal(": %s, Version - %s\n", 342 ixv_strings[ent->index], ixv_driver_version); 343 344 /* Core Lock Init*/ 345 IXGBE_CORE_LOCK_INIT(adapter, device_xname(dev)); 346 347 /* Do base PCI setup - map BAR0 */ 348 if (ixv_allocate_pci_resources(adapter, pa)) { 349 aprint_error_dev(dev, "ixv_allocate_pci_resources() failed!\n"); 350 error = ENXIO; 351 goto err_out; 352 } 353 354 /* SYSCTL APIs */ 355 ixv_add_device_sysctls(adapter); 356 357 /* Set up the timer callout */ 358 callout_init(&adapter->timer, IXGBE_CALLOUT_FLAGS); 359 360 /* Save off the information about this board */ 361 id = pci_conf_read(pa->pa_pc, pa->pa_tag, PCI_ID_REG); 362 subid = pci_conf_read(pa->pa_pc, pa->pa_tag, PCI_SUBSYS_ID_REG); 363 hw->vendor_id = PCI_VENDOR(id); 364 hw->device_id = PCI_PRODUCT(id); 365 hw->revision_id = 366 PCI_REVISION(pci_conf_read(pa->pa_pc, pa->pa_tag, PCI_CLASS_REG)); 367 hw->subsystem_vendor_id = PCI_SUBSYS_VENDOR(subid); 368 hw->subsystem_device_id = PCI_SUBSYS_ID(subid); 369 370 /* A subset of set_mac_type */ 371 switch (hw->device_id) { 372 case IXGBE_DEV_ID_82599_VF: 373 hw->mac.type = ixgbe_mac_82599_vf; 374 str = "82599 VF"; 375 break; 376 case IXGBE_DEV_ID_X540_VF: 377 hw->mac.type = ixgbe_mac_X540_vf; 378 str = "X540 VF"; 379 break; 380 case IXGBE_DEV_ID_X550_VF: 381 hw->mac.type = ixgbe_mac_X550_vf; 382 str = "X550 VF"; 383 break; 384 case IXGBE_DEV_ID_X550EM_X_VF: 385 hw->mac.type = ixgbe_mac_X550EM_x_vf; 386 str = "X550EM X VF"; 387 break; 388 case IXGBE_DEV_ID_X550EM_A_VF: 389 hw->mac.type = ixgbe_mac_X550EM_a_vf; 390 str = "X550EM A VF"; 391 break; 392 default: 393 /* Shouldn't get here since probe succeeded */ 394 aprint_error_dev(dev, "Unknown device ID!\n"); 395 error = ENXIO; 396 goto err_out; 397 break; 398 } 399 aprint_normal_dev(dev, "device %s\n", str); 400 401 ixv_init_device_features(adapter); 402 403 /* Initialize the shared code */ 404 error = ixgbe_init_ops_vf(hw); 405 if (error) { 406 aprint_error_dev(dev, "ixgbe_init_ops_vf() failed!\n"); 407 error = EIO; 408 goto err_out; 409 } 410 411 /* Setup the mailbox */ 412 ixgbe_init_mbx_params_vf(hw); 413 414 /* Set the right number of segments */ 415 adapter->num_segs = IXGBE_82599_SCATTER; 416 417 /* Reset mbox api to 1.0 */ 418 error = hw->mac.ops.reset_hw(hw); 419 if (error == IXGBE_ERR_RESET_FAILED) 420 aprint_error_dev(dev, "...reset_hw() failure: Reset Failed!\n"); 421 else if (error) 422 aprint_error_dev(dev, "...reset_hw() failed with error %d\n", 423 error); 424 if (error) { 425 error = EIO; 426 goto err_out; 427 } 428 429 error = hw->mac.ops.init_hw(hw); 430 if (error) { 431 aprint_error_dev(dev, "...init_hw() failed!\n"); 432 error = EIO; 433 goto err_out; 434 } 435 436 /* Negotiate mailbox API version */ 437 error = ixv_negotiate_api(adapter); 438 if (error) 439 aprint_normal_dev(dev, 440 "MBX API negotiation failed during attach!\n"); 441 switch (hw->api_version) { 442 case ixgbe_mbox_api_10: 443 apivstr = "1.0"; 444 break; 445 case ixgbe_mbox_api_20: 446 apivstr = "2.0"; 447 break; 448 case ixgbe_mbox_api_11: 449 apivstr = "1.1"; 450 break; 451 case ixgbe_mbox_api_12: 452 apivstr = "1.2"; 453 break; 454 case ixgbe_mbox_api_13: 455 apivstr = "1.3"; 456 break; 457 default: 458 apivstr = "unknown"; 459 break; 460 } 461 aprint_normal_dev(dev, "Mailbox API %s\n", apivstr); 462 463 /* If no mac address was assigned, make a random one */ 464 if (!ixv_check_ether_addr(hw->mac.addr)) { 465 u8 addr[ETHER_ADDR_LEN]; 466 uint64_t rndval = cprng_strong64(); 467 468 memcpy(addr, &rndval, sizeof(addr)); 469 addr[0] &= 0xFE; 470 addr[0] |= 0x02; 471 bcopy(addr, hw->mac.addr, sizeof(addr)); 472 } 473 474 /* Register for VLAN events */ 475 ether_set_vlan_cb(&adapter->osdep.ec, ixv_vlan_cb); 476 477 /* Sysctls for limiting the amount of work done in the taskqueues */ 478 ixv_set_sysctl_value(adapter, "rx_processing_limit", 479 "max number of rx packets to process", 480 &adapter->rx_process_limit, ixv_rx_process_limit); 481 482 ixv_set_sysctl_value(adapter, "tx_processing_limit", 483 "max number of tx packets to process", 484 &adapter->tx_process_limit, ixv_tx_process_limit); 485 486 /* Do descriptor calc and sanity checks */ 487 if (((ixv_txd * sizeof(union ixgbe_adv_tx_desc)) % DBA_ALIGN) != 0 || 488 ixv_txd < MIN_TXD || ixv_txd > MAX_TXD) { 489 aprint_error_dev(dev, "TXD config issue, using default!\n"); 490 adapter->num_tx_desc = DEFAULT_TXD; 491 } else 492 adapter->num_tx_desc = ixv_txd; 493 494 if (((ixv_rxd * sizeof(union ixgbe_adv_rx_desc)) % DBA_ALIGN) != 0 || 495 ixv_rxd < MIN_RXD || ixv_rxd > MAX_RXD) { 496 aprint_error_dev(dev, "RXD config issue, using default!\n"); 497 adapter->num_rx_desc = DEFAULT_RXD; 498 } else 499 adapter->num_rx_desc = ixv_rxd; 500 501 /* Setup MSI-X */ 502 error = ixv_configure_interrupts(adapter); 503 if (error) 504 goto err_out; 505 506 /* Allocate our TX/RX Queues */ 507 if (ixgbe_allocate_queues(adapter)) { 508 aprint_error_dev(dev, "ixgbe_allocate_queues() failed!\n"); 509 error = ENOMEM; 510 goto err_out; 511 } 512 513 /* hw.ix defaults init */ 514 adapter->enable_aim = ixv_enable_aim; 515 516 adapter->txrx_use_workqueue = ixv_txrx_workqueue; 517 518 error = ixv_allocate_msix(adapter, pa); 519 if (error) { 520 aprint_error_dev(dev, "ixv_allocate_msix() failed!\n"); 521 goto err_late; 522 } 523 524 /* Setup OS specific network interface */ 525 error = ixv_setup_interface(dev, adapter); 526 if (error != 0) { 527 aprint_error_dev(dev, "ixv_setup_interface() failed!\n"); 528 goto err_late; 529 } 530 531 /* Do the stats setup */ 532 ixv_save_stats(adapter); 533 ixv_init_stats(adapter); 534 ixv_add_stats_sysctls(adapter); 535 536 if (adapter->feat_en & IXGBE_FEATURE_NETMAP) 537 ixgbe_netmap_attach(adapter); 538 539 snprintb(buf, sizeof(buf), IXGBE_FEATURE_FLAGS, adapter->feat_cap); 540 aprint_verbose_dev(dev, "feature cap %s\n", buf); 541 snprintb(buf, sizeof(buf), IXGBE_FEATURE_FLAGS, adapter->feat_en); 542 aprint_verbose_dev(dev, "feature ena %s\n", buf); 543 544 INIT_DEBUGOUT("ixv_attach: end"); 545 adapter->osdep.attached = true; 546 547 return; 548 549 err_late: 550 ixgbe_free_queues(adapter); 551 err_out: 552 ixv_free_pci_resources(adapter); 553 IXGBE_CORE_LOCK_DESTROY(adapter); 554 555 return; 556 } /* ixv_attach */ 557 558 /************************************************************************ 559 * ixv_detach - Device removal routine 560 * 561 * Called when the driver is being removed. 562 * Stops the adapter and deallocates all the resources 563 * that were allocated for driver operation. 564 * 565 * return 0 on success, positive on failure 566 ************************************************************************/ 567 static int 568 ixv_detach(device_t dev, int flags) 569 { 570 struct adapter *adapter = device_private(dev); 571 struct ixgbe_hw *hw = &adapter->hw; 572 struct ix_queue *que = adapter->queues; 573 struct tx_ring *txr = adapter->tx_rings; 574 struct rx_ring *rxr = adapter->rx_rings; 575 struct ixgbevf_hw_stats *stats = &adapter->stats.vf; 576 577 INIT_DEBUGOUT("ixv_detach: begin"); 578 if (adapter->osdep.attached == false) 579 return 0; 580 581 /* Stop the interface. Callouts are stopped in it. */ 582 ixv_ifstop(adapter->ifp, 1); 583 584 #if NVLAN > 0 585 /* Make sure VLANs are not using driver */ 586 if (!VLAN_ATTACHED(&adapter->osdep.ec)) 587 ; /* nothing to do: no VLANs */ 588 else if ((flags & (DETACH_SHUTDOWN | DETACH_FORCE)) != 0) 589 vlan_ifdetach(adapter->ifp); 590 else { 591 aprint_error_dev(dev, "VLANs in use, detach first\n"); 592 return EBUSY; 593 } 594 #endif 595 596 for (int i = 0; i < adapter->num_queues; i++, que++, txr++) { 597 if (!(adapter->feat_en & IXGBE_FEATURE_LEGACY_TX)) 598 softint_disestablish(txr->txr_si); 599 softint_disestablish(que->que_si); 600 } 601 if (adapter->txr_wq != NULL) 602 workqueue_destroy(adapter->txr_wq); 603 if (adapter->txr_wq_enqueued != NULL) 604 percpu_free(adapter->txr_wq_enqueued, sizeof(u_int)); 605 if (adapter->que_wq != NULL) 606 workqueue_destroy(adapter->que_wq); 607 608 /* Drain the Mailbox(link) queue */ 609 softint_disestablish(adapter->link_si); 610 611 ether_ifdetach(adapter->ifp); 612 callout_halt(&adapter->timer, NULL); 613 614 if (adapter->feat_en & IXGBE_FEATURE_NETMAP) 615 netmap_detach(adapter->ifp); 616 617 ixv_free_pci_resources(adapter); 618 #if 0 /* XXX the NetBSD port is probably missing something here */ 619 bus_generic_detach(dev); 620 #endif 621 if_detach(adapter->ifp); 622 ifmedia_fini(&adapter->media); 623 if_percpuq_destroy(adapter->ipq); 624 625 sysctl_teardown(&adapter->sysctllog); 626 evcnt_detach(&adapter->efbig_tx_dma_setup); 627 evcnt_detach(&adapter->mbuf_defrag_failed); 628 evcnt_detach(&adapter->efbig2_tx_dma_setup); 629 evcnt_detach(&adapter->einval_tx_dma_setup); 630 evcnt_detach(&adapter->other_tx_dma_setup); 631 evcnt_detach(&adapter->eagain_tx_dma_setup); 632 evcnt_detach(&adapter->enomem_tx_dma_setup); 633 evcnt_detach(&adapter->watchdog_events); 634 evcnt_detach(&adapter->tso_err); 635 evcnt_detach(&adapter->link_irq); 636 637 txr = adapter->tx_rings; 638 for (int i = 0; i < adapter->num_queues; i++, rxr++, txr++) { 639 evcnt_detach(&adapter->queues[i].irqs); 640 evcnt_detach(&adapter->queues[i].handleq); 641 evcnt_detach(&adapter->queues[i].req); 642 evcnt_detach(&txr->no_desc_avail); 643 evcnt_detach(&txr->total_packets); 644 evcnt_detach(&txr->tso_tx); 645 #ifndef IXGBE_LEGACY_TX 646 evcnt_detach(&txr->pcq_drops); 647 #endif 648 649 evcnt_detach(&rxr->rx_packets); 650 evcnt_detach(&rxr->rx_bytes); 651 evcnt_detach(&rxr->rx_copies); 652 evcnt_detach(&rxr->no_jmbuf); 653 evcnt_detach(&rxr->rx_discarded); 654 } 655 evcnt_detach(&stats->ipcs); 656 evcnt_detach(&stats->l4cs); 657 evcnt_detach(&stats->ipcs_bad); 658 evcnt_detach(&stats->l4cs_bad); 659 660 /* Packet Reception Stats */ 661 evcnt_detach(&stats->vfgorc); 662 evcnt_detach(&stats->vfgprc); 663 evcnt_detach(&stats->vfmprc); 664 665 /* Packet Transmission Stats */ 666 evcnt_detach(&stats->vfgotc); 667 evcnt_detach(&stats->vfgptc); 668 669 /* Mailbox Stats */ 670 evcnt_detach(&hw->mbx.stats.msgs_tx); 671 evcnt_detach(&hw->mbx.stats.msgs_rx); 672 evcnt_detach(&hw->mbx.stats.acks); 673 evcnt_detach(&hw->mbx.stats.reqs); 674 evcnt_detach(&hw->mbx.stats.rsts); 675 676 ixgbe_free_queues(adapter); 677 678 IXGBE_CORE_LOCK_DESTROY(adapter); 679 680 return (0); 681 } /* ixv_detach */ 682 683 /************************************************************************ 684 * ixv_init_locked - Init entry point 685 * 686 * Used in two ways: It is used by the stack as an init entry 687 * point in network interface structure. It is also used 688 * by the driver as a hw/sw initialization routine to get 689 * to a consistent state. 690 * 691 * return 0 on success, positive on failure 692 ************************************************************************/ 693 static void 694 ixv_init_locked(struct adapter *adapter) 695 { 696 struct ifnet *ifp = adapter->ifp; 697 device_t dev = adapter->dev; 698 struct ixgbe_hw *hw = &adapter->hw; 699 struct ix_queue *que; 700 int error = 0; 701 uint32_t mask; 702 int i; 703 704 INIT_DEBUGOUT("ixv_init_locked: begin"); 705 KASSERT(mutex_owned(&adapter->core_mtx)); 706 hw->adapter_stopped = FALSE; 707 hw->mac.ops.stop_adapter(hw); 708 callout_stop(&adapter->timer); 709 for (i = 0, que = adapter->queues; i < adapter->num_queues; i++, que++) 710 que->disabled_count = 0; 711 712 adapter->max_frame_size = 713 ifp->if_mtu + ETHER_HDR_LEN + ETHER_CRC_LEN; 714 715 /* reprogram the RAR[0] in case user changed it. */ 716 hw->mac.ops.set_rar(hw, 0, hw->mac.addr, 0, IXGBE_RAH_AV); 717 718 /* Get the latest mac address, User can use a LAA */ 719 memcpy(hw->mac.addr, CLLADDR(ifp->if_sadl), 720 IXGBE_ETH_LENGTH_OF_ADDRESS); 721 hw->mac.ops.set_rar(hw, 0, hw->mac.addr, 0, 1); 722 723 /* Prepare transmit descriptors and buffers */ 724 if (ixgbe_setup_transmit_structures(adapter)) { 725 aprint_error_dev(dev, "Could not setup transmit structures\n"); 726 ixv_stop(adapter); 727 return; 728 } 729 730 /* Reset VF and renegotiate mailbox API version */ 731 hw->mac.ops.reset_hw(hw); 732 hw->mac.ops.start_hw(hw); 733 error = ixv_negotiate_api(adapter); 734 if (error) 735 device_printf(dev, 736 "Mailbox API negotiation failed in init_locked!\n"); 737 738 ixv_initialize_transmit_units(adapter); 739 740 /* Setup Multicast table */ 741 ixv_set_rxfilter(adapter); 742 743 /* 744 * Determine the correct mbuf pool 745 * for doing jumbo/headersplit 746 */ 747 if (adapter->max_frame_size <= MCLBYTES) 748 adapter->rx_mbuf_sz = MCLBYTES; 749 else 750 adapter->rx_mbuf_sz = MJUMPAGESIZE; 751 752 /* Prepare receive descriptors and buffers */ 753 if (ixgbe_setup_receive_structures(adapter)) { 754 device_printf(dev, "Could not setup receive structures\n"); 755 ixv_stop(adapter); 756 return; 757 } 758 759 /* Configure RX settings */ 760 ixv_initialize_receive_units(adapter); 761 762 /* Set up VLAN offload and filter */ 763 ixv_setup_vlan_support(adapter); 764 765 /* Set up MSI-X routing */ 766 ixv_configure_ivars(adapter); 767 768 /* Set up auto-mask */ 769 mask = (1 << adapter->vector); 770 for (i = 0, que = adapter->queues; i < adapter->num_queues; i++, que++) 771 mask |= (1 << que->msix); 772 IXGBE_WRITE_REG(hw, IXGBE_VTEIAM, mask); 773 774 /* Set moderation on the Link interrupt */ 775 ixv_eitr_write(adapter, adapter->vector, IXGBE_LINK_ITR); 776 777 /* Stats init */ 778 ixv_init_stats(adapter); 779 780 /* Config/Enable Link */ 781 hw->mac.get_link_status = TRUE; 782 hw->mac.ops.check_link(hw, &adapter->link_speed, &adapter->link_up, 783 FALSE); 784 785 /* Start watchdog */ 786 callout_reset(&adapter->timer, hz, ixv_local_timer, adapter); 787 788 /* And now turn on interrupts */ 789 ixv_enable_intr(adapter); 790 791 /* Update saved flags. See ixgbe_ifflags_cb() */ 792 adapter->if_flags = ifp->if_flags; 793 adapter->ec_capenable = adapter->osdep.ec.ec_capenable; 794 795 /* Now inform the stack we're ready */ 796 ifp->if_flags |= IFF_RUNNING; 797 ifp->if_flags &= ~IFF_OACTIVE; 798 799 return; 800 } /* ixv_init_locked */ 801 802 /************************************************************************ 803 * ixv_enable_queue 804 ************************************************************************/ 805 static inline void 806 ixv_enable_queue(struct adapter *adapter, u32 vector) 807 { 808 struct ixgbe_hw *hw = &adapter->hw; 809 struct ix_queue *que = &adapter->queues[vector]; 810 u32 queue = 1UL << vector; 811 u32 mask; 812 813 mutex_enter(&que->dc_mtx); 814 if (que->disabled_count > 0 && --que->disabled_count > 0) 815 goto out; 816 817 mask = (IXGBE_EIMS_RTX_QUEUE & queue); 818 IXGBE_WRITE_REG(hw, IXGBE_VTEIMS, mask); 819 out: 820 mutex_exit(&que->dc_mtx); 821 } /* ixv_enable_queue */ 822 823 /************************************************************************ 824 * ixv_disable_queue 825 ************************************************************************/ 826 static inline void 827 ixv_disable_queue(struct adapter *adapter, u32 vector) 828 { 829 struct ixgbe_hw *hw = &adapter->hw; 830 struct ix_queue *que = &adapter->queues[vector]; 831 u32 queue = 1UL << vector; 832 u32 mask; 833 834 mutex_enter(&que->dc_mtx); 835 if (que->disabled_count++ > 0) 836 goto out; 837 838 mask = (IXGBE_EIMS_RTX_QUEUE & queue); 839 IXGBE_WRITE_REG(hw, IXGBE_VTEIMC, mask); 840 out: 841 mutex_exit(&que->dc_mtx); 842 } /* ixv_disable_queue */ 843 844 #if 0 845 static inline void 846 ixv_rearm_queues(struct adapter *adapter, u64 queues) 847 { 848 u32 mask = (IXGBE_EIMS_RTX_QUEUE & queues); 849 IXGBE_WRITE_REG(&adapter->hw, IXGBE_VTEICS, mask); 850 } /* ixv_rearm_queues */ 851 #endif 852 853 854 /************************************************************************ 855 * ixv_msix_que - MSI-X Queue Interrupt Service routine 856 ************************************************************************/ 857 static int 858 ixv_msix_que(void *arg) 859 { 860 struct ix_queue *que = arg; 861 struct adapter *adapter = que->adapter; 862 struct tx_ring *txr = que->txr; 863 struct rx_ring *rxr = que->rxr; 864 bool more; 865 u32 newitr = 0; 866 867 ixv_disable_queue(adapter, que->msix); 868 ++que->irqs.ev_count; 869 870 #ifdef __NetBSD__ 871 /* Don't run ixgbe_rxeof in interrupt context */ 872 more = true; 873 #else 874 more = ixgbe_rxeof(que); 875 #endif 876 877 IXGBE_TX_LOCK(txr); 878 ixgbe_txeof(txr); 879 IXGBE_TX_UNLOCK(txr); 880 881 /* Do AIM now? */ 882 883 if (adapter->enable_aim == false) 884 goto no_calc; 885 /* 886 * Do Adaptive Interrupt Moderation: 887 * - Write out last calculated setting 888 * - Calculate based on average size over 889 * the last interval. 890 */ 891 if (que->eitr_setting) 892 ixv_eitr_write(adapter, que->msix, que->eitr_setting); 893 894 que->eitr_setting = 0; 895 896 /* Idle, do nothing */ 897 if ((txr->bytes == 0) && (rxr->bytes == 0)) 898 goto no_calc; 899 900 if ((txr->bytes) && (txr->packets)) 901 newitr = txr->bytes/txr->packets; 902 if ((rxr->bytes) && (rxr->packets)) 903 newitr = uimax(newitr, (rxr->bytes / rxr->packets)); 904 newitr += 24; /* account for hardware frame, crc */ 905 906 /* set an upper boundary */ 907 newitr = uimin(newitr, 3000); 908 909 /* Be nice to the mid range */ 910 if ((newitr > 300) && (newitr < 1200)) 911 newitr = (newitr / 3); 912 else 913 newitr = (newitr / 2); 914 915 /* 916 * When RSC is used, ITR interval must be larger than RSC_DELAY. 917 * Currently, we use 2us for RSC_DELAY. The minimum value is always 918 * greater than 2us on 100M (and 10M?(not documented)), but it's not 919 * on 1G and higher. 920 */ 921 if ((adapter->link_speed != IXGBE_LINK_SPEED_100_FULL) 922 && (adapter->link_speed != IXGBE_LINK_SPEED_10_FULL)) { 923 if (newitr < IXGBE_MIN_RSC_EITR_10G1G) 924 newitr = IXGBE_MIN_RSC_EITR_10G1G; 925 } 926 927 /* save for next interrupt */ 928 que->eitr_setting = newitr; 929 930 /* Reset state */ 931 txr->bytes = 0; 932 txr->packets = 0; 933 rxr->bytes = 0; 934 rxr->packets = 0; 935 936 no_calc: 937 if (more) 938 softint_schedule(que->que_si); 939 else /* Re-enable this interrupt */ 940 ixv_enable_queue(adapter, que->msix); 941 942 return 1; 943 } /* ixv_msix_que */ 944 945 /************************************************************************ 946 * ixv_msix_mbx 947 ************************************************************************/ 948 static int 949 ixv_msix_mbx(void *arg) 950 { 951 struct adapter *adapter = arg; 952 struct ixgbe_hw *hw = &adapter->hw; 953 954 ++adapter->link_irq.ev_count; 955 /* NetBSD: We use auto-clear, so it's not required to write VTEICR */ 956 957 /* Link status change */ 958 hw->mac.get_link_status = TRUE; 959 softint_schedule(adapter->link_si); 960 961 IXGBE_WRITE_REG(hw, IXGBE_VTEIMS, (1 << adapter->vector)); 962 963 return 1; 964 } /* ixv_msix_mbx */ 965 966 static void 967 ixv_eitr_write(struct adapter *adapter, uint32_t index, uint32_t itr) 968 { 969 970 /* 971 * Newer devices than 82598 have VF function, so this function is 972 * simple. 973 */ 974 itr |= IXGBE_EITR_CNT_WDIS; 975 976 IXGBE_WRITE_REG(&adapter->hw, IXGBE_VTEITR(index), itr); 977 } 978 979 980 /************************************************************************ 981 * ixv_media_status - Media Ioctl callback 982 * 983 * Called whenever the user queries the status of 984 * the interface using ifconfig. 985 ************************************************************************/ 986 static void 987 ixv_media_status(struct ifnet *ifp, struct ifmediareq *ifmr) 988 { 989 struct adapter *adapter = ifp->if_softc; 990 991 INIT_DEBUGOUT("ixv_media_status: begin"); 992 ixv_update_link_status(adapter); 993 994 ifmr->ifm_status = IFM_AVALID; 995 ifmr->ifm_active = IFM_ETHER; 996 997 if (adapter->link_active != LINK_STATE_UP) { 998 ifmr->ifm_active |= IFM_NONE; 999 return; 1000 } 1001 1002 ifmr->ifm_status |= IFM_ACTIVE; 1003 1004 switch (adapter->link_speed) { 1005 case IXGBE_LINK_SPEED_10GB_FULL: 1006 ifmr->ifm_active |= IFM_10G_T | IFM_FDX; 1007 break; 1008 case IXGBE_LINK_SPEED_5GB_FULL: 1009 ifmr->ifm_active |= IFM_5000_T | IFM_FDX; 1010 break; 1011 case IXGBE_LINK_SPEED_2_5GB_FULL: 1012 ifmr->ifm_active |= IFM_2500_T | IFM_FDX; 1013 break; 1014 case IXGBE_LINK_SPEED_1GB_FULL: 1015 ifmr->ifm_active |= IFM_1000_T | IFM_FDX; 1016 break; 1017 case IXGBE_LINK_SPEED_100_FULL: 1018 ifmr->ifm_active |= IFM_100_TX | IFM_FDX; 1019 break; 1020 case IXGBE_LINK_SPEED_10_FULL: 1021 ifmr->ifm_active |= IFM_10_T | IFM_FDX; 1022 break; 1023 } 1024 1025 ifp->if_baudrate = ifmedia_baudrate(ifmr->ifm_active); 1026 } /* ixv_media_status */ 1027 1028 /************************************************************************ 1029 * ixv_media_change - Media Ioctl callback 1030 * 1031 * Called when the user changes speed/duplex using 1032 * media/mediopt option with ifconfig. 1033 ************************************************************************/ 1034 static int 1035 ixv_media_change(struct ifnet *ifp) 1036 { 1037 struct adapter *adapter = ifp->if_softc; 1038 struct ifmedia *ifm = &adapter->media; 1039 1040 INIT_DEBUGOUT("ixv_media_change: begin"); 1041 1042 if (IFM_TYPE(ifm->ifm_media) != IFM_ETHER) 1043 return (EINVAL); 1044 1045 switch (IFM_SUBTYPE(ifm->ifm_media)) { 1046 case IFM_AUTO: 1047 break; 1048 default: 1049 device_printf(adapter->dev, "Only auto media type\n"); 1050 return (EINVAL); 1051 } 1052 1053 return (0); 1054 } /* ixv_media_change */ 1055 1056 /************************************************************************ 1057 * ixv_negotiate_api 1058 * 1059 * Negotiate the Mailbox API with the PF; 1060 * start with the most featured API first. 1061 ************************************************************************/ 1062 static int 1063 ixv_negotiate_api(struct adapter *adapter) 1064 { 1065 struct ixgbe_hw *hw = &adapter->hw; 1066 int mbx_api[] = { ixgbe_mbox_api_13, 1067 ixgbe_mbox_api_12, 1068 ixgbe_mbox_api_11, 1069 ixgbe_mbox_api_10, 1070 ixgbe_mbox_api_unknown }; 1071 int i = 0; 1072 1073 while (mbx_api[i] != ixgbe_mbox_api_unknown) { 1074 if (ixgbevf_negotiate_api_version(hw, mbx_api[i]) == 0) 1075 return (0); 1076 i++; 1077 } 1078 1079 return (EINVAL); 1080 } /* ixv_negotiate_api */ 1081 1082 1083 /************************************************************************ 1084 * ixv_set_multi - Multicast Update 1085 * 1086 * Called whenever multicast address list is updated. 1087 ************************************************************************/ 1088 static int 1089 ixv_set_rxfilter(struct adapter *adapter) 1090 { 1091 u8 mta[IXGBE_MAX_VF_MC * IXGBE_ETH_LENGTH_OF_ADDRESS]; 1092 struct ifnet *ifp = adapter->ifp; 1093 struct ixgbe_hw *hw = &adapter->hw; 1094 u8 *update_ptr; 1095 int mcnt = 0; 1096 struct ethercom *ec = &adapter->osdep.ec; 1097 struct ether_multi *enm; 1098 struct ether_multistep step; 1099 bool overflow = false; 1100 int error, rc = 0; 1101 1102 KASSERT(mutex_owned(&adapter->core_mtx)); 1103 IOCTL_DEBUGOUT("ixv_set_rxfilter: begin"); 1104 1105 /* 1: For PROMISC */ 1106 if (ifp->if_flags & IFF_PROMISC) { 1107 error = hw->mac.ops.update_xcast_mode(hw, 1108 IXGBEVF_XCAST_MODE_PROMISC); 1109 if (error == IXGBE_ERR_NOT_TRUSTED) { 1110 device_printf(adapter->dev, 1111 "this interface is not trusted\n"); 1112 error = EPERM; 1113 } else if (error == IXGBE_ERR_FEATURE_NOT_SUPPORTED) { 1114 device_printf(adapter->dev, 1115 "the PF doesn't support promisc mode\n"); 1116 error = EOPNOTSUPP; 1117 } else if (error == IXGBE_ERR_NOT_IN_PROMISC) { 1118 device_printf(adapter->dev, 1119 "the PF may not in promisc mode\n"); 1120 error = EINVAL; 1121 } else if (error) { 1122 device_printf(adapter->dev, 1123 "failed to set promisc mode. error = %d\n", 1124 error); 1125 error = EIO; 1126 } else 1127 return 0; 1128 rc = error; 1129 } 1130 1131 /* 2: For ALLMULTI or normal */ 1132 ETHER_LOCK(ec); 1133 ETHER_FIRST_MULTI(step, ec, enm); 1134 while (enm != NULL) { 1135 if ((mcnt >= IXGBE_MAX_VF_MC) || 1136 (memcmp(enm->enm_addrlo, enm->enm_addrhi, 1137 ETHER_ADDR_LEN) != 0)) { 1138 overflow = true; 1139 break; 1140 } 1141 bcopy(enm->enm_addrlo, 1142 &mta[mcnt * IXGBE_ETH_LENGTH_OF_ADDRESS], 1143 IXGBE_ETH_LENGTH_OF_ADDRESS); 1144 mcnt++; 1145 ETHER_NEXT_MULTI(step, enm); 1146 } 1147 ETHER_UNLOCK(ec); 1148 1149 /* 3: For ALLMULTI */ 1150 if (overflow) { 1151 error = hw->mac.ops.update_xcast_mode(hw, 1152 IXGBEVF_XCAST_MODE_ALLMULTI); 1153 if (error == IXGBE_ERR_NOT_TRUSTED) { 1154 device_printf(adapter->dev, 1155 "this interface is not trusted\n"); 1156 error = EPERM; 1157 } else if (error == IXGBE_ERR_FEATURE_NOT_SUPPORTED) { 1158 device_printf(adapter->dev, 1159 "the PF doesn't support allmulti mode\n"); 1160 error = EOPNOTSUPP; 1161 } else if (error) { 1162 device_printf(adapter->dev, 1163 "number of Ethernet multicast addresses " 1164 "exceeds the limit (%d). error = %d\n", 1165 IXGBE_MAX_VF_MC, error); 1166 error = ENOSPC; 1167 } else { 1168 ETHER_LOCK(ec); 1169 ec->ec_flags |= ETHER_F_ALLMULTI; 1170 ETHER_UNLOCK(ec); 1171 return rc; /* Promisc might have failed */ 1172 } 1173 1174 if (rc == 0) 1175 rc = error; 1176 1177 /* Continue to update the multicast table as many as we can */ 1178 } 1179 1180 /* 4: For normal operation */ 1181 error = hw->mac.ops.update_xcast_mode(hw, IXGBEVF_XCAST_MODE_MULTI); 1182 if ((error == IXGBE_ERR_FEATURE_NOT_SUPPORTED) || (error == 0)) { 1183 /* Normal operation */ 1184 ETHER_LOCK(ec); 1185 ec->ec_flags &= ~ETHER_F_ALLMULTI; 1186 ETHER_UNLOCK(ec); 1187 error = 0; 1188 } else if (error) { 1189 device_printf(adapter->dev, 1190 "failed to set Ethernet multicast address " 1191 "operation to normal. error = %d\n", error); 1192 } 1193 1194 update_ptr = mta; 1195 1196 error = adapter->hw.mac.ops.update_mc_addr_list(&adapter->hw, 1197 update_ptr, mcnt, ixv_mc_array_itr, TRUE); 1198 if (rc == 0) 1199 rc = error; 1200 1201 return rc; 1202 } /* ixv_set_rxfilter */ 1203 1204 /************************************************************************ 1205 * ixv_mc_array_itr 1206 * 1207 * An iterator function needed by the multicast shared code. 1208 * It feeds the shared code routine the addresses in the 1209 * array of ixv_set_rxfilter() one by one. 1210 ************************************************************************/ 1211 static u8 * 1212 ixv_mc_array_itr(struct ixgbe_hw *hw, u8 **update_ptr, u32 *vmdq) 1213 { 1214 u8 *addr = *update_ptr; 1215 u8 *newptr; 1216 1217 *vmdq = 0; 1218 1219 newptr = addr + IXGBE_ETH_LENGTH_OF_ADDRESS; 1220 *update_ptr = newptr; 1221 1222 return addr; 1223 } /* ixv_mc_array_itr */ 1224 1225 /************************************************************************ 1226 * ixv_local_timer - Timer routine 1227 * 1228 * Checks for link status, updates statistics, 1229 * and runs the watchdog check. 1230 ************************************************************************/ 1231 static void 1232 ixv_local_timer(void *arg) 1233 { 1234 struct adapter *adapter = arg; 1235 1236 IXGBE_CORE_LOCK(adapter); 1237 ixv_local_timer_locked(adapter); 1238 IXGBE_CORE_UNLOCK(adapter); 1239 } 1240 1241 static void 1242 ixv_local_timer_locked(void *arg) 1243 { 1244 struct adapter *adapter = arg; 1245 device_t dev = adapter->dev; 1246 struct ix_queue *que = adapter->queues; 1247 u64 queues = 0; 1248 u64 v0, v1, v2, v3, v4, v5, v6, v7; 1249 int hung = 0; 1250 int i; 1251 1252 KASSERT(mutex_owned(&adapter->core_mtx)); 1253 1254 if (ixv_check_link(adapter)) { 1255 ixv_init_locked(adapter); 1256 return; 1257 } 1258 1259 /* Stats Update */ 1260 ixv_update_stats(adapter); 1261 1262 /* Update some event counters */ 1263 v0 = v1 = v2 = v3 = v4 = v5 = v6 = v7 = 0; 1264 que = adapter->queues; 1265 for (i = 0; i < adapter->num_queues; i++, que++) { 1266 struct tx_ring *txr = que->txr; 1267 1268 v0 += txr->q_efbig_tx_dma_setup; 1269 v1 += txr->q_mbuf_defrag_failed; 1270 v2 += txr->q_efbig2_tx_dma_setup; 1271 v3 += txr->q_einval_tx_dma_setup; 1272 v4 += txr->q_other_tx_dma_setup; 1273 v5 += txr->q_eagain_tx_dma_setup; 1274 v6 += txr->q_enomem_tx_dma_setup; 1275 v7 += txr->q_tso_err; 1276 } 1277 adapter->efbig_tx_dma_setup.ev_count = v0; 1278 adapter->mbuf_defrag_failed.ev_count = v1; 1279 adapter->efbig2_tx_dma_setup.ev_count = v2; 1280 adapter->einval_tx_dma_setup.ev_count = v3; 1281 adapter->other_tx_dma_setup.ev_count = v4; 1282 adapter->eagain_tx_dma_setup.ev_count = v5; 1283 adapter->enomem_tx_dma_setup.ev_count = v6; 1284 adapter->tso_err.ev_count = v7; 1285 1286 /* 1287 * Check the TX queues status 1288 * - mark hung queues so we don't schedule on them 1289 * - watchdog only if all queues show hung 1290 */ 1291 que = adapter->queues; 1292 for (i = 0; i < adapter->num_queues; i++, que++) { 1293 /* Keep track of queues with work for soft irq */ 1294 if (que->txr->busy) 1295 queues |= ((u64)1 << que->me); 1296 /* 1297 * Each time txeof runs without cleaning, but there 1298 * are uncleaned descriptors it increments busy. If 1299 * we get to the MAX we declare it hung. 1300 */ 1301 if (que->busy == IXGBE_QUEUE_HUNG) { 1302 ++hung; 1303 /* Mark the queue as inactive */ 1304 adapter->active_queues &= ~((u64)1 << que->me); 1305 continue; 1306 } else { 1307 /* Check if we've come back from hung */ 1308 if ((adapter->active_queues & ((u64)1 << que->me)) == 0) 1309 adapter->active_queues |= ((u64)1 << que->me); 1310 } 1311 if (que->busy >= IXGBE_MAX_TX_BUSY) { 1312 device_printf(dev, 1313 "Warning queue %d appears to be hung!\n", i); 1314 que->txr->busy = IXGBE_QUEUE_HUNG; 1315 ++hung; 1316 } 1317 } 1318 1319 /* Only truly watchdog if all queues show hung */ 1320 if (hung == adapter->num_queues) 1321 goto watchdog; 1322 #if 0 1323 else if (queues != 0) { /* Force an IRQ on queues with work */ 1324 ixv_rearm_queues(adapter, queues); 1325 } 1326 #endif 1327 1328 callout_reset(&adapter->timer, hz, ixv_local_timer, adapter); 1329 1330 return; 1331 1332 watchdog: 1333 1334 device_printf(adapter->dev, "Watchdog timeout -- resetting\n"); 1335 adapter->ifp->if_flags &= ~IFF_RUNNING; 1336 adapter->watchdog_events.ev_count++; 1337 ixv_init_locked(adapter); 1338 } /* ixv_local_timer */ 1339 1340 /************************************************************************ 1341 * ixv_update_link_status - Update OS on link state 1342 * 1343 * Note: Only updates the OS on the cached link state. 1344 * The real check of the hardware only happens with 1345 * a link interrupt. 1346 ************************************************************************/ 1347 static void 1348 ixv_update_link_status(struct adapter *adapter) 1349 { 1350 struct ifnet *ifp = adapter->ifp; 1351 device_t dev = adapter->dev; 1352 1353 KASSERT(mutex_owned(&adapter->core_mtx)); 1354 1355 if (adapter->link_up) { 1356 if (adapter->link_active != LINK_STATE_UP) { 1357 if (bootverbose) { 1358 const char *bpsmsg; 1359 1360 switch (adapter->link_speed) { 1361 case IXGBE_LINK_SPEED_10GB_FULL: 1362 bpsmsg = "10 Gbps"; 1363 break; 1364 case IXGBE_LINK_SPEED_5GB_FULL: 1365 bpsmsg = "5 Gbps"; 1366 break; 1367 case IXGBE_LINK_SPEED_2_5GB_FULL: 1368 bpsmsg = "2.5 Gbps"; 1369 break; 1370 case IXGBE_LINK_SPEED_1GB_FULL: 1371 bpsmsg = "1 Gbps"; 1372 break; 1373 case IXGBE_LINK_SPEED_100_FULL: 1374 bpsmsg = "100 Mbps"; 1375 break; 1376 case IXGBE_LINK_SPEED_10_FULL: 1377 bpsmsg = "10 Mbps"; 1378 break; 1379 default: 1380 bpsmsg = "unknown speed"; 1381 break; 1382 } 1383 device_printf(dev, "Link is up %s %s \n", 1384 bpsmsg, "Full Duplex"); 1385 } 1386 adapter->link_active = LINK_STATE_UP; 1387 if_link_state_change(ifp, LINK_STATE_UP); 1388 } 1389 } else { 1390 /* 1391 * Do it when link active changes to DOWN. i.e. 1392 * a) LINK_STATE_UNKNOWN -> LINK_STATE_DOWN 1393 * b) LINK_STATE_UP -> LINK_STATE_DOWN 1394 */ 1395 if (adapter->link_active != LINK_STATE_DOWN) { 1396 if (bootverbose) 1397 device_printf(dev, "Link is Down\n"); 1398 if_link_state_change(ifp, LINK_STATE_DOWN); 1399 adapter->link_active = LINK_STATE_DOWN; 1400 } 1401 } 1402 } /* ixv_update_link_status */ 1403 1404 1405 /************************************************************************ 1406 * ixv_stop - Stop the hardware 1407 * 1408 * Disables all traffic on the adapter by issuing a 1409 * global reset on the MAC and deallocates TX/RX buffers. 1410 ************************************************************************/ 1411 static void 1412 ixv_ifstop(struct ifnet *ifp, int disable) 1413 { 1414 struct adapter *adapter = ifp->if_softc; 1415 1416 IXGBE_CORE_LOCK(adapter); 1417 ixv_stop(adapter); 1418 IXGBE_CORE_UNLOCK(adapter); 1419 } 1420 1421 static void 1422 ixv_stop(void *arg) 1423 { 1424 struct ifnet *ifp; 1425 struct adapter *adapter = arg; 1426 struct ixgbe_hw *hw = &adapter->hw; 1427 1428 ifp = adapter->ifp; 1429 1430 KASSERT(mutex_owned(&adapter->core_mtx)); 1431 1432 INIT_DEBUGOUT("ixv_stop: begin\n"); 1433 ixv_disable_intr(adapter); 1434 1435 /* Tell the stack that the interface is no longer active */ 1436 ifp->if_flags &= ~(IFF_RUNNING | IFF_OACTIVE); 1437 1438 hw->mac.ops.reset_hw(hw); 1439 adapter->hw.adapter_stopped = FALSE; 1440 hw->mac.ops.stop_adapter(hw); 1441 callout_stop(&adapter->timer); 1442 1443 /* reprogram the RAR[0] in case user changed it. */ 1444 hw->mac.ops.set_rar(hw, 0, hw->mac.addr, 0, IXGBE_RAH_AV); 1445 1446 return; 1447 } /* ixv_stop */ 1448 1449 1450 /************************************************************************ 1451 * ixv_allocate_pci_resources 1452 ************************************************************************/ 1453 static int 1454 ixv_allocate_pci_resources(struct adapter *adapter, 1455 const struct pci_attach_args *pa) 1456 { 1457 pcireg_t memtype, csr; 1458 device_t dev = adapter->dev; 1459 bus_addr_t addr; 1460 int flags; 1461 1462 memtype = pci_mapreg_type(pa->pa_pc, pa->pa_tag, PCI_BAR(0)); 1463 switch (memtype) { 1464 case PCI_MAPREG_TYPE_MEM | PCI_MAPREG_MEM_TYPE_32BIT: 1465 case PCI_MAPREG_TYPE_MEM | PCI_MAPREG_MEM_TYPE_64BIT: 1466 adapter->osdep.mem_bus_space_tag = pa->pa_memt; 1467 if (pci_mapreg_info(pa->pa_pc, pa->pa_tag, PCI_BAR(0), 1468 memtype, &addr, &adapter->osdep.mem_size, &flags) != 0) 1469 goto map_err; 1470 if ((flags & BUS_SPACE_MAP_PREFETCHABLE) != 0) { 1471 aprint_normal_dev(dev, "clearing prefetchable bit\n"); 1472 flags &= ~BUS_SPACE_MAP_PREFETCHABLE; 1473 } 1474 if (bus_space_map(adapter->osdep.mem_bus_space_tag, addr, 1475 adapter->osdep.mem_size, flags, 1476 &adapter->osdep.mem_bus_space_handle) != 0) { 1477 map_err: 1478 adapter->osdep.mem_size = 0; 1479 aprint_error_dev(dev, "unable to map BAR0\n"); 1480 return ENXIO; 1481 } 1482 /* 1483 * Enable address decoding for memory range in case it's not 1484 * set. 1485 */ 1486 csr = pci_conf_read(pa->pa_pc, pa->pa_tag, 1487 PCI_COMMAND_STATUS_REG); 1488 csr |= PCI_COMMAND_MEM_ENABLE; 1489 pci_conf_write(pa->pa_pc, pa->pa_tag, PCI_COMMAND_STATUS_REG, 1490 csr); 1491 break; 1492 default: 1493 aprint_error_dev(dev, "unexpected type on BAR0\n"); 1494 return ENXIO; 1495 } 1496 1497 /* Pick up the tuneable queues */ 1498 adapter->num_queues = ixv_num_queues; 1499 1500 return (0); 1501 } /* ixv_allocate_pci_resources */ 1502 1503 /************************************************************************ 1504 * ixv_free_pci_resources 1505 ************************************************************************/ 1506 static void 1507 ixv_free_pci_resources(struct adapter * adapter) 1508 { 1509 struct ix_queue *que = adapter->queues; 1510 int rid; 1511 1512 /* 1513 * Release all msix queue resources: 1514 */ 1515 for (int i = 0; i < adapter->num_queues; i++, que++) { 1516 if (que->res != NULL) 1517 pci_intr_disestablish(adapter->osdep.pc, 1518 adapter->osdep.ihs[i]); 1519 } 1520 1521 1522 /* Clean the Mailbox interrupt last */ 1523 rid = adapter->vector; 1524 1525 if (adapter->osdep.ihs[rid] != NULL) { 1526 pci_intr_disestablish(adapter->osdep.pc, 1527 adapter->osdep.ihs[rid]); 1528 adapter->osdep.ihs[rid] = NULL; 1529 } 1530 1531 pci_intr_release(adapter->osdep.pc, adapter->osdep.intrs, 1532 adapter->osdep.nintrs); 1533 1534 if (adapter->osdep.mem_size != 0) { 1535 bus_space_unmap(adapter->osdep.mem_bus_space_tag, 1536 adapter->osdep.mem_bus_space_handle, 1537 adapter->osdep.mem_size); 1538 } 1539 1540 return; 1541 } /* ixv_free_pci_resources */ 1542 1543 /************************************************************************ 1544 * ixv_setup_interface 1545 * 1546 * Setup networking device structure and register an interface. 1547 ************************************************************************/ 1548 static int 1549 ixv_setup_interface(device_t dev, struct adapter *adapter) 1550 { 1551 struct ethercom *ec = &adapter->osdep.ec; 1552 struct ifnet *ifp; 1553 int rv; 1554 1555 INIT_DEBUGOUT("ixv_setup_interface: begin"); 1556 1557 ifp = adapter->ifp = &ec->ec_if; 1558 strlcpy(ifp->if_xname, device_xname(dev), IFNAMSIZ); 1559 ifp->if_baudrate = IF_Gbps(10); 1560 ifp->if_init = ixv_init; 1561 ifp->if_stop = ixv_ifstop; 1562 ifp->if_softc = adapter; 1563 ifp->if_flags = IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST; 1564 #ifdef IXGBE_MPSAFE 1565 ifp->if_extflags = IFEF_MPSAFE; 1566 #endif 1567 ifp->if_ioctl = ixv_ioctl; 1568 if (adapter->feat_en & IXGBE_FEATURE_LEGACY_TX) { 1569 #if 0 1570 ixv_start_locked = ixgbe_legacy_start_locked; 1571 #endif 1572 } else { 1573 ifp->if_transmit = ixgbe_mq_start; 1574 #if 0 1575 ixv_start_locked = ixgbe_mq_start_locked; 1576 #endif 1577 } 1578 ifp->if_start = ixgbe_legacy_start; 1579 IFQ_SET_MAXLEN(&ifp->if_snd, adapter->num_tx_desc - 2); 1580 IFQ_SET_READY(&ifp->if_snd); 1581 1582 rv = if_initialize(ifp); 1583 if (rv != 0) { 1584 aprint_error_dev(dev, "if_initialize failed(%d)\n", rv); 1585 return rv; 1586 } 1587 adapter->ipq = if_percpuq_create(&adapter->osdep.ec.ec_if); 1588 ether_ifattach(ifp, adapter->hw.mac.addr); 1589 aprint_normal_dev(dev, "Ethernet address %s\n", 1590 ether_sprintf(adapter->hw.mac.addr)); 1591 /* 1592 * We use per TX queue softint, so if_deferred_start_init() isn't 1593 * used. 1594 */ 1595 ether_set_ifflags_cb(ec, ixv_ifflags_cb); 1596 1597 adapter->max_frame_size = ifp->if_mtu + IXGBE_MTU_HDR; 1598 1599 /* 1600 * Tell the upper layer(s) we support long frames. 1601 */ 1602 ifp->if_hdrlen = sizeof(struct ether_vlan_header); 1603 1604 /* Set capability flags */ 1605 ifp->if_capabilities |= IFCAP_HWCSUM 1606 | IFCAP_TSOv4 1607 | IFCAP_TSOv6; 1608 ifp->if_capenable = 0; 1609 1610 ec->ec_capabilities |= ETHERCAP_VLAN_HWFILTER 1611 | ETHERCAP_VLAN_HWTAGGING 1612 | ETHERCAP_VLAN_HWCSUM 1613 | ETHERCAP_JUMBO_MTU 1614 | ETHERCAP_VLAN_MTU; 1615 1616 /* Enable the above capabilities by default */ 1617 ec->ec_capenable = ec->ec_capabilities; 1618 1619 /* Don't enable LRO by default */ 1620 #if 0 1621 /* NetBSD doesn't support LRO yet */ 1622 ifp->if_capabilities |= IFCAP_LRO; 1623 #endif 1624 1625 /* 1626 * Specify the media types supported by this adapter and register 1627 * callbacks to update media and link information 1628 */ 1629 ec->ec_ifmedia = &adapter->media; 1630 ifmedia_init_with_lock(&adapter->media, IFM_IMASK, ixv_media_change, 1631 ixv_media_status, &adapter->core_mtx); 1632 ifmedia_add(&adapter->media, IFM_ETHER | IFM_AUTO, 0, NULL); 1633 ifmedia_set(&adapter->media, IFM_ETHER | IFM_AUTO); 1634 1635 if_register(ifp); 1636 1637 return 0; 1638 } /* ixv_setup_interface */ 1639 1640 1641 /************************************************************************ 1642 * ixv_initialize_transmit_units - Enable transmit unit. 1643 ************************************************************************/ 1644 static void 1645 ixv_initialize_transmit_units(struct adapter *adapter) 1646 { 1647 struct tx_ring *txr = adapter->tx_rings; 1648 struct ixgbe_hw *hw = &adapter->hw; 1649 int i; 1650 1651 for (i = 0; i < adapter->num_queues; i++, txr++) { 1652 u64 tdba = txr->txdma.dma_paddr; 1653 u32 txctrl, txdctl; 1654 int j = txr->me; 1655 1656 /* Set WTHRESH to 8, burst writeback */ 1657 txdctl = IXGBE_READ_REG(hw, IXGBE_VFTXDCTL(j)); 1658 txdctl |= (8 << 16); 1659 IXGBE_WRITE_REG(hw, IXGBE_VFTXDCTL(j), txdctl); 1660 1661 /* Set the HW Tx Head and Tail indices */ 1662 IXGBE_WRITE_REG(&adapter->hw, IXGBE_VFTDH(j), 0); 1663 IXGBE_WRITE_REG(&adapter->hw, IXGBE_VFTDT(j), 0); 1664 1665 /* Set Tx Tail register */ 1666 txr->tail = IXGBE_VFTDT(j); 1667 1668 txr->txr_no_space = false; 1669 1670 /* Set Ring parameters */ 1671 IXGBE_WRITE_REG(hw, IXGBE_VFTDBAL(j), 1672 (tdba & 0x00000000ffffffffULL)); 1673 IXGBE_WRITE_REG(hw, IXGBE_VFTDBAH(j), (tdba >> 32)); 1674 IXGBE_WRITE_REG(hw, IXGBE_VFTDLEN(j), 1675 adapter->num_tx_desc * sizeof(struct ixgbe_legacy_tx_desc)); 1676 txctrl = IXGBE_READ_REG(hw, IXGBE_VFDCA_TXCTRL(j)); 1677 txctrl &= ~IXGBE_DCA_TXCTRL_DESC_WRO_EN; 1678 IXGBE_WRITE_REG(hw, IXGBE_VFDCA_TXCTRL(j), txctrl); 1679 1680 /* Now enable */ 1681 txdctl = IXGBE_READ_REG(hw, IXGBE_VFTXDCTL(j)); 1682 txdctl |= IXGBE_TXDCTL_ENABLE; 1683 IXGBE_WRITE_REG(hw, IXGBE_VFTXDCTL(j), txdctl); 1684 } 1685 1686 return; 1687 } /* ixv_initialize_transmit_units */ 1688 1689 1690 /************************************************************************ 1691 * ixv_initialize_rss_mapping 1692 ************************************************************************/ 1693 static void 1694 ixv_initialize_rss_mapping(struct adapter *adapter) 1695 { 1696 struct ixgbe_hw *hw = &adapter->hw; 1697 u32 reta = 0, mrqc, rss_key[10]; 1698 int queue_id; 1699 int i, j; 1700 u32 rss_hash_config; 1701 1702 /* force use default RSS key. */ 1703 #ifdef __NetBSD__ 1704 rss_getkey((uint8_t *) &rss_key); 1705 #else 1706 if (adapter->feat_en & IXGBE_FEATURE_RSS) { 1707 /* Fetch the configured RSS key */ 1708 rss_getkey((uint8_t *)&rss_key); 1709 } else { 1710 /* set up random bits */ 1711 cprng_fast(&rss_key, sizeof(rss_key)); 1712 } 1713 #endif 1714 1715 /* Now fill out hash function seeds */ 1716 for (i = 0; i < 10; i++) 1717 IXGBE_WRITE_REG(hw, IXGBE_VFRSSRK(i), rss_key[i]); 1718 1719 /* Set up the redirection table */ 1720 for (i = 0, j = 0; i < 64; i++, j++) { 1721 if (j == adapter->num_queues) 1722 j = 0; 1723 1724 if (adapter->feat_en & IXGBE_FEATURE_RSS) { 1725 /* 1726 * Fetch the RSS bucket id for the given indirection 1727 * entry. Cap it at the number of configured buckets 1728 * (which is num_queues.) 1729 */ 1730 queue_id = rss_get_indirection_to_bucket(i); 1731 queue_id = queue_id % adapter->num_queues; 1732 } else 1733 queue_id = j; 1734 1735 /* 1736 * The low 8 bits are for hash value (n+0); 1737 * The next 8 bits are for hash value (n+1), etc. 1738 */ 1739 reta >>= 8; 1740 reta |= ((uint32_t)queue_id) << 24; 1741 if ((i & 3) == 3) { 1742 IXGBE_WRITE_REG(hw, IXGBE_VFRETA(i >> 2), reta); 1743 reta = 0; 1744 } 1745 } 1746 1747 /* Perform hash on these packet types */ 1748 if (adapter->feat_en & IXGBE_FEATURE_RSS) 1749 rss_hash_config = rss_gethashconfig(); 1750 else { 1751 /* 1752 * Disable UDP - IP fragments aren't currently being handled 1753 * and so we end up with a mix of 2-tuple and 4-tuple 1754 * traffic. 1755 */ 1756 rss_hash_config = RSS_HASHTYPE_RSS_IPV4 1757 | RSS_HASHTYPE_RSS_TCP_IPV4 1758 | RSS_HASHTYPE_RSS_IPV6 1759 | RSS_HASHTYPE_RSS_TCP_IPV6; 1760 } 1761 1762 mrqc = IXGBE_MRQC_RSSEN; 1763 if (rss_hash_config & RSS_HASHTYPE_RSS_IPV4) 1764 mrqc |= IXGBE_MRQC_RSS_FIELD_IPV4; 1765 if (rss_hash_config & RSS_HASHTYPE_RSS_TCP_IPV4) 1766 mrqc |= IXGBE_MRQC_RSS_FIELD_IPV4_TCP; 1767 if (rss_hash_config & RSS_HASHTYPE_RSS_IPV6) 1768 mrqc |= IXGBE_MRQC_RSS_FIELD_IPV6; 1769 if (rss_hash_config & RSS_HASHTYPE_RSS_TCP_IPV6) 1770 mrqc |= IXGBE_MRQC_RSS_FIELD_IPV6_TCP; 1771 if (rss_hash_config & RSS_HASHTYPE_RSS_IPV6_EX) 1772 device_printf(adapter->dev, "%s: RSS_HASHTYPE_RSS_IPV6_EX defined, but not supported\n", 1773 __func__); 1774 if (rss_hash_config & RSS_HASHTYPE_RSS_TCP_IPV6_EX) 1775 device_printf(adapter->dev, "%s: RSS_HASHTYPE_RSS_TCP_IPV6_EX defined, but not supported\n", 1776 __func__); 1777 if (rss_hash_config & RSS_HASHTYPE_RSS_UDP_IPV4) 1778 mrqc |= IXGBE_MRQC_RSS_FIELD_IPV4_UDP; 1779 if (rss_hash_config & RSS_HASHTYPE_RSS_UDP_IPV6) 1780 mrqc |= IXGBE_MRQC_RSS_FIELD_IPV6_UDP; 1781 if (rss_hash_config & RSS_HASHTYPE_RSS_UDP_IPV6_EX) 1782 device_printf(adapter->dev, "%s: RSS_HASHTYPE_RSS_UDP_IPV6_EX defined, but not supported\n", 1783 __func__); 1784 IXGBE_WRITE_REG(hw, IXGBE_VFMRQC, mrqc); 1785 } /* ixv_initialize_rss_mapping */ 1786 1787 1788 /************************************************************************ 1789 * ixv_initialize_receive_units - Setup receive registers and features. 1790 ************************************************************************/ 1791 static void 1792 ixv_initialize_receive_units(struct adapter *adapter) 1793 { 1794 struct rx_ring *rxr = adapter->rx_rings; 1795 struct ixgbe_hw *hw = &adapter->hw; 1796 struct ifnet *ifp = adapter->ifp; 1797 u32 bufsz, psrtype; 1798 1799 if (ifp->if_mtu > ETHERMTU) 1800 bufsz = 4096 >> IXGBE_SRRCTL_BSIZEPKT_SHIFT; 1801 else 1802 bufsz = 2048 >> IXGBE_SRRCTL_BSIZEPKT_SHIFT; 1803 1804 psrtype = IXGBE_PSRTYPE_TCPHDR 1805 | IXGBE_PSRTYPE_UDPHDR 1806 | IXGBE_PSRTYPE_IPV4HDR 1807 | IXGBE_PSRTYPE_IPV6HDR 1808 | IXGBE_PSRTYPE_L2HDR; 1809 1810 if (adapter->num_queues > 1) 1811 psrtype |= 1 << 29; 1812 1813 IXGBE_WRITE_REG(hw, IXGBE_VFPSRTYPE, psrtype); 1814 1815 /* Tell PF our max_frame size */ 1816 if (ixgbevf_rlpml_set_vf(hw, adapter->max_frame_size) != 0) { 1817 device_printf(adapter->dev, "There is a problem with the PF setup. It is likely the receive unit for this VF will not function correctly.\n"); 1818 } 1819 1820 for (int i = 0; i < adapter->num_queues; i++, rxr++) { 1821 u64 rdba = rxr->rxdma.dma_paddr; 1822 u32 reg, rxdctl; 1823 int j = rxr->me; 1824 1825 /* Disable the queue */ 1826 rxdctl = IXGBE_READ_REG(hw, IXGBE_VFRXDCTL(j)); 1827 rxdctl &= ~IXGBE_RXDCTL_ENABLE; 1828 IXGBE_WRITE_REG(hw, IXGBE_VFRXDCTL(j), rxdctl); 1829 for (int k = 0; k < 10; k++) { 1830 if (IXGBE_READ_REG(hw, IXGBE_VFRXDCTL(j)) & 1831 IXGBE_RXDCTL_ENABLE) 1832 msec_delay(1); 1833 else 1834 break; 1835 } 1836 IXGBE_WRITE_BARRIER(hw); 1837 /* Setup the Base and Length of the Rx Descriptor Ring */ 1838 IXGBE_WRITE_REG(hw, IXGBE_VFRDBAL(j), 1839 (rdba & 0x00000000ffffffffULL)); 1840 IXGBE_WRITE_REG(hw, IXGBE_VFRDBAH(j), (rdba >> 32)); 1841 IXGBE_WRITE_REG(hw, IXGBE_VFRDLEN(j), 1842 adapter->num_rx_desc * sizeof(union ixgbe_adv_rx_desc)); 1843 1844 /* Reset the ring indices */ 1845 IXGBE_WRITE_REG(hw, IXGBE_VFRDH(rxr->me), 0); 1846 IXGBE_WRITE_REG(hw, IXGBE_VFRDT(rxr->me), 0); 1847 1848 /* Set up the SRRCTL register */ 1849 reg = IXGBE_READ_REG(hw, IXGBE_VFSRRCTL(j)); 1850 reg &= ~IXGBE_SRRCTL_BSIZEHDR_MASK; 1851 reg &= ~IXGBE_SRRCTL_BSIZEPKT_MASK; 1852 reg |= bufsz; 1853 reg |= IXGBE_SRRCTL_DESCTYPE_ADV_ONEBUF; 1854 IXGBE_WRITE_REG(hw, IXGBE_VFSRRCTL(j), reg); 1855 1856 /* Capture Rx Tail index */ 1857 rxr->tail = IXGBE_VFRDT(rxr->me); 1858 1859 /* Do the queue enabling last */ 1860 rxdctl |= IXGBE_RXDCTL_ENABLE | IXGBE_RXDCTL_VME; 1861 IXGBE_WRITE_REG(hw, IXGBE_VFRXDCTL(j), rxdctl); 1862 for (int k = 0; k < 10; k++) { 1863 if (IXGBE_READ_REG(hw, IXGBE_VFRXDCTL(j)) & 1864 IXGBE_RXDCTL_ENABLE) 1865 break; 1866 msec_delay(1); 1867 } 1868 IXGBE_WRITE_BARRIER(hw); 1869 1870 /* Set the Tail Pointer */ 1871 #ifdef DEV_NETMAP 1872 /* 1873 * In netmap mode, we must preserve the buffers made 1874 * available to userspace before the if_init() 1875 * (this is true by default on the TX side, because 1876 * init makes all buffers available to userspace). 1877 * 1878 * netmap_reset() and the device specific routines 1879 * (e.g. ixgbe_setup_receive_rings()) map these 1880 * buffers at the end of the NIC ring, so here we 1881 * must set the RDT (tail) register to make sure 1882 * they are not overwritten. 1883 * 1884 * In this driver the NIC ring starts at RDH = 0, 1885 * RDT points to the last slot available for reception (?), 1886 * so RDT = num_rx_desc - 1 means the whole ring is available. 1887 */ 1888 if ((adapter->feat_en & IXGBE_FEATURE_NETMAP) && 1889 (ifp->if_capenable & IFCAP_NETMAP)) { 1890 struct netmap_adapter *na = NA(adapter->ifp); 1891 struct netmap_kring *kring = na->rx_rings[i]; 1892 int t = na->num_rx_desc - 1 - nm_kr_rxspace(kring); 1893 1894 IXGBE_WRITE_REG(hw, IXGBE_VFRDT(rxr->me), t); 1895 } else 1896 #endif /* DEV_NETMAP */ 1897 IXGBE_WRITE_REG(hw, IXGBE_VFRDT(rxr->me), 1898 adapter->num_rx_desc - 1); 1899 } 1900 1901 if (adapter->hw.mac.type >= ixgbe_mac_X550_vf) 1902 ixv_initialize_rss_mapping(adapter); 1903 } /* ixv_initialize_receive_units */ 1904 1905 /************************************************************************ 1906 * ixv_sysctl_tdh_handler - Transmit Descriptor Head handler function 1907 * 1908 * Retrieves the TDH value from the hardware 1909 ************************************************************************/ 1910 static int 1911 ixv_sysctl_tdh_handler(SYSCTLFN_ARGS) 1912 { 1913 struct sysctlnode node = *rnode; 1914 struct tx_ring *txr = (struct tx_ring *)node.sysctl_data; 1915 uint32_t val; 1916 1917 if (!txr) 1918 return (0); 1919 1920 val = IXGBE_READ_REG(&txr->adapter->hw, IXGBE_VFTDH(txr->me)); 1921 node.sysctl_data = &val; 1922 return sysctl_lookup(SYSCTLFN_CALL(&node)); 1923 } /* ixv_sysctl_tdh_handler */ 1924 1925 /************************************************************************ 1926 * ixgbe_sysctl_tdt_handler - Transmit Descriptor Tail handler function 1927 * 1928 * Retrieves the TDT value from the hardware 1929 ************************************************************************/ 1930 static int 1931 ixv_sysctl_tdt_handler(SYSCTLFN_ARGS) 1932 { 1933 struct sysctlnode node = *rnode; 1934 struct tx_ring *txr = (struct tx_ring *)node.sysctl_data; 1935 uint32_t val; 1936 1937 if (!txr) 1938 return (0); 1939 1940 val = IXGBE_READ_REG(&txr->adapter->hw, IXGBE_VFTDT(txr->me)); 1941 node.sysctl_data = &val; 1942 return sysctl_lookup(SYSCTLFN_CALL(&node)); 1943 } /* ixv_sysctl_tdt_handler */ 1944 1945 /************************************************************************ 1946 * ixv_sysctl_next_to_check_handler - Receive Descriptor next to check 1947 * handler function 1948 * 1949 * Retrieves the next_to_check value 1950 ************************************************************************/ 1951 static int 1952 ixv_sysctl_next_to_check_handler(SYSCTLFN_ARGS) 1953 { 1954 struct sysctlnode node = *rnode; 1955 struct rx_ring *rxr = (struct rx_ring *)node.sysctl_data; 1956 uint32_t val; 1957 1958 if (!rxr) 1959 return (0); 1960 1961 val = rxr->next_to_check; 1962 node.sysctl_data = &val; 1963 return sysctl_lookup(SYSCTLFN_CALL(&node)); 1964 } /* ixv_sysctl_next_to_check_handler */ 1965 1966 /************************************************************************ 1967 * ixv_sysctl_rdh_handler - Receive Descriptor Head handler function 1968 * 1969 * Retrieves the RDH value from the hardware 1970 ************************************************************************/ 1971 static int 1972 ixv_sysctl_rdh_handler(SYSCTLFN_ARGS) 1973 { 1974 struct sysctlnode node = *rnode; 1975 struct rx_ring *rxr = (struct rx_ring *)node.sysctl_data; 1976 uint32_t val; 1977 1978 if (!rxr) 1979 return (0); 1980 1981 val = IXGBE_READ_REG(&rxr->adapter->hw, IXGBE_VFRDH(rxr->me)); 1982 node.sysctl_data = &val; 1983 return sysctl_lookup(SYSCTLFN_CALL(&node)); 1984 } /* ixv_sysctl_rdh_handler */ 1985 1986 /************************************************************************ 1987 * ixv_sysctl_rdt_handler - Receive Descriptor Tail handler function 1988 * 1989 * Retrieves the RDT value from the hardware 1990 ************************************************************************/ 1991 static int 1992 ixv_sysctl_rdt_handler(SYSCTLFN_ARGS) 1993 { 1994 struct sysctlnode node = *rnode; 1995 struct rx_ring *rxr = (struct rx_ring *)node.sysctl_data; 1996 uint32_t val; 1997 1998 if (!rxr) 1999 return (0); 2000 2001 val = IXGBE_READ_REG(&rxr->adapter->hw, IXGBE_VFRDT(rxr->me)); 2002 node.sysctl_data = &val; 2003 return sysctl_lookup(SYSCTLFN_CALL(&node)); 2004 } /* ixv_sysctl_rdt_handler */ 2005 2006 static void 2007 ixv_setup_vlan_tagging(struct adapter *adapter) 2008 { 2009 struct ethercom *ec = &adapter->osdep.ec; 2010 struct ixgbe_hw *hw = &adapter->hw; 2011 struct rx_ring *rxr; 2012 u32 ctrl; 2013 int i; 2014 bool hwtagging; 2015 2016 /* Enable HW tagging only if any vlan is attached */ 2017 hwtagging = (ec->ec_capenable & ETHERCAP_VLAN_HWTAGGING) 2018 && VLAN_ATTACHED(ec); 2019 2020 /* Enable the queues */ 2021 for (i = 0; i < adapter->num_queues; i++) { 2022 rxr = &adapter->rx_rings[i]; 2023 ctrl = IXGBE_READ_REG(hw, IXGBE_VFRXDCTL(rxr->me)); 2024 if (hwtagging) 2025 ctrl |= IXGBE_RXDCTL_VME; 2026 else 2027 ctrl &= ~IXGBE_RXDCTL_VME; 2028 IXGBE_WRITE_REG(hw, IXGBE_VFRXDCTL(rxr->me), ctrl); 2029 /* 2030 * Let Rx path know that it needs to store VLAN tag 2031 * as part of extra mbuf info. 2032 */ 2033 rxr->vtag_strip = hwtagging ? TRUE : FALSE; 2034 } 2035 } /* ixv_setup_vlan_tagging */ 2036 2037 /************************************************************************ 2038 * ixv_setup_vlan_support 2039 ************************************************************************/ 2040 static int 2041 ixv_setup_vlan_support(struct adapter *adapter) 2042 { 2043 struct ethercom *ec = &adapter->osdep.ec; 2044 struct ixgbe_hw *hw = &adapter->hw; 2045 u32 vid, vfta, retry; 2046 struct vlanid_list *vlanidp; 2047 int rv, error = 0; 2048 2049 /* 2050 * This function is called from both if_init and ifflags_cb() 2051 * on NetBSD. 2052 */ 2053 2054 /* 2055 * Part 1: 2056 * Setup VLAN HW tagging 2057 */ 2058 ixv_setup_vlan_tagging(adapter); 2059 2060 if (!VLAN_ATTACHED(ec)) 2061 return 0; 2062 2063 /* 2064 * Part 2: 2065 * Setup VLAN HW filter 2066 */ 2067 /* Cleanup shadow_vfta */ 2068 for (int i = 0; i < IXGBE_VFTA_SIZE; i++) 2069 adapter->shadow_vfta[i] = 0; 2070 /* Generate shadow_vfta from ec_vids */ 2071 ETHER_LOCK(ec); 2072 SIMPLEQ_FOREACH(vlanidp, &ec->ec_vids, vid_list) { 2073 uint32_t idx; 2074 2075 idx = vlanidp->vid / 32; 2076 KASSERT(idx < IXGBE_VFTA_SIZE); 2077 adapter->shadow_vfta[idx] |= (u32)1 << (vlanidp->vid % 32); 2078 } 2079 ETHER_UNLOCK(ec); 2080 2081 /* 2082 * A soft reset zero's out the VFTA, so 2083 * we need to repopulate it now. 2084 */ 2085 for (int i = 0; i < IXGBE_VFTA_SIZE; i++) { 2086 if (adapter->shadow_vfta[i] == 0) 2087 continue; 2088 vfta = adapter->shadow_vfta[i]; 2089 /* 2090 * Reconstruct the vlan id's 2091 * based on the bits set in each 2092 * of the array ints. 2093 */ 2094 for (int j = 0; j < 32; j++) { 2095 retry = 0; 2096 if ((vfta & ((u32)1 << j)) == 0) 2097 continue; 2098 vid = (i * 32) + j; 2099 2100 /* Call the shared code mailbox routine */ 2101 while ((rv = hw->mac.ops.set_vfta(hw, vid, 0, TRUE, 2102 FALSE)) != 0) { 2103 if (++retry > 5) { 2104 device_printf(adapter->dev, 2105 "%s: max retry exceeded\n", 2106 __func__); 2107 break; 2108 } 2109 } 2110 if (rv != 0) { 2111 device_printf(adapter->dev, 2112 "failed to set vlan %d\n", vid); 2113 error = EACCES; 2114 } 2115 } 2116 } 2117 return error; 2118 } /* ixv_setup_vlan_support */ 2119 2120 static int 2121 ixv_vlan_cb(struct ethercom *ec, uint16_t vid, bool set) 2122 { 2123 struct ifnet *ifp = &ec->ec_if; 2124 struct adapter *adapter = ifp->if_softc; 2125 int rv; 2126 2127 if (set) 2128 rv = ixv_register_vlan(adapter, vid); 2129 else 2130 rv = ixv_unregister_vlan(adapter, vid); 2131 2132 if (rv != 0) 2133 return rv; 2134 2135 /* 2136 * Control VLAN HW tagging when ec_nvlan is changed from 1 to 0 2137 * or 0 to 1. 2138 */ 2139 if ((set && (ec->ec_nvlans == 1)) || (!set && (ec->ec_nvlans == 0))) 2140 ixv_setup_vlan_tagging(adapter); 2141 2142 return rv; 2143 } 2144 2145 /************************************************************************ 2146 * ixv_register_vlan 2147 * 2148 * Run via a vlan config EVENT, it enables us to use the 2149 * HW Filter table since we can get the vlan id. This just 2150 * creates the entry in the soft version of the VFTA, init 2151 * will repopulate the real table. 2152 ************************************************************************/ 2153 static int 2154 ixv_register_vlan(struct adapter *adapter, u16 vtag) 2155 { 2156 struct ixgbe_hw *hw = &adapter->hw; 2157 u16 index, bit; 2158 int error; 2159 2160 if ((vtag == 0) || (vtag > 4095)) /* Invalid */ 2161 return EINVAL; 2162 IXGBE_CORE_LOCK(adapter); 2163 index = (vtag >> 5) & 0x7F; 2164 bit = vtag & 0x1F; 2165 adapter->shadow_vfta[index] |= ((u32)1 << bit); 2166 error = hw->mac.ops.set_vfta(hw, vtag, 0, true, false); 2167 IXGBE_CORE_UNLOCK(adapter); 2168 2169 if (error != 0) { 2170 device_printf(adapter->dev, "failed to register vlan %hu\n", 2171 vtag); 2172 error = EACCES; 2173 } 2174 return error; 2175 } /* ixv_register_vlan */ 2176 2177 /************************************************************************ 2178 * ixv_unregister_vlan 2179 * 2180 * Run via a vlan unconfig EVENT, remove our entry 2181 * in the soft vfta. 2182 ************************************************************************/ 2183 static int 2184 ixv_unregister_vlan(struct adapter *adapter, u16 vtag) 2185 { 2186 struct ixgbe_hw *hw = &adapter->hw; 2187 u16 index, bit; 2188 int error; 2189 2190 if ((vtag == 0) || (vtag > 4095)) /* Invalid */ 2191 return EINVAL; 2192 2193 IXGBE_CORE_LOCK(adapter); 2194 index = (vtag >> 5) & 0x7F; 2195 bit = vtag & 0x1F; 2196 adapter->shadow_vfta[index] &= ~((u32)1 << bit); 2197 error = hw->mac.ops.set_vfta(hw, vtag, 0, false, false); 2198 IXGBE_CORE_UNLOCK(adapter); 2199 2200 if (error != 0) { 2201 device_printf(adapter->dev, "failed to unregister vlan %hu\n", 2202 vtag); 2203 error = EIO; 2204 } 2205 return error; 2206 } /* ixv_unregister_vlan */ 2207 2208 /************************************************************************ 2209 * ixv_enable_intr 2210 ************************************************************************/ 2211 static void 2212 ixv_enable_intr(struct adapter *adapter) 2213 { 2214 struct ixgbe_hw *hw = &adapter->hw; 2215 struct ix_queue *que = adapter->queues; 2216 u32 mask; 2217 int i; 2218 2219 /* For VTEIAC */ 2220 mask = (1 << adapter->vector); 2221 for (i = 0; i < adapter->num_queues; i++, que++) 2222 mask |= (1 << que->msix); 2223 IXGBE_WRITE_REG(hw, IXGBE_VTEIAC, mask); 2224 2225 /* For VTEIMS */ 2226 IXGBE_WRITE_REG(hw, IXGBE_VTEIMS, (1 << adapter->vector)); 2227 que = adapter->queues; 2228 for (i = 0; i < adapter->num_queues; i++, que++) 2229 ixv_enable_queue(adapter, que->msix); 2230 2231 IXGBE_WRITE_FLUSH(hw); 2232 } /* ixv_enable_intr */ 2233 2234 /************************************************************************ 2235 * ixv_disable_intr 2236 ************************************************************************/ 2237 static void 2238 ixv_disable_intr(struct adapter *adapter) 2239 { 2240 struct ix_queue *que = adapter->queues; 2241 2242 IXGBE_WRITE_REG(&adapter->hw, IXGBE_VTEIAC, 0); 2243 2244 /* disable interrupts other than queues */ 2245 IXGBE_WRITE_REG(&adapter->hw, IXGBE_VTEIMC, adapter->vector); 2246 2247 for (int i = 0; i < adapter->num_queues; i++, que++) 2248 ixv_disable_queue(adapter, que->msix); 2249 2250 IXGBE_WRITE_FLUSH(&adapter->hw); 2251 } /* ixv_disable_intr */ 2252 2253 /************************************************************************ 2254 * ixv_set_ivar 2255 * 2256 * Setup the correct IVAR register for a particular MSI-X interrupt 2257 * - entry is the register array entry 2258 * - vector is the MSI-X vector for this queue 2259 * - type is RX/TX/MISC 2260 ************************************************************************/ 2261 static void 2262 ixv_set_ivar(struct adapter *adapter, u8 entry, u8 vector, s8 type) 2263 { 2264 struct ixgbe_hw *hw = &adapter->hw; 2265 u32 ivar, index; 2266 2267 vector |= IXGBE_IVAR_ALLOC_VAL; 2268 2269 if (type == -1) { /* MISC IVAR */ 2270 ivar = IXGBE_READ_REG(hw, IXGBE_VTIVAR_MISC); 2271 ivar &= ~0xFF; 2272 ivar |= vector; 2273 IXGBE_WRITE_REG(hw, IXGBE_VTIVAR_MISC, ivar); 2274 } else { /* RX/TX IVARS */ 2275 index = (16 * (entry & 1)) + (8 * type); 2276 ivar = IXGBE_READ_REG(hw, IXGBE_VTIVAR(entry >> 1)); 2277 ivar &= ~(0xffUL << index); 2278 ivar |= ((u32)vector << index); 2279 IXGBE_WRITE_REG(hw, IXGBE_VTIVAR(entry >> 1), ivar); 2280 } 2281 } /* ixv_set_ivar */ 2282 2283 /************************************************************************ 2284 * ixv_configure_ivars 2285 ************************************************************************/ 2286 static void 2287 ixv_configure_ivars(struct adapter *adapter) 2288 { 2289 struct ix_queue *que = adapter->queues; 2290 2291 /* XXX We should sync EITR value calculation with ixgbe.c? */ 2292 2293 for (int i = 0; i < adapter->num_queues; i++, que++) { 2294 /* First the RX queue entry */ 2295 ixv_set_ivar(adapter, i, que->msix, 0); 2296 /* ... and the TX */ 2297 ixv_set_ivar(adapter, i, que->msix, 1); 2298 /* Set an initial value in EITR */ 2299 ixv_eitr_write(adapter, que->msix, IXGBE_EITR_DEFAULT); 2300 } 2301 2302 /* For the mailbox interrupt */ 2303 ixv_set_ivar(adapter, 1, adapter->vector, -1); 2304 } /* ixv_configure_ivars */ 2305 2306 2307 /************************************************************************ 2308 * ixv_save_stats 2309 * 2310 * The VF stats registers never have a truly virgin 2311 * starting point, so this routine tries to make an 2312 * artificial one, marking ground zero on attach as 2313 * it were. 2314 ************************************************************************/ 2315 static void 2316 ixv_save_stats(struct adapter *adapter) 2317 { 2318 struct ixgbevf_hw_stats *stats = &adapter->stats.vf; 2319 2320 if (stats->vfgprc.ev_count || stats->vfgptc.ev_count) { 2321 stats->saved_reset_vfgprc += 2322 stats->vfgprc.ev_count - stats->base_vfgprc; 2323 stats->saved_reset_vfgptc += 2324 stats->vfgptc.ev_count - stats->base_vfgptc; 2325 stats->saved_reset_vfgorc += 2326 stats->vfgorc.ev_count - stats->base_vfgorc; 2327 stats->saved_reset_vfgotc += 2328 stats->vfgotc.ev_count - stats->base_vfgotc; 2329 stats->saved_reset_vfmprc += 2330 stats->vfmprc.ev_count - stats->base_vfmprc; 2331 } 2332 } /* ixv_save_stats */ 2333 2334 /************************************************************************ 2335 * ixv_init_stats 2336 ************************************************************************/ 2337 static void 2338 ixv_init_stats(struct adapter *adapter) 2339 { 2340 struct ixgbe_hw *hw = &adapter->hw; 2341 2342 adapter->stats.vf.last_vfgprc = IXGBE_READ_REG(hw, IXGBE_VFGPRC); 2343 adapter->stats.vf.last_vfgorc = IXGBE_READ_REG(hw, IXGBE_VFGORC_LSB); 2344 adapter->stats.vf.last_vfgorc |= 2345 (((u64)(IXGBE_READ_REG(hw, IXGBE_VFGORC_MSB))) << 32); 2346 2347 adapter->stats.vf.last_vfgptc = IXGBE_READ_REG(hw, IXGBE_VFGPTC); 2348 adapter->stats.vf.last_vfgotc = IXGBE_READ_REG(hw, IXGBE_VFGOTC_LSB); 2349 adapter->stats.vf.last_vfgotc |= 2350 (((u64)(IXGBE_READ_REG(hw, IXGBE_VFGOTC_MSB))) << 32); 2351 2352 adapter->stats.vf.last_vfmprc = IXGBE_READ_REG(hw, IXGBE_VFMPRC); 2353 2354 adapter->stats.vf.base_vfgprc = adapter->stats.vf.last_vfgprc; 2355 adapter->stats.vf.base_vfgorc = adapter->stats.vf.last_vfgorc; 2356 adapter->stats.vf.base_vfgptc = adapter->stats.vf.last_vfgptc; 2357 adapter->stats.vf.base_vfgotc = adapter->stats.vf.last_vfgotc; 2358 adapter->stats.vf.base_vfmprc = adapter->stats.vf.last_vfmprc; 2359 } /* ixv_init_stats */ 2360 2361 #define UPDATE_STAT_32(reg, last, count) \ 2362 { \ 2363 u32 current = IXGBE_READ_REG(hw, (reg)); \ 2364 if (current < (last)) \ 2365 count.ev_count += 0x100000000LL; \ 2366 (last) = current; \ 2367 count.ev_count &= 0xFFFFFFFF00000000LL; \ 2368 count.ev_count |= current; \ 2369 } 2370 2371 #define UPDATE_STAT_36(lsb, msb, last, count) \ 2372 { \ 2373 u64 cur_lsb = IXGBE_READ_REG(hw, (lsb)); \ 2374 u64 cur_msb = IXGBE_READ_REG(hw, (msb)); \ 2375 u64 current = ((cur_msb << 32) | cur_lsb); \ 2376 if (current < (last)) \ 2377 count.ev_count += 0x1000000000LL; \ 2378 (last) = current; \ 2379 count.ev_count &= 0xFFFFFFF000000000LL; \ 2380 count.ev_count |= current; \ 2381 } 2382 2383 /************************************************************************ 2384 * ixv_update_stats - Update the board statistics counters. 2385 ************************************************************************/ 2386 void 2387 ixv_update_stats(struct adapter *adapter) 2388 { 2389 struct ixgbe_hw *hw = &adapter->hw; 2390 struct ixgbevf_hw_stats *stats = &adapter->stats.vf; 2391 2392 UPDATE_STAT_32(IXGBE_VFGPRC, stats->last_vfgprc, stats->vfgprc); 2393 UPDATE_STAT_32(IXGBE_VFGPTC, stats->last_vfgptc, stats->vfgptc); 2394 UPDATE_STAT_36(IXGBE_VFGORC_LSB, IXGBE_VFGORC_MSB, stats->last_vfgorc, 2395 stats->vfgorc); 2396 UPDATE_STAT_36(IXGBE_VFGOTC_LSB, IXGBE_VFGOTC_MSB, stats->last_vfgotc, 2397 stats->vfgotc); 2398 UPDATE_STAT_32(IXGBE_VFMPRC, stats->last_vfmprc, stats->vfmprc); 2399 2400 /* VF doesn't count errors by hardware */ 2401 2402 } /* ixv_update_stats */ 2403 2404 /************************************************************************ 2405 * ixv_sysctl_interrupt_rate_handler 2406 ************************************************************************/ 2407 static int 2408 ixv_sysctl_interrupt_rate_handler(SYSCTLFN_ARGS) 2409 { 2410 struct sysctlnode node = *rnode; 2411 struct ix_queue *que = (struct ix_queue *)node.sysctl_data; 2412 struct adapter *adapter = que->adapter; 2413 uint32_t reg, usec, rate; 2414 int error; 2415 2416 if (que == NULL) 2417 return 0; 2418 reg = IXGBE_READ_REG(&que->adapter->hw, IXGBE_VTEITR(que->msix)); 2419 usec = ((reg & 0x0FF8) >> 3); 2420 if (usec > 0) 2421 rate = 500000 / usec; 2422 else 2423 rate = 0; 2424 node.sysctl_data = &rate; 2425 error = sysctl_lookup(SYSCTLFN_CALL(&node)); 2426 if (error || newp == NULL) 2427 return error; 2428 reg &= ~0xfff; /* default, no limitation */ 2429 if (rate > 0 && rate < 500000) { 2430 if (rate < 1000) 2431 rate = 1000; 2432 reg |= ((4000000 / rate) & 0xff8); 2433 /* 2434 * When RSC is used, ITR interval must be larger than 2435 * RSC_DELAY. Currently, we use 2us for RSC_DELAY. 2436 * The minimum value is always greater than 2us on 100M 2437 * (and 10M?(not documented)), but it's not on 1G and higher. 2438 */ 2439 if ((adapter->link_speed != IXGBE_LINK_SPEED_100_FULL) 2440 && (adapter->link_speed != IXGBE_LINK_SPEED_10_FULL)) { 2441 if ((adapter->num_queues > 1) 2442 && (reg < IXGBE_MIN_RSC_EITR_10G1G)) 2443 return EINVAL; 2444 } 2445 ixv_max_interrupt_rate = rate; 2446 } else 2447 ixv_max_interrupt_rate = 0; 2448 ixv_eitr_write(adapter, que->msix, reg); 2449 2450 return (0); 2451 } /* ixv_sysctl_interrupt_rate_handler */ 2452 2453 const struct sysctlnode * 2454 ixv_sysctl_instance(struct adapter *adapter) 2455 { 2456 const char *dvname; 2457 struct sysctllog **log; 2458 int rc; 2459 const struct sysctlnode *rnode; 2460 2461 log = &adapter->sysctllog; 2462 dvname = device_xname(adapter->dev); 2463 2464 if ((rc = sysctl_createv(log, 0, NULL, &rnode, 2465 0, CTLTYPE_NODE, dvname, 2466 SYSCTL_DESCR("ixv information and settings"), 2467 NULL, 0, NULL, 0, CTL_HW, CTL_CREATE, CTL_EOL)) != 0) 2468 goto err; 2469 2470 return rnode; 2471 err: 2472 device_printf(adapter->dev, 2473 "%s: sysctl_createv failed, rc = %d\n", __func__, rc); 2474 return NULL; 2475 } 2476 2477 static void 2478 ixv_add_device_sysctls(struct adapter *adapter) 2479 { 2480 struct sysctllog **log; 2481 const struct sysctlnode *rnode, *cnode; 2482 device_t dev; 2483 2484 dev = adapter->dev; 2485 log = &adapter->sysctllog; 2486 2487 if ((rnode = ixv_sysctl_instance(adapter)) == NULL) { 2488 aprint_error_dev(dev, "could not create sysctl root\n"); 2489 return; 2490 } 2491 2492 if (sysctl_createv(log, 0, &rnode, &cnode, 2493 CTLFLAG_READWRITE, CTLTYPE_INT, 2494 "debug", SYSCTL_DESCR("Debug Info"), 2495 ixv_sysctl_debug, 0, (void *)adapter, 0, CTL_CREATE, CTL_EOL) != 0) 2496 aprint_error_dev(dev, "could not create sysctl\n"); 2497 2498 if (sysctl_createv(log, 0, &rnode, &cnode, 2499 CTLFLAG_READWRITE, CTLTYPE_BOOL, 2500 "enable_aim", SYSCTL_DESCR("Interrupt Moderation"), 2501 NULL, 0, &adapter->enable_aim, 0, CTL_CREATE, CTL_EOL) != 0) 2502 aprint_error_dev(dev, "could not create sysctl\n"); 2503 2504 if (sysctl_createv(log, 0, &rnode, &cnode, 2505 CTLFLAG_READWRITE, CTLTYPE_BOOL, 2506 "txrx_workqueue", SYSCTL_DESCR("Use workqueue for packet processing"), 2507 NULL, 0, &adapter->txrx_use_workqueue, 0, CTL_CREATE, CTL_EOL) != 0) 2508 aprint_error_dev(dev, "could not create sysctl\n"); 2509 } 2510 2511 /************************************************************************ 2512 * ixv_add_stats_sysctls - Add statistic sysctls for the VF. 2513 ************************************************************************/ 2514 static void 2515 ixv_add_stats_sysctls(struct adapter *adapter) 2516 { 2517 device_t dev = adapter->dev; 2518 struct tx_ring *txr = adapter->tx_rings; 2519 struct rx_ring *rxr = adapter->rx_rings; 2520 struct ixgbevf_hw_stats *stats = &adapter->stats.vf; 2521 struct ixgbe_hw *hw = &adapter->hw; 2522 const struct sysctlnode *rnode, *cnode; 2523 struct sysctllog **log = &adapter->sysctllog; 2524 const char *xname = device_xname(dev); 2525 2526 /* Driver Statistics */ 2527 evcnt_attach_dynamic(&adapter->efbig_tx_dma_setup, EVCNT_TYPE_MISC, 2528 NULL, xname, "Driver tx dma soft fail EFBIG"); 2529 evcnt_attach_dynamic(&adapter->mbuf_defrag_failed, EVCNT_TYPE_MISC, 2530 NULL, xname, "m_defrag() failed"); 2531 evcnt_attach_dynamic(&adapter->efbig2_tx_dma_setup, EVCNT_TYPE_MISC, 2532 NULL, xname, "Driver tx dma hard fail EFBIG"); 2533 evcnt_attach_dynamic(&adapter->einval_tx_dma_setup, EVCNT_TYPE_MISC, 2534 NULL, xname, "Driver tx dma hard fail EINVAL"); 2535 evcnt_attach_dynamic(&adapter->other_tx_dma_setup, EVCNT_TYPE_MISC, 2536 NULL, xname, "Driver tx dma hard fail other"); 2537 evcnt_attach_dynamic(&adapter->eagain_tx_dma_setup, EVCNT_TYPE_MISC, 2538 NULL, xname, "Driver tx dma soft fail EAGAIN"); 2539 evcnt_attach_dynamic(&adapter->enomem_tx_dma_setup, EVCNT_TYPE_MISC, 2540 NULL, xname, "Driver tx dma soft fail ENOMEM"); 2541 evcnt_attach_dynamic(&adapter->watchdog_events, EVCNT_TYPE_MISC, 2542 NULL, xname, "Watchdog timeouts"); 2543 evcnt_attach_dynamic(&adapter->tso_err, EVCNT_TYPE_MISC, 2544 NULL, xname, "TSO errors"); 2545 evcnt_attach_dynamic(&adapter->link_irq, EVCNT_TYPE_INTR, 2546 NULL, xname, "Link MSI-X IRQ Handled"); 2547 2548 for (int i = 0; i < adapter->num_queues; i++, rxr++, txr++) { 2549 snprintf(adapter->queues[i].evnamebuf, 2550 sizeof(adapter->queues[i].evnamebuf), "%s q%d", 2551 xname, i); 2552 snprintf(adapter->queues[i].namebuf, 2553 sizeof(adapter->queues[i].namebuf), "q%d", i); 2554 2555 if ((rnode = ixv_sysctl_instance(adapter)) == NULL) { 2556 aprint_error_dev(dev, "could not create sysctl root\n"); 2557 break; 2558 } 2559 2560 if (sysctl_createv(log, 0, &rnode, &rnode, 2561 0, CTLTYPE_NODE, 2562 adapter->queues[i].namebuf, SYSCTL_DESCR("Queue Name"), 2563 NULL, 0, NULL, 0, CTL_CREATE, CTL_EOL) != 0) 2564 break; 2565 2566 if (sysctl_createv(log, 0, &rnode, &cnode, 2567 CTLFLAG_READWRITE, CTLTYPE_INT, 2568 "interrupt_rate", SYSCTL_DESCR("Interrupt Rate"), 2569 ixv_sysctl_interrupt_rate_handler, 0, 2570 (void *)&adapter->queues[i], 0, CTL_CREATE, CTL_EOL) != 0) 2571 break; 2572 2573 if (sysctl_createv(log, 0, &rnode, &cnode, 2574 CTLFLAG_READONLY, CTLTYPE_INT, 2575 "txd_head", SYSCTL_DESCR("Transmit Descriptor Head"), 2576 ixv_sysctl_tdh_handler, 0, (void *)txr, 2577 0, CTL_CREATE, CTL_EOL) != 0) 2578 break; 2579 2580 if (sysctl_createv(log, 0, &rnode, &cnode, 2581 CTLFLAG_READONLY, CTLTYPE_INT, 2582 "txd_tail", SYSCTL_DESCR("Transmit Descriptor Tail"), 2583 ixv_sysctl_tdt_handler, 0, (void *)txr, 2584 0, CTL_CREATE, CTL_EOL) != 0) 2585 break; 2586 2587 evcnt_attach_dynamic(&adapter->queues[i].irqs, EVCNT_TYPE_INTR, 2588 NULL, adapter->queues[i].evnamebuf, "IRQs on queue"); 2589 evcnt_attach_dynamic(&adapter->queues[i].handleq, 2590 EVCNT_TYPE_MISC, NULL, adapter->queues[i].evnamebuf, 2591 "Handled queue in softint"); 2592 evcnt_attach_dynamic(&adapter->queues[i].req, EVCNT_TYPE_MISC, 2593 NULL, adapter->queues[i].evnamebuf, "Requeued in softint"); 2594 evcnt_attach_dynamic(&txr->tso_tx, EVCNT_TYPE_MISC, 2595 NULL, adapter->queues[i].evnamebuf, "TSO"); 2596 evcnt_attach_dynamic(&txr->no_desc_avail, EVCNT_TYPE_MISC, 2597 NULL, adapter->queues[i].evnamebuf, 2598 "Queue No Descriptor Available"); 2599 evcnt_attach_dynamic(&txr->total_packets, EVCNT_TYPE_MISC, 2600 NULL, adapter->queues[i].evnamebuf, 2601 "Queue Packets Transmitted"); 2602 #ifndef IXGBE_LEGACY_TX 2603 evcnt_attach_dynamic(&txr->pcq_drops, EVCNT_TYPE_MISC, 2604 NULL, adapter->queues[i].evnamebuf, 2605 "Packets dropped in pcq"); 2606 #endif 2607 2608 #ifdef LRO 2609 struct lro_ctrl *lro = &rxr->lro; 2610 #endif /* LRO */ 2611 2612 if (sysctl_createv(log, 0, &rnode, &cnode, 2613 CTLFLAG_READONLY, 2614 CTLTYPE_INT, 2615 "rxd_nxck", SYSCTL_DESCR("Receive Descriptor next to check"), 2616 ixv_sysctl_next_to_check_handler, 0, (void *)rxr, 0, 2617 CTL_CREATE, CTL_EOL) != 0) 2618 break; 2619 2620 if (sysctl_createv(log, 0, &rnode, &cnode, 2621 CTLFLAG_READONLY, 2622 CTLTYPE_INT, 2623 "rxd_head", SYSCTL_DESCR("Receive Descriptor Head"), 2624 ixv_sysctl_rdh_handler, 0, (void *)rxr, 0, 2625 CTL_CREATE, CTL_EOL) != 0) 2626 break; 2627 2628 if (sysctl_createv(log, 0, &rnode, &cnode, 2629 CTLFLAG_READONLY, 2630 CTLTYPE_INT, 2631 "rxd_tail", SYSCTL_DESCR("Receive Descriptor Tail"), 2632 ixv_sysctl_rdt_handler, 0, (void *)rxr, 0, 2633 CTL_CREATE, CTL_EOL) != 0) 2634 break; 2635 2636 evcnt_attach_dynamic(&rxr->rx_packets, EVCNT_TYPE_MISC, 2637 NULL, adapter->queues[i].evnamebuf, "Queue Packets Received"); 2638 evcnt_attach_dynamic(&rxr->rx_bytes, EVCNT_TYPE_MISC, 2639 NULL, adapter->queues[i].evnamebuf, "Queue Bytes Received"); 2640 evcnt_attach_dynamic(&rxr->rx_copies, EVCNT_TYPE_MISC, 2641 NULL, adapter->queues[i].evnamebuf, "Copied RX Frames"); 2642 evcnt_attach_dynamic(&rxr->no_jmbuf, EVCNT_TYPE_MISC, 2643 NULL, adapter->queues[i].evnamebuf, "Rx no jumbo mbuf"); 2644 evcnt_attach_dynamic(&rxr->rx_discarded, EVCNT_TYPE_MISC, 2645 NULL, adapter->queues[i].evnamebuf, "Rx discarded"); 2646 #ifdef LRO 2647 SYSCTL_ADD_INT(ctx, queue_list, OID_AUTO, "lro_queued", 2648 CTLFLAG_RD, &lro->lro_queued, 0, 2649 "LRO Queued"); 2650 SYSCTL_ADD_INT(ctx, queue_list, OID_AUTO, "lro_flushed", 2651 CTLFLAG_RD, &lro->lro_flushed, 0, 2652 "LRO Flushed"); 2653 #endif /* LRO */ 2654 } 2655 2656 /* MAC stats get their own sub node */ 2657 2658 snprintf(stats->namebuf, 2659 sizeof(stats->namebuf), "%s MAC Statistics", xname); 2660 2661 evcnt_attach_dynamic(&stats->ipcs, EVCNT_TYPE_MISC, NULL, 2662 stats->namebuf, "rx csum offload - IP"); 2663 evcnt_attach_dynamic(&stats->l4cs, EVCNT_TYPE_MISC, NULL, 2664 stats->namebuf, "rx csum offload - L4"); 2665 evcnt_attach_dynamic(&stats->ipcs_bad, EVCNT_TYPE_MISC, NULL, 2666 stats->namebuf, "rx csum offload - IP bad"); 2667 evcnt_attach_dynamic(&stats->l4cs_bad, EVCNT_TYPE_MISC, NULL, 2668 stats->namebuf, "rx csum offload - L4 bad"); 2669 2670 /* Packet Reception Stats */ 2671 evcnt_attach_dynamic(&stats->vfgprc, EVCNT_TYPE_MISC, NULL, 2672 xname, "Good Packets Received"); 2673 evcnt_attach_dynamic(&stats->vfgorc, EVCNT_TYPE_MISC, NULL, 2674 xname, "Good Octets Received"); 2675 evcnt_attach_dynamic(&stats->vfmprc, EVCNT_TYPE_MISC, NULL, 2676 xname, "Multicast Packets Received"); 2677 evcnt_attach_dynamic(&stats->vfgptc, EVCNT_TYPE_MISC, NULL, 2678 xname, "Good Packets Transmitted"); 2679 evcnt_attach_dynamic(&stats->vfgotc, EVCNT_TYPE_MISC, NULL, 2680 xname, "Good Octets Transmitted"); 2681 2682 /* Mailbox Stats */ 2683 evcnt_attach_dynamic(&hw->mbx.stats.msgs_tx, EVCNT_TYPE_MISC, NULL, 2684 xname, "message TXs"); 2685 evcnt_attach_dynamic(&hw->mbx.stats.msgs_rx, EVCNT_TYPE_MISC, NULL, 2686 xname, "message RXs"); 2687 evcnt_attach_dynamic(&hw->mbx.stats.acks, EVCNT_TYPE_MISC, NULL, 2688 xname, "ACKs"); 2689 evcnt_attach_dynamic(&hw->mbx.stats.reqs, EVCNT_TYPE_MISC, NULL, 2690 xname, "REQs"); 2691 evcnt_attach_dynamic(&hw->mbx.stats.rsts, EVCNT_TYPE_MISC, NULL, 2692 xname, "RSTs"); 2693 2694 } /* ixv_add_stats_sysctls */ 2695 2696 static void 2697 ixv_clear_evcnt(struct adapter *adapter) 2698 { 2699 struct tx_ring *txr = adapter->tx_rings; 2700 struct rx_ring *rxr = adapter->rx_rings; 2701 struct ixgbevf_hw_stats *stats = &adapter->stats.vf; 2702 struct ixgbe_hw *hw = &adapter->hw; 2703 int i; 2704 2705 /* Driver Statistics */ 2706 adapter->efbig_tx_dma_setup.ev_count = 0; 2707 adapter->mbuf_defrag_failed.ev_count = 0; 2708 adapter->efbig2_tx_dma_setup.ev_count = 0; 2709 adapter->einval_tx_dma_setup.ev_count = 0; 2710 adapter->other_tx_dma_setup.ev_count = 0; 2711 adapter->eagain_tx_dma_setup.ev_count = 0; 2712 adapter->enomem_tx_dma_setup.ev_count = 0; 2713 adapter->watchdog_events.ev_count = 0; 2714 adapter->tso_err.ev_count = 0; 2715 adapter->link_irq.ev_count = 0; 2716 2717 for (i = 0; i < adapter->num_queues; i++, rxr++, txr++) { 2718 adapter->queues[i].irqs.ev_count = 0; 2719 adapter->queues[i].handleq.ev_count = 0; 2720 adapter->queues[i].req.ev_count = 0; 2721 txr->tso_tx.ev_count = 0; 2722 txr->no_desc_avail.ev_count = 0; 2723 txr->total_packets.ev_count = 0; 2724 #ifndef IXGBE_LEGACY_TX 2725 txr->pcq_drops.ev_count = 0; 2726 #endif 2727 txr->q_efbig_tx_dma_setup = 0; 2728 txr->q_mbuf_defrag_failed = 0; 2729 txr->q_efbig2_tx_dma_setup = 0; 2730 txr->q_einval_tx_dma_setup = 0; 2731 txr->q_other_tx_dma_setup = 0; 2732 txr->q_eagain_tx_dma_setup = 0; 2733 txr->q_enomem_tx_dma_setup = 0; 2734 txr->q_tso_err = 0; 2735 2736 rxr->rx_packets.ev_count = 0; 2737 rxr->rx_bytes.ev_count = 0; 2738 rxr->rx_copies.ev_count = 0; 2739 rxr->no_jmbuf.ev_count = 0; 2740 rxr->rx_discarded.ev_count = 0; 2741 } 2742 2743 /* MAC stats get their own sub node */ 2744 2745 stats->ipcs.ev_count = 0; 2746 stats->l4cs.ev_count = 0; 2747 stats->ipcs_bad.ev_count = 0; 2748 stats->l4cs_bad.ev_count = 0; 2749 2750 /* Packet Reception Stats */ 2751 stats->vfgprc.ev_count = 0; 2752 stats->vfgorc.ev_count = 0; 2753 stats->vfmprc.ev_count = 0; 2754 stats->vfgptc.ev_count = 0; 2755 stats->vfgotc.ev_count = 0; 2756 2757 /* Mailbox Stats */ 2758 hw->mbx.stats.msgs_tx.ev_count = 0; 2759 hw->mbx.stats.msgs_rx.ev_count = 0; 2760 hw->mbx.stats.acks.ev_count = 0; 2761 hw->mbx.stats.reqs.ev_count = 0; 2762 hw->mbx.stats.rsts.ev_count = 0; 2763 2764 } /* ixv_clear_evcnt */ 2765 2766 /************************************************************************ 2767 * ixv_set_sysctl_value 2768 ************************************************************************/ 2769 static void 2770 ixv_set_sysctl_value(struct adapter *adapter, const char *name, 2771 const char *description, int *limit, int value) 2772 { 2773 device_t dev = adapter->dev; 2774 struct sysctllog **log; 2775 const struct sysctlnode *rnode, *cnode; 2776 2777 log = &adapter->sysctllog; 2778 if ((rnode = ixv_sysctl_instance(adapter)) == NULL) { 2779 aprint_error_dev(dev, "could not create sysctl root\n"); 2780 return; 2781 } 2782 if (sysctl_createv(log, 0, &rnode, &cnode, 2783 CTLFLAG_READWRITE, CTLTYPE_INT, 2784 name, SYSCTL_DESCR(description), 2785 NULL, 0, limit, 0, CTL_CREATE, CTL_EOL) != 0) 2786 aprint_error_dev(dev, "could not create sysctl\n"); 2787 *limit = value; 2788 } /* ixv_set_sysctl_value */ 2789 2790 /************************************************************************ 2791 * ixv_print_debug_info 2792 * 2793 * Called only when em_display_debug_stats is enabled. 2794 * Provides a way to take a look at important statistics 2795 * maintained by the driver and hardware. 2796 ************************************************************************/ 2797 static void 2798 ixv_print_debug_info(struct adapter *adapter) 2799 { 2800 device_t dev = adapter->dev; 2801 struct ix_queue *que = adapter->queues; 2802 struct rx_ring *rxr; 2803 struct tx_ring *txr; 2804 #ifdef LRO 2805 struct lro_ctrl *lro; 2806 #endif /* LRO */ 2807 2808 for (int i = 0; i < adapter->num_queues; i++, que++) { 2809 txr = que->txr; 2810 rxr = que->rxr; 2811 #ifdef LRO 2812 lro = &rxr->lro; 2813 #endif /* LRO */ 2814 device_printf(dev, "QUE(%d) IRQs Handled: %lu\n", 2815 que->msix, (long)que->irqs.ev_count); 2816 device_printf(dev, "RX(%d) Packets Received: %lld\n", 2817 rxr->me, (long long)rxr->rx_packets.ev_count); 2818 device_printf(dev, "RX(%d) Bytes Received: %lu\n", 2819 rxr->me, (long)rxr->rx_bytes.ev_count); 2820 #ifdef LRO 2821 device_printf(dev, "RX(%d) LRO Queued= %ju\n", 2822 rxr->me, (uintmax_t)lro->lro_queued); 2823 device_printf(dev, "RX(%d) LRO Flushed= %ju\n", 2824 rxr->me, (uintmax_t)lro->lro_flushed); 2825 #endif /* LRO */ 2826 device_printf(dev, "TX(%d) Packets Sent: %lu\n", 2827 txr->me, (long)txr->total_packets.ev_count); 2828 device_printf(dev, "TX(%d) NO Desc Avail: %lu\n", 2829 txr->me, (long)txr->no_desc_avail.ev_count); 2830 } 2831 2832 device_printf(dev, "MBX IRQ Handled: %lu\n", 2833 (long)adapter->link_irq.ev_count); 2834 } /* ixv_print_debug_info */ 2835 2836 /************************************************************************ 2837 * ixv_sysctl_debug 2838 ************************************************************************/ 2839 static int 2840 ixv_sysctl_debug(SYSCTLFN_ARGS) 2841 { 2842 struct sysctlnode node = *rnode; 2843 struct adapter *adapter = (struct adapter *)node.sysctl_data; 2844 int error, result; 2845 2846 node.sysctl_data = &result; 2847 error = sysctl_lookup(SYSCTLFN_CALL(&node)); 2848 2849 if (error || newp == NULL) 2850 return error; 2851 2852 if (result == 1) 2853 ixv_print_debug_info(adapter); 2854 2855 return 0; 2856 } /* ixv_sysctl_debug */ 2857 2858 /************************************************************************ 2859 * ixv_init_device_features 2860 ************************************************************************/ 2861 static void 2862 ixv_init_device_features(struct adapter *adapter) 2863 { 2864 adapter->feat_cap = IXGBE_FEATURE_NETMAP 2865 | IXGBE_FEATURE_VF 2866 | IXGBE_FEATURE_RSS 2867 | IXGBE_FEATURE_LEGACY_TX; 2868 2869 /* A tad short on feature flags for VFs, atm. */ 2870 switch (adapter->hw.mac.type) { 2871 case ixgbe_mac_82599_vf: 2872 break; 2873 case ixgbe_mac_X540_vf: 2874 break; 2875 case ixgbe_mac_X550_vf: 2876 case ixgbe_mac_X550EM_x_vf: 2877 case ixgbe_mac_X550EM_a_vf: 2878 adapter->feat_cap |= IXGBE_FEATURE_NEEDS_CTXD; 2879 break; 2880 default: 2881 break; 2882 } 2883 2884 /* Enabled by default... */ 2885 /* Is a virtual function (VF) */ 2886 if (adapter->feat_cap & IXGBE_FEATURE_VF) 2887 adapter->feat_en |= IXGBE_FEATURE_VF; 2888 /* Netmap */ 2889 if (adapter->feat_cap & IXGBE_FEATURE_NETMAP) 2890 adapter->feat_en |= IXGBE_FEATURE_NETMAP; 2891 /* Receive-Side Scaling (RSS) */ 2892 if (adapter->feat_cap & IXGBE_FEATURE_RSS) 2893 adapter->feat_en |= IXGBE_FEATURE_RSS; 2894 /* Needs advanced context descriptor regardless of offloads req'd */ 2895 if (adapter->feat_cap & IXGBE_FEATURE_NEEDS_CTXD) 2896 adapter->feat_en |= IXGBE_FEATURE_NEEDS_CTXD; 2897 2898 /* Enabled via sysctl... */ 2899 /* Legacy (single queue) transmit */ 2900 if ((adapter->feat_cap & IXGBE_FEATURE_LEGACY_TX) && 2901 ixv_enable_legacy_tx) 2902 adapter->feat_en |= IXGBE_FEATURE_LEGACY_TX; 2903 } /* ixv_init_device_features */ 2904 2905 /************************************************************************ 2906 * ixv_shutdown - Shutdown entry point 2907 ************************************************************************/ 2908 #if 0 /* XXX NetBSD ought to register something like this through pmf(9) */ 2909 static int 2910 ixv_shutdown(device_t dev) 2911 { 2912 struct adapter *adapter = device_private(dev); 2913 IXGBE_CORE_LOCK(adapter); 2914 ixv_stop(adapter); 2915 IXGBE_CORE_UNLOCK(adapter); 2916 2917 return (0); 2918 } /* ixv_shutdown */ 2919 #endif 2920 2921 static int 2922 ixv_ifflags_cb(struct ethercom *ec) 2923 { 2924 struct ifnet *ifp = &ec->ec_if; 2925 struct adapter *adapter = ifp->if_softc; 2926 u_short saved_flags; 2927 u_short change; 2928 int rv = 0; 2929 2930 IXGBE_CORE_LOCK(adapter); 2931 2932 saved_flags = adapter->if_flags; 2933 change = ifp->if_flags ^ adapter->if_flags; 2934 if (change != 0) 2935 adapter->if_flags = ifp->if_flags; 2936 2937 if ((change & ~(IFF_CANTCHANGE | IFF_DEBUG)) != 0) { 2938 rv = ENETRESET; 2939 goto out; 2940 } else if ((change & IFF_PROMISC) != 0) { 2941 rv = ixv_set_rxfilter(adapter); 2942 if (rv != 0) { 2943 /* Restore previous */ 2944 adapter->if_flags = saved_flags; 2945 goto out; 2946 } 2947 } 2948 2949 /* Check for ec_capenable. */ 2950 change = ec->ec_capenable ^ adapter->ec_capenable; 2951 adapter->ec_capenable = ec->ec_capenable; 2952 if ((change & ~(ETHERCAP_VLAN_MTU | ETHERCAP_VLAN_HWTAGGING 2953 | ETHERCAP_VLAN_HWFILTER)) != 0) { 2954 rv = ENETRESET; 2955 goto out; 2956 } 2957 2958 /* 2959 * Special handling is not required for ETHERCAP_VLAN_MTU. 2960 * PF's MAXFRS(MHADD) does not include the 4bytes of the VLAN header. 2961 */ 2962 2963 /* Set up VLAN support and filter */ 2964 if ((change & (ETHERCAP_VLAN_HWTAGGING | ETHERCAP_VLAN_HWFILTER)) != 0) 2965 rv = ixv_setup_vlan_support(adapter); 2966 2967 out: 2968 IXGBE_CORE_UNLOCK(adapter); 2969 2970 return rv; 2971 } 2972 2973 2974 /************************************************************************ 2975 * ixv_ioctl - Ioctl entry point 2976 * 2977 * Called when the user wants to configure the interface. 2978 * 2979 * return 0 on success, positive on failure 2980 ************************************************************************/ 2981 static int 2982 ixv_ioctl(struct ifnet *ifp, u_long command, void *data) 2983 { 2984 struct adapter *adapter = ifp->if_softc; 2985 struct ixgbe_hw *hw = &adapter->hw; 2986 struct ifcapreq *ifcr = data; 2987 int error; 2988 int l4csum_en; 2989 const int l4csum = IFCAP_CSUM_TCPv4_Rx | IFCAP_CSUM_UDPv4_Rx | 2990 IFCAP_CSUM_TCPv6_Rx | IFCAP_CSUM_UDPv6_Rx; 2991 2992 switch (command) { 2993 case SIOCSIFFLAGS: 2994 IOCTL_DEBUGOUT("ioctl: SIOCSIFFLAGS (Set Interface Flags)"); 2995 break; 2996 case SIOCADDMULTI: { 2997 struct ether_multi *enm; 2998 struct ether_multistep step; 2999 struct ethercom *ec = &adapter->osdep.ec; 3000 bool overflow = false; 3001 int mcnt = 0; 3002 3003 /* 3004 * Check the number of multicast address. If it exceeds, 3005 * return ENOSPC. 3006 * Update this code when we support API 1.3. 3007 */ 3008 ETHER_LOCK(ec); 3009 ETHER_FIRST_MULTI(step, ec, enm); 3010 while (enm != NULL) { 3011 mcnt++; 3012 3013 /* 3014 * This code is before adding, so one room is required 3015 * at least. 3016 */ 3017 if (mcnt > (IXGBE_MAX_VF_MC - 1)) { 3018 overflow = true; 3019 break; 3020 } 3021 ETHER_NEXT_MULTI(step, enm); 3022 } 3023 ETHER_UNLOCK(ec); 3024 error = 0; 3025 if (overflow && ((ec->ec_flags & ETHER_F_ALLMULTI) == 0)) { 3026 error = hw->mac.ops.update_xcast_mode(hw, 3027 IXGBEVF_XCAST_MODE_ALLMULTI); 3028 if (error == IXGBE_ERR_NOT_TRUSTED) { 3029 device_printf(adapter->dev, 3030 "this interface is not trusted\n"); 3031 error = EPERM; 3032 } else if (error == IXGBE_ERR_FEATURE_NOT_SUPPORTED) { 3033 device_printf(adapter->dev, 3034 "the PF doesn't support allmulti mode\n"); 3035 error = EOPNOTSUPP; 3036 } else if (error) { 3037 device_printf(adapter->dev, 3038 "number of Ethernet multicast addresses " 3039 "exceeds the limit (%d). error = %d\n", 3040 IXGBE_MAX_VF_MC, error); 3041 error = ENOSPC; 3042 } else 3043 ec->ec_flags |= ETHER_F_ALLMULTI; 3044 } 3045 if (error) 3046 return error; 3047 } 3048 /*FALLTHROUGH*/ 3049 case SIOCDELMULTI: 3050 IOCTL_DEBUGOUT("ioctl: SIOC(ADD|DEL)MULTI"); 3051 break; 3052 case SIOCSIFMEDIA: 3053 case SIOCGIFMEDIA: 3054 IOCTL_DEBUGOUT("ioctl: SIOCxIFMEDIA (Get/Set Interface Media)"); 3055 break; 3056 case SIOCSIFCAP: 3057 IOCTL_DEBUGOUT("ioctl: SIOCSIFCAP (Set Capabilities)"); 3058 break; 3059 case SIOCSIFMTU: 3060 IOCTL_DEBUGOUT("ioctl: SIOCSIFMTU (Set Interface MTU)"); 3061 break; 3062 case SIOCZIFDATA: 3063 IOCTL_DEBUGOUT("ioctl: SIOCZIFDATA (Zero counter)"); 3064 ixv_update_stats(adapter); 3065 ixv_clear_evcnt(adapter); 3066 break; 3067 default: 3068 IOCTL_DEBUGOUT1("ioctl: UNKNOWN (0x%X)", (int)command); 3069 break; 3070 } 3071 3072 switch (command) { 3073 case SIOCSIFCAP: 3074 /* Layer-4 Rx checksum offload has to be turned on and 3075 * off as a unit. 3076 */ 3077 l4csum_en = ifcr->ifcr_capenable & l4csum; 3078 if (l4csum_en != l4csum && l4csum_en != 0) 3079 return EINVAL; 3080 /*FALLTHROUGH*/ 3081 case SIOCADDMULTI: 3082 case SIOCDELMULTI: 3083 case SIOCSIFFLAGS: 3084 case SIOCSIFMTU: 3085 default: 3086 if ((error = ether_ioctl(ifp, command, data)) != ENETRESET) 3087 return error; 3088 if ((ifp->if_flags & IFF_RUNNING) == 0) 3089 ; 3090 else if (command == SIOCSIFCAP || command == SIOCSIFMTU) { 3091 IXGBE_CORE_LOCK(adapter); 3092 ixv_init_locked(adapter); 3093 IXGBE_CORE_UNLOCK(adapter); 3094 } else if (command == SIOCADDMULTI || command == SIOCDELMULTI) { 3095 /* 3096 * Multicast list has changed; set the hardware filter 3097 * accordingly. 3098 */ 3099 IXGBE_CORE_LOCK(adapter); 3100 ixv_disable_intr(adapter); 3101 ixv_set_rxfilter(adapter); 3102 ixv_enable_intr(adapter); 3103 IXGBE_CORE_UNLOCK(adapter); 3104 } 3105 return 0; 3106 } 3107 } /* ixv_ioctl */ 3108 3109 /************************************************************************ 3110 * ixv_init 3111 ************************************************************************/ 3112 static int 3113 ixv_init(struct ifnet *ifp) 3114 { 3115 struct adapter *adapter = ifp->if_softc; 3116 3117 IXGBE_CORE_LOCK(adapter); 3118 ixv_init_locked(adapter); 3119 IXGBE_CORE_UNLOCK(adapter); 3120 3121 return 0; 3122 } /* ixv_init */ 3123 3124 /************************************************************************ 3125 * ixv_handle_que 3126 ************************************************************************/ 3127 static void 3128 ixv_handle_que(void *context) 3129 { 3130 struct ix_queue *que = context; 3131 struct adapter *adapter = que->adapter; 3132 struct tx_ring *txr = que->txr; 3133 struct ifnet *ifp = adapter->ifp; 3134 bool more; 3135 3136 que->handleq.ev_count++; 3137 3138 if (ifp->if_flags & IFF_RUNNING) { 3139 more = ixgbe_rxeof(que); 3140 IXGBE_TX_LOCK(txr); 3141 more |= ixgbe_txeof(txr); 3142 if (!(adapter->feat_en & IXGBE_FEATURE_LEGACY_TX)) 3143 if (!ixgbe_mq_ring_empty(ifp, txr->txr_interq)) 3144 ixgbe_mq_start_locked(ifp, txr); 3145 /* Only for queue 0 */ 3146 /* NetBSD still needs this for CBQ */ 3147 if ((&adapter->queues[0] == que) 3148 && (!ixgbe_legacy_ring_empty(ifp, NULL))) 3149 ixgbe_legacy_start_locked(ifp, txr); 3150 IXGBE_TX_UNLOCK(txr); 3151 if (more) { 3152 que->req.ev_count++; 3153 if (adapter->txrx_use_workqueue) { 3154 /* 3155 * "enqueued flag" is not required here 3156 * the same as ixg(4). See ixgbe_msix_que(). 3157 */ 3158 workqueue_enqueue(adapter->que_wq, 3159 &que->wq_cookie, curcpu()); 3160 } else 3161 softint_schedule(que->que_si); 3162 return; 3163 } 3164 } 3165 3166 /* Re-enable this interrupt */ 3167 ixv_enable_queue(adapter, que->msix); 3168 3169 return; 3170 } /* ixv_handle_que */ 3171 3172 /************************************************************************ 3173 * ixv_handle_que_work 3174 ************************************************************************/ 3175 static void 3176 ixv_handle_que_work(struct work *wk, void *context) 3177 { 3178 struct ix_queue *que = container_of(wk, struct ix_queue, wq_cookie); 3179 3180 /* 3181 * "enqueued flag" is not required here the same as ixg(4). 3182 * See ixgbe_msix_que(). 3183 */ 3184 ixv_handle_que(que); 3185 } 3186 3187 /************************************************************************ 3188 * ixv_allocate_msix - Setup MSI-X Interrupt resources and handlers 3189 ************************************************************************/ 3190 static int 3191 ixv_allocate_msix(struct adapter *adapter, const struct pci_attach_args *pa) 3192 { 3193 device_t dev = adapter->dev; 3194 struct ix_queue *que = adapter->queues; 3195 struct tx_ring *txr = adapter->tx_rings; 3196 int error, msix_ctrl, rid, vector = 0; 3197 pci_chipset_tag_t pc; 3198 pcitag_t tag; 3199 char intrbuf[PCI_INTRSTR_LEN]; 3200 char wqname[MAXCOMLEN]; 3201 char intr_xname[32]; 3202 const char *intrstr = NULL; 3203 kcpuset_t *affinity; 3204 int cpu_id = 0; 3205 3206 pc = adapter->osdep.pc; 3207 tag = adapter->osdep.tag; 3208 3209 adapter->osdep.nintrs = adapter->num_queues + 1; 3210 if (pci_msix_alloc_exact(pa, &adapter->osdep.intrs, 3211 adapter->osdep.nintrs) != 0) { 3212 aprint_error_dev(dev, 3213 "failed to allocate MSI-X interrupt\n"); 3214 return (ENXIO); 3215 } 3216 3217 kcpuset_create(&affinity, false); 3218 for (int i = 0; i < adapter->num_queues; i++, vector++, que++, txr++) { 3219 snprintf(intr_xname, sizeof(intr_xname), "%s TXRX%d", 3220 device_xname(dev), i); 3221 intrstr = pci_intr_string(pc, adapter->osdep.intrs[i], intrbuf, 3222 sizeof(intrbuf)); 3223 #ifdef IXGBE_MPSAFE 3224 pci_intr_setattr(pc, &adapter->osdep.intrs[i], PCI_INTR_MPSAFE, 3225 true); 3226 #endif 3227 /* Set the handler function */ 3228 que->res = adapter->osdep.ihs[i] = pci_intr_establish_xname(pc, 3229 adapter->osdep.intrs[i], IPL_NET, ixv_msix_que, que, 3230 intr_xname); 3231 if (que->res == NULL) { 3232 pci_intr_release(pc, adapter->osdep.intrs, 3233 adapter->osdep.nintrs); 3234 aprint_error_dev(dev, 3235 "Failed to register QUE handler\n"); 3236 kcpuset_destroy(affinity); 3237 return (ENXIO); 3238 } 3239 que->msix = vector; 3240 adapter->active_queues |= (u64)(1 << que->msix); 3241 3242 cpu_id = i; 3243 /* Round-robin affinity */ 3244 kcpuset_zero(affinity); 3245 kcpuset_set(affinity, cpu_id % ncpu); 3246 error = interrupt_distribute(adapter->osdep.ihs[i], affinity, 3247 NULL); 3248 aprint_normal_dev(dev, "for TX/RX, interrupting at %s", 3249 intrstr); 3250 if (error == 0) 3251 aprint_normal(", bound queue %d to cpu %d\n", 3252 i, cpu_id % ncpu); 3253 else 3254 aprint_normal("\n"); 3255 3256 #ifndef IXGBE_LEGACY_TX 3257 txr->txr_si 3258 = softint_establish(SOFTINT_NET | IXGBE_SOFTINT_FLAGS, 3259 ixgbe_deferred_mq_start, txr); 3260 #endif 3261 que->que_si 3262 = softint_establish(SOFTINT_NET | IXGBE_SOFTINT_FLAGS, 3263 ixv_handle_que, que); 3264 if (que->que_si == NULL) { 3265 aprint_error_dev(dev, 3266 "could not establish software interrupt\n"); 3267 } 3268 } 3269 snprintf(wqname, sizeof(wqname), "%sdeferTx", device_xname(dev)); 3270 error = workqueue_create(&adapter->txr_wq, wqname, 3271 ixgbe_deferred_mq_start_work, adapter, IXGBE_WORKQUEUE_PRI, IPL_NET, 3272 IXGBE_WORKQUEUE_FLAGS); 3273 if (error) { 3274 aprint_error_dev(dev, "couldn't create workqueue for deferred Tx\n"); 3275 } 3276 adapter->txr_wq_enqueued = percpu_alloc(sizeof(u_int)); 3277 3278 snprintf(wqname, sizeof(wqname), "%sTxRx", device_xname(dev)); 3279 error = workqueue_create(&adapter->que_wq, wqname, 3280 ixv_handle_que_work, adapter, IXGBE_WORKQUEUE_PRI, IPL_NET, 3281 IXGBE_WORKQUEUE_FLAGS); 3282 if (error) { 3283 aprint_error_dev(dev, 3284 "couldn't create workqueue\n"); 3285 } 3286 3287 /* and Mailbox */ 3288 cpu_id++; 3289 snprintf(intr_xname, sizeof(intr_xname), "%s link", device_xname(dev)); 3290 adapter->vector = vector; 3291 intrstr = pci_intr_string(pc, adapter->osdep.intrs[vector], intrbuf, 3292 sizeof(intrbuf)); 3293 #ifdef IXGBE_MPSAFE 3294 pci_intr_setattr(pc, &adapter->osdep.intrs[vector], PCI_INTR_MPSAFE, 3295 true); 3296 #endif 3297 /* Set the mbx handler function */ 3298 adapter->osdep.ihs[vector] = pci_intr_establish_xname(pc, 3299 adapter->osdep.intrs[vector], IPL_NET, ixv_msix_mbx, adapter, 3300 intr_xname); 3301 if (adapter->osdep.ihs[vector] == NULL) { 3302 aprint_error_dev(dev, "Failed to register LINK handler\n"); 3303 kcpuset_destroy(affinity); 3304 return (ENXIO); 3305 } 3306 /* Round-robin affinity */ 3307 kcpuset_zero(affinity); 3308 kcpuset_set(affinity, cpu_id % ncpu); 3309 error = interrupt_distribute(adapter->osdep.ihs[vector], 3310 affinity, NULL); 3311 3312 aprint_normal_dev(dev, 3313 "for link, interrupting at %s", intrstr); 3314 if (error == 0) 3315 aprint_normal(", affinity to cpu %d\n", cpu_id % ncpu); 3316 else 3317 aprint_normal("\n"); 3318 3319 /* Tasklets for Mailbox */ 3320 adapter->link_si = softint_establish(SOFTINT_NET |IXGBE_SOFTINT_FLAGS, 3321 ixv_handle_link, adapter); 3322 /* 3323 * Due to a broken design QEMU will fail to properly 3324 * enable the guest for MSI-X unless the vectors in 3325 * the table are all set up, so we must rewrite the 3326 * ENABLE in the MSI-X control register again at this 3327 * point to cause it to successfully initialize us. 3328 */ 3329 if (adapter->hw.mac.type == ixgbe_mac_82599_vf) { 3330 pci_get_capability(pc, tag, PCI_CAP_MSIX, &rid, NULL); 3331 rid += PCI_MSIX_CTL; 3332 msix_ctrl = pci_conf_read(pc, tag, rid); 3333 msix_ctrl |= PCI_MSIX_CTL_ENABLE; 3334 pci_conf_write(pc, tag, rid, msix_ctrl); 3335 } 3336 3337 kcpuset_destroy(affinity); 3338 return (0); 3339 } /* ixv_allocate_msix */ 3340 3341 /************************************************************************ 3342 * ixv_configure_interrupts - Setup MSI-X resources 3343 * 3344 * Note: The VF device MUST use MSI-X, there is no fallback. 3345 ************************************************************************/ 3346 static int 3347 ixv_configure_interrupts(struct adapter *adapter) 3348 { 3349 device_t dev = adapter->dev; 3350 int want, queues, msgs; 3351 3352 /* Must have at least 2 MSI-X vectors */ 3353 msgs = pci_msix_count(adapter->osdep.pc, adapter->osdep.tag); 3354 if (msgs < 2) { 3355 aprint_error_dev(dev, "MSIX config error\n"); 3356 return (ENXIO); 3357 } 3358 msgs = MIN(msgs, IXG_MAX_NINTR); 3359 3360 /* Figure out a reasonable auto config value */ 3361 queues = (ncpu > (msgs - 1)) ? (msgs - 1) : ncpu; 3362 3363 if (ixv_num_queues != 0) 3364 queues = ixv_num_queues; 3365 else if ((ixv_num_queues == 0) && (queues > IXGBE_VF_MAX_TX_QUEUES)) 3366 queues = IXGBE_VF_MAX_TX_QUEUES; 3367 3368 /* 3369 * Want vectors for the queues, 3370 * plus an additional for mailbox. 3371 */ 3372 want = queues + 1; 3373 if (msgs >= want) 3374 msgs = want; 3375 else { 3376 aprint_error_dev(dev, 3377 "MSI-X Configuration Problem, " 3378 "%d vectors but %d queues wanted!\n", 3379 msgs, want); 3380 return -1; 3381 } 3382 3383 adapter->msix_mem = (void *)1; /* XXX */ 3384 aprint_normal_dev(dev, 3385 "Using MSI-X interrupts with %d vectors\n", msgs); 3386 adapter->num_queues = queues; 3387 3388 return (0); 3389 } /* ixv_configure_interrupts */ 3390 3391 3392 /************************************************************************ 3393 * ixv_handle_link - Tasklet handler for MSI-X MBX interrupts 3394 * 3395 * Done outside of interrupt context since the driver might sleep 3396 ************************************************************************/ 3397 static void 3398 ixv_handle_link(void *context) 3399 { 3400 struct adapter *adapter = context; 3401 3402 IXGBE_CORE_LOCK(adapter); 3403 3404 adapter->hw.mac.ops.check_link(&adapter->hw, &adapter->link_speed, 3405 &adapter->link_up, FALSE); 3406 ixv_update_link_status(adapter); 3407 3408 IXGBE_CORE_UNLOCK(adapter); 3409 } /* ixv_handle_link */ 3410 3411 /************************************************************************ 3412 * ixv_check_link - Used in the local timer to poll for link changes 3413 ************************************************************************/ 3414 static s32 3415 ixv_check_link(struct adapter *adapter) 3416 { 3417 s32 error; 3418 3419 KASSERT(mutex_owned(&adapter->core_mtx)); 3420 3421 adapter->hw.mac.get_link_status = TRUE; 3422 3423 error = adapter->hw.mac.ops.check_link(&adapter->hw, 3424 &adapter->link_speed, &adapter->link_up, FALSE); 3425 ixv_update_link_status(adapter); 3426 3427 return error; 3428 } /* ixv_check_link */ 3429
Indexes created Tue Sep 30 20:09:53 GMT 2025