ixv.c revision 1.144
1 /*$NetBSD: ixv.c,v 1.144 2020/01/21 14:55:55 msaitoh Exp $*/ 2 3 /****************************************************************************** 4 5 Copyright (c) 2001-2017, Intel Corporation 6 All rights reserved. 7 8 Redistribution and use in source and binary forms, with or without 9 modification, are permitted provided that the following conditions are met: 10 11 1. Redistributions of source code must retain the above copyright notice, 12 this list of conditions and the following disclaimer. 13 14 2. Redistributions in binary form must reproduce the above copyright 15 notice, this list of conditions and the following disclaimer in the 16 documentation and/or other materials provided with the distribution. 17 18 3. Neither the name of the Intel Corporation nor the names of its 19 contributors may be used to endorse or promote products derived from 20 this software without specific prior written permission. 21 22 THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 23 AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 24 IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 25 ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE 26 LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 27 CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 28 SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 29 INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 30 CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 31 ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 32 POSSIBILITY OF SUCH DAMAGE. 33 34 ******************************************************************************/ 35 /*$FreeBSD: head/sys/dev/ixgbe/if_ixv.c 331224 2018-03-19 20:55:05Z erj $*/ 36 37 #ifdef _KERNEL_OPT 38 #include "opt_inet.h" 39 #include "opt_inet6.h" 40 #include "opt_net_mpsafe.h" 41 #endif 42 43 #include "ixgbe.h" 44 #include "vlan.h" 45 46 /************************************************************************ 47 * Driver version 48 ************************************************************************/ 49 static const char ixv_driver_version[] = "2.0.1-k"; 50 /* XXX NetBSD: + 1.5.17 */ 51 52 /************************************************************************ 53 * PCI Device ID Table 54 * 55 * Used by probe to select devices to load on 56 * Last field stores an index into ixv_strings 57 * Last entry must be all 0s 58 * 59 * { Vendor ID, Device ID, SubVendor ID, SubDevice ID, String Index } 60 ************************************************************************/ 61 static const ixgbe_vendor_info_t ixv_vendor_info_array[] = 62 { 63 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_VF, 0, 0, 0}, 64 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_X540_VF, 0, 0, 0}, 65 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_X550_VF, 0, 0, 0}, 66 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_X550EM_X_VF, 0, 0, 0}, 67 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_X550EM_A_VF, 0, 0, 0}, 68 /* required last entry */ 69 {0, 0, 0, 0, 0} 70 }; 71 72 /************************************************************************ 73 * Table of branding strings 74 ************************************************************************/ 75 static const char *ixv_strings[] = { 76 "Intel(R) PRO/10GbE Virtual Function Network Driver" 77 }; 78 79 /********************************************************************* 80 * Function prototypes 81 *********************************************************************/ 82 static int ixv_probe(device_t, cfdata_t, void *); 83 static void ixv_attach(device_t, device_t, void *); 84 static int ixv_detach(device_t, int); 85 #if 0 86 static int ixv_shutdown(device_t); 87 #endif 88 static int ixv_ifflags_cb(struct ethercom *); 89 static int ixv_ioctl(struct ifnet *, u_long, void *); 90 static int ixv_init(struct ifnet *); 91 static void ixv_init_locked(struct adapter *); 92 static void ixv_ifstop(struct ifnet *, int); 93 static void ixv_stop(void *); 94 static void ixv_init_device_features(struct adapter *); 95 static void ixv_media_status(struct ifnet *, struct ifmediareq *); 96 static int ixv_media_change(struct ifnet *); 97 static int ixv_allocate_pci_resources(struct adapter *, 98 const struct pci_attach_args *); 99 static int ixv_allocate_msix(struct adapter *, 100 const struct pci_attach_args *); 101 static int ixv_configure_interrupts(struct adapter *); 102 static void ixv_free_pci_resources(struct adapter *); 103 static void ixv_local_timer(void *); 104 static void ixv_local_timer_locked(void *); 105 static int ixv_setup_interface(device_t, struct adapter *); 106 static int ixv_negotiate_api(struct adapter *); 107 108 static void ixv_initialize_transmit_units(struct adapter *); 109 static void ixv_initialize_receive_units(struct adapter *); 110 static void ixv_initialize_rss_mapping(struct adapter *); 111 static s32 ixv_check_link(struct adapter *); 112 113 static void ixv_enable_intr(struct adapter *); 114 static void ixv_disable_intr(struct adapter *); 115 static int ixv_set_rxfilter(struct adapter *); 116 static void ixv_update_link_status(struct adapter *); 117 static int ixv_sysctl_debug(SYSCTLFN_PROTO); 118 static void ixv_set_ivar(struct adapter *, u8, u8, s8); 119 static void ixv_configure_ivars(struct adapter *); 120 static u8 * ixv_mc_array_itr(struct ixgbe_hw *, u8 **, u32 *); 121 static void ixv_eitr_write(struct adapter *, uint32_t, uint32_t); 122 123 static void ixv_setup_vlan_tagging(struct adapter *); 124 static int ixv_setup_vlan_support(struct adapter *); 125 static int ixv_vlan_cb(struct ethercom *, uint16_t, bool); 126 static int ixv_register_vlan(struct adapter *, u16); 127 static int ixv_unregister_vlan(struct adapter *, u16); 128 129 static void ixv_add_device_sysctls(struct adapter *); 130 static void ixv_save_stats(struct adapter *); 131 static void ixv_init_stats(struct adapter *); 132 static void ixv_update_stats(struct adapter *); 133 static void ixv_add_stats_sysctls(struct adapter *); 134 static void ixv_clear_evcnt(struct adapter *); 135 136 /* Sysctl handlers */ 137 static void ixv_set_sysctl_value(struct adapter *, const char *, 138 const char *, int *, int); 139 static int ixv_sysctl_interrupt_rate_handler(SYSCTLFN_PROTO); 140 static int ixv_sysctl_next_to_check_handler(SYSCTLFN_PROTO); 141 static int ixv_sysctl_rdh_handler(SYSCTLFN_PROTO); 142 static int ixv_sysctl_rdt_handler(SYSCTLFN_PROTO); 143 static int ixv_sysctl_tdt_handler(SYSCTLFN_PROTO); 144 static int ixv_sysctl_tdh_handler(SYSCTLFN_PROTO); 145 146 /* The MSI-X Interrupt handlers */ 147 static int ixv_msix_que(void *); 148 static int ixv_msix_mbx(void *); 149 150 /* Deferred interrupt tasklets */ 151 static void ixv_handle_que(void *); 152 static void ixv_handle_link(void *); 153 154 /* Workqueue handler for deferred work */ 155 static void ixv_handle_que_work(struct work *, void *); 156 157 const struct sysctlnode *ixv_sysctl_instance(struct adapter *); 158 static const ixgbe_vendor_info_t *ixv_lookup(const struct pci_attach_args *); 159 160 /************************************************************************ 161 * FreeBSD Device Interface Entry Points 162 ************************************************************************/ 163 CFATTACH_DECL3_NEW(ixv, sizeof(struct adapter), 164 ixv_probe, ixv_attach, ixv_detach, NULL, NULL, NULL, 165 DVF_DETACH_SHUTDOWN); 166 167 #if 0 168 static driver_t ixv_driver = { 169 "ixv", ixv_methods, sizeof(struct adapter), 170 }; 171 172 devclass_t ixv_devclass; 173 DRIVER_MODULE(ixv, pci, ixv_driver, ixv_devclass, 0, 0); 174 MODULE_DEPEND(ixv, pci, 1, 1, 1); 175 MODULE_DEPEND(ixv, ether, 1, 1, 1); 176 #endif 177 178 /* 179 * TUNEABLE PARAMETERS: 180 */ 181 182 /* Number of Queues - do not exceed MSI-X vectors - 1 */ 183 static int ixv_num_queues = 0; 184 #define TUNABLE_INT(__x, __y) 185 TUNABLE_INT("hw.ixv.num_queues", &ixv_num_queues); 186 187 /* 188 * AIM: Adaptive Interrupt Moderation 189 * which means that the interrupt rate 190 * is varied over time based on the 191 * traffic for that interrupt vector 192 */ 193 static bool ixv_enable_aim = false; 194 TUNABLE_INT("hw.ixv.enable_aim", &ixv_enable_aim); 195 196 static int ixv_max_interrupt_rate = (4000000 / IXGBE_LOW_LATENCY); 197 TUNABLE_INT("hw.ixv.max_interrupt_rate", &ixv_max_interrupt_rate); 198 199 /* How many packets rxeof tries to clean at a time */ 200 static int ixv_rx_process_limit = 256; 201 TUNABLE_INT("hw.ixv.rx_process_limit", &ixv_rx_process_limit); 202 203 /* How many packets txeof tries to clean at a time */ 204 static int ixv_tx_process_limit = 256; 205 TUNABLE_INT("hw.ixv.tx_process_limit", &ixv_tx_process_limit); 206 207 /* Which packet processing uses workqueue or softint */ 208 static bool ixv_txrx_workqueue = false; 209 210 /* 211 * Number of TX descriptors per ring, 212 * setting higher than RX as this seems 213 * the better performing choice. 214 */ 215 static int ixv_txd = PERFORM_TXD; 216 TUNABLE_INT("hw.ixv.txd", &ixv_txd); 217 218 /* Number of RX descriptors per ring */ 219 static int ixv_rxd = PERFORM_RXD; 220 TUNABLE_INT("hw.ixv.rxd", &ixv_rxd); 221 222 /* Legacy Transmit (single queue) */ 223 static int ixv_enable_legacy_tx = 0; 224 TUNABLE_INT("hw.ixv.enable_legacy_tx", &ixv_enable_legacy_tx); 225 226 #ifdef NET_MPSAFE 227 #define IXGBE_MPSAFE 1 228 #define IXGBE_CALLOUT_FLAGS CALLOUT_MPSAFE 229 #define IXGBE_SOFTINFT_FLAGS SOFTINT_MPSAFE 230 #define IXGBE_WORKQUEUE_FLAGS WQ_PERCPU | WQ_MPSAFE 231 #else 232 #define IXGBE_CALLOUT_FLAGS 0 233 #define IXGBE_SOFTINFT_FLAGS 0 234 #define IXGBE_WORKQUEUE_FLAGS WQ_PERCPU 235 #endif 236 #define IXGBE_WORKQUEUE_PRI PRI_SOFTNET 237 238 #if 0 239 static int (*ixv_start_locked)(struct ifnet *, struct tx_ring *); 240 static int (*ixv_ring_empty)(struct ifnet *, struct buf_ring *); 241 #endif 242 243 /************************************************************************ 244 * ixv_probe - Device identification routine 245 * 246 * Determines if the driver should be loaded on 247 * adapter based on its PCI vendor/device ID. 248 * 249 * return BUS_PROBE_DEFAULT on success, positive on failure 250 ************************************************************************/ 251 static int 252 ixv_probe(device_t dev, cfdata_t cf, void *aux) 253 { 254 #ifdef __HAVE_PCI_MSI_MSIX 255 const struct pci_attach_args *pa = aux; 256 257 return (ixv_lookup(pa) != NULL) ? 1 : 0; 258 #else 259 return 0; 260 #endif 261 } /* ixv_probe */ 262 263 static const ixgbe_vendor_info_t * 264 ixv_lookup(const struct pci_attach_args *pa) 265 { 266 const ixgbe_vendor_info_t *ent; 267 pcireg_t subid; 268 269 INIT_DEBUGOUT("ixv_lookup: begin"); 270 271 if (PCI_VENDOR(pa->pa_id) != IXGBE_INTEL_VENDOR_ID) 272 return NULL; 273 274 subid = pci_conf_read(pa->pa_pc, pa->pa_tag, PCI_SUBSYS_ID_REG); 275 276 for (ent = ixv_vendor_info_array; ent->vendor_id != 0; ent++) { 277 if ((PCI_VENDOR(pa->pa_id) == ent->vendor_id) && 278 (PCI_PRODUCT(pa->pa_id) == ent->device_id) && 279 ((PCI_SUBSYS_VENDOR(subid) == ent->subvendor_id) || 280 (ent->subvendor_id == 0)) && 281 ((PCI_SUBSYS_ID(subid) == ent->subdevice_id) || 282 (ent->subdevice_id == 0))) { 283 return ent; 284 } 285 } 286 287 return NULL; 288 } 289 290 /************************************************************************ 291 * ixv_attach - Device initialization routine 292 * 293 * Called when the driver is being loaded. 294 * Identifies the type of hardware, allocates all resources 295 * and initializes the hardware. 296 * 297 * return 0 on success, positive on failure 298 ************************************************************************/ 299 static void 300 ixv_attach(device_t parent, device_t dev, void *aux) 301 { 302 struct adapter *adapter; 303 struct ixgbe_hw *hw; 304 int error = 0; 305 pcireg_t id, subid; 306 const ixgbe_vendor_info_t *ent; 307 const struct pci_attach_args *pa = aux; 308 const char *apivstr; 309 const char *str; 310 char buf[256]; 311 312 INIT_DEBUGOUT("ixv_attach: begin"); 313 314 /* 315 * Make sure BUSMASTER is set, on a VM under 316 * KVM it may not be and will break things. 317 */ 318 ixgbe_pci_enable_busmaster(pa->pa_pc, pa->pa_tag); 319 320 /* Allocate, clear, and link in our adapter structure */ 321 adapter = device_private(dev); 322 adapter->dev = dev; 323 adapter->hw.back = adapter; 324 hw = &adapter->hw; 325 326 adapter->init_locked = ixv_init_locked; 327 adapter->stop_locked = ixv_stop; 328 329 adapter->osdep.pc = pa->pa_pc; 330 adapter->osdep.tag = pa->pa_tag; 331 if (pci_dma64_available(pa)) 332 adapter->osdep.dmat = pa->pa_dmat64; 333 else 334 adapter->osdep.dmat = pa->pa_dmat; 335 adapter->osdep.attached = false; 336 337 ent = ixv_lookup(pa); 338 339 KASSERT(ent != NULL); 340 341 aprint_normal(": %s, Version - %s\n", 342 ixv_strings[ent->index], ixv_driver_version); 343 344 /* Core Lock Init*/ 345 IXGBE_CORE_LOCK_INIT(adapter, device_xname(dev)); 346 347 /* Do base PCI setup - map BAR0 */ 348 if (ixv_allocate_pci_resources(adapter, pa)) { 349 aprint_error_dev(dev, "ixv_allocate_pci_resources() failed!\n"); 350 error = ENXIO; 351 goto err_out; 352 } 353 354 /* SYSCTL APIs */ 355 ixv_add_device_sysctls(adapter); 356 357 /* Set up the timer callout */ 358 callout_init(&adapter->timer, IXGBE_CALLOUT_FLAGS); 359 360 /* Save off the information about this board */ 361 id = pci_conf_read(pa->pa_pc, pa->pa_tag, PCI_ID_REG); 362 subid = pci_conf_read(pa->pa_pc, pa->pa_tag, PCI_SUBSYS_ID_REG); 363 hw->vendor_id = PCI_VENDOR(id); 364 hw->device_id = PCI_PRODUCT(id); 365 hw->revision_id = 366 PCI_REVISION(pci_conf_read(pa->pa_pc, pa->pa_tag, PCI_CLASS_REG)); 367 hw->subsystem_vendor_id = PCI_SUBSYS_VENDOR(subid); 368 hw->subsystem_device_id = PCI_SUBSYS_ID(subid); 369 370 /* A subset of set_mac_type */ 371 switch (hw->device_id) { 372 case IXGBE_DEV_ID_82599_VF: 373 hw->mac.type = ixgbe_mac_82599_vf; 374 str = "82599 VF"; 375 break; 376 case IXGBE_DEV_ID_X540_VF: 377 hw->mac.type = ixgbe_mac_X540_vf; 378 str = "X540 VF"; 379 break; 380 case IXGBE_DEV_ID_X550_VF: 381 hw->mac.type = ixgbe_mac_X550_vf; 382 str = "X550 VF"; 383 break; 384 case IXGBE_DEV_ID_X550EM_X_VF: 385 hw->mac.type = ixgbe_mac_X550EM_x_vf; 386 str = "X550EM X VF"; 387 break; 388 case IXGBE_DEV_ID_X550EM_A_VF: 389 hw->mac.type = ixgbe_mac_X550EM_a_vf; 390 str = "X550EM A VF"; 391 break; 392 default: 393 /* Shouldn't get here since probe succeeded */ 394 aprint_error_dev(dev, "Unknown device ID!\n"); 395 error = ENXIO; 396 goto err_out; 397 break; 398 } 399 aprint_normal_dev(dev, "device %s\n", str); 400 401 ixv_init_device_features(adapter); 402 403 /* Initialize the shared code */ 404 error = ixgbe_init_ops_vf(hw); 405 if (error) { 406 aprint_error_dev(dev, "ixgbe_init_ops_vf() failed!\n"); 407 error = EIO; 408 goto err_out; 409 } 410 411 /* Setup the mailbox */ 412 ixgbe_init_mbx_params_vf(hw); 413 414 /* Set the right number of segments */ 415 adapter->num_segs = IXGBE_82599_SCATTER; 416 417 /* Reset mbox api to 1.0 */ 418 error = hw->mac.ops.reset_hw(hw); 419 if (error == IXGBE_ERR_RESET_FAILED) 420 aprint_error_dev(dev, "...reset_hw() failure: Reset Failed!\n"); 421 else if (error) 422 aprint_error_dev(dev, "...reset_hw() failed with error %d\n", 423 error); 424 if (error) { 425 error = EIO; 426 goto err_out; 427 } 428 429 error = hw->mac.ops.init_hw(hw); 430 if (error) { 431 aprint_error_dev(dev, "...init_hw() failed!\n"); 432 error = EIO; 433 goto err_out; 434 } 435 436 /* Negotiate mailbox API version */ 437 error = ixv_negotiate_api(adapter); 438 if (error) 439 aprint_normal_dev(dev, 440 "MBX API negotiation failed during attach!\n"); 441 switch (hw->api_version) { 442 case ixgbe_mbox_api_10: 443 apivstr = "1.0"; 444 break; 445 case ixgbe_mbox_api_20: 446 apivstr = "2.0"; 447 break; 448 case ixgbe_mbox_api_11: 449 apivstr = "1.1"; 450 break; 451 case ixgbe_mbox_api_12: 452 apivstr = "1.2"; 453 break; 454 case ixgbe_mbox_api_13: 455 apivstr = "1.3"; 456 break; 457 default: 458 apivstr = "unknown"; 459 break; 460 } 461 aprint_normal_dev(dev, "Mailbox API %s\n", apivstr); 462 463 /* If no mac address was assigned, make a random one */ 464 if (!ixv_check_ether_addr(hw->mac.addr)) { 465 u8 addr[ETHER_ADDR_LEN]; 466 uint64_t rndval = cprng_strong64(); 467 468 memcpy(addr, &rndval, sizeof(addr)); 469 addr[0] &= 0xFE; 470 addr[0] |= 0x02; 471 bcopy(addr, hw->mac.addr, sizeof(addr)); 472 } 473 474 /* Register for VLAN events */ 475 ether_set_vlan_cb(&adapter->osdep.ec, ixv_vlan_cb); 476 477 /* Sysctls for limiting the amount of work done in the taskqueues */ 478 ixv_set_sysctl_value(adapter, "rx_processing_limit", 479 "max number of rx packets to process", 480 &adapter->rx_process_limit, ixv_rx_process_limit); 481 482 ixv_set_sysctl_value(adapter, "tx_processing_limit", 483 "max number of tx packets to process", 484 &adapter->tx_process_limit, ixv_tx_process_limit); 485 486 /* Do descriptor calc and sanity checks */ 487 if (((ixv_txd * sizeof(union ixgbe_adv_tx_desc)) % DBA_ALIGN) != 0 || 488 ixv_txd < MIN_TXD || ixv_txd > MAX_TXD) { 489 aprint_error_dev(dev, "TXD config issue, using default!\n"); 490 adapter->num_tx_desc = DEFAULT_TXD; 491 } else 492 adapter->num_tx_desc = ixv_txd; 493 494 if (((ixv_rxd * sizeof(union ixgbe_adv_rx_desc)) % DBA_ALIGN) != 0 || 495 ixv_rxd < MIN_RXD || ixv_rxd > MAX_RXD) { 496 aprint_error_dev(dev, "RXD config issue, using default!\n"); 497 adapter->num_rx_desc = DEFAULT_RXD; 498 } else 499 adapter->num_rx_desc = ixv_rxd; 500 501 /* Setup MSI-X */ 502 error = ixv_configure_interrupts(adapter); 503 if (error) 504 goto err_out; 505 506 /* Allocate our TX/RX Queues */ 507 if (ixgbe_allocate_queues(adapter)) { 508 aprint_error_dev(dev, "ixgbe_allocate_queues() failed!\n"); 509 error = ENOMEM; 510 goto err_out; 511 } 512 513 /* hw.ix defaults init */ 514 adapter->enable_aim = ixv_enable_aim; 515 516 adapter->txrx_use_workqueue = ixv_txrx_workqueue; 517 518 error = ixv_allocate_msix(adapter, pa); 519 if (error) { 520 aprint_error_dev(dev, "ixv_allocate_msix() failed!\n"); 521 goto err_late; 522 } 523 524 /* Setup OS specific network interface */ 525 error = ixv_setup_interface(dev, adapter); 526 if (error != 0) { 527 aprint_error_dev(dev, "ixv_setup_interface() failed!\n"); 528 goto err_late; 529 } 530 531 /* Do the stats setup */ 532 ixv_save_stats(adapter); 533 ixv_init_stats(adapter); 534 ixv_add_stats_sysctls(adapter); 535 536 if (adapter->feat_en & IXGBE_FEATURE_NETMAP) 537 ixgbe_netmap_attach(adapter); 538 539 snprintb(buf, sizeof(buf), IXGBE_FEATURE_FLAGS, adapter->feat_cap); 540 aprint_verbose_dev(dev, "feature cap %s\n", buf); 541 snprintb(buf, sizeof(buf), IXGBE_FEATURE_FLAGS, adapter->feat_en); 542 aprint_verbose_dev(dev, "feature ena %s\n", buf); 543 544 INIT_DEBUGOUT("ixv_attach: end"); 545 adapter->osdep.attached = true; 546 547 return; 548 549 err_late: 550 ixgbe_free_queues(adapter); 551 err_out: 552 ixv_free_pci_resources(adapter); 553 IXGBE_CORE_LOCK_DESTROY(adapter); 554 555 return; 556 } /* ixv_attach */ 557 558 /************************************************************************ 559 * ixv_detach - Device removal routine 560 * 561 * Called when the driver is being removed. 562 * Stops the adapter and deallocates all the resources 563 * that were allocated for driver operation. 564 * 565 * return 0 on success, positive on failure 566 ************************************************************************/ 567 static int 568 ixv_detach(device_t dev, int flags) 569 { 570 struct adapter *adapter = device_private(dev); 571 struct ixgbe_hw *hw = &adapter->hw; 572 struct ix_queue *que = adapter->queues; 573 struct tx_ring *txr = adapter->tx_rings; 574 struct rx_ring *rxr = adapter->rx_rings; 575 struct ixgbevf_hw_stats *stats = &adapter->stats.vf; 576 577 INIT_DEBUGOUT("ixv_detach: begin"); 578 if (adapter->osdep.attached == false) 579 return 0; 580 581 /* Stop the interface. Callouts are stopped in it. */ 582 ixv_ifstop(adapter->ifp, 1); 583 584 #if NVLAN > 0 585 /* Make sure VLANs are not using driver */ 586 if (!VLAN_ATTACHED(&adapter->osdep.ec)) 587 ; /* nothing to do: no VLANs */ 588 else if ((flags & (DETACH_SHUTDOWN | DETACH_FORCE)) != 0) 589 vlan_ifdetach(adapter->ifp); 590 else { 591 aprint_error_dev(dev, "VLANs in use, detach first\n"); 592 return EBUSY; 593 } 594 #endif 595 596 for (int i = 0; i < adapter->num_queues; i++, que++, txr++) { 597 if (!(adapter->feat_en & IXGBE_FEATURE_LEGACY_TX)) 598 softint_disestablish(txr->txr_si); 599 softint_disestablish(que->que_si); 600 } 601 if (adapter->txr_wq != NULL) 602 workqueue_destroy(adapter->txr_wq); 603 if (adapter->txr_wq_enqueued != NULL) 604 percpu_free(adapter->txr_wq_enqueued, sizeof(u_int)); 605 if (adapter->que_wq != NULL) 606 workqueue_destroy(adapter->que_wq); 607 608 /* Drain the Mailbox(link) queue */ 609 softint_disestablish(adapter->link_si); 610 611 ether_ifdetach(adapter->ifp); 612 callout_halt(&adapter->timer, NULL); 613 614 if (adapter->feat_en & IXGBE_FEATURE_NETMAP) 615 netmap_detach(adapter->ifp); 616 617 ixv_free_pci_resources(adapter); 618 #if 0 /* XXX the NetBSD port is probably missing something here */ 619 bus_generic_detach(dev); 620 #endif 621 if_detach(adapter->ifp); 622 if_percpuq_destroy(adapter->ipq); 623 624 sysctl_teardown(&adapter->sysctllog); 625 evcnt_detach(&adapter->efbig_tx_dma_setup); 626 evcnt_detach(&adapter->mbuf_defrag_failed); 627 evcnt_detach(&adapter->efbig2_tx_dma_setup); 628 evcnt_detach(&adapter->einval_tx_dma_setup); 629 evcnt_detach(&adapter->other_tx_dma_setup); 630 evcnt_detach(&adapter->eagain_tx_dma_setup); 631 evcnt_detach(&adapter->enomem_tx_dma_setup); 632 evcnt_detach(&adapter->watchdog_events); 633 evcnt_detach(&adapter->tso_err); 634 evcnt_detach(&adapter->link_irq); 635 636 txr = adapter->tx_rings; 637 for (int i = 0; i < adapter->num_queues; i++, rxr++, txr++) { 638 evcnt_detach(&adapter->queues[i].irqs); 639 evcnt_detach(&adapter->queues[i].handleq); 640 evcnt_detach(&adapter->queues[i].req); 641 evcnt_detach(&txr->no_desc_avail); 642 evcnt_detach(&txr->total_packets); 643 evcnt_detach(&txr->tso_tx); 644 #ifndef IXGBE_LEGACY_TX 645 evcnt_detach(&txr->pcq_drops); 646 #endif 647 648 evcnt_detach(&rxr->rx_packets); 649 evcnt_detach(&rxr->rx_bytes); 650 evcnt_detach(&rxr->rx_copies); 651 evcnt_detach(&rxr->no_jmbuf); 652 evcnt_detach(&rxr->rx_discarded); 653 } 654 evcnt_detach(&stats->ipcs); 655 evcnt_detach(&stats->l4cs); 656 evcnt_detach(&stats->ipcs_bad); 657 evcnt_detach(&stats->l4cs_bad); 658 659 /* Packet Reception Stats */ 660 evcnt_detach(&stats->vfgorc); 661 evcnt_detach(&stats->vfgprc); 662 evcnt_detach(&stats->vfmprc); 663 664 /* Packet Transmission Stats */ 665 evcnt_detach(&stats->vfgotc); 666 evcnt_detach(&stats->vfgptc); 667 668 /* Mailbox Stats */ 669 evcnt_detach(&hw->mbx.stats.msgs_tx); 670 evcnt_detach(&hw->mbx.stats.msgs_rx); 671 evcnt_detach(&hw->mbx.stats.acks); 672 evcnt_detach(&hw->mbx.stats.reqs); 673 evcnt_detach(&hw->mbx.stats.rsts); 674 675 ixgbe_free_queues(adapter); 676 677 IXGBE_CORE_LOCK_DESTROY(adapter); 678 679 return (0); 680 } /* ixv_detach */ 681 682 /************************************************************************ 683 * ixv_init_locked - Init entry point 684 * 685 * Used in two ways: It is used by the stack as an init entry 686 * point in network interface structure. It is also used 687 * by the driver as a hw/sw initialization routine to get 688 * to a consistent state. 689 * 690 * return 0 on success, positive on failure 691 ************************************************************************/ 692 static void 693 ixv_init_locked(struct adapter *adapter) 694 { 695 struct ifnet *ifp = adapter->ifp; 696 device_t dev = adapter->dev; 697 struct ixgbe_hw *hw = &adapter->hw; 698 struct ix_queue *que; 699 int error = 0; 700 uint32_t mask; 701 int i; 702 703 INIT_DEBUGOUT("ixv_init_locked: begin"); 704 KASSERT(mutex_owned(&adapter->core_mtx)); 705 hw->adapter_stopped = FALSE; 706 hw->mac.ops.stop_adapter(hw); 707 callout_stop(&adapter->timer); 708 for (i = 0, que = adapter->queues; i < adapter->num_queues; i++, que++) 709 que->disabled_count = 0; 710 711 adapter->max_frame_size = 712 ifp->if_mtu + ETHER_HDR_LEN + ETHER_CRC_LEN; 713 714 /* reprogram the RAR[0] in case user changed it. */ 715 hw->mac.ops.set_rar(hw, 0, hw->mac.addr, 0, IXGBE_RAH_AV); 716 717 /* Get the latest mac address, User can use a LAA */ 718 memcpy(hw->mac.addr, CLLADDR(ifp->if_sadl), 719 IXGBE_ETH_LENGTH_OF_ADDRESS); 720 hw->mac.ops.set_rar(hw, 0, hw->mac.addr, 0, 1); 721 722 /* Prepare transmit descriptors and buffers */ 723 if (ixgbe_setup_transmit_structures(adapter)) { 724 aprint_error_dev(dev, "Could not setup transmit structures\n"); 725 ixv_stop(adapter); 726 return; 727 } 728 729 /* Reset VF and renegotiate mailbox API version */ 730 hw->mac.ops.reset_hw(hw); 731 hw->mac.ops.start_hw(hw); 732 error = ixv_negotiate_api(adapter); 733 if (error) 734 device_printf(dev, 735 "Mailbox API negotiation failed in init_locked!\n"); 736 737 ixv_initialize_transmit_units(adapter); 738 739 /* Setup Multicast table */ 740 ixv_set_rxfilter(adapter); 741 742 /* 743 * Determine the correct mbuf pool 744 * for doing jumbo/headersplit 745 */ 746 if (adapter->max_frame_size <= MCLBYTES) 747 adapter->rx_mbuf_sz = MCLBYTES; 748 else 749 adapter->rx_mbuf_sz = MJUMPAGESIZE; 750 751 /* Prepare receive descriptors and buffers */ 752 if (ixgbe_setup_receive_structures(adapter)) { 753 device_printf(dev, "Could not setup receive structures\n"); 754 ixv_stop(adapter); 755 return; 756 } 757 758 /* Configure RX settings */ 759 ixv_initialize_receive_units(adapter); 760 761 /* Set up VLAN offload and filter */ 762 ixv_setup_vlan_support(adapter); 763 764 /* Set up MSI-X routing */ 765 ixv_configure_ivars(adapter); 766 767 /* Set up auto-mask */ 768 mask = (1 << adapter->vector); 769 for (i = 0, que = adapter->queues; i < adapter->num_queues; i++, que++) 770 mask |= (1 << que->msix); 771 IXGBE_WRITE_REG(hw, IXGBE_VTEIAM, mask); 772 773 /* Set moderation on the Link interrupt */ 774 ixv_eitr_write(adapter, adapter->vector, IXGBE_LINK_ITR); 775 776 /* Stats init */ 777 ixv_init_stats(adapter); 778 779 /* Config/Enable Link */ 780 hw->mac.get_link_status = TRUE; 781 hw->mac.ops.check_link(hw, &adapter->link_speed, &adapter->link_up, 782 FALSE); 783 784 /* Start watchdog */ 785 callout_reset(&adapter->timer, hz, ixv_local_timer, adapter); 786 787 /* And now turn on interrupts */ 788 ixv_enable_intr(adapter); 789 790 /* Update saved flags. See ixgbe_ifflags_cb() */ 791 adapter->if_flags = ifp->if_flags; 792 adapter->ec_capenable = adapter->osdep.ec.ec_capenable; 793 794 /* Now inform the stack we're ready */ 795 ifp->if_flags |= IFF_RUNNING; 796 ifp->if_flags &= ~IFF_OACTIVE; 797 798 return; 799 } /* ixv_init_locked */ 800 801 /************************************************************************ 802 * ixv_enable_queue 803 ************************************************************************/ 804 static inline void 805 ixv_enable_queue(struct adapter *adapter, u32 vector) 806 { 807 struct ixgbe_hw *hw = &adapter->hw; 808 struct ix_queue *que = &adapter->queues[vector]; 809 u32 queue = 1UL << vector; 810 u32 mask; 811 812 mutex_enter(&que->dc_mtx); 813 if (que->disabled_count > 0 && --que->disabled_count > 0) 814 goto out; 815 816 mask = (IXGBE_EIMS_RTX_QUEUE & queue); 817 IXGBE_WRITE_REG(hw, IXGBE_VTEIMS, mask); 818 out: 819 mutex_exit(&que->dc_mtx); 820 } /* ixv_enable_queue */ 821 822 /************************************************************************ 823 * ixv_disable_queue 824 ************************************************************************/ 825 static inline void 826 ixv_disable_queue(struct adapter *adapter, u32 vector) 827 { 828 struct ixgbe_hw *hw = &adapter->hw; 829 struct ix_queue *que = &adapter->queues[vector]; 830 u32 queue = 1UL << vector; 831 u32 mask; 832 833 mutex_enter(&que->dc_mtx); 834 if (que->disabled_count++ > 0) 835 goto out; 836 837 mask = (IXGBE_EIMS_RTX_QUEUE & queue); 838 IXGBE_WRITE_REG(hw, IXGBE_VTEIMC, mask); 839 out: 840 mutex_exit(&que->dc_mtx); 841 } /* ixv_disable_queue */ 842 843 #if 0 844 static inline void 845 ixv_rearm_queues(struct adapter *adapter, u64 queues) 846 { 847 u32 mask = (IXGBE_EIMS_RTX_QUEUE & queues); 848 IXGBE_WRITE_REG(&adapter->hw, IXGBE_VTEICS, mask); 849 } /* ixv_rearm_queues */ 850 #endif 851 852 853 /************************************************************************ 854 * ixv_msix_que - MSI-X Queue Interrupt Service routine 855 ************************************************************************/ 856 static int 857 ixv_msix_que(void *arg) 858 { 859 struct ix_queue *que = arg; 860 struct adapter *adapter = que->adapter; 861 struct tx_ring *txr = que->txr; 862 struct rx_ring *rxr = que->rxr; 863 bool more; 864 u32 newitr = 0; 865 866 ixv_disable_queue(adapter, que->msix); 867 ++que->irqs.ev_count; 868 869 #ifdef __NetBSD__ 870 /* Don't run ixgbe_rxeof in interrupt context */ 871 more = true; 872 #else 873 more = ixgbe_rxeof(que); 874 #endif 875 876 IXGBE_TX_LOCK(txr); 877 ixgbe_txeof(txr); 878 IXGBE_TX_UNLOCK(txr); 879 880 /* Do AIM now? */ 881 882 if (adapter->enable_aim == false) 883 goto no_calc; 884 /* 885 * Do Adaptive Interrupt Moderation: 886 * - Write out last calculated setting 887 * - Calculate based on average size over 888 * the last interval. 889 */ 890 if (que->eitr_setting) 891 ixv_eitr_write(adapter, que->msix, que->eitr_setting); 892 893 que->eitr_setting = 0; 894 895 /* Idle, do nothing */ 896 if ((txr->bytes == 0) && (rxr->bytes == 0)) 897 goto no_calc; 898 899 if ((txr->bytes) && (txr->packets)) 900 newitr = txr->bytes/txr->packets; 901 if ((rxr->bytes) && (rxr->packets)) 902 newitr = uimax(newitr, (rxr->bytes / rxr->packets)); 903 newitr += 24; /* account for hardware frame, crc */ 904 905 /* set an upper boundary */ 906 newitr = uimin(newitr, 3000); 907 908 /* Be nice to the mid range */ 909 if ((newitr > 300) && (newitr < 1200)) 910 newitr = (newitr / 3); 911 else 912 newitr = (newitr / 2); 913 914 /* 915 * When RSC is used, ITR interval must be larger than RSC_DELAY. 916 * Currently, we use 2us for RSC_DELAY. The minimum value is always 917 * greater than 2us on 100M (and 10M?(not documented)), but it's not 918 * on 1G and higher. 919 */ 920 if ((adapter->link_speed != IXGBE_LINK_SPEED_100_FULL) 921 && (adapter->link_speed != IXGBE_LINK_SPEED_10_FULL)) { 922 if (newitr < IXGBE_MIN_RSC_EITR_10G1G) 923 newitr = IXGBE_MIN_RSC_EITR_10G1G; 924 } 925 926 /* save for next interrupt */ 927 que->eitr_setting = newitr; 928 929 /* Reset state */ 930 txr->bytes = 0; 931 txr->packets = 0; 932 rxr->bytes = 0; 933 rxr->packets = 0; 934 935 no_calc: 936 if (more) 937 softint_schedule(que->que_si); 938 else /* Re-enable this interrupt */ 939 ixv_enable_queue(adapter, que->msix); 940 941 return 1; 942 } /* ixv_msix_que */ 943 944 /************************************************************************ 945 * ixv_msix_mbx 946 ************************************************************************/ 947 static int 948 ixv_msix_mbx(void *arg) 949 { 950 struct adapter *adapter = arg; 951 struct ixgbe_hw *hw = &adapter->hw; 952 953 ++adapter->link_irq.ev_count; 954 /* NetBSD: We use auto-clear, so it's not required to write VTEICR */ 955 956 /* Link status change */ 957 hw->mac.get_link_status = TRUE; 958 softint_schedule(adapter->link_si); 959 960 IXGBE_WRITE_REG(hw, IXGBE_VTEIMS, (1 << adapter->vector)); 961 962 return 1; 963 } /* ixv_msix_mbx */ 964 965 static void 966 ixv_eitr_write(struct adapter *adapter, uint32_t index, uint32_t itr) 967 { 968 969 /* 970 * Newer devices than 82598 have VF function, so this function is 971 * simple. 972 */ 973 itr |= IXGBE_EITR_CNT_WDIS; 974 975 IXGBE_WRITE_REG(&adapter->hw, IXGBE_VTEITR(index), itr); 976 } 977 978 979 /************************************************************************ 980 * ixv_media_status - Media Ioctl callback 981 * 982 * Called whenever the user queries the status of 983 * the interface using ifconfig. 984 ************************************************************************/ 985 static void 986 ixv_media_status(struct ifnet *ifp, struct ifmediareq *ifmr) 987 { 988 struct adapter *adapter = ifp->if_softc; 989 990 INIT_DEBUGOUT("ixv_media_status: begin"); 991 IXGBE_CORE_LOCK(adapter); 992 ixv_update_link_status(adapter); 993 994 ifmr->ifm_status = IFM_AVALID; 995 ifmr->ifm_active = IFM_ETHER; 996 997 if (adapter->link_active != LINK_STATE_UP) { 998 ifmr->ifm_active |= IFM_NONE; 999 IXGBE_CORE_UNLOCK(adapter); 1000 return; 1001 } 1002 1003 ifmr->ifm_status |= IFM_ACTIVE; 1004 1005 switch (adapter->link_speed) { 1006 case IXGBE_LINK_SPEED_10GB_FULL: 1007 ifmr->ifm_active |= IFM_10G_T | IFM_FDX; 1008 break; 1009 case IXGBE_LINK_SPEED_5GB_FULL: 1010 ifmr->ifm_active |= IFM_5000_T | IFM_FDX; 1011 break; 1012 case IXGBE_LINK_SPEED_2_5GB_FULL: 1013 ifmr->ifm_active |= IFM_2500_T | IFM_FDX; 1014 break; 1015 case IXGBE_LINK_SPEED_1GB_FULL: 1016 ifmr->ifm_active |= IFM_1000_T | IFM_FDX; 1017 break; 1018 case IXGBE_LINK_SPEED_100_FULL: 1019 ifmr->ifm_active |= IFM_100_TX | IFM_FDX; 1020 break; 1021 case IXGBE_LINK_SPEED_10_FULL: 1022 ifmr->ifm_active |= IFM_10_T | IFM_FDX; 1023 break; 1024 } 1025 1026 ifp->if_baudrate = ifmedia_baudrate(ifmr->ifm_active); 1027 1028 IXGBE_CORE_UNLOCK(adapter); 1029 } /* ixv_media_status */ 1030 1031 /************************************************************************ 1032 * ixv_media_change - Media Ioctl callback 1033 * 1034 * Called when the user changes speed/duplex using 1035 * media/mediopt option with ifconfig. 1036 ************************************************************************/ 1037 static int 1038 ixv_media_change(struct ifnet *ifp) 1039 { 1040 struct adapter *adapter = ifp->if_softc; 1041 struct ifmedia *ifm = &adapter->media; 1042 1043 INIT_DEBUGOUT("ixv_media_change: begin"); 1044 1045 if (IFM_TYPE(ifm->ifm_media) != IFM_ETHER) 1046 return (EINVAL); 1047 1048 switch (IFM_SUBTYPE(ifm->ifm_media)) { 1049 case IFM_AUTO: 1050 break; 1051 default: 1052 device_printf(adapter->dev, "Only auto media type\n"); 1053 return (EINVAL); 1054 } 1055 1056 return (0); 1057 } /* ixv_media_change */ 1058 1059 /************************************************************************ 1060 * ixv_negotiate_api 1061 * 1062 * Negotiate the Mailbox API with the PF; 1063 * start with the most featured API first. 1064 ************************************************************************/ 1065 static int 1066 ixv_negotiate_api(struct adapter *adapter) 1067 { 1068 struct ixgbe_hw *hw = &adapter->hw; 1069 int mbx_api[] = { ixgbe_mbox_api_13, 1070 ixgbe_mbox_api_12, 1071 ixgbe_mbox_api_11, 1072 ixgbe_mbox_api_10, 1073 ixgbe_mbox_api_unknown }; 1074 int i = 0; 1075 1076 while (mbx_api[i] != ixgbe_mbox_api_unknown) { 1077 if (ixgbevf_negotiate_api_version(hw, mbx_api[i]) == 0) 1078 return (0); 1079 i++; 1080 } 1081 1082 return (EINVAL); 1083 } /* ixv_negotiate_api */ 1084 1085 1086 /************************************************************************ 1087 * ixv_set_multi - Multicast Update 1088 * 1089 * Called whenever multicast address list is updated. 1090 ************************************************************************/ 1091 static int 1092 ixv_set_rxfilter(struct adapter *adapter) 1093 { 1094 u8 mta[IXGBE_MAX_VF_MC * IXGBE_ETH_LENGTH_OF_ADDRESS]; 1095 struct ifnet *ifp = adapter->ifp; 1096 struct ixgbe_hw *hw = &adapter->hw; 1097 u8 *update_ptr; 1098 int mcnt = 0; 1099 struct ethercom *ec = &adapter->osdep.ec; 1100 struct ether_multi *enm; 1101 struct ether_multistep step; 1102 bool overflow = false; 1103 int error, rc = 0; 1104 1105 KASSERT(mutex_owned(&adapter->core_mtx)); 1106 IOCTL_DEBUGOUT("ixv_set_rxfilter: begin"); 1107 1108 /* 1: For PROMISC */ 1109 if (ifp->if_flags & IFF_PROMISC) { 1110 error = hw->mac.ops.update_xcast_mode(hw, 1111 IXGBEVF_XCAST_MODE_PROMISC); 1112 if (error == IXGBE_ERR_NOT_TRUSTED) { 1113 device_printf(adapter->dev, 1114 "this interface is not trusted\n"); 1115 error = EPERM; 1116 } else if (error == IXGBE_ERR_FEATURE_NOT_SUPPORTED) { 1117 device_printf(adapter->dev, 1118 "the PF doesn't support promisc mode\n"); 1119 error = EOPNOTSUPP; 1120 } else if (error == IXGBE_ERR_NOT_IN_PROMISC) { 1121 device_printf(adapter->dev, 1122 "the PF may not in promisc mode\n"); 1123 error = EINVAL; 1124 } else if (error) { 1125 device_printf(adapter->dev, 1126 "failed to set promisc mode. error = %d\n", 1127 error); 1128 error = EIO; 1129 } else 1130 return 0; 1131 rc = error; 1132 } 1133 1134 /* 2: For ALLMULTI or normal */ 1135 ETHER_LOCK(ec); 1136 ETHER_FIRST_MULTI(step, ec, enm); 1137 while (enm != NULL) { 1138 if ((mcnt >= IXGBE_MAX_VF_MC) || 1139 (memcmp(enm->enm_addrlo, enm->enm_addrhi, 1140 ETHER_ADDR_LEN) != 0)) { 1141 overflow = true; 1142 break; 1143 } 1144 bcopy(enm->enm_addrlo, 1145 &mta[mcnt * IXGBE_ETH_LENGTH_OF_ADDRESS], 1146 IXGBE_ETH_LENGTH_OF_ADDRESS); 1147 mcnt++; 1148 ETHER_NEXT_MULTI(step, enm); 1149 } 1150 ETHER_UNLOCK(ec); 1151 1152 /* 3: For ALLMULTI */ 1153 if (overflow) { 1154 error = hw->mac.ops.update_xcast_mode(hw, 1155 IXGBEVF_XCAST_MODE_ALLMULTI); 1156 if (error == IXGBE_ERR_NOT_TRUSTED) { 1157 device_printf(adapter->dev, 1158 "this interface is not trusted\n"); 1159 error = EPERM; 1160 } else if (error == IXGBE_ERR_FEATURE_NOT_SUPPORTED) { 1161 device_printf(adapter->dev, 1162 "the PF doesn't support allmulti mode\n"); 1163 error = EOPNOTSUPP; 1164 } else if (error) { 1165 device_printf(adapter->dev, 1166 "number of Ethernet multicast addresses " 1167 "exceeds the limit (%d). error = %d\n", 1168 IXGBE_MAX_VF_MC, error); 1169 error = ENOSPC; 1170 } else { 1171 ETHER_LOCK(ec); 1172 ec->ec_flags |= ETHER_F_ALLMULTI; 1173 ETHER_UNLOCK(ec); 1174 return rc; /* Promisc might failed */ 1175 } 1176 1177 if (rc == 0) 1178 rc = error; 1179 1180 /* Continue to update the multicast table as many as we can */ 1181 } 1182 1183 /* 4: For normal operation */ 1184 error = hw->mac.ops.update_xcast_mode(hw, IXGBEVF_XCAST_MODE_MULTI); 1185 if ((error == IXGBE_ERR_FEATURE_NOT_SUPPORTED) || (error == 0)) { 1186 /* Normal operation */ 1187 ETHER_LOCK(ec); 1188 ec->ec_flags &= ~ETHER_F_ALLMULTI; 1189 ETHER_UNLOCK(ec); 1190 error = 0; 1191 } else if (error) { 1192 device_printf(adapter->dev, 1193 "failed to set Ethernet multicast address " 1194 "operation to normal. error = %d\n", error); 1195 } 1196 1197 update_ptr = mta; 1198 1199 error = adapter->hw.mac.ops.update_mc_addr_list(&adapter->hw, 1200 update_ptr, mcnt, ixv_mc_array_itr, TRUE); 1201 if (rc == 0) 1202 rc = error; 1203 1204 return rc; 1205 } /* ixv_set_rxfilter */ 1206 1207 /************************************************************************ 1208 * ixv_mc_array_itr 1209 * 1210 * An iterator function needed by the multicast shared code. 1211 * It feeds the shared code routine the addresses in the 1212 * array of ixv_set_rxfilter() one by one. 1213 ************************************************************************/ 1214 static u8 * 1215 ixv_mc_array_itr(struct ixgbe_hw *hw, u8 **update_ptr, u32 *vmdq) 1216 { 1217 u8 *addr = *update_ptr; 1218 u8 *newptr; 1219 1220 *vmdq = 0; 1221 1222 newptr = addr + IXGBE_ETH_LENGTH_OF_ADDRESS; 1223 *update_ptr = newptr; 1224 1225 return addr; 1226 } /* ixv_mc_array_itr */ 1227 1228 /************************************************************************ 1229 * ixv_local_timer - Timer routine 1230 * 1231 * Checks for link status, updates statistics, 1232 * and runs the watchdog check. 1233 ************************************************************************/ 1234 static void 1235 ixv_local_timer(void *arg) 1236 { 1237 struct adapter *adapter = arg; 1238 1239 IXGBE_CORE_LOCK(adapter); 1240 ixv_local_timer_locked(adapter); 1241 IXGBE_CORE_UNLOCK(adapter); 1242 } 1243 1244 static void 1245 ixv_local_timer_locked(void *arg) 1246 { 1247 struct adapter *adapter = arg; 1248 device_t dev = adapter->dev; 1249 struct ix_queue *que = adapter->queues; 1250 u64 queues = 0; 1251 u64 v0, v1, v2, v3, v4, v5, v6, v7; 1252 int hung = 0; 1253 int i; 1254 1255 KASSERT(mutex_owned(&adapter->core_mtx)); 1256 1257 if (ixv_check_link(adapter)) { 1258 ixv_init_locked(adapter); 1259 return; 1260 } 1261 1262 /* Stats Update */ 1263 ixv_update_stats(adapter); 1264 1265 /* Update some event counters */ 1266 v0 = v1 = v2 = v3 = v4 = v5 = v6 = v7 = 0; 1267 que = adapter->queues; 1268 for (i = 0; i < adapter->num_queues; i++, que++) { 1269 struct tx_ring *txr = que->txr; 1270 1271 v0 += txr->q_efbig_tx_dma_setup; 1272 v1 += txr->q_mbuf_defrag_failed; 1273 v2 += txr->q_efbig2_tx_dma_setup; 1274 v3 += txr->q_einval_tx_dma_setup; 1275 v4 += txr->q_other_tx_dma_setup; 1276 v5 += txr->q_eagain_tx_dma_setup; 1277 v6 += txr->q_enomem_tx_dma_setup; 1278 v7 += txr->q_tso_err; 1279 } 1280 adapter->efbig_tx_dma_setup.ev_count = v0; 1281 adapter->mbuf_defrag_failed.ev_count = v1; 1282 adapter->efbig2_tx_dma_setup.ev_count = v2; 1283 adapter->einval_tx_dma_setup.ev_count = v3; 1284 adapter->other_tx_dma_setup.ev_count = v4; 1285 adapter->eagain_tx_dma_setup.ev_count = v5; 1286 adapter->enomem_tx_dma_setup.ev_count = v6; 1287 adapter->tso_err.ev_count = v7; 1288 1289 /* 1290 * Check the TX queues status 1291 * - mark hung queues so we don't schedule on them 1292 * - watchdog only if all queues show hung 1293 */ 1294 que = adapter->queues; 1295 for (i = 0; i < adapter->num_queues; i++, que++) { 1296 /* Keep track of queues with work for soft irq */ 1297 if (que->txr->busy) 1298 queues |= ((u64)1 << que->me); 1299 /* 1300 * Each time txeof runs without cleaning, but there 1301 * are uncleaned descriptors it increments busy. If 1302 * we get to the MAX we declare it hung. 1303 */ 1304 if (que->busy == IXGBE_QUEUE_HUNG) { 1305 ++hung; 1306 /* Mark the queue as inactive */ 1307 adapter->active_queues &= ~((u64)1 << que->me); 1308 continue; 1309 } else { 1310 /* Check if we've come back from hung */ 1311 if ((adapter->active_queues & ((u64)1 << que->me)) == 0) 1312 adapter->active_queues |= ((u64)1 << que->me); 1313 } 1314 if (que->busy >= IXGBE_MAX_TX_BUSY) { 1315 device_printf(dev, 1316 "Warning queue %d appears to be hung!\n", i); 1317 que->txr->busy = IXGBE_QUEUE_HUNG; 1318 ++hung; 1319 } 1320 } 1321 1322 /* Only truly watchdog if all queues show hung */ 1323 if (hung == adapter->num_queues) 1324 goto watchdog; 1325 #if 0 1326 else if (queues != 0) { /* Force an IRQ on queues with work */ 1327 ixv_rearm_queues(adapter, queues); 1328 } 1329 #endif 1330 1331 callout_reset(&adapter->timer, hz, ixv_local_timer, adapter); 1332 1333 return; 1334 1335 watchdog: 1336 1337 device_printf(adapter->dev, "Watchdog timeout -- resetting\n"); 1338 adapter->ifp->if_flags &= ~IFF_RUNNING; 1339 adapter->watchdog_events.ev_count++; 1340 ixv_init_locked(adapter); 1341 } /* ixv_local_timer */ 1342 1343 /************************************************************************ 1344 * ixv_update_link_status - Update OS on link state 1345 * 1346 * Note: Only updates the OS on the cached link state. 1347 * The real check of the hardware only happens with 1348 * a link interrupt. 1349 ************************************************************************/ 1350 static void 1351 ixv_update_link_status(struct adapter *adapter) 1352 { 1353 struct ifnet *ifp = adapter->ifp; 1354 device_t dev = adapter->dev; 1355 1356 KASSERT(mutex_owned(&adapter->core_mtx)); 1357 1358 if (adapter->link_up) { 1359 if (adapter->link_active != LINK_STATE_UP) { 1360 if (bootverbose) { 1361 const char *bpsmsg; 1362 1363 switch (adapter->link_speed) { 1364 case IXGBE_LINK_SPEED_10GB_FULL: 1365 bpsmsg = "10 Gbps"; 1366 break; 1367 case IXGBE_LINK_SPEED_5GB_FULL: 1368 bpsmsg = "5 Gbps"; 1369 break; 1370 case IXGBE_LINK_SPEED_2_5GB_FULL: 1371 bpsmsg = "2.5 Gbps"; 1372 break; 1373 case IXGBE_LINK_SPEED_1GB_FULL: 1374 bpsmsg = "1 Gbps"; 1375 break; 1376 case IXGBE_LINK_SPEED_100_FULL: 1377 bpsmsg = "100 Mbps"; 1378 break; 1379 case IXGBE_LINK_SPEED_10_FULL: 1380 bpsmsg = "10 Mbps"; 1381 break; 1382 default: 1383 bpsmsg = "unknown speed"; 1384 break; 1385 } 1386 device_printf(dev, "Link is up %s %s \n", 1387 bpsmsg, "Full Duplex"); 1388 } 1389 adapter->link_active = LINK_STATE_UP; 1390 if_link_state_change(ifp, LINK_STATE_UP); 1391 } 1392 } else { 1393 /* 1394 * Do it when link active changes to DOWN. i.e. 1395 * a) LINK_STATE_UNKNOWN -> LINK_STATE_DOWN 1396 * b) LINK_STATE_UP -> LINK_STATE_DOWN 1397 */ 1398 if (adapter->link_active != LINK_STATE_DOWN) { 1399 if (bootverbose) 1400 device_printf(dev, "Link is Down\n"); 1401 if_link_state_change(ifp, LINK_STATE_DOWN); 1402 adapter->link_active = LINK_STATE_DOWN; 1403 } 1404 } 1405 } /* ixv_update_link_status */ 1406 1407 1408 /************************************************************************ 1409 * ixv_stop - Stop the hardware 1410 * 1411 * Disables all traffic on the adapter by issuing a 1412 * global reset on the MAC and deallocates TX/RX buffers. 1413 ************************************************************************/ 1414 static void 1415 ixv_ifstop(struct ifnet *ifp, int disable) 1416 { 1417 struct adapter *adapter = ifp->if_softc; 1418 1419 IXGBE_CORE_LOCK(adapter); 1420 ixv_stop(adapter); 1421 IXGBE_CORE_UNLOCK(adapter); 1422 } 1423 1424 static void 1425 ixv_stop(void *arg) 1426 { 1427 struct ifnet *ifp; 1428 struct adapter *adapter = arg; 1429 struct ixgbe_hw *hw = &adapter->hw; 1430 1431 ifp = adapter->ifp; 1432 1433 KASSERT(mutex_owned(&adapter->core_mtx)); 1434 1435 INIT_DEBUGOUT("ixv_stop: begin\n"); 1436 ixv_disable_intr(adapter); 1437 1438 /* Tell the stack that the interface is no longer active */ 1439 ifp->if_flags &= ~(IFF_RUNNING | IFF_OACTIVE); 1440 1441 hw->mac.ops.reset_hw(hw); 1442 adapter->hw.adapter_stopped = FALSE; 1443 hw->mac.ops.stop_adapter(hw); 1444 callout_stop(&adapter->timer); 1445 1446 /* reprogram the RAR[0] in case user changed it. */ 1447 hw->mac.ops.set_rar(hw, 0, hw->mac.addr, 0, IXGBE_RAH_AV); 1448 1449 return; 1450 } /* ixv_stop */ 1451 1452 1453 /************************************************************************ 1454 * ixv_allocate_pci_resources 1455 ************************************************************************/ 1456 static int 1457 ixv_allocate_pci_resources(struct adapter *adapter, 1458 const struct pci_attach_args *pa) 1459 { 1460 pcireg_t memtype, csr; 1461 device_t dev = adapter->dev; 1462 bus_addr_t addr; 1463 int flags; 1464 1465 memtype = pci_mapreg_type(pa->pa_pc, pa->pa_tag, PCI_BAR(0)); 1466 switch (memtype) { 1467 case PCI_MAPREG_TYPE_MEM | PCI_MAPREG_MEM_TYPE_32BIT: 1468 case PCI_MAPREG_TYPE_MEM | PCI_MAPREG_MEM_TYPE_64BIT: 1469 adapter->osdep.mem_bus_space_tag = pa->pa_memt; 1470 if (pci_mapreg_info(pa->pa_pc, pa->pa_tag, PCI_BAR(0), 1471 memtype, &addr, &adapter->osdep.mem_size, &flags) != 0) 1472 goto map_err; 1473 if ((flags & BUS_SPACE_MAP_PREFETCHABLE) != 0) { 1474 aprint_normal_dev(dev, "clearing prefetchable bit\n"); 1475 flags &= ~BUS_SPACE_MAP_PREFETCHABLE; 1476 } 1477 if (bus_space_map(adapter->osdep.mem_bus_space_tag, addr, 1478 adapter->osdep.mem_size, flags, 1479 &adapter->osdep.mem_bus_space_handle) != 0) { 1480 map_err: 1481 adapter->osdep.mem_size = 0; 1482 aprint_error_dev(dev, "unable to map BAR0\n"); 1483 return ENXIO; 1484 } 1485 /* 1486 * Enable address decoding for memory range in case it's not 1487 * set. 1488 */ 1489 csr = pci_conf_read(pa->pa_pc, pa->pa_tag, 1490 PCI_COMMAND_STATUS_REG); 1491 csr |= PCI_COMMAND_MEM_ENABLE; 1492 pci_conf_write(pa->pa_pc, pa->pa_tag, PCI_COMMAND_STATUS_REG, 1493 csr); 1494 break; 1495 default: 1496 aprint_error_dev(dev, "unexpected type on BAR0\n"); 1497 return ENXIO; 1498 } 1499 1500 /* Pick up the tuneable queues */ 1501 adapter->num_queues = ixv_num_queues; 1502 1503 return (0); 1504 } /* ixv_allocate_pci_resources */ 1505 1506 /************************************************************************ 1507 * ixv_free_pci_resources 1508 ************************************************************************/ 1509 static void 1510 ixv_free_pci_resources(struct adapter * adapter) 1511 { 1512 struct ix_queue *que = adapter->queues; 1513 int rid; 1514 1515 /* 1516 * Release all msix queue resources: 1517 */ 1518 for (int i = 0; i < adapter->num_queues; i++, que++) { 1519 if (que->res != NULL) 1520 pci_intr_disestablish(adapter->osdep.pc, 1521 adapter->osdep.ihs[i]); 1522 } 1523 1524 1525 /* Clean the Mailbox interrupt last */ 1526 rid = adapter->vector; 1527 1528 if (adapter->osdep.ihs[rid] != NULL) { 1529 pci_intr_disestablish(adapter->osdep.pc, 1530 adapter->osdep.ihs[rid]); 1531 adapter->osdep.ihs[rid] = NULL; 1532 } 1533 1534 pci_intr_release(adapter->osdep.pc, adapter->osdep.intrs, 1535 adapter->osdep.nintrs); 1536 1537 if (adapter->osdep.mem_size != 0) { 1538 bus_space_unmap(adapter->osdep.mem_bus_space_tag, 1539 adapter->osdep.mem_bus_space_handle, 1540 adapter->osdep.mem_size); 1541 } 1542 1543 return; 1544 } /* ixv_free_pci_resources */ 1545 1546 /************************************************************************ 1547 * ixv_setup_interface 1548 * 1549 * Setup networking device structure and register an interface. 1550 ************************************************************************/ 1551 static int 1552 ixv_setup_interface(device_t dev, struct adapter *adapter) 1553 { 1554 struct ethercom *ec = &adapter->osdep.ec; 1555 struct ifnet *ifp; 1556 int rv; 1557 1558 INIT_DEBUGOUT("ixv_setup_interface: begin"); 1559 1560 ifp = adapter->ifp = &ec->ec_if; 1561 strlcpy(ifp->if_xname, device_xname(dev), IFNAMSIZ); 1562 ifp->if_baudrate = IF_Gbps(10); 1563 ifp->if_init = ixv_init; 1564 ifp->if_stop = ixv_ifstop; 1565 ifp->if_softc = adapter; 1566 ifp->if_flags = IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST; 1567 #ifdef IXGBE_MPSAFE 1568 ifp->if_extflags = IFEF_MPSAFE; 1569 #endif 1570 ifp->if_ioctl = ixv_ioctl; 1571 if (adapter->feat_en & IXGBE_FEATURE_LEGACY_TX) { 1572 #if 0 1573 ixv_start_locked = ixgbe_legacy_start_locked; 1574 #endif 1575 } else { 1576 ifp->if_transmit = ixgbe_mq_start; 1577 #if 0 1578 ixv_start_locked = ixgbe_mq_start_locked; 1579 #endif 1580 } 1581 ifp->if_start = ixgbe_legacy_start; 1582 IFQ_SET_MAXLEN(&ifp->if_snd, adapter->num_tx_desc - 2); 1583 IFQ_SET_READY(&ifp->if_snd); 1584 1585 rv = if_initialize(ifp); 1586 if (rv != 0) { 1587 aprint_error_dev(dev, "if_initialize failed(%d)\n", rv); 1588 return rv; 1589 } 1590 adapter->ipq = if_percpuq_create(&adapter->osdep.ec.ec_if); 1591 ether_ifattach(ifp, adapter->hw.mac.addr); 1592 aprint_normal_dev(dev, "Ethernet address %s\n", 1593 ether_sprintf(adapter->hw.mac.addr)); 1594 /* 1595 * We use per TX queue softint, so if_deferred_start_init() isn't 1596 * used. 1597 */ 1598 ether_set_ifflags_cb(ec, ixv_ifflags_cb); 1599 1600 adapter->max_frame_size = ifp->if_mtu + IXGBE_MTU_HDR; 1601 1602 /* 1603 * Tell the upper layer(s) we support long frames. 1604 */ 1605 ifp->if_hdrlen = sizeof(struct ether_vlan_header); 1606 1607 /* Set capability flags */ 1608 ifp->if_capabilities |= IFCAP_HWCSUM 1609 | IFCAP_TSOv4 1610 | IFCAP_TSOv6; 1611 ifp->if_capenable = 0; 1612 1613 ec->ec_capabilities |= ETHERCAP_VLAN_HWFILTER 1614 | ETHERCAP_VLAN_HWTAGGING 1615 | ETHERCAP_VLAN_HWCSUM 1616 | ETHERCAP_JUMBO_MTU 1617 | ETHERCAP_VLAN_MTU; 1618 1619 /* Enable the above capabilities by default */ 1620 ec->ec_capenable = ec->ec_capabilities; 1621 1622 /* Don't enable LRO by default */ 1623 #if 0 1624 /* NetBSD doesn't support LRO yet */ 1625 ifp->if_capabilities |= IFCAP_LRO; 1626 #endif 1627 1628 /* 1629 * Specify the media types supported by this adapter and register 1630 * callbacks to update media and link information 1631 */ 1632 ec->ec_ifmedia = &adapter->media; 1633 ifmedia_init(&adapter->media, IFM_IMASK, ixv_media_change, 1634 ixv_media_status); 1635 ifmedia_add(&adapter->media, IFM_ETHER | IFM_AUTO, 0, NULL); 1636 ifmedia_set(&adapter->media, IFM_ETHER | IFM_AUTO); 1637 1638 if_register(ifp); 1639 1640 return 0; 1641 } /* ixv_setup_interface */ 1642 1643 1644 /************************************************************************ 1645 * ixv_initialize_transmit_units - Enable transmit unit. 1646 ************************************************************************/ 1647 static void 1648 ixv_initialize_transmit_units(struct adapter *adapter) 1649 { 1650 struct tx_ring *txr = adapter->tx_rings; 1651 struct ixgbe_hw *hw = &adapter->hw; 1652 int i; 1653 1654 for (i = 0; i < adapter->num_queues; i++, txr++) { 1655 u64 tdba = txr->txdma.dma_paddr; 1656 u32 txctrl, txdctl; 1657 int j = txr->me; 1658 1659 /* Set WTHRESH to 8, burst writeback */ 1660 txdctl = IXGBE_READ_REG(hw, IXGBE_VFTXDCTL(j)); 1661 txdctl |= (8 << 16); 1662 IXGBE_WRITE_REG(hw, IXGBE_VFTXDCTL(j), txdctl); 1663 1664 /* Set the HW Tx Head and Tail indices */ 1665 IXGBE_WRITE_REG(&adapter->hw, IXGBE_VFTDH(j), 0); 1666 IXGBE_WRITE_REG(&adapter->hw, IXGBE_VFTDT(j), 0); 1667 1668 /* Set Tx Tail register */ 1669 txr->tail = IXGBE_VFTDT(j); 1670 1671 txr->txr_no_space = false; 1672 1673 /* Set Ring parameters */ 1674 IXGBE_WRITE_REG(hw, IXGBE_VFTDBAL(j), 1675 (tdba & 0x00000000ffffffffULL)); 1676 IXGBE_WRITE_REG(hw, IXGBE_VFTDBAH(j), (tdba >> 32)); 1677 IXGBE_WRITE_REG(hw, IXGBE_VFTDLEN(j), 1678 adapter->num_tx_desc * sizeof(struct ixgbe_legacy_tx_desc)); 1679 txctrl = IXGBE_READ_REG(hw, IXGBE_VFDCA_TXCTRL(j)); 1680 txctrl &= ~IXGBE_DCA_TXCTRL_DESC_WRO_EN; 1681 IXGBE_WRITE_REG(hw, IXGBE_VFDCA_TXCTRL(j), txctrl); 1682 1683 /* Now enable */ 1684 txdctl = IXGBE_READ_REG(hw, IXGBE_VFTXDCTL(j)); 1685 txdctl |= IXGBE_TXDCTL_ENABLE; 1686 IXGBE_WRITE_REG(hw, IXGBE_VFTXDCTL(j), txdctl); 1687 } 1688 1689 return; 1690 } /* ixv_initialize_transmit_units */ 1691 1692 1693 /************************************************************************ 1694 * ixv_initialize_rss_mapping 1695 ************************************************************************/ 1696 static void 1697 ixv_initialize_rss_mapping(struct adapter *adapter) 1698 { 1699 struct ixgbe_hw *hw = &adapter->hw; 1700 u32 reta = 0, mrqc, rss_key[10]; 1701 int queue_id; 1702 int i, j; 1703 u32 rss_hash_config; 1704 1705 /* force use default RSS key. */ 1706 #ifdef __NetBSD__ 1707 rss_getkey((uint8_t *) &rss_key); 1708 #else 1709 if (adapter->feat_en & IXGBE_FEATURE_RSS) { 1710 /* Fetch the configured RSS key */ 1711 rss_getkey((uint8_t *)&rss_key); 1712 } else { 1713 /* set up random bits */ 1714 cprng_fast(&rss_key, sizeof(rss_key)); 1715 } 1716 #endif 1717 1718 /* Now fill out hash function seeds */ 1719 for (i = 0; i < 10; i++) 1720 IXGBE_WRITE_REG(hw, IXGBE_VFRSSRK(i), rss_key[i]); 1721 1722 /* Set up the redirection table */ 1723 for (i = 0, j = 0; i < 64; i++, j++) { 1724 if (j == adapter->num_queues) 1725 j = 0; 1726 1727 if (adapter->feat_en & IXGBE_FEATURE_RSS) { 1728 /* 1729 * Fetch the RSS bucket id for the given indirection 1730 * entry. Cap it at the number of configured buckets 1731 * (which is num_queues.) 1732 */ 1733 queue_id = rss_get_indirection_to_bucket(i); 1734 queue_id = queue_id % adapter->num_queues; 1735 } else 1736 queue_id = j; 1737 1738 /* 1739 * The low 8 bits are for hash value (n+0); 1740 * The next 8 bits are for hash value (n+1), etc. 1741 */ 1742 reta >>= 8; 1743 reta |= ((uint32_t)queue_id) << 24; 1744 if ((i & 3) == 3) { 1745 IXGBE_WRITE_REG(hw, IXGBE_VFRETA(i >> 2), reta); 1746 reta = 0; 1747 } 1748 } 1749 1750 /* Perform hash on these packet types */ 1751 if (adapter->feat_en & IXGBE_FEATURE_RSS) 1752 rss_hash_config = rss_gethashconfig(); 1753 else { 1754 /* 1755 * Disable UDP - IP fragments aren't currently being handled 1756 * and so we end up with a mix of 2-tuple and 4-tuple 1757 * traffic. 1758 */ 1759 rss_hash_config = RSS_HASHTYPE_RSS_IPV4 1760 | RSS_HASHTYPE_RSS_TCP_IPV4 1761 | RSS_HASHTYPE_RSS_IPV6 1762 | RSS_HASHTYPE_RSS_TCP_IPV6; 1763 } 1764 1765 mrqc = IXGBE_MRQC_RSSEN; 1766 if (rss_hash_config & RSS_HASHTYPE_RSS_IPV4) 1767 mrqc |= IXGBE_MRQC_RSS_FIELD_IPV4; 1768 if (rss_hash_config & RSS_HASHTYPE_RSS_TCP_IPV4) 1769 mrqc |= IXGBE_MRQC_RSS_FIELD_IPV4_TCP; 1770 if (rss_hash_config & RSS_HASHTYPE_RSS_IPV6) 1771 mrqc |= IXGBE_MRQC_RSS_FIELD_IPV6; 1772 if (rss_hash_config & RSS_HASHTYPE_RSS_TCP_IPV6) 1773 mrqc |= IXGBE_MRQC_RSS_FIELD_IPV6_TCP; 1774 if (rss_hash_config & RSS_HASHTYPE_RSS_IPV6_EX) 1775 device_printf(adapter->dev, "%s: RSS_HASHTYPE_RSS_IPV6_EX defined, but not supported\n", 1776 __func__); 1777 if (rss_hash_config & RSS_HASHTYPE_RSS_TCP_IPV6_EX) 1778 device_printf(adapter->dev, "%s: RSS_HASHTYPE_RSS_TCP_IPV6_EX defined, but not supported\n", 1779 __func__); 1780 if (rss_hash_config & RSS_HASHTYPE_RSS_UDP_IPV4) 1781 mrqc |= IXGBE_MRQC_RSS_FIELD_IPV4_UDP; 1782 if (rss_hash_config & RSS_HASHTYPE_RSS_UDP_IPV6) 1783 mrqc |= IXGBE_MRQC_RSS_FIELD_IPV6_UDP; 1784 if (rss_hash_config & RSS_HASHTYPE_RSS_UDP_IPV6_EX) 1785 device_printf(adapter->dev, "%s: RSS_HASHTYPE_RSS_UDP_IPV6_EX defined, but not supported\n", 1786 __func__); 1787 IXGBE_WRITE_REG(hw, IXGBE_VFMRQC, mrqc); 1788 } /* ixv_initialize_rss_mapping */ 1789 1790 1791 /************************************************************************ 1792 * ixv_initialize_receive_units - Setup receive registers and features. 1793 ************************************************************************/ 1794 static void 1795 ixv_initialize_receive_units(struct adapter *adapter) 1796 { 1797 struct rx_ring *rxr = adapter->rx_rings; 1798 struct ixgbe_hw *hw = &adapter->hw; 1799 struct ifnet *ifp = adapter->ifp; 1800 u32 bufsz, psrtype; 1801 1802 if (ifp->if_mtu > ETHERMTU) 1803 bufsz = 4096 >> IXGBE_SRRCTL_BSIZEPKT_SHIFT; 1804 else 1805 bufsz = 2048 >> IXGBE_SRRCTL_BSIZEPKT_SHIFT; 1806 1807 psrtype = IXGBE_PSRTYPE_TCPHDR 1808 | IXGBE_PSRTYPE_UDPHDR 1809 | IXGBE_PSRTYPE_IPV4HDR 1810 | IXGBE_PSRTYPE_IPV6HDR 1811 | IXGBE_PSRTYPE_L2HDR; 1812 1813 if (adapter->num_queues > 1) 1814 psrtype |= 1 << 29; 1815 1816 IXGBE_WRITE_REG(hw, IXGBE_VFPSRTYPE, psrtype); 1817 1818 /* Tell PF our max_frame size */ 1819 if (ixgbevf_rlpml_set_vf(hw, adapter->max_frame_size) != 0) { 1820 device_printf(adapter->dev, "There is a problem with the PF setup. It is likely the receive unit for this VF will not function correctly.\n"); 1821 } 1822 1823 for (int i = 0; i < adapter->num_queues; i++, rxr++) { 1824 u64 rdba = rxr->rxdma.dma_paddr; 1825 u32 reg, rxdctl; 1826 int j = rxr->me; 1827 1828 /* Disable the queue */ 1829 rxdctl = IXGBE_READ_REG(hw, IXGBE_VFRXDCTL(j)); 1830 rxdctl &= ~IXGBE_RXDCTL_ENABLE; 1831 IXGBE_WRITE_REG(hw, IXGBE_VFRXDCTL(j), rxdctl); 1832 for (int k = 0; k < 10; k++) { 1833 if (IXGBE_READ_REG(hw, IXGBE_VFRXDCTL(j)) & 1834 IXGBE_RXDCTL_ENABLE) 1835 msec_delay(1); 1836 else 1837 break; 1838 } 1839 IXGBE_WRITE_BARRIER(hw); 1840 /* Setup the Base and Length of the Rx Descriptor Ring */ 1841 IXGBE_WRITE_REG(hw, IXGBE_VFRDBAL(j), 1842 (rdba & 0x00000000ffffffffULL)); 1843 IXGBE_WRITE_REG(hw, IXGBE_VFRDBAH(j), (rdba >> 32)); 1844 IXGBE_WRITE_REG(hw, IXGBE_VFRDLEN(j), 1845 adapter->num_rx_desc * sizeof(union ixgbe_adv_rx_desc)); 1846 1847 /* Reset the ring indices */ 1848 IXGBE_WRITE_REG(hw, IXGBE_VFRDH(rxr->me), 0); 1849 IXGBE_WRITE_REG(hw, IXGBE_VFRDT(rxr->me), 0); 1850 1851 /* Set up the SRRCTL register */ 1852 reg = IXGBE_READ_REG(hw, IXGBE_VFSRRCTL(j)); 1853 reg &= ~IXGBE_SRRCTL_BSIZEHDR_MASK; 1854 reg &= ~IXGBE_SRRCTL_BSIZEPKT_MASK; 1855 reg |= bufsz; 1856 reg |= IXGBE_SRRCTL_DESCTYPE_ADV_ONEBUF; 1857 IXGBE_WRITE_REG(hw, IXGBE_VFSRRCTL(j), reg); 1858 1859 /* Capture Rx Tail index */ 1860 rxr->tail = IXGBE_VFRDT(rxr->me); 1861 1862 /* Do the queue enabling last */ 1863 rxdctl |= IXGBE_RXDCTL_ENABLE | IXGBE_RXDCTL_VME; 1864 IXGBE_WRITE_REG(hw, IXGBE_VFRXDCTL(j), rxdctl); 1865 for (int k = 0; k < 10; k++) { 1866 if (IXGBE_READ_REG(hw, IXGBE_VFRXDCTL(j)) & 1867 IXGBE_RXDCTL_ENABLE) 1868 break; 1869 msec_delay(1); 1870 } 1871 IXGBE_WRITE_BARRIER(hw); 1872 1873 /* Set the Tail Pointer */ 1874 #ifdef DEV_NETMAP 1875 /* 1876 * In netmap mode, we must preserve the buffers made 1877 * available to userspace before the if_init() 1878 * (this is true by default on the TX side, because 1879 * init makes all buffers available to userspace). 1880 * 1881 * netmap_reset() and the device specific routines 1882 * (e.g. ixgbe_setup_receive_rings()) map these 1883 * buffers at the end of the NIC ring, so here we 1884 * must set the RDT (tail) register to make sure 1885 * they are not overwritten. 1886 * 1887 * In this driver the NIC ring starts at RDH = 0, 1888 * RDT points to the last slot available for reception (?), 1889 * so RDT = num_rx_desc - 1 means the whole ring is available. 1890 */ 1891 if ((adapter->feat_en & IXGBE_FEATURE_NETMAP) && 1892 (ifp->if_capenable & IFCAP_NETMAP)) { 1893 struct netmap_adapter *na = NA(adapter->ifp); 1894 struct netmap_kring *kring = na->rx_rings[i]; 1895 int t = na->num_rx_desc - 1 - nm_kr_rxspace(kring); 1896 1897 IXGBE_WRITE_REG(hw, IXGBE_VFRDT(rxr->me), t); 1898 } else 1899 #endif /* DEV_NETMAP */ 1900 IXGBE_WRITE_REG(hw, IXGBE_VFRDT(rxr->me), 1901 adapter->num_rx_desc - 1); 1902 } 1903 1904 if (adapter->hw.mac.type >= ixgbe_mac_X550_vf) 1905 ixv_initialize_rss_mapping(adapter); 1906 } /* ixv_initialize_receive_units */ 1907 1908 /************************************************************************ 1909 * ixv_sysctl_tdh_handler - Transmit Descriptor Head handler function 1910 * 1911 * Retrieves the TDH value from the hardware 1912 ************************************************************************/ 1913 static int 1914 ixv_sysctl_tdh_handler(SYSCTLFN_ARGS) 1915 { 1916 struct sysctlnode node = *rnode; 1917 struct tx_ring *txr = (struct tx_ring *)node.sysctl_data; 1918 uint32_t val; 1919 1920 if (!txr) 1921 return (0); 1922 1923 val = IXGBE_READ_REG(&txr->adapter->hw, IXGBE_VFTDH(txr->me)); 1924 node.sysctl_data = &val; 1925 return sysctl_lookup(SYSCTLFN_CALL(&node)); 1926 } /* ixv_sysctl_tdh_handler */ 1927 1928 /************************************************************************ 1929 * ixgbe_sysctl_tdt_handler - Transmit Descriptor Tail handler function 1930 * 1931 * Retrieves the TDT value from the hardware 1932 ************************************************************************/ 1933 static int 1934 ixv_sysctl_tdt_handler(SYSCTLFN_ARGS) 1935 { 1936 struct sysctlnode node = *rnode; 1937 struct tx_ring *txr = (struct tx_ring *)node.sysctl_data; 1938 uint32_t val; 1939 1940 if (!txr) 1941 return (0); 1942 1943 val = IXGBE_READ_REG(&txr->adapter->hw, IXGBE_VFTDT(txr->me)); 1944 node.sysctl_data = &val; 1945 return sysctl_lookup(SYSCTLFN_CALL(&node)); 1946 } /* ixv_sysctl_tdt_handler */ 1947 1948 /************************************************************************ 1949 * ixv_sysctl_next_to_check_handler - Receive Descriptor next to check 1950 * handler function 1951 * 1952 * Retrieves the next_to_check value 1953 ************************************************************************/ 1954 static int 1955 ixv_sysctl_next_to_check_handler(SYSCTLFN_ARGS) 1956 { 1957 struct sysctlnode node = *rnode; 1958 struct rx_ring *rxr = (struct rx_ring *)node.sysctl_data; 1959 uint32_t val; 1960 1961 if (!rxr) 1962 return (0); 1963 1964 val = rxr->next_to_check; 1965 node.sysctl_data = &val; 1966 return sysctl_lookup(SYSCTLFN_CALL(&node)); 1967 } /* ixv_sysctl_next_to_check_handler */ 1968 1969 /************************************************************************ 1970 * ixv_sysctl_rdh_handler - Receive Descriptor Head handler function 1971 * 1972 * Retrieves the RDH value from the hardware 1973 ************************************************************************/ 1974 static int 1975 ixv_sysctl_rdh_handler(SYSCTLFN_ARGS) 1976 { 1977 struct sysctlnode node = *rnode; 1978 struct rx_ring *rxr = (struct rx_ring *)node.sysctl_data; 1979 uint32_t val; 1980 1981 if (!rxr) 1982 return (0); 1983 1984 val = IXGBE_READ_REG(&rxr->adapter->hw, IXGBE_VFRDH(rxr->me)); 1985 node.sysctl_data = &val; 1986 return sysctl_lookup(SYSCTLFN_CALL(&node)); 1987 } /* ixv_sysctl_rdh_handler */ 1988 1989 /************************************************************************ 1990 * ixv_sysctl_rdt_handler - Receive Descriptor Tail handler function 1991 * 1992 * Retrieves the RDT value from the hardware 1993 ************************************************************************/ 1994 static int 1995 ixv_sysctl_rdt_handler(SYSCTLFN_ARGS) 1996 { 1997 struct sysctlnode node = *rnode; 1998 struct rx_ring *rxr = (struct rx_ring *)node.sysctl_data; 1999 uint32_t val; 2000 2001 if (!rxr) 2002 return (0); 2003 2004 val = IXGBE_READ_REG(&rxr->adapter->hw, IXGBE_VFRDT(rxr->me)); 2005 node.sysctl_data = &val; 2006 return sysctl_lookup(SYSCTLFN_CALL(&node)); 2007 } /* ixv_sysctl_rdt_handler */ 2008 2009 static void 2010 ixv_setup_vlan_tagging(struct adapter *adapter) 2011 { 2012 struct ethercom *ec = &adapter->osdep.ec; 2013 struct ixgbe_hw *hw = &adapter->hw; 2014 struct rx_ring *rxr; 2015 u32 ctrl; 2016 int i; 2017 bool hwtagging; 2018 2019 /* Enable HW tagging only if any vlan is attached */ 2020 hwtagging = (ec->ec_capenable & ETHERCAP_VLAN_HWTAGGING) 2021 && VLAN_ATTACHED(ec); 2022 2023 /* Enable the queues */ 2024 for (i = 0; i < adapter->num_queues; i++) { 2025 rxr = &adapter->rx_rings[i]; 2026 ctrl = IXGBE_READ_REG(hw, IXGBE_VFRXDCTL(rxr->me)); 2027 if (hwtagging) 2028 ctrl |= IXGBE_RXDCTL_VME; 2029 else 2030 ctrl &= ~IXGBE_RXDCTL_VME; 2031 IXGBE_WRITE_REG(hw, IXGBE_VFRXDCTL(rxr->me), ctrl); 2032 /* 2033 * Let Rx path know that it needs to store VLAN tag 2034 * as part of extra mbuf info. 2035 */ 2036 rxr->vtag_strip = hwtagging ? TRUE : FALSE; 2037 } 2038 } /* ixv_setup_vlan_tagging */ 2039 2040 /************************************************************************ 2041 * ixv_setup_vlan_support 2042 ************************************************************************/ 2043 static int 2044 ixv_setup_vlan_support(struct adapter *adapter) 2045 { 2046 struct ethercom *ec = &adapter->osdep.ec; 2047 struct ixgbe_hw *hw = &adapter->hw; 2048 u32 vid, vfta, retry; 2049 struct vlanid_list *vlanidp; 2050 int rv, error = 0; 2051 2052 /* 2053 * This function is called from both if_init and ifflags_cb() 2054 * on NetBSD. 2055 */ 2056 2057 /* 2058 * Part 1: 2059 * Setup VLAN HW tagging 2060 */ 2061 ixv_setup_vlan_tagging(adapter); 2062 2063 if (!VLAN_ATTACHED(ec)) 2064 return 0; 2065 2066 /* 2067 * Part 2: 2068 * Setup VLAN HW filter 2069 */ 2070 /* Cleanup shadow_vfta */ 2071 for (int i = 0; i < IXGBE_VFTA_SIZE; i++) 2072 adapter->shadow_vfta[i] = 0; 2073 /* Generate shadow_vfta from ec_vids */ 2074 ETHER_LOCK(ec); 2075 SIMPLEQ_FOREACH(vlanidp, &ec->ec_vids, vid_list) { 2076 uint32_t idx; 2077 2078 idx = vlanidp->vid / 32; 2079 KASSERT(idx < IXGBE_VFTA_SIZE); 2080 adapter->shadow_vfta[idx] |= (u32)1 << (vlanidp->vid % 32); 2081 } 2082 ETHER_UNLOCK(ec); 2083 2084 /* 2085 * A soft reset zero's out the VFTA, so 2086 * we need to repopulate it now. 2087 */ 2088 for (int i = 0; i < IXGBE_VFTA_SIZE; i++) { 2089 if (adapter->shadow_vfta[i] == 0) 2090 continue; 2091 vfta = adapter->shadow_vfta[i]; 2092 /* 2093 * Reconstruct the vlan id's 2094 * based on the bits set in each 2095 * of the array ints. 2096 */ 2097 for (int j = 0; j < 32; j++) { 2098 retry = 0; 2099 if ((vfta & ((u32)1 << j)) == 0) 2100 continue; 2101 vid = (i * 32) + j; 2102 2103 /* Call the shared code mailbox routine */ 2104 while ((rv = hw->mac.ops.set_vfta(hw, vid, 0, TRUE, 2105 FALSE)) != 0) { 2106 if (++retry > 5) { 2107 device_printf(adapter->dev, 2108 "%s: max retry exceeded\n", 2109 __func__); 2110 break; 2111 } 2112 } 2113 if (rv != 0) { 2114 device_printf(adapter->dev, 2115 "failed to set vlan %d\n", vid); 2116 error = EACCES; 2117 } 2118 } 2119 } 2120 return error; 2121 } /* ixv_setup_vlan_support */ 2122 2123 static int 2124 ixv_vlan_cb(struct ethercom *ec, uint16_t vid, bool set) 2125 { 2126 struct ifnet *ifp = &ec->ec_if; 2127 struct adapter *adapter = ifp->if_softc; 2128 int rv; 2129 2130 if (set) 2131 rv = ixv_register_vlan(adapter, vid); 2132 else 2133 rv = ixv_unregister_vlan(adapter, vid); 2134 2135 if (rv != 0) 2136 return rv; 2137 2138 /* 2139 * Control VLAN HW tagging when ec_nvlan is changed from 1 to 0 2140 * or 0 to 1. 2141 */ 2142 if ((set && (ec->ec_nvlans == 1)) || (!set && (ec->ec_nvlans == 0))) 2143 ixv_setup_vlan_tagging(adapter); 2144 2145 return rv; 2146 } 2147 2148 /************************************************************************ 2149 * ixv_register_vlan 2150 * 2151 * Run via a vlan config EVENT, it enables us to use the 2152 * HW Filter table since we can get the vlan id. This just 2153 * creates the entry in the soft version of the VFTA, init 2154 * will repopulate the real table. 2155 ************************************************************************/ 2156 static int 2157 ixv_register_vlan(struct adapter *adapter, u16 vtag) 2158 { 2159 struct ixgbe_hw *hw = &adapter->hw; 2160 u16 index, bit; 2161 int error; 2162 2163 if ((vtag == 0) || (vtag > 4095)) /* Invalid */ 2164 return EINVAL; 2165 IXGBE_CORE_LOCK(adapter); 2166 index = (vtag >> 5) & 0x7F; 2167 bit = vtag & 0x1F; 2168 adapter->shadow_vfta[index] |= ((u32)1 << bit); 2169 error = hw->mac.ops.set_vfta(hw, vtag, 0, true, false); 2170 IXGBE_CORE_UNLOCK(adapter); 2171 2172 if (error != 0) { 2173 device_printf(adapter->dev, "failed to register vlan %hu\n", 2174 vtag); 2175 error = EACCES; 2176 } 2177 return error; 2178 } /* ixv_register_vlan */ 2179 2180 /************************************************************************ 2181 * ixv_unregister_vlan 2182 * 2183 * Run via a vlan unconfig EVENT, remove our entry 2184 * in the soft vfta. 2185 ************************************************************************/ 2186 static int 2187 ixv_unregister_vlan(struct adapter *adapter, u16 vtag) 2188 { 2189 struct ixgbe_hw *hw = &adapter->hw; 2190 u16 index, bit; 2191 int error; 2192 2193 if ((vtag == 0) || (vtag > 4095)) /* Invalid */ 2194 return EINVAL; 2195 2196 IXGBE_CORE_LOCK(adapter); 2197 index = (vtag >> 5) & 0x7F; 2198 bit = vtag & 0x1F; 2199 adapter->shadow_vfta[index] &= ~((u32)1 << bit); 2200 error = hw->mac.ops.set_vfta(hw, vtag, 0, false, false); 2201 IXGBE_CORE_UNLOCK(adapter); 2202 2203 if (error != 0) { 2204 device_printf(adapter->dev, "failed to unregister vlan %hu\n", 2205 vtag); 2206 error = EIO; 2207 } 2208 return error; 2209 } /* ixv_unregister_vlan */ 2210 2211 /************************************************************************ 2212 * ixv_enable_intr 2213 ************************************************************************/ 2214 static void 2215 ixv_enable_intr(struct adapter *adapter) 2216 { 2217 struct ixgbe_hw *hw = &adapter->hw; 2218 struct ix_queue *que = adapter->queues; 2219 u32 mask; 2220 int i; 2221 2222 /* For VTEIAC */ 2223 mask = (1 << adapter->vector); 2224 for (i = 0; i < adapter->num_queues; i++, que++) 2225 mask |= (1 << que->msix); 2226 IXGBE_WRITE_REG(hw, IXGBE_VTEIAC, mask); 2227 2228 /* For VTEIMS */ 2229 IXGBE_WRITE_REG(hw, IXGBE_VTEIMS, (1 << adapter->vector)); 2230 que = adapter->queues; 2231 for (i = 0; i < adapter->num_queues; i++, que++) 2232 ixv_enable_queue(adapter, que->msix); 2233 2234 IXGBE_WRITE_FLUSH(hw); 2235 } /* ixv_enable_intr */ 2236 2237 /************************************************************************ 2238 * ixv_disable_intr 2239 ************************************************************************/ 2240 static void 2241 ixv_disable_intr(struct adapter *adapter) 2242 { 2243 struct ix_queue *que = adapter->queues; 2244 2245 IXGBE_WRITE_REG(&adapter->hw, IXGBE_VTEIAC, 0); 2246 2247 /* disable interrupts other than queues */ 2248 IXGBE_WRITE_REG(&adapter->hw, IXGBE_VTEIMC, adapter->vector); 2249 2250 for (int i = 0; i < adapter->num_queues; i++, que++) 2251 ixv_disable_queue(adapter, que->msix); 2252 2253 IXGBE_WRITE_FLUSH(&adapter->hw); 2254 } /* ixv_disable_intr */ 2255 2256 /************************************************************************ 2257 * ixv_set_ivar 2258 * 2259 * Setup the correct IVAR register for a particular MSI-X interrupt 2260 * - entry is the register array entry 2261 * - vector is the MSI-X vector for this queue 2262 * - type is RX/TX/MISC 2263 ************************************************************************/ 2264 static void 2265 ixv_set_ivar(struct adapter *adapter, u8 entry, u8 vector, s8 type) 2266 { 2267 struct ixgbe_hw *hw = &adapter->hw; 2268 u32 ivar, index; 2269 2270 vector |= IXGBE_IVAR_ALLOC_VAL; 2271 2272 if (type == -1) { /* MISC IVAR */ 2273 ivar = IXGBE_READ_REG(hw, IXGBE_VTIVAR_MISC); 2274 ivar &= ~0xFF; 2275 ivar |= vector; 2276 IXGBE_WRITE_REG(hw, IXGBE_VTIVAR_MISC, ivar); 2277 } else { /* RX/TX IVARS */ 2278 index = (16 * (entry & 1)) + (8 * type); 2279 ivar = IXGBE_READ_REG(hw, IXGBE_VTIVAR(entry >> 1)); 2280 ivar &= ~(0xffUL << index); 2281 ivar |= ((u32)vector << index); 2282 IXGBE_WRITE_REG(hw, IXGBE_VTIVAR(entry >> 1), ivar); 2283 } 2284 } /* ixv_set_ivar */ 2285 2286 /************************************************************************ 2287 * ixv_configure_ivars 2288 ************************************************************************/ 2289 static void 2290 ixv_configure_ivars(struct adapter *adapter) 2291 { 2292 struct ix_queue *que = adapter->queues; 2293 2294 /* XXX We should sync EITR value calculation with ixgbe.c? */ 2295 2296 for (int i = 0; i < adapter->num_queues; i++, que++) { 2297 /* First the RX queue entry */ 2298 ixv_set_ivar(adapter, i, que->msix, 0); 2299 /* ... and the TX */ 2300 ixv_set_ivar(adapter, i, que->msix, 1); 2301 /* Set an initial value in EITR */ 2302 ixv_eitr_write(adapter, que->msix, IXGBE_EITR_DEFAULT); 2303 } 2304 2305 /* For the mailbox interrupt */ 2306 ixv_set_ivar(adapter, 1, adapter->vector, -1); 2307 } /* ixv_configure_ivars */ 2308 2309 2310 /************************************************************************ 2311 * ixv_save_stats 2312 * 2313 * The VF stats registers never have a truly virgin 2314 * starting point, so this routine tries to make an 2315 * artificial one, marking ground zero on attach as 2316 * it were. 2317 ************************************************************************/ 2318 static void 2319 ixv_save_stats(struct adapter *adapter) 2320 { 2321 struct ixgbevf_hw_stats *stats = &adapter->stats.vf; 2322 2323 if (stats->vfgprc.ev_count || stats->vfgptc.ev_count) { 2324 stats->saved_reset_vfgprc += 2325 stats->vfgprc.ev_count - stats->base_vfgprc; 2326 stats->saved_reset_vfgptc += 2327 stats->vfgptc.ev_count - stats->base_vfgptc; 2328 stats->saved_reset_vfgorc += 2329 stats->vfgorc.ev_count - stats->base_vfgorc; 2330 stats->saved_reset_vfgotc += 2331 stats->vfgotc.ev_count - stats->base_vfgotc; 2332 stats->saved_reset_vfmprc += 2333 stats->vfmprc.ev_count - stats->base_vfmprc; 2334 } 2335 } /* ixv_save_stats */ 2336 2337 /************************************************************************ 2338 * ixv_init_stats 2339 ************************************************************************/ 2340 static void 2341 ixv_init_stats(struct adapter *adapter) 2342 { 2343 struct ixgbe_hw *hw = &adapter->hw; 2344 2345 adapter->stats.vf.last_vfgprc = IXGBE_READ_REG(hw, IXGBE_VFGPRC); 2346 adapter->stats.vf.last_vfgorc = IXGBE_READ_REG(hw, IXGBE_VFGORC_LSB); 2347 adapter->stats.vf.last_vfgorc |= 2348 (((u64)(IXGBE_READ_REG(hw, IXGBE_VFGORC_MSB))) << 32); 2349 2350 adapter->stats.vf.last_vfgptc = IXGBE_READ_REG(hw, IXGBE_VFGPTC); 2351 adapter->stats.vf.last_vfgotc = IXGBE_READ_REG(hw, IXGBE_VFGOTC_LSB); 2352 adapter->stats.vf.last_vfgotc |= 2353 (((u64)(IXGBE_READ_REG(hw, IXGBE_VFGOTC_MSB))) << 32); 2354 2355 adapter->stats.vf.last_vfmprc = IXGBE_READ_REG(hw, IXGBE_VFMPRC); 2356 2357 adapter->stats.vf.base_vfgprc = adapter->stats.vf.last_vfgprc; 2358 adapter->stats.vf.base_vfgorc = adapter->stats.vf.last_vfgorc; 2359 adapter->stats.vf.base_vfgptc = adapter->stats.vf.last_vfgptc; 2360 adapter->stats.vf.base_vfgotc = adapter->stats.vf.last_vfgotc; 2361 adapter->stats.vf.base_vfmprc = adapter->stats.vf.last_vfmprc; 2362 } /* ixv_init_stats */ 2363 2364 #define UPDATE_STAT_32(reg, last, count) \ 2365 { \ 2366 u32 current = IXGBE_READ_REG(hw, (reg)); \ 2367 if (current < (last)) \ 2368 count.ev_count += 0x100000000LL; \ 2369 (last) = current; \ 2370 count.ev_count &= 0xFFFFFFFF00000000LL; \ 2371 count.ev_count |= current; \ 2372 } 2373 2374 #define UPDATE_STAT_36(lsb, msb, last, count) \ 2375 { \ 2376 u64 cur_lsb = IXGBE_READ_REG(hw, (lsb)); \ 2377 u64 cur_msb = IXGBE_READ_REG(hw, (msb)); \ 2378 u64 current = ((cur_msb << 32) | cur_lsb); \ 2379 if (current < (last)) \ 2380 count.ev_count += 0x1000000000LL; \ 2381 (last) = current; \ 2382 count.ev_count &= 0xFFFFFFF000000000LL; \ 2383 count.ev_count |= current; \ 2384 } 2385 2386 /************************************************************************ 2387 * ixv_update_stats - Update the board statistics counters. 2388 ************************************************************************/ 2389 void 2390 ixv_update_stats(struct adapter *adapter) 2391 { 2392 struct ixgbe_hw *hw = &adapter->hw; 2393 struct ixgbevf_hw_stats *stats = &adapter->stats.vf; 2394 2395 UPDATE_STAT_32(IXGBE_VFGPRC, stats->last_vfgprc, stats->vfgprc); 2396 UPDATE_STAT_32(IXGBE_VFGPTC, stats->last_vfgptc, stats->vfgptc); 2397 UPDATE_STAT_36(IXGBE_VFGORC_LSB, IXGBE_VFGORC_MSB, stats->last_vfgorc, 2398 stats->vfgorc); 2399 UPDATE_STAT_36(IXGBE_VFGOTC_LSB, IXGBE_VFGOTC_MSB, stats->last_vfgotc, 2400 stats->vfgotc); 2401 UPDATE_STAT_32(IXGBE_VFMPRC, stats->last_vfmprc, stats->vfmprc); 2402 2403 /* Fill out the OS statistics structure */ 2404 /* 2405 * NetBSD: Don't override if_{i|o}{packets|bytes|mcasts} with 2406 * adapter->stats counters. It's required to make ifconfig -z 2407 * (SOICZIFDATA) work. 2408 */ 2409 } /* ixv_update_stats */ 2410 2411 /************************************************************************ 2412 * ixv_sysctl_interrupt_rate_handler 2413 ************************************************************************/ 2414 static int 2415 ixv_sysctl_interrupt_rate_handler(SYSCTLFN_ARGS) 2416 { 2417 struct sysctlnode node = *rnode; 2418 struct ix_queue *que = (struct ix_queue *)node.sysctl_data; 2419 struct adapter *adapter = que->adapter; 2420 uint32_t reg, usec, rate; 2421 int error; 2422 2423 if (que == NULL) 2424 return 0; 2425 reg = IXGBE_READ_REG(&que->adapter->hw, IXGBE_VTEITR(que->msix)); 2426 usec = ((reg & 0x0FF8) >> 3); 2427 if (usec > 0) 2428 rate = 500000 / usec; 2429 else 2430 rate = 0; 2431 node.sysctl_data = &rate; 2432 error = sysctl_lookup(SYSCTLFN_CALL(&node)); 2433 if (error || newp == NULL) 2434 return error; 2435 reg &= ~0xfff; /* default, no limitation */ 2436 if (rate > 0 && rate < 500000) { 2437 if (rate < 1000) 2438 rate = 1000; 2439 reg |= ((4000000/rate) & 0xff8); 2440 /* 2441 * When RSC is used, ITR interval must be larger than 2442 * RSC_DELAY. Currently, we use 2us for RSC_DELAY. 2443 * The minimum value is always greater than 2us on 100M 2444 * (and 10M?(not documented)), but it's not on 1G and higher. 2445 */ 2446 if ((adapter->link_speed != IXGBE_LINK_SPEED_100_FULL) 2447 && (adapter->link_speed != IXGBE_LINK_SPEED_10_FULL)) { 2448 if ((adapter->num_queues > 1) 2449 && (reg < IXGBE_MIN_RSC_EITR_10G1G)) 2450 return EINVAL; 2451 } 2452 ixv_max_interrupt_rate = rate; 2453 } else 2454 ixv_max_interrupt_rate = 0; 2455 ixv_eitr_write(adapter, que->msix, reg); 2456 2457 return (0); 2458 } /* ixv_sysctl_interrupt_rate_handler */ 2459 2460 const struct sysctlnode * 2461 ixv_sysctl_instance(struct adapter *adapter) 2462 { 2463 const char *dvname; 2464 struct sysctllog **log; 2465 int rc; 2466 const struct sysctlnode *rnode; 2467 2468 log = &adapter->sysctllog; 2469 dvname = device_xname(adapter->dev); 2470 2471 if ((rc = sysctl_createv(log, 0, NULL, &rnode, 2472 0, CTLTYPE_NODE, dvname, 2473 SYSCTL_DESCR("ixv information and settings"), 2474 NULL, 0, NULL, 0, CTL_HW, CTL_CREATE, CTL_EOL)) != 0) 2475 goto err; 2476 2477 return rnode; 2478 err: 2479 device_printf(adapter->dev, 2480 "%s: sysctl_createv failed, rc = %d\n", __func__, rc); 2481 return NULL; 2482 } 2483 2484 static void 2485 ixv_add_device_sysctls(struct adapter *adapter) 2486 { 2487 struct sysctllog **log; 2488 const struct sysctlnode *rnode, *cnode; 2489 device_t dev; 2490 2491 dev = adapter->dev; 2492 log = &adapter->sysctllog; 2493 2494 if ((rnode = ixv_sysctl_instance(adapter)) == NULL) { 2495 aprint_error_dev(dev, "could not create sysctl root\n"); 2496 return; 2497 } 2498 2499 if (sysctl_createv(log, 0, &rnode, &cnode, 2500 CTLFLAG_READWRITE, CTLTYPE_INT, 2501 "debug", SYSCTL_DESCR("Debug Info"), 2502 ixv_sysctl_debug, 0, (void *)adapter, 0, CTL_CREATE, CTL_EOL) != 0) 2503 aprint_error_dev(dev, "could not create sysctl\n"); 2504 2505 if (sysctl_createv(log, 0, &rnode, &cnode, 2506 CTLFLAG_READWRITE, CTLTYPE_BOOL, 2507 "enable_aim", SYSCTL_DESCR("Interrupt Moderation"), 2508 NULL, 0, &adapter->enable_aim, 0, CTL_CREATE, CTL_EOL) != 0) 2509 aprint_error_dev(dev, "could not create sysctl\n"); 2510 2511 if (sysctl_createv(log, 0, &rnode, &cnode, 2512 CTLFLAG_READWRITE, CTLTYPE_BOOL, 2513 "txrx_workqueue", SYSCTL_DESCR("Use workqueue for packet processing"), 2514 NULL, 0, &adapter->txrx_use_workqueue, 0, CTL_CREATE, CTL_EOL) != 0) 2515 aprint_error_dev(dev, "could not create sysctl\n"); 2516 } 2517 2518 /************************************************************************ 2519 * ixv_add_stats_sysctls - Add statistic sysctls for the VF. 2520 ************************************************************************/ 2521 static void 2522 ixv_add_stats_sysctls(struct adapter *adapter) 2523 { 2524 device_t dev = adapter->dev; 2525 struct tx_ring *txr = adapter->tx_rings; 2526 struct rx_ring *rxr = adapter->rx_rings; 2527 struct ixgbevf_hw_stats *stats = &adapter->stats.vf; 2528 struct ixgbe_hw *hw = &adapter->hw; 2529 const struct sysctlnode *rnode, *cnode; 2530 struct sysctllog **log = &adapter->sysctllog; 2531 const char *xname = device_xname(dev); 2532 2533 /* Driver Statistics */ 2534 evcnt_attach_dynamic(&adapter->efbig_tx_dma_setup, EVCNT_TYPE_MISC, 2535 NULL, xname, "Driver tx dma soft fail EFBIG"); 2536 evcnt_attach_dynamic(&adapter->mbuf_defrag_failed, EVCNT_TYPE_MISC, 2537 NULL, xname, "m_defrag() failed"); 2538 evcnt_attach_dynamic(&adapter->efbig2_tx_dma_setup, EVCNT_TYPE_MISC, 2539 NULL, xname, "Driver tx dma hard fail EFBIG"); 2540 evcnt_attach_dynamic(&adapter->einval_tx_dma_setup, EVCNT_TYPE_MISC, 2541 NULL, xname, "Driver tx dma hard fail EINVAL"); 2542 evcnt_attach_dynamic(&adapter->other_tx_dma_setup, EVCNT_TYPE_MISC, 2543 NULL, xname, "Driver tx dma hard fail other"); 2544 evcnt_attach_dynamic(&adapter->eagain_tx_dma_setup, EVCNT_TYPE_MISC, 2545 NULL, xname, "Driver tx dma soft fail EAGAIN"); 2546 evcnt_attach_dynamic(&adapter->enomem_tx_dma_setup, EVCNT_TYPE_MISC, 2547 NULL, xname, "Driver tx dma soft fail ENOMEM"); 2548 evcnt_attach_dynamic(&adapter->watchdog_events, EVCNT_TYPE_MISC, 2549 NULL, xname, "Watchdog timeouts"); 2550 evcnt_attach_dynamic(&adapter->tso_err, EVCNT_TYPE_MISC, 2551 NULL, xname, "TSO errors"); 2552 evcnt_attach_dynamic(&adapter->link_irq, EVCNT_TYPE_INTR, 2553 NULL, xname, "Link MSI-X IRQ Handled"); 2554 2555 for (int i = 0; i < adapter->num_queues; i++, rxr++, txr++) { 2556 snprintf(adapter->queues[i].evnamebuf, 2557 sizeof(adapter->queues[i].evnamebuf), "%s q%d", 2558 xname, i); 2559 snprintf(adapter->queues[i].namebuf, 2560 sizeof(adapter->queues[i].namebuf), "q%d", i); 2561 2562 if ((rnode = ixv_sysctl_instance(adapter)) == NULL) { 2563 aprint_error_dev(dev, "could not create sysctl root\n"); 2564 break; 2565 } 2566 2567 if (sysctl_createv(log, 0, &rnode, &rnode, 2568 0, CTLTYPE_NODE, 2569 adapter->queues[i].namebuf, SYSCTL_DESCR("Queue Name"), 2570 NULL, 0, NULL, 0, CTL_CREATE, CTL_EOL) != 0) 2571 break; 2572 2573 if (sysctl_createv(log, 0, &rnode, &cnode, 2574 CTLFLAG_READWRITE, CTLTYPE_INT, 2575 "interrupt_rate", SYSCTL_DESCR("Interrupt Rate"), 2576 ixv_sysctl_interrupt_rate_handler, 0, 2577 (void *)&adapter->queues[i], 0, CTL_CREATE, CTL_EOL) != 0) 2578 break; 2579 2580 if (sysctl_createv(log, 0, &rnode, &cnode, 2581 CTLFLAG_READONLY, CTLTYPE_INT, 2582 "txd_head", SYSCTL_DESCR("Transmit Descriptor Head"), 2583 ixv_sysctl_tdh_handler, 0, (void *)txr, 2584 0, CTL_CREATE, CTL_EOL) != 0) 2585 break; 2586 2587 if (sysctl_createv(log, 0, &rnode, &cnode, 2588 CTLFLAG_READONLY, CTLTYPE_INT, 2589 "txd_tail", SYSCTL_DESCR("Transmit Descriptor Tail"), 2590 ixv_sysctl_tdt_handler, 0, (void *)txr, 2591 0, CTL_CREATE, CTL_EOL) != 0) 2592 break; 2593 2594 evcnt_attach_dynamic(&adapter->queues[i].irqs, EVCNT_TYPE_INTR, 2595 NULL, adapter->queues[i].evnamebuf, "IRQs on queue"); 2596 evcnt_attach_dynamic(&adapter->queues[i].handleq, 2597 EVCNT_TYPE_MISC, NULL, adapter->queues[i].evnamebuf, 2598 "Handled queue in softint"); 2599 evcnt_attach_dynamic(&adapter->queues[i].req, EVCNT_TYPE_MISC, 2600 NULL, adapter->queues[i].evnamebuf, "Requeued in softint"); 2601 evcnt_attach_dynamic(&txr->tso_tx, EVCNT_TYPE_MISC, 2602 NULL, adapter->queues[i].evnamebuf, "TSO"); 2603 evcnt_attach_dynamic(&txr->no_desc_avail, EVCNT_TYPE_MISC, 2604 NULL, adapter->queues[i].evnamebuf, 2605 "Queue No Descriptor Available"); 2606 evcnt_attach_dynamic(&txr->total_packets, EVCNT_TYPE_MISC, 2607 NULL, adapter->queues[i].evnamebuf, 2608 "Queue Packets Transmitted"); 2609 #ifndef IXGBE_LEGACY_TX 2610 evcnt_attach_dynamic(&txr->pcq_drops, EVCNT_TYPE_MISC, 2611 NULL, adapter->queues[i].evnamebuf, 2612 "Packets dropped in pcq"); 2613 #endif 2614 2615 #ifdef LRO 2616 struct lro_ctrl *lro = &rxr->lro; 2617 #endif /* LRO */ 2618 2619 if (sysctl_createv(log, 0, &rnode, &cnode, 2620 CTLFLAG_READONLY, 2621 CTLTYPE_INT, 2622 "rxd_nxck", SYSCTL_DESCR("Receive Descriptor next to check"), 2623 ixv_sysctl_next_to_check_handler, 0, (void *)rxr, 0, 2624 CTL_CREATE, CTL_EOL) != 0) 2625 break; 2626 2627 if (sysctl_createv(log, 0, &rnode, &cnode, 2628 CTLFLAG_READONLY, 2629 CTLTYPE_INT, 2630 "rxd_head", SYSCTL_DESCR("Receive Descriptor Head"), 2631 ixv_sysctl_rdh_handler, 0, (void *)rxr, 0, 2632 CTL_CREATE, CTL_EOL) != 0) 2633 break; 2634 2635 if (sysctl_createv(log, 0, &rnode, &cnode, 2636 CTLFLAG_READONLY, 2637 CTLTYPE_INT, 2638 "rxd_tail", SYSCTL_DESCR("Receive Descriptor Tail"), 2639 ixv_sysctl_rdt_handler, 0, (void *)rxr, 0, 2640 CTL_CREATE, CTL_EOL) != 0) 2641 break; 2642 2643 evcnt_attach_dynamic(&rxr->rx_packets, EVCNT_TYPE_MISC, 2644 NULL, adapter->queues[i].evnamebuf, "Queue Packets Received"); 2645 evcnt_attach_dynamic(&rxr->rx_bytes, EVCNT_TYPE_MISC, 2646 NULL, adapter->queues[i].evnamebuf, "Queue Bytes Received"); 2647 evcnt_attach_dynamic(&rxr->rx_copies, EVCNT_TYPE_MISC, 2648 NULL, adapter->queues[i].evnamebuf, "Copied RX Frames"); 2649 evcnt_attach_dynamic(&rxr->no_jmbuf, EVCNT_TYPE_MISC, 2650 NULL, adapter->queues[i].evnamebuf, "Rx no jumbo mbuf"); 2651 evcnt_attach_dynamic(&rxr->rx_discarded, EVCNT_TYPE_MISC, 2652 NULL, adapter->queues[i].evnamebuf, "Rx discarded"); 2653 #ifdef LRO 2654 SYSCTL_ADD_INT(ctx, queue_list, OID_AUTO, "lro_queued", 2655 CTLFLAG_RD, &lro->lro_queued, 0, 2656 "LRO Queued"); 2657 SYSCTL_ADD_INT(ctx, queue_list, OID_AUTO, "lro_flushed", 2658 CTLFLAG_RD, &lro->lro_flushed, 0, 2659 "LRO Flushed"); 2660 #endif /* LRO */ 2661 } 2662 2663 /* MAC stats get their own sub node */ 2664 2665 snprintf(stats->namebuf, 2666 sizeof(stats->namebuf), "%s MAC Statistics", xname); 2667 2668 evcnt_attach_dynamic(&stats->ipcs, EVCNT_TYPE_MISC, NULL, 2669 stats->namebuf, "rx csum offload - IP"); 2670 evcnt_attach_dynamic(&stats->l4cs, EVCNT_TYPE_MISC, NULL, 2671 stats->namebuf, "rx csum offload - L4"); 2672 evcnt_attach_dynamic(&stats->ipcs_bad, EVCNT_TYPE_MISC, NULL, 2673 stats->namebuf, "rx csum offload - IP bad"); 2674 evcnt_attach_dynamic(&stats->l4cs_bad, EVCNT_TYPE_MISC, NULL, 2675 stats->namebuf, "rx csum offload - L4 bad"); 2676 2677 /* Packet Reception Stats */ 2678 evcnt_attach_dynamic(&stats->vfgprc, EVCNT_TYPE_MISC, NULL, 2679 xname, "Good Packets Received"); 2680 evcnt_attach_dynamic(&stats->vfgorc, EVCNT_TYPE_MISC, NULL, 2681 xname, "Good Octets Received"); 2682 evcnt_attach_dynamic(&stats->vfmprc, EVCNT_TYPE_MISC, NULL, 2683 xname, "Multicast Packets Received"); 2684 evcnt_attach_dynamic(&stats->vfgptc, EVCNT_TYPE_MISC, NULL, 2685 xname, "Good Packets Transmitted"); 2686 evcnt_attach_dynamic(&stats->vfgotc, EVCNT_TYPE_MISC, NULL, 2687 xname, "Good Octets Transmitted"); 2688 2689 /* Mailbox Stats */ 2690 evcnt_attach_dynamic(&hw->mbx.stats.msgs_tx, EVCNT_TYPE_MISC, NULL, 2691 xname, "message TXs"); 2692 evcnt_attach_dynamic(&hw->mbx.stats.msgs_rx, EVCNT_TYPE_MISC, NULL, 2693 xname, "message RXs"); 2694 evcnt_attach_dynamic(&hw->mbx.stats.acks, EVCNT_TYPE_MISC, NULL, 2695 xname, "ACKs"); 2696 evcnt_attach_dynamic(&hw->mbx.stats.reqs, EVCNT_TYPE_MISC, NULL, 2697 xname, "REQs"); 2698 evcnt_attach_dynamic(&hw->mbx.stats.rsts, EVCNT_TYPE_MISC, NULL, 2699 xname, "RSTs"); 2700 2701 } /* ixv_add_stats_sysctls */ 2702 2703 static void 2704 ixv_clear_evcnt(struct adapter *adapter) 2705 { 2706 struct tx_ring *txr = adapter->tx_rings; 2707 struct rx_ring *rxr = adapter->rx_rings; 2708 struct ixgbevf_hw_stats *stats = &adapter->stats.vf; 2709 struct ixgbe_hw *hw = &adapter->hw; 2710 int i; 2711 2712 /* Driver Statistics */ 2713 adapter->efbig_tx_dma_setup.ev_count = 0; 2714 adapter->mbuf_defrag_failed.ev_count = 0; 2715 adapter->efbig2_tx_dma_setup.ev_count = 0; 2716 adapter->einval_tx_dma_setup.ev_count = 0; 2717 adapter->other_tx_dma_setup.ev_count = 0; 2718 adapter->eagain_tx_dma_setup.ev_count = 0; 2719 adapter->enomem_tx_dma_setup.ev_count = 0; 2720 adapter->watchdog_events.ev_count = 0; 2721 adapter->tso_err.ev_count = 0; 2722 adapter->link_irq.ev_count = 0; 2723 2724 for (i = 0; i < adapter->num_queues; i++, rxr++, txr++) { 2725 adapter->queues[i].irqs.ev_count = 0; 2726 adapter->queues[i].handleq.ev_count = 0; 2727 adapter->queues[i].req.ev_count = 0; 2728 txr->tso_tx.ev_count = 0; 2729 txr->no_desc_avail.ev_count = 0; 2730 txr->total_packets.ev_count = 0; 2731 #ifndef IXGBE_LEGACY_TX 2732 txr->pcq_drops.ev_count = 0; 2733 #endif 2734 txr->q_efbig_tx_dma_setup = 0; 2735 txr->q_mbuf_defrag_failed = 0; 2736 txr->q_efbig2_tx_dma_setup = 0; 2737 txr->q_einval_tx_dma_setup = 0; 2738 txr->q_other_tx_dma_setup = 0; 2739 txr->q_eagain_tx_dma_setup = 0; 2740 txr->q_enomem_tx_dma_setup = 0; 2741 txr->q_tso_err = 0; 2742 2743 rxr->rx_packets.ev_count = 0; 2744 rxr->rx_bytes.ev_count = 0; 2745 rxr->rx_copies.ev_count = 0; 2746 rxr->no_jmbuf.ev_count = 0; 2747 rxr->rx_discarded.ev_count = 0; 2748 } 2749 2750 /* MAC stats get their own sub node */ 2751 2752 stats->ipcs.ev_count = 0; 2753 stats->l4cs.ev_count = 0; 2754 stats->ipcs_bad.ev_count = 0; 2755 stats->l4cs_bad.ev_count = 0; 2756 2757 /* Packet Reception Stats */ 2758 stats->vfgprc.ev_count = 0; 2759 stats->vfgorc.ev_count = 0; 2760 stats->vfmprc.ev_count = 0; 2761 stats->vfgptc.ev_count = 0; 2762 stats->vfgotc.ev_count = 0; 2763 2764 /* Mailbox Stats */ 2765 hw->mbx.stats.msgs_tx.ev_count = 0; 2766 hw->mbx.stats.msgs_rx.ev_count = 0; 2767 hw->mbx.stats.acks.ev_count = 0; 2768 hw->mbx.stats.reqs.ev_count = 0; 2769 hw->mbx.stats.rsts.ev_count = 0; 2770 2771 } /* ixv_clear_evcnt */ 2772 2773 /************************************************************************ 2774 * ixv_set_sysctl_value 2775 ************************************************************************/ 2776 static void 2777 ixv_set_sysctl_value(struct adapter *adapter, const char *name, 2778 const char *description, int *limit, int value) 2779 { 2780 device_t dev = adapter->dev; 2781 struct sysctllog **log; 2782 const struct sysctlnode *rnode, *cnode; 2783 2784 log = &adapter->sysctllog; 2785 if ((rnode = ixv_sysctl_instance(adapter)) == NULL) { 2786 aprint_error_dev(dev, "could not create sysctl root\n"); 2787 return; 2788 } 2789 if (sysctl_createv(log, 0, &rnode, &cnode, 2790 CTLFLAG_READWRITE, CTLTYPE_INT, 2791 name, SYSCTL_DESCR(description), 2792 NULL, 0, limit, 0, CTL_CREATE, CTL_EOL) != 0) 2793 aprint_error_dev(dev, "could not create sysctl\n"); 2794 *limit = value; 2795 } /* ixv_set_sysctl_value */ 2796 2797 /************************************************************************ 2798 * ixv_print_debug_info 2799 * 2800 * Called only when em_display_debug_stats is enabled. 2801 * Provides a way to take a look at important statistics 2802 * maintained by the driver and hardware. 2803 ************************************************************************/ 2804 static void 2805 ixv_print_debug_info(struct adapter *adapter) 2806 { 2807 device_t dev = adapter->dev; 2808 struct ix_queue *que = adapter->queues; 2809 struct rx_ring *rxr; 2810 struct tx_ring *txr; 2811 #ifdef LRO 2812 struct lro_ctrl *lro; 2813 #endif /* LRO */ 2814 2815 for (int i = 0; i < adapter->num_queues; i++, que++) { 2816 txr = que->txr; 2817 rxr = que->rxr; 2818 #ifdef LRO 2819 lro = &rxr->lro; 2820 #endif /* LRO */ 2821 device_printf(dev, "QUE(%d) IRQs Handled: %lu\n", 2822 que->msix, (long)que->irqs.ev_count); 2823 device_printf(dev, "RX(%d) Packets Received: %lld\n", 2824 rxr->me, (long long)rxr->rx_packets.ev_count); 2825 device_printf(dev, "RX(%d) Bytes Received: %lu\n", 2826 rxr->me, (long)rxr->rx_bytes.ev_count); 2827 #ifdef LRO 2828 device_printf(dev, "RX(%d) LRO Queued= %ju\n", 2829 rxr->me, (uintmax_t)lro->lro_queued); 2830 device_printf(dev, "RX(%d) LRO Flushed= %ju\n", 2831 rxr->me, (uintmax_t)lro->lro_flushed); 2832 #endif /* LRO */ 2833 device_printf(dev, "TX(%d) Packets Sent: %lu\n", 2834 txr->me, (long)txr->total_packets.ev_count); 2835 device_printf(dev, "TX(%d) NO Desc Avail: %lu\n", 2836 txr->me, (long)txr->no_desc_avail.ev_count); 2837 } 2838 2839 device_printf(dev, "MBX IRQ Handled: %lu\n", 2840 (long)adapter->link_irq.ev_count); 2841 } /* ixv_print_debug_info */ 2842 2843 /************************************************************************ 2844 * ixv_sysctl_debug 2845 ************************************************************************/ 2846 static int 2847 ixv_sysctl_debug(SYSCTLFN_ARGS) 2848 { 2849 struct sysctlnode node = *rnode; 2850 struct adapter *adapter = (struct adapter *)node.sysctl_data; 2851 int error, result; 2852 2853 node.sysctl_data = &result; 2854 error = sysctl_lookup(SYSCTLFN_CALL(&node)); 2855 2856 if (error || newp == NULL) 2857 return error; 2858 2859 if (result == 1) 2860 ixv_print_debug_info(adapter); 2861 2862 return 0; 2863 } /* ixv_sysctl_debug */ 2864 2865 /************************************************************************ 2866 * ixv_init_device_features 2867 ************************************************************************/ 2868 static void 2869 ixv_init_device_features(struct adapter *adapter) 2870 { 2871 adapter->feat_cap = IXGBE_FEATURE_NETMAP 2872 | IXGBE_FEATURE_VF 2873 | IXGBE_FEATURE_RSS 2874 | IXGBE_FEATURE_LEGACY_TX; 2875 2876 /* A tad short on feature flags for VFs, atm. */ 2877 switch (adapter->hw.mac.type) { 2878 case ixgbe_mac_82599_vf: 2879 break; 2880 case ixgbe_mac_X540_vf: 2881 break; 2882 case ixgbe_mac_X550_vf: 2883 case ixgbe_mac_X550EM_x_vf: 2884 case ixgbe_mac_X550EM_a_vf: 2885 adapter->feat_cap |= IXGBE_FEATURE_NEEDS_CTXD; 2886 break; 2887 default: 2888 break; 2889 } 2890 2891 /* Enabled by default... */ 2892 /* Is a virtual function (VF) */ 2893 if (adapter->feat_cap & IXGBE_FEATURE_VF) 2894 adapter->feat_en |= IXGBE_FEATURE_VF; 2895 /* Netmap */ 2896 if (adapter->feat_cap & IXGBE_FEATURE_NETMAP) 2897 adapter->feat_en |= IXGBE_FEATURE_NETMAP; 2898 /* Receive-Side Scaling (RSS) */ 2899 if (adapter->feat_cap & IXGBE_FEATURE_RSS) 2900 adapter->feat_en |= IXGBE_FEATURE_RSS; 2901 /* Needs advanced context descriptor regardless of offloads req'd */ 2902 if (adapter->feat_cap & IXGBE_FEATURE_NEEDS_CTXD) 2903 adapter->feat_en |= IXGBE_FEATURE_NEEDS_CTXD; 2904 2905 /* Enabled via sysctl... */ 2906 /* Legacy (single queue) transmit */ 2907 if ((adapter->feat_cap & IXGBE_FEATURE_LEGACY_TX) && 2908 ixv_enable_legacy_tx) 2909 adapter->feat_en |= IXGBE_FEATURE_LEGACY_TX; 2910 } /* ixv_init_device_features */ 2911 2912 /************************************************************************ 2913 * ixv_shutdown - Shutdown entry point 2914 ************************************************************************/ 2915 #if 0 /* XXX NetBSD ought to register something like this through pmf(9) */ 2916 static int 2917 ixv_shutdown(device_t dev) 2918 { 2919 struct adapter *adapter = device_private(dev); 2920 IXGBE_CORE_LOCK(adapter); 2921 ixv_stop(adapter); 2922 IXGBE_CORE_UNLOCK(adapter); 2923 2924 return (0); 2925 } /* ixv_shutdown */ 2926 #endif 2927 2928 static int 2929 ixv_ifflags_cb(struct ethercom *ec) 2930 { 2931 struct ifnet *ifp = &ec->ec_if; 2932 struct adapter *adapter = ifp->if_softc; 2933 u_short saved_flags; 2934 u_short change; 2935 int rv = 0; 2936 2937 IXGBE_CORE_LOCK(adapter); 2938 2939 saved_flags = adapter->if_flags; 2940 change = ifp->if_flags ^ adapter->if_flags; 2941 if (change != 0) 2942 adapter->if_flags = ifp->if_flags; 2943 2944 if ((change & ~(IFF_CANTCHANGE | IFF_DEBUG)) != 0) { 2945 rv = ENETRESET; 2946 goto out; 2947 } else if ((change & IFF_PROMISC) != 0) { 2948 rv = ixv_set_rxfilter(adapter); 2949 if (rv != 0) { 2950 /* Restore previous */ 2951 adapter->if_flags = saved_flags; 2952 goto out; 2953 } 2954 } 2955 2956 /* Check for ec_capenable. */ 2957 change = ec->ec_capenable ^ adapter->ec_capenable; 2958 adapter->ec_capenable = ec->ec_capenable; 2959 if ((change & ~(ETHERCAP_VLAN_MTU | ETHERCAP_VLAN_HWTAGGING 2960 | ETHERCAP_VLAN_HWFILTER)) != 0) { 2961 rv = ENETRESET; 2962 goto out; 2963 } 2964 2965 /* 2966 * Special handling is not required for ETHERCAP_VLAN_MTU. 2967 * PF's MAXFRS(MHADD) does not include the 4bytes of the VLAN header. 2968 */ 2969 2970 /* Set up VLAN support and filter */ 2971 if ((change & (ETHERCAP_VLAN_HWTAGGING | ETHERCAP_VLAN_HWFILTER)) != 0) 2972 rv = ixv_setup_vlan_support(adapter); 2973 2974 out: 2975 IXGBE_CORE_UNLOCK(adapter); 2976 2977 return rv; 2978 } 2979 2980 2981 /************************************************************************ 2982 * ixv_ioctl - Ioctl entry point 2983 * 2984 * Called when the user wants to configure the interface. 2985 * 2986 * return 0 on success, positive on failure 2987 ************************************************************************/ 2988 static int 2989 ixv_ioctl(struct ifnet *ifp, u_long command, void *data) 2990 { 2991 struct adapter *adapter = ifp->if_softc; 2992 struct ixgbe_hw *hw = &adapter->hw; 2993 struct ifcapreq *ifcr = data; 2994 int error; 2995 int l4csum_en; 2996 const int l4csum = IFCAP_CSUM_TCPv4_Rx | IFCAP_CSUM_UDPv4_Rx | 2997 IFCAP_CSUM_TCPv6_Rx | IFCAP_CSUM_UDPv6_Rx; 2998 2999 switch (command) { 3000 case SIOCSIFFLAGS: 3001 IOCTL_DEBUGOUT("ioctl: SIOCSIFFLAGS (Set Interface Flags)"); 3002 break; 3003 case SIOCADDMULTI: { 3004 struct ether_multi *enm; 3005 struct ether_multistep step; 3006 struct ethercom *ec = &adapter->osdep.ec; 3007 bool overflow = false; 3008 int mcnt = 0; 3009 3010 /* 3011 * Check the number of multicast address. If it exceeds, 3012 * return ENOSPC. 3013 * Update this code when we support API 1.3. 3014 */ 3015 ETHER_LOCK(ec); 3016 ETHER_FIRST_MULTI(step, ec, enm); 3017 while (enm != NULL) { 3018 mcnt++; 3019 3020 /* 3021 * This code is before adding, so one room is required 3022 * at least. 3023 */ 3024 if (mcnt > (IXGBE_MAX_VF_MC - 1)) { 3025 overflow = true; 3026 break; 3027 } 3028 ETHER_NEXT_MULTI(step, enm); 3029 } 3030 ETHER_UNLOCK(ec); 3031 error = 0; 3032 if (overflow && ((ec->ec_flags & ETHER_F_ALLMULTI) == 0)) { 3033 error = hw->mac.ops.update_xcast_mode(hw, 3034 IXGBEVF_XCAST_MODE_ALLMULTI); 3035 if (error == IXGBE_ERR_NOT_TRUSTED) { 3036 device_printf(adapter->dev, 3037 "this interface is not trusted\n"); 3038 error = EPERM; 3039 } else if (error == IXGBE_ERR_FEATURE_NOT_SUPPORTED) { 3040 device_printf(adapter->dev, 3041 "the PF doesn't support allmulti mode\n"); 3042 error = EOPNOTSUPP; 3043 } else if (error) { 3044 device_printf(adapter->dev, 3045 "number of Ethernet multicast addresses " 3046 "exceeds the limit (%d). error = %d\n", 3047 IXGBE_MAX_VF_MC, error); 3048 error = ENOSPC; 3049 } else 3050 ec->ec_flags |= ETHER_F_ALLMULTI; 3051 } 3052 if (error) 3053 return error; 3054 } 3055 /*FALLTHROUGH*/ 3056 case SIOCDELMULTI: 3057 IOCTL_DEBUGOUT("ioctl: SIOC(ADD|DEL)MULTI"); 3058 break; 3059 case SIOCSIFMEDIA: 3060 case SIOCGIFMEDIA: 3061 IOCTL_DEBUGOUT("ioctl: SIOCxIFMEDIA (Get/Set Interface Media)"); 3062 break; 3063 case SIOCSIFCAP: 3064 IOCTL_DEBUGOUT("ioctl: SIOCSIFCAP (Set Capabilities)"); 3065 break; 3066 case SIOCSIFMTU: 3067 IOCTL_DEBUGOUT("ioctl: SIOCSIFMTU (Set Interface MTU)"); 3068 break; 3069 case SIOCZIFDATA: 3070 IOCTL_DEBUGOUT("ioctl: SIOCZIFDATA (Zero counter)"); 3071 ixv_update_stats(adapter); 3072 ixv_clear_evcnt(adapter); 3073 break; 3074 default: 3075 IOCTL_DEBUGOUT1("ioctl: UNKNOWN (0x%X)", (int)command); 3076 break; 3077 } 3078 3079 switch (command) { 3080 case SIOCSIFCAP: 3081 /* Layer-4 Rx checksum offload has to be turned on and 3082 * off as a unit. 3083 */ 3084 l4csum_en = ifcr->ifcr_capenable & l4csum; 3085 if (l4csum_en != l4csum && l4csum_en != 0) 3086 return EINVAL; 3087 /*FALLTHROUGH*/ 3088 case SIOCADDMULTI: 3089 case SIOCDELMULTI: 3090 case SIOCSIFFLAGS: 3091 case SIOCSIFMTU: 3092 default: 3093 if ((error = ether_ioctl(ifp, command, data)) != ENETRESET) 3094 return error; 3095 if ((ifp->if_flags & IFF_RUNNING) == 0) 3096 ; 3097 else if (command == SIOCSIFCAP || command == SIOCSIFMTU) { 3098 IXGBE_CORE_LOCK(adapter); 3099 ixv_init_locked(adapter); 3100 IXGBE_CORE_UNLOCK(adapter); 3101 } else if (command == SIOCADDMULTI || command == SIOCDELMULTI) { 3102 /* 3103 * Multicast list has changed; set the hardware filter 3104 * accordingly. 3105 */ 3106 IXGBE_CORE_LOCK(adapter); 3107 ixv_disable_intr(adapter); 3108 ixv_set_rxfilter(adapter); 3109 ixv_enable_intr(adapter); 3110 IXGBE_CORE_UNLOCK(adapter); 3111 } 3112 return 0; 3113 } 3114 } /* ixv_ioctl */ 3115 3116 /************************************************************************ 3117 * ixv_init 3118 ************************************************************************/ 3119 static int 3120 ixv_init(struct ifnet *ifp) 3121 { 3122 struct adapter *adapter = ifp->if_softc; 3123 3124 IXGBE_CORE_LOCK(adapter); 3125 ixv_init_locked(adapter); 3126 IXGBE_CORE_UNLOCK(adapter); 3127 3128 return 0; 3129 } /* ixv_init */ 3130 3131 /************************************************************************ 3132 * ixv_handle_que 3133 ************************************************************************/ 3134 static void 3135 ixv_handle_que(void *context) 3136 { 3137 struct ix_queue *que = context; 3138 struct adapter *adapter = que->adapter; 3139 struct tx_ring *txr = que->txr; 3140 struct ifnet *ifp = adapter->ifp; 3141 bool more; 3142 3143 que->handleq.ev_count++; 3144 3145 if (ifp->if_flags & IFF_RUNNING) { 3146 more = ixgbe_rxeof(que); 3147 IXGBE_TX_LOCK(txr); 3148 more |= ixgbe_txeof(txr); 3149 if (!(adapter->feat_en & IXGBE_FEATURE_LEGACY_TX)) 3150 if (!ixgbe_mq_ring_empty(ifp, txr->txr_interq)) 3151 ixgbe_mq_start_locked(ifp, txr); 3152 /* Only for queue 0 */ 3153 /* NetBSD still needs this for CBQ */ 3154 if ((&adapter->queues[0] == que) 3155 && (!ixgbe_legacy_ring_empty(ifp, NULL))) 3156 ixgbe_legacy_start_locked(ifp, txr); 3157 IXGBE_TX_UNLOCK(txr); 3158 if (more) { 3159 que->req.ev_count++; 3160 if (adapter->txrx_use_workqueue) { 3161 /* 3162 * "enqueued flag" is not required here 3163 * the same as ixg(4). See ixgbe_msix_que(). 3164 */ 3165 workqueue_enqueue(adapter->que_wq, 3166 &que->wq_cookie, curcpu()); 3167 } else 3168 softint_schedule(que->que_si); 3169 return; 3170 } 3171 } 3172 3173 /* Re-enable this interrupt */ 3174 ixv_enable_queue(adapter, que->msix); 3175 3176 return; 3177 } /* ixv_handle_que */ 3178 3179 /************************************************************************ 3180 * ixv_handle_que_work 3181 ************************************************************************/ 3182 static void 3183 ixv_handle_que_work(struct work *wk, void *context) 3184 { 3185 struct ix_queue *que = container_of(wk, struct ix_queue, wq_cookie); 3186 3187 /* 3188 * "enqueued flag" is not required here the same as ixg(4). 3189 * See ixgbe_msix_que(). 3190 */ 3191 ixv_handle_que(que); 3192 } 3193 3194 /************************************************************************ 3195 * ixv_allocate_msix - Setup MSI-X Interrupt resources and handlers 3196 ************************************************************************/ 3197 static int 3198 ixv_allocate_msix(struct adapter *adapter, const struct pci_attach_args *pa) 3199 { 3200 device_t dev = adapter->dev; 3201 struct ix_queue *que = adapter->queues; 3202 struct tx_ring *txr = adapter->tx_rings; 3203 int error, msix_ctrl, rid, vector = 0; 3204 pci_chipset_tag_t pc; 3205 pcitag_t tag; 3206 char intrbuf[PCI_INTRSTR_LEN]; 3207 char wqname[MAXCOMLEN]; 3208 char intr_xname[32]; 3209 const char *intrstr = NULL; 3210 kcpuset_t *affinity; 3211 int cpu_id = 0; 3212 3213 pc = adapter->osdep.pc; 3214 tag = adapter->osdep.tag; 3215 3216 adapter->osdep.nintrs = adapter->num_queues + 1; 3217 if (pci_msix_alloc_exact(pa, &adapter->osdep.intrs, 3218 adapter->osdep.nintrs) != 0) { 3219 aprint_error_dev(dev, 3220 "failed to allocate MSI-X interrupt\n"); 3221 return (ENXIO); 3222 } 3223 3224 kcpuset_create(&affinity, false); 3225 for (int i = 0; i < adapter->num_queues; i++, vector++, que++, txr++) { 3226 snprintf(intr_xname, sizeof(intr_xname), "%s TXRX%d", 3227 device_xname(dev), i); 3228 intrstr = pci_intr_string(pc, adapter->osdep.intrs[i], intrbuf, 3229 sizeof(intrbuf)); 3230 #ifdef IXGBE_MPSAFE 3231 pci_intr_setattr(pc, &adapter->osdep.intrs[i], PCI_INTR_MPSAFE, 3232 true); 3233 #endif 3234 /* Set the handler function */ 3235 que->res = adapter->osdep.ihs[i] = pci_intr_establish_xname(pc, 3236 adapter->osdep.intrs[i], IPL_NET, ixv_msix_que, que, 3237 intr_xname); 3238 if (que->res == NULL) { 3239 pci_intr_release(pc, adapter->osdep.intrs, 3240 adapter->osdep.nintrs); 3241 aprint_error_dev(dev, 3242 "Failed to register QUE handler\n"); 3243 kcpuset_destroy(affinity); 3244 return (ENXIO); 3245 } 3246 que->msix = vector; 3247 adapter->active_queues |= (u64)(1 << que->msix); 3248 3249 cpu_id = i; 3250 /* Round-robin affinity */ 3251 kcpuset_zero(affinity); 3252 kcpuset_set(affinity, cpu_id % ncpu); 3253 error = interrupt_distribute(adapter->osdep.ihs[i], affinity, 3254 NULL); 3255 aprint_normal_dev(dev, "for TX/RX, interrupting at %s", 3256 intrstr); 3257 if (error == 0) 3258 aprint_normal(", bound queue %d to cpu %d\n", 3259 i, cpu_id % ncpu); 3260 else 3261 aprint_normal("\n"); 3262 3263 #ifndef IXGBE_LEGACY_TX 3264 txr->txr_si 3265 = softint_establish(SOFTINT_NET | IXGBE_SOFTINFT_FLAGS, 3266 ixgbe_deferred_mq_start, txr); 3267 #endif 3268 que->que_si 3269 = softint_establish(SOFTINT_NET | IXGBE_SOFTINFT_FLAGS, 3270 ixv_handle_que, que); 3271 if (que->que_si == NULL) { 3272 aprint_error_dev(dev, 3273 "could not establish software interrupt\n"); 3274 } 3275 } 3276 snprintf(wqname, sizeof(wqname), "%sdeferTx", device_xname(dev)); 3277 error = workqueue_create(&adapter->txr_wq, wqname, 3278 ixgbe_deferred_mq_start_work, adapter, IXGBE_WORKQUEUE_PRI, IPL_NET, 3279 IXGBE_WORKQUEUE_FLAGS); 3280 if (error) { 3281 aprint_error_dev(dev, "couldn't create workqueue for deferred Tx\n"); 3282 } 3283 adapter->txr_wq_enqueued = percpu_alloc(sizeof(u_int)); 3284 3285 snprintf(wqname, sizeof(wqname), "%sTxRx", device_xname(dev)); 3286 error = workqueue_create(&adapter->que_wq, wqname, 3287 ixv_handle_que_work, adapter, IXGBE_WORKQUEUE_PRI, IPL_NET, 3288 IXGBE_WORKQUEUE_FLAGS); 3289 if (error) { 3290 aprint_error_dev(dev, 3291 "couldn't create workqueue\n"); 3292 } 3293 3294 /* and Mailbox */ 3295 cpu_id++; 3296 snprintf(intr_xname, sizeof(intr_xname), "%s link", device_xname(dev)); 3297 adapter->vector = vector; 3298 intrstr = pci_intr_string(pc, adapter->osdep.intrs[vector], intrbuf, 3299 sizeof(intrbuf)); 3300 #ifdef IXGBE_MPSAFE 3301 pci_intr_setattr(pc, &adapter->osdep.intrs[vector], PCI_INTR_MPSAFE, 3302 true); 3303 #endif 3304 /* Set the mbx handler function */ 3305 adapter->osdep.ihs[vector] = pci_intr_establish_xname(pc, 3306 adapter->osdep.intrs[vector], IPL_NET, ixv_msix_mbx, adapter, 3307 intr_xname); 3308 if (adapter->osdep.ihs[vector] == NULL) { 3309 aprint_error_dev(dev, "Failed to register LINK handler\n"); 3310 kcpuset_destroy(affinity); 3311 return (ENXIO); 3312 } 3313 /* Round-robin affinity */ 3314 kcpuset_zero(affinity); 3315 kcpuset_set(affinity, cpu_id % ncpu); 3316 error = interrupt_distribute(adapter->osdep.ihs[vector], 3317 affinity, NULL); 3318 3319 aprint_normal_dev(dev, 3320 "for link, interrupting at %s", intrstr); 3321 if (error == 0) 3322 aprint_normal(", affinity to cpu %d\n", cpu_id % ncpu); 3323 else 3324 aprint_normal("\n"); 3325 3326 /* Tasklets for Mailbox */ 3327 adapter->link_si = softint_establish(SOFTINT_NET |IXGBE_SOFTINFT_FLAGS, 3328 ixv_handle_link, adapter); 3329 /* 3330 * Due to a broken design QEMU will fail to properly 3331 * enable the guest for MSI-X unless the vectors in 3332 * the table are all set up, so we must rewrite the 3333 * ENABLE in the MSI-X control register again at this 3334 * point to cause it to successfully initialize us. 3335 */ 3336 if (adapter->hw.mac.type == ixgbe_mac_82599_vf) { 3337 pci_get_capability(pc, tag, PCI_CAP_MSIX, &rid, NULL); 3338 rid += PCI_MSIX_CTL; 3339 msix_ctrl = pci_conf_read(pc, tag, rid); 3340 msix_ctrl |= PCI_MSIX_CTL_ENABLE; 3341 pci_conf_write(pc, tag, rid, msix_ctrl); 3342 } 3343 3344 kcpuset_destroy(affinity); 3345 return (0); 3346 } /* ixv_allocate_msix */ 3347 3348 /************************************************************************ 3349 * ixv_configure_interrupts - Setup MSI-X resources 3350 * 3351 * Note: The VF device MUST use MSI-X, there is no fallback. 3352 ************************************************************************/ 3353 static int 3354 ixv_configure_interrupts(struct adapter *adapter) 3355 { 3356 device_t dev = adapter->dev; 3357 int want, queues, msgs; 3358 3359 /* Must have at least 2 MSI-X vectors */ 3360 msgs = pci_msix_count(adapter->osdep.pc, adapter->osdep.tag); 3361 if (msgs < 2) { 3362 aprint_error_dev(dev, "MSIX config error\n"); 3363 return (ENXIO); 3364 } 3365 msgs = MIN(msgs, IXG_MAX_NINTR); 3366 3367 /* Figure out a reasonable auto config value */ 3368 queues = (ncpu > (msgs - 1)) ? (msgs - 1) : ncpu; 3369 3370 if (ixv_num_queues != 0) 3371 queues = ixv_num_queues; 3372 else if ((ixv_num_queues == 0) && (queues > IXGBE_VF_MAX_TX_QUEUES)) 3373 queues = IXGBE_VF_MAX_TX_QUEUES; 3374 3375 /* 3376 * Want vectors for the queues, 3377 * plus an additional for mailbox. 3378 */ 3379 want = queues + 1; 3380 if (msgs >= want) 3381 msgs = want; 3382 else { 3383 aprint_error_dev(dev, 3384 "MSI-X Configuration Problem, " 3385 "%d vectors but %d queues wanted!\n", 3386 msgs, want); 3387 return -1; 3388 } 3389 3390 adapter->msix_mem = (void *)1; /* XXX */ 3391 aprint_normal_dev(dev, 3392 "Using MSI-X interrupts with %d vectors\n", msgs); 3393 adapter->num_queues = queues; 3394 3395 return (0); 3396 } /* ixv_configure_interrupts */ 3397 3398 3399 /************************************************************************ 3400 * ixv_handle_link - Tasklet handler for MSI-X MBX interrupts 3401 * 3402 * Done outside of interrupt context since the driver might sleep 3403 ************************************************************************/ 3404 static void 3405 ixv_handle_link(void *context) 3406 { 3407 struct adapter *adapter = context; 3408 3409 IXGBE_CORE_LOCK(adapter); 3410 3411 adapter->hw.mac.ops.check_link(&adapter->hw, &adapter->link_speed, 3412 &adapter->link_up, FALSE); 3413 ixv_update_link_status(adapter); 3414 3415 IXGBE_CORE_UNLOCK(adapter); 3416 } /* ixv_handle_link */ 3417 3418 /************************************************************************ 3419 * ixv_check_link - Used in the local timer to poll for link changes 3420 ************************************************************************/ 3421 static s32 3422 ixv_check_link(struct adapter *adapter) 3423 { 3424 s32 error; 3425 3426 KASSERT(mutex_owned(&adapter->core_mtx)); 3427 3428 adapter->hw.mac.get_link_status = TRUE; 3429 3430 error = adapter->hw.mac.ops.check_link(&adapter->hw, 3431 &adapter->link_speed, &adapter->link_up, FALSE); 3432 ixv_update_link_status(adapter); 3433 3434 return error; 3435 } /* ixv_check_link */ 3436
Indexes created Tue Sep 30 20:09:53 GMT 2025