ixv.c revision 1.139
1 /*$NetBSD: ixv.c,v 1.139 2019/10/01 10:04:11 msaitoh Exp $*/ 2 3 /****************************************************************************** 4 5 Copyright (c) 2001-2017, Intel Corporation 6 All rights reserved. 7 8 Redistribution and use in source and binary forms, with or without 9 modification, are permitted provided that the following conditions are met: 10 11 1. Redistributions of source code must retain the above copyright notice, 12 this list of conditions and the following disclaimer. 13 14 2. Redistributions in binary form must reproduce the above copyright 15 notice, this list of conditions and the following disclaimer in the 16 documentation and/or other materials provided with the distribution. 17 18 3. Neither the name of the Intel Corporation nor the names of its 19 contributors may be used to endorse or promote products derived from 20 this software without specific prior written permission. 21 22 THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 23 AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 24 IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 25 ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE 26 LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 27 CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 28 SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 29 INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 30 CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 31 ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 32 POSSIBILITY OF SUCH DAMAGE. 33 34 ******************************************************************************/ 35 /*$FreeBSD: head/sys/dev/ixgbe/if_ixv.c 331224 2018-03-19 20:55:05Z erj $*/ 36 37 #ifdef _KERNEL_OPT 38 #include "opt_inet.h" 39 #include "opt_inet6.h" 40 #include "opt_net_mpsafe.h" 41 #endif 42 43 #include "ixgbe.h" 44 #include "vlan.h" 45 46 /************************************************************************ 47 * Driver version 48 ************************************************************************/ 49 static const char ixv_driver_version[] = "2.0.1-k"; 50 /* XXX NetBSD: + 1.5.17 */ 51 52 /************************************************************************ 53 * PCI Device ID Table 54 * 55 * Used by probe to select devices to load on 56 * Last field stores an index into ixv_strings 57 * Last entry must be all 0s 58 * 59 * { Vendor ID, Device ID, SubVendor ID, SubDevice ID, String Index } 60 ************************************************************************/ 61 static const ixgbe_vendor_info_t ixv_vendor_info_array[] = 62 { 63 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_VF, 0, 0, 0}, 64 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_X540_VF, 0, 0, 0}, 65 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_X550_VF, 0, 0, 0}, 66 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_X550EM_X_VF, 0, 0, 0}, 67 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_X550EM_A_VF, 0, 0, 0}, 68 /* required last entry */ 69 {0, 0, 0, 0, 0} 70 }; 71 72 /************************************************************************ 73 * Table of branding strings 74 ************************************************************************/ 75 static const char *ixv_strings[] = { 76 "Intel(R) PRO/10GbE Virtual Function Network Driver" 77 }; 78 79 /********************************************************************* 80 * Function prototypes 81 *********************************************************************/ 82 static int ixv_probe(device_t, cfdata_t, void *); 83 static void ixv_attach(device_t, device_t, void *); 84 static int ixv_detach(device_t, int); 85 #if 0 86 static int ixv_shutdown(device_t); 87 #endif 88 static int ixv_ifflags_cb(struct ethercom *); 89 static int ixv_ioctl(struct ifnet *, u_long, void *); 90 static int ixv_init(struct ifnet *); 91 static void ixv_init_locked(struct adapter *); 92 static void ixv_ifstop(struct ifnet *, int); 93 static void ixv_stop(void *); 94 static void ixv_init_device_features(struct adapter *); 95 static void ixv_media_status(struct ifnet *, struct ifmediareq *); 96 static int ixv_media_change(struct ifnet *); 97 static int ixv_allocate_pci_resources(struct adapter *, 98 const struct pci_attach_args *); 99 static int ixv_allocate_msix(struct adapter *, 100 const struct pci_attach_args *); 101 static int ixv_configure_interrupts(struct adapter *); 102 static void ixv_free_pci_resources(struct adapter *); 103 static void ixv_local_timer(void *); 104 static void ixv_local_timer_locked(void *); 105 static int ixv_setup_interface(device_t, struct adapter *); 106 static int ixv_negotiate_api(struct adapter *); 107 108 static void ixv_initialize_transmit_units(struct adapter *); 109 static void ixv_initialize_receive_units(struct adapter *); 110 static void ixv_initialize_rss_mapping(struct adapter *); 111 static s32 ixv_check_link(struct adapter *); 112 113 static void ixv_enable_intr(struct adapter *); 114 static void ixv_disable_intr(struct adapter *); 115 static int ixv_set_rxfilter(struct adapter *); 116 static void ixv_update_link_status(struct adapter *); 117 static int ixv_sysctl_debug(SYSCTLFN_PROTO); 118 static void ixv_set_ivar(struct adapter *, u8, u8, s8); 119 static void ixv_configure_ivars(struct adapter *); 120 static u8 * ixv_mc_array_itr(struct ixgbe_hw *, u8 **, u32 *); 121 static void ixv_eitr_write(struct adapter *, uint32_t, uint32_t); 122 123 static void ixv_setup_vlan_tagging(struct adapter *); 124 static int ixv_setup_vlan_support(struct adapter *); 125 static int ixv_vlan_cb(struct ethercom *, uint16_t, bool); 126 static int ixv_register_vlan(struct adapter *, u16); 127 static int ixv_unregister_vlan(struct adapter *, u16); 128 129 static void ixv_add_device_sysctls(struct adapter *); 130 static void ixv_save_stats(struct adapter *); 131 static void ixv_init_stats(struct adapter *); 132 static void ixv_update_stats(struct adapter *); 133 static void ixv_add_stats_sysctls(struct adapter *); 134 static void ixv_clear_evcnt(struct adapter *); 135 136 /* Sysctl handlers */ 137 static void ixv_set_sysctl_value(struct adapter *, const char *, 138 const char *, int *, int); 139 static int ixv_sysctl_interrupt_rate_handler(SYSCTLFN_PROTO); 140 static int ixv_sysctl_next_to_check_handler(SYSCTLFN_PROTO); 141 static int ixv_sysctl_rdh_handler(SYSCTLFN_PROTO); 142 static int ixv_sysctl_rdt_handler(SYSCTLFN_PROTO); 143 static int ixv_sysctl_tdt_handler(SYSCTLFN_PROTO); 144 static int ixv_sysctl_tdh_handler(SYSCTLFN_PROTO); 145 146 /* The MSI-X Interrupt handlers */ 147 static int ixv_msix_que(void *); 148 static int ixv_msix_mbx(void *); 149 150 /* Deferred interrupt tasklets */ 151 static void ixv_handle_que(void *); 152 static void ixv_handle_link(void *); 153 154 /* Workqueue handler for deferred work */ 155 static void ixv_handle_que_work(struct work *, void *); 156 157 const struct sysctlnode *ixv_sysctl_instance(struct adapter *); 158 static const ixgbe_vendor_info_t *ixv_lookup(const struct pci_attach_args *); 159 160 /************************************************************************ 161 * FreeBSD Device Interface Entry Points 162 ************************************************************************/ 163 CFATTACH_DECL3_NEW(ixv, sizeof(struct adapter), 164 ixv_probe, ixv_attach, ixv_detach, NULL, NULL, NULL, 165 DVF_DETACH_SHUTDOWN); 166 167 #if 0 168 static driver_t ixv_driver = { 169 "ixv", ixv_methods, sizeof(struct adapter), 170 }; 171 172 devclass_t ixv_devclass; 173 DRIVER_MODULE(ixv, pci, ixv_driver, ixv_devclass, 0, 0); 174 MODULE_DEPEND(ixv, pci, 1, 1, 1); 175 MODULE_DEPEND(ixv, ether, 1, 1, 1); 176 #endif 177 178 /* 179 * TUNEABLE PARAMETERS: 180 */ 181 182 /* Number of Queues - do not exceed MSI-X vectors - 1 */ 183 static int ixv_num_queues = 0; 184 #define TUNABLE_INT(__x, __y) 185 TUNABLE_INT("hw.ixv.num_queues", &ixv_num_queues); 186 187 /* 188 * AIM: Adaptive Interrupt Moderation 189 * which means that the interrupt rate 190 * is varied over time based on the 191 * traffic for that interrupt vector 192 */ 193 static bool ixv_enable_aim = false; 194 TUNABLE_INT("hw.ixv.enable_aim", &ixv_enable_aim); 195 196 static int ixv_max_interrupt_rate = (4000000 / IXGBE_LOW_LATENCY); 197 TUNABLE_INT("hw.ixv.max_interrupt_rate", &ixv_max_interrupt_rate); 198 199 /* How many packets rxeof tries to clean at a time */ 200 static int ixv_rx_process_limit = 256; 201 TUNABLE_INT("hw.ixv.rx_process_limit", &ixv_rx_process_limit); 202 203 /* How many packets txeof tries to clean at a time */ 204 static int ixv_tx_process_limit = 256; 205 TUNABLE_INT("hw.ixv.tx_process_limit", &ixv_tx_process_limit); 206 207 /* Which packet processing uses workqueue or softint */ 208 static bool ixv_txrx_workqueue = false; 209 210 /* 211 * Number of TX descriptors per ring, 212 * setting higher than RX as this seems 213 * the better performing choice. 214 */ 215 static int ixv_txd = PERFORM_TXD; 216 TUNABLE_INT("hw.ixv.txd", &ixv_txd); 217 218 /* Number of RX descriptors per ring */ 219 static int ixv_rxd = PERFORM_RXD; 220 TUNABLE_INT("hw.ixv.rxd", &ixv_rxd); 221 222 /* Legacy Transmit (single queue) */ 223 static int ixv_enable_legacy_tx = 0; 224 TUNABLE_INT("hw.ixv.enable_legacy_tx", &ixv_enable_legacy_tx); 225 226 #ifdef NET_MPSAFE 227 #define IXGBE_MPSAFE 1 228 #define IXGBE_CALLOUT_FLAGS CALLOUT_MPSAFE 229 #define IXGBE_SOFTINFT_FLAGS SOFTINT_MPSAFE 230 #define IXGBE_WORKQUEUE_FLAGS WQ_PERCPU | WQ_MPSAFE 231 #else 232 #define IXGBE_CALLOUT_FLAGS 0 233 #define IXGBE_SOFTINFT_FLAGS 0 234 #define IXGBE_WORKQUEUE_FLAGS WQ_PERCPU 235 #endif 236 #define IXGBE_WORKQUEUE_PRI PRI_SOFTNET 237 238 #if 0 239 static int (*ixv_start_locked)(struct ifnet *, struct tx_ring *); 240 static int (*ixv_ring_empty)(struct ifnet *, struct buf_ring *); 241 #endif 242 243 /************************************************************************ 244 * ixv_probe - Device identification routine 245 * 246 * Determines if the driver should be loaded on 247 * adapter based on its PCI vendor/device ID. 248 * 249 * return BUS_PROBE_DEFAULT on success, positive on failure 250 ************************************************************************/ 251 static int 252 ixv_probe(device_t dev, cfdata_t cf, void *aux) 253 { 254 #ifdef __HAVE_PCI_MSI_MSIX 255 const struct pci_attach_args *pa = aux; 256 257 return (ixv_lookup(pa) != NULL) ? 1 : 0; 258 #else 259 return 0; 260 #endif 261 } /* ixv_probe */ 262 263 static const ixgbe_vendor_info_t * 264 ixv_lookup(const struct pci_attach_args *pa) 265 { 266 const ixgbe_vendor_info_t *ent; 267 pcireg_t subid; 268 269 INIT_DEBUGOUT("ixv_lookup: begin"); 270 271 if (PCI_VENDOR(pa->pa_id) != IXGBE_INTEL_VENDOR_ID) 272 return NULL; 273 274 subid = pci_conf_read(pa->pa_pc, pa->pa_tag, PCI_SUBSYS_ID_REG); 275 276 for (ent = ixv_vendor_info_array; ent->vendor_id != 0; ent++) { 277 if ((PCI_VENDOR(pa->pa_id) == ent->vendor_id) && 278 (PCI_PRODUCT(pa->pa_id) == ent->device_id) && 279 ((PCI_SUBSYS_VENDOR(subid) == ent->subvendor_id) || 280 (ent->subvendor_id == 0)) && 281 ((PCI_SUBSYS_ID(subid) == ent->subdevice_id) || 282 (ent->subdevice_id == 0))) { 283 return ent; 284 } 285 } 286 287 return NULL; 288 } 289 290 /************************************************************************ 291 * ixv_attach - Device initialization routine 292 * 293 * Called when the driver is being loaded. 294 * Identifies the type of hardware, allocates all resources 295 * and initializes the hardware. 296 * 297 * return 0 on success, positive on failure 298 ************************************************************************/ 299 static void 300 ixv_attach(device_t parent, device_t dev, void *aux) 301 { 302 struct adapter *adapter; 303 struct ixgbe_hw *hw; 304 int error = 0; 305 pcireg_t id, subid; 306 const ixgbe_vendor_info_t *ent; 307 const struct pci_attach_args *pa = aux; 308 const char *apivstr; 309 const char *str; 310 char buf[256]; 311 312 INIT_DEBUGOUT("ixv_attach: begin"); 313 314 /* 315 * Make sure BUSMASTER is set, on a VM under 316 * KVM it may not be and will break things. 317 */ 318 ixgbe_pci_enable_busmaster(pa->pa_pc, pa->pa_tag); 319 320 /* Allocate, clear, and link in our adapter structure */ 321 adapter = device_private(dev); 322 adapter->dev = dev; 323 adapter->hw.back = adapter; 324 hw = &adapter->hw; 325 326 adapter->init_locked = ixv_init_locked; 327 adapter->stop_locked = ixv_stop; 328 329 adapter->osdep.pc = pa->pa_pc; 330 adapter->osdep.tag = pa->pa_tag; 331 if (pci_dma64_available(pa)) 332 adapter->osdep.dmat = pa->pa_dmat64; 333 else 334 adapter->osdep.dmat = pa->pa_dmat; 335 adapter->osdep.attached = false; 336 337 ent = ixv_lookup(pa); 338 339 KASSERT(ent != NULL); 340 341 aprint_normal(": %s, Version - %s\n", 342 ixv_strings[ent->index], ixv_driver_version); 343 344 /* Core Lock Init*/ 345 IXGBE_CORE_LOCK_INIT(adapter, device_xname(dev)); 346 347 /* Do base PCI setup - map BAR0 */ 348 if (ixv_allocate_pci_resources(adapter, pa)) { 349 aprint_error_dev(dev, "ixv_allocate_pci_resources() failed!\n"); 350 error = ENXIO; 351 goto err_out; 352 } 353 354 /* SYSCTL APIs */ 355 ixv_add_device_sysctls(adapter); 356 357 /* Set up the timer callout */ 358 callout_init(&adapter->timer, IXGBE_CALLOUT_FLAGS); 359 360 /* Save off the information about this board */ 361 id = pci_conf_read(pa->pa_pc, pa->pa_tag, PCI_ID_REG); 362 subid = pci_conf_read(pa->pa_pc, pa->pa_tag, PCI_SUBSYS_ID_REG); 363 hw->vendor_id = PCI_VENDOR(id); 364 hw->device_id = PCI_PRODUCT(id); 365 hw->revision_id = 366 PCI_REVISION(pci_conf_read(pa->pa_pc, pa->pa_tag, PCI_CLASS_REG)); 367 hw->subsystem_vendor_id = PCI_SUBSYS_VENDOR(subid); 368 hw->subsystem_device_id = PCI_SUBSYS_ID(subid); 369 370 /* A subset of set_mac_type */ 371 switch (hw->device_id) { 372 case IXGBE_DEV_ID_82599_VF: 373 hw->mac.type = ixgbe_mac_82599_vf; 374 str = "82599 VF"; 375 break; 376 case IXGBE_DEV_ID_X540_VF: 377 hw->mac.type = ixgbe_mac_X540_vf; 378 str = "X540 VF"; 379 break; 380 case IXGBE_DEV_ID_X550_VF: 381 hw->mac.type = ixgbe_mac_X550_vf; 382 str = "X550 VF"; 383 break; 384 case IXGBE_DEV_ID_X550EM_X_VF: 385 hw->mac.type = ixgbe_mac_X550EM_x_vf; 386 str = "X550EM X VF"; 387 break; 388 case IXGBE_DEV_ID_X550EM_A_VF: 389 hw->mac.type = ixgbe_mac_X550EM_a_vf; 390 str = "X550EM A VF"; 391 break; 392 default: 393 /* Shouldn't get here since probe succeeded */ 394 aprint_error_dev(dev, "Unknown device ID!\n"); 395 error = ENXIO; 396 goto err_out; 397 break; 398 } 399 aprint_normal_dev(dev, "device %s\n", str); 400 401 ixv_init_device_features(adapter); 402 403 /* Initialize the shared code */ 404 error = ixgbe_init_ops_vf(hw); 405 if (error) { 406 aprint_error_dev(dev, "ixgbe_init_ops_vf() failed!\n"); 407 error = EIO; 408 goto err_out; 409 } 410 411 /* Setup the mailbox */ 412 ixgbe_init_mbx_params_vf(hw); 413 414 /* Set the right number of segments */ 415 adapter->num_segs = IXGBE_82599_SCATTER; 416 417 /* Reset mbox api to 1.0 */ 418 error = hw->mac.ops.reset_hw(hw); 419 if (error == IXGBE_ERR_RESET_FAILED) 420 aprint_error_dev(dev, "...reset_hw() failure: Reset Failed!\n"); 421 else if (error) 422 aprint_error_dev(dev, "...reset_hw() failed with error %d\n", 423 error); 424 if (error) { 425 error = EIO; 426 goto err_out; 427 } 428 429 error = hw->mac.ops.init_hw(hw); 430 if (error) { 431 aprint_error_dev(dev, "...init_hw() failed!\n"); 432 error = EIO; 433 goto err_out; 434 } 435 436 /* Negotiate mailbox API version */ 437 error = ixv_negotiate_api(adapter); 438 if (error) 439 aprint_normal_dev(dev, 440 "MBX API negotiation failed during attach!\n"); 441 switch (hw->api_version) { 442 case ixgbe_mbox_api_10: 443 apivstr = "1.0"; 444 break; 445 case ixgbe_mbox_api_20: 446 apivstr = "2.0"; 447 break; 448 case ixgbe_mbox_api_11: 449 apivstr = "1.1"; 450 break; 451 case ixgbe_mbox_api_12: 452 apivstr = "1.2"; 453 break; 454 case ixgbe_mbox_api_13: 455 apivstr = "1.3"; 456 break; 457 default: 458 apivstr = "unknown"; 459 break; 460 } 461 aprint_normal_dev(dev, "Mailbox API %s\n", apivstr); 462 463 /* If no mac address was assigned, make a random one */ 464 if (!ixv_check_ether_addr(hw->mac.addr)) { 465 u8 addr[ETHER_ADDR_LEN]; 466 uint64_t rndval = cprng_strong64(); 467 468 memcpy(addr, &rndval, sizeof(addr)); 469 addr[0] &= 0xFE; 470 addr[0] |= 0x02; 471 bcopy(addr, hw->mac.addr, sizeof(addr)); 472 } 473 474 /* Register for VLAN events */ 475 ether_set_vlan_cb(&adapter->osdep.ec, ixv_vlan_cb); 476 477 /* Sysctls for limiting the amount of work done in the taskqueues */ 478 ixv_set_sysctl_value(adapter, "rx_processing_limit", 479 "max number of rx packets to process", 480 &adapter->rx_process_limit, ixv_rx_process_limit); 481 482 ixv_set_sysctl_value(adapter, "tx_processing_limit", 483 "max number of tx packets to process", 484 &adapter->tx_process_limit, ixv_tx_process_limit); 485 486 /* Do descriptor calc and sanity checks */ 487 if (((ixv_txd * sizeof(union ixgbe_adv_tx_desc)) % DBA_ALIGN) != 0 || 488 ixv_txd < MIN_TXD || ixv_txd > MAX_TXD) { 489 aprint_error_dev(dev, "TXD config issue, using default!\n"); 490 adapter->num_tx_desc = DEFAULT_TXD; 491 } else 492 adapter->num_tx_desc = ixv_txd; 493 494 if (((ixv_rxd * sizeof(union ixgbe_adv_rx_desc)) % DBA_ALIGN) != 0 || 495 ixv_rxd < MIN_RXD || ixv_rxd > MAX_RXD) { 496 aprint_error_dev(dev, "RXD config issue, using default!\n"); 497 adapter->num_rx_desc = DEFAULT_RXD; 498 } else 499 adapter->num_rx_desc = ixv_rxd; 500 501 /* Setup MSI-X */ 502 error = ixv_configure_interrupts(adapter); 503 if (error) 504 goto err_out; 505 506 /* Allocate our TX/RX Queues */ 507 if (ixgbe_allocate_queues(adapter)) { 508 aprint_error_dev(dev, "ixgbe_allocate_queues() failed!\n"); 509 error = ENOMEM; 510 goto err_out; 511 } 512 513 /* hw.ix defaults init */ 514 adapter->enable_aim = ixv_enable_aim; 515 516 adapter->txrx_use_workqueue = ixv_txrx_workqueue; 517 518 error = ixv_allocate_msix(adapter, pa); 519 if (error) { 520 aprint_error_dev(dev, "ixv_allocate_msix() failed!\n"); 521 goto err_late; 522 } 523 524 /* Setup OS specific network interface */ 525 error = ixv_setup_interface(dev, adapter); 526 if (error != 0) { 527 aprint_error_dev(dev, "ixv_setup_interface() failed!\n"); 528 goto err_late; 529 } 530 531 /* Do the stats setup */ 532 ixv_save_stats(adapter); 533 ixv_init_stats(adapter); 534 ixv_add_stats_sysctls(adapter); 535 536 if (adapter->feat_en & IXGBE_FEATURE_NETMAP) 537 ixgbe_netmap_attach(adapter); 538 539 snprintb(buf, sizeof(buf), IXGBE_FEATURE_FLAGS, adapter->feat_cap); 540 aprint_verbose_dev(dev, "feature cap %s\n", buf); 541 snprintb(buf, sizeof(buf), IXGBE_FEATURE_FLAGS, adapter->feat_en); 542 aprint_verbose_dev(dev, "feature ena %s\n", buf); 543 544 INIT_DEBUGOUT("ixv_attach: end"); 545 adapter->osdep.attached = true; 546 547 return; 548 549 err_late: 550 ixgbe_free_transmit_structures(adapter); 551 ixgbe_free_receive_structures(adapter); 552 free(adapter->queues, M_DEVBUF); 553 err_out: 554 ixv_free_pci_resources(adapter); 555 IXGBE_CORE_LOCK_DESTROY(adapter); 556 557 return; 558 } /* ixv_attach */ 559 560 /************************************************************************ 561 * ixv_detach - Device removal routine 562 * 563 * Called when the driver is being removed. 564 * Stops the adapter and deallocates all the resources 565 * that were allocated for driver operation. 566 * 567 * return 0 on success, positive on failure 568 ************************************************************************/ 569 static int 570 ixv_detach(device_t dev, int flags) 571 { 572 struct adapter *adapter = device_private(dev); 573 struct ixgbe_hw *hw = &adapter->hw; 574 struct ix_queue *que = adapter->queues; 575 struct tx_ring *txr = adapter->tx_rings; 576 struct rx_ring *rxr = adapter->rx_rings; 577 struct ixgbevf_hw_stats *stats = &adapter->stats.vf; 578 579 INIT_DEBUGOUT("ixv_detach: begin"); 580 if (adapter->osdep.attached == false) 581 return 0; 582 583 /* Stop the interface. Callouts are stopped in it. */ 584 ixv_ifstop(adapter->ifp, 1); 585 586 #if NVLAN > 0 587 /* Make sure VLANs are not using driver */ 588 if (!VLAN_ATTACHED(&adapter->osdep.ec)) 589 ; /* nothing to do: no VLANs */ 590 else if ((flags & (DETACH_SHUTDOWN | DETACH_FORCE)) != 0) 591 vlan_ifdetach(adapter->ifp); 592 else { 593 aprint_error_dev(dev, "VLANs in use, detach first\n"); 594 return EBUSY; 595 } 596 #endif 597 598 for (int i = 0; i < adapter->num_queues; i++, que++, txr++) { 599 if (!(adapter->feat_en & IXGBE_FEATURE_LEGACY_TX)) 600 softint_disestablish(txr->txr_si); 601 softint_disestablish(que->que_si); 602 } 603 if (adapter->txr_wq != NULL) 604 workqueue_destroy(adapter->txr_wq); 605 if (adapter->txr_wq_enqueued != NULL) 606 percpu_free(adapter->txr_wq_enqueued, sizeof(u_int)); 607 if (adapter->que_wq != NULL) 608 workqueue_destroy(adapter->que_wq); 609 610 /* Drain the Mailbox(link) queue */ 611 softint_disestablish(adapter->link_si); 612 613 ether_ifdetach(adapter->ifp); 614 callout_halt(&adapter->timer, NULL); 615 616 if (adapter->feat_en & IXGBE_FEATURE_NETMAP) 617 netmap_detach(adapter->ifp); 618 619 ixv_free_pci_resources(adapter); 620 #if 0 /* XXX the NetBSD port is probably missing something here */ 621 bus_generic_detach(dev); 622 #endif 623 if_detach(adapter->ifp); 624 if_percpuq_destroy(adapter->ipq); 625 626 sysctl_teardown(&adapter->sysctllog); 627 evcnt_detach(&adapter->efbig_tx_dma_setup); 628 evcnt_detach(&adapter->mbuf_defrag_failed); 629 evcnt_detach(&adapter->efbig2_tx_dma_setup); 630 evcnt_detach(&adapter->einval_tx_dma_setup); 631 evcnt_detach(&adapter->other_tx_dma_setup); 632 evcnt_detach(&adapter->eagain_tx_dma_setup); 633 evcnt_detach(&adapter->enomem_tx_dma_setup); 634 evcnt_detach(&adapter->watchdog_events); 635 evcnt_detach(&adapter->tso_err); 636 evcnt_detach(&adapter->link_irq); 637 638 txr = adapter->tx_rings; 639 for (int i = 0; i < adapter->num_queues; i++, rxr++, txr++) { 640 evcnt_detach(&adapter->queues[i].irqs); 641 evcnt_detach(&adapter->queues[i].handleq); 642 evcnt_detach(&adapter->queues[i].req); 643 evcnt_detach(&txr->no_desc_avail); 644 evcnt_detach(&txr->total_packets); 645 evcnt_detach(&txr->tso_tx); 646 #ifndef IXGBE_LEGACY_TX 647 evcnt_detach(&txr->pcq_drops); 648 #endif 649 650 evcnt_detach(&rxr->rx_packets); 651 evcnt_detach(&rxr->rx_bytes); 652 evcnt_detach(&rxr->rx_copies); 653 evcnt_detach(&rxr->no_jmbuf); 654 evcnt_detach(&rxr->rx_discarded); 655 } 656 evcnt_detach(&stats->ipcs); 657 evcnt_detach(&stats->l4cs); 658 evcnt_detach(&stats->ipcs_bad); 659 evcnt_detach(&stats->l4cs_bad); 660 661 /* Packet Reception Stats */ 662 evcnt_detach(&stats->vfgorc); 663 evcnt_detach(&stats->vfgprc); 664 evcnt_detach(&stats->vfmprc); 665 666 /* Packet Transmission Stats */ 667 evcnt_detach(&stats->vfgotc); 668 evcnt_detach(&stats->vfgptc); 669 670 /* Mailbox Stats */ 671 evcnt_detach(&hw->mbx.stats.msgs_tx); 672 evcnt_detach(&hw->mbx.stats.msgs_rx); 673 evcnt_detach(&hw->mbx.stats.acks); 674 evcnt_detach(&hw->mbx.stats.reqs); 675 evcnt_detach(&hw->mbx.stats.rsts); 676 677 ixgbe_free_transmit_structures(adapter); 678 ixgbe_free_receive_structures(adapter); 679 for (int i = 0; i < adapter->num_queues; i++) { 680 struct ix_queue *lque = &adapter->queues[i]; 681 mutex_destroy(&lque->dc_mtx); 682 } 683 free(adapter->queues, M_DEVBUF); 684 685 IXGBE_CORE_LOCK_DESTROY(adapter); 686 687 return (0); 688 } /* ixv_detach */ 689 690 /************************************************************************ 691 * ixv_init_locked - Init entry point 692 * 693 * Used in two ways: It is used by the stack as an init entry 694 * point in network interface structure. It is also used 695 * by the driver as a hw/sw initialization routine to get 696 * to a consistent state. 697 * 698 * return 0 on success, positive on failure 699 ************************************************************************/ 700 static void 701 ixv_init_locked(struct adapter *adapter) 702 { 703 struct ifnet *ifp = adapter->ifp; 704 device_t dev = adapter->dev; 705 struct ixgbe_hw *hw = &adapter->hw; 706 struct ix_queue *que; 707 int error = 0; 708 uint32_t mask; 709 int i; 710 711 INIT_DEBUGOUT("ixv_init_locked: begin"); 712 KASSERT(mutex_owned(&adapter->core_mtx)); 713 hw->adapter_stopped = FALSE; 714 hw->mac.ops.stop_adapter(hw); 715 callout_stop(&adapter->timer); 716 for (i = 0, que = adapter->queues; i < adapter->num_queues; i++, que++) 717 que->disabled_count = 0; 718 719 adapter->max_frame_size = 720 ifp->if_mtu + ETHER_HDR_LEN + ETHER_CRC_LEN; 721 722 /* reprogram the RAR[0] in case user changed it. */ 723 hw->mac.ops.set_rar(hw, 0, hw->mac.addr, 0, IXGBE_RAH_AV); 724 725 /* Get the latest mac address, User can use a LAA */ 726 memcpy(hw->mac.addr, CLLADDR(ifp->if_sadl), 727 IXGBE_ETH_LENGTH_OF_ADDRESS); 728 hw->mac.ops.set_rar(hw, 0, hw->mac.addr, 0, 1); 729 730 /* Prepare transmit descriptors and buffers */ 731 if (ixgbe_setup_transmit_structures(adapter)) { 732 aprint_error_dev(dev, "Could not setup transmit structures\n"); 733 ixv_stop(adapter); 734 return; 735 } 736 737 /* Reset VF and renegotiate mailbox API version */ 738 hw->mac.ops.reset_hw(hw); 739 hw->mac.ops.start_hw(hw); 740 error = ixv_negotiate_api(adapter); 741 if (error) 742 device_printf(dev, 743 "Mailbox API negotiation failed in init_locked!\n"); 744 745 ixv_initialize_transmit_units(adapter); 746 747 /* Setup Multicast table */ 748 ixv_set_rxfilter(adapter); 749 750 /* 751 * Determine the correct mbuf pool 752 * for doing jumbo/headersplit 753 */ 754 if (adapter->max_frame_size <= MCLBYTES) 755 adapter->rx_mbuf_sz = MCLBYTES; 756 else 757 adapter->rx_mbuf_sz = MJUMPAGESIZE; 758 759 /* Prepare receive descriptors and buffers */ 760 if (ixgbe_setup_receive_structures(adapter)) { 761 device_printf(dev, "Could not setup receive structures\n"); 762 ixv_stop(adapter); 763 return; 764 } 765 766 /* Configure RX settings */ 767 ixv_initialize_receive_units(adapter); 768 769 #if 0 /* XXX isn't it required? -- msaitoh */ 770 /* Set the various hardware offload abilities */ 771 ifp->if_hwassist = 0; 772 if (ifp->if_capenable & IFCAP_TSO4) 773 ifp->if_hwassist |= CSUM_TSO; 774 if (ifp->if_capenable & IFCAP_TXCSUM) { 775 ifp->if_hwassist |= (CSUM_TCP | CSUM_UDP); 776 #if __FreeBSD_version >= 800000 777 ifp->if_hwassist |= CSUM_SCTP; 778 #endif 779 } 780 #endif 781 782 /* Set up VLAN offload and filter */ 783 ixv_setup_vlan_support(adapter); 784 785 /* Set up MSI-X routing */ 786 ixv_configure_ivars(adapter); 787 788 /* Set up auto-mask */ 789 mask = (1 << adapter->vector); 790 for (i = 0, que = adapter->queues; i < adapter->num_queues; i++, que++) 791 mask |= (1 << que->msix); 792 IXGBE_WRITE_REG(hw, IXGBE_VTEIAM, mask); 793 794 /* Set moderation on the Link interrupt */ 795 ixv_eitr_write(adapter, adapter->vector, IXGBE_LINK_ITR); 796 797 /* Stats init */ 798 ixv_init_stats(adapter); 799 800 /* Config/Enable Link */ 801 hw->mac.get_link_status = TRUE; 802 hw->mac.ops.check_link(hw, &adapter->link_speed, &adapter->link_up, 803 FALSE); 804 805 /* Start watchdog */ 806 callout_reset(&adapter->timer, hz, ixv_local_timer, adapter); 807 808 /* And now turn on interrupts */ 809 ixv_enable_intr(adapter); 810 811 /* Update saved flags. See ixgbe_ifflags_cb() */ 812 adapter->if_flags = ifp->if_flags; 813 adapter->ec_capenable = adapter->osdep.ec.ec_capenable; 814 815 /* Now inform the stack we're ready */ 816 ifp->if_flags |= IFF_RUNNING; 817 ifp->if_flags &= ~IFF_OACTIVE; 818 819 return; 820 } /* ixv_init_locked */ 821 822 /************************************************************************ 823 * ixv_enable_queue 824 ************************************************************************/ 825 static inline void 826 ixv_enable_queue(struct adapter *adapter, u32 vector) 827 { 828 struct ixgbe_hw *hw = &adapter->hw; 829 struct ix_queue *que = &adapter->queues[vector]; 830 u32 queue = 1UL << vector; 831 u32 mask; 832 833 mutex_enter(&que->dc_mtx); 834 if (que->disabled_count > 0 && --que->disabled_count > 0) 835 goto out; 836 837 mask = (IXGBE_EIMS_RTX_QUEUE & queue); 838 IXGBE_WRITE_REG(hw, IXGBE_VTEIMS, mask); 839 out: 840 mutex_exit(&que->dc_mtx); 841 } /* ixv_enable_queue */ 842 843 /************************************************************************ 844 * ixv_disable_queue 845 ************************************************************************/ 846 static inline void 847 ixv_disable_queue(struct adapter *adapter, u32 vector) 848 { 849 struct ixgbe_hw *hw = &adapter->hw; 850 struct ix_queue *que = &adapter->queues[vector]; 851 u32 queue = 1UL << vector; 852 u32 mask; 853 854 mutex_enter(&que->dc_mtx); 855 if (que->disabled_count++ > 0) 856 goto out; 857 858 mask = (IXGBE_EIMS_RTX_QUEUE & queue); 859 IXGBE_WRITE_REG(hw, IXGBE_VTEIMC, mask); 860 out: 861 mutex_exit(&que->dc_mtx); 862 } /* ixv_disable_queue */ 863 864 #if 0 865 static inline void 866 ixv_rearm_queues(struct adapter *adapter, u64 queues) 867 { 868 u32 mask = (IXGBE_EIMS_RTX_QUEUE & queues); 869 IXGBE_WRITE_REG(&adapter->hw, IXGBE_VTEICS, mask); 870 } /* ixv_rearm_queues */ 871 #endif 872 873 874 /************************************************************************ 875 * ixv_msix_que - MSI-X Queue Interrupt Service routine 876 ************************************************************************/ 877 static int 878 ixv_msix_que(void *arg) 879 { 880 struct ix_queue *que = arg; 881 struct adapter *adapter = que->adapter; 882 struct tx_ring *txr = que->txr; 883 struct rx_ring *rxr = que->rxr; 884 bool more; 885 u32 newitr = 0; 886 887 ixv_disable_queue(adapter, que->msix); 888 ++que->irqs.ev_count; 889 890 #ifdef __NetBSD__ 891 /* Don't run ixgbe_rxeof in interrupt context */ 892 more = true; 893 #else 894 more = ixgbe_rxeof(que); 895 #endif 896 897 IXGBE_TX_LOCK(txr); 898 ixgbe_txeof(txr); 899 IXGBE_TX_UNLOCK(txr); 900 901 /* Do AIM now? */ 902 903 if (adapter->enable_aim == false) 904 goto no_calc; 905 /* 906 * Do Adaptive Interrupt Moderation: 907 * - Write out last calculated setting 908 * - Calculate based on average size over 909 * the last interval. 910 */ 911 if (que->eitr_setting) 912 ixv_eitr_write(adapter, que->msix, que->eitr_setting); 913 914 que->eitr_setting = 0; 915 916 /* Idle, do nothing */ 917 if ((txr->bytes == 0) && (rxr->bytes == 0)) 918 goto no_calc; 919 920 if ((txr->bytes) && (txr->packets)) 921 newitr = txr->bytes/txr->packets; 922 if ((rxr->bytes) && (rxr->packets)) 923 newitr = uimax(newitr, (rxr->bytes / rxr->packets)); 924 newitr += 24; /* account for hardware frame, crc */ 925 926 /* set an upper boundary */ 927 newitr = uimin(newitr, 3000); 928 929 /* Be nice to the mid range */ 930 if ((newitr > 300) && (newitr < 1200)) 931 newitr = (newitr / 3); 932 else 933 newitr = (newitr / 2); 934 935 /* 936 * When RSC is used, ITR interval must be larger than RSC_DELAY. 937 * Currently, we use 2us for RSC_DELAY. The minimum value is always 938 * greater than 2us on 100M (and 10M?(not documented)), but it's not 939 * on 1G and higher. 940 */ 941 if ((adapter->link_speed != IXGBE_LINK_SPEED_100_FULL) 942 && (adapter->link_speed != IXGBE_LINK_SPEED_10_FULL)) { 943 if (newitr < IXGBE_MIN_RSC_EITR_10G1G) 944 newitr = IXGBE_MIN_RSC_EITR_10G1G; 945 } 946 947 /* save for next interrupt */ 948 que->eitr_setting = newitr; 949 950 /* Reset state */ 951 txr->bytes = 0; 952 txr->packets = 0; 953 rxr->bytes = 0; 954 rxr->packets = 0; 955 956 no_calc: 957 if (more) 958 softint_schedule(que->que_si); 959 else /* Re-enable this interrupt */ 960 ixv_enable_queue(adapter, que->msix); 961 962 return 1; 963 } /* ixv_msix_que */ 964 965 /************************************************************************ 966 * ixv_msix_mbx 967 ************************************************************************/ 968 static int 969 ixv_msix_mbx(void *arg) 970 { 971 struct adapter *adapter = arg; 972 struct ixgbe_hw *hw = &adapter->hw; 973 974 ++adapter->link_irq.ev_count; 975 /* NetBSD: We use auto-clear, so it's not required to write VTEICR */ 976 977 /* Link status change */ 978 hw->mac.get_link_status = TRUE; 979 softint_schedule(adapter->link_si); 980 981 IXGBE_WRITE_REG(hw, IXGBE_VTEIMS, (1 << adapter->vector)); 982 983 return 1; 984 } /* ixv_msix_mbx */ 985 986 static void 987 ixv_eitr_write(struct adapter *adapter, uint32_t index, uint32_t itr) 988 { 989 990 /* 991 * Newer devices than 82598 have VF function, so this function is 992 * simple. 993 */ 994 itr |= IXGBE_EITR_CNT_WDIS; 995 996 IXGBE_WRITE_REG(&adapter->hw, IXGBE_VTEITR(index), itr); 997 } 998 999 1000 /************************************************************************ 1001 * ixv_media_status - Media Ioctl callback 1002 * 1003 * Called whenever the user queries the status of 1004 * the interface using ifconfig. 1005 ************************************************************************/ 1006 static void 1007 ixv_media_status(struct ifnet *ifp, struct ifmediareq *ifmr) 1008 { 1009 struct adapter *adapter = ifp->if_softc; 1010 1011 INIT_DEBUGOUT("ixv_media_status: begin"); 1012 IXGBE_CORE_LOCK(adapter); 1013 ixv_update_link_status(adapter); 1014 1015 ifmr->ifm_status = IFM_AVALID; 1016 ifmr->ifm_active = IFM_ETHER; 1017 1018 if (adapter->link_active != LINK_STATE_UP) { 1019 ifmr->ifm_active |= IFM_NONE; 1020 IXGBE_CORE_UNLOCK(adapter); 1021 return; 1022 } 1023 1024 ifmr->ifm_status |= IFM_ACTIVE; 1025 1026 switch (adapter->link_speed) { 1027 case IXGBE_LINK_SPEED_10GB_FULL: 1028 ifmr->ifm_active |= IFM_10G_T | IFM_FDX; 1029 break; 1030 case IXGBE_LINK_SPEED_5GB_FULL: 1031 ifmr->ifm_active |= IFM_5000_T | IFM_FDX; 1032 break; 1033 case IXGBE_LINK_SPEED_2_5GB_FULL: 1034 ifmr->ifm_active |= IFM_2500_T | IFM_FDX; 1035 break; 1036 case IXGBE_LINK_SPEED_1GB_FULL: 1037 ifmr->ifm_active |= IFM_1000_T | IFM_FDX; 1038 break; 1039 case IXGBE_LINK_SPEED_100_FULL: 1040 ifmr->ifm_active |= IFM_100_TX | IFM_FDX; 1041 break; 1042 case IXGBE_LINK_SPEED_10_FULL: 1043 ifmr->ifm_active |= IFM_10_T | IFM_FDX; 1044 break; 1045 } 1046 1047 ifp->if_baudrate = ifmedia_baudrate(ifmr->ifm_active); 1048 1049 IXGBE_CORE_UNLOCK(adapter); 1050 } /* ixv_media_status */ 1051 1052 /************************************************************************ 1053 * ixv_media_change - Media Ioctl callback 1054 * 1055 * Called when the user changes speed/duplex using 1056 * media/mediopt option with ifconfig. 1057 ************************************************************************/ 1058 static int 1059 ixv_media_change(struct ifnet *ifp) 1060 { 1061 struct adapter *adapter = ifp->if_softc; 1062 struct ifmedia *ifm = &adapter->media; 1063 1064 INIT_DEBUGOUT("ixv_media_change: begin"); 1065 1066 if (IFM_TYPE(ifm->ifm_media) != IFM_ETHER) 1067 return (EINVAL); 1068 1069 switch (IFM_SUBTYPE(ifm->ifm_media)) { 1070 case IFM_AUTO: 1071 break; 1072 default: 1073 device_printf(adapter->dev, "Only auto media type\n"); 1074 return (EINVAL); 1075 } 1076 1077 return (0); 1078 } /* ixv_media_change */ 1079 1080 /************************************************************************ 1081 * ixv_negotiate_api 1082 * 1083 * Negotiate the Mailbox API with the PF; 1084 * start with the most featured API first. 1085 ************************************************************************/ 1086 static int 1087 ixv_negotiate_api(struct adapter *adapter) 1088 { 1089 struct ixgbe_hw *hw = &adapter->hw; 1090 int mbx_api[] = { ixgbe_mbox_api_13, 1091 ixgbe_mbox_api_12, 1092 ixgbe_mbox_api_11, 1093 ixgbe_mbox_api_10, 1094 ixgbe_mbox_api_unknown }; 1095 int i = 0; 1096 1097 while (mbx_api[i] != ixgbe_mbox_api_unknown) { 1098 if (ixgbevf_negotiate_api_version(hw, mbx_api[i]) == 0) 1099 return (0); 1100 i++; 1101 } 1102 1103 return (EINVAL); 1104 } /* ixv_negotiate_api */ 1105 1106 1107 /************************************************************************ 1108 * ixv_set_multi - Multicast Update 1109 * 1110 * Called whenever multicast address list is updated. 1111 ************************************************************************/ 1112 static int 1113 ixv_set_rxfilter(struct adapter *adapter) 1114 { 1115 u8 mta[IXGBE_MAX_VF_MC * IXGBE_ETH_LENGTH_OF_ADDRESS]; 1116 struct ifnet *ifp = adapter->ifp; 1117 struct ixgbe_hw *hw = &adapter->hw; 1118 u8 *update_ptr; 1119 int mcnt = 0; 1120 struct ethercom *ec = &adapter->osdep.ec; 1121 struct ether_multi *enm; 1122 struct ether_multistep step; 1123 bool overflow = false; 1124 int error, rc = 0; 1125 1126 KASSERT(mutex_owned(&adapter->core_mtx)); 1127 IOCTL_DEBUGOUT("ixv_set_rxfilter: begin"); 1128 1129 /* 1: For PROMISC */ 1130 if (ifp->if_flags & IFF_PROMISC) { 1131 error = hw->mac.ops.update_xcast_mode(hw, 1132 IXGBEVF_XCAST_MODE_PROMISC); 1133 if (error == IXGBE_ERR_NOT_TRUSTED) { 1134 device_printf(adapter->dev, 1135 "this interface is not trusted\n"); 1136 error = EPERM; 1137 } else if (error == IXGBE_ERR_FEATURE_NOT_SUPPORTED) { 1138 device_printf(adapter->dev, 1139 "the PF doesn't support promisc mode\n"); 1140 error = EOPNOTSUPP; 1141 } else if (error == IXGBE_ERR_NOT_IN_PROMISC) { 1142 device_printf(adapter->dev, 1143 "the PF may not in promisc mode\n"); 1144 error = EINVAL; 1145 } else if (error) { 1146 device_printf(adapter->dev, 1147 "failed to set promisc mode. error = %d\n", 1148 error); 1149 error = EIO; 1150 } else 1151 return 0; 1152 rc = error; 1153 } 1154 1155 /* 2: For ALLMULTI or normal */ 1156 ETHER_LOCK(ec); 1157 ETHER_FIRST_MULTI(step, ec, enm); 1158 while (enm != NULL) { 1159 if ((mcnt >= IXGBE_MAX_VF_MC) || 1160 (memcmp(enm->enm_addrlo, enm->enm_addrhi, 1161 ETHER_ADDR_LEN) != 0)) { 1162 overflow = true; 1163 break; 1164 } 1165 bcopy(enm->enm_addrlo, 1166 &mta[mcnt * IXGBE_ETH_LENGTH_OF_ADDRESS], 1167 IXGBE_ETH_LENGTH_OF_ADDRESS); 1168 mcnt++; 1169 ETHER_NEXT_MULTI(step, enm); 1170 } 1171 ETHER_UNLOCK(ec); 1172 1173 /* 3: For ALLMULTI */ 1174 if (overflow) { 1175 error = hw->mac.ops.update_xcast_mode(hw, 1176 IXGBEVF_XCAST_MODE_ALLMULTI); 1177 if (error == IXGBE_ERR_NOT_TRUSTED) { 1178 device_printf(adapter->dev, 1179 "this interface is not trusted\n"); 1180 error = EPERM; 1181 } else if (error == IXGBE_ERR_FEATURE_NOT_SUPPORTED) { 1182 device_printf(adapter->dev, 1183 "the PF doesn't support allmulti mode\n"); 1184 error = EOPNOTSUPP; 1185 } else if (error) { 1186 device_printf(adapter->dev, 1187 "number of Ethernet multicast addresses " 1188 "exceeds the limit (%d). error = %d\n", 1189 IXGBE_MAX_VF_MC, error); 1190 error = ENOSPC; 1191 } else { 1192 ETHER_LOCK(ec); 1193 ec->ec_flags |= ETHER_F_ALLMULTI; 1194 ETHER_UNLOCK(ec); 1195 return rc; /* Promisc might failed */ 1196 } 1197 1198 if (rc == 0) 1199 rc = error; 1200 1201 /* Continue to update the multicast table as many as we can */ 1202 } 1203 1204 /* 4: For normal operation */ 1205 error = hw->mac.ops.update_xcast_mode(hw, IXGBEVF_XCAST_MODE_MULTI); 1206 if ((error == IXGBE_ERR_FEATURE_NOT_SUPPORTED) || (error == 0)) { 1207 /* Normal operation */ 1208 ETHER_LOCK(ec); 1209 ec->ec_flags &= ~ETHER_F_ALLMULTI; 1210 ETHER_UNLOCK(ec); 1211 error = 0; 1212 } else if (error) { 1213 device_printf(adapter->dev, 1214 "failed to set Ethernet multicast address " 1215 "operation to normal. error = %d\n", error); 1216 } 1217 1218 update_ptr = mta; 1219 1220 error = adapter->hw.mac.ops.update_mc_addr_list(&adapter->hw, 1221 update_ptr, mcnt, ixv_mc_array_itr, TRUE); 1222 if (rc == 0) 1223 rc = error; 1224 1225 return rc; 1226 } /* ixv_set_rxfilter */ 1227 1228 /************************************************************************ 1229 * ixv_mc_array_itr 1230 * 1231 * An iterator function needed by the multicast shared code. 1232 * It feeds the shared code routine the addresses in the 1233 * array of ixv_set_rxfilter() one by one. 1234 ************************************************************************/ 1235 static u8 * 1236 ixv_mc_array_itr(struct ixgbe_hw *hw, u8 **update_ptr, u32 *vmdq) 1237 { 1238 u8 *addr = *update_ptr; 1239 u8 *newptr; 1240 1241 *vmdq = 0; 1242 1243 newptr = addr + IXGBE_ETH_LENGTH_OF_ADDRESS; 1244 *update_ptr = newptr; 1245 1246 return addr; 1247 } /* ixv_mc_array_itr */ 1248 1249 /************************************************************************ 1250 * ixv_local_timer - Timer routine 1251 * 1252 * Checks for link status, updates statistics, 1253 * and runs the watchdog check. 1254 ************************************************************************/ 1255 static void 1256 ixv_local_timer(void *arg) 1257 { 1258 struct adapter *adapter = arg; 1259 1260 IXGBE_CORE_LOCK(adapter); 1261 ixv_local_timer_locked(adapter); 1262 IXGBE_CORE_UNLOCK(adapter); 1263 } 1264 1265 static void 1266 ixv_local_timer_locked(void *arg) 1267 { 1268 struct adapter *adapter = arg; 1269 device_t dev = adapter->dev; 1270 struct ix_queue *que = adapter->queues; 1271 u64 queues = 0; 1272 u64 v0, v1, v2, v3, v4, v5, v6, v7; 1273 int hung = 0; 1274 int i; 1275 1276 KASSERT(mutex_owned(&adapter->core_mtx)); 1277 1278 if (ixv_check_link(adapter)) { 1279 ixv_init_locked(adapter); 1280 return; 1281 } 1282 1283 /* Stats Update */ 1284 ixv_update_stats(adapter); 1285 1286 /* Update some event counters */ 1287 v0 = v1 = v2 = v3 = v4 = v5 = v6 = v7 = 0; 1288 que = adapter->queues; 1289 for (i = 0; i < adapter->num_queues; i++, que++) { 1290 struct tx_ring *txr = que->txr; 1291 1292 v0 += txr->q_efbig_tx_dma_setup; 1293 v1 += txr->q_mbuf_defrag_failed; 1294 v2 += txr->q_efbig2_tx_dma_setup; 1295 v3 += txr->q_einval_tx_dma_setup; 1296 v4 += txr->q_other_tx_dma_setup; 1297 v5 += txr->q_eagain_tx_dma_setup; 1298 v6 += txr->q_enomem_tx_dma_setup; 1299 v7 += txr->q_tso_err; 1300 } 1301 adapter->efbig_tx_dma_setup.ev_count = v0; 1302 adapter->mbuf_defrag_failed.ev_count = v1; 1303 adapter->efbig2_tx_dma_setup.ev_count = v2; 1304 adapter->einval_tx_dma_setup.ev_count = v3; 1305 adapter->other_tx_dma_setup.ev_count = v4; 1306 adapter->eagain_tx_dma_setup.ev_count = v5; 1307 adapter->enomem_tx_dma_setup.ev_count = v6; 1308 adapter->tso_err.ev_count = v7; 1309 1310 /* 1311 * Check the TX queues status 1312 * - mark hung queues so we don't schedule on them 1313 * - watchdog only if all queues show hung 1314 */ 1315 que = adapter->queues; 1316 for (i = 0; i < adapter->num_queues; i++, que++) { 1317 /* Keep track of queues with work for soft irq */ 1318 if (que->txr->busy) 1319 queues |= ((u64)1 << que->me); 1320 /* 1321 * Each time txeof runs without cleaning, but there 1322 * are uncleaned descriptors it increments busy. If 1323 * we get to the MAX we declare it hung. 1324 */ 1325 if (que->busy == IXGBE_QUEUE_HUNG) { 1326 ++hung; 1327 /* Mark the queue as inactive */ 1328 adapter->active_queues &= ~((u64)1 << que->me); 1329 continue; 1330 } else { 1331 /* Check if we've come back from hung */ 1332 if ((adapter->active_queues & ((u64)1 << que->me)) == 0) 1333 adapter->active_queues |= ((u64)1 << que->me); 1334 } 1335 if (que->busy >= IXGBE_MAX_TX_BUSY) { 1336 device_printf(dev, 1337 "Warning queue %d appears to be hung!\n", i); 1338 que->txr->busy = IXGBE_QUEUE_HUNG; 1339 ++hung; 1340 } 1341 } 1342 1343 /* Only truly watchdog if all queues show hung */ 1344 if (hung == adapter->num_queues) 1345 goto watchdog; 1346 #if 0 1347 else if (queues != 0) { /* Force an IRQ on queues with work */ 1348 ixv_rearm_queues(adapter, queues); 1349 } 1350 #endif 1351 1352 callout_reset(&adapter->timer, hz, ixv_local_timer, adapter); 1353 1354 return; 1355 1356 watchdog: 1357 1358 device_printf(adapter->dev, "Watchdog timeout -- resetting\n"); 1359 adapter->ifp->if_flags &= ~IFF_RUNNING; 1360 adapter->watchdog_events.ev_count++; 1361 ixv_init_locked(adapter); 1362 } /* ixv_local_timer */ 1363 1364 /************************************************************************ 1365 * ixv_update_link_status - Update OS on link state 1366 * 1367 * Note: Only updates the OS on the cached link state. 1368 * The real check of the hardware only happens with 1369 * a link interrupt. 1370 ************************************************************************/ 1371 static void 1372 ixv_update_link_status(struct adapter *adapter) 1373 { 1374 struct ifnet *ifp = adapter->ifp; 1375 device_t dev = adapter->dev; 1376 1377 KASSERT(mutex_owned(&adapter->core_mtx)); 1378 1379 if (adapter->link_up) { 1380 if (adapter->link_active != LINK_STATE_UP) { 1381 if (bootverbose) { 1382 const char *bpsmsg; 1383 1384 switch (adapter->link_speed) { 1385 case IXGBE_LINK_SPEED_10GB_FULL: 1386 bpsmsg = "10 Gbps"; 1387 break; 1388 case IXGBE_LINK_SPEED_5GB_FULL: 1389 bpsmsg = "5 Gbps"; 1390 break; 1391 case IXGBE_LINK_SPEED_2_5GB_FULL: 1392 bpsmsg = "2.5 Gbps"; 1393 break; 1394 case IXGBE_LINK_SPEED_1GB_FULL: 1395 bpsmsg = "1 Gbps"; 1396 break; 1397 case IXGBE_LINK_SPEED_100_FULL: 1398 bpsmsg = "100 Mbps"; 1399 break; 1400 case IXGBE_LINK_SPEED_10_FULL: 1401 bpsmsg = "10 Mbps"; 1402 break; 1403 default: 1404 bpsmsg = "unknown speed"; 1405 break; 1406 } 1407 device_printf(dev, "Link is up %s %s \n", 1408 bpsmsg, "Full Duplex"); 1409 } 1410 adapter->link_active = LINK_STATE_UP; 1411 if_link_state_change(ifp, LINK_STATE_UP); 1412 } 1413 } else { 1414 /* 1415 * Do it when link active changes to DOWN. i.e. 1416 * a) LINK_STATE_UNKNOWN -> LINK_STATE_DOWN 1417 * b) LINK_STATE_UP -> LINK_STATE_DOWN 1418 */ 1419 if (adapter->link_active != LINK_STATE_DOWN) { 1420 if (bootverbose) 1421 device_printf(dev, "Link is Down\n"); 1422 if_link_state_change(ifp, LINK_STATE_DOWN); 1423 adapter->link_active = LINK_STATE_DOWN; 1424 } 1425 } 1426 } /* ixv_update_link_status */ 1427 1428 1429 /************************************************************************ 1430 * ixv_stop - Stop the hardware 1431 * 1432 * Disables all traffic on the adapter by issuing a 1433 * global reset on the MAC and deallocates TX/RX buffers. 1434 ************************************************************************/ 1435 static void 1436 ixv_ifstop(struct ifnet *ifp, int disable) 1437 { 1438 struct adapter *adapter = ifp->if_softc; 1439 1440 IXGBE_CORE_LOCK(adapter); 1441 ixv_stop(adapter); 1442 IXGBE_CORE_UNLOCK(adapter); 1443 } 1444 1445 static void 1446 ixv_stop(void *arg) 1447 { 1448 struct ifnet *ifp; 1449 struct adapter *adapter = arg; 1450 struct ixgbe_hw *hw = &adapter->hw; 1451 1452 ifp = adapter->ifp; 1453 1454 KASSERT(mutex_owned(&adapter->core_mtx)); 1455 1456 INIT_DEBUGOUT("ixv_stop: begin\n"); 1457 ixv_disable_intr(adapter); 1458 1459 /* Tell the stack that the interface is no longer active */ 1460 ifp->if_flags &= ~(IFF_RUNNING | IFF_OACTIVE); 1461 1462 hw->mac.ops.reset_hw(hw); 1463 adapter->hw.adapter_stopped = FALSE; 1464 hw->mac.ops.stop_adapter(hw); 1465 callout_stop(&adapter->timer); 1466 1467 /* reprogram the RAR[0] in case user changed it. */ 1468 hw->mac.ops.set_rar(hw, 0, hw->mac.addr, 0, IXGBE_RAH_AV); 1469 1470 return; 1471 } /* ixv_stop */ 1472 1473 1474 /************************************************************************ 1475 * ixv_allocate_pci_resources 1476 ************************************************************************/ 1477 static int 1478 ixv_allocate_pci_resources(struct adapter *adapter, 1479 const struct pci_attach_args *pa) 1480 { 1481 pcireg_t memtype, csr; 1482 device_t dev = adapter->dev; 1483 bus_addr_t addr; 1484 int flags; 1485 1486 memtype = pci_mapreg_type(pa->pa_pc, pa->pa_tag, PCI_BAR(0)); 1487 switch (memtype) { 1488 case PCI_MAPREG_TYPE_MEM | PCI_MAPREG_MEM_TYPE_32BIT: 1489 case PCI_MAPREG_TYPE_MEM | PCI_MAPREG_MEM_TYPE_64BIT: 1490 adapter->osdep.mem_bus_space_tag = pa->pa_memt; 1491 if (pci_mapreg_info(pa->pa_pc, pa->pa_tag, PCI_BAR(0), 1492 memtype, &addr, &adapter->osdep.mem_size, &flags) != 0) 1493 goto map_err; 1494 if ((flags & BUS_SPACE_MAP_PREFETCHABLE) != 0) { 1495 aprint_normal_dev(dev, "clearing prefetchable bit\n"); 1496 flags &= ~BUS_SPACE_MAP_PREFETCHABLE; 1497 } 1498 if (bus_space_map(adapter->osdep.mem_bus_space_tag, addr, 1499 adapter->osdep.mem_size, flags, 1500 &adapter->osdep.mem_bus_space_handle) != 0) { 1501 map_err: 1502 adapter->osdep.mem_size = 0; 1503 aprint_error_dev(dev, "unable to map BAR0\n"); 1504 return ENXIO; 1505 } 1506 /* 1507 * Enable address decoding for memory range in case it's not 1508 * set. 1509 */ 1510 csr = pci_conf_read(pa->pa_pc, pa->pa_tag, 1511 PCI_COMMAND_STATUS_REG); 1512 csr |= PCI_COMMAND_MEM_ENABLE; 1513 pci_conf_write(pa->pa_pc, pa->pa_tag, PCI_COMMAND_STATUS_REG, 1514 csr); 1515 break; 1516 default: 1517 aprint_error_dev(dev, "unexpected type on BAR0\n"); 1518 return ENXIO; 1519 } 1520 1521 /* Pick up the tuneable queues */ 1522 adapter->num_queues = ixv_num_queues; 1523 1524 return (0); 1525 } /* ixv_allocate_pci_resources */ 1526 1527 /************************************************************************ 1528 * ixv_free_pci_resources 1529 ************************************************************************/ 1530 static void 1531 ixv_free_pci_resources(struct adapter * adapter) 1532 { 1533 struct ix_queue *que = adapter->queues; 1534 int rid; 1535 1536 /* 1537 * Release all msix queue resources: 1538 */ 1539 for (int i = 0; i < adapter->num_queues; i++, que++) { 1540 if (que->res != NULL) 1541 pci_intr_disestablish(adapter->osdep.pc, 1542 adapter->osdep.ihs[i]); 1543 } 1544 1545 1546 /* Clean the Mailbox interrupt last */ 1547 rid = adapter->vector; 1548 1549 if (adapter->osdep.ihs[rid] != NULL) { 1550 pci_intr_disestablish(adapter->osdep.pc, 1551 adapter->osdep.ihs[rid]); 1552 adapter->osdep.ihs[rid] = NULL; 1553 } 1554 1555 pci_intr_release(adapter->osdep.pc, adapter->osdep.intrs, 1556 adapter->osdep.nintrs); 1557 1558 if (adapter->osdep.mem_size != 0) { 1559 bus_space_unmap(adapter->osdep.mem_bus_space_tag, 1560 adapter->osdep.mem_bus_space_handle, 1561 adapter->osdep.mem_size); 1562 } 1563 1564 return; 1565 } /* ixv_free_pci_resources */ 1566 1567 /************************************************************************ 1568 * ixv_setup_interface 1569 * 1570 * Setup networking device structure and register an interface. 1571 ************************************************************************/ 1572 static int 1573 ixv_setup_interface(device_t dev, struct adapter *adapter) 1574 { 1575 struct ethercom *ec = &adapter->osdep.ec; 1576 struct ifnet *ifp; 1577 int rv; 1578 1579 INIT_DEBUGOUT("ixv_setup_interface: begin"); 1580 1581 ifp = adapter->ifp = &ec->ec_if; 1582 strlcpy(ifp->if_xname, device_xname(dev), IFNAMSIZ); 1583 ifp->if_baudrate = IF_Gbps(10); 1584 ifp->if_init = ixv_init; 1585 ifp->if_stop = ixv_ifstop; 1586 ifp->if_softc = adapter; 1587 ifp->if_flags = IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST; 1588 #ifdef IXGBE_MPSAFE 1589 ifp->if_extflags = IFEF_MPSAFE; 1590 #endif 1591 ifp->if_ioctl = ixv_ioctl; 1592 if (adapter->feat_en & IXGBE_FEATURE_LEGACY_TX) { 1593 #if 0 1594 ixv_start_locked = ixgbe_legacy_start_locked; 1595 #endif 1596 } else { 1597 ifp->if_transmit = ixgbe_mq_start; 1598 #if 0 1599 ixv_start_locked = ixgbe_mq_start_locked; 1600 #endif 1601 } 1602 ifp->if_start = ixgbe_legacy_start; 1603 IFQ_SET_MAXLEN(&ifp->if_snd, adapter->num_tx_desc - 2); 1604 IFQ_SET_READY(&ifp->if_snd); 1605 1606 rv = if_initialize(ifp); 1607 if (rv != 0) { 1608 aprint_error_dev(dev, "if_initialize failed(%d)\n", rv); 1609 return rv; 1610 } 1611 adapter->ipq = if_percpuq_create(&adapter->osdep.ec.ec_if); 1612 ether_ifattach(ifp, adapter->hw.mac.addr); 1613 /* 1614 * We use per TX queue softint, so if_deferred_start_init() isn't 1615 * used. 1616 */ 1617 ether_set_ifflags_cb(ec, ixv_ifflags_cb); 1618 1619 adapter->max_frame_size = ifp->if_mtu + IXGBE_MTU_HDR; 1620 1621 /* 1622 * Tell the upper layer(s) we support long frames. 1623 */ 1624 ifp->if_hdrlen = sizeof(struct ether_vlan_header); 1625 1626 /* Set capability flags */ 1627 ifp->if_capabilities |= IFCAP_HWCSUM 1628 | IFCAP_TSOv4 1629 | IFCAP_TSOv6; 1630 ifp->if_capenable = 0; 1631 1632 ec->ec_capabilities |= ETHERCAP_VLAN_HWFILTER 1633 | ETHERCAP_VLAN_HWTAGGING 1634 | ETHERCAP_VLAN_HWCSUM 1635 | ETHERCAP_JUMBO_MTU 1636 | ETHERCAP_VLAN_MTU; 1637 1638 /* Enable the above capabilities by default */ 1639 ec->ec_capenable = ec->ec_capabilities; 1640 1641 /* Don't enable LRO by default */ 1642 #if 0 1643 /* NetBSD doesn't support LRO yet */ 1644 ifp->if_capabilities |= IFCAP_LRO; 1645 #endif 1646 1647 /* 1648 * Specify the media types supported by this adapter and register 1649 * callbacks to update media and link information 1650 */ 1651 ec->ec_ifmedia = &adapter->media; 1652 ifmedia_init(&adapter->media, IFM_IMASK, ixv_media_change, 1653 ixv_media_status); 1654 ifmedia_add(&adapter->media, IFM_ETHER | IFM_AUTO, 0, NULL); 1655 ifmedia_set(&adapter->media, IFM_ETHER | IFM_AUTO); 1656 1657 if_register(ifp); 1658 1659 return 0; 1660 } /* ixv_setup_interface */ 1661 1662 1663 /************************************************************************ 1664 * ixv_initialize_transmit_units - Enable transmit unit. 1665 ************************************************************************/ 1666 static void 1667 ixv_initialize_transmit_units(struct adapter *adapter) 1668 { 1669 struct tx_ring *txr = adapter->tx_rings; 1670 struct ixgbe_hw *hw = &adapter->hw; 1671 int i; 1672 1673 for (i = 0; i < adapter->num_queues; i++, txr++) { 1674 u64 tdba = txr->txdma.dma_paddr; 1675 u32 txctrl, txdctl; 1676 int j = txr->me; 1677 1678 /* Set WTHRESH to 8, burst writeback */ 1679 txdctl = IXGBE_READ_REG(hw, IXGBE_VFTXDCTL(j)); 1680 txdctl |= (8 << 16); 1681 IXGBE_WRITE_REG(hw, IXGBE_VFTXDCTL(j), txdctl); 1682 1683 /* Set the HW Tx Head and Tail indices */ 1684 IXGBE_WRITE_REG(&adapter->hw, IXGBE_VFTDH(j), 0); 1685 IXGBE_WRITE_REG(&adapter->hw, IXGBE_VFTDT(j), 0); 1686 1687 /* Set Tx Tail register */ 1688 txr->tail = IXGBE_VFTDT(j); 1689 1690 txr->txr_no_space = false; 1691 1692 /* Set Ring parameters */ 1693 IXGBE_WRITE_REG(hw, IXGBE_VFTDBAL(j), 1694 (tdba & 0x00000000ffffffffULL)); 1695 IXGBE_WRITE_REG(hw, IXGBE_VFTDBAH(j), (tdba >> 32)); 1696 IXGBE_WRITE_REG(hw, IXGBE_VFTDLEN(j), 1697 adapter->num_tx_desc * sizeof(struct ixgbe_legacy_tx_desc)); 1698 txctrl = IXGBE_READ_REG(hw, IXGBE_VFDCA_TXCTRL(j)); 1699 txctrl &= ~IXGBE_DCA_TXCTRL_DESC_WRO_EN; 1700 IXGBE_WRITE_REG(hw, IXGBE_VFDCA_TXCTRL(j), txctrl); 1701 1702 /* Now enable */ 1703 txdctl = IXGBE_READ_REG(hw, IXGBE_VFTXDCTL(j)); 1704 txdctl |= IXGBE_TXDCTL_ENABLE; 1705 IXGBE_WRITE_REG(hw, IXGBE_VFTXDCTL(j), txdctl); 1706 } 1707 1708 return; 1709 } /* ixv_initialize_transmit_units */ 1710 1711 1712 /************************************************************************ 1713 * ixv_initialize_rss_mapping 1714 ************************************************************************/ 1715 static void 1716 ixv_initialize_rss_mapping(struct adapter *adapter) 1717 { 1718 struct ixgbe_hw *hw = &adapter->hw; 1719 u32 reta = 0, mrqc, rss_key[10]; 1720 int queue_id; 1721 int i, j; 1722 u32 rss_hash_config; 1723 1724 /* force use default RSS key. */ 1725 #ifdef __NetBSD__ 1726 rss_getkey((uint8_t *) &rss_key); 1727 #else 1728 if (adapter->feat_en & IXGBE_FEATURE_RSS) { 1729 /* Fetch the configured RSS key */ 1730 rss_getkey((uint8_t *)&rss_key); 1731 } else { 1732 /* set up random bits */ 1733 cprng_fast(&rss_key, sizeof(rss_key)); 1734 } 1735 #endif 1736 1737 /* Now fill out hash function seeds */ 1738 for (i = 0; i < 10; i++) 1739 IXGBE_WRITE_REG(hw, IXGBE_VFRSSRK(i), rss_key[i]); 1740 1741 /* Set up the redirection table */ 1742 for (i = 0, j = 0; i < 64; i++, j++) { 1743 if (j == adapter->num_queues) 1744 j = 0; 1745 1746 if (adapter->feat_en & IXGBE_FEATURE_RSS) { 1747 /* 1748 * Fetch the RSS bucket id for the given indirection 1749 * entry. Cap it at the number of configured buckets 1750 * (which is num_queues.) 1751 */ 1752 queue_id = rss_get_indirection_to_bucket(i); 1753 queue_id = queue_id % adapter->num_queues; 1754 } else 1755 queue_id = j; 1756 1757 /* 1758 * The low 8 bits are for hash value (n+0); 1759 * The next 8 bits are for hash value (n+1), etc. 1760 */ 1761 reta >>= 8; 1762 reta |= ((uint32_t)queue_id) << 24; 1763 if ((i & 3) == 3) { 1764 IXGBE_WRITE_REG(hw, IXGBE_VFRETA(i >> 2), reta); 1765 reta = 0; 1766 } 1767 } 1768 1769 /* Perform hash on these packet types */ 1770 if (adapter->feat_en & IXGBE_FEATURE_RSS) 1771 rss_hash_config = rss_gethashconfig(); 1772 else { 1773 /* 1774 * Disable UDP - IP fragments aren't currently being handled 1775 * and so we end up with a mix of 2-tuple and 4-tuple 1776 * traffic. 1777 */ 1778 rss_hash_config = RSS_HASHTYPE_RSS_IPV4 1779 | RSS_HASHTYPE_RSS_TCP_IPV4 1780 | RSS_HASHTYPE_RSS_IPV6 1781 | RSS_HASHTYPE_RSS_TCP_IPV6; 1782 } 1783 1784 mrqc = IXGBE_MRQC_RSSEN; 1785 if (rss_hash_config & RSS_HASHTYPE_RSS_IPV4) 1786 mrqc |= IXGBE_MRQC_RSS_FIELD_IPV4; 1787 if (rss_hash_config & RSS_HASHTYPE_RSS_TCP_IPV4) 1788 mrqc |= IXGBE_MRQC_RSS_FIELD_IPV4_TCP; 1789 if (rss_hash_config & RSS_HASHTYPE_RSS_IPV6) 1790 mrqc |= IXGBE_MRQC_RSS_FIELD_IPV6; 1791 if (rss_hash_config & RSS_HASHTYPE_RSS_TCP_IPV6) 1792 mrqc |= IXGBE_MRQC_RSS_FIELD_IPV6_TCP; 1793 if (rss_hash_config & RSS_HASHTYPE_RSS_IPV6_EX) 1794 device_printf(adapter->dev, "%s: RSS_HASHTYPE_RSS_IPV6_EX defined, but not supported\n", 1795 __func__); 1796 if (rss_hash_config & RSS_HASHTYPE_RSS_TCP_IPV6_EX) 1797 device_printf(adapter->dev, "%s: RSS_HASHTYPE_RSS_TCP_IPV6_EX defined, but not supported\n", 1798 __func__); 1799 if (rss_hash_config & RSS_HASHTYPE_RSS_UDP_IPV4) 1800 mrqc |= IXGBE_MRQC_RSS_FIELD_IPV4_UDP; 1801 if (rss_hash_config & RSS_HASHTYPE_RSS_UDP_IPV6) 1802 mrqc |= IXGBE_MRQC_RSS_FIELD_IPV6_UDP; 1803 if (rss_hash_config & RSS_HASHTYPE_RSS_UDP_IPV6_EX) 1804 device_printf(adapter->dev, "%s: RSS_HASHTYPE_RSS_UDP_IPV6_EX defined, but not supported\n", 1805 __func__); 1806 IXGBE_WRITE_REG(hw, IXGBE_VFMRQC, mrqc); 1807 } /* ixv_initialize_rss_mapping */ 1808 1809 1810 /************************************************************************ 1811 * ixv_initialize_receive_units - Setup receive registers and features. 1812 ************************************************************************/ 1813 static void 1814 ixv_initialize_receive_units(struct adapter *adapter) 1815 { 1816 struct rx_ring *rxr = adapter->rx_rings; 1817 struct ixgbe_hw *hw = &adapter->hw; 1818 struct ifnet *ifp = adapter->ifp; 1819 u32 bufsz, psrtype; 1820 1821 if (ifp->if_mtu > ETHERMTU) 1822 bufsz = 4096 >> IXGBE_SRRCTL_BSIZEPKT_SHIFT; 1823 else 1824 bufsz = 2048 >> IXGBE_SRRCTL_BSIZEPKT_SHIFT; 1825 1826 psrtype = IXGBE_PSRTYPE_TCPHDR 1827 | IXGBE_PSRTYPE_UDPHDR 1828 | IXGBE_PSRTYPE_IPV4HDR 1829 | IXGBE_PSRTYPE_IPV6HDR 1830 | IXGBE_PSRTYPE_L2HDR; 1831 1832 if (adapter->num_queues > 1) 1833 psrtype |= 1 << 29; 1834 1835 IXGBE_WRITE_REG(hw, IXGBE_VFPSRTYPE, psrtype); 1836 1837 /* Tell PF our max_frame size */ 1838 if (ixgbevf_rlpml_set_vf(hw, adapter->max_frame_size) != 0) { 1839 device_printf(adapter->dev, "There is a problem with the PF setup. It is likely the receive unit for this VF will not function correctly.\n"); 1840 } 1841 1842 for (int i = 0; i < adapter->num_queues; i++, rxr++) { 1843 u64 rdba = rxr->rxdma.dma_paddr; 1844 u32 reg, rxdctl; 1845 int j = rxr->me; 1846 1847 /* Disable the queue */ 1848 rxdctl = IXGBE_READ_REG(hw, IXGBE_VFRXDCTL(j)); 1849 rxdctl &= ~IXGBE_RXDCTL_ENABLE; 1850 IXGBE_WRITE_REG(hw, IXGBE_VFRXDCTL(j), rxdctl); 1851 for (int k = 0; k < 10; k++) { 1852 if (IXGBE_READ_REG(hw, IXGBE_VFRXDCTL(j)) & 1853 IXGBE_RXDCTL_ENABLE) 1854 msec_delay(1); 1855 else 1856 break; 1857 } 1858 wmb(); 1859 /* Setup the Base and Length of the Rx Descriptor Ring */ 1860 IXGBE_WRITE_REG(hw, IXGBE_VFRDBAL(j), 1861 (rdba & 0x00000000ffffffffULL)); 1862 IXGBE_WRITE_REG(hw, IXGBE_VFRDBAH(j), (rdba >> 32)); 1863 IXGBE_WRITE_REG(hw, IXGBE_VFRDLEN(j), 1864 adapter->num_rx_desc * sizeof(union ixgbe_adv_rx_desc)); 1865 1866 /* Reset the ring indices */ 1867 IXGBE_WRITE_REG(hw, IXGBE_VFRDH(rxr->me), 0); 1868 IXGBE_WRITE_REG(hw, IXGBE_VFRDT(rxr->me), 0); 1869 1870 /* Set up the SRRCTL register */ 1871 reg = IXGBE_READ_REG(hw, IXGBE_VFSRRCTL(j)); 1872 reg &= ~IXGBE_SRRCTL_BSIZEHDR_MASK; 1873 reg &= ~IXGBE_SRRCTL_BSIZEPKT_MASK; 1874 reg |= bufsz; 1875 reg |= IXGBE_SRRCTL_DESCTYPE_ADV_ONEBUF; 1876 IXGBE_WRITE_REG(hw, IXGBE_VFSRRCTL(j), reg); 1877 1878 /* Capture Rx Tail index */ 1879 rxr->tail = IXGBE_VFRDT(rxr->me); 1880 1881 /* Do the queue enabling last */ 1882 rxdctl |= IXGBE_RXDCTL_ENABLE | IXGBE_RXDCTL_VME; 1883 IXGBE_WRITE_REG(hw, IXGBE_VFRXDCTL(j), rxdctl); 1884 for (int k = 0; k < 10; k++) { 1885 if (IXGBE_READ_REG(hw, IXGBE_VFRXDCTL(j)) & 1886 IXGBE_RXDCTL_ENABLE) 1887 break; 1888 msec_delay(1); 1889 } 1890 wmb(); 1891 1892 /* Set the Tail Pointer */ 1893 #ifdef DEV_NETMAP 1894 /* 1895 * In netmap mode, we must preserve the buffers made 1896 * available to userspace before the if_init() 1897 * (this is true by default on the TX side, because 1898 * init makes all buffers available to userspace). 1899 * 1900 * netmap_reset() and the device specific routines 1901 * (e.g. ixgbe_setup_receive_rings()) map these 1902 * buffers at the end of the NIC ring, so here we 1903 * must set the RDT (tail) register to make sure 1904 * they are not overwritten. 1905 * 1906 * In this driver the NIC ring starts at RDH = 0, 1907 * RDT points to the last slot available for reception (?), 1908 * so RDT = num_rx_desc - 1 means the whole ring is available. 1909 */ 1910 if ((adapter->feat_en & IXGBE_FEATURE_NETMAP) && 1911 (ifp->if_capenable & IFCAP_NETMAP)) { 1912 struct netmap_adapter *na = NA(adapter->ifp); 1913 struct netmap_kring *kring = na->rx_rings[i]; 1914 int t = na->num_rx_desc - 1 - nm_kr_rxspace(kring); 1915 1916 IXGBE_WRITE_REG(hw, IXGBE_VFRDT(rxr->me), t); 1917 } else 1918 #endif /* DEV_NETMAP */ 1919 IXGBE_WRITE_REG(hw, IXGBE_VFRDT(rxr->me), 1920 adapter->num_rx_desc - 1); 1921 } 1922 1923 ixv_initialize_rss_mapping(adapter); 1924 } /* ixv_initialize_receive_units */ 1925 1926 /************************************************************************ 1927 * ixv_sysctl_tdh_handler - Transmit Descriptor Head handler function 1928 * 1929 * Retrieves the TDH value from the hardware 1930 ************************************************************************/ 1931 static int 1932 ixv_sysctl_tdh_handler(SYSCTLFN_ARGS) 1933 { 1934 struct sysctlnode node = *rnode; 1935 struct tx_ring *txr = (struct tx_ring *)node.sysctl_data; 1936 uint32_t val; 1937 1938 if (!txr) 1939 return (0); 1940 1941 val = IXGBE_READ_REG(&txr->adapter->hw, IXGBE_VFTDH(txr->me)); 1942 node.sysctl_data = &val; 1943 return sysctl_lookup(SYSCTLFN_CALL(&node)); 1944 } /* ixv_sysctl_tdh_handler */ 1945 1946 /************************************************************************ 1947 * ixgbe_sysctl_tdt_handler - Transmit Descriptor Tail handler function 1948 * 1949 * Retrieves the TDT value from the hardware 1950 ************************************************************************/ 1951 static int 1952 ixv_sysctl_tdt_handler(SYSCTLFN_ARGS) 1953 { 1954 struct sysctlnode node = *rnode; 1955 struct tx_ring *txr = (struct tx_ring *)node.sysctl_data; 1956 uint32_t val; 1957 1958 if (!txr) 1959 return (0); 1960 1961 val = IXGBE_READ_REG(&txr->adapter->hw, IXGBE_VFTDT(txr->me)); 1962 node.sysctl_data = &val; 1963 return sysctl_lookup(SYSCTLFN_CALL(&node)); 1964 } /* ixv_sysctl_tdt_handler */ 1965 1966 /************************************************************************ 1967 * ixv_sysctl_next_to_check_handler - Receive Descriptor next to check 1968 * handler function 1969 * 1970 * Retrieves the next_to_check value 1971 ************************************************************************/ 1972 static int 1973 ixv_sysctl_next_to_check_handler(SYSCTLFN_ARGS) 1974 { 1975 struct sysctlnode node = *rnode; 1976 struct rx_ring *rxr = (struct rx_ring *)node.sysctl_data; 1977 uint32_t val; 1978 1979 if (!rxr) 1980 return (0); 1981 1982 val = rxr->next_to_check; 1983 node.sysctl_data = &val; 1984 return sysctl_lookup(SYSCTLFN_CALL(&node)); 1985 } /* ixv_sysctl_next_to_check_handler */ 1986 1987 /************************************************************************ 1988 * ixv_sysctl_rdh_handler - Receive Descriptor Head handler function 1989 * 1990 * Retrieves the RDH value from the hardware 1991 ************************************************************************/ 1992 static int 1993 ixv_sysctl_rdh_handler(SYSCTLFN_ARGS) 1994 { 1995 struct sysctlnode node = *rnode; 1996 struct rx_ring *rxr = (struct rx_ring *)node.sysctl_data; 1997 uint32_t val; 1998 1999 if (!rxr) 2000 return (0); 2001 2002 val = IXGBE_READ_REG(&rxr->adapter->hw, IXGBE_VFRDH(rxr->me)); 2003 node.sysctl_data = &val; 2004 return sysctl_lookup(SYSCTLFN_CALL(&node)); 2005 } /* ixv_sysctl_rdh_handler */ 2006 2007 /************************************************************************ 2008 * ixv_sysctl_rdt_handler - Receive Descriptor Tail handler function 2009 * 2010 * Retrieves the RDT value from the hardware 2011 ************************************************************************/ 2012 static int 2013 ixv_sysctl_rdt_handler(SYSCTLFN_ARGS) 2014 { 2015 struct sysctlnode node = *rnode; 2016 struct rx_ring *rxr = (struct rx_ring *)node.sysctl_data; 2017 uint32_t val; 2018 2019 if (!rxr) 2020 return (0); 2021 2022 val = IXGBE_READ_REG(&rxr->adapter->hw, IXGBE_VFRDT(rxr->me)); 2023 node.sysctl_data = &val; 2024 return sysctl_lookup(SYSCTLFN_CALL(&node)); 2025 } /* ixv_sysctl_rdt_handler */ 2026 2027 static void 2028 ixv_setup_vlan_tagging(struct adapter *adapter) 2029 { 2030 struct ethercom *ec = &adapter->osdep.ec; 2031 struct ixgbe_hw *hw = &adapter->hw; 2032 struct rx_ring *rxr; 2033 u32 ctrl; 2034 int i; 2035 bool hwtagging; 2036 2037 /* Enable HW tagging only if any vlan is attached */ 2038 hwtagging = (ec->ec_capenable & ETHERCAP_VLAN_HWTAGGING) 2039 && VLAN_ATTACHED(ec); 2040 2041 /* Enable the queues */ 2042 for (i = 0; i < adapter->num_queues; i++) { 2043 rxr = &adapter->rx_rings[i]; 2044 ctrl = IXGBE_READ_REG(hw, IXGBE_VFRXDCTL(rxr->me)); 2045 if (hwtagging) 2046 ctrl |= IXGBE_RXDCTL_VME; 2047 else 2048 ctrl &= ~IXGBE_RXDCTL_VME; 2049 IXGBE_WRITE_REG(hw, IXGBE_VFRXDCTL(rxr->me), ctrl); 2050 /* 2051 * Let Rx path know that it needs to store VLAN tag 2052 * as part of extra mbuf info. 2053 */ 2054 rxr->vtag_strip = hwtagging ? TRUE : FALSE; 2055 } 2056 } /* ixv_setup_vlan_tagging */ 2057 2058 /************************************************************************ 2059 * ixv_setup_vlan_support 2060 ************************************************************************/ 2061 static int 2062 ixv_setup_vlan_support(struct adapter *adapter) 2063 { 2064 struct ethercom *ec = &adapter->osdep.ec; 2065 struct ixgbe_hw *hw = &adapter->hw; 2066 u32 vid, vfta, retry; 2067 struct vlanid_list *vlanidp; 2068 int rv, error = 0; 2069 2070 /* 2071 * This function is called from both if_init and ifflags_cb() 2072 * on NetBSD. 2073 */ 2074 2075 /* 2076 * Part 1: 2077 * Setup VLAN HW tagging 2078 */ 2079 ixv_setup_vlan_tagging(adapter); 2080 2081 if (!VLAN_ATTACHED(ec)) 2082 return 0; 2083 2084 /* 2085 * Part 2: 2086 * Setup VLAN HW filter 2087 */ 2088 /* Cleanup shadow_vfta */ 2089 for (int i = 0; i < IXGBE_VFTA_SIZE; i++) 2090 adapter->shadow_vfta[i] = 0; 2091 /* Generate shadow_vfta from ec_vids */ 2092 ETHER_LOCK(ec); 2093 SIMPLEQ_FOREACH(vlanidp, &ec->ec_vids, vid_list) { 2094 uint32_t idx; 2095 2096 idx = vlanidp->vid / 32; 2097 KASSERT(idx < IXGBE_VFTA_SIZE); 2098 adapter->shadow_vfta[idx] |= (u32)1 << (vlanidp->vid % 32); 2099 } 2100 ETHER_UNLOCK(ec); 2101 2102 /* 2103 * A soft reset zero's out the VFTA, so 2104 * we need to repopulate it now. 2105 */ 2106 for (int i = 0; i < IXGBE_VFTA_SIZE; i++) { 2107 if (adapter->shadow_vfta[i] == 0) 2108 continue; 2109 vfta = adapter->shadow_vfta[i]; 2110 /* 2111 * Reconstruct the vlan id's 2112 * based on the bits set in each 2113 * of the array ints. 2114 */ 2115 for (int j = 0; j < 32; j++) { 2116 retry = 0; 2117 if ((vfta & ((u32)1 << j)) == 0) 2118 continue; 2119 vid = (i * 32) + j; 2120 2121 /* Call the shared code mailbox routine */ 2122 while ((rv = hw->mac.ops.set_vfta(hw, vid, 0, TRUE, 2123 FALSE)) != 0) { 2124 if (++retry > 5) { 2125 device_printf(adapter->dev, 2126 "%s: max retry exceeded\n", 2127 __func__); 2128 break; 2129 } 2130 } 2131 if (rv != 0) { 2132 device_printf(adapter->dev, 2133 "failed to set vlan %d\n", vid); 2134 error = EACCES; 2135 } 2136 } 2137 } 2138 return error; 2139 } /* ixv_setup_vlan_support */ 2140 2141 static int 2142 ixv_vlan_cb(struct ethercom *ec, uint16_t vid, bool set) 2143 { 2144 struct ifnet *ifp = &ec->ec_if; 2145 struct adapter *adapter = ifp->if_softc; 2146 int rv; 2147 2148 if (set) 2149 rv = ixv_register_vlan(adapter, vid); 2150 else 2151 rv = ixv_unregister_vlan(adapter, vid); 2152 2153 if (rv != 0) 2154 return rv; 2155 2156 /* 2157 * Control VLAN HW tagging when ec_nvlan is changed from 1 to 0 2158 * or 0 to 1. 2159 */ 2160 if ((set && (ec->ec_nvlans == 1)) || (!set && (ec->ec_nvlans == 0))) 2161 ixv_setup_vlan_tagging(adapter); 2162 2163 return rv; 2164 } 2165 2166 /************************************************************************ 2167 * ixv_register_vlan 2168 * 2169 * Run via a vlan config EVENT, it enables us to use the 2170 * HW Filter table since we can get the vlan id. This just 2171 * creates the entry in the soft version of the VFTA, init 2172 * will repopulate the real table. 2173 ************************************************************************/ 2174 static int 2175 ixv_register_vlan(struct adapter *adapter, u16 vtag) 2176 { 2177 struct ixgbe_hw *hw = &adapter->hw; 2178 u16 index, bit; 2179 int error; 2180 2181 if ((vtag == 0) || (vtag > 4095)) /* Invalid */ 2182 return EINVAL; 2183 IXGBE_CORE_LOCK(adapter); 2184 index = (vtag >> 5) & 0x7F; 2185 bit = vtag & 0x1F; 2186 adapter->shadow_vfta[index] |= ((u32)1 << bit); 2187 error = hw->mac.ops.set_vfta(hw, vtag, 0, true, false); 2188 IXGBE_CORE_UNLOCK(adapter); 2189 2190 if (error != 0) { 2191 device_printf(adapter->dev, "failed to register vlan %hu\n", 2192 vtag); 2193 error = EACCES; 2194 } 2195 return error; 2196 } /* ixv_register_vlan */ 2197 2198 /************************************************************************ 2199 * ixv_unregister_vlan 2200 * 2201 * Run via a vlan unconfig EVENT, remove our entry 2202 * in the soft vfta. 2203 ************************************************************************/ 2204 static int 2205 ixv_unregister_vlan(struct adapter *adapter, u16 vtag) 2206 { 2207 struct ixgbe_hw *hw = &adapter->hw; 2208 u16 index, bit; 2209 int error; 2210 2211 if ((vtag == 0) || (vtag > 4095)) /* Invalid */ 2212 return EINVAL; 2213 2214 IXGBE_CORE_LOCK(adapter); 2215 index = (vtag >> 5) & 0x7F; 2216 bit = vtag & 0x1F; 2217 adapter->shadow_vfta[index] &= ~((u32)1 << bit); 2218 error = hw->mac.ops.set_vfta(hw, vtag, 0, false, false); 2219 IXGBE_CORE_UNLOCK(adapter); 2220 2221 if (error != 0) { 2222 device_printf(adapter->dev, "failed to unregister vlan %hu\n", 2223 vtag); 2224 error = EIO; 2225 } 2226 return error; 2227 } /* ixv_unregister_vlan */ 2228 2229 /************************************************************************ 2230 * ixv_enable_intr 2231 ************************************************************************/ 2232 static void 2233 ixv_enable_intr(struct adapter *adapter) 2234 { 2235 struct ixgbe_hw *hw = &adapter->hw; 2236 struct ix_queue *que = adapter->queues; 2237 u32 mask; 2238 int i; 2239 2240 /* For VTEIAC */ 2241 mask = (1 << adapter->vector); 2242 for (i = 0; i < adapter->num_queues; i++, que++) 2243 mask |= (1 << que->msix); 2244 IXGBE_WRITE_REG(hw, IXGBE_VTEIAC, mask); 2245 2246 /* For VTEIMS */ 2247 IXGBE_WRITE_REG(hw, IXGBE_VTEIMS, (1 << adapter->vector)); 2248 que = adapter->queues; 2249 for (i = 0; i < adapter->num_queues; i++, que++) 2250 ixv_enable_queue(adapter, que->msix); 2251 2252 IXGBE_WRITE_FLUSH(hw); 2253 } /* ixv_enable_intr */ 2254 2255 /************************************************************************ 2256 * ixv_disable_intr 2257 ************************************************************************/ 2258 static void 2259 ixv_disable_intr(struct adapter *adapter) 2260 { 2261 struct ix_queue *que = adapter->queues; 2262 2263 IXGBE_WRITE_REG(&adapter->hw, IXGBE_VTEIAC, 0); 2264 2265 /* disable interrupts other than queues */ 2266 IXGBE_WRITE_REG(&adapter->hw, IXGBE_VTEIMC, adapter->vector); 2267 2268 for (int i = 0; i < adapter->num_queues; i++, que++) 2269 ixv_disable_queue(adapter, que->msix); 2270 2271 IXGBE_WRITE_FLUSH(&adapter->hw); 2272 } /* ixv_disable_intr */ 2273 2274 /************************************************************************ 2275 * ixv_set_ivar 2276 * 2277 * Setup the correct IVAR register for a particular MSI-X interrupt 2278 * - entry is the register array entry 2279 * - vector is the MSI-X vector for this queue 2280 * - type is RX/TX/MISC 2281 ************************************************************************/ 2282 static void 2283 ixv_set_ivar(struct adapter *adapter, u8 entry, u8 vector, s8 type) 2284 { 2285 struct ixgbe_hw *hw = &adapter->hw; 2286 u32 ivar, index; 2287 2288 vector |= IXGBE_IVAR_ALLOC_VAL; 2289 2290 if (type == -1) { /* MISC IVAR */ 2291 ivar = IXGBE_READ_REG(hw, IXGBE_VTIVAR_MISC); 2292 ivar &= ~0xFF; 2293 ivar |= vector; 2294 IXGBE_WRITE_REG(hw, IXGBE_VTIVAR_MISC, ivar); 2295 } else { /* RX/TX IVARS */ 2296 index = (16 * (entry & 1)) + (8 * type); 2297 ivar = IXGBE_READ_REG(hw, IXGBE_VTIVAR(entry >> 1)); 2298 ivar &= ~(0xffUL << index); 2299 ivar |= ((u32)vector << index); 2300 IXGBE_WRITE_REG(hw, IXGBE_VTIVAR(entry >> 1), ivar); 2301 } 2302 } /* ixv_set_ivar */ 2303 2304 /************************************************************************ 2305 * ixv_configure_ivars 2306 ************************************************************************/ 2307 static void 2308 ixv_configure_ivars(struct adapter *adapter) 2309 { 2310 struct ix_queue *que = adapter->queues; 2311 2312 /* XXX We should sync EITR value calculation with ixgbe.c? */ 2313 2314 for (int i = 0; i < adapter->num_queues; i++, que++) { 2315 /* First the RX queue entry */ 2316 ixv_set_ivar(adapter, i, que->msix, 0); 2317 /* ... and the TX */ 2318 ixv_set_ivar(adapter, i, que->msix, 1); 2319 /* Set an initial value in EITR */ 2320 ixv_eitr_write(adapter, que->msix, IXGBE_EITR_DEFAULT); 2321 } 2322 2323 /* For the mailbox interrupt */ 2324 ixv_set_ivar(adapter, 1, adapter->vector, -1); 2325 } /* ixv_configure_ivars */ 2326 2327 2328 /************************************************************************ 2329 * ixv_save_stats 2330 * 2331 * The VF stats registers never have a truly virgin 2332 * starting point, so this routine tries to make an 2333 * artificial one, marking ground zero on attach as 2334 * it were. 2335 ************************************************************************/ 2336 static void 2337 ixv_save_stats(struct adapter *adapter) 2338 { 2339 struct ixgbevf_hw_stats *stats = &adapter->stats.vf; 2340 2341 if (stats->vfgprc.ev_count || stats->vfgptc.ev_count) { 2342 stats->saved_reset_vfgprc += 2343 stats->vfgprc.ev_count - stats->base_vfgprc; 2344 stats->saved_reset_vfgptc += 2345 stats->vfgptc.ev_count - stats->base_vfgptc; 2346 stats->saved_reset_vfgorc += 2347 stats->vfgorc.ev_count - stats->base_vfgorc; 2348 stats->saved_reset_vfgotc += 2349 stats->vfgotc.ev_count - stats->base_vfgotc; 2350 stats->saved_reset_vfmprc += 2351 stats->vfmprc.ev_count - stats->base_vfmprc; 2352 } 2353 } /* ixv_save_stats */ 2354 2355 /************************************************************************ 2356 * ixv_init_stats 2357 ************************************************************************/ 2358 static void 2359 ixv_init_stats(struct adapter *adapter) 2360 { 2361 struct ixgbe_hw *hw = &adapter->hw; 2362 2363 adapter->stats.vf.last_vfgprc = IXGBE_READ_REG(hw, IXGBE_VFGPRC); 2364 adapter->stats.vf.last_vfgorc = IXGBE_READ_REG(hw, IXGBE_VFGORC_LSB); 2365 adapter->stats.vf.last_vfgorc |= 2366 (((u64)(IXGBE_READ_REG(hw, IXGBE_VFGORC_MSB))) << 32); 2367 2368 adapter->stats.vf.last_vfgptc = IXGBE_READ_REG(hw, IXGBE_VFGPTC); 2369 adapter->stats.vf.last_vfgotc = IXGBE_READ_REG(hw, IXGBE_VFGOTC_LSB); 2370 adapter->stats.vf.last_vfgotc |= 2371 (((u64)(IXGBE_READ_REG(hw, IXGBE_VFGOTC_MSB))) << 32); 2372 2373 adapter->stats.vf.last_vfmprc = IXGBE_READ_REG(hw, IXGBE_VFMPRC); 2374 2375 adapter->stats.vf.base_vfgprc = adapter->stats.vf.last_vfgprc; 2376 adapter->stats.vf.base_vfgorc = adapter->stats.vf.last_vfgorc; 2377 adapter->stats.vf.base_vfgptc = adapter->stats.vf.last_vfgptc; 2378 adapter->stats.vf.base_vfgotc = adapter->stats.vf.last_vfgotc; 2379 adapter->stats.vf.base_vfmprc = adapter->stats.vf.last_vfmprc; 2380 } /* ixv_init_stats */ 2381 2382 #define UPDATE_STAT_32(reg, last, count) \ 2383 { \ 2384 u32 current = IXGBE_READ_REG(hw, (reg)); \ 2385 if (current < (last)) \ 2386 count.ev_count += 0x100000000LL; \ 2387 (last) = current; \ 2388 count.ev_count &= 0xFFFFFFFF00000000LL; \ 2389 count.ev_count |= current; \ 2390 } 2391 2392 #define UPDATE_STAT_36(lsb, msb, last, count) \ 2393 { \ 2394 u64 cur_lsb = IXGBE_READ_REG(hw, (lsb)); \ 2395 u64 cur_msb = IXGBE_READ_REG(hw, (msb)); \ 2396 u64 current = ((cur_msb << 32) | cur_lsb); \ 2397 if (current < (last)) \ 2398 count.ev_count += 0x1000000000LL; \ 2399 (last) = current; \ 2400 count.ev_count &= 0xFFFFFFF000000000LL; \ 2401 count.ev_count |= current; \ 2402 } 2403 2404 /************************************************************************ 2405 * ixv_update_stats - Update the board statistics counters. 2406 ************************************************************************/ 2407 void 2408 ixv_update_stats(struct adapter *adapter) 2409 { 2410 struct ixgbe_hw *hw = &adapter->hw; 2411 struct ixgbevf_hw_stats *stats = &adapter->stats.vf; 2412 2413 UPDATE_STAT_32(IXGBE_VFGPRC, stats->last_vfgprc, stats->vfgprc); 2414 UPDATE_STAT_32(IXGBE_VFGPTC, stats->last_vfgptc, stats->vfgptc); 2415 UPDATE_STAT_36(IXGBE_VFGORC_LSB, IXGBE_VFGORC_MSB, stats->last_vfgorc, 2416 stats->vfgorc); 2417 UPDATE_STAT_36(IXGBE_VFGOTC_LSB, IXGBE_VFGOTC_MSB, stats->last_vfgotc, 2418 stats->vfgotc); 2419 UPDATE_STAT_32(IXGBE_VFMPRC, stats->last_vfmprc, stats->vfmprc); 2420 2421 /* Fill out the OS statistics structure */ 2422 /* 2423 * NetBSD: Don't override if_{i|o}{packets|bytes|mcasts} with 2424 * adapter->stats counters. It's required to make ifconfig -z 2425 * (SOICZIFDATA) work. 2426 */ 2427 } /* ixv_update_stats */ 2428 2429 /************************************************************************ 2430 * ixv_sysctl_interrupt_rate_handler 2431 ************************************************************************/ 2432 static int 2433 ixv_sysctl_interrupt_rate_handler(SYSCTLFN_ARGS) 2434 { 2435 struct sysctlnode node = *rnode; 2436 struct ix_queue *que = (struct ix_queue *)node.sysctl_data; 2437 struct adapter *adapter = que->adapter; 2438 uint32_t reg, usec, rate; 2439 int error; 2440 2441 if (que == NULL) 2442 return 0; 2443 reg = IXGBE_READ_REG(&que->adapter->hw, IXGBE_VTEITR(que->msix)); 2444 usec = ((reg & 0x0FF8) >> 3); 2445 if (usec > 0) 2446 rate = 500000 / usec; 2447 else 2448 rate = 0; 2449 node.sysctl_data = &rate; 2450 error = sysctl_lookup(SYSCTLFN_CALL(&node)); 2451 if (error || newp == NULL) 2452 return error; 2453 reg &= ~0xfff; /* default, no limitation */ 2454 if (rate > 0 && rate < 500000) { 2455 if (rate < 1000) 2456 rate = 1000; 2457 reg |= ((4000000/rate) & 0xff8); 2458 /* 2459 * When RSC is used, ITR interval must be larger than 2460 * RSC_DELAY. Currently, we use 2us for RSC_DELAY. 2461 * The minimum value is always greater than 2us on 100M 2462 * (and 10M?(not documented)), but it's not on 1G and higher. 2463 */ 2464 if ((adapter->link_speed != IXGBE_LINK_SPEED_100_FULL) 2465 && (adapter->link_speed != IXGBE_LINK_SPEED_10_FULL)) { 2466 if ((adapter->num_queues > 1) 2467 && (reg < IXGBE_MIN_RSC_EITR_10G1G)) 2468 return EINVAL; 2469 } 2470 ixv_max_interrupt_rate = rate; 2471 } else 2472 ixv_max_interrupt_rate = 0; 2473 ixv_eitr_write(adapter, que->msix, reg); 2474 2475 return (0); 2476 } /* ixv_sysctl_interrupt_rate_handler */ 2477 2478 const struct sysctlnode * 2479 ixv_sysctl_instance(struct adapter *adapter) 2480 { 2481 const char *dvname; 2482 struct sysctllog **log; 2483 int rc; 2484 const struct sysctlnode *rnode; 2485 2486 log = &adapter->sysctllog; 2487 dvname = device_xname(adapter->dev); 2488 2489 if ((rc = sysctl_createv(log, 0, NULL, &rnode, 2490 0, CTLTYPE_NODE, dvname, 2491 SYSCTL_DESCR("ixv information and settings"), 2492 NULL, 0, NULL, 0, CTL_HW, CTL_CREATE, CTL_EOL)) != 0) 2493 goto err; 2494 2495 return rnode; 2496 err: 2497 device_printf(adapter->dev, 2498 "%s: sysctl_createv failed, rc = %d\n", __func__, rc); 2499 return NULL; 2500 } 2501 2502 static void 2503 ixv_add_device_sysctls(struct adapter *adapter) 2504 { 2505 struct sysctllog **log; 2506 const struct sysctlnode *rnode, *cnode; 2507 device_t dev; 2508 2509 dev = adapter->dev; 2510 log = &adapter->sysctllog; 2511 2512 if ((rnode = ixv_sysctl_instance(adapter)) == NULL) { 2513 aprint_error_dev(dev, "could not create sysctl root\n"); 2514 return; 2515 } 2516 2517 if (sysctl_createv(log, 0, &rnode, &cnode, 2518 CTLFLAG_READWRITE, CTLTYPE_INT, 2519 "debug", SYSCTL_DESCR("Debug Info"), 2520 ixv_sysctl_debug, 0, (void *)adapter, 0, CTL_CREATE, CTL_EOL) != 0) 2521 aprint_error_dev(dev, "could not create sysctl\n"); 2522 2523 if (sysctl_createv(log, 0, &rnode, &cnode, 2524 CTLFLAG_READWRITE, CTLTYPE_BOOL, 2525 "enable_aim", SYSCTL_DESCR("Interrupt Moderation"), 2526 NULL, 0, &adapter->enable_aim, 0, CTL_CREATE, CTL_EOL) != 0) 2527 aprint_error_dev(dev, "could not create sysctl\n"); 2528 2529 if (sysctl_createv(log, 0, &rnode, &cnode, 2530 CTLFLAG_READWRITE, CTLTYPE_BOOL, 2531 "txrx_workqueue", SYSCTL_DESCR("Use workqueue for packet processing"), 2532 NULL, 0, &adapter->txrx_use_workqueue, 0, CTL_CREATE, CTL_EOL) != 0) 2533 aprint_error_dev(dev, "could not create sysctl\n"); 2534 } 2535 2536 /************************************************************************ 2537 * ixv_add_stats_sysctls - Add statistic sysctls for the VF. 2538 ************************************************************************/ 2539 static void 2540 ixv_add_stats_sysctls(struct adapter *adapter) 2541 { 2542 device_t dev = adapter->dev; 2543 struct tx_ring *txr = adapter->tx_rings; 2544 struct rx_ring *rxr = adapter->rx_rings; 2545 struct ixgbevf_hw_stats *stats = &adapter->stats.vf; 2546 struct ixgbe_hw *hw = &adapter->hw; 2547 const struct sysctlnode *rnode, *cnode; 2548 struct sysctllog **log = &adapter->sysctllog; 2549 const char *xname = device_xname(dev); 2550 2551 /* Driver Statistics */ 2552 evcnt_attach_dynamic(&adapter->efbig_tx_dma_setup, EVCNT_TYPE_MISC, 2553 NULL, xname, "Driver tx dma soft fail EFBIG"); 2554 evcnt_attach_dynamic(&adapter->mbuf_defrag_failed, EVCNT_TYPE_MISC, 2555 NULL, xname, "m_defrag() failed"); 2556 evcnt_attach_dynamic(&adapter->efbig2_tx_dma_setup, EVCNT_TYPE_MISC, 2557 NULL, xname, "Driver tx dma hard fail EFBIG"); 2558 evcnt_attach_dynamic(&adapter->einval_tx_dma_setup, EVCNT_TYPE_MISC, 2559 NULL, xname, "Driver tx dma hard fail EINVAL"); 2560 evcnt_attach_dynamic(&adapter->other_tx_dma_setup, EVCNT_TYPE_MISC, 2561 NULL, xname, "Driver tx dma hard fail other"); 2562 evcnt_attach_dynamic(&adapter->eagain_tx_dma_setup, EVCNT_TYPE_MISC, 2563 NULL, xname, "Driver tx dma soft fail EAGAIN"); 2564 evcnt_attach_dynamic(&adapter->enomem_tx_dma_setup, EVCNT_TYPE_MISC, 2565 NULL, xname, "Driver tx dma soft fail ENOMEM"); 2566 evcnt_attach_dynamic(&adapter->watchdog_events, EVCNT_TYPE_MISC, 2567 NULL, xname, "Watchdog timeouts"); 2568 evcnt_attach_dynamic(&adapter->tso_err, EVCNT_TYPE_MISC, 2569 NULL, xname, "TSO errors"); 2570 evcnt_attach_dynamic(&adapter->link_irq, EVCNT_TYPE_INTR, 2571 NULL, xname, "Link MSI-X IRQ Handled"); 2572 2573 for (int i = 0; i < adapter->num_queues; i++, rxr++, txr++) { 2574 snprintf(adapter->queues[i].evnamebuf, 2575 sizeof(adapter->queues[i].evnamebuf), "%s q%d", 2576 xname, i); 2577 snprintf(adapter->queues[i].namebuf, 2578 sizeof(adapter->queues[i].namebuf), "q%d", i); 2579 2580 if ((rnode = ixv_sysctl_instance(adapter)) == NULL) { 2581 aprint_error_dev(dev, "could not create sysctl root\n"); 2582 break; 2583 } 2584 2585 if (sysctl_createv(log, 0, &rnode, &rnode, 2586 0, CTLTYPE_NODE, 2587 adapter->queues[i].namebuf, SYSCTL_DESCR("Queue Name"), 2588 NULL, 0, NULL, 0, CTL_CREATE, CTL_EOL) != 0) 2589 break; 2590 2591 if (sysctl_createv(log, 0, &rnode, &cnode, 2592 CTLFLAG_READWRITE, CTLTYPE_INT, 2593 "interrupt_rate", SYSCTL_DESCR("Interrupt Rate"), 2594 ixv_sysctl_interrupt_rate_handler, 0, 2595 (void *)&adapter->queues[i], 0, CTL_CREATE, CTL_EOL) != 0) 2596 break; 2597 2598 if (sysctl_createv(log, 0, &rnode, &cnode, 2599 CTLFLAG_READONLY, CTLTYPE_INT, 2600 "txd_head", SYSCTL_DESCR("Transmit Descriptor Head"), 2601 ixv_sysctl_tdh_handler, 0, (void *)txr, 2602 0, CTL_CREATE, CTL_EOL) != 0) 2603 break; 2604 2605 if (sysctl_createv(log, 0, &rnode, &cnode, 2606 CTLFLAG_READONLY, CTLTYPE_INT, 2607 "txd_tail", SYSCTL_DESCR("Transmit Descriptor Tail"), 2608 ixv_sysctl_tdt_handler, 0, (void *)txr, 2609 0, CTL_CREATE, CTL_EOL) != 0) 2610 break; 2611 2612 evcnt_attach_dynamic(&adapter->queues[i].irqs, EVCNT_TYPE_INTR, 2613 NULL, adapter->queues[i].evnamebuf, "IRQs on queue"); 2614 evcnt_attach_dynamic(&adapter->queues[i].handleq, 2615 EVCNT_TYPE_MISC, NULL, adapter->queues[i].evnamebuf, 2616 "Handled queue in softint"); 2617 evcnt_attach_dynamic(&adapter->queues[i].req, EVCNT_TYPE_MISC, 2618 NULL, adapter->queues[i].evnamebuf, "Requeued in softint"); 2619 evcnt_attach_dynamic(&txr->tso_tx, EVCNT_TYPE_MISC, 2620 NULL, adapter->queues[i].evnamebuf, "TSO"); 2621 evcnt_attach_dynamic(&txr->no_desc_avail, EVCNT_TYPE_MISC, 2622 NULL, adapter->queues[i].evnamebuf, 2623 "Queue No Descriptor Available"); 2624 evcnt_attach_dynamic(&txr->total_packets, EVCNT_TYPE_MISC, 2625 NULL, adapter->queues[i].evnamebuf, 2626 "Queue Packets Transmitted"); 2627 #ifndef IXGBE_LEGACY_TX 2628 evcnt_attach_dynamic(&txr->pcq_drops, EVCNT_TYPE_MISC, 2629 NULL, adapter->queues[i].evnamebuf, 2630 "Packets dropped in pcq"); 2631 #endif 2632 2633 #ifdef LRO 2634 struct lro_ctrl *lro = &rxr->lro; 2635 #endif /* LRO */ 2636 2637 if (sysctl_createv(log, 0, &rnode, &cnode, 2638 CTLFLAG_READONLY, 2639 CTLTYPE_INT, 2640 "rxd_nxck", SYSCTL_DESCR("Receive Descriptor next to check"), 2641 ixv_sysctl_next_to_check_handler, 0, (void *)rxr, 0, 2642 CTL_CREATE, CTL_EOL) != 0) 2643 break; 2644 2645 if (sysctl_createv(log, 0, &rnode, &cnode, 2646 CTLFLAG_READONLY, 2647 CTLTYPE_INT, 2648 "rxd_head", SYSCTL_DESCR("Receive Descriptor Head"), 2649 ixv_sysctl_rdh_handler, 0, (void *)rxr, 0, 2650 CTL_CREATE, CTL_EOL) != 0) 2651 break; 2652 2653 if (sysctl_createv(log, 0, &rnode, &cnode, 2654 CTLFLAG_READONLY, 2655 CTLTYPE_INT, 2656 "rxd_tail", SYSCTL_DESCR("Receive Descriptor Tail"), 2657 ixv_sysctl_rdt_handler, 0, (void *)rxr, 0, 2658 CTL_CREATE, CTL_EOL) != 0) 2659 break; 2660 2661 evcnt_attach_dynamic(&rxr->rx_packets, EVCNT_TYPE_MISC, 2662 NULL, adapter->queues[i].evnamebuf, "Queue Packets Received"); 2663 evcnt_attach_dynamic(&rxr->rx_bytes, EVCNT_TYPE_MISC, 2664 NULL, adapter->queues[i].evnamebuf, "Queue Bytes Received"); 2665 evcnt_attach_dynamic(&rxr->rx_copies, EVCNT_TYPE_MISC, 2666 NULL, adapter->queues[i].evnamebuf, "Copied RX Frames"); 2667 evcnt_attach_dynamic(&rxr->no_jmbuf, EVCNT_TYPE_MISC, 2668 NULL, adapter->queues[i].evnamebuf, "Rx no jumbo mbuf"); 2669 evcnt_attach_dynamic(&rxr->rx_discarded, EVCNT_TYPE_MISC, 2670 NULL, adapter->queues[i].evnamebuf, "Rx discarded"); 2671 #ifdef LRO 2672 SYSCTL_ADD_INT(ctx, queue_list, OID_AUTO, "lro_queued", 2673 CTLFLAG_RD, &lro->lro_queued, 0, 2674 "LRO Queued"); 2675 SYSCTL_ADD_INT(ctx, queue_list, OID_AUTO, "lro_flushed", 2676 CTLFLAG_RD, &lro->lro_flushed, 0, 2677 "LRO Flushed"); 2678 #endif /* LRO */ 2679 } 2680 2681 /* MAC stats get their own sub node */ 2682 2683 snprintf(stats->namebuf, 2684 sizeof(stats->namebuf), "%s MAC Statistics", xname); 2685 2686 evcnt_attach_dynamic(&stats->ipcs, EVCNT_TYPE_MISC, NULL, 2687 stats->namebuf, "rx csum offload - IP"); 2688 evcnt_attach_dynamic(&stats->l4cs, EVCNT_TYPE_MISC, NULL, 2689 stats->namebuf, "rx csum offload - L4"); 2690 evcnt_attach_dynamic(&stats->ipcs_bad, EVCNT_TYPE_MISC, NULL, 2691 stats->namebuf, "rx csum offload - IP bad"); 2692 evcnt_attach_dynamic(&stats->l4cs_bad, EVCNT_TYPE_MISC, NULL, 2693 stats->namebuf, "rx csum offload - L4 bad"); 2694 2695 /* Packet Reception Stats */ 2696 evcnt_attach_dynamic(&stats->vfgprc, EVCNT_TYPE_MISC, NULL, 2697 xname, "Good Packets Received"); 2698 evcnt_attach_dynamic(&stats->vfgorc, EVCNT_TYPE_MISC, NULL, 2699 xname, "Good Octets Received"); 2700 evcnt_attach_dynamic(&stats->vfmprc, EVCNT_TYPE_MISC, NULL, 2701 xname, "Multicast Packets Received"); 2702 evcnt_attach_dynamic(&stats->vfgptc, EVCNT_TYPE_MISC, NULL, 2703 xname, "Good Packets Transmitted"); 2704 evcnt_attach_dynamic(&stats->vfgotc, EVCNT_TYPE_MISC, NULL, 2705 xname, "Good Octets Transmitted"); 2706 2707 /* Mailbox Stats */ 2708 evcnt_attach_dynamic(&hw->mbx.stats.msgs_tx, EVCNT_TYPE_MISC, NULL, 2709 xname, "message TXs"); 2710 evcnt_attach_dynamic(&hw->mbx.stats.msgs_rx, EVCNT_TYPE_MISC, NULL, 2711 xname, "message RXs"); 2712 evcnt_attach_dynamic(&hw->mbx.stats.acks, EVCNT_TYPE_MISC, NULL, 2713 xname, "ACKs"); 2714 evcnt_attach_dynamic(&hw->mbx.stats.reqs, EVCNT_TYPE_MISC, NULL, 2715 xname, "REQs"); 2716 evcnt_attach_dynamic(&hw->mbx.stats.rsts, EVCNT_TYPE_MISC, NULL, 2717 xname, "RSTs"); 2718 2719 } /* ixv_add_stats_sysctls */ 2720 2721 static void 2722 ixv_clear_evcnt(struct adapter *adapter) 2723 { 2724 struct tx_ring *txr = adapter->tx_rings; 2725 struct rx_ring *rxr = adapter->rx_rings; 2726 struct ixgbevf_hw_stats *stats = &adapter->stats.vf; 2727 struct ixgbe_hw *hw = &adapter->hw; 2728 int i; 2729 2730 /* Driver Statistics */ 2731 adapter->efbig_tx_dma_setup.ev_count = 0; 2732 adapter->mbuf_defrag_failed.ev_count = 0; 2733 adapter->efbig2_tx_dma_setup.ev_count = 0; 2734 adapter->einval_tx_dma_setup.ev_count = 0; 2735 adapter->other_tx_dma_setup.ev_count = 0; 2736 adapter->eagain_tx_dma_setup.ev_count = 0; 2737 adapter->enomem_tx_dma_setup.ev_count = 0; 2738 adapter->watchdog_events.ev_count = 0; 2739 adapter->tso_err.ev_count = 0; 2740 adapter->link_irq.ev_count = 0; 2741 2742 for (i = 0; i < adapter->num_queues; i++, rxr++, txr++) { 2743 adapter->queues[i].irqs.ev_count = 0; 2744 adapter->queues[i].handleq.ev_count = 0; 2745 adapter->queues[i].req.ev_count = 0; 2746 txr->tso_tx.ev_count = 0; 2747 txr->no_desc_avail.ev_count = 0; 2748 txr->total_packets.ev_count = 0; 2749 #ifndef IXGBE_LEGACY_TX 2750 txr->pcq_drops.ev_count = 0; 2751 #endif 2752 txr->q_efbig_tx_dma_setup = 0; 2753 txr->q_mbuf_defrag_failed = 0; 2754 txr->q_efbig2_tx_dma_setup = 0; 2755 txr->q_einval_tx_dma_setup = 0; 2756 txr->q_other_tx_dma_setup = 0; 2757 txr->q_eagain_tx_dma_setup = 0; 2758 txr->q_enomem_tx_dma_setup = 0; 2759 txr->q_tso_err = 0; 2760 2761 rxr->rx_packets.ev_count = 0; 2762 rxr->rx_bytes.ev_count = 0; 2763 rxr->rx_copies.ev_count = 0; 2764 rxr->no_jmbuf.ev_count = 0; 2765 rxr->rx_discarded.ev_count = 0; 2766 } 2767 2768 /* MAC stats get their own sub node */ 2769 2770 stats->ipcs.ev_count = 0; 2771 stats->l4cs.ev_count = 0; 2772 stats->ipcs_bad.ev_count = 0; 2773 stats->l4cs_bad.ev_count = 0; 2774 2775 /* Packet Reception Stats */ 2776 stats->vfgprc.ev_count = 0; 2777 stats->vfgorc.ev_count = 0; 2778 stats->vfmprc.ev_count = 0; 2779 stats->vfgptc.ev_count = 0; 2780 stats->vfgotc.ev_count = 0; 2781 2782 /* Mailbox Stats */ 2783 hw->mbx.stats.msgs_tx.ev_count = 0; 2784 hw->mbx.stats.msgs_rx.ev_count = 0; 2785 hw->mbx.stats.acks.ev_count = 0; 2786 hw->mbx.stats.reqs.ev_count = 0; 2787 hw->mbx.stats.rsts.ev_count = 0; 2788 2789 } /* ixv_clear_evcnt */ 2790 2791 /************************************************************************ 2792 * ixv_set_sysctl_value 2793 ************************************************************************/ 2794 static void 2795 ixv_set_sysctl_value(struct adapter *adapter, const char *name, 2796 const char *description, int *limit, int value) 2797 { 2798 device_t dev = adapter->dev; 2799 struct sysctllog **log; 2800 const struct sysctlnode *rnode, *cnode; 2801 2802 log = &adapter->sysctllog; 2803 if ((rnode = ixv_sysctl_instance(adapter)) == NULL) { 2804 aprint_error_dev(dev, "could not create sysctl root\n"); 2805 return; 2806 } 2807 if (sysctl_createv(log, 0, &rnode, &cnode, 2808 CTLFLAG_READWRITE, CTLTYPE_INT, 2809 name, SYSCTL_DESCR(description), 2810 NULL, 0, limit, 0, CTL_CREATE, CTL_EOL) != 0) 2811 aprint_error_dev(dev, "could not create sysctl\n"); 2812 *limit = value; 2813 } /* ixv_set_sysctl_value */ 2814 2815 /************************************************************************ 2816 * ixv_print_debug_info 2817 * 2818 * Called only when em_display_debug_stats is enabled. 2819 * Provides a way to take a look at important statistics 2820 * maintained by the driver and hardware. 2821 ************************************************************************/ 2822 static void 2823 ixv_print_debug_info(struct adapter *adapter) 2824 { 2825 device_t dev = adapter->dev; 2826 struct ix_queue *que = adapter->queues; 2827 struct rx_ring *rxr; 2828 struct tx_ring *txr; 2829 #ifdef LRO 2830 struct lro_ctrl *lro; 2831 #endif /* LRO */ 2832 2833 for (int i = 0; i < adapter->num_queues; i++, que++) { 2834 txr = que->txr; 2835 rxr = que->rxr; 2836 #ifdef LRO 2837 lro = &rxr->lro; 2838 #endif /* LRO */ 2839 device_printf(dev, "QUE(%d) IRQs Handled: %lu\n", 2840 que->msix, (long)que->irqs.ev_count); 2841 device_printf(dev, "RX(%d) Packets Received: %lld\n", 2842 rxr->me, (long long)rxr->rx_packets.ev_count); 2843 device_printf(dev, "RX(%d) Bytes Received: %lu\n", 2844 rxr->me, (long)rxr->rx_bytes.ev_count); 2845 #ifdef LRO 2846 device_printf(dev, "RX(%d) LRO Queued= %ju\n", 2847 rxr->me, (uintmax_t)lro->lro_queued); 2848 device_printf(dev, "RX(%d) LRO Flushed= %ju\n", 2849 rxr->me, (uintmax_t)lro->lro_flushed); 2850 #endif /* LRO */ 2851 device_printf(dev, "TX(%d) Packets Sent: %lu\n", 2852 txr->me, (long)txr->total_packets.ev_count); 2853 device_printf(dev, "TX(%d) NO Desc Avail: %lu\n", 2854 txr->me, (long)txr->no_desc_avail.ev_count); 2855 } 2856 2857 device_printf(dev, "MBX IRQ Handled: %lu\n", 2858 (long)adapter->link_irq.ev_count); 2859 } /* ixv_print_debug_info */ 2860 2861 /************************************************************************ 2862 * ixv_sysctl_debug 2863 ************************************************************************/ 2864 static int 2865 ixv_sysctl_debug(SYSCTLFN_ARGS) 2866 { 2867 struct sysctlnode node = *rnode; 2868 struct adapter *adapter = (struct adapter *)node.sysctl_data; 2869 int error, result; 2870 2871 node.sysctl_data = &result; 2872 error = sysctl_lookup(SYSCTLFN_CALL(&node)); 2873 2874 if (error || newp == NULL) 2875 return error; 2876 2877 if (result == 1) 2878 ixv_print_debug_info(adapter); 2879 2880 return 0; 2881 } /* ixv_sysctl_debug */ 2882 2883 /************************************************************************ 2884 * ixv_init_device_features 2885 ************************************************************************/ 2886 static void 2887 ixv_init_device_features(struct adapter *adapter) 2888 { 2889 adapter->feat_cap = IXGBE_FEATURE_NETMAP 2890 | IXGBE_FEATURE_VF 2891 | IXGBE_FEATURE_RSS 2892 | IXGBE_FEATURE_LEGACY_TX; 2893 2894 /* A tad short on feature flags for VFs, atm. */ 2895 switch (adapter->hw.mac.type) { 2896 case ixgbe_mac_82599_vf: 2897 break; 2898 case ixgbe_mac_X540_vf: 2899 break; 2900 case ixgbe_mac_X550_vf: 2901 case ixgbe_mac_X550EM_x_vf: 2902 case ixgbe_mac_X550EM_a_vf: 2903 adapter->feat_cap |= IXGBE_FEATURE_NEEDS_CTXD; 2904 break; 2905 default: 2906 break; 2907 } 2908 2909 /* Enabled by default... */ 2910 /* Is a virtual function (VF) */ 2911 if (adapter->feat_cap & IXGBE_FEATURE_VF) 2912 adapter->feat_en |= IXGBE_FEATURE_VF; 2913 /* Netmap */ 2914 if (adapter->feat_cap & IXGBE_FEATURE_NETMAP) 2915 adapter->feat_en |= IXGBE_FEATURE_NETMAP; 2916 /* Receive-Side Scaling (RSS) */ 2917 if (adapter->feat_cap & IXGBE_FEATURE_RSS) 2918 adapter->feat_en |= IXGBE_FEATURE_RSS; 2919 /* Needs advanced context descriptor regardless of offloads req'd */ 2920 if (adapter->feat_cap & IXGBE_FEATURE_NEEDS_CTXD) 2921 adapter->feat_en |= IXGBE_FEATURE_NEEDS_CTXD; 2922 2923 /* Enabled via sysctl... */ 2924 /* Legacy (single queue) transmit */ 2925 if ((adapter->feat_cap & IXGBE_FEATURE_LEGACY_TX) && 2926 ixv_enable_legacy_tx) 2927 adapter->feat_en |= IXGBE_FEATURE_LEGACY_TX; 2928 } /* ixv_init_device_features */ 2929 2930 /************************************************************************ 2931 * ixv_shutdown - Shutdown entry point 2932 ************************************************************************/ 2933 #if 0 /* XXX NetBSD ought to register something like this through pmf(9) */ 2934 static int 2935 ixv_shutdown(device_t dev) 2936 { 2937 struct adapter *adapter = device_private(dev); 2938 IXGBE_CORE_LOCK(adapter); 2939 ixv_stop(adapter); 2940 IXGBE_CORE_UNLOCK(adapter); 2941 2942 return (0); 2943 } /* ixv_shutdown */ 2944 #endif 2945 2946 static int 2947 ixv_ifflags_cb(struct ethercom *ec) 2948 { 2949 struct ifnet *ifp = &ec->ec_if; 2950 struct adapter *adapter = ifp->if_softc; 2951 u_short saved_flags; 2952 u_short change; 2953 int rv = 0; 2954 2955 IXGBE_CORE_LOCK(adapter); 2956 2957 saved_flags = adapter->if_flags; 2958 change = ifp->if_flags ^ adapter->if_flags; 2959 if (change != 0) 2960 adapter->if_flags = ifp->if_flags; 2961 2962 if ((change & ~(IFF_CANTCHANGE | IFF_DEBUG)) != 0) { 2963 rv = ENETRESET; 2964 goto out; 2965 } else if ((change & IFF_PROMISC) != 0) { 2966 rv = ixv_set_rxfilter(adapter); 2967 if (rv != 0) { 2968 /* Restore previous */ 2969 adapter->if_flags = saved_flags; 2970 goto out; 2971 } 2972 } 2973 2974 /* Check for ec_capenable. */ 2975 change = ec->ec_capenable ^ adapter->ec_capenable; 2976 adapter->ec_capenable = ec->ec_capenable; 2977 if ((change & ~(ETHERCAP_VLAN_MTU | ETHERCAP_VLAN_HWTAGGING 2978 | ETHERCAP_VLAN_HWFILTER)) != 0) { 2979 rv = ENETRESET; 2980 goto out; 2981 } 2982 2983 /* 2984 * Special handling is not required for ETHERCAP_VLAN_MTU. 2985 * PF's MAXFRS(MHADD) does not include the 4bytes of the VLAN header. 2986 */ 2987 2988 /* Set up VLAN support and filter */ 2989 if ((change & (ETHERCAP_VLAN_HWTAGGING | ETHERCAP_VLAN_HWFILTER)) != 0) 2990 rv = ixv_setup_vlan_support(adapter); 2991 2992 out: 2993 IXGBE_CORE_UNLOCK(adapter); 2994 2995 return rv; 2996 } 2997 2998 2999 /************************************************************************ 3000 * ixv_ioctl - Ioctl entry point 3001 * 3002 * Called when the user wants to configure the interface. 3003 * 3004 * return 0 on success, positive on failure 3005 ************************************************************************/ 3006 static int 3007 ixv_ioctl(struct ifnet *ifp, u_long command, void *data) 3008 { 3009 struct adapter *adapter = ifp->if_softc; 3010 struct ixgbe_hw *hw = &adapter->hw; 3011 struct ifcapreq *ifcr = data; 3012 int error; 3013 int l4csum_en; 3014 const int l4csum = IFCAP_CSUM_TCPv4_Rx | IFCAP_CSUM_UDPv4_Rx | 3015 IFCAP_CSUM_TCPv6_Rx | IFCAP_CSUM_UDPv6_Rx; 3016 3017 switch (command) { 3018 case SIOCSIFFLAGS: 3019 IOCTL_DEBUGOUT("ioctl: SIOCSIFFLAGS (Set Interface Flags)"); 3020 break; 3021 case SIOCADDMULTI: { 3022 struct ether_multi *enm; 3023 struct ether_multistep step; 3024 struct ethercom *ec = &adapter->osdep.ec; 3025 bool overflow = false; 3026 int mcnt = 0; 3027 3028 /* 3029 * Check the number of multicast address. If it exceeds, 3030 * return ENOSPC. 3031 * Update this code when we support API 1.3. 3032 */ 3033 ETHER_LOCK(ec); 3034 ETHER_FIRST_MULTI(step, ec, enm); 3035 while (enm != NULL) { 3036 mcnt++; 3037 3038 /* 3039 * This code is before adding, so one room is required 3040 * at least. 3041 */ 3042 if (mcnt > (IXGBE_MAX_VF_MC - 1)) { 3043 overflow = true; 3044 break; 3045 } 3046 ETHER_NEXT_MULTI(step, enm); 3047 } 3048 ETHER_UNLOCK(ec); 3049 error = 0; 3050 if (overflow && ((ec->ec_flags & ETHER_F_ALLMULTI) == 0)) { 3051 error = hw->mac.ops.update_xcast_mode(hw, 3052 IXGBEVF_XCAST_MODE_ALLMULTI); 3053 if (error == IXGBE_ERR_NOT_TRUSTED) { 3054 device_printf(adapter->dev, 3055 "this interface is not trusted\n"); 3056 error = EPERM; 3057 } else if (error == IXGBE_ERR_FEATURE_NOT_SUPPORTED) { 3058 device_printf(adapter->dev, 3059 "the PF doesn't support allmulti mode\n"); 3060 error = EOPNOTSUPP; 3061 } else if (error) { 3062 device_printf(adapter->dev, 3063 "number of Ethernet multicast addresses " 3064 "exceeds the limit (%d). error = %d\n", 3065 IXGBE_MAX_VF_MC, error); 3066 error = ENOSPC; 3067 } else 3068 ec->ec_flags |= ETHER_F_ALLMULTI; 3069 } 3070 if (error) 3071 return error; 3072 } 3073 /*FALLTHROUGH*/ 3074 case SIOCDELMULTI: 3075 IOCTL_DEBUGOUT("ioctl: SIOC(ADD|DEL)MULTI"); 3076 break; 3077 case SIOCSIFMEDIA: 3078 case SIOCGIFMEDIA: 3079 IOCTL_DEBUGOUT("ioctl: SIOCxIFMEDIA (Get/Set Interface Media)"); 3080 break; 3081 case SIOCSIFCAP: 3082 IOCTL_DEBUGOUT("ioctl: SIOCSIFCAP (Set Capabilities)"); 3083 break; 3084 case SIOCSIFMTU: 3085 IOCTL_DEBUGOUT("ioctl: SIOCSIFMTU (Set Interface MTU)"); 3086 break; 3087 case SIOCZIFDATA: 3088 IOCTL_DEBUGOUT("ioctl: SIOCZIFDATA (Zero counter)"); 3089 ixv_update_stats(adapter); 3090 ixv_clear_evcnt(adapter); 3091 break; 3092 default: 3093 IOCTL_DEBUGOUT1("ioctl: UNKNOWN (0x%X)", (int)command); 3094 break; 3095 } 3096 3097 switch (command) { 3098 case SIOCSIFCAP: 3099 /* Layer-4 Rx checksum offload has to be turned on and 3100 * off as a unit. 3101 */ 3102 l4csum_en = ifcr->ifcr_capenable & l4csum; 3103 if (l4csum_en != l4csum && l4csum_en != 0) 3104 return EINVAL; 3105 /*FALLTHROUGH*/ 3106 case SIOCADDMULTI: 3107 case SIOCDELMULTI: 3108 case SIOCSIFFLAGS: 3109 case SIOCSIFMTU: 3110 default: 3111 if ((error = ether_ioctl(ifp, command, data)) != ENETRESET) 3112 return error; 3113 if ((ifp->if_flags & IFF_RUNNING) == 0) 3114 ; 3115 else if (command == SIOCSIFCAP || command == SIOCSIFMTU) { 3116 IXGBE_CORE_LOCK(adapter); 3117 ixv_init_locked(adapter); 3118 IXGBE_CORE_UNLOCK(adapter); 3119 } else if (command == SIOCADDMULTI || command == SIOCDELMULTI) { 3120 /* 3121 * Multicast list has changed; set the hardware filter 3122 * accordingly. 3123 */ 3124 IXGBE_CORE_LOCK(adapter); 3125 ixv_disable_intr(adapter); 3126 ixv_set_rxfilter(adapter); 3127 ixv_enable_intr(adapter); 3128 IXGBE_CORE_UNLOCK(adapter); 3129 } 3130 return 0; 3131 } 3132 } /* ixv_ioctl */ 3133 3134 /************************************************************************ 3135 * ixv_init 3136 ************************************************************************/ 3137 static int 3138 ixv_init(struct ifnet *ifp) 3139 { 3140 struct adapter *adapter = ifp->if_softc; 3141 3142 IXGBE_CORE_LOCK(adapter); 3143 ixv_init_locked(adapter); 3144 IXGBE_CORE_UNLOCK(adapter); 3145 3146 return 0; 3147 } /* ixv_init */ 3148 3149 /************************************************************************ 3150 * ixv_handle_que 3151 ************************************************************************/ 3152 static void 3153 ixv_handle_que(void *context) 3154 { 3155 struct ix_queue *que = context; 3156 struct adapter *adapter = que->adapter; 3157 struct tx_ring *txr = que->txr; 3158 struct ifnet *ifp = adapter->ifp; 3159 bool more; 3160 3161 que->handleq.ev_count++; 3162 3163 if (ifp->if_flags & IFF_RUNNING) { 3164 more = ixgbe_rxeof(que); 3165 IXGBE_TX_LOCK(txr); 3166 more |= ixgbe_txeof(txr); 3167 if (!(adapter->feat_en & IXGBE_FEATURE_LEGACY_TX)) 3168 if (!ixgbe_mq_ring_empty(ifp, txr->txr_interq)) 3169 ixgbe_mq_start_locked(ifp, txr); 3170 /* Only for queue 0 */ 3171 /* NetBSD still needs this for CBQ */ 3172 if ((&adapter->queues[0] == que) 3173 && (!ixgbe_legacy_ring_empty(ifp, NULL))) 3174 ixgbe_legacy_start_locked(ifp, txr); 3175 IXGBE_TX_UNLOCK(txr); 3176 if (more) { 3177 que->req.ev_count++; 3178 if (adapter->txrx_use_workqueue) { 3179 /* 3180 * "enqueued flag" is not required here 3181 * the same as ixg(4). See ixgbe_msix_que(). 3182 */ 3183 workqueue_enqueue(adapter->que_wq, 3184 &que->wq_cookie, curcpu()); 3185 } else 3186 softint_schedule(que->que_si); 3187 return; 3188 } 3189 } 3190 3191 /* Re-enable this interrupt */ 3192 ixv_enable_queue(adapter, que->msix); 3193 3194 return; 3195 } /* ixv_handle_que */ 3196 3197 /************************************************************************ 3198 * ixv_handle_que_work 3199 ************************************************************************/ 3200 static void 3201 ixv_handle_que_work(struct work *wk, void *context) 3202 { 3203 struct ix_queue *que = container_of(wk, struct ix_queue, wq_cookie); 3204 3205 /* 3206 * "enqueued flag" is not required here the same as ixg(4). 3207 * See ixgbe_msix_que(). 3208 */ 3209 ixv_handle_que(que); 3210 } 3211 3212 /************************************************************************ 3213 * ixv_allocate_msix - Setup MSI-X Interrupt resources and handlers 3214 ************************************************************************/ 3215 static int 3216 ixv_allocate_msix(struct adapter *adapter, const struct pci_attach_args *pa) 3217 { 3218 device_t dev = adapter->dev; 3219 struct ix_queue *que = adapter->queues; 3220 struct tx_ring *txr = adapter->tx_rings; 3221 int error, msix_ctrl, rid, vector = 0; 3222 pci_chipset_tag_t pc; 3223 pcitag_t tag; 3224 char intrbuf[PCI_INTRSTR_LEN]; 3225 char wqname[MAXCOMLEN]; 3226 char intr_xname[32]; 3227 const char *intrstr = NULL; 3228 kcpuset_t *affinity; 3229 int cpu_id = 0; 3230 3231 pc = adapter->osdep.pc; 3232 tag = adapter->osdep.tag; 3233 3234 adapter->osdep.nintrs = adapter->num_queues + 1; 3235 if (pci_msix_alloc_exact(pa, &adapter->osdep.intrs, 3236 adapter->osdep.nintrs) != 0) { 3237 aprint_error_dev(dev, 3238 "failed to allocate MSI-X interrupt\n"); 3239 return (ENXIO); 3240 } 3241 3242 kcpuset_create(&affinity, false); 3243 for (int i = 0; i < adapter->num_queues; i++, vector++, que++, txr++) { 3244 snprintf(intr_xname, sizeof(intr_xname), "%s TXRX%d", 3245 device_xname(dev), i); 3246 intrstr = pci_intr_string(pc, adapter->osdep.intrs[i], intrbuf, 3247 sizeof(intrbuf)); 3248 #ifdef IXGBE_MPSAFE 3249 pci_intr_setattr(pc, &adapter->osdep.intrs[i], PCI_INTR_MPSAFE, 3250 true); 3251 #endif 3252 /* Set the handler function */ 3253 que->res = adapter->osdep.ihs[i] = pci_intr_establish_xname(pc, 3254 adapter->osdep.intrs[i], IPL_NET, ixv_msix_que, que, 3255 intr_xname); 3256 if (que->res == NULL) { 3257 pci_intr_release(pc, adapter->osdep.intrs, 3258 adapter->osdep.nintrs); 3259 aprint_error_dev(dev, 3260 "Failed to register QUE handler\n"); 3261 kcpuset_destroy(affinity); 3262 return (ENXIO); 3263 } 3264 que->msix = vector; 3265 adapter->active_queues |= (u64)(1 << que->msix); 3266 3267 cpu_id = i; 3268 /* Round-robin affinity */ 3269 kcpuset_zero(affinity); 3270 kcpuset_set(affinity, cpu_id % ncpu); 3271 error = interrupt_distribute(adapter->osdep.ihs[i], affinity, 3272 NULL); 3273 aprint_normal_dev(dev, "for TX/RX, interrupting at %s", 3274 intrstr); 3275 if (error == 0) 3276 aprint_normal(", bound queue %d to cpu %d\n", 3277 i, cpu_id % ncpu); 3278 else 3279 aprint_normal("\n"); 3280 3281 #ifndef IXGBE_LEGACY_TX 3282 txr->txr_si 3283 = softint_establish(SOFTINT_NET | IXGBE_SOFTINFT_FLAGS, 3284 ixgbe_deferred_mq_start, txr); 3285 #endif 3286 que->que_si 3287 = softint_establish(SOFTINT_NET | IXGBE_SOFTINFT_FLAGS, 3288 ixv_handle_que, que); 3289 if (que->que_si == NULL) { 3290 aprint_error_dev(dev, 3291 "could not establish software interrupt\n"); 3292 } 3293 } 3294 snprintf(wqname, sizeof(wqname), "%sdeferTx", device_xname(dev)); 3295 error = workqueue_create(&adapter->txr_wq, wqname, 3296 ixgbe_deferred_mq_start_work, adapter, IXGBE_WORKQUEUE_PRI, IPL_NET, 3297 IXGBE_WORKQUEUE_FLAGS); 3298 if (error) { 3299 aprint_error_dev(dev, "couldn't create workqueue for deferred Tx\n"); 3300 } 3301 adapter->txr_wq_enqueued = percpu_alloc(sizeof(u_int)); 3302 3303 snprintf(wqname, sizeof(wqname), "%sTxRx", device_xname(dev)); 3304 error = workqueue_create(&adapter->que_wq, wqname, 3305 ixv_handle_que_work, adapter, IXGBE_WORKQUEUE_PRI, IPL_NET, 3306 IXGBE_WORKQUEUE_FLAGS); 3307 if (error) { 3308 aprint_error_dev(dev, 3309 "couldn't create workqueue\n"); 3310 } 3311 3312 /* and Mailbox */ 3313 cpu_id++; 3314 snprintf(intr_xname, sizeof(intr_xname), "%s link", device_xname(dev)); 3315 adapter->vector = vector; 3316 intrstr = pci_intr_string(pc, adapter->osdep.intrs[vector], intrbuf, 3317 sizeof(intrbuf)); 3318 #ifdef IXGBE_MPSAFE 3319 pci_intr_setattr(pc, &adapter->osdep.intrs[vector], PCI_INTR_MPSAFE, 3320 true); 3321 #endif 3322 /* Set the mbx handler function */ 3323 adapter->osdep.ihs[vector] = pci_intr_establish_xname(pc, 3324 adapter->osdep.intrs[vector], IPL_NET, ixv_msix_mbx, adapter, 3325 intr_xname); 3326 if (adapter->osdep.ihs[vector] == NULL) { 3327 aprint_error_dev(dev, "Failed to register LINK handler\n"); 3328 kcpuset_destroy(affinity); 3329 return (ENXIO); 3330 } 3331 /* Round-robin affinity */ 3332 kcpuset_zero(affinity); 3333 kcpuset_set(affinity, cpu_id % ncpu); 3334 error = interrupt_distribute(adapter->osdep.ihs[vector], 3335 affinity, NULL); 3336 3337 aprint_normal_dev(dev, 3338 "for link, interrupting at %s", intrstr); 3339 if (error == 0) 3340 aprint_normal(", affinity to cpu %d\n", cpu_id % ncpu); 3341 else 3342 aprint_normal("\n"); 3343 3344 /* Tasklets for Mailbox */ 3345 adapter->link_si = softint_establish(SOFTINT_NET |IXGBE_SOFTINFT_FLAGS, 3346 ixv_handle_link, adapter); 3347 /* 3348 * Due to a broken design QEMU will fail to properly 3349 * enable the guest for MSI-X unless the vectors in 3350 * the table are all set up, so we must rewrite the 3351 * ENABLE in the MSI-X control register again at this 3352 * point to cause it to successfully initialize us. 3353 */ 3354 if (adapter->hw.mac.type == ixgbe_mac_82599_vf) { 3355 pci_get_capability(pc, tag, PCI_CAP_MSIX, &rid, NULL); 3356 rid += PCI_MSIX_CTL; 3357 msix_ctrl = pci_conf_read(pc, tag, rid); 3358 msix_ctrl |= PCI_MSIX_CTL_ENABLE; 3359 pci_conf_write(pc, tag, rid, msix_ctrl); 3360 } 3361 3362 kcpuset_destroy(affinity); 3363 return (0); 3364 } /* ixv_allocate_msix */ 3365 3366 /************************************************************************ 3367 * ixv_configure_interrupts - Setup MSI-X resources 3368 * 3369 * Note: The VF device MUST use MSI-X, there is no fallback. 3370 ************************************************************************/ 3371 static int 3372 ixv_configure_interrupts(struct adapter *adapter) 3373 { 3374 device_t dev = adapter->dev; 3375 int want, queues, msgs; 3376 3377 /* Must have at least 2 MSI-X vectors */ 3378 msgs = pci_msix_count(adapter->osdep.pc, adapter->osdep.tag); 3379 if (msgs < 2) { 3380 aprint_error_dev(dev, "MSIX config error\n"); 3381 return (ENXIO); 3382 } 3383 msgs = MIN(msgs, IXG_MAX_NINTR); 3384 3385 /* Figure out a reasonable auto config value */ 3386 queues = (ncpu > (msgs - 1)) ? (msgs - 1) : ncpu; 3387 3388 if (ixv_num_queues != 0) 3389 queues = ixv_num_queues; 3390 else if ((ixv_num_queues == 0) && (queues > IXGBE_VF_MAX_TX_QUEUES)) 3391 queues = IXGBE_VF_MAX_TX_QUEUES; 3392 3393 /* 3394 * Want vectors for the queues, 3395 * plus an additional for mailbox. 3396 */ 3397 want = queues + 1; 3398 if (msgs >= want) 3399 msgs = want; 3400 else { 3401 aprint_error_dev(dev, 3402 "MSI-X Configuration Problem, " 3403 "%d vectors but %d queues wanted!\n", 3404 msgs, want); 3405 return -1; 3406 } 3407 3408 adapter->msix_mem = (void *)1; /* XXX */ 3409 aprint_normal_dev(dev, 3410 "Using MSI-X interrupts with %d vectors\n", msgs); 3411 adapter->num_queues = queues; 3412 3413 return (0); 3414 } /* ixv_configure_interrupts */ 3415 3416 3417 /************************************************************************ 3418 * ixv_handle_link - Tasklet handler for MSI-X MBX interrupts 3419 * 3420 * Done outside of interrupt context since the driver might sleep 3421 ************************************************************************/ 3422 static void 3423 ixv_handle_link(void *context) 3424 { 3425 struct adapter *adapter = context; 3426 3427 IXGBE_CORE_LOCK(adapter); 3428 3429 adapter->hw.mac.ops.check_link(&adapter->hw, &adapter->link_speed, 3430 &adapter->link_up, FALSE); 3431 ixv_update_link_status(adapter); 3432 3433 IXGBE_CORE_UNLOCK(adapter); 3434 } /* ixv_handle_link */ 3435 3436 /************************************************************************ 3437 * ixv_check_link - Used in the local timer to poll for link changes 3438 ************************************************************************/ 3439 static s32 3440 ixv_check_link(struct adapter *adapter) 3441 { 3442 s32 error; 3443 3444 KASSERT(mutex_owned(&adapter->core_mtx)); 3445 3446 adapter->hw.mac.get_link_status = TRUE; 3447 3448 error = adapter->hw.mac.ops.check_link(&adapter->hw, 3449 &adapter->link_speed, &adapter->link_up, FALSE); 3450 ixv_update_link_status(adapter); 3451 3452 return error; 3453 } /* ixv_check_link */ 3454
Indexes created Tue Sep 30 20:09:53 GMT 2025