ixv.c revision 1.168
1 /* $NetBSD: ixv.c,v 1.168 2021/09/16 09:55:28 msaitoh Exp $ */ 2 3 /****************************************************************************** 4 5 Copyright (c) 2001-2017, Intel Corporation 6 All rights reserved. 7 8 Redistribution and use in source and binary forms, with or without 9 modification, are permitted provided that the following conditions are met: 10 11 1. Redistributions of source code must retain the above copyright notice, 12 this list of conditions and the following disclaimer. 13 14 2. Redistributions in binary form must reproduce the above copyright 15 notice, this list of conditions and the following disclaimer in the 16 documentation and/or other materials provided with the distribution. 17 18 3. Neither the name of the Intel Corporation nor the names of its 19 contributors may be used to endorse or promote products derived from 20 this software without specific prior written permission. 21 22 THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 23 AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 24 IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 25 ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE 26 LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 27 CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 28 SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 29 INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 30 CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 31 ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 32 POSSIBILITY OF SUCH DAMAGE. 33 34 ******************************************************************************/ 35 /*$FreeBSD: head/sys/dev/ixgbe/if_ixv.c 331224 2018-03-19 20:55:05Z erj $*/ 36 37 #include <sys/cdefs.h> 38 __KERNEL_RCSID(0, "$NetBSD: ixv.c,v 1.168 2021/09/16 09:55:28 msaitoh Exp $"); 39 40 #ifdef _KERNEL_OPT 41 #include "opt_inet.h" 42 #include "opt_inet6.h" 43 #include "opt_net_mpsafe.h" 44 #endif 45 46 #include "ixgbe.h" 47 #include "vlan.h" 48 49 /************************************************************************ 50 * Driver version 51 ************************************************************************/ 52 static const char ixv_driver_version[] = "2.0.1-k"; 53 /* XXX NetBSD: + 1.5.17 */ 54 55 /************************************************************************ 56 * PCI Device ID Table 57 * 58 * Used by probe to select devices to load on 59 * Last field stores an index into ixv_strings 60 * Last entry must be all 0s 61 * 62 * { Vendor ID, Device ID, SubVendor ID, SubDevice ID, String Index } 63 ************************************************************************/ 64 static const ixgbe_vendor_info_t ixv_vendor_info_array[] = 65 { 66 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_VF, 0, 0, 0}, 67 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_X540_VF, 0, 0, 0}, 68 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_X550_VF, 0, 0, 0}, 69 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_X550EM_X_VF, 0, 0, 0}, 70 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_X550EM_A_VF, 0, 0, 0}, 71 /* required last entry */ 72 {0, 0, 0, 0, 0} 73 }; 74 75 /************************************************************************ 76 * Table of branding strings 77 ************************************************************************/ 78 static const char *ixv_strings[] = { 79 "Intel(R) PRO/10GbE Virtual Function Network Driver" 80 }; 81 82 /********************************************************************* 83 * Function prototypes 84 *********************************************************************/ 85 static int ixv_probe(device_t, cfdata_t, void *); 86 static void ixv_attach(device_t, device_t, void *); 87 static int ixv_detach(device_t, int); 88 #if 0 89 static int ixv_shutdown(device_t); 90 #endif 91 static int ixv_ifflags_cb(struct ethercom *); 92 static int ixv_ioctl(struct ifnet *, u_long, void *); 93 static int ixv_init(struct ifnet *); 94 static void ixv_init_locked(struct adapter *); 95 static void ixv_ifstop(struct ifnet *, int); 96 static void ixv_stop_locked(void *); 97 static void ixv_init_device_features(struct adapter *); 98 static void ixv_media_status(struct ifnet *, struct ifmediareq *); 99 static int ixv_media_change(struct ifnet *); 100 static int ixv_allocate_pci_resources(struct adapter *, 101 const struct pci_attach_args *); 102 static void ixv_free_deferred_handlers(struct adapter *); 103 static int ixv_allocate_msix(struct adapter *, 104 const struct pci_attach_args *); 105 static int ixv_configure_interrupts(struct adapter *); 106 static void ixv_free_pci_resources(struct adapter *); 107 static void ixv_local_timer(void *); 108 static void ixv_handle_timer(struct work *, void *); 109 static int ixv_setup_interface(device_t, struct adapter *); 110 static void ixv_schedule_admin_tasklet(struct adapter *); 111 static int ixv_negotiate_api(struct adapter *); 112 113 static void ixv_initialize_transmit_units(struct adapter *); 114 static void ixv_initialize_receive_units(struct adapter *); 115 static void ixv_initialize_rss_mapping(struct adapter *); 116 static s32 ixv_check_link(struct adapter *); 117 118 static void ixv_enable_intr(struct adapter *); 119 static void ixv_disable_intr(struct adapter *); 120 static int ixv_set_rxfilter(struct adapter *); 121 static void ixv_update_link_status(struct adapter *); 122 static int ixv_sysctl_debug(SYSCTLFN_PROTO); 123 static void ixv_set_ivar(struct adapter *, u8, u8, s8); 124 static void ixv_configure_ivars(struct adapter *); 125 static u8 * ixv_mc_array_itr(struct ixgbe_hw *, u8 **, u32 *); 126 static void ixv_eitr_write(struct adapter *, uint32_t, uint32_t); 127 128 static void ixv_setup_vlan_tagging(struct adapter *); 129 static int ixv_setup_vlan_support(struct adapter *); 130 static int ixv_vlan_cb(struct ethercom *, uint16_t, bool); 131 static int ixv_register_vlan(struct adapter *, u16); 132 static int ixv_unregister_vlan(struct adapter *, u16); 133 134 static void ixv_add_device_sysctls(struct adapter *); 135 static void ixv_save_stats(struct adapter *); 136 static void ixv_init_stats(struct adapter *); 137 static void ixv_update_stats(struct adapter *); 138 static void ixv_add_stats_sysctls(struct adapter *); 139 static void ixv_clear_evcnt(struct adapter *); 140 141 /* Sysctl handlers */ 142 static void ixv_set_sysctl_value(struct adapter *, const char *, 143 const char *, int *, int); 144 static int ixv_sysctl_interrupt_rate_handler(SYSCTLFN_PROTO); 145 static int ixv_sysctl_next_to_check_handler(SYSCTLFN_PROTO); 146 static int ixv_sysctl_next_to_refresh_handler(SYSCTLFN_PROTO); 147 static int ixv_sysctl_rdh_handler(SYSCTLFN_PROTO); 148 static int ixv_sysctl_rdt_handler(SYSCTLFN_PROTO); 149 static int ixv_sysctl_tdt_handler(SYSCTLFN_PROTO); 150 static int ixv_sysctl_tdh_handler(SYSCTLFN_PROTO); 151 static int ixv_sysctl_rx_copy_len(SYSCTLFN_PROTO); 152 153 /* The MSI-X Interrupt handlers */ 154 static int ixv_msix_que(void *); 155 static int ixv_msix_mbx(void *); 156 157 /* Event handlers running on workqueue */ 158 static void ixv_handle_que(void *); 159 160 /* Deferred workqueue handlers */ 161 static void ixv_handle_admin(struct work *, void *); 162 static void ixv_handle_que_work(struct work *, void *); 163 164 const struct sysctlnode *ixv_sysctl_instance(struct adapter *); 165 static const ixgbe_vendor_info_t *ixv_lookup(const struct pci_attach_args *); 166 167 /************************************************************************ 168 * NetBSD Device Interface Entry Points 169 ************************************************************************/ 170 CFATTACH_DECL3_NEW(ixv, sizeof(struct adapter), 171 ixv_probe, ixv_attach, ixv_detach, NULL, NULL, NULL, 172 DVF_DETACH_SHUTDOWN); 173 174 #if 0 175 static driver_t ixv_driver = { 176 "ixv", ixv_methods, sizeof(struct adapter), 177 }; 178 179 devclass_t ixv_devclass; 180 DRIVER_MODULE(ixv, pci, ixv_driver, ixv_devclass, 0, 0); 181 MODULE_DEPEND(ixv, pci, 1, 1, 1); 182 MODULE_DEPEND(ixv, ether, 1, 1, 1); 183 #endif 184 185 /* 186 * TUNEABLE PARAMETERS: 187 */ 188 189 /* Number of Queues - do not exceed MSI-X vectors - 1 */ 190 static int ixv_num_queues = 0; 191 #define TUNABLE_INT(__x, __y) 192 TUNABLE_INT("hw.ixv.num_queues", &ixv_num_queues); 193 194 /* 195 * AIM: Adaptive Interrupt Moderation 196 * which means that the interrupt rate 197 * is varied over time based on the 198 * traffic for that interrupt vector 199 */ 200 static bool ixv_enable_aim = false; 201 TUNABLE_INT("hw.ixv.enable_aim", &ixv_enable_aim); 202 203 static int ixv_max_interrupt_rate = (4000000 / IXGBE_LOW_LATENCY); 204 TUNABLE_INT("hw.ixv.max_interrupt_rate", &ixv_max_interrupt_rate); 205 206 /* How many packets rxeof tries to clean at a time */ 207 static int ixv_rx_process_limit = 256; 208 TUNABLE_INT("hw.ixv.rx_process_limit", &ixv_rx_process_limit); 209 210 /* How many packets txeof tries to clean at a time */ 211 static int ixv_tx_process_limit = 256; 212 TUNABLE_INT("hw.ixv.tx_process_limit", &ixv_tx_process_limit); 213 214 /* Which packet processing uses workqueue or softint */ 215 static bool ixv_txrx_workqueue = false; 216 217 /* 218 * Number of TX descriptors per ring, 219 * setting higher than RX as this seems 220 * the better performing choice. 221 */ 222 static int ixv_txd = PERFORM_TXD; 223 TUNABLE_INT("hw.ixv.txd", &ixv_txd); 224 225 /* Number of RX descriptors per ring */ 226 static int ixv_rxd = PERFORM_RXD; 227 TUNABLE_INT("hw.ixv.rxd", &ixv_rxd); 228 229 /* Legacy Transmit (single queue) */ 230 static int ixv_enable_legacy_tx = 0; 231 TUNABLE_INT("hw.ixv.enable_legacy_tx", &ixv_enable_legacy_tx); 232 233 #ifdef NET_MPSAFE 234 #define IXGBE_MPSAFE 1 235 #define IXGBE_CALLOUT_FLAGS CALLOUT_MPSAFE 236 #define IXGBE_SOFTINT_FLAGS SOFTINT_MPSAFE 237 #define IXGBE_WORKQUEUE_FLAGS WQ_PERCPU | WQ_MPSAFE 238 #define IXGBE_TASKLET_WQ_FLAGS WQ_MPSAFE 239 #else 240 #define IXGBE_CALLOUT_FLAGS 0 241 #define IXGBE_SOFTINT_FLAGS 0 242 #define IXGBE_WORKQUEUE_FLAGS WQ_PERCPU 243 #define IXGBE_TASKLET_WQ_FLAGS 0 244 #endif 245 #define IXGBE_WORKQUEUE_PRI PRI_SOFTNET 246 247 #if 0 248 static int (*ixv_start_locked)(struct ifnet *, struct tx_ring *); 249 static int (*ixv_ring_empty)(struct ifnet *, struct buf_ring *); 250 #endif 251 252 /************************************************************************ 253 * ixv_probe - Device identification routine 254 * 255 * Determines if the driver should be loaded on 256 * adapter based on its PCI vendor/device ID. 257 * 258 * return BUS_PROBE_DEFAULT on success, positive on failure 259 ************************************************************************/ 260 static int 261 ixv_probe(device_t dev, cfdata_t cf, void *aux) 262 { 263 #ifdef __HAVE_PCI_MSI_MSIX 264 const struct pci_attach_args *pa = aux; 265 266 return (ixv_lookup(pa) != NULL) ? 1 : 0; 267 #else 268 return 0; 269 #endif 270 } /* ixv_probe */ 271 272 static const ixgbe_vendor_info_t * 273 ixv_lookup(const struct pci_attach_args *pa) 274 { 275 const ixgbe_vendor_info_t *ent; 276 pcireg_t subid; 277 278 INIT_DEBUGOUT("ixv_lookup: begin"); 279 280 if (PCI_VENDOR(pa->pa_id) != IXGBE_INTEL_VENDOR_ID) 281 return NULL; 282 283 subid = pci_conf_read(pa->pa_pc, pa->pa_tag, PCI_SUBSYS_ID_REG); 284 285 for (ent = ixv_vendor_info_array; ent->vendor_id != 0; ent++) { 286 if ((PCI_VENDOR(pa->pa_id) == ent->vendor_id) && 287 (PCI_PRODUCT(pa->pa_id) == ent->device_id) && 288 ((PCI_SUBSYS_VENDOR(subid) == ent->subvendor_id) || 289 (ent->subvendor_id == 0)) && 290 ((PCI_SUBSYS_ID(subid) == ent->subdevice_id) || 291 (ent->subdevice_id == 0))) { 292 return ent; 293 } 294 } 295 296 return NULL; 297 } 298 299 /************************************************************************ 300 * ixv_attach - Device initialization routine 301 * 302 * Called when the driver is being loaded. 303 * Identifies the type of hardware, allocates all resources 304 * and initializes the hardware. 305 * 306 * return 0 on success, positive on failure 307 ************************************************************************/ 308 static void 309 ixv_attach(device_t parent, device_t dev, void *aux) 310 { 311 struct adapter *adapter; 312 struct ixgbe_hw *hw; 313 int error = 0; 314 pcireg_t id, subid; 315 const ixgbe_vendor_info_t *ent; 316 const struct pci_attach_args *pa = aux; 317 const char *apivstr; 318 const char *str; 319 char wqname[MAXCOMLEN]; 320 char buf[256]; 321 322 INIT_DEBUGOUT("ixv_attach: begin"); 323 324 /* 325 * Make sure BUSMASTER is set, on a VM under 326 * KVM it may not be and will break things. 327 */ 328 ixgbe_pci_enable_busmaster(pa->pa_pc, pa->pa_tag); 329 330 /* Allocate, clear, and link in our adapter structure */ 331 adapter = device_private(dev); 332 adapter->hw.back = adapter; 333 adapter->dev = dev; 334 hw = &adapter->hw; 335 336 adapter->init_locked = ixv_init_locked; 337 adapter->stop_locked = ixv_stop_locked; 338 339 adapter->osdep.pc = pa->pa_pc; 340 adapter->osdep.tag = pa->pa_tag; 341 if (pci_dma64_available(pa)) 342 adapter->osdep.dmat = pa->pa_dmat64; 343 else 344 adapter->osdep.dmat = pa->pa_dmat; 345 adapter->osdep.attached = false; 346 347 ent = ixv_lookup(pa); 348 349 KASSERT(ent != NULL); 350 351 aprint_normal(": %s, Version - %s\n", 352 ixv_strings[ent->index], ixv_driver_version); 353 354 /* Core Lock Init */ 355 IXGBE_CORE_LOCK_INIT(adapter, device_xname(dev)); 356 357 /* Do base PCI setup - map BAR0 */ 358 if (ixv_allocate_pci_resources(adapter, pa)) { 359 aprint_error_dev(dev, "ixv_allocate_pci_resources() failed!\n"); 360 error = ENXIO; 361 goto err_out; 362 } 363 364 /* SYSCTL APIs */ 365 ixv_add_device_sysctls(adapter); 366 367 /* Set up the timer callout and workqueue */ 368 callout_init(&adapter->timer, IXGBE_CALLOUT_FLAGS); 369 snprintf(wqname, sizeof(wqname), "%s-timer", device_xname(dev)); 370 error = workqueue_create(&adapter->timer_wq, wqname, 371 ixv_handle_timer, adapter, IXGBE_WORKQUEUE_PRI, IPL_NET, 372 IXGBE_TASKLET_WQ_FLAGS); 373 if (error) { 374 aprint_error_dev(dev, 375 "could not create timer workqueue (%d)\n", error); 376 goto err_out; 377 } 378 379 /* Save off the information about this board */ 380 id = pci_conf_read(pa->pa_pc, pa->pa_tag, PCI_ID_REG); 381 subid = pci_conf_read(pa->pa_pc, pa->pa_tag, PCI_SUBSYS_ID_REG); 382 hw->vendor_id = PCI_VENDOR(id); 383 hw->device_id = PCI_PRODUCT(id); 384 hw->revision_id = 385 PCI_REVISION(pci_conf_read(pa->pa_pc, pa->pa_tag, PCI_CLASS_REG)); 386 hw->subsystem_vendor_id = PCI_SUBSYS_VENDOR(subid); 387 hw->subsystem_device_id = PCI_SUBSYS_ID(subid); 388 389 /* A subset of set_mac_type */ 390 switch (hw->device_id) { 391 case IXGBE_DEV_ID_82599_VF: 392 hw->mac.type = ixgbe_mac_82599_vf; 393 str = "82599 VF"; 394 break; 395 case IXGBE_DEV_ID_X540_VF: 396 hw->mac.type = ixgbe_mac_X540_vf; 397 str = "X540 VF"; 398 break; 399 case IXGBE_DEV_ID_X550_VF: 400 hw->mac.type = ixgbe_mac_X550_vf; 401 str = "X550 VF"; 402 break; 403 case IXGBE_DEV_ID_X550EM_X_VF: 404 hw->mac.type = ixgbe_mac_X550EM_x_vf; 405 str = "X550EM X VF"; 406 break; 407 case IXGBE_DEV_ID_X550EM_A_VF: 408 hw->mac.type = ixgbe_mac_X550EM_a_vf; 409 str = "X550EM A VF"; 410 break; 411 default: 412 /* Shouldn't get here since probe succeeded */ 413 aprint_error_dev(dev, "Unknown device ID!\n"); 414 error = ENXIO; 415 goto err_out; 416 break; 417 } 418 aprint_normal_dev(dev, "device %s\n", str); 419 420 ixv_init_device_features(adapter); 421 422 /* Initialize the shared code */ 423 error = ixgbe_init_ops_vf(hw); 424 if (error) { 425 aprint_error_dev(dev, "ixgbe_init_ops_vf() failed!\n"); 426 error = EIO; 427 goto err_out; 428 } 429 430 /* Setup the mailbox */ 431 ixgbe_init_mbx_params_vf(hw); 432 433 /* Set the right number of segments */ 434 KASSERT(IXGBE_82599_SCATTER_MAX >= IXGBE_SCATTER_DEFAULT); 435 adapter->num_segs = IXGBE_SCATTER_DEFAULT; 436 437 /* Reset mbox api to 1.0 */ 438 error = hw->mac.ops.reset_hw(hw); 439 if (error == IXGBE_ERR_RESET_FAILED) 440 aprint_error_dev(dev, "...reset_hw() failure: Reset Failed!\n"); 441 else if (error) 442 aprint_error_dev(dev, "...reset_hw() failed with error %d\n", 443 error); 444 if (error) { 445 error = EIO; 446 goto err_out; 447 } 448 449 error = hw->mac.ops.init_hw(hw); 450 if (error) { 451 aprint_error_dev(dev, "...init_hw() failed!\n"); 452 error = EIO; 453 goto err_out; 454 } 455 456 /* Negotiate mailbox API version */ 457 error = ixv_negotiate_api(adapter); 458 if (error) 459 aprint_normal_dev(dev, 460 "MBX API negotiation failed during attach!\n"); 461 switch (hw->api_version) { 462 case ixgbe_mbox_api_10: 463 apivstr = "1.0"; 464 break; 465 case ixgbe_mbox_api_20: 466 apivstr = "2.0"; 467 break; 468 case ixgbe_mbox_api_11: 469 apivstr = "1.1"; 470 break; 471 case ixgbe_mbox_api_12: 472 apivstr = "1.2"; 473 break; 474 case ixgbe_mbox_api_13: 475 apivstr = "1.3"; 476 break; 477 default: 478 apivstr = "unknown"; 479 break; 480 } 481 aprint_normal_dev(dev, "Mailbox API %s\n", apivstr); 482 483 /* If no mac address was assigned, make a random one */ 484 if (!ixv_check_ether_addr(hw->mac.addr)) { 485 u8 addr[ETHER_ADDR_LEN]; 486 uint64_t rndval = cprng_strong64(); 487 488 memcpy(addr, &rndval, sizeof(addr)); 489 addr[0] &= 0xFE; 490 addr[0] |= 0x02; 491 bcopy(addr, hw->mac.addr, sizeof(addr)); 492 } 493 494 /* Register for VLAN events */ 495 ether_set_vlan_cb(&adapter->osdep.ec, ixv_vlan_cb); 496 497 /* Sysctls for limiting the amount of work done in the taskqueues */ 498 ixv_set_sysctl_value(adapter, "rx_processing_limit", 499 "max number of rx packets to process", 500 &adapter->rx_process_limit, ixv_rx_process_limit); 501 502 ixv_set_sysctl_value(adapter, "tx_processing_limit", 503 "max number of tx packets to process", 504 &adapter->tx_process_limit, ixv_tx_process_limit); 505 506 /* Do descriptor calc and sanity checks */ 507 if (((ixv_txd * sizeof(union ixgbe_adv_tx_desc)) % DBA_ALIGN) != 0 || 508 ixv_txd < MIN_TXD || ixv_txd > MAX_TXD) { 509 aprint_error_dev(dev, "TXD config issue, using default!\n"); 510 adapter->num_tx_desc = DEFAULT_TXD; 511 } else 512 adapter->num_tx_desc = ixv_txd; 513 514 if (((ixv_rxd * sizeof(union ixgbe_adv_rx_desc)) % DBA_ALIGN) != 0 || 515 ixv_rxd < MIN_RXD || ixv_rxd > MAX_RXD) { 516 aprint_error_dev(dev, "RXD config issue, using default!\n"); 517 adapter->num_rx_desc = DEFAULT_RXD; 518 } else 519 adapter->num_rx_desc = ixv_rxd; 520 521 /* Set default high limit of copying mbuf in rxeof */ 522 adapter->rx_copy_len = IXGBE_RX_COPY_LEN_MAX; 523 524 /* Setup MSI-X */ 525 error = ixv_configure_interrupts(adapter); 526 if (error) 527 goto err_out; 528 529 /* Allocate our TX/RX Queues */ 530 if (ixgbe_allocate_queues(adapter)) { 531 aprint_error_dev(dev, "ixgbe_allocate_queues() failed!\n"); 532 error = ENOMEM; 533 goto err_out; 534 } 535 536 /* hw.ix defaults init */ 537 adapter->enable_aim = ixv_enable_aim; 538 539 adapter->txrx_use_workqueue = ixv_txrx_workqueue; 540 541 error = ixv_allocate_msix(adapter, pa); 542 if (error) { 543 aprint_error_dev(dev, "ixv_allocate_msix() failed!\n"); 544 goto err_late; 545 } 546 547 /* Setup OS specific network interface */ 548 error = ixv_setup_interface(dev, adapter); 549 if (error != 0) { 550 aprint_error_dev(dev, "ixv_setup_interface() failed!\n"); 551 goto err_late; 552 } 553 554 /* Do the stats setup */ 555 ixv_save_stats(adapter); 556 ixv_init_stats(adapter); 557 ixv_add_stats_sysctls(adapter); 558 559 if (adapter->feat_en & IXGBE_FEATURE_NETMAP) 560 ixgbe_netmap_attach(adapter); 561 562 snprintb(buf, sizeof(buf), IXGBE_FEATURE_FLAGS, adapter->feat_cap); 563 aprint_verbose_dev(dev, "feature cap %s\n", buf); 564 snprintb(buf, sizeof(buf), IXGBE_FEATURE_FLAGS, adapter->feat_en); 565 aprint_verbose_dev(dev, "feature ena %s\n", buf); 566 567 INIT_DEBUGOUT("ixv_attach: end"); 568 adapter->osdep.attached = true; 569 570 return; 571 572 err_late: 573 ixgbe_free_queues(adapter); 574 err_out: 575 ixv_free_pci_resources(adapter); 576 IXGBE_CORE_LOCK_DESTROY(adapter); 577 578 return; 579 } /* ixv_attach */ 580 581 /************************************************************************ 582 * ixv_detach - Device removal routine 583 * 584 * Called when the driver is being removed. 585 * Stops the adapter and deallocates all the resources 586 * that were allocated for driver operation. 587 * 588 * return 0 on success, positive on failure 589 ************************************************************************/ 590 static int 591 ixv_detach(device_t dev, int flags) 592 { 593 struct adapter *adapter = device_private(dev); 594 struct ixgbe_hw *hw = &adapter->hw; 595 struct tx_ring *txr = adapter->tx_rings; 596 struct rx_ring *rxr = adapter->rx_rings; 597 struct ixgbevf_hw_stats *stats = &adapter->stats.vf; 598 599 INIT_DEBUGOUT("ixv_detach: begin"); 600 if (adapter->osdep.attached == false) 601 return 0; 602 603 /* Stop the interface. Callouts are stopped in it. */ 604 ixv_ifstop(adapter->ifp, 1); 605 606 #if NVLAN > 0 607 /* Make sure VLANs are not using driver */ 608 if (!VLAN_ATTACHED(&adapter->osdep.ec)) 609 ; /* nothing to do: no VLANs */ 610 else if ((flags & (DETACH_SHUTDOWN | DETACH_FORCE)) != 0) 611 vlan_ifdetach(adapter->ifp); 612 else { 613 aprint_error_dev(dev, "VLANs in use, detach first\n"); 614 return EBUSY; 615 } 616 #endif 617 618 ether_ifdetach(adapter->ifp); 619 callout_halt(&adapter->timer, NULL); 620 ixv_free_deferred_handlers(adapter); 621 622 if (adapter->feat_en & IXGBE_FEATURE_NETMAP) 623 netmap_detach(adapter->ifp); 624 625 ixv_free_pci_resources(adapter); 626 #if 0 /* XXX the NetBSD port is probably missing something here */ 627 bus_generic_detach(dev); 628 #endif 629 if_detach(adapter->ifp); 630 ifmedia_fini(&adapter->media); 631 if_percpuq_destroy(adapter->ipq); 632 633 sysctl_teardown(&adapter->sysctllog); 634 evcnt_detach(&adapter->efbig_tx_dma_setup); 635 evcnt_detach(&adapter->mbuf_defrag_failed); 636 evcnt_detach(&adapter->efbig2_tx_dma_setup); 637 evcnt_detach(&adapter->einval_tx_dma_setup); 638 evcnt_detach(&adapter->other_tx_dma_setup); 639 evcnt_detach(&adapter->eagain_tx_dma_setup); 640 evcnt_detach(&adapter->enomem_tx_dma_setup); 641 evcnt_detach(&adapter->watchdog_events); 642 evcnt_detach(&adapter->tso_err); 643 evcnt_detach(&adapter->admin_irqev); 644 evcnt_detach(&adapter->link_workev); 645 646 txr = adapter->tx_rings; 647 for (int i = 0; i < adapter->num_queues; i++, rxr++, txr++) { 648 evcnt_detach(&adapter->queues[i].irqs); 649 evcnt_detach(&adapter->queues[i].handleq); 650 evcnt_detach(&adapter->queues[i].req); 651 evcnt_detach(&txr->no_desc_avail); 652 evcnt_detach(&txr->total_packets); 653 evcnt_detach(&txr->tso_tx); 654 #ifndef IXGBE_LEGACY_TX 655 evcnt_detach(&txr->pcq_drops); 656 #endif 657 658 evcnt_detach(&rxr->rx_packets); 659 evcnt_detach(&rxr->rx_bytes); 660 evcnt_detach(&rxr->rx_copies); 661 evcnt_detach(&rxr->no_mbuf); 662 evcnt_detach(&rxr->rx_discarded); 663 } 664 evcnt_detach(&stats->ipcs); 665 evcnt_detach(&stats->l4cs); 666 evcnt_detach(&stats->ipcs_bad); 667 evcnt_detach(&stats->l4cs_bad); 668 669 /* Packet Reception Stats */ 670 evcnt_detach(&stats->vfgorc); 671 evcnt_detach(&stats->vfgprc); 672 evcnt_detach(&stats->vfmprc); 673 674 /* Packet Transmission Stats */ 675 evcnt_detach(&stats->vfgotc); 676 evcnt_detach(&stats->vfgptc); 677 678 /* Mailbox Stats */ 679 evcnt_detach(&hw->mbx.stats.msgs_tx); 680 evcnt_detach(&hw->mbx.stats.msgs_rx); 681 evcnt_detach(&hw->mbx.stats.acks); 682 evcnt_detach(&hw->mbx.stats.reqs); 683 evcnt_detach(&hw->mbx.stats.rsts); 684 685 ixgbe_free_queues(adapter); 686 687 IXGBE_CORE_LOCK_DESTROY(adapter); 688 689 return (0); 690 } /* ixv_detach */ 691 692 /************************************************************************ 693 * ixv_init_locked - Init entry point 694 * 695 * Used in two ways: It is used by the stack as an init entry 696 * point in network interface structure. It is also used 697 * by the driver as a hw/sw initialization routine to get 698 * to a consistent state. 699 * 700 * return 0 on success, positive on failure 701 ************************************************************************/ 702 static void 703 ixv_init_locked(struct adapter *adapter) 704 { 705 struct ifnet *ifp = adapter->ifp; 706 device_t dev = adapter->dev; 707 struct ixgbe_hw *hw = &adapter->hw; 708 struct ix_queue *que; 709 int error = 0; 710 uint32_t mask; 711 int i; 712 713 INIT_DEBUGOUT("ixv_init_locked: begin"); 714 KASSERT(mutex_owned(&adapter->core_mtx)); 715 hw->adapter_stopped = FALSE; 716 hw->mac.ops.stop_adapter(hw); 717 callout_stop(&adapter->timer); 718 for (i = 0, que = adapter->queues; i < adapter->num_queues; i++, que++) 719 que->disabled_count = 0; 720 721 adapter->max_frame_size = 722 ifp->if_mtu + ETHER_HDR_LEN + ETHER_CRC_LEN; 723 724 /* reprogram the RAR[0] in case user changed it. */ 725 hw->mac.ops.set_rar(hw, 0, hw->mac.addr, 0, IXGBE_RAH_AV); 726 727 /* Get the latest mac address, User can use a LAA */ 728 memcpy(hw->mac.addr, CLLADDR(ifp->if_sadl), 729 IXGBE_ETH_LENGTH_OF_ADDRESS); 730 hw->mac.ops.set_rar(hw, 0, hw->mac.addr, 0, 1); 731 732 /* Prepare transmit descriptors and buffers */ 733 if (ixgbe_setup_transmit_structures(adapter)) { 734 aprint_error_dev(dev, "Could not setup transmit structures\n"); 735 ixv_stop_locked(adapter); 736 return; 737 } 738 739 /* Reset VF and renegotiate mailbox API version */ 740 hw->mac.ops.reset_hw(hw); 741 hw->mac.ops.start_hw(hw); 742 error = ixv_negotiate_api(adapter); 743 if (error) 744 device_printf(dev, 745 "Mailbox API negotiation failed in init_locked!\n"); 746 747 ixv_initialize_transmit_units(adapter); 748 749 /* Setup Multicast table */ 750 ixv_set_rxfilter(adapter); 751 752 /* Use fixed buffer size, even for jumbo frames */ 753 adapter->rx_mbuf_sz = MCLBYTES; 754 755 /* Prepare receive descriptors and buffers */ 756 error = ixgbe_setup_receive_structures(adapter); 757 if (error) { 758 device_printf(dev, 759 "Could not setup receive structures (err = %d)\n", error); 760 ixv_stop_locked(adapter); 761 return; 762 } 763 764 /* Configure RX settings */ 765 ixv_initialize_receive_units(adapter); 766 767 /* Initialize variable holding task enqueue requests interrupts */ 768 adapter->task_requests = 0; 769 770 /* Set up VLAN offload and filter */ 771 ixv_setup_vlan_support(adapter); 772 773 /* Set up MSI-X routing */ 774 ixv_configure_ivars(adapter); 775 776 /* Set up auto-mask */ 777 mask = (1 << adapter->vector); 778 for (i = 0, que = adapter->queues; i < adapter->num_queues; i++, que++) 779 mask |= (1 << que->msix); 780 IXGBE_WRITE_REG(hw, IXGBE_VTEIAM, mask); 781 782 /* Set moderation on the Link interrupt */ 783 ixv_eitr_write(adapter, adapter->vector, IXGBE_LINK_ITR); 784 785 /* Stats init */ 786 ixv_init_stats(adapter); 787 788 /* Config/Enable Link */ 789 hw->mac.get_link_status = TRUE; 790 hw->mac.ops.check_link(hw, &adapter->link_speed, &adapter->link_up, 791 FALSE); 792 793 /* Start watchdog */ 794 callout_reset(&adapter->timer, hz, ixv_local_timer, adapter); 795 atomic_store_relaxed(&adapter->timer_pending, 0); 796 797 /* OK to schedule workqueues. */ 798 adapter->schedule_wqs_ok = true; 799 800 /* And now turn on interrupts */ 801 ixv_enable_intr(adapter); 802 803 /* Update saved flags. See ixgbe_ifflags_cb() */ 804 adapter->if_flags = ifp->if_flags; 805 adapter->ec_capenable = adapter->osdep.ec.ec_capenable; 806 807 /* Now inform the stack we're ready */ 808 ifp->if_flags |= IFF_RUNNING; 809 ifp->if_flags &= ~IFF_OACTIVE; 810 811 return; 812 } /* ixv_init_locked */ 813 814 /************************************************************************ 815 * ixv_enable_queue 816 ************************************************************************/ 817 static inline void 818 ixv_enable_queue(struct adapter *adapter, u32 vector) 819 { 820 struct ixgbe_hw *hw = &adapter->hw; 821 struct ix_queue *que = &adapter->queues[vector]; 822 u32 queue = 1UL << vector; 823 u32 mask; 824 825 mutex_enter(&que->dc_mtx); 826 if (que->disabled_count > 0 && --que->disabled_count > 0) 827 goto out; 828 829 mask = (IXGBE_EIMS_RTX_QUEUE & queue); 830 IXGBE_WRITE_REG(hw, IXGBE_VTEIMS, mask); 831 out: 832 mutex_exit(&que->dc_mtx); 833 } /* ixv_enable_queue */ 834 835 /************************************************************************ 836 * ixv_disable_queue 837 ************************************************************************/ 838 static inline void 839 ixv_disable_queue(struct adapter *adapter, u32 vector) 840 { 841 struct ixgbe_hw *hw = &adapter->hw; 842 struct ix_queue *que = &adapter->queues[vector]; 843 u32 queue = 1UL << vector; 844 u32 mask; 845 846 mutex_enter(&que->dc_mtx); 847 if (que->disabled_count++ > 0) 848 goto out; 849 850 mask = (IXGBE_EIMS_RTX_QUEUE & queue); 851 IXGBE_WRITE_REG(hw, IXGBE_VTEIMC, mask); 852 out: 853 mutex_exit(&que->dc_mtx); 854 } /* ixv_disable_queue */ 855 856 #if 0 857 static inline void 858 ixv_rearm_queues(struct adapter *adapter, u64 queues) 859 { 860 u32 mask = (IXGBE_EIMS_RTX_QUEUE & queues); 861 IXGBE_WRITE_REG(&adapter->hw, IXGBE_VTEICS, mask); 862 } /* ixv_rearm_queues */ 863 #endif 864 865 866 /************************************************************************ 867 * ixv_msix_que - MSI-X Queue Interrupt Service routine 868 ************************************************************************/ 869 static int 870 ixv_msix_que(void *arg) 871 { 872 struct ix_queue *que = arg; 873 struct adapter *adapter = que->adapter; 874 struct tx_ring *txr = que->txr; 875 struct rx_ring *rxr = que->rxr; 876 bool more; 877 u32 newitr = 0; 878 879 ixv_disable_queue(adapter, que->msix); 880 ++que->irqs.ev_count; 881 882 #ifdef __NetBSD__ 883 /* Don't run ixgbe_rxeof in interrupt context */ 884 more = true; 885 #else 886 more = ixgbe_rxeof(que); 887 #endif 888 889 IXGBE_TX_LOCK(txr); 890 ixgbe_txeof(txr); 891 IXGBE_TX_UNLOCK(txr); 892 893 /* Do AIM now? */ 894 895 if (adapter->enable_aim == false) 896 goto no_calc; 897 /* 898 * Do Adaptive Interrupt Moderation: 899 * - Write out last calculated setting 900 * - Calculate based on average size over 901 * the last interval. 902 */ 903 if (que->eitr_setting) 904 ixv_eitr_write(adapter, que->msix, que->eitr_setting); 905 906 que->eitr_setting = 0; 907 908 /* Idle, do nothing */ 909 if ((txr->bytes == 0) && (rxr->bytes == 0)) 910 goto no_calc; 911 912 if ((txr->bytes) && (txr->packets)) 913 newitr = txr->bytes/txr->packets; 914 if ((rxr->bytes) && (rxr->packets)) 915 newitr = uimax(newitr, (rxr->bytes / rxr->packets)); 916 newitr += 24; /* account for hardware frame, crc */ 917 918 /* set an upper boundary */ 919 newitr = uimin(newitr, 3000); 920 921 /* Be nice to the mid range */ 922 if ((newitr > 300) && (newitr < 1200)) 923 newitr = (newitr / 3); 924 else 925 newitr = (newitr / 2); 926 927 /* 928 * When RSC is used, ITR interval must be larger than RSC_DELAY. 929 * Currently, we use 2us for RSC_DELAY. The minimum value is always 930 * greater than 2us on 100M (and 10M?(not documented)), but it's not 931 * on 1G and higher. 932 */ 933 if ((adapter->link_speed != IXGBE_LINK_SPEED_100_FULL) 934 && (adapter->link_speed != IXGBE_LINK_SPEED_10_FULL)) { 935 if (newitr < IXGBE_MIN_RSC_EITR_10G1G) 936 newitr = IXGBE_MIN_RSC_EITR_10G1G; 937 } 938 939 /* save for next interrupt */ 940 que->eitr_setting = newitr; 941 942 /* Reset state */ 943 txr->bytes = 0; 944 txr->packets = 0; 945 rxr->bytes = 0; 946 rxr->packets = 0; 947 948 no_calc: 949 if (more) 950 softint_schedule(que->que_si); 951 else /* Re-enable this interrupt */ 952 ixv_enable_queue(adapter, que->msix); 953 954 return 1; 955 } /* ixv_msix_que */ 956 957 /************************************************************************ 958 * ixv_msix_mbx 959 ************************************************************************/ 960 static int 961 ixv_msix_mbx(void *arg) 962 { 963 struct adapter *adapter = arg; 964 struct ixgbe_hw *hw = &adapter->hw; 965 966 ++adapter->admin_irqev.ev_count; 967 /* NetBSD: We use auto-clear, so it's not required to write VTEICR */ 968 969 /* Link status change */ 970 hw->mac.get_link_status = TRUE; 971 atomic_or_32(&adapter->task_requests, IXGBE_REQUEST_TASK_MBX); 972 ixv_schedule_admin_tasklet(adapter); 973 974 return 1; 975 } /* ixv_msix_mbx */ 976 977 static void 978 ixv_eitr_write(struct adapter *adapter, uint32_t index, uint32_t itr) 979 { 980 981 /* 982 * Newer devices than 82598 have VF function, so this function is 983 * simple. 984 */ 985 itr |= IXGBE_EITR_CNT_WDIS; 986 987 IXGBE_WRITE_REG(&adapter->hw, IXGBE_VTEITR(index), itr); 988 } 989 990 991 /************************************************************************ 992 * ixv_media_status - Media Ioctl callback 993 * 994 * Called whenever the user queries the status of 995 * the interface using ifconfig. 996 ************************************************************************/ 997 static void 998 ixv_media_status(struct ifnet *ifp, struct ifmediareq *ifmr) 999 { 1000 struct adapter *adapter = ifp->if_softc; 1001 1002 INIT_DEBUGOUT("ixv_media_status: begin"); 1003 ixv_update_link_status(adapter); 1004 1005 ifmr->ifm_status = IFM_AVALID; 1006 ifmr->ifm_active = IFM_ETHER; 1007 1008 if (adapter->link_active != LINK_STATE_UP) { 1009 ifmr->ifm_active |= IFM_NONE; 1010 return; 1011 } 1012 1013 ifmr->ifm_status |= IFM_ACTIVE; 1014 1015 switch (adapter->link_speed) { 1016 case IXGBE_LINK_SPEED_10GB_FULL: 1017 ifmr->ifm_active |= IFM_10G_T | IFM_FDX; 1018 break; 1019 case IXGBE_LINK_SPEED_5GB_FULL: 1020 ifmr->ifm_active |= IFM_5000_T | IFM_FDX; 1021 break; 1022 case IXGBE_LINK_SPEED_2_5GB_FULL: 1023 ifmr->ifm_active |= IFM_2500_T | IFM_FDX; 1024 break; 1025 case IXGBE_LINK_SPEED_1GB_FULL: 1026 ifmr->ifm_active |= IFM_1000_T | IFM_FDX; 1027 break; 1028 case IXGBE_LINK_SPEED_100_FULL: 1029 ifmr->ifm_active |= IFM_100_TX | IFM_FDX; 1030 break; 1031 case IXGBE_LINK_SPEED_10_FULL: 1032 ifmr->ifm_active |= IFM_10_T | IFM_FDX; 1033 break; 1034 } 1035 1036 ifp->if_baudrate = ifmedia_baudrate(ifmr->ifm_active); 1037 } /* ixv_media_status */ 1038 1039 /************************************************************************ 1040 * ixv_media_change - Media Ioctl callback 1041 * 1042 * Called when the user changes speed/duplex using 1043 * media/mediopt option with ifconfig. 1044 ************************************************************************/ 1045 static int 1046 ixv_media_change(struct ifnet *ifp) 1047 { 1048 struct adapter *adapter = ifp->if_softc; 1049 struct ifmedia *ifm = &adapter->media; 1050 1051 INIT_DEBUGOUT("ixv_media_change: begin"); 1052 1053 if (IFM_TYPE(ifm->ifm_media) != IFM_ETHER) 1054 return (EINVAL); 1055 1056 switch (IFM_SUBTYPE(ifm->ifm_media)) { 1057 case IFM_AUTO: 1058 break; 1059 default: 1060 device_printf(adapter->dev, "Only auto media type\n"); 1061 return (EINVAL); 1062 } 1063 1064 return (0); 1065 } /* ixv_media_change */ 1066 1067 static void 1068 ixv_schedule_admin_tasklet(struct adapter *adapter) 1069 { 1070 if (adapter->schedule_wqs_ok) { 1071 if (atomic_cas_uint(&adapter->admin_pending, 0, 1) == 0) 1072 workqueue_enqueue(adapter->admin_wq, 1073 &adapter->admin_wc, NULL); 1074 } 1075 } 1076 1077 /************************************************************************ 1078 * ixv_negotiate_api 1079 * 1080 * Negotiate the Mailbox API with the PF; 1081 * start with the most featured API first. 1082 ************************************************************************/ 1083 static int 1084 ixv_negotiate_api(struct adapter *adapter) 1085 { 1086 struct ixgbe_hw *hw = &adapter->hw; 1087 int mbx_api[] = { ixgbe_mbox_api_13, 1088 ixgbe_mbox_api_12, 1089 ixgbe_mbox_api_11, 1090 ixgbe_mbox_api_10, 1091 ixgbe_mbox_api_unknown }; 1092 int i = 0; 1093 1094 while (mbx_api[i] != ixgbe_mbox_api_unknown) { 1095 if (ixgbevf_negotiate_api_version(hw, mbx_api[i]) == 0) 1096 return (0); 1097 i++; 1098 } 1099 1100 return (EINVAL); 1101 } /* ixv_negotiate_api */ 1102 1103 1104 /************************************************************************ 1105 * ixv_set_rxfilter - Multicast Update 1106 * 1107 * Called whenever multicast address list is updated. 1108 ************************************************************************/ 1109 static int 1110 ixv_set_rxfilter(struct adapter *adapter) 1111 { 1112 u8 mta[IXGBE_MAX_VF_MC * IXGBE_ETH_LENGTH_OF_ADDRESS]; 1113 struct ifnet *ifp = adapter->ifp; 1114 struct ixgbe_hw *hw = &adapter->hw; 1115 u8 *update_ptr; 1116 int mcnt = 0; 1117 struct ethercom *ec = &adapter->osdep.ec; 1118 struct ether_multi *enm; 1119 struct ether_multistep step; 1120 bool overflow = false; 1121 int error, rc = 0; 1122 1123 KASSERT(mutex_owned(&adapter->core_mtx)); 1124 IOCTL_DEBUGOUT("ixv_set_rxfilter: begin"); 1125 1126 /* 1: For PROMISC */ 1127 if (ifp->if_flags & IFF_PROMISC) { 1128 error = hw->mac.ops.update_xcast_mode(hw, 1129 IXGBEVF_XCAST_MODE_PROMISC); 1130 if (error == IXGBE_ERR_NOT_TRUSTED) { 1131 device_printf(adapter->dev, 1132 "this interface is not trusted\n"); 1133 error = EPERM; 1134 } else if (error == IXGBE_ERR_FEATURE_NOT_SUPPORTED) { 1135 device_printf(adapter->dev, 1136 "the PF doesn't support promisc mode\n"); 1137 error = EOPNOTSUPP; 1138 } else if (error == IXGBE_ERR_NOT_IN_PROMISC) { 1139 device_printf(adapter->dev, 1140 "the PF may not in promisc mode\n"); 1141 error = EINVAL; 1142 } else if (error) { 1143 device_printf(adapter->dev, 1144 "failed to set promisc mode. error = %d\n", 1145 error); 1146 error = EIO; 1147 } else 1148 return 0; 1149 rc = error; 1150 } 1151 1152 /* 2: For ALLMULTI or normal */ 1153 ETHER_LOCK(ec); 1154 ETHER_FIRST_MULTI(step, ec, enm); 1155 while (enm != NULL) { 1156 if ((mcnt >= IXGBE_MAX_VF_MC) || 1157 (memcmp(enm->enm_addrlo, enm->enm_addrhi, 1158 ETHER_ADDR_LEN) != 0)) { 1159 overflow = true; 1160 break; 1161 } 1162 bcopy(enm->enm_addrlo, 1163 &mta[mcnt * IXGBE_ETH_LENGTH_OF_ADDRESS], 1164 IXGBE_ETH_LENGTH_OF_ADDRESS); 1165 mcnt++; 1166 ETHER_NEXT_MULTI(step, enm); 1167 } 1168 ETHER_UNLOCK(ec); 1169 1170 /* 3: For ALLMULTI */ 1171 if (overflow) { 1172 error = hw->mac.ops.update_xcast_mode(hw, 1173 IXGBEVF_XCAST_MODE_ALLMULTI); 1174 if (error == IXGBE_ERR_NOT_TRUSTED) { 1175 device_printf(adapter->dev, 1176 "this interface is not trusted\n"); 1177 error = EPERM; 1178 } else if (error == IXGBE_ERR_FEATURE_NOT_SUPPORTED) { 1179 device_printf(adapter->dev, 1180 "the PF doesn't support allmulti mode\n"); 1181 error = EOPNOTSUPP; 1182 } else if (error) { 1183 device_printf(adapter->dev, 1184 "number of Ethernet multicast addresses " 1185 "exceeds the limit (%d). error = %d\n", 1186 IXGBE_MAX_VF_MC, error); 1187 error = ENOSPC; 1188 } else { 1189 ETHER_LOCK(ec); 1190 ec->ec_flags |= ETHER_F_ALLMULTI; 1191 ETHER_UNLOCK(ec); 1192 return rc; /* Promisc might have failed */ 1193 } 1194 1195 if (rc == 0) 1196 rc = error; 1197 1198 /* Continue to update the multicast table as many as we can */ 1199 } 1200 1201 /* 4: For normal operation */ 1202 error = hw->mac.ops.update_xcast_mode(hw, IXGBEVF_XCAST_MODE_MULTI); 1203 if ((error == IXGBE_ERR_FEATURE_NOT_SUPPORTED) || (error == 0)) { 1204 /* Normal operation */ 1205 ETHER_LOCK(ec); 1206 ec->ec_flags &= ~ETHER_F_ALLMULTI; 1207 ETHER_UNLOCK(ec); 1208 error = 0; 1209 } else if (error) { 1210 device_printf(adapter->dev, 1211 "failed to set Ethernet multicast address " 1212 "operation to normal. error = %d\n", error); 1213 } 1214 1215 update_ptr = mta; 1216 1217 error = adapter->hw.mac.ops.update_mc_addr_list(&adapter->hw, 1218 update_ptr, mcnt, ixv_mc_array_itr, TRUE); 1219 if (rc == 0) 1220 rc = error; 1221 1222 return rc; 1223 } /* ixv_set_rxfilter */ 1224 1225 /************************************************************************ 1226 * ixv_mc_array_itr 1227 * 1228 * An iterator function needed by the multicast shared code. 1229 * It feeds the shared code routine the addresses in the 1230 * array of ixv_set_rxfilter() one by one. 1231 ************************************************************************/ 1232 static u8 * 1233 ixv_mc_array_itr(struct ixgbe_hw *hw, u8 **update_ptr, u32 *vmdq) 1234 { 1235 u8 *addr = *update_ptr; 1236 u8 *newptr; 1237 1238 *vmdq = 0; 1239 1240 newptr = addr + IXGBE_ETH_LENGTH_OF_ADDRESS; 1241 *update_ptr = newptr; 1242 1243 return addr; 1244 } /* ixv_mc_array_itr */ 1245 1246 /************************************************************************ 1247 * ixv_local_timer - Timer routine 1248 * 1249 * Checks for link status, updates statistics, 1250 * and runs the watchdog check. 1251 ************************************************************************/ 1252 static void 1253 ixv_local_timer(void *arg) 1254 { 1255 struct adapter *adapter = arg; 1256 1257 if (adapter->schedule_wqs_ok) { 1258 if (atomic_cas_uint(&adapter->timer_pending, 0, 1) == 0) 1259 workqueue_enqueue(adapter->timer_wq, 1260 &adapter->timer_wc, NULL); 1261 } 1262 } 1263 1264 static void 1265 ixv_handle_timer(struct work *wk, void *context) 1266 { 1267 struct adapter *adapter = context; 1268 device_t dev = adapter->dev; 1269 struct ix_queue *que = adapter->queues; 1270 u64 queues = 0; 1271 u64 v0, v1, v2, v3, v4, v5, v6, v7; 1272 int hung = 0; 1273 int i; 1274 1275 IXGBE_CORE_LOCK(adapter); 1276 1277 if (ixv_check_link(adapter)) { 1278 ixv_init_locked(adapter); 1279 IXGBE_CORE_UNLOCK(adapter); 1280 return; 1281 } 1282 1283 /* Stats Update */ 1284 ixv_update_stats(adapter); 1285 1286 /* Update some event counters */ 1287 v0 = v1 = v2 = v3 = v4 = v5 = v6 = v7 = 0; 1288 que = adapter->queues; 1289 for (i = 0; i < adapter->num_queues; i++, que++) { 1290 struct tx_ring *txr = que->txr; 1291 1292 v0 += txr->q_efbig_tx_dma_setup; 1293 v1 += txr->q_mbuf_defrag_failed; 1294 v2 += txr->q_efbig2_tx_dma_setup; 1295 v3 += txr->q_einval_tx_dma_setup; 1296 v4 += txr->q_other_tx_dma_setup; 1297 v5 += txr->q_eagain_tx_dma_setup; 1298 v6 += txr->q_enomem_tx_dma_setup; 1299 v7 += txr->q_tso_err; 1300 } 1301 adapter->efbig_tx_dma_setup.ev_count = v0; 1302 adapter->mbuf_defrag_failed.ev_count = v1; 1303 adapter->efbig2_tx_dma_setup.ev_count = v2; 1304 adapter->einval_tx_dma_setup.ev_count = v3; 1305 adapter->other_tx_dma_setup.ev_count = v4; 1306 adapter->eagain_tx_dma_setup.ev_count = v5; 1307 adapter->enomem_tx_dma_setup.ev_count = v6; 1308 adapter->tso_err.ev_count = v7; 1309 1310 /* 1311 * Check the TX queues status 1312 * - mark hung queues so we don't schedule on them 1313 * - watchdog only if all queues show hung 1314 */ 1315 que = adapter->queues; 1316 for (i = 0; i < adapter->num_queues; i++, que++) { 1317 /* Keep track of queues with work for soft irq */ 1318 if (que->txr->busy) 1319 queues |= ((u64)1 << que->me); 1320 /* 1321 * Each time txeof runs without cleaning, but there 1322 * are uncleaned descriptors it increments busy. If 1323 * we get to the MAX we declare it hung. 1324 */ 1325 if (que->busy == IXGBE_QUEUE_HUNG) { 1326 ++hung; 1327 /* Mark the queue as inactive */ 1328 adapter->active_queues &= ~((u64)1 << que->me); 1329 continue; 1330 } else { 1331 /* Check if we've come back from hung */ 1332 if ((adapter->active_queues & ((u64)1 << que->me)) == 0) 1333 adapter->active_queues |= ((u64)1 << que->me); 1334 } 1335 if (que->busy >= IXGBE_MAX_TX_BUSY) { 1336 device_printf(dev, 1337 "Warning queue %d appears to be hung!\n", i); 1338 que->txr->busy = IXGBE_QUEUE_HUNG; 1339 ++hung; 1340 } 1341 } 1342 1343 /* Only truly watchdog if all queues show hung */ 1344 if (hung == adapter->num_queues) 1345 goto watchdog; 1346 #if 0 1347 else if (queues != 0) { /* Force an IRQ on queues with work */ 1348 ixv_rearm_queues(adapter, queues); 1349 } 1350 #endif 1351 1352 atomic_store_relaxed(&adapter->timer_pending, 0); 1353 IXGBE_CORE_UNLOCK(adapter); 1354 callout_reset(&adapter->timer, hz, ixv_local_timer, adapter); 1355 1356 return; 1357 1358 watchdog: 1359 device_printf(adapter->dev, "Watchdog timeout -- resetting\n"); 1360 adapter->ifp->if_flags &= ~IFF_RUNNING; 1361 adapter->watchdog_events.ev_count++; 1362 ixv_init_locked(adapter); 1363 IXGBE_CORE_UNLOCK(adapter); 1364 } /* ixv_handle_timer */ 1365 1366 /************************************************************************ 1367 * ixv_update_link_status - Update OS on link state 1368 * 1369 * Note: Only updates the OS on the cached link state. 1370 * The real check of the hardware only happens with 1371 * a link interrupt. 1372 ************************************************************************/ 1373 static void 1374 ixv_update_link_status(struct adapter *adapter) 1375 { 1376 struct ifnet *ifp = adapter->ifp; 1377 device_t dev = adapter->dev; 1378 1379 KASSERT(mutex_owned(&adapter->core_mtx)); 1380 1381 if (adapter->link_up) { 1382 if (adapter->link_active != LINK_STATE_UP) { 1383 if (bootverbose) { 1384 const char *bpsmsg; 1385 1386 switch (adapter->link_speed) { 1387 case IXGBE_LINK_SPEED_10GB_FULL: 1388 bpsmsg = "10 Gbps"; 1389 break; 1390 case IXGBE_LINK_SPEED_5GB_FULL: 1391 bpsmsg = "5 Gbps"; 1392 break; 1393 case IXGBE_LINK_SPEED_2_5GB_FULL: 1394 bpsmsg = "2.5 Gbps"; 1395 break; 1396 case IXGBE_LINK_SPEED_1GB_FULL: 1397 bpsmsg = "1 Gbps"; 1398 break; 1399 case IXGBE_LINK_SPEED_100_FULL: 1400 bpsmsg = "100 Mbps"; 1401 break; 1402 case IXGBE_LINK_SPEED_10_FULL: 1403 bpsmsg = "10 Mbps"; 1404 break; 1405 default: 1406 bpsmsg = "unknown speed"; 1407 break; 1408 } 1409 device_printf(dev, "Link is up %s %s \n", 1410 bpsmsg, "Full Duplex"); 1411 } 1412 adapter->link_active = LINK_STATE_UP; 1413 if_link_state_change(ifp, LINK_STATE_UP); 1414 } 1415 } else { 1416 /* 1417 * Do it when link active changes to DOWN. i.e. 1418 * a) LINK_STATE_UNKNOWN -> LINK_STATE_DOWN 1419 * b) LINK_STATE_UP -> LINK_STATE_DOWN 1420 */ 1421 if (adapter->link_active != LINK_STATE_DOWN) { 1422 if (bootverbose) 1423 device_printf(dev, "Link is Down\n"); 1424 if_link_state_change(ifp, LINK_STATE_DOWN); 1425 adapter->link_active = LINK_STATE_DOWN; 1426 } 1427 } 1428 } /* ixv_update_link_status */ 1429 1430 1431 /************************************************************************ 1432 * ixv_stop - Stop the hardware 1433 * 1434 * Disables all traffic on the adapter by issuing a 1435 * global reset on the MAC and deallocates TX/RX buffers. 1436 ************************************************************************/ 1437 static void 1438 ixv_ifstop(struct ifnet *ifp, int disable) 1439 { 1440 struct adapter *adapter = ifp->if_softc; 1441 1442 IXGBE_CORE_LOCK(adapter); 1443 ixv_stop_locked(adapter); 1444 IXGBE_CORE_UNLOCK(adapter); 1445 1446 workqueue_wait(adapter->admin_wq, &adapter->admin_wc); 1447 atomic_store_relaxed(&adapter->admin_pending, 0); 1448 workqueue_wait(adapter->timer_wq, &adapter->timer_wc); 1449 atomic_store_relaxed(&adapter->timer_pending, 0); 1450 } 1451 1452 static void 1453 ixv_stop_locked(void *arg) 1454 { 1455 struct ifnet *ifp; 1456 struct adapter *adapter = arg; 1457 struct ixgbe_hw *hw = &adapter->hw; 1458 1459 ifp = adapter->ifp; 1460 1461 KASSERT(mutex_owned(&adapter->core_mtx)); 1462 1463 INIT_DEBUGOUT("ixv_stop_locked: begin\n"); 1464 ixv_disable_intr(adapter); 1465 1466 /* Tell the stack that the interface is no longer active */ 1467 ifp->if_flags &= ~(IFF_RUNNING | IFF_OACTIVE); 1468 1469 hw->mac.ops.reset_hw(hw); 1470 adapter->hw.adapter_stopped = FALSE; 1471 hw->mac.ops.stop_adapter(hw); 1472 callout_stop(&adapter->timer); 1473 1474 /* Don't schedule workqueues. */ 1475 adapter->schedule_wqs_ok = false; 1476 1477 /* reprogram the RAR[0] in case user changed it. */ 1478 hw->mac.ops.set_rar(hw, 0, hw->mac.addr, 0, IXGBE_RAH_AV); 1479 1480 return; 1481 } /* ixv_stop_locked */ 1482 1483 1484 /************************************************************************ 1485 * ixv_allocate_pci_resources 1486 ************************************************************************/ 1487 static int 1488 ixv_allocate_pci_resources(struct adapter *adapter, 1489 const struct pci_attach_args *pa) 1490 { 1491 pcireg_t memtype, csr; 1492 device_t dev = adapter->dev; 1493 bus_addr_t addr; 1494 int flags; 1495 1496 memtype = pci_mapreg_type(pa->pa_pc, pa->pa_tag, PCI_BAR(0)); 1497 switch (memtype) { 1498 case PCI_MAPREG_TYPE_MEM | PCI_MAPREG_MEM_TYPE_32BIT: 1499 case PCI_MAPREG_TYPE_MEM | PCI_MAPREG_MEM_TYPE_64BIT: 1500 adapter->osdep.mem_bus_space_tag = pa->pa_memt; 1501 if (pci_mapreg_info(pa->pa_pc, pa->pa_tag, PCI_BAR(0), 1502 memtype, &addr, &adapter->osdep.mem_size, &flags) != 0) 1503 goto map_err; 1504 if ((flags & BUS_SPACE_MAP_PREFETCHABLE) != 0) { 1505 aprint_normal_dev(dev, "clearing prefetchable bit\n"); 1506 flags &= ~BUS_SPACE_MAP_PREFETCHABLE; 1507 } 1508 if (bus_space_map(adapter->osdep.mem_bus_space_tag, addr, 1509 adapter->osdep.mem_size, flags, 1510 &adapter->osdep.mem_bus_space_handle) != 0) { 1511 map_err: 1512 adapter->osdep.mem_size = 0; 1513 aprint_error_dev(dev, "unable to map BAR0\n"); 1514 return ENXIO; 1515 } 1516 /* 1517 * Enable address decoding for memory range in case it's not 1518 * set. 1519 */ 1520 csr = pci_conf_read(pa->pa_pc, pa->pa_tag, 1521 PCI_COMMAND_STATUS_REG); 1522 csr |= PCI_COMMAND_MEM_ENABLE; 1523 pci_conf_write(pa->pa_pc, pa->pa_tag, PCI_COMMAND_STATUS_REG, 1524 csr); 1525 break; 1526 default: 1527 aprint_error_dev(dev, "unexpected type on BAR0\n"); 1528 return ENXIO; 1529 } 1530 1531 /* Pick up the tuneable queues */ 1532 adapter->num_queues = ixv_num_queues; 1533 1534 return (0); 1535 } /* ixv_allocate_pci_resources */ 1536 1537 static void 1538 ixv_free_deferred_handlers(struct adapter *adapter) 1539 { 1540 struct ix_queue *que = adapter->queues; 1541 struct tx_ring *txr = adapter->tx_rings; 1542 int i; 1543 1544 for (i = 0; i < adapter->num_queues; i++, que++, txr++) { 1545 if (!(adapter->feat_en & IXGBE_FEATURE_LEGACY_TX)) { 1546 if (txr->txr_si != NULL) 1547 softint_disestablish(txr->txr_si); 1548 } 1549 if (que->que_si != NULL) 1550 softint_disestablish(que->que_si); 1551 } 1552 if (adapter->txr_wq != NULL) 1553 workqueue_destroy(adapter->txr_wq); 1554 if (adapter->txr_wq_enqueued != NULL) 1555 percpu_free(adapter->txr_wq_enqueued, sizeof(u_int)); 1556 if (adapter->que_wq != NULL) 1557 workqueue_destroy(adapter->que_wq); 1558 1559 /* Drain the Mailbox(link) queue */ 1560 if (adapter->admin_wq != NULL) { 1561 workqueue_destroy(adapter->admin_wq); 1562 adapter->admin_wq = NULL; 1563 } 1564 if (adapter->timer_wq != NULL) { 1565 workqueue_destroy(adapter->timer_wq); 1566 adapter->timer_wq = NULL; 1567 } 1568 } /* ixv_free_deferred_handlers */ 1569 1570 /************************************************************************ 1571 * ixv_free_pci_resources 1572 ************************************************************************/ 1573 static void 1574 ixv_free_pci_resources(struct adapter * adapter) 1575 { 1576 struct ix_queue *que = adapter->queues; 1577 int rid; 1578 1579 /* 1580 * Release all msix queue resources: 1581 */ 1582 for (int i = 0; i < adapter->num_queues; i++, que++) { 1583 if (que->res != NULL) 1584 pci_intr_disestablish(adapter->osdep.pc, 1585 adapter->osdep.ihs[i]); 1586 } 1587 1588 1589 /* Clean the Mailbox interrupt last */ 1590 rid = adapter->vector; 1591 1592 if (adapter->osdep.ihs[rid] != NULL) { 1593 pci_intr_disestablish(adapter->osdep.pc, 1594 adapter->osdep.ihs[rid]); 1595 adapter->osdep.ihs[rid] = NULL; 1596 } 1597 1598 pci_intr_release(adapter->osdep.pc, adapter->osdep.intrs, 1599 adapter->osdep.nintrs); 1600 1601 if (adapter->osdep.mem_size != 0) { 1602 bus_space_unmap(adapter->osdep.mem_bus_space_tag, 1603 adapter->osdep.mem_bus_space_handle, 1604 adapter->osdep.mem_size); 1605 } 1606 1607 return; 1608 } /* ixv_free_pci_resources */ 1609 1610 /************************************************************************ 1611 * ixv_setup_interface 1612 * 1613 * Setup networking device structure and register an interface. 1614 ************************************************************************/ 1615 static int 1616 ixv_setup_interface(device_t dev, struct adapter *adapter) 1617 { 1618 struct ethercom *ec = &adapter->osdep.ec; 1619 struct ifnet *ifp; 1620 1621 INIT_DEBUGOUT("ixv_setup_interface: begin"); 1622 1623 ifp = adapter->ifp = &ec->ec_if; 1624 strlcpy(ifp->if_xname, device_xname(dev), IFNAMSIZ); 1625 ifp->if_baudrate = IF_Gbps(10); 1626 ifp->if_init = ixv_init; 1627 ifp->if_stop = ixv_ifstop; 1628 ifp->if_softc = adapter; 1629 ifp->if_flags = IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST; 1630 #ifdef IXGBE_MPSAFE 1631 ifp->if_extflags = IFEF_MPSAFE; 1632 #endif 1633 ifp->if_ioctl = ixv_ioctl; 1634 if (adapter->feat_en & IXGBE_FEATURE_LEGACY_TX) { 1635 #if 0 1636 ixv_start_locked = ixgbe_legacy_start_locked; 1637 #endif 1638 } else { 1639 ifp->if_transmit = ixgbe_mq_start; 1640 #if 0 1641 ixv_start_locked = ixgbe_mq_start_locked; 1642 #endif 1643 } 1644 ifp->if_start = ixgbe_legacy_start; 1645 IFQ_SET_MAXLEN(&ifp->if_snd, adapter->num_tx_desc - 2); 1646 IFQ_SET_READY(&ifp->if_snd); 1647 1648 if_initialize(ifp); 1649 adapter->ipq = if_percpuq_create(&adapter->osdep.ec.ec_if); 1650 ether_ifattach(ifp, adapter->hw.mac.addr); 1651 aprint_normal_dev(dev, "Ethernet address %s\n", 1652 ether_sprintf(adapter->hw.mac.addr)); 1653 /* 1654 * We use per TX queue softint, so if_deferred_start_init() isn't 1655 * used. 1656 */ 1657 ether_set_ifflags_cb(ec, ixv_ifflags_cb); 1658 1659 adapter->max_frame_size = ifp->if_mtu + IXGBE_MTU_HDR; 1660 1661 /* 1662 * Tell the upper layer(s) we support long frames. 1663 */ 1664 ifp->if_hdrlen = sizeof(struct ether_vlan_header); 1665 1666 /* Set capability flags */ 1667 ifp->if_capabilities |= IFCAP_HWCSUM 1668 | IFCAP_TSOv4 1669 | IFCAP_TSOv6; 1670 ifp->if_capenable = 0; 1671 1672 ec->ec_capabilities |= ETHERCAP_VLAN_HWFILTER 1673 | ETHERCAP_VLAN_HWTAGGING 1674 | ETHERCAP_VLAN_HWCSUM 1675 | ETHERCAP_JUMBO_MTU 1676 | ETHERCAP_VLAN_MTU; 1677 1678 /* Enable the above capabilities by default */ 1679 ec->ec_capenable = ec->ec_capabilities; 1680 1681 /* Don't enable LRO by default */ 1682 #if 0 1683 /* NetBSD doesn't support LRO yet */ 1684 ifp->if_capabilities |= IFCAP_LRO; 1685 #endif 1686 1687 /* 1688 * Specify the media types supported by this adapter and register 1689 * callbacks to update media and link information 1690 */ 1691 ec->ec_ifmedia = &adapter->media; 1692 ifmedia_init_with_lock(&adapter->media, IFM_IMASK, ixv_media_change, 1693 ixv_media_status, &adapter->core_mtx); 1694 ifmedia_add(&adapter->media, IFM_ETHER | IFM_AUTO, 0, NULL); 1695 ifmedia_set(&adapter->media, IFM_ETHER | IFM_AUTO); 1696 1697 if_register(ifp); 1698 1699 return 0; 1700 } /* ixv_setup_interface */ 1701 1702 1703 /************************************************************************ 1704 * ixv_initialize_transmit_units - Enable transmit unit. 1705 ************************************************************************/ 1706 static void 1707 ixv_initialize_transmit_units(struct adapter *adapter) 1708 { 1709 struct tx_ring *txr = adapter->tx_rings; 1710 struct ixgbe_hw *hw = &adapter->hw; 1711 int i; 1712 1713 for (i = 0; i < adapter->num_queues; i++, txr++) { 1714 u64 tdba = txr->txdma.dma_paddr; 1715 u32 txctrl, txdctl; 1716 int j = txr->me; 1717 1718 /* Set WTHRESH to 8, burst writeback */ 1719 txdctl = IXGBE_READ_REG(hw, IXGBE_VFTXDCTL(j)); 1720 txdctl |= IXGBE_TX_WTHRESH << IXGBE_TXDCTL_WTHRESH_SHIFT; 1721 IXGBE_WRITE_REG(hw, IXGBE_VFTXDCTL(j), txdctl); 1722 1723 /* Set the HW Tx Head and Tail indices */ 1724 IXGBE_WRITE_REG(&adapter->hw, IXGBE_VFTDH(j), 0); 1725 IXGBE_WRITE_REG(&adapter->hw, IXGBE_VFTDT(j), 0); 1726 1727 /* Set Tx Tail register */ 1728 txr->tail = IXGBE_VFTDT(j); 1729 1730 txr->txr_no_space = false; 1731 1732 /* Set Ring parameters */ 1733 IXGBE_WRITE_REG(hw, IXGBE_VFTDBAL(j), 1734 (tdba & 0x00000000ffffffffULL)); 1735 IXGBE_WRITE_REG(hw, IXGBE_VFTDBAH(j), (tdba >> 32)); 1736 IXGBE_WRITE_REG(hw, IXGBE_VFTDLEN(j), 1737 adapter->num_tx_desc * sizeof(struct ixgbe_legacy_tx_desc)); 1738 txctrl = IXGBE_READ_REG(hw, IXGBE_VFDCA_TXCTRL(j)); 1739 txctrl &= ~IXGBE_DCA_TXCTRL_DESC_WRO_EN; 1740 IXGBE_WRITE_REG(hw, IXGBE_VFDCA_TXCTRL(j), txctrl); 1741 1742 /* Now enable */ 1743 txdctl = IXGBE_READ_REG(hw, IXGBE_VFTXDCTL(j)); 1744 txdctl |= IXGBE_TXDCTL_ENABLE; 1745 IXGBE_WRITE_REG(hw, IXGBE_VFTXDCTL(j), txdctl); 1746 } 1747 1748 return; 1749 } /* ixv_initialize_transmit_units */ 1750 1751 1752 /************************************************************************ 1753 * ixv_initialize_rss_mapping 1754 ************************************************************************/ 1755 static void 1756 ixv_initialize_rss_mapping(struct adapter *adapter) 1757 { 1758 struct ixgbe_hw *hw = &adapter->hw; 1759 u32 reta = 0, mrqc, rss_key[10]; 1760 int queue_id; 1761 int i, j; 1762 u32 rss_hash_config; 1763 1764 /* force use default RSS key. */ 1765 #ifdef __NetBSD__ 1766 rss_getkey((uint8_t *) &rss_key); 1767 #else 1768 if (adapter->feat_en & IXGBE_FEATURE_RSS) { 1769 /* Fetch the configured RSS key */ 1770 rss_getkey((uint8_t *)&rss_key); 1771 } else { 1772 /* set up random bits */ 1773 cprng_fast(&rss_key, sizeof(rss_key)); 1774 } 1775 #endif 1776 1777 /* Now fill out hash function seeds */ 1778 for (i = 0; i < 10; i++) 1779 IXGBE_WRITE_REG(hw, IXGBE_VFRSSRK(i), rss_key[i]); 1780 1781 /* Set up the redirection table */ 1782 for (i = 0, j = 0; i < 64; i++, j++) { 1783 if (j == adapter->num_queues) 1784 j = 0; 1785 1786 if (adapter->feat_en & IXGBE_FEATURE_RSS) { 1787 /* 1788 * Fetch the RSS bucket id for the given indirection 1789 * entry. Cap it at the number of configured buckets 1790 * (which is num_queues.) 1791 */ 1792 queue_id = rss_get_indirection_to_bucket(i); 1793 queue_id = queue_id % adapter->num_queues; 1794 } else 1795 queue_id = j; 1796 1797 /* 1798 * The low 8 bits are for hash value (n+0); 1799 * The next 8 bits are for hash value (n+1), etc. 1800 */ 1801 reta >>= 8; 1802 reta |= ((uint32_t)queue_id) << 24; 1803 if ((i & 3) == 3) { 1804 IXGBE_WRITE_REG(hw, IXGBE_VFRETA(i >> 2), reta); 1805 reta = 0; 1806 } 1807 } 1808 1809 /* Perform hash on these packet types */ 1810 if (adapter->feat_en & IXGBE_FEATURE_RSS) 1811 rss_hash_config = rss_gethashconfig(); 1812 else { 1813 /* 1814 * Disable UDP - IP fragments aren't currently being handled 1815 * and so we end up with a mix of 2-tuple and 4-tuple 1816 * traffic. 1817 */ 1818 rss_hash_config = RSS_HASHTYPE_RSS_IPV4 1819 | RSS_HASHTYPE_RSS_TCP_IPV4 1820 | RSS_HASHTYPE_RSS_IPV6 1821 | RSS_HASHTYPE_RSS_TCP_IPV6; 1822 } 1823 1824 mrqc = IXGBE_MRQC_RSSEN; 1825 if (rss_hash_config & RSS_HASHTYPE_RSS_IPV4) 1826 mrqc |= IXGBE_MRQC_RSS_FIELD_IPV4; 1827 if (rss_hash_config & RSS_HASHTYPE_RSS_TCP_IPV4) 1828 mrqc |= IXGBE_MRQC_RSS_FIELD_IPV4_TCP; 1829 if (rss_hash_config & RSS_HASHTYPE_RSS_IPV6) 1830 mrqc |= IXGBE_MRQC_RSS_FIELD_IPV6; 1831 if (rss_hash_config & RSS_HASHTYPE_RSS_TCP_IPV6) 1832 mrqc |= IXGBE_MRQC_RSS_FIELD_IPV6_TCP; 1833 if (rss_hash_config & RSS_HASHTYPE_RSS_IPV6_EX) 1834 device_printf(adapter->dev, "%s: RSS_HASHTYPE_RSS_IPV6_EX defined, but not supported\n", 1835 __func__); 1836 if (rss_hash_config & RSS_HASHTYPE_RSS_TCP_IPV6_EX) 1837 device_printf(adapter->dev, "%s: RSS_HASHTYPE_RSS_TCP_IPV6_EX defined, but not supported\n", 1838 __func__); 1839 if (rss_hash_config & RSS_HASHTYPE_RSS_UDP_IPV4) 1840 mrqc |= IXGBE_MRQC_RSS_FIELD_IPV4_UDP; 1841 if (rss_hash_config & RSS_HASHTYPE_RSS_UDP_IPV6) 1842 mrqc |= IXGBE_MRQC_RSS_FIELD_IPV6_UDP; 1843 if (rss_hash_config & RSS_HASHTYPE_RSS_UDP_IPV6_EX) 1844 device_printf(adapter->dev, "%s: RSS_HASHTYPE_RSS_UDP_IPV6_EX defined, but not supported\n", 1845 __func__); 1846 IXGBE_WRITE_REG(hw, IXGBE_VFMRQC, mrqc); 1847 } /* ixv_initialize_rss_mapping */ 1848 1849 1850 /************************************************************************ 1851 * ixv_initialize_receive_units - Setup receive registers and features. 1852 ************************************************************************/ 1853 static void 1854 ixv_initialize_receive_units(struct adapter *adapter) 1855 { 1856 struct rx_ring *rxr = adapter->rx_rings; 1857 struct ixgbe_hw *hw = &adapter->hw; 1858 struct ifnet *ifp = adapter->ifp; 1859 u32 bufsz, psrtype; 1860 1861 if (ifp->if_mtu > ETHERMTU) 1862 bufsz = 4096 >> IXGBE_SRRCTL_BSIZEPKT_SHIFT; 1863 else 1864 bufsz = 2048 >> IXGBE_SRRCTL_BSIZEPKT_SHIFT; 1865 1866 psrtype = IXGBE_PSRTYPE_TCPHDR 1867 | IXGBE_PSRTYPE_UDPHDR 1868 | IXGBE_PSRTYPE_IPV4HDR 1869 | IXGBE_PSRTYPE_IPV6HDR 1870 | IXGBE_PSRTYPE_L2HDR; 1871 1872 if (adapter->num_queues > 1) 1873 psrtype |= 1 << 29; 1874 1875 IXGBE_WRITE_REG(hw, IXGBE_VFPSRTYPE, psrtype); 1876 1877 /* Tell PF our max_frame size */ 1878 if (ixgbevf_rlpml_set_vf(hw, adapter->max_frame_size) != 0) { 1879 device_printf(adapter->dev, "There is a problem with the PF setup. It is likely the receive unit for this VF will not function correctly.\n"); 1880 } 1881 1882 for (int i = 0; i < adapter->num_queues; i++, rxr++) { 1883 u64 rdba = rxr->rxdma.dma_paddr; 1884 u32 reg, rxdctl; 1885 int j = rxr->me; 1886 1887 /* Disable the queue */ 1888 rxdctl = IXGBE_READ_REG(hw, IXGBE_VFRXDCTL(j)); 1889 rxdctl &= ~IXGBE_RXDCTL_ENABLE; 1890 IXGBE_WRITE_REG(hw, IXGBE_VFRXDCTL(j), rxdctl); 1891 for (int k = 0; k < 10; k++) { 1892 if (IXGBE_READ_REG(hw, IXGBE_VFRXDCTL(j)) & 1893 IXGBE_RXDCTL_ENABLE) 1894 msec_delay(1); 1895 else 1896 break; 1897 } 1898 IXGBE_WRITE_BARRIER(hw); 1899 /* Setup the Base and Length of the Rx Descriptor Ring */ 1900 IXGBE_WRITE_REG(hw, IXGBE_VFRDBAL(j), 1901 (rdba & 0x00000000ffffffffULL)); 1902 IXGBE_WRITE_REG(hw, IXGBE_VFRDBAH(j), (rdba >> 32)); 1903 IXGBE_WRITE_REG(hw, IXGBE_VFRDLEN(j), 1904 adapter->num_rx_desc * sizeof(union ixgbe_adv_rx_desc)); 1905 1906 /* Reset the ring indices */ 1907 IXGBE_WRITE_REG(hw, IXGBE_VFRDH(rxr->me), 0); 1908 IXGBE_WRITE_REG(hw, IXGBE_VFRDT(rxr->me), 0); 1909 1910 /* Set up the SRRCTL register */ 1911 reg = IXGBE_READ_REG(hw, IXGBE_VFSRRCTL(j)); 1912 reg &= ~IXGBE_SRRCTL_BSIZEHDR_MASK; 1913 reg &= ~IXGBE_SRRCTL_BSIZEPKT_MASK; 1914 reg |= bufsz; 1915 reg |= IXGBE_SRRCTL_DESCTYPE_ADV_ONEBUF; 1916 IXGBE_WRITE_REG(hw, IXGBE_VFSRRCTL(j), reg); 1917 1918 /* Capture Rx Tail index */ 1919 rxr->tail = IXGBE_VFRDT(rxr->me); 1920 1921 /* Do the queue enabling last */ 1922 rxdctl |= IXGBE_RXDCTL_ENABLE | IXGBE_RXDCTL_VME; 1923 IXGBE_WRITE_REG(hw, IXGBE_VFRXDCTL(j), rxdctl); 1924 for (int k = 0; k < 10; k++) { 1925 if (IXGBE_READ_REG(hw, IXGBE_VFRXDCTL(j)) & 1926 IXGBE_RXDCTL_ENABLE) 1927 break; 1928 msec_delay(1); 1929 } 1930 IXGBE_WRITE_BARRIER(hw); 1931 1932 /* Set the Tail Pointer */ 1933 #ifdef DEV_NETMAP 1934 /* 1935 * In netmap mode, we must preserve the buffers made 1936 * available to userspace before the if_init() 1937 * (this is true by default on the TX side, because 1938 * init makes all buffers available to userspace). 1939 * 1940 * netmap_reset() and the device specific routines 1941 * (e.g. ixgbe_setup_receive_rings()) map these 1942 * buffers at the end of the NIC ring, so here we 1943 * must set the RDT (tail) register to make sure 1944 * they are not overwritten. 1945 * 1946 * In this driver the NIC ring starts at RDH = 0, 1947 * RDT points to the last slot available for reception (?), 1948 * so RDT = num_rx_desc - 1 means the whole ring is available. 1949 */ 1950 if ((adapter->feat_en & IXGBE_FEATURE_NETMAP) && 1951 (ifp->if_capenable & IFCAP_NETMAP)) { 1952 struct netmap_adapter *na = NA(adapter->ifp); 1953 struct netmap_kring *kring = na->rx_rings[i]; 1954 int t = na->num_rx_desc - 1 - nm_kr_rxspace(kring); 1955 1956 IXGBE_WRITE_REG(hw, IXGBE_VFRDT(rxr->me), t); 1957 } else 1958 #endif /* DEV_NETMAP */ 1959 IXGBE_WRITE_REG(hw, IXGBE_VFRDT(rxr->me), 1960 adapter->num_rx_desc - 1); 1961 } 1962 1963 if (adapter->hw.mac.type >= ixgbe_mac_X550_vf) 1964 ixv_initialize_rss_mapping(adapter); 1965 } /* ixv_initialize_receive_units */ 1966 1967 /************************************************************************ 1968 * ixv_sysctl_tdh_handler - Transmit Descriptor Head handler function 1969 * 1970 * Retrieves the TDH value from the hardware 1971 ************************************************************************/ 1972 static int 1973 ixv_sysctl_tdh_handler(SYSCTLFN_ARGS) 1974 { 1975 struct sysctlnode node = *rnode; 1976 struct tx_ring *txr = (struct tx_ring *)node.sysctl_data; 1977 uint32_t val; 1978 1979 if (!txr) 1980 return (0); 1981 1982 val = IXGBE_READ_REG(&txr->adapter->hw, IXGBE_VFTDH(txr->me)); 1983 node.sysctl_data = &val; 1984 return sysctl_lookup(SYSCTLFN_CALL(&node)); 1985 } /* ixv_sysctl_tdh_handler */ 1986 1987 /************************************************************************ 1988 * ixgbe_sysctl_tdt_handler - Transmit Descriptor Tail handler function 1989 * 1990 * Retrieves the TDT value from the hardware 1991 ************************************************************************/ 1992 static int 1993 ixv_sysctl_tdt_handler(SYSCTLFN_ARGS) 1994 { 1995 struct sysctlnode node = *rnode; 1996 struct tx_ring *txr = (struct tx_ring *)node.sysctl_data; 1997 uint32_t val; 1998 1999 if (!txr) 2000 return (0); 2001 2002 val = IXGBE_READ_REG(&txr->adapter->hw, IXGBE_VFTDT(txr->me)); 2003 node.sysctl_data = &val; 2004 return sysctl_lookup(SYSCTLFN_CALL(&node)); 2005 } /* ixv_sysctl_tdt_handler */ 2006 2007 /************************************************************************ 2008 * ixv_sysctl_next_to_check_handler - Receive Descriptor next to check 2009 * handler function 2010 * 2011 * Retrieves the next_to_check value 2012 ************************************************************************/ 2013 static int 2014 ixv_sysctl_next_to_check_handler(SYSCTLFN_ARGS) 2015 { 2016 struct sysctlnode node = *rnode; 2017 struct rx_ring *rxr = (struct rx_ring *)node.sysctl_data; 2018 uint32_t val; 2019 2020 if (!rxr) 2021 return (0); 2022 2023 val = rxr->next_to_check; 2024 node.sysctl_data = &val; 2025 return sysctl_lookup(SYSCTLFN_CALL(&node)); 2026 } /* ixv_sysctl_next_to_check_handler */ 2027 2028 /************************************************************************ 2029 * ixv_sysctl_next_to_refresh_handler - Receive Descriptor next to refresh 2030 * handler function 2031 * 2032 * Retrieves the next_to_refresh value 2033 ************************************************************************/ 2034 static int 2035 ixv_sysctl_next_to_refresh_handler(SYSCTLFN_ARGS) 2036 { 2037 struct sysctlnode node = *rnode; 2038 struct rx_ring *rxr = (struct rx_ring *)node.sysctl_data; 2039 struct adapter *adapter; 2040 uint32_t val; 2041 2042 if (!rxr) 2043 return (0); 2044 2045 adapter = rxr->adapter; 2046 if (ixgbe_fw_recovery_mode_swflag(adapter)) 2047 return (EPERM); 2048 2049 val = rxr->next_to_refresh; 2050 node.sysctl_data = &val; 2051 return sysctl_lookup(SYSCTLFN_CALL(&node)); 2052 } /* ixv_sysctl_next_to_refresh_handler */ 2053 2054 /************************************************************************ 2055 * ixv_sysctl_rdh_handler - Receive Descriptor Head handler function 2056 * 2057 * Retrieves the RDH value from the hardware 2058 ************************************************************************/ 2059 static int 2060 ixv_sysctl_rdh_handler(SYSCTLFN_ARGS) 2061 { 2062 struct sysctlnode node = *rnode; 2063 struct rx_ring *rxr = (struct rx_ring *)node.sysctl_data; 2064 uint32_t val; 2065 2066 if (!rxr) 2067 return (0); 2068 2069 val = IXGBE_READ_REG(&rxr->adapter->hw, IXGBE_VFRDH(rxr->me)); 2070 node.sysctl_data = &val; 2071 return sysctl_lookup(SYSCTLFN_CALL(&node)); 2072 } /* ixv_sysctl_rdh_handler */ 2073 2074 /************************************************************************ 2075 * ixv_sysctl_rdt_handler - Receive Descriptor Tail handler function 2076 * 2077 * Retrieves the RDT value from the hardware 2078 ************************************************************************/ 2079 static int 2080 ixv_sysctl_rdt_handler(SYSCTLFN_ARGS) 2081 { 2082 struct sysctlnode node = *rnode; 2083 struct rx_ring *rxr = (struct rx_ring *)node.sysctl_data; 2084 uint32_t val; 2085 2086 if (!rxr) 2087 return (0); 2088 2089 val = IXGBE_READ_REG(&rxr->adapter->hw, IXGBE_VFRDT(rxr->me)); 2090 node.sysctl_data = &val; 2091 return sysctl_lookup(SYSCTLFN_CALL(&node)); 2092 } /* ixv_sysctl_rdt_handler */ 2093 2094 static void 2095 ixv_setup_vlan_tagging(struct adapter *adapter) 2096 { 2097 struct ethercom *ec = &adapter->osdep.ec; 2098 struct ixgbe_hw *hw = &adapter->hw; 2099 struct rx_ring *rxr; 2100 u32 ctrl; 2101 int i; 2102 bool hwtagging; 2103 2104 /* Enable HW tagging only if any vlan is attached */ 2105 hwtagging = (ec->ec_capenable & ETHERCAP_VLAN_HWTAGGING) 2106 && VLAN_ATTACHED(ec); 2107 2108 /* Enable the queues */ 2109 for (i = 0; i < adapter->num_queues; i++) { 2110 rxr = &adapter->rx_rings[i]; 2111 ctrl = IXGBE_READ_REG(hw, IXGBE_VFRXDCTL(rxr->me)); 2112 if (hwtagging) 2113 ctrl |= IXGBE_RXDCTL_VME; 2114 else 2115 ctrl &= ~IXGBE_RXDCTL_VME; 2116 IXGBE_WRITE_REG(hw, IXGBE_VFRXDCTL(rxr->me), ctrl); 2117 /* 2118 * Let Rx path know that it needs to store VLAN tag 2119 * as part of extra mbuf info. 2120 */ 2121 rxr->vtag_strip = hwtagging ? TRUE : FALSE; 2122 } 2123 } /* ixv_setup_vlan_tagging */ 2124 2125 /************************************************************************ 2126 * ixv_setup_vlan_support 2127 ************************************************************************/ 2128 static int 2129 ixv_setup_vlan_support(struct adapter *adapter) 2130 { 2131 struct ethercom *ec = &adapter->osdep.ec; 2132 struct ixgbe_hw *hw = &adapter->hw; 2133 u32 vid, vfta, retry; 2134 struct vlanid_list *vlanidp; 2135 int rv, error = 0; 2136 2137 /* 2138 * This function is called from both if_init and ifflags_cb() 2139 * on NetBSD. 2140 */ 2141 2142 /* 2143 * Part 1: 2144 * Setup VLAN HW tagging 2145 */ 2146 ixv_setup_vlan_tagging(adapter); 2147 2148 if (!VLAN_ATTACHED(ec)) 2149 return 0; 2150 2151 /* 2152 * Part 2: 2153 * Setup VLAN HW filter 2154 */ 2155 /* Cleanup shadow_vfta */ 2156 for (int i = 0; i < IXGBE_VFTA_SIZE; i++) 2157 adapter->shadow_vfta[i] = 0; 2158 /* Generate shadow_vfta from ec_vids */ 2159 ETHER_LOCK(ec); 2160 SIMPLEQ_FOREACH(vlanidp, &ec->ec_vids, vid_list) { 2161 uint32_t idx; 2162 2163 idx = vlanidp->vid / 32; 2164 KASSERT(idx < IXGBE_VFTA_SIZE); 2165 adapter->shadow_vfta[idx] |= (u32)1 << (vlanidp->vid % 32); 2166 } 2167 ETHER_UNLOCK(ec); 2168 2169 /* 2170 * A soft reset zero's out the VFTA, so 2171 * we need to repopulate it now. 2172 */ 2173 for (int i = 0; i < IXGBE_VFTA_SIZE; i++) { 2174 if (adapter->shadow_vfta[i] == 0) 2175 continue; 2176 vfta = adapter->shadow_vfta[i]; 2177 /* 2178 * Reconstruct the vlan id's 2179 * based on the bits set in each 2180 * of the array ints. 2181 */ 2182 for (int j = 0; j < 32; j++) { 2183 retry = 0; 2184 if ((vfta & ((u32)1 << j)) == 0) 2185 continue; 2186 vid = (i * 32) + j; 2187 2188 /* Call the shared code mailbox routine */ 2189 while ((rv = hw->mac.ops.set_vfta(hw, vid, 0, TRUE, 2190 FALSE)) != 0) { 2191 if (++retry > 5) { 2192 device_printf(adapter->dev, 2193 "%s: max retry exceeded\n", 2194 __func__); 2195 break; 2196 } 2197 } 2198 if (rv != 0) { 2199 device_printf(adapter->dev, 2200 "failed to set vlan %d\n", vid); 2201 error = EACCES; 2202 } 2203 } 2204 } 2205 return error; 2206 } /* ixv_setup_vlan_support */ 2207 2208 static int 2209 ixv_vlan_cb(struct ethercom *ec, uint16_t vid, bool set) 2210 { 2211 struct ifnet *ifp = &ec->ec_if; 2212 struct adapter *adapter = ifp->if_softc; 2213 int rv; 2214 2215 if (set) 2216 rv = ixv_register_vlan(adapter, vid); 2217 else 2218 rv = ixv_unregister_vlan(adapter, vid); 2219 2220 if (rv != 0) 2221 return rv; 2222 2223 /* 2224 * Control VLAN HW tagging when ec_nvlan is changed from 1 to 0 2225 * or 0 to 1. 2226 */ 2227 if ((set && (ec->ec_nvlans == 1)) || (!set && (ec->ec_nvlans == 0))) 2228 ixv_setup_vlan_tagging(adapter); 2229 2230 return rv; 2231 } 2232 2233 /************************************************************************ 2234 * ixv_register_vlan 2235 * 2236 * Run via a vlan config EVENT, it enables us to use the 2237 * HW Filter table since we can get the vlan id. This just 2238 * creates the entry in the soft version of the VFTA, init 2239 * will repopulate the real table. 2240 ************************************************************************/ 2241 static int 2242 ixv_register_vlan(struct adapter *adapter, u16 vtag) 2243 { 2244 struct ixgbe_hw *hw = &adapter->hw; 2245 u16 index, bit; 2246 int error; 2247 2248 if ((vtag == 0) || (vtag > 4095)) /* Invalid */ 2249 return EINVAL; 2250 IXGBE_CORE_LOCK(adapter); 2251 index = (vtag >> 5) & 0x7F; 2252 bit = vtag & 0x1F; 2253 adapter->shadow_vfta[index] |= ((u32)1 << bit); 2254 error = hw->mac.ops.set_vfta(hw, vtag, 0, true, false); 2255 IXGBE_CORE_UNLOCK(adapter); 2256 2257 if (error != 0) { 2258 device_printf(adapter->dev, "failed to register vlan %hu\n", 2259 vtag); 2260 error = EACCES; 2261 } 2262 return error; 2263 } /* ixv_register_vlan */ 2264 2265 /************************************************************************ 2266 * ixv_unregister_vlan 2267 * 2268 * Run via a vlan unconfig EVENT, remove our entry 2269 * in the soft vfta. 2270 ************************************************************************/ 2271 static int 2272 ixv_unregister_vlan(struct adapter *adapter, u16 vtag) 2273 { 2274 struct ixgbe_hw *hw = &adapter->hw; 2275 u16 index, bit; 2276 int error; 2277 2278 if ((vtag == 0) || (vtag > 4095)) /* Invalid */ 2279 return EINVAL; 2280 2281 IXGBE_CORE_LOCK(adapter); 2282 index = (vtag >> 5) & 0x7F; 2283 bit = vtag & 0x1F; 2284 adapter->shadow_vfta[index] &= ~((u32)1 << bit); 2285 error = hw->mac.ops.set_vfta(hw, vtag, 0, false, false); 2286 IXGBE_CORE_UNLOCK(adapter); 2287 2288 if (error != 0) { 2289 device_printf(adapter->dev, "failed to unregister vlan %hu\n", 2290 vtag); 2291 error = EIO; 2292 } 2293 return error; 2294 } /* ixv_unregister_vlan */ 2295 2296 /************************************************************************ 2297 * ixv_enable_intr 2298 ************************************************************************/ 2299 static void 2300 ixv_enable_intr(struct adapter *adapter) 2301 { 2302 struct ixgbe_hw *hw = &adapter->hw; 2303 struct ix_queue *que = adapter->queues; 2304 u32 mask; 2305 int i; 2306 2307 /* For VTEIAC */ 2308 mask = (1 << adapter->vector); 2309 for (i = 0; i < adapter->num_queues; i++, que++) 2310 mask |= (1 << que->msix); 2311 IXGBE_WRITE_REG(hw, IXGBE_VTEIAC, mask); 2312 2313 /* For VTEIMS */ 2314 IXGBE_WRITE_REG(hw, IXGBE_VTEIMS, (1 << adapter->vector)); 2315 que = adapter->queues; 2316 for (i = 0; i < adapter->num_queues; i++, que++) 2317 ixv_enable_queue(adapter, que->msix); 2318 2319 IXGBE_WRITE_FLUSH(hw); 2320 } /* ixv_enable_intr */ 2321 2322 /************************************************************************ 2323 * ixv_disable_intr 2324 ************************************************************************/ 2325 static void 2326 ixv_disable_intr(struct adapter *adapter) 2327 { 2328 struct ix_queue *que = adapter->queues; 2329 2330 IXGBE_WRITE_REG(&adapter->hw, IXGBE_VTEIAC, 0); 2331 2332 /* disable interrupts other than queues */ 2333 IXGBE_WRITE_REG(&adapter->hw, IXGBE_VTEIMC, adapter->vector); 2334 2335 for (int i = 0; i < adapter->num_queues; i++, que++) 2336 ixv_disable_queue(adapter, que->msix); 2337 2338 IXGBE_WRITE_FLUSH(&adapter->hw); 2339 } /* ixv_disable_intr */ 2340 2341 /************************************************************************ 2342 * ixv_set_ivar 2343 * 2344 * Setup the correct IVAR register for a particular MSI-X interrupt 2345 * - entry is the register array entry 2346 * - vector is the MSI-X vector for this queue 2347 * - type is RX/TX/MISC 2348 ************************************************************************/ 2349 static void 2350 ixv_set_ivar(struct adapter *adapter, u8 entry, u8 vector, s8 type) 2351 { 2352 struct ixgbe_hw *hw = &adapter->hw; 2353 u32 ivar, index; 2354 2355 vector |= IXGBE_IVAR_ALLOC_VAL; 2356 2357 if (type == -1) { /* MISC IVAR */ 2358 ivar = IXGBE_READ_REG(hw, IXGBE_VTIVAR_MISC); 2359 ivar &= ~0xFF; 2360 ivar |= vector; 2361 IXGBE_WRITE_REG(hw, IXGBE_VTIVAR_MISC, ivar); 2362 } else { /* RX/TX IVARS */ 2363 index = (16 * (entry & 1)) + (8 * type); 2364 ivar = IXGBE_READ_REG(hw, IXGBE_VTIVAR(entry >> 1)); 2365 ivar &= ~(0xffUL << index); 2366 ivar |= ((u32)vector << index); 2367 IXGBE_WRITE_REG(hw, IXGBE_VTIVAR(entry >> 1), ivar); 2368 } 2369 } /* ixv_set_ivar */ 2370 2371 /************************************************************************ 2372 * ixv_configure_ivars 2373 ************************************************************************/ 2374 static void 2375 ixv_configure_ivars(struct adapter *adapter) 2376 { 2377 struct ix_queue *que = adapter->queues; 2378 2379 /* XXX We should sync EITR value calculation with ixgbe.c? */ 2380 2381 for (int i = 0; i < adapter->num_queues; i++, que++) { 2382 /* First the RX queue entry */ 2383 ixv_set_ivar(adapter, i, que->msix, 0); 2384 /* ... and the TX */ 2385 ixv_set_ivar(adapter, i, que->msix, 1); 2386 /* Set an initial value in EITR */ 2387 ixv_eitr_write(adapter, que->msix, IXGBE_EITR_DEFAULT); 2388 } 2389 2390 /* For the mailbox interrupt */ 2391 ixv_set_ivar(adapter, 1, adapter->vector, -1); 2392 } /* ixv_configure_ivars */ 2393 2394 2395 /************************************************************************ 2396 * ixv_save_stats 2397 * 2398 * The VF stats registers never have a truly virgin 2399 * starting point, so this routine tries to make an 2400 * artificial one, marking ground zero on attach as 2401 * it were. 2402 ************************************************************************/ 2403 static void 2404 ixv_save_stats(struct adapter *adapter) 2405 { 2406 struct ixgbevf_hw_stats *stats = &adapter->stats.vf; 2407 2408 if (stats->vfgprc.ev_count || stats->vfgptc.ev_count) { 2409 stats->saved_reset_vfgprc += 2410 stats->vfgprc.ev_count - stats->base_vfgprc; 2411 stats->saved_reset_vfgptc += 2412 stats->vfgptc.ev_count - stats->base_vfgptc; 2413 stats->saved_reset_vfgorc += 2414 stats->vfgorc.ev_count - stats->base_vfgorc; 2415 stats->saved_reset_vfgotc += 2416 stats->vfgotc.ev_count - stats->base_vfgotc; 2417 stats->saved_reset_vfmprc += 2418 stats->vfmprc.ev_count - stats->base_vfmprc; 2419 } 2420 } /* ixv_save_stats */ 2421 2422 /************************************************************************ 2423 * ixv_init_stats 2424 ************************************************************************/ 2425 static void 2426 ixv_init_stats(struct adapter *adapter) 2427 { 2428 struct ixgbe_hw *hw = &adapter->hw; 2429 2430 adapter->stats.vf.last_vfgprc = IXGBE_READ_REG(hw, IXGBE_VFGPRC); 2431 adapter->stats.vf.last_vfgorc = IXGBE_READ_REG(hw, IXGBE_VFGORC_LSB); 2432 adapter->stats.vf.last_vfgorc |= 2433 (((u64)(IXGBE_READ_REG(hw, IXGBE_VFGORC_MSB))) << 32); 2434 2435 adapter->stats.vf.last_vfgptc = IXGBE_READ_REG(hw, IXGBE_VFGPTC); 2436 adapter->stats.vf.last_vfgotc = IXGBE_READ_REG(hw, IXGBE_VFGOTC_LSB); 2437 adapter->stats.vf.last_vfgotc |= 2438 (((u64)(IXGBE_READ_REG(hw, IXGBE_VFGOTC_MSB))) << 32); 2439 2440 adapter->stats.vf.last_vfmprc = IXGBE_READ_REG(hw, IXGBE_VFMPRC); 2441 2442 adapter->stats.vf.base_vfgprc = adapter->stats.vf.last_vfgprc; 2443 adapter->stats.vf.base_vfgorc = adapter->stats.vf.last_vfgorc; 2444 adapter->stats.vf.base_vfgptc = adapter->stats.vf.last_vfgptc; 2445 adapter->stats.vf.base_vfgotc = adapter->stats.vf.last_vfgotc; 2446 adapter->stats.vf.base_vfmprc = adapter->stats.vf.last_vfmprc; 2447 } /* ixv_init_stats */ 2448 2449 #define UPDATE_STAT_32(reg, last, count) \ 2450 { \ 2451 u32 current = IXGBE_READ_REG(hw, (reg)); \ 2452 if (current < (last)) \ 2453 count.ev_count += 0x100000000LL; \ 2454 (last) = current; \ 2455 count.ev_count &= 0xFFFFFFFF00000000LL; \ 2456 count.ev_count |= current; \ 2457 } 2458 2459 #define UPDATE_STAT_36(lsb, msb, last, count) \ 2460 { \ 2461 u64 cur_lsb = IXGBE_READ_REG(hw, (lsb)); \ 2462 u64 cur_msb = IXGBE_READ_REG(hw, (msb)); \ 2463 u64 current = ((cur_msb << 32) | cur_lsb); \ 2464 if (current < (last)) \ 2465 count.ev_count += 0x1000000000LL; \ 2466 (last) = current; \ 2467 count.ev_count &= 0xFFFFFFF000000000LL; \ 2468 count.ev_count |= current; \ 2469 } 2470 2471 /************************************************************************ 2472 * ixv_update_stats - Update the board statistics counters. 2473 ************************************************************************/ 2474 void 2475 ixv_update_stats(struct adapter *adapter) 2476 { 2477 struct ixgbe_hw *hw = &adapter->hw; 2478 struct ixgbevf_hw_stats *stats = &adapter->stats.vf; 2479 2480 UPDATE_STAT_32(IXGBE_VFGPRC, stats->last_vfgprc, stats->vfgprc); 2481 UPDATE_STAT_32(IXGBE_VFGPTC, stats->last_vfgptc, stats->vfgptc); 2482 UPDATE_STAT_36(IXGBE_VFGORC_LSB, IXGBE_VFGORC_MSB, stats->last_vfgorc, 2483 stats->vfgorc); 2484 UPDATE_STAT_36(IXGBE_VFGOTC_LSB, IXGBE_VFGOTC_MSB, stats->last_vfgotc, 2485 stats->vfgotc); 2486 UPDATE_STAT_32(IXGBE_VFMPRC, stats->last_vfmprc, stats->vfmprc); 2487 2488 /* VF doesn't count errors by hardware */ 2489 2490 } /* ixv_update_stats */ 2491 2492 /************************************************************************ 2493 * ixv_sysctl_interrupt_rate_handler 2494 ************************************************************************/ 2495 static int 2496 ixv_sysctl_interrupt_rate_handler(SYSCTLFN_ARGS) 2497 { 2498 struct sysctlnode node = *rnode; 2499 struct ix_queue *que = (struct ix_queue *)node.sysctl_data; 2500 struct adapter *adapter = que->adapter; 2501 uint32_t reg, usec, rate; 2502 int error; 2503 2504 if (que == NULL) 2505 return 0; 2506 reg = IXGBE_READ_REG(&que->adapter->hw, IXGBE_VTEITR(que->msix)); 2507 usec = ((reg & 0x0FF8) >> 3); 2508 if (usec > 0) 2509 rate = 500000 / usec; 2510 else 2511 rate = 0; 2512 node.sysctl_data = &rate; 2513 error = sysctl_lookup(SYSCTLFN_CALL(&node)); 2514 if (error || newp == NULL) 2515 return error; 2516 reg &= ~0xfff; /* default, no limitation */ 2517 if (rate > 0 && rate < 500000) { 2518 if (rate < 1000) 2519 rate = 1000; 2520 reg |= ((4000000 / rate) & 0xff8); 2521 /* 2522 * When RSC is used, ITR interval must be larger than 2523 * RSC_DELAY. Currently, we use 2us for RSC_DELAY. 2524 * The minimum value is always greater than 2us on 100M 2525 * (and 10M?(not documented)), but it's not on 1G and higher. 2526 */ 2527 if ((adapter->link_speed != IXGBE_LINK_SPEED_100_FULL) 2528 && (adapter->link_speed != IXGBE_LINK_SPEED_10_FULL)) { 2529 if ((adapter->num_queues > 1) 2530 && (reg < IXGBE_MIN_RSC_EITR_10G1G)) 2531 return EINVAL; 2532 } 2533 ixv_max_interrupt_rate = rate; 2534 } else 2535 ixv_max_interrupt_rate = 0; 2536 ixv_eitr_write(adapter, que->msix, reg); 2537 2538 return (0); 2539 } /* ixv_sysctl_interrupt_rate_handler */ 2540 2541 const struct sysctlnode * 2542 ixv_sysctl_instance(struct adapter *adapter) 2543 { 2544 const char *dvname; 2545 struct sysctllog **log; 2546 int rc; 2547 const struct sysctlnode *rnode; 2548 2549 log = &adapter->sysctllog; 2550 dvname = device_xname(adapter->dev); 2551 2552 if ((rc = sysctl_createv(log, 0, NULL, &rnode, 2553 0, CTLTYPE_NODE, dvname, 2554 SYSCTL_DESCR("ixv information and settings"), 2555 NULL, 0, NULL, 0, CTL_HW, CTL_CREATE, CTL_EOL)) != 0) 2556 goto err; 2557 2558 return rnode; 2559 err: 2560 device_printf(adapter->dev, 2561 "%s: sysctl_createv failed, rc = %d\n", __func__, rc); 2562 return NULL; 2563 } 2564 2565 static void 2566 ixv_add_device_sysctls(struct adapter *adapter) 2567 { 2568 struct sysctllog **log; 2569 const struct sysctlnode *rnode, *cnode; 2570 device_t dev; 2571 2572 dev = adapter->dev; 2573 log = &adapter->sysctllog; 2574 2575 if ((rnode = ixv_sysctl_instance(adapter)) == NULL) { 2576 aprint_error_dev(dev, "could not create sysctl root\n"); 2577 return; 2578 } 2579 2580 if (sysctl_createv(log, 0, &rnode, &cnode, 2581 CTLFLAG_READWRITE, CTLTYPE_INT, "debug", 2582 SYSCTL_DESCR("Debug Info"), 2583 ixv_sysctl_debug, 0, (void *)adapter, 0, CTL_CREATE, CTL_EOL) != 0) 2584 aprint_error_dev(dev, "could not create sysctl\n"); 2585 2586 if (sysctl_createv(log, 0, &rnode, &cnode, 2587 CTLFLAG_READWRITE, CTLTYPE_INT, 2588 "rx_copy_len", SYSCTL_DESCR("RX Copy Length"), 2589 ixv_sysctl_rx_copy_len, 0, 2590 (void *)adapter, 0, CTL_CREATE, CTL_EOL) != 0) 2591 aprint_error_dev(dev, "could not create sysctl\n"); 2592 2593 if (sysctl_createv(log, 0, &rnode, &cnode, 2594 CTLFLAG_READWRITE, CTLTYPE_BOOL, "enable_aim", 2595 SYSCTL_DESCR("Interrupt Moderation"), 2596 NULL, 0, &adapter->enable_aim, 0, CTL_CREATE, CTL_EOL) != 0) 2597 aprint_error_dev(dev, "could not create sysctl\n"); 2598 2599 if (sysctl_createv(log, 0, &rnode, &cnode, 2600 CTLFLAG_READWRITE, CTLTYPE_BOOL, "txrx_workqueue", 2601 SYSCTL_DESCR("Use workqueue for packet processing"), 2602 NULL, 0, &adapter->txrx_use_workqueue, 0, CTL_CREATE, CTL_EOL) 2603 != 0) 2604 aprint_error_dev(dev, "could not create sysctl\n"); 2605 } 2606 2607 /************************************************************************ 2608 * ixv_add_stats_sysctls - Add statistic sysctls for the VF. 2609 ************************************************************************/ 2610 static void 2611 ixv_add_stats_sysctls(struct adapter *adapter) 2612 { 2613 device_t dev = adapter->dev; 2614 struct tx_ring *txr = adapter->tx_rings; 2615 struct rx_ring *rxr = adapter->rx_rings; 2616 struct ixgbevf_hw_stats *stats = &adapter->stats.vf; 2617 struct ixgbe_hw *hw = &adapter->hw; 2618 const struct sysctlnode *rnode, *cnode; 2619 struct sysctllog **log = &adapter->sysctllog; 2620 const char *xname = device_xname(dev); 2621 2622 /* Driver Statistics */ 2623 evcnt_attach_dynamic(&adapter->efbig_tx_dma_setup, EVCNT_TYPE_MISC, 2624 NULL, xname, "Driver tx dma soft fail EFBIG"); 2625 evcnt_attach_dynamic(&adapter->mbuf_defrag_failed, EVCNT_TYPE_MISC, 2626 NULL, xname, "m_defrag() failed"); 2627 evcnt_attach_dynamic(&adapter->efbig2_tx_dma_setup, EVCNT_TYPE_MISC, 2628 NULL, xname, "Driver tx dma hard fail EFBIG"); 2629 evcnt_attach_dynamic(&adapter->einval_tx_dma_setup, EVCNT_TYPE_MISC, 2630 NULL, xname, "Driver tx dma hard fail EINVAL"); 2631 evcnt_attach_dynamic(&adapter->other_tx_dma_setup, EVCNT_TYPE_MISC, 2632 NULL, xname, "Driver tx dma hard fail other"); 2633 evcnt_attach_dynamic(&adapter->eagain_tx_dma_setup, EVCNT_TYPE_MISC, 2634 NULL, xname, "Driver tx dma soft fail EAGAIN"); 2635 evcnt_attach_dynamic(&adapter->enomem_tx_dma_setup, EVCNT_TYPE_MISC, 2636 NULL, xname, "Driver tx dma soft fail ENOMEM"); 2637 evcnt_attach_dynamic(&adapter->watchdog_events, EVCNT_TYPE_MISC, 2638 NULL, xname, "Watchdog timeouts"); 2639 evcnt_attach_dynamic(&adapter->tso_err, EVCNT_TYPE_MISC, 2640 NULL, xname, "TSO errors"); 2641 evcnt_attach_dynamic(&adapter->admin_irqev, EVCNT_TYPE_INTR, 2642 NULL, xname, "Admin MSI-X IRQ Handled"); 2643 evcnt_attach_dynamic(&adapter->link_workev, EVCNT_TYPE_INTR, 2644 NULL, xname, "Admin event"); 2645 2646 for (int i = 0; i < adapter->num_queues; i++, rxr++, txr++) { 2647 snprintf(adapter->queues[i].evnamebuf, 2648 sizeof(adapter->queues[i].evnamebuf), "%s q%d", 2649 xname, i); 2650 snprintf(adapter->queues[i].namebuf, 2651 sizeof(adapter->queues[i].namebuf), "q%d", i); 2652 2653 if ((rnode = ixv_sysctl_instance(adapter)) == NULL) { 2654 aprint_error_dev(dev, "could not create sysctl root\n"); 2655 break; 2656 } 2657 2658 if (sysctl_createv(log, 0, &rnode, &rnode, 2659 0, CTLTYPE_NODE, 2660 adapter->queues[i].namebuf, SYSCTL_DESCR("Queue Name"), 2661 NULL, 0, NULL, 0, CTL_CREATE, CTL_EOL) != 0) 2662 break; 2663 2664 if (sysctl_createv(log, 0, &rnode, &cnode, 2665 CTLFLAG_READWRITE, CTLTYPE_INT, 2666 "interrupt_rate", SYSCTL_DESCR("Interrupt Rate"), 2667 ixv_sysctl_interrupt_rate_handler, 0, 2668 (void *)&adapter->queues[i], 0, CTL_CREATE, CTL_EOL) != 0) 2669 break; 2670 2671 if (sysctl_createv(log, 0, &rnode, &cnode, 2672 CTLFLAG_READONLY, CTLTYPE_INT, 2673 "txd_head", SYSCTL_DESCR("Transmit Descriptor Head"), 2674 ixv_sysctl_tdh_handler, 0, (void *)txr, 2675 0, CTL_CREATE, CTL_EOL) != 0) 2676 break; 2677 2678 if (sysctl_createv(log, 0, &rnode, &cnode, 2679 CTLFLAG_READONLY, CTLTYPE_INT, 2680 "txd_tail", SYSCTL_DESCR("Transmit Descriptor Tail"), 2681 ixv_sysctl_tdt_handler, 0, (void *)txr, 2682 0, CTL_CREATE, CTL_EOL) != 0) 2683 break; 2684 2685 evcnt_attach_dynamic(&adapter->queues[i].irqs, EVCNT_TYPE_INTR, 2686 NULL, adapter->queues[i].evnamebuf, "IRQs on queue"); 2687 evcnt_attach_dynamic(&adapter->queues[i].handleq, 2688 EVCNT_TYPE_MISC, NULL, adapter->queues[i].evnamebuf, 2689 "Handled queue in softint"); 2690 evcnt_attach_dynamic(&adapter->queues[i].req, EVCNT_TYPE_MISC, 2691 NULL, adapter->queues[i].evnamebuf, "Requeued in softint"); 2692 evcnt_attach_dynamic(&txr->tso_tx, EVCNT_TYPE_MISC, 2693 NULL, adapter->queues[i].evnamebuf, "TSO"); 2694 evcnt_attach_dynamic(&txr->no_desc_avail, EVCNT_TYPE_MISC, 2695 NULL, adapter->queues[i].evnamebuf, 2696 "TX Queue No Descriptor Available"); 2697 evcnt_attach_dynamic(&txr->total_packets, EVCNT_TYPE_MISC, 2698 NULL, adapter->queues[i].evnamebuf, 2699 "Queue Packets Transmitted"); 2700 #ifndef IXGBE_LEGACY_TX 2701 evcnt_attach_dynamic(&txr->pcq_drops, EVCNT_TYPE_MISC, 2702 NULL, adapter->queues[i].evnamebuf, 2703 "Packets dropped in pcq"); 2704 #endif 2705 2706 #ifdef LRO 2707 struct lro_ctrl *lro = &rxr->lro; 2708 #endif /* LRO */ 2709 2710 if (sysctl_createv(log, 0, &rnode, &cnode, 2711 CTLFLAG_READONLY, CTLTYPE_INT, "rxd_nxck", 2712 SYSCTL_DESCR("Receive Descriptor next to check"), 2713 ixv_sysctl_next_to_check_handler, 0, (void *)rxr, 0, 2714 CTL_CREATE, CTL_EOL) != 0) 2715 break; 2716 2717 if (sysctl_createv(log, 0, &rnode, &cnode, 2718 CTLFLAG_READONLY, CTLTYPE_INT, "rxd_nxrf", 2719 SYSCTL_DESCR("Receive Descriptor next to refresh"), 2720 ixv_sysctl_next_to_refresh_handler, 0, (void *)rxr, 0, 2721 CTL_CREATE, CTL_EOL) != 0) 2722 break; 2723 2724 if (sysctl_createv(log, 0, &rnode, &cnode, 2725 CTLFLAG_READONLY, CTLTYPE_INT, "rxd_head", 2726 SYSCTL_DESCR("Receive Descriptor Head"), 2727 ixv_sysctl_rdh_handler, 0, (void *)rxr, 0, 2728 CTL_CREATE, CTL_EOL) != 0) 2729 break; 2730 2731 if (sysctl_createv(log, 0, &rnode, &cnode, 2732 CTLFLAG_READONLY, CTLTYPE_INT, "rxd_tail", 2733 SYSCTL_DESCR("Receive Descriptor Tail"), 2734 ixv_sysctl_rdt_handler, 0, (void *)rxr, 0, 2735 CTL_CREATE, CTL_EOL) != 0) 2736 break; 2737 2738 evcnt_attach_dynamic(&rxr->rx_packets, EVCNT_TYPE_MISC, 2739 NULL, adapter->queues[i].evnamebuf, 2740 "Queue Packets Received"); 2741 evcnt_attach_dynamic(&rxr->rx_bytes, EVCNT_TYPE_MISC, 2742 NULL, adapter->queues[i].evnamebuf, 2743 "Queue Bytes Received"); 2744 evcnt_attach_dynamic(&rxr->rx_copies, EVCNT_TYPE_MISC, 2745 NULL, adapter->queues[i].evnamebuf, "Copied RX Frames"); 2746 evcnt_attach_dynamic(&rxr->no_mbuf, EVCNT_TYPE_MISC, 2747 NULL, adapter->queues[i].evnamebuf, "Rx no mbuf"); 2748 evcnt_attach_dynamic(&rxr->rx_discarded, EVCNT_TYPE_MISC, 2749 NULL, adapter->queues[i].evnamebuf, "Rx discarded"); 2750 #ifdef LRO 2751 SYSCTL_ADD_INT(ctx, queue_list, OID_AUTO, "lro_queued", 2752 CTLFLAG_RD, &lro->lro_queued, 0, 2753 "LRO Queued"); 2754 SYSCTL_ADD_INT(ctx, queue_list, OID_AUTO, "lro_flushed", 2755 CTLFLAG_RD, &lro->lro_flushed, 0, 2756 "LRO Flushed"); 2757 #endif /* LRO */ 2758 } 2759 2760 /* MAC stats get their own sub node */ 2761 2762 snprintf(stats->namebuf, 2763 sizeof(stats->namebuf), "%s MAC Statistics", xname); 2764 2765 evcnt_attach_dynamic(&stats->ipcs, EVCNT_TYPE_MISC, NULL, 2766 stats->namebuf, "rx csum offload - IP"); 2767 evcnt_attach_dynamic(&stats->l4cs, EVCNT_TYPE_MISC, NULL, 2768 stats->namebuf, "rx csum offload - L4"); 2769 evcnt_attach_dynamic(&stats->ipcs_bad, EVCNT_TYPE_MISC, NULL, 2770 stats->namebuf, "rx csum offload - IP bad"); 2771 evcnt_attach_dynamic(&stats->l4cs_bad, EVCNT_TYPE_MISC, NULL, 2772 stats->namebuf, "rx csum offload - L4 bad"); 2773 2774 /* Packet Reception Stats */ 2775 evcnt_attach_dynamic(&stats->vfgprc, EVCNT_TYPE_MISC, NULL, 2776 xname, "Good Packets Received"); 2777 evcnt_attach_dynamic(&stats->vfgorc, EVCNT_TYPE_MISC, NULL, 2778 xname, "Good Octets Received"); 2779 evcnt_attach_dynamic(&stats->vfmprc, EVCNT_TYPE_MISC, NULL, 2780 xname, "Multicast Packets Received"); 2781 evcnt_attach_dynamic(&stats->vfgptc, EVCNT_TYPE_MISC, NULL, 2782 xname, "Good Packets Transmitted"); 2783 evcnt_attach_dynamic(&stats->vfgotc, EVCNT_TYPE_MISC, NULL, 2784 xname, "Good Octets Transmitted"); 2785 2786 /* Mailbox Stats */ 2787 evcnt_attach_dynamic(&hw->mbx.stats.msgs_tx, EVCNT_TYPE_MISC, NULL, 2788 xname, "message TXs"); 2789 evcnt_attach_dynamic(&hw->mbx.stats.msgs_rx, EVCNT_TYPE_MISC, NULL, 2790 xname, "message RXs"); 2791 evcnt_attach_dynamic(&hw->mbx.stats.acks, EVCNT_TYPE_MISC, NULL, 2792 xname, "ACKs"); 2793 evcnt_attach_dynamic(&hw->mbx.stats.reqs, EVCNT_TYPE_MISC, NULL, 2794 xname, "REQs"); 2795 evcnt_attach_dynamic(&hw->mbx.stats.rsts, EVCNT_TYPE_MISC, NULL, 2796 xname, "RSTs"); 2797 2798 } /* ixv_add_stats_sysctls */ 2799 2800 static void 2801 ixv_clear_evcnt(struct adapter *adapter) 2802 { 2803 struct tx_ring *txr = adapter->tx_rings; 2804 struct rx_ring *rxr = adapter->rx_rings; 2805 struct ixgbevf_hw_stats *stats = &adapter->stats.vf; 2806 struct ixgbe_hw *hw = &adapter->hw; 2807 int i; 2808 2809 /* Driver Statistics */ 2810 adapter->efbig_tx_dma_setup.ev_count = 0; 2811 adapter->mbuf_defrag_failed.ev_count = 0; 2812 adapter->efbig2_tx_dma_setup.ev_count = 0; 2813 adapter->einval_tx_dma_setup.ev_count = 0; 2814 adapter->other_tx_dma_setup.ev_count = 0; 2815 adapter->eagain_tx_dma_setup.ev_count = 0; 2816 adapter->enomem_tx_dma_setup.ev_count = 0; 2817 adapter->watchdog_events.ev_count = 0; 2818 adapter->tso_err.ev_count = 0; 2819 adapter->admin_irqev.ev_count = 0; 2820 adapter->link_workev.ev_count = 0; 2821 2822 for (i = 0; i < adapter->num_queues; i++, rxr++, txr++) { 2823 adapter->queues[i].irqs.ev_count = 0; 2824 adapter->queues[i].handleq.ev_count = 0; 2825 adapter->queues[i].req.ev_count = 0; 2826 txr->tso_tx.ev_count = 0; 2827 txr->no_desc_avail.ev_count = 0; 2828 txr->total_packets.ev_count = 0; 2829 #ifndef IXGBE_LEGACY_TX 2830 txr->pcq_drops.ev_count = 0; 2831 #endif 2832 txr->q_efbig_tx_dma_setup = 0; 2833 txr->q_mbuf_defrag_failed = 0; 2834 txr->q_efbig2_tx_dma_setup = 0; 2835 txr->q_einval_tx_dma_setup = 0; 2836 txr->q_other_tx_dma_setup = 0; 2837 txr->q_eagain_tx_dma_setup = 0; 2838 txr->q_enomem_tx_dma_setup = 0; 2839 txr->q_tso_err = 0; 2840 2841 rxr->rx_packets.ev_count = 0; 2842 rxr->rx_bytes.ev_count = 0; 2843 rxr->rx_copies.ev_count = 0; 2844 rxr->no_mbuf.ev_count = 0; 2845 rxr->rx_discarded.ev_count = 0; 2846 } 2847 2848 /* MAC stats get their own sub node */ 2849 2850 stats->ipcs.ev_count = 0; 2851 stats->l4cs.ev_count = 0; 2852 stats->ipcs_bad.ev_count = 0; 2853 stats->l4cs_bad.ev_count = 0; 2854 2855 /* Packet Reception Stats */ 2856 stats->vfgprc.ev_count = 0; 2857 stats->vfgorc.ev_count = 0; 2858 stats->vfmprc.ev_count = 0; 2859 stats->vfgptc.ev_count = 0; 2860 stats->vfgotc.ev_count = 0; 2861 2862 /* Mailbox Stats */ 2863 hw->mbx.stats.msgs_tx.ev_count = 0; 2864 hw->mbx.stats.msgs_rx.ev_count = 0; 2865 hw->mbx.stats.acks.ev_count = 0; 2866 hw->mbx.stats.reqs.ev_count = 0; 2867 hw->mbx.stats.rsts.ev_count = 0; 2868 2869 } /* ixv_clear_evcnt */ 2870 2871 /************************************************************************ 2872 * ixv_set_sysctl_value 2873 ************************************************************************/ 2874 static void 2875 ixv_set_sysctl_value(struct adapter *adapter, const char *name, 2876 const char *description, int *limit, int value) 2877 { 2878 device_t dev = adapter->dev; 2879 struct sysctllog **log; 2880 const struct sysctlnode *rnode, *cnode; 2881 2882 log = &adapter->sysctllog; 2883 if ((rnode = ixv_sysctl_instance(adapter)) == NULL) { 2884 aprint_error_dev(dev, "could not create sysctl root\n"); 2885 return; 2886 } 2887 if (sysctl_createv(log, 0, &rnode, &cnode, 2888 CTLFLAG_READWRITE, CTLTYPE_INT, 2889 name, SYSCTL_DESCR(description), 2890 NULL, 0, limit, 0, CTL_CREATE, CTL_EOL) != 0) 2891 aprint_error_dev(dev, "could not create sysctl\n"); 2892 *limit = value; 2893 } /* ixv_set_sysctl_value */ 2894 2895 /************************************************************************ 2896 * ixv_print_debug_info 2897 * 2898 * Called only when em_display_debug_stats is enabled. 2899 * Provides a way to take a look at important statistics 2900 * maintained by the driver and hardware. 2901 ************************************************************************/ 2902 static void 2903 ixv_print_debug_info(struct adapter *adapter) 2904 { 2905 device_t dev = adapter->dev; 2906 struct ix_queue *que = adapter->queues; 2907 struct rx_ring *rxr; 2908 struct tx_ring *txr; 2909 #ifdef LRO 2910 struct lro_ctrl *lro; 2911 #endif /* LRO */ 2912 2913 for (int i = 0; i < adapter->num_queues; i++, que++) { 2914 txr = que->txr; 2915 rxr = que->rxr; 2916 #ifdef LRO 2917 lro = &rxr->lro; 2918 #endif /* LRO */ 2919 device_printf(dev, "QUE(%d) IRQs Handled: %lu\n", 2920 que->msix, (long)que->irqs.ev_count); 2921 device_printf(dev, "RX(%d) Packets Received: %lld\n", 2922 rxr->me, (long long)rxr->rx_packets.ev_count); 2923 device_printf(dev, "RX(%d) Bytes Received: %lu\n", 2924 rxr->me, (long)rxr->rx_bytes.ev_count); 2925 #ifdef LRO 2926 device_printf(dev, "RX(%d) LRO Queued= %ju\n", 2927 rxr->me, (uintmax_t)lro->lro_queued); 2928 device_printf(dev, "RX(%d) LRO Flushed= %ju\n", 2929 rxr->me, (uintmax_t)lro->lro_flushed); 2930 #endif /* LRO */ 2931 device_printf(dev, "TX(%d) Packets Sent: %lu\n", 2932 txr->me, (long)txr->total_packets.ev_count); 2933 device_printf(dev, "TX(%d) NO Desc Avail: %lu\n", 2934 txr->me, (long)txr->no_desc_avail.ev_count); 2935 } 2936 2937 device_printf(dev, "Admin IRQ Handled: %lu\n", 2938 (long)adapter->admin_irqev.ev_count); 2939 device_printf(dev, "Admin work Handled: %lu\n", 2940 (long)adapter->link_workev.ev_count); 2941 } /* ixv_print_debug_info */ 2942 2943 /************************************************************************ 2944 * ixv_sysctl_debug 2945 ************************************************************************/ 2946 static int 2947 ixv_sysctl_debug(SYSCTLFN_ARGS) 2948 { 2949 struct sysctlnode node = *rnode; 2950 struct adapter *adapter = (struct adapter *)node.sysctl_data; 2951 int error, result; 2952 2953 node.sysctl_data = &result; 2954 error = sysctl_lookup(SYSCTLFN_CALL(&node)); 2955 2956 if (error || newp == NULL) 2957 return error; 2958 2959 if (result == 1) 2960 ixv_print_debug_info(adapter); 2961 2962 return 0; 2963 } /* ixv_sysctl_debug */ 2964 2965 /************************************************************************ 2966 * ixv_sysctl_rx_copy_len 2967 ************************************************************************/ 2968 static int 2969 ixv_sysctl_rx_copy_len(SYSCTLFN_ARGS) 2970 { 2971 struct sysctlnode node = *rnode; 2972 struct adapter *adapter = (struct adapter *)node.sysctl_data; 2973 int error; 2974 int result = adapter->rx_copy_len; 2975 2976 node.sysctl_data = &result; 2977 error = sysctl_lookup(SYSCTLFN_CALL(&node)); 2978 2979 if (error || newp == NULL) 2980 return error; 2981 2982 if ((result < 0) || (result > IXGBE_RX_COPY_LEN_MAX)) 2983 return EINVAL; 2984 2985 adapter->rx_copy_len = result; 2986 2987 return 0; 2988 } /* ixgbe_sysctl_rx_copy_len */ 2989 2990 /************************************************************************ 2991 * ixv_init_device_features 2992 ************************************************************************/ 2993 static void 2994 ixv_init_device_features(struct adapter *adapter) 2995 { 2996 adapter->feat_cap = IXGBE_FEATURE_NETMAP 2997 | IXGBE_FEATURE_VF 2998 | IXGBE_FEATURE_RSS 2999 | IXGBE_FEATURE_LEGACY_TX; 3000 3001 /* A tad short on feature flags for VFs, atm. */ 3002 switch (adapter->hw.mac.type) { 3003 case ixgbe_mac_82599_vf: 3004 break; 3005 case ixgbe_mac_X540_vf: 3006 break; 3007 case ixgbe_mac_X550_vf: 3008 case ixgbe_mac_X550EM_x_vf: 3009 case ixgbe_mac_X550EM_a_vf: 3010 adapter->feat_cap |= IXGBE_FEATURE_NEEDS_CTXD; 3011 break; 3012 default: 3013 break; 3014 } 3015 3016 /* Enabled by default... */ 3017 /* Is a virtual function (VF) */ 3018 if (adapter->feat_cap & IXGBE_FEATURE_VF) 3019 adapter->feat_en |= IXGBE_FEATURE_VF; 3020 /* Netmap */ 3021 if (adapter->feat_cap & IXGBE_FEATURE_NETMAP) 3022 adapter->feat_en |= IXGBE_FEATURE_NETMAP; 3023 /* Receive-Side Scaling (RSS) */ 3024 if (adapter->feat_cap & IXGBE_FEATURE_RSS) 3025 adapter->feat_en |= IXGBE_FEATURE_RSS; 3026 /* Needs advanced context descriptor regardless of offloads req'd */ 3027 if (adapter->feat_cap & IXGBE_FEATURE_NEEDS_CTXD) 3028 adapter->feat_en |= IXGBE_FEATURE_NEEDS_CTXD; 3029 3030 /* Enabled via sysctl... */ 3031 /* Legacy (single queue) transmit */ 3032 if ((adapter->feat_cap & IXGBE_FEATURE_LEGACY_TX) && 3033 ixv_enable_legacy_tx) 3034 adapter->feat_en |= IXGBE_FEATURE_LEGACY_TX; 3035 } /* ixv_init_device_features */ 3036 3037 /************************************************************************ 3038 * ixv_shutdown - Shutdown entry point 3039 ************************************************************************/ 3040 #if 0 /* XXX NetBSD ought to register something like this through pmf(9) */ 3041 static int 3042 ixv_shutdown(device_t dev) 3043 { 3044 struct adapter *adapter = device_private(dev); 3045 IXGBE_CORE_LOCK(adapter); 3046 ixv_stop_locked(adapter); 3047 IXGBE_CORE_UNLOCK(adapter); 3048 3049 return (0); 3050 } /* ixv_shutdown */ 3051 #endif 3052 3053 static int 3054 ixv_ifflags_cb(struct ethercom *ec) 3055 { 3056 struct ifnet *ifp = &ec->ec_if; 3057 struct adapter *adapter = ifp->if_softc; 3058 u_short saved_flags; 3059 u_short change; 3060 int rv = 0; 3061 3062 IXGBE_CORE_LOCK(adapter); 3063 3064 saved_flags = adapter->if_flags; 3065 change = ifp->if_flags ^ adapter->if_flags; 3066 if (change != 0) 3067 adapter->if_flags = ifp->if_flags; 3068 3069 if ((change & ~(IFF_CANTCHANGE | IFF_DEBUG)) != 0) { 3070 rv = ENETRESET; 3071 goto out; 3072 } else if ((change & IFF_PROMISC) != 0) { 3073 rv = ixv_set_rxfilter(adapter); 3074 if (rv != 0) { 3075 /* Restore previous */ 3076 adapter->if_flags = saved_flags; 3077 goto out; 3078 } 3079 } 3080 3081 /* Check for ec_capenable. */ 3082 change = ec->ec_capenable ^ adapter->ec_capenable; 3083 adapter->ec_capenable = ec->ec_capenable; 3084 if ((change & ~(ETHERCAP_VLAN_MTU | ETHERCAP_VLAN_HWTAGGING 3085 | ETHERCAP_VLAN_HWFILTER)) != 0) { 3086 rv = ENETRESET; 3087 goto out; 3088 } 3089 3090 /* 3091 * Special handling is not required for ETHERCAP_VLAN_MTU. 3092 * PF's MAXFRS(MHADD) does not include the 4bytes of the VLAN header. 3093 */ 3094 3095 /* Set up VLAN support and filter */ 3096 if ((change & (ETHERCAP_VLAN_HWTAGGING | ETHERCAP_VLAN_HWFILTER)) != 0) 3097 rv = ixv_setup_vlan_support(adapter); 3098 3099 out: 3100 IXGBE_CORE_UNLOCK(adapter); 3101 3102 return rv; 3103 } 3104 3105 3106 /************************************************************************ 3107 * ixv_ioctl - Ioctl entry point 3108 * 3109 * Called when the user wants to configure the interface. 3110 * 3111 * return 0 on success, positive on failure 3112 ************************************************************************/ 3113 static int 3114 ixv_ioctl(struct ifnet *ifp, u_long command, void *data) 3115 { 3116 struct adapter *adapter = ifp->if_softc; 3117 struct ixgbe_hw *hw = &adapter->hw; 3118 struct ifcapreq *ifcr = data; 3119 int error; 3120 int l4csum_en; 3121 const int l4csum = IFCAP_CSUM_TCPv4_Rx | IFCAP_CSUM_UDPv4_Rx | 3122 IFCAP_CSUM_TCPv6_Rx | IFCAP_CSUM_UDPv6_Rx; 3123 3124 switch (command) { 3125 case SIOCSIFFLAGS: 3126 IOCTL_DEBUGOUT("ioctl: SIOCSIFFLAGS (Set Interface Flags)"); 3127 break; 3128 case SIOCADDMULTI: { 3129 struct ether_multi *enm; 3130 struct ether_multistep step; 3131 struct ethercom *ec = &adapter->osdep.ec; 3132 bool overflow = false; 3133 int mcnt = 0; 3134 3135 /* 3136 * Check the number of multicast address. If it exceeds, 3137 * return ENOSPC. 3138 * Update this code when we support API 1.3. 3139 */ 3140 ETHER_LOCK(ec); 3141 ETHER_FIRST_MULTI(step, ec, enm); 3142 while (enm != NULL) { 3143 mcnt++; 3144 3145 /* 3146 * This code is before adding, so one room is required 3147 * at least. 3148 */ 3149 if (mcnt > (IXGBE_MAX_VF_MC - 1)) { 3150 overflow = true; 3151 break; 3152 } 3153 ETHER_NEXT_MULTI(step, enm); 3154 } 3155 ETHER_UNLOCK(ec); 3156 error = 0; 3157 if (overflow && ((ec->ec_flags & ETHER_F_ALLMULTI) == 0)) { 3158 error = hw->mac.ops.update_xcast_mode(hw, 3159 IXGBEVF_XCAST_MODE_ALLMULTI); 3160 if (error == IXGBE_ERR_NOT_TRUSTED) { 3161 device_printf(adapter->dev, 3162 "this interface is not trusted\n"); 3163 error = EPERM; 3164 } else if (error == IXGBE_ERR_FEATURE_NOT_SUPPORTED) { 3165 device_printf(adapter->dev, 3166 "the PF doesn't support allmulti mode\n"); 3167 error = EOPNOTSUPP; 3168 } else if (error) { 3169 device_printf(adapter->dev, 3170 "number of Ethernet multicast addresses " 3171 "exceeds the limit (%d). error = %d\n", 3172 IXGBE_MAX_VF_MC, error); 3173 error = ENOSPC; 3174 } else 3175 ec->ec_flags |= ETHER_F_ALLMULTI; 3176 } 3177 if (error) 3178 return error; 3179 } 3180 /*FALLTHROUGH*/ 3181 case SIOCDELMULTI: 3182 IOCTL_DEBUGOUT("ioctl: SIOC(ADD|DEL)MULTI"); 3183 break; 3184 case SIOCSIFMEDIA: 3185 case SIOCGIFMEDIA: 3186 IOCTL_DEBUGOUT("ioctl: SIOCxIFMEDIA (Get/Set Interface Media)"); 3187 break; 3188 case SIOCSIFCAP: 3189 IOCTL_DEBUGOUT("ioctl: SIOCSIFCAP (Set Capabilities)"); 3190 break; 3191 case SIOCSIFMTU: 3192 IOCTL_DEBUGOUT("ioctl: SIOCSIFMTU (Set Interface MTU)"); 3193 break; 3194 case SIOCZIFDATA: 3195 IOCTL_DEBUGOUT("ioctl: SIOCZIFDATA (Zero counter)"); 3196 ixv_update_stats(adapter); 3197 ixv_clear_evcnt(adapter); 3198 break; 3199 default: 3200 IOCTL_DEBUGOUT1("ioctl: UNKNOWN (0x%X)", (int)command); 3201 break; 3202 } 3203 3204 switch (command) { 3205 case SIOCSIFCAP: 3206 /* Layer-4 Rx checksum offload has to be turned on and 3207 * off as a unit. 3208 */ 3209 l4csum_en = ifcr->ifcr_capenable & l4csum; 3210 if (l4csum_en != l4csum && l4csum_en != 0) 3211 return EINVAL; 3212 /*FALLTHROUGH*/ 3213 case SIOCADDMULTI: 3214 case SIOCDELMULTI: 3215 case SIOCSIFFLAGS: 3216 case SIOCSIFMTU: 3217 default: 3218 if ((error = ether_ioctl(ifp, command, data)) != ENETRESET) 3219 return error; 3220 if ((ifp->if_flags & IFF_RUNNING) == 0) 3221 ; 3222 else if (command == SIOCSIFCAP || command == SIOCSIFMTU) { 3223 IXGBE_CORE_LOCK(adapter); 3224 ixv_init_locked(adapter); 3225 IXGBE_CORE_UNLOCK(adapter); 3226 } else if (command == SIOCADDMULTI || command == SIOCDELMULTI) { 3227 /* 3228 * Multicast list has changed; set the hardware filter 3229 * accordingly. 3230 */ 3231 IXGBE_CORE_LOCK(adapter); 3232 ixv_disable_intr(adapter); 3233 ixv_set_rxfilter(adapter); 3234 ixv_enable_intr(adapter); 3235 IXGBE_CORE_UNLOCK(adapter); 3236 } 3237 return 0; 3238 } 3239 } /* ixv_ioctl */ 3240 3241 /************************************************************************ 3242 * ixv_init 3243 ************************************************************************/ 3244 static int 3245 ixv_init(struct ifnet *ifp) 3246 { 3247 struct adapter *adapter = ifp->if_softc; 3248 3249 IXGBE_CORE_LOCK(adapter); 3250 ixv_init_locked(adapter); 3251 IXGBE_CORE_UNLOCK(adapter); 3252 3253 return 0; 3254 } /* ixv_init */ 3255 3256 /************************************************************************ 3257 * ixv_handle_que 3258 ************************************************************************/ 3259 static void 3260 ixv_handle_que(void *context) 3261 { 3262 struct ix_queue *que = context; 3263 struct adapter *adapter = que->adapter; 3264 struct tx_ring *txr = que->txr; 3265 struct ifnet *ifp = adapter->ifp; 3266 bool more; 3267 3268 que->handleq.ev_count++; 3269 3270 if (ifp->if_flags & IFF_RUNNING) { 3271 more = ixgbe_rxeof(que); 3272 IXGBE_TX_LOCK(txr); 3273 more |= ixgbe_txeof(txr); 3274 if (!(adapter->feat_en & IXGBE_FEATURE_LEGACY_TX)) 3275 if (!ixgbe_mq_ring_empty(ifp, txr->txr_interq)) 3276 ixgbe_mq_start_locked(ifp, txr); 3277 /* Only for queue 0 */ 3278 /* NetBSD still needs this for CBQ */ 3279 if ((&adapter->queues[0] == que) 3280 && (!ixgbe_legacy_ring_empty(ifp, NULL))) 3281 ixgbe_legacy_start_locked(ifp, txr); 3282 IXGBE_TX_UNLOCK(txr); 3283 if (more) { 3284 que->req.ev_count++; 3285 if (adapter->txrx_use_workqueue) { 3286 /* 3287 * "enqueued flag" is not required here 3288 * the same as ixg(4). See ixgbe_msix_que(). 3289 */ 3290 workqueue_enqueue(adapter->que_wq, 3291 &que->wq_cookie, curcpu()); 3292 } else 3293 softint_schedule(que->que_si); 3294 return; 3295 } 3296 } 3297 3298 /* Re-enable this interrupt */ 3299 ixv_enable_queue(adapter, que->msix); 3300 3301 return; 3302 } /* ixv_handle_que */ 3303 3304 /************************************************************************ 3305 * ixv_handle_que_work 3306 ************************************************************************/ 3307 static void 3308 ixv_handle_que_work(struct work *wk, void *context) 3309 { 3310 struct ix_queue *que = container_of(wk, struct ix_queue, wq_cookie); 3311 3312 /* 3313 * "enqueued flag" is not required here the same as ixg(4). 3314 * See ixgbe_msix_que(). 3315 */ 3316 ixv_handle_que(que); 3317 } 3318 3319 /************************************************************************ 3320 * ixv_allocate_msix - Setup MSI-X Interrupt resources and handlers 3321 ************************************************************************/ 3322 static int 3323 ixv_allocate_msix(struct adapter *adapter, const struct pci_attach_args *pa) 3324 { 3325 device_t dev = adapter->dev; 3326 struct ix_queue *que = adapter->queues; 3327 struct tx_ring *txr = adapter->tx_rings; 3328 int error, msix_ctrl, rid, vector = 0; 3329 pci_chipset_tag_t pc; 3330 pcitag_t tag; 3331 char intrbuf[PCI_INTRSTR_LEN]; 3332 char wqname[MAXCOMLEN]; 3333 char intr_xname[32]; 3334 const char *intrstr = NULL; 3335 kcpuset_t *affinity; 3336 int cpu_id = 0; 3337 3338 pc = adapter->osdep.pc; 3339 tag = adapter->osdep.tag; 3340 3341 adapter->osdep.nintrs = adapter->num_queues + 1; 3342 if (pci_msix_alloc_exact(pa, &adapter->osdep.intrs, 3343 adapter->osdep.nintrs) != 0) { 3344 aprint_error_dev(dev, 3345 "failed to allocate MSI-X interrupt\n"); 3346 return (ENXIO); 3347 } 3348 3349 kcpuset_create(&affinity, false); 3350 for (int i = 0; i < adapter->num_queues; i++, vector++, que++, txr++) { 3351 snprintf(intr_xname, sizeof(intr_xname), "%s TXRX%d", 3352 device_xname(dev), i); 3353 intrstr = pci_intr_string(pc, adapter->osdep.intrs[i], intrbuf, 3354 sizeof(intrbuf)); 3355 #ifdef IXGBE_MPSAFE 3356 pci_intr_setattr(pc, &adapter->osdep.intrs[i], PCI_INTR_MPSAFE, 3357 true); 3358 #endif 3359 /* Set the handler function */ 3360 que->res = adapter->osdep.ihs[i] = pci_intr_establish_xname(pc, 3361 adapter->osdep.intrs[i], IPL_NET, ixv_msix_que, que, 3362 intr_xname); 3363 if (que->res == NULL) { 3364 pci_intr_release(pc, adapter->osdep.intrs, 3365 adapter->osdep.nintrs); 3366 aprint_error_dev(dev, 3367 "Failed to register QUE handler\n"); 3368 kcpuset_destroy(affinity); 3369 return (ENXIO); 3370 } 3371 que->msix = vector; 3372 adapter->active_queues |= (u64)(1 << que->msix); 3373 3374 cpu_id = i; 3375 /* Round-robin affinity */ 3376 kcpuset_zero(affinity); 3377 kcpuset_set(affinity, cpu_id % ncpu); 3378 error = interrupt_distribute(adapter->osdep.ihs[i], affinity, 3379 NULL); 3380 aprint_normal_dev(dev, "for TX/RX, interrupting at %s", 3381 intrstr); 3382 if (error == 0) 3383 aprint_normal(", bound queue %d to cpu %d\n", 3384 i, cpu_id % ncpu); 3385 else 3386 aprint_normal("\n"); 3387 3388 #ifndef IXGBE_LEGACY_TX 3389 txr->txr_si 3390 = softint_establish(SOFTINT_NET | IXGBE_SOFTINT_FLAGS, 3391 ixgbe_deferred_mq_start, txr); 3392 #endif 3393 que->que_si 3394 = softint_establish(SOFTINT_NET | IXGBE_SOFTINT_FLAGS, 3395 ixv_handle_que, que); 3396 if (que->que_si == NULL) { 3397 aprint_error_dev(dev, 3398 "could not establish software interrupt\n"); 3399 } 3400 } 3401 snprintf(wqname, sizeof(wqname), "%sdeferTx", device_xname(dev)); 3402 error = workqueue_create(&adapter->txr_wq, wqname, 3403 ixgbe_deferred_mq_start_work, adapter, IXGBE_WORKQUEUE_PRI, IPL_NET, 3404 IXGBE_WORKQUEUE_FLAGS); 3405 if (error) { 3406 aprint_error_dev(dev, 3407 "couldn't create workqueue for deferred Tx\n"); 3408 } 3409 adapter->txr_wq_enqueued = percpu_alloc(sizeof(u_int)); 3410 3411 snprintf(wqname, sizeof(wqname), "%sTxRx", device_xname(dev)); 3412 error = workqueue_create(&adapter->que_wq, wqname, 3413 ixv_handle_que_work, adapter, IXGBE_WORKQUEUE_PRI, IPL_NET, 3414 IXGBE_WORKQUEUE_FLAGS); 3415 if (error) { 3416 aprint_error_dev(dev, "couldn't create workqueue for Tx/Rx\n"); 3417 } 3418 3419 /* and Mailbox */ 3420 cpu_id++; 3421 snprintf(intr_xname, sizeof(intr_xname), "%s link", device_xname(dev)); 3422 adapter->vector = vector; 3423 intrstr = pci_intr_string(pc, adapter->osdep.intrs[vector], intrbuf, 3424 sizeof(intrbuf)); 3425 #ifdef IXGBE_MPSAFE 3426 pci_intr_setattr(pc, &adapter->osdep.intrs[vector], PCI_INTR_MPSAFE, 3427 true); 3428 #endif 3429 /* Set the mbx handler function */ 3430 adapter->osdep.ihs[vector] = pci_intr_establish_xname(pc, 3431 adapter->osdep.intrs[vector], IPL_NET, ixv_msix_mbx, adapter, 3432 intr_xname); 3433 if (adapter->osdep.ihs[vector] == NULL) { 3434 aprint_error_dev(dev, "Failed to register LINK handler\n"); 3435 kcpuset_destroy(affinity); 3436 return (ENXIO); 3437 } 3438 /* Round-robin affinity */ 3439 kcpuset_zero(affinity); 3440 kcpuset_set(affinity, cpu_id % ncpu); 3441 error = interrupt_distribute(adapter->osdep.ihs[vector], affinity, 3442 NULL); 3443 3444 aprint_normal_dev(dev, 3445 "for link, interrupting at %s", intrstr); 3446 if (error == 0) 3447 aprint_normal(", affinity to cpu %d\n", cpu_id % ncpu); 3448 else 3449 aprint_normal("\n"); 3450 3451 /* Tasklets for Mailbox */ 3452 snprintf(wqname, sizeof(wqname), "%s-admin", device_xname(dev)); 3453 error = workqueue_create(&adapter->admin_wq, wqname, 3454 ixv_handle_admin, adapter, IXGBE_WORKQUEUE_PRI, IPL_NET, 3455 IXGBE_TASKLET_WQ_FLAGS); 3456 if (error) { 3457 aprint_error_dev(dev, 3458 "could not create admin workqueue (%d)\n", error); 3459 goto err_out; 3460 } 3461 3462 /* 3463 * Due to a broken design QEMU will fail to properly 3464 * enable the guest for MSI-X unless the vectors in 3465 * the table are all set up, so we must rewrite the 3466 * ENABLE in the MSI-X control register again at this 3467 * point to cause it to successfully initialize us. 3468 */ 3469 if (adapter->hw.mac.type == ixgbe_mac_82599_vf) { 3470 pci_get_capability(pc, tag, PCI_CAP_MSIX, &rid, NULL); 3471 rid += PCI_MSIX_CTL; 3472 msix_ctrl = pci_conf_read(pc, tag, rid); 3473 msix_ctrl |= PCI_MSIX_CTL_ENABLE; 3474 pci_conf_write(pc, tag, rid, msix_ctrl); 3475 } 3476 3477 kcpuset_destroy(affinity); 3478 return (0); 3479 err_out: 3480 kcpuset_destroy(affinity); 3481 ixv_free_deferred_handlers(adapter); 3482 ixv_free_pci_resources(adapter); 3483 return (error); 3484 } /* ixv_allocate_msix */ 3485 3486 /************************************************************************ 3487 * ixv_configure_interrupts - Setup MSI-X resources 3488 * 3489 * Note: The VF device MUST use MSI-X, there is no fallback. 3490 ************************************************************************/ 3491 static int 3492 ixv_configure_interrupts(struct adapter *adapter) 3493 { 3494 device_t dev = adapter->dev; 3495 int want, queues, msgs; 3496 3497 /* Must have at least 2 MSI-X vectors */ 3498 msgs = pci_msix_count(adapter->osdep.pc, adapter->osdep.tag); 3499 if (msgs < 2) { 3500 aprint_error_dev(dev, "MSIX config error\n"); 3501 return (ENXIO); 3502 } 3503 msgs = MIN(msgs, IXG_MAX_NINTR); 3504 3505 /* Figure out a reasonable auto config value */ 3506 queues = (ncpu > (msgs - 1)) ? (msgs - 1) : ncpu; 3507 3508 if (ixv_num_queues != 0) 3509 queues = ixv_num_queues; 3510 else if ((ixv_num_queues == 0) && (queues > IXGBE_VF_MAX_TX_QUEUES)) 3511 queues = IXGBE_VF_MAX_TX_QUEUES; 3512 3513 /* 3514 * Want vectors for the queues, 3515 * plus an additional for mailbox. 3516 */ 3517 want = queues + 1; 3518 if (msgs >= want) 3519 msgs = want; 3520 else { 3521 aprint_error_dev(dev, 3522 "MSI-X Configuration Problem, " 3523 "%d vectors but %d queues wanted!\n", 3524 msgs, want); 3525 return -1; 3526 } 3527 3528 adapter->msix_mem = (void *)1; /* XXX */ 3529 aprint_normal_dev(dev, 3530 "Using MSI-X interrupts with %d vectors\n", msgs); 3531 adapter->num_queues = queues; 3532 3533 return (0); 3534 } /* ixv_configure_interrupts */ 3535 3536 3537 /************************************************************************ 3538 * ixv_handle_admin - Tasklet handler for MSI-X MBX interrupts 3539 * 3540 * Done outside of interrupt context since the driver might sleep 3541 ************************************************************************/ 3542 static void 3543 ixv_handle_admin(struct work *wk, void *context) 3544 { 3545 struct adapter *adapter = context; 3546 struct ixgbe_hw *hw = &adapter->hw; 3547 3548 IXGBE_CORE_LOCK(adapter); 3549 3550 ++adapter->link_workev.ev_count; 3551 adapter->hw.mac.ops.check_link(&adapter->hw, &adapter->link_speed, 3552 &adapter->link_up, FALSE); 3553 ixv_update_link_status(adapter); 3554 3555 adapter->task_requests = 0; 3556 atomic_store_relaxed(&adapter->admin_pending, 0); 3557 3558 /* Re-enable interrupts */ 3559 IXGBE_WRITE_REG(hw, IXGBE_VTEIMS, (1 << adapter->vector)); 3560 3561 IXGBE_CORE_UNLOCK(adapter); 3562 } /* ixv_handle_admin */ 3563 3564 /************************************************************************ 3565 * ixv_check_link - Used in the local timer to poll for link changes 3566 ************************************************************************/ 3567 static s32 3568 ixv_check_link(struct adapter *adapter) 3569 { 3570 s32 error; 3571 3572 KASSERT(mutex_owned(&adapter->core_mtx)); 3573 3574 adapter->hw.mac.get_link_status = TRUE; 3575 3576 error = adapter->hw.mac.ops.check_link(&adapter->hw, 3577 &adapter->link_speed, &adapter->link_up, FALSE); 3578 ixv_update_link_status(adapter); 3579 3580 return error; 3581 } /* ixv_check_link */ 3582
Indexes created Tue Sep 30 20:09:53 GMT 2025