ixv.c revision 1.185
1 /* $NetBSD: ixv.c,v 1.185 2023/10/06 14:34:23 msaitoh Exp $ */ 2 3 /****************************************************************************** 4 5 Copyright (c) 2001-2017, Intel Corporation 6 All rights reserved. 7 8 Redistribution and use in source and binary forms, with or without 9 modification, are permitted provided that the following conditions are met: 10 11 1. Redistributions of source code must retain the above copyright notice, 12 this list of conditions and the following disclaimer. 13 14 2. Redistributions in binary form must reproduce the above copyright 15 notice, this list of conditions and the following disclaimer in the 16 documentation and/or other materials provided with the distribution. 17 18 3. Neither the name of the Intel Corporation nor the names of its 19 contributors may be used to endorse or promote products derived from 20 this software without specific prior written permission. 21 22 THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 23 AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 24 IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 25 ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE 26 LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 27 CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 28 SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 29 INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 30 CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 31 ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 32 POSSIBILITY OF SUCH DAMAGE. 33 34 ******************************************************************************/ 35 /*$FreeBSD: head/sys/dev/ixgbe/if_ixv.c 331224 2018-03-19 20:55:05Z erj $*/ 36 37 #include <sys/cdefs.h> 38 __KERNEL_RCSID(0, "$NetBSD: ixv.c,v 1.185 2023/10/06 14:34:23 msaitoh Exp $"); 39 40 #ifdef _KERNEL_OPT 41 #include "opt_inet.h" 42 #include "opt_inet6.h" 43 #include "opt_net_mpsafe.h" 44 #endif 45 46 #include "ixgbe.h" 47 48 /************************************************************************ 49 * Driver version 50 ************************************************************************/ 51 static const char ixv_driver_version[] = "2.0.1-k"; 52 /* XXX NetBSD: + 1.5.17 */ 53 54 /************************************************************************ 55 * PCI Device ID Table 56 * 57 * Used by probe to select devices to load on 58 * Last field stores an index into ixv_strings 59 * Last entry must be all 0s 60 * 61 * { Vendor ID, Device ID, SubVendor ID, SubDevice ID, String Index } 62 ************************************************************************/ 63 static const ixgbe_vendor_info_t ixv_vendor_info_array[] = 64 { 65 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_VF, 0, 0, 0}, 66 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_X540_VF, 0, 0, 0}, 67 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_X550_VF, 0, 0, 0}, 68 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_X550EM_X_VF, 0, 0, 0}, 69 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_X550EM_A_VF, 0, 0, 0}, 70 /* required last entry */ 71 {0, 0, 0, 0, 0} 72 }; 73 74 /************************************************************************ 75 * Table of branding strings 76 ************************************************************************/ 77 static const char *ixv_strings[] = { 78 "Intel(R) PRO/10GbE Virtual Function Network Driver" 79 }; 80 81 /********************************************************************* 82 * Function prototypes 83 *********************************************************************/ 84 static int ixv_probe(device_t, cfdata_t, void *); 85 static void ixv_attach(device_t, device_t, void *); 86 static int ixv_detach(device_t, int); 87 #if 0 88 static int ixv_shutdown(device_t); 89 #endif 90 static int ixv_ifflags_cb(struct ethercom *); 91 static int ixv_ioctl(struct ifnet *, u_long, void *); 92 static int ixv_init(struct ifnet *); 93 static void ixv_init_locked(struct adapter *); 94 static void ixv_ifstop(struct ifnet *, int); 95 static void ixv_stop_locked(void *); 96 static void ixv_init_device_features(struct adapter *); 97 static void ixv_media_status(struct ifnet *, struct ifmediareq *); 98 static int ixv_media_change(struct ifnet *); 99 static int ixv_allocate_pci_resources(struct adapter *, 100 const struct pci_attach_args *); 101 static void ixv_free_deferred_handlers(struct adapter *); 102 static int ixv_allocate_msix(struct adapter *, 103 const struct pci_attach_args *); 104 static int ixv_configure_interrupts(struct adapter *); 105 static void ixv_free_pci_resources(struct adapter *); 106 static void ixv_local_timer(void *); 107 static void ixv_handle_timer(struct work *, void *); 108 static int ixv_setup_interface(device_t, struct adapter *); 109 static void ixv_schedule_admin_tasklet(struct adapter *); 110 static int ixv_negotiate_api(struct adapter *); 111 112 static void ixv_initialize_transmit_units(struct adapter *); 113 static void ixv_initialize_receive_units(struct adapter *); 114 static void ixv_initialize_rss_mapping(struct adapter *); 115 static s32 ixv_check_link(struct adapter *); 116 117 static void ixv_enable_intr(struct adapter *); 118 static void ixv_disable_intr(struct adapter *); 119 static int ixv_set_rxfilter(struct adapter *); 120 static void ixv_update_link_status(struct adapter *); 121 static int ixv_sysctl_debug(SYSCTLFN_PROTO); 122 static void ixv_set_ivar(struct adapter *, u8, u8, s8); 123 static void ixv_configure_ivars(struct adapter *); 124 static u8 * ixv_mc_array_itr(struct ixgbe_hw *, u8 **, u32 *); 125 static void ixv_eitr_write(struct adapter *, uint32_t, uint32_t); 126 127 static void ixv_setup_vlan_tagging(struct adapter *); 128 static int ixv_setup_vlan_support(struct adapter *); 129 static int ixv_vlan_cb(struct ethercom *, uint16_t, bool); 130 static int ixv_register_vlan(struct adapter *, u16); 131 static int ixv_unregister_vlan(struct adapter *, u16); 132 133 static void ixv_add_device_sysctls(struct adapter *); 134 static void ixv_init_stats(struct adapter *); 135 static void ixv_update_stats(struct adapter *); 136 static void ixv_add_stats_sysctls(struct adapter *); 137 static void ixv_clear_evcnt(struct adapter *); 138 139 /* Sysctl handlers */ 140 static int ixv_sysctl_interrupt_rate_handler(SYSCTLFN_PROTO); 141 static int ixv_sysctl_next_to_check_handler(SYSCTLFN_PROTO); 142 static int ixv_sysctl_next_to_refresh_handler(SYSCTLFN_PROTO); 143 static int ixv_sysctl_rdh_handler(SYSCTLFN_PROTO); 144 static int ixv_sysctl_rdt_handler(SYSCTLFN_PROTO); 145 static int ixv_sysctl_tdt_handler(SYSCTLFN_PROTO); 146 static int ixv_sysctl_tdh_handler(SYSCTLFN_PROTO); 147 static int ixv_sysctl_tx_process_limit(SYSCTLFN_PROTO); 148 static int ixv_sysctl_rx_process_limit(SYSCTLFN_PROTO); 149 static int ixv_sysctl_rx_copy_len(SYSCTLFN_PROTO); 150 151 /* The MSI-X Interrupt handlers */ 152 static int ixv_msix_que(void *); 153 static int ixv_msix_mbx(void *); 154 155 /* Event handlers running on workqueue */ 156 static void ixv_handle_que(void *); 157 158 /* Deferred workqueue handlers */ 159 static void ixv_handle_admin(struct work *, void *); 160 static void ixv_handle_que_work(struct work *, void *); 161 162 const struct sysctlnode *ixv_sysctl_instance(struct adapter *); 163 static const ixgbe_vendor_info_t *ixv_lookup(const struct pci_attach_args *); 164 165 /************************************************************************ 166 * NetBSD Device Interface Entry Points 167 ************************************************************************/ 168 CFATTACH_DECL3_NEW(ixv, sizeof(struct adapter), 169 ixv_probe, ixv_attach, ixv_detach, NULL, NULL, NULL, 170 DVF_DETACH_SHUTDOWN); 171 172 #if 0 173 static driver_t ixv_driver = { 174 "ixv", ixv_methods, sizeof(struct adapter), 175 }; 176 177 devclass_t ixv_devclass; 178 DRIVER_MODULE(ixv, pci, ixv_driver, ixv_devclass, 0, 0); 179 MODULE_DEPEND(ixv, pci, 1, 1, 1); 180 MODULE_DEPEND(ixv, ether, 1, 1, 1); 181 #endif 182 183 /* 184 * TUNEABLE PARAMETERS: 185 */ 186 187 /* Number of Queues - do not exceed MSI-X vectors - 1 */ 188 static int ixv_num_queues = 0; 189 #define TUNABLE_INT(__x, __y) 190 TUNABLE_INT("hw.ixv.num_queues", &ixv_num_queues); 191 192 /* 193 * AIM: Adaptive Interrupt Moderation 194 * which means that the interrupt rate 195 * is varied over time based on the 196 * traffic for that interrupt vector 197 */ 198 static bool ixv_enable_aim = false; 199 TUNABLE_INT("hw.ixv.enable_aim", &ixv_enable_aim); 200 201 static int ixv_max_interrupt_rate = (4000000 / IXGBE_LOW_LATENCY); 202 TUNABLE_INT("hw.ixv.max_interrupt_rate", &ixv_max_interrupt_rate); 203 204 /* How many packets rxeof tries to clean at a time */ 205 static int ixv_rx_process_limit = 256; 206 TUNABLE_INT("hw.ixv.rx_process_limit", &ixv_rx_process_limit); 207 208 /* How many packets txeof tries to clean at a time */ 209 static int ixv_tx_process_limit = 256; 210 TUNABLE_INT("hw.ixv.tx_process_limit", &ixv_tx_process_limit); 211 212 /* Which packet processing uses workqueue or softint */ 213 static bool ixv_txrx_workqueue = false; 214 215 /* 216 * Number of TX descriptors per ring, 217 * setting higher than RX as this seems 218 * the better performing choice. 219 */ 220 static int ixv_txd = PERFORM_TXD; 221 TUNABLE_INT("hw.ixv.txd", &ixv_txd); 222 223 /* Number of RX descriptors per ring */ 224 static int ixv_rxd = PERFORM_RXD; 225 TUNABLE_INT("hw.ixv.rxd", &ixv_rxd); 226 227 /* Legacy Transmit (single queue) */ 228 static int ixv_enable_legacy_tx = 0; 229 TUNABLE_INT("hw.ixv.enable_legacy_tx", &ixv_enable_legacy_tx); 230 231 #ifdef NET_MPSAFE 232 #define IXGBE_MPSAFE 1 233 #define IXGBE_CALLOUT_FLAGS CALLOUT_MPSAFE 234 #define IXGBE_SOFTINT_FLAGS SOFTINT_MPSAFE 235 #define IXGBE_WORKQUEUE_FLAGS WQ_PERCPU | WQ_MPSAFE 236 #define IXGBE_TASKLET_WQ_FLAGS WQ_MPSAFE 237 #else 238 #define IXGBE_CALLOUT_FLAGS 0 239 #define IXGBE_SOFTINT_FLAGS 0 240 #define IXGBE_WORKQUEUE_FLAGS WQ_PERCPU 241 #define IXGBE_TASKLET_WQ_FLAGS 0 242 #endif 243 #define IXGBE_WORKQUEUE_PRI PRI_SOFTNET 244 245 #if 0 246 static int (*ixv_start_locked)(struct ifnet *, struct tx_ring *); 247 static int (*ixv_ring_empty)(struct ifnet *, struct buf_ring *); 248 #endif 249 250 /************************************************************************ 251 * ixv_probe - Device identification routine 252 * 253 * Determines if the driver should be loaded on 254 * adapter based on its PCI vendor/device ID. 255 * 256 * return BUS_PROBE_DEFAULT on success, positive on failure 257 ************************************************************************/ 258 static int 259 ixv_probe(device_t dev, cfdata_t cf, void *aux) 260 { 261 #ifdef __HAVE_PCI_MSI_MSIX 262 const struct pci_attach_args *pa = aux; 263 264 return (ixv_lookup(pa) != NULL) ? 1 : 0; 265 #else 266 return 0; 267 #endif 268 } /* ixv_probe */ 269 270 static const ixgbe_vendor_info_t * 271 ixv_lookup(const struct pci_attach_args *pa) 272 { 273 const ixgbe_vendor_info_t *ent; 274 pcireg_t subid; 275 276 INIT_DEBUGOUT("ixv_lookup: begin"); 277 278 if (PCI_VENDOR(pa->pa_id) != IXGBE_INTEL_VENDOR_ID) 279 return NULL; 280 281 subid = pci_conf_read(pa->pa_pc, pa->pa_tag, PCI_SUBSYS_ID_REG); 282 283 for (ent = ixv_vendor_info_array; ent->vendor_id != 0; ent++) { 284 if ((PCI_VENDOR(pa->pa_id) == ent->vendor_id) && 285 (PCI_PRODUCT(pa->pa_id) == ent->device_id) && 286 ((PCI_SUBSYS_VENDOR(subid) == ent->subvendor_id) || 287 (ent->subvendor_id == 0)) && 288 ((PCI_SUBSYS_ID(subid) == ent->subdevice_id) || 289 (ent->subdevice_id == 0))) { 290 return ent; 291 } 292 } 293 294 return NULL; 295 } 296 297 /************************************************************************ 298 * ixv_attach - Device initialization routine 299 * 300 * Called when the driver is being loaded. 301 * Identifies the type of hardware, allocates all resources 302 * and initializes the hardware. 303 * 304 * return 0 on success, positive on failure 305 ************************************************************************/ 306 static void 307 ixv_attach(device_t parent, device_t dev, void *aux) 308 { 309 struct adapter *adapter; 310 struct ixgbe_hw *hw; 311 int error = 0; 312 pcireg_t id, subid; 313 const ixgbe_vendor_info_t *ent; 314 const struct pci_attach_args *pa = aux; 315 const char *apivstr; 316 const char *str; 317 char wqname[MAXCOMLEN]; 318 char buf[256]; 319 320 INIT_DEBUGOUT("ixv_attach: begin"); 321 322 /* 323 * Make sure BUSMASTER is set, on a VM under 324 * KVM it may not be and will break things. 325 */ 326 ixgbe_pci_enable_busmaster(pa->pa_pc, pa->pa_tag); 327 328 /* Allocate, clear, and link in our adapter structure */ 329 adapter = device_private(dev); 330 adapter->hw.back = adapter; 331 adapter->dev = dev; 332 hw = &adapter->hw; 333 334 adapter->init_locked = ixv_init_locked; 335 adapter->stop_locked = ixv_stop_locked; 336 337 adapter->osdep.pc = pa->pa_pc; 338 adapter->osdep.tag = pa->pa_tag; 339 if (pci_dma64_available(pa)) 340 adapter->osdep.dmat = pa->pa_dmat64; 341 else 342 adapter->osdep.dmat = pa->pa_dmat; 343 adapter->osdep.attached = false; 344 345 ent = ixv_lookup(pa); 346 347 KASSERT(ent != NULL); 348 349 aprint_normal(": %s, Version - %s\n", 350 ixv_strings[ent->index], ixv_driver_version); 351 352 /* Core Lock Init */ 353 IXGBE_CORE_LOCK_INIT(adapter, device_xname(dev)); 354 355 /* Do base PCI setup - map BAR0 */ 356 if (ixv_allocate_pci_resources(adapter, pa)) { 357 aprint_error_dev(dev, "ixv_allocate_pci_resources() failed!\n"); 358 error = ENXIO; 359 goto err_out; 360 } 361 362 /* SYSCTL APIs */ 363 ixv_add_device_sysctls(adapter); 364 365 /* Set up the timer callout and workqueue */ 366 callout_init(&adapter->timer, IXGBE_CALLOUT_FLAGS); 367 snprintf(wqname, sizeof(wqname), "%s-timer", device_xname(dev)); 368 error = workqueue_create(&adapter->timer_wq, wqname, 369 ixv_handle_timer, adapter, IXGBE_WORKQUEUE_PRI, IPL_NET, 370 IXGBE_TASKLET_WQ_FLAGS); 371 if (error) { 372 aprint_error_dev(dev, 373 "could not create timer workqueue (%d)\n", error); 374 goto err_out; 375 } 376 377 /* Save off the information about this board */ 378 id = pci_conf_read(pa->pa_pc, pa->pa_tag, PCI_ID_REG); 379 subid = pci_conf_read(pa->pa_pc, pa->pa_tag, PCI_SUBSYS_ID_REG); 380 hw->vendor_id = PCI_VENDOR(id); 381 hw->device_id = PCI_PRODUCT(id); 382 hw->revision_id = 383 PCI_REVISION(pci_conf_read(pa->pa_pc, pa->pa_tag, PCI_CLASS_REG)); 384 hw->subsystem_vendor_id = PCI_SUBSYS_VENDOR(subid); 385 hw->subsystem_device_id = PCI_SUBSYS_ID(subid); 386 387 /* A subset of set_mac_type */ 388 switch (hw->device_id) { 389 case IXGBE_DEV_ID_82599_VF: 390 hw->mac.type = ixgbe_mac_82599_vf; 391 str = "82599 VF"; 392 break; 393 case IXGBE_DEV_ID_X540_VF: 394 hw->mac.type = ixgbe_mac_X540_vf; 395 str = "X540 VF"; 396 break; 397 case IXGBE_DEV_ID_X550_VF: 398 hw->mac.type = ixgbe_mac_X550_vf; 399 str = "X550 VF"; 400 break; 401 case IXGBE_DEV_ID_X550EM_X_VF: 402 hw->mac.type = ixgbe_mac_X550EM_x_vf; 403 str = "X550EM X VF"; 404 break; 405 case IXGBE_DEV_ID_X550EM_A_VF: 406 hw->mac.type = ixgbe_mac_X550EM_a_vf; 407 str = "X550EM A VF"; 408 break; 409 default: 410 /* Shouldn't get here since probe succeeded */ 411 aprint_error_dev(dev, "Unknown device ID!\n"); 412 error = ENXIO; 413 goto err_out; 414 break; 415 } 416 aprint_normal_dev(dev, "device %s\n", str); 417 418 ixv_init_device_features(adapter); 419 420 /* Initialize the shared code */ 421 error = ixgbe_init_ops_vf(hw); 422 if (error) { 423 aprint_error_dev(dev, "ixgbe_init_ops_vf() failed!\n"); 424 error = EIO; 425 goto err_out; 426 } 427 428 /* Setup the mailbox */ 429 ixgbe_init_mbx_params_vf(hw); 430 431 /* Set the right number of segments */ 432 KASSERT(IXGBE_82599_SCATTER_MAX >= IXGBE_SCATTER_DEFAULT); 433 adapter->num_segs = IXGBE_SCATTER_DEFAULT; 434 435 /* Reset mbox api to 1.0 */ 436 error = hw->mac.ops.reset_hw(hw); 437 if (error == IXGBE_ERR_RESET_FAILED) 438 aprint_error_dev(dev, "...reset_hw() failure: Reset Failed!\n"); 439 else if (error) 440 aprint_error_dev(dev, "...reset_hw() failed with error %d\n", 441 error); 442 if (error) { 443 error = EIO; 444 goto err_out; 445 } 446 447 error = hw->mac.ops.init_hw(hw); 448 if (error) { 449 aprint_error_dev(dev, "...init_hw() failed!\n"); 450 error = EIO; 451 goto err_out; 452 } 453 454 /* Negotiate mailbox API version */ 455 error = ixv_negotiate_api(adapter); 456 if (error) 457 aprint_normal_dev(dev, 458 "MBX API negotiation failed during attach!\n"); 459 switch (hw->api_version) { 460 case ixgbe_mbox_api_10: 461 apivstr = "1.0"; 462 break; 463 case ixgbe_mbox_api_20: 464 apivstr = "2.0"; 465 break; 466 case ixgbe_mbox_api_11: 467 apivstr = "1.1"; 468 break; 469 case ixgbe_mbox_api_12: 470 apivstr = "1.2"; 471 break; 472 case ixgbe_mbox_api_13: 473 apivstr = "1.3"; 474 break; 475 case ixgbe_mbox_api_14: 476 apivstr = "1.4"; 477 break; 478 case ixgbe_mbox_api_15: 479 apivstr = "1.5"; 480 break; 481 default: 482 apivstr = "unknown"; 483 break; 484 } 485 aprint_normal_dev(dev, "Mailbox API %s\n", apivstr); 486 487 /* If no mac address was assigned, make a random one */ 488 if (!ixv_check_ether_addr(hw->mac.addr)) { 489 u8 addr[ETHER_ADDR_LEN]; 490 uint64_t rndval = cprng_strong64(); 491 492 memcpy(addr, &rndval, sizeof(addr)); 493 addr[0] &= 0xFE; 494 addr[0] |= 0x02; 495 bcopy(addr, hw->mac.addr, sizeof(addr)); 496 } 497 498 /* Register for VLAN events */ 499 ether_set_vlan_cb(&adapter->osdep.ec, ixv_vlan_cb); 500 501 /* Do descriptor calc and sanity checks */ 502 if (((ixv_txd * sizeof(union ixgbe_adv_tx_desc)) % DBA_ALIGN) != 0 || 503 ixv_txd < MIN_TXD || ixv_txd > MAX_TXD) { 504 aprint_error_dev(dev, "TXD config issue, using default!\n"); 505 adapter->num_tx_desc = DEFAULT_TXD; 506 } else 507 adapter->num_tx_desc = ixv_txd; 508 509 if (((ixv_rxd * sizeof(union ixgbe_adv_rx_desc)) % DBA_ALIGN) != 0 || 510 ixv_rxd < MIN_RXD || ixv_rxd > MAX_RXD) { 511 aprint_error_dev(dev, "RXD config issue, using default!\n"); 512 adapter->num_rx_desc = DEFAULT_RXD; 513 } else 514 adapter->num_rx_desc = ixv_rxd; 515 516 /* Sysctls for limiting the amount of work done in the taskqueues */ 517 adapter->rx_process_limit 518 = (ixv_rx_process_limit <= adapter->num_rx_desc) 519 ? ixv_rx_process_limit : adapter->num_rx_desc; 520 adapter->tx_process_limit 521 = (ixv_tx_process_limit <= adapter->num_tx_desc) 522 ? ixv_tx_process_limit : adapter->num_tx_desc; 523 524 /* Set default high limit of copying mbuf in rxeof */ 525 adapter->rx_copy_len = IXGBE_RX_COPY_LEN_MAX; 526 527 /* Setup MSI-X */ 528 error = ixv_configure_interrupts(adapter); 529 if (error) 530 goto err_out; 531 532 /* Allocate our TX/RX Queues */ 533 if (ixgbe_allocate_queues(adapter)) { 534 aprint_error_dev(dev, "ixgbe_allocate_queues() failed!\n"); 535 error = ENOMEM; 536 goto err_out; 537 } 538 539 /* hw.ix defaults init */ 540 adapter->enable_aim = ixv_enable_aim; 541 542 adapter->txrx_use_workqueue = ixv_txrx_workqueue; 543 544 error = ixv_allocate_msix(adapter, pa); 545 if (error) { 546 aprint_error_dev(dev, "ixv_allocate_msix() failed!\n"); 547 goto err_late; 548 } 549 550 /* Setup OS specific network interface */ 551 error = ixv_setup_interface(dev, adapter); 552 if (error != 0) { 553 aprint_error_dev(dev, "ixv_setup_interface() failed!\n"); 554 goto err_late; 555 } 556 557 /* Allocate multicast array memory */ 558 adapter->mta = malloc(sizeof(*adapter->mta) * 559 IXGBE_MAX_VF_MC, M_DEVBUF, M_WAITOK); 560 561 /* Check if VF was disabled by PF */ 562 error = hw->mac.ops.get_link_state(hw, &adapter->link_enabled); 563 if (error) { 564 /* PF is not capable of controlling VF state. Enable the link. */ 565 adapter->link_enabled = TRUE; 566 } 567 568 /* Do the stats setup */ 569 ixv_init_stats(adapter); 570 ixv_add_stats_sysctls(adapter); 571 572 if (adapter->feat_en & IXGBE_FEATURE_NETMAP) 573 ixgbe_netmap_attach(adapter); 574 575 snprintb(buf, sizeof(buf), IXGBE_FEATURE_FLAGS, adapter->feat_cap); 576 aprint_verbose_dev(dev, "feature cap %s\n", buf); 577 snprintb(buf, sizeof(buf), IXGBE_FEATURE_FLAGS, adapter->feat_en); 578 aprint_verbose_dev(dev, "feature ena %s\n", buf); 579 580 INIT_DEBUGOUT("ixv_attach: end"); 581 adapter->osdep.attached = true; 582 583 return; 584 585 err_late: 586 ixgbe_free_queues(adapter); 587 err_out: 588 ixv_free_pci_resources(adapter); 589 IXGBE_CORE_LOCK_DESTROY(adapter); 590 591 return; 592 } /* ixv_attach */ 593 594 /************************************************************************ 595 * ixv_detach - Device removal routine 596 * 597 * Called when the driver is being removed. 598 * Stops the adapter and deallocates all the resources 599 * that were allocated for driver operation. 600 * 601 * return 0 on success, positive on failure 602 ************************************************************************/ 603 static int 604 ixv_detach(device_t dev, int flags) 605 { 606 struct adapter *adapter = device_private(dev); 607 struct ixgbe_hw *hw = &adapter->hw; 608 struct tx_ring *txr = adapter->tx_rings; 609 struct rx_ring *rxr = adapter->rx_rings; 610 struct ixgbevf_hw_stats *stats = &adapter->stats.vf; 611 612 INIT_DEBUGOUT("ixv_detach: begin"); 613 if (adapter->osdep.attached == false) 614 return 0; 615 616 /* Stop the interface. Callouts are stopped in it. */ 617 ixv_ifstop(adapter->ifp, 1); 618 619 if (VLAN_ATTACHED(&adapter->osdep.ec) && 620 (flags & (DETACH_SHUTDOWN | DETACH_FORCE)) == 0) { 621 aprint_error_dev(dev, "VLANs in use, detach first\n"); 622 return EBUSY; 623 } 624 625 ether_ifdetach(adapter->ifp); 626 callout_halt(&adapter->timer, NULL); 627 ixv_free_deferred_handlers(adapter); 628 629 if (adapter->feat_en & IXGBE_FEATURE_NETMAP) 630 netmap_detach(adapter->ifp); 631 632 ixv_free_pci_resources(adapter); 633 #if 0 /* XXX the NetBSD port is probably missing something here */ 634 bus_generic_detach(dev); 635 #endif 636 if_detach(adapter->ifp); 637 ifmedia_fini(&adapter->media); 638 if_percpuq_destroy(adapter->ipq); 639 640 sysctl_teardown(&adapter->sysctllog); 641 evcnt_detach(&adapter->efbig_tx_dma_setup); 642 evcnt_detach(&adapter->mbuf_defrag_failed); 643 evcnt_detach(&adapter->efbig2_tx_dma_setup); 644 evcnt_detach(&adapter->einval_tx_dma_setup); 645 evcnt_detach(&adapter->other_tx_dma_setup); 646 evcnt_detach(&adapter->eagain_tx_dma_setup); 647 evcnt_detach(&adapter->enomem_tx_dma_setup); 648 evcnt_detach(&adapter->watchdog_events); 649 evcnt_detach(&adapter->tso_err); 650 evcnt_detach(&adapter->admin_irqev); 651 evcnt_detach(&adapter->link_workev); 652 653 txr = adapter->tx_rings; 654 for (int i = 0; i < adapter->num_queues; i++, rxr++, txr++) { 655 evcnt_detach(&adapter->queues[i].irqs); 656 evcnt_detach(&adapter->queues[i].handleq); 657 evcnt_detach(&adapter->queues[i].req); 658 evcnt_detach(&txr->total_packets); 659 #ifndef IXGBE_LEGACY_TX 660 evcnt_detach(&txr->pcq_drops); 661 #endif 662 evcnt_detach(&txr->no_desc_avail); 663 evcnt_detach(&txr->tso_tx); 664 665 evcnt_detach(&rxr->rx_packets); 666 evcnt_detach(&rxr->rx_bytes); 667 evcnt_detach(&rxr->rx_copies); 668 evcnt_detach(&rxr->no_mbuf); 669 evcnt_detach(&rxr->rx_discarded); 670 } 671 evcnt_detach(&stats->ipcs); 672 evcnt_detach(&stats->l4cs); 673 evcnt_detach(&stats->ipcs_bad); 674 evcnt_detach(&stats->l4cs_bad); 675 676 /* Packet Reception Stats */ 677 evcnt_detach(&stats->vfgorc); 678 evcnt_detach(&stats->vfgprc); 679 evcnt_detach(&stats->vfmprc); 680 681 /* Packet Transmission Stats */ 682 evcnt_detach(&stats->vfgotc); 683 evcnt_detach(&stats->vfgptc); 684 685 /* Mailbox Stats */ 686 evcnt_detach(&hw->mbx.stats.msgs_tx); 687 evcnt_detach(&hw->mbx.stats.msgs_rx); 688 evcnt_detach(&hw->mbx.stats.acks); 689 evcnt_detach(&hw->mbx.stats.reqs); 690 evcnt_detach(&hw->mbx.stats.rsts); 691 692 ixgbe_free_queues(adapter); 693 694 IXGBE_CORE_LOCK_DESTROY(adapter); 695 696 return (0); 697 } /* ixv_detach */ 698 699 /************************************************************************ 700 * ixv_init_locked - Init entry point 701 * 702 * Used in two ways: It is used by the stack as an init entry 703 * point in network interface structure. It is also used 704 * by the driver as a hw/sw initialization routine to get 705 * to a consistent state. 706 * 707 * return 0 on success, positive on failure 708 ************************************************************************/ 709 static void 710 ixv_init_locked(struct adapter *adapter) 711 { 712 struct ifnet *ifp = adapter->ifp; 713 device_t dev = adapter->dev; 714 struct ixgbe_hw *hw = &adapter->hw; 715 struct ix_queue *que; 716 int error = 0; 717 uint32_t mask; 718 int i; 719 720 INIT_DEBUGOUT("ixv_init_locked: begin"); 721 KASSERT(mutex_owned(&adapter->core_mtx)); 722 hw->adapter_stopped = FALSE; 723 hw->mac.ops.stop_adapter(hw); 724 callout_stop(&adapter->timer); 725 for (i = 0, que = adapter->queues; i < adapter->num_queues; i++, que++) 726 que->disabled_count = 0; 727 728 adapter->max_frame_size = 729 ifp->if_mtu + ETHER_HDR_LEN + ETHER_CRC_LEN; 730 731 /* reprogram the RAR[0] in case user changed it. */ 732 hw->mac.ops.set_rar(hw, 0, hw->mac.addr, 0, IXGBE_RAH_AV); 733 734 /* Get the latest mac address, User can use a LAA */ 735 memcpy(hw->mac.addr, CLLADDR(ifp->if_sadl), 736 IXGBE_ETH_LENGTH_OF_ADDRESS); 737 hw->mac.ops.set_rar(hw, 0, hw->mac.addr, 0, 1); 738 739 /* Prepare transmit descriptors and buffers */ 740 if (ixgbe_setup_transmit_structures(adapter)) { 741 aprint_error_dev(dev, "Could not setup transmit structures\n"); 742 ixv_stop_locked(adapter); 743 return; 744 } 745 746 /* Reset VF and renegotiate mailbox API version */ 747 hw->mac.ops.reset_hw(hw); 748 hw->mac.ops.start_hw(hw); 749 error = ixv_negotiate_api(adapter); 750 if (error) 751 device_printf(dev, 752 "Mailbox API negotiation failed in init_locked!\n"); 753 754 ixv_initialize_transmit_units(adapter); 755 756 /* Setup Multicast table */ 757 ixv_set_rxfilter(adapter); 758 759 /* Use fixed buffer size, even for jumbo frames */ 760 adapter->rx_mbuf_sz = MCLBYTES; 761 762 /* Prepare receive descriptors and buffers */ 763 error = ixgbe_setup_receive_structures(adapter); 764 if (error) { 765 device_printf(dev, 766 "Could not setup receive structures (err = %d)\n", error); 767 ixv_stop_locked(adapter); 768 return; 769 } 770 771 /* Configure RX settings */ 772 ixv_initialize_receive_units(adapter); 773 774 /* Initialize variable holding task enqueue requests interrupts */ 775 adapter->task_requests = 0; 776 777 /* Set up VLAN offload and filter */ 778 ixv_setup_vlan_support(adapter); 779 780 /* Set up MSI-X routing */ 781 ixv_configure_ivars(adapter); 782 783 /* Set up auto-mask */ 784 mask = (1 << adapter->vector); 785 for (i = 0, que = adapter->queues; i < adapter->num_queues; i++, que++) 786 mask |= (1 << que->msix); 787 IXGBE_WRITE_REG(hw, IXGBE_VTEIAM, mask); 788 789 /* Set moderation on the Link interrupt */ 790 ixv_eitr_write(adapter, adapter->vector, IXGBE_LINK_ITR); 791 792 /* Stats init */ 793 ixv_init_stats(adapter); 794 795 /* Config/Enable Link */ 796 error = hw->mac.ops.get_link_state(hw, &adapter->link_enabled); 797 if (error) { 798 /* PF is not capable of controlling VF state. Enable the link. */ 799 adapter->link_enabled = TRUE; 800 } else if (adapter->link_enabled == FALSE) 801 device_printf(dev, "VF is disabled by PF\n"); 802 803 hw->mac.get_link_status = TRUE; 804 hw->mac.ops.check_link(hw, &adapter->link_speed, &adapter->link_up, 805 FALSE); 806 807 /* Start watchdog */ 808 callout_reset(&adapter->timer, hz, ixv_local_timer, adapter); 809 atomic_store_relaxed(&adapter->timer_pending, 0); 810 811 /* OK to schedule workqueues. */ 812 adapter->schedule_wqs_ok = true; 813 814 /* And now turn on interrupts */ 815 ixv_enable_intr(adapter); 816 817 /* Update saved flags. See ixgbe_ifflags_cb() */ 818 adapter->if_flags = ifp->if_flags; 819 adapter->ec_capenable = adapter->osdep.ec.ec_capenable; 820 821 /* Now inform the stack we're ready */ 822 ifp->if_flags |= IFF_RUNNING; 823 ifp->if_flags &= ~IFF_OACTIVE; 824 825 return; 826 } /* ixv_init_locked */ 827 828 /************************************************************************ 829 * ixv_enable_queue 830 ************************************************************************/ 831 static inline void 832 ixv_enable_queue(struct adapter *adapter, u32 vector) 833 { 834 struct ixgbe_hw *hw = &adapter->hw; 835 struct ix_queue *que = &adapter->queues[vector]; 836 u32 queue = 1UL << vector; 837 u32 mask; 838 839 mutex_enter(&que->dc_mtx); 840 if (que->disabled_count > 0 && --que->disabled_count > 0) 841 goto out; 842 843 mask = (IXGBE_EIMS_RTX_QUEUE & queue); 844 IXGBE_WRITE_REG(hw, IXGBE_VTEIMS, mask); 845 out: 846 mutex_exit(&que->dc_mtx); 847 } /* ixv_enable_queue */ 848 849 /************************************************************************ 850 * ixv_disable_queue 851 ************************************************************************/ 852 static inline void 853 ixv_disable_queue(struct adapter *adapter, u32 vector) 854 { 855 struct ixgbe_hw *hw = &adapter->hw; 856 struct ix_queue *que = &adapter->queues[vector]; 857 u32 queue = 1UL << vector; 858 u32 mask; 859 860 mutex_enter(&que->dc_mtx); 861 if (que->disabled_count++ > 0) 862 goto out; 863 864 mask = (IXGBE_EIMS_RTX_QUEUE & queue); 865 IXGBE_WRITE_REG(hw, IXGBE_VTEIMC, mask); 866 out: 867 mutex_exit(&que->dc_mtx); 868 } /* ixv_disable_queue */ 869 870 #if 0 871 static inline void 872 ixv_rearm_queues(struct adapter *adapter, u64 queues) 873 { 874 u32 mask = (IXGBE_EIMS_RTX_QUEUE & queues); 875 IXGBE_WRITE_REG(&adapter->hw, IXGBE_VTEICS, mask); 876 } /* ixv_rearm_queues */ 877 #endif 878 879 880 /************************************************************************ 881 * ixv_msix_que - MSI-X Queue Interrupt Service routine 882 ************************************************************************/ 883 static int 884 ixv_msix_que(void *arg) 885 { 886 struct ix_queue *que = arg; 887 struct adapter *adapter = que->adapter; 888 struct tx_ring *txr = que->txr; 889 struct rx_ring *rxr = que->rxr; 890 bool more; 891 u32 newitr = 0; 892 893 ixv_disable_queue(adapter, que->msix); 894 IXGBE_EVC_ADD(&que->irqs, 1); 895 896 #ifdef __NetBSD__ 897 /* Don't run ixgbe_rxeof in interrupt context */ 898 more = true; 899 #else 900 more = ixgbe_rxeof(que); 901 #endif 902 903 IXGBE_TX_LOCK(txr); 904 ixgbe_txeof(txr); 905 IXGBE_TX_UNLOCK(txr); 906 907 /* Do AIM now? */ 908 909 if (adapter->enable_aim == false) 910 goto no_calc; 911 /* 912 * Do Adaptive Interrupt Moderation: 913 * - Write out last calculated setting 914 * - Calculate based on average size over 915 * the last interval. 916 */ 917 if (que->eitr_setting) 918 ixv_eitr_write(adapter, que->msix, que->eitr_setting); 919 920 que->eitr_setting = 0; 921 922 /* Idle, do nothing */ 923 if ((txr->bytes == 0) && (rxr->bytes == 0)) 924 goto no_calc; 925 926 if ((txr->bytes) && (txr->packets)) 927 newitr = txr->bytes/txr->packets; 928 if ((rxr->bytes) && (rxr->packets)) 929 newitr = uimax(newitr, (rxr->bytes / rxr->packets)); 930 newitr += 24; /* account for hardware frame, crc */ 931 932 /* set an upper boundary */ 933 newitr = uimin(newitr, 3000); 934 935 /* Be nice to the mid range */ 936 if ((newitr > 300) && (newitr < 1200)) 937 newitr = (newitr / 3); 938 else 939 newitr = (newitr / 2); 940 941 /* 942 * When RSC is used, ITR interval must be larger than RSC_DELAY. 943 * Currently, we use 2us for RSC_DELAY. The minimum value is always 944 * greater than 2us on 100M (and 10M?(not documented)), but it's not 945 * on 1G and higher. 946 */ 947 if ((adapter->link_speed != IXGBE_LINK_SPEED_100_FULL) 948 && (adapter->link_speed != IXGBE_LINK_SPEED_10_FULL)) { 949 if (newitr < IXGBE_MIN_RSC_EITR_10G1G) 950 newitr = IXGBE_MIN_RSC_EITR_10G1G; 951 } 952 953 /* save for next interrupt */ 954 que->eitr_setting = newitr; 955 956 /* Reset state */ 957 txr->bytes = 0; 958 txr->packets = 0; 959 rxr->bytes = 0; 960 rxr->packets = 0; 961 962 no_calc: 963 if (more) 964 softint_schedule(que->que_si); 965 else /* Re-enable this interrupt */ 966 ixv_enable_queue(adapter, que->msix); 967 968 return 1; 969 } /* ixv_msix_que */ 970 971 /************************************************************************ 972 * ixv_msix_mbx 973 ************************************************************************/ 974 static int 975 ixv_msix_mbx(void *arg) 976 { 977 struct adapter *adapter = arg; 978 struct ixgbe_hw *hw = &adapter->hw; 979 980 IXGBE_EVC_ADD(&adapter->admin_irqev, 1); 981 /* NetBSD: We use auto-clear, so it's not required to write VTEICR */ 982 983 /* Link status change */ 984 hw->mac.get_link_status = TRUE; 985 atomic_or_32(&adapter->task_requests, IXGBE_REQUEST_TASK_MBX); 986 ixv_schedule_admin_tasklet(adapter); 987 988 return 1; 989 } /* ixv_msix_mbx */ 990 991 static void 992 ixv_eitr_write(struct adapter *adapter, uint32_t index, uint32_t itr) 993 { 994 995 /* 996 * Newer devices than 82598 have VF function, so this function is 997 * simple. 998 */ 999 itr |= IXGBE_EITR_CNT_WDIS; 1000 1001 IXGBE_WRITE_REG(&adapter->hw, IXGBE_VTEITR(index), itr); 1002 } 1003 1004 1005 /************************************************************************ 1006 * ixv_media_status - Media Ioctl callback 1007 * 1008 * Called whenever the user queries the status of 1009 * the interface using ifconfig. 1010 ************************************************************************/ 1011 static void 1012 ixv_media_status(struct ifnet *ifp, struct ifmediareq *ifmr) 1013 { 1014 struct adapter *adapter = ifp->if_softc; 1015 1016 INIT_DEBUGOUT("ixv_media_status: begin"); 1017 ixv_update_link_status(adapter); 1018 1019 ifmr->ifm_status = IFM_AVALID; 1020 ifmr->ifm_active = IFM_ETHER; 1021 1022 if (adapter->link_active != LINK_STATE_UP) { 1023 ifmr->ifm_active |= IFM_NONE; 1024 return; 1025 } 1026 1027 ifmr->ifm_status |= IFM_ACTIVE; 1028 1029 switch (adapter->link_speed) { 1030 case IXGBE_LINK_SPEED_10GB_FULL: 1031 ifmr->ifm_active |= IFM_10G_T | IFM_FDX; 1032 break; 1033 case IXGBE_LINK_SPEED_5GB_FULL: 1034 ifmr->ifm_active |= IFM_5000_T | IFM_FDX; 1035 break; 1036 case IXGBE_LINK_SPEED_2_5GB_FULL: 1037 ifmr->ifm_active |= IFM_2500_T | IFM_FDX; 1038 break; 1039 case IXGBE_LINK_SPEED_1GB_FULL: 1040 ifmr->ifm_active |= IFM_1000_T | IFM_FDX; 1041 break; 1042 case IXGBE_LINK_SPEED_100_FULL: 1043 ifmr->ifm_active |= IFM_100_TX | IFM_FDX; 1044 break; 1045 case IXGBE_LINK_SPEED_10_FULL: 1046 ifmr->ifm_active |= IFM_10_T | IFM_FDX; 1047 break; 1048 } 1049 1050 ifp->if_baudrate = ifmedia_baudrate(ifmr->ifm_active); 1051 } /* ixv_media_status */ 1052 1053 /************************************************************************ 1054 * ixv_media_change - Media Ioctl callback 1055 * 1056 * Called when the user changes speed/duplex using 1057 * media/mediopt option with ifconfig. 1058 ************************************************************************/ 1059 static int 1060 ixv_media_change(struct ifnet *ifp) 1061 { 1062 struct adapter *adapter = ifp->if_softc; 1063 struct ifmedia *ifm = &adapter->media; 1064 1065 INIT_DEBUGOUT("ixv_media_change: begin"); 1066 1067 if (IFM_TYPE(ifm->ifm_media) != IFM_ETHER) 1068 return (EINVAL); 1069 1070 switch (IFM_SUBTYPE(ifm->ifm_media)) { 1071 case IFM_AUTO: 1072 break; 1073 default: 1074 device_printf(adapter->dev, "Only auto media type\n"); 1075 return (EINVAL); 1076 } 1077 1078 return (0); 1079 } /* ixv_media_change */ 1080 1081 static void 1082 ixv_schedule_admin_tasklet(struct adapter *adapter) 1083 { 1084 if (adapter->schedule_wqs_ok) { 1085 if (atomic_cas_uint(&adapter->admin_pending, 0, 1) == 0) 1086 workqueue_enqueue(adapter->admin_wq, 1087 &adapter->admin_wc, NULL); 1088 } 1089 } 1090 1091 /************************************************************************ 1092 * ixv_negotiate_api 1093 * 1094 * Negotiate the Mailbox API with the PF; 1095 * start with the most featured API first. 1096 ************************************************************************/ 1097 static int 1098 ixv_negotiate_api(struct adapter *adapter) 1099 { 1100 struct ixgbe_hw *hw = &adapter->hw; 1101 int mbx_api[] = { ixgbe_mbox_api_15, 1102 ixgbe_mbox_api_13, 1103 ixgbe_mbox_api_12, 1104 ixgbe_mbox_api_11, 1105 ixgbe_mbox_api_10, 1106 ixgbe_mbox_api_unknown }; 1107 int i = 0; 1108 1109 while (mbx_api[i] != ixgbe_mbox_api_unknown) { 1110 if (ixgbevf_negotiate_api_version(hw, mbx_api[i]) == 0) { 1111 if (hw->api_version >= ixgbe_mbox_api_15) 1112 ixgbe_upgrade_mbx_params_vf(hw); 1113 return (0); 1114 } 1115 i++; 1116 } 1117 1118 return (EINVAL); 1119 } /* ixv_negotiate_api */ 1120 1121 1122 /************************************************************************ 1123 * ixv_set_rxfilter - Multicast Update 1124 * 1125 * Called whenever multicast address list is updated. 1126 ************************************************************************/ 1127 static int 1128 ixv_set_rxfilter(struct adapter *adapter) 1129 { 1130 struct ixgbe_mc_addr *mta; 1131 struct ifnet *ifp = adapter->ifp; 1132 struct ixgbe_hw *hw = &adapter->hw; 1133 u8 *update_ptr; 1134 int mcnt = 0; 1135 struct ethercom *ec = &adapter->osdep.ec; 1136 struct ether_multi *enm; 1137 struct ether_multistep step; 1138 bool overflow = false; 1139 int error, rc = 0; 1140 1141 KASSERT(mutex_owned(&adapter->core_mtx)); 1142 IOCTL_DEBUGOUT("ixv_set_rxfilter: begin"); 1143 1144 mta = adapter->mta; 1145 bzero(mta, sizeof(*mta) * IXGBE_MAX_VF_MC); 1146 1147 /* 1: For PROMISC */ 1148 if (ifp->if_flags & IFF_PROMISC) { 1149 error = hw->mac.ops.update_xcast_mode(hw, 1150 IXGBEVF_XCAST_MODE_PROMISC); 1151 if (error == IXGBE_ERR_NOT_TRUSTED) { 1152 device_printf(adapter->dev, 1153 "this interface is not trusted\n"); 1154 error = EPERM; 1155 } else if (error == IXGBE_ERR_FEATURE_NOT_SUPPORTED) { 1156 device_printf(adapter->dev, 1157 "the PF doesn't support promisc mode\n"); 1158 error = EOPNOTSUPP; 1159 } else if (error == IXGBE_ERR_NOT_IN_PROMISC) { 1160 device_printf(adapter->dev, 1161 "the PF may not in promisc mode\n"); 1162 error = EINVAL; 1163 } else if (error) { 1164 device_printf(adapter->dev, 1165 "failed to set promisc mode. error = %d\n", 1166 error); 1167 error = EIO; 1168 } else 1169 return 0; 1170 rc = error; 1171 } 1172 1173 /* 2: For ALLMULTI or normal */ 1174 ETHER_LOCK(ec); 1175 ETHER_FIRST_MULTI(step, ec, enm); 1176 while (enm != NULL) { 1177 if ((mcnt >= IXGBE_MAX_VF_MC) || 1178 (memcmp(enm->enm_addrlo, enm->enm_addrhi, 1179 ETHER_ADDR_LEN) != 0)) { 1180 overflow = true; 1181 break; 1182 } 1183 bcopy(enm->enm_addrlo, 1184 mta[mcnt].addr, IXGBE_ETH_LENGTH_OF_ADDRESS); 1185 mcnt++; 1186 ETHER_NEXT_MULTI(step, enm); 1187 } 1188 ETHER_UNLOCK(ec); 1189 1190 /* 3: For ALLMULTI */ 1191 if (overflow) { 1192 error = hw->mac.ops.update_xcast_mode(hw, 1193 IXGBEVF_XCAST_MODE_ALLMULTI); 1194 if (error == IXGBE_ERR_NOT_TRUSTED) { 1195 device_printf(adapter->dev, 1196 "this interface is not trusted\n"); 1197 error = EPERM; 1198 } else if (error == IXGBE_ERR_FEATURE_NOT_SUPPORTED) { 1199 device_printf(adapter->dev, 1200 "the PF doesn't support allmulti mode\n"); 1201 error = EOPNOTSUPP; 1202 } else if (error) { 1203 device_printf(adapter->dev, 1204 "number of Ethernet multicast addresses " 1205 "exceeds the limit (%d). error = %d\n", 1206 IXGBE_MAX_VF_MC, error); 1207 error = ENOSPC; 1208 } else { 1209 ETHER_LOCK(ec); 1210 ec->ec_flags |= ETHER_F_ALLMULTI; 1211 ETHER_UNLOCK(ec); 1212 return rc; /* Promisc might have failed */ 1213 } 1214 1215 if (rc == 0) 1216 rc = error; 1217 1218 /* Continue to update the multicast table as many as we can */ 1219 } 1220 1221 /* 4: For normal operation */ 1222 error = hw->mac.ops.update_xcast_mode(hw, IXGBEVF_XCAST_MODE_MULTI); 1223 if ((error == IXGBE_ERR_FEATURE_NOT_SUPPORTED) || (error == 0)) { 1224 /* Normal operation */ 1225 ETHER_LOCK(ec); 1226 ec->ec_flags &= ~ETHER_F_ALLMULTI; 1227 ETHER_UNLOCK(ec); 1228 error = 0; 1229 } else if (error) { 1230 device_printf(adapter->dev, 1231 "failed to set Ethernet multicast address " 1232 "operation to normal. error = %d\n", error); 1233 } 1234 1235 update_ptr = (u8 *)mta; 1236 error = adapter->hw.mac.ops.update_mc_addr_list(&adapter->hw, 1237 update_ptr, mcnt, ixv_mc_array_itr, TRUE); 1238 if (rc == 0) 1239 rc = error; 1240 1241 return rc; 1242 } /* ixv_set_rxfilter */ 1243 1244 /************************************************************************ 1245 * ixv_mc_array_itr 1246 * 1247 * An iterator function needed by the multicast shared code. 1248 * It feeds the shared code routine the addresses in the 1249 * array of ixv_set_rxfilter() one by one. 1250 ************************************************************************/ 1251 static u8 * 1252 ixv_mc_array_itr(struct ixgbe_hw *hw, u8 **update_ptr, u32 *vmdq) 1253 { 1254 struct ixgbe_mc_addr *mta; 1255 1256 mta = (struct ixgbe_mc_addr *)*update_ptr; 1257 1258 *vmdq = 0; 1259 *update_ptr = (u8*)(mta + 1); 1260 1261 return (mta->addr); 1262 } /* ixv_mc_array_itr */ 1263 1264 /************************************************************************ 1265 * ixv_local_timer - Timer routine 1266 * 1267 * Checks for link status, updates statistics, 1268 * and runs the watchdog check. 1269 ************************************************************************/ 1270 static void 1271 ixv_local_timer(void *arg) 1272 { 1273 struct adapter *adapter = arg; 1274 1275 if (adapter->schedule_wqs_ok) { 1276 if (atomic_cas_uint(&adapter->timer_pending, 0, 1) == 0) 1277 workqueue_enqueue(adapter->timer_wq, 1278 &adapter->timer_wc, NULL); 1279 } 1280 } 1281 1282 static void 1283 ixv_handle_timer(struct work *wk, void *context) 1284 { 1285 struct adapter *adapter = context; 1286 device_t dev = adapter->dev; 1287 struct ix_queue *que = adapter->queues; 1288 u64 queues = 0; 1289 u64 v0, v1, v2, v3, v4, v5, v6, v7; 1290 int hung = 0; 1291 int i; 1292 1293 IXGBE_CORE_LOCK(adapter); 1294 1295 if (ixv_check_link(adapter)) { 1296 ixv_init_locked(adapter); 1297 IXGBE_CORE_UNLOCK(adapter); 1298 return; 1299 } 1300 1301 /* Stats Update */ 1302 ixv_update_stats(adapter); 1303 1304 /* Update some event counters */ 1305 v0 = v1 = v2 = v3 = v4 = v5 = v6 = v7 = 0; 1306 que = adapter->queues; 1307 for (i = 0; i < adapter->num_queues; i++, que++) { 1308 struct tx_ring *txr = que->txr; 1309 1310 v0 += txr->q_efbig_tx_dma_setup; 1311 v1 += txr->q_mbuf_defrag_failed; 1312 v2 += txr->q_efbig2_tx_dma_setup; 1313 v3 += txr->q_einval_tx_dma_setup; 1314 v4 += txr->q_other_tx_dma_setup; 1315 v5 += txr->q_eagain_tx_dma_setup; 1316 v6 += txr->q_enomem_tx_dma_setup; 1317 v7 += txr->q_tso_err; 1318 } 1319 IXGBE_EVC_STORE(&adapter->efbig_tx_dma_setup, v0); 1320 IXGBE_EVC_STORE(&adapter->mbuf_defrag_failed, v1); 1321 IXGBE_EVC_STORE(&adapter->efbig2_tx_dma_setup, v2); 1322 IXGBE_EVC_STORE(&adapter->einval_tx_dma_setup, v3); 1323 IXGBE_EVC_STORE(&adapter->other_tx_dma_setup, v4); 1324 IXGBE_EVC_STORE(&adapter->eagain_tx_dma_setup, v5); 1325 IXGBE_EVC_STORE(&adapter->enomem_tx_dma_setup, v6); 1326 IXGBE_EVC_STORE(&adapter->tso_err, v7); 1327 1328 /* 1329 * Check the TX queues status 1330 * - mark hung queues so we don't schedule on them 1331 * - watchdog only if all queues show hung 1332 */ 1333 que = adapter->queues; 1334 for (i = 0; i < adapter->num_queues; i++, que++) { 1335 /* Keep track of queues with work for soft irq */ 1336 if (que->txr->busy) 1337 queues |= ((u64)1 << que->me); 1338 /* 1339 * Each time txeof runs without cleaning, but there 1340 * are uncleaned descriptors it increments busy. If 1341 * we get to the MAX we declare it hung. 1342 */ 1343 if (que->busy == IXGBE_QUEUE_HUNG) { 1344 ++hung; 1345 /* Mark the queue as inactive */ 1346 adapter->active_queues &= ~((u64)1 << que->me); 1347 continue; 1348 } else { 1349 /* Check if we've come back from hung */ 1350 if ((adapter->active_queues & ((u64)1 << que->me)) == 0) 1351 adapter->active_queues |= ((u64)1 << que->me); 1352 } 1353 if (que->busy >= IXGBE_MAX_TX_BUSY) { 1354 device_printf(dev, 1355 "Warning queue %d appears to be hung!\n", i); 1356 que->txr->busy = IXGBE_QUEUE_HUNG; 1357 ++hung; 1358 } 1359 } 1360 1361 /* Only truly watchdog if all queues show hung */ 1362 if (hung == adapter->num_queues) 1363 goto watchdog; 1364 #if 0 1365 else if (queues != 0) { /* Force an IRQ on queues with work */ 1366 ixv_rearm_queues(adapter, queues); 1367 } 1368 #endif 1369 1370 atomic_store_relaxed(&adapter->timer_pending, 0); 1371 IXGBE_CORE_UNLOCK(adapter); 1372 callout_reset(&adapter->timer, hz, ixv_local_timer, adapter); 1373 1374 return; 1375 1376 watchdog: 1377 device_printf(adapter->dev, "Watchdog timeout -- resetting\n"); 1378 adapter->ifp->if_flags &= ~IFF_RUNNING; 1379 IXGBE_EVC_ADD(&adapter->watchdog_events, 1); 1380 ixv_init_locked(adapter); 1381 IXGBE_CORE_UNLOCK(adapter); 1382 } /* ixv_handle_timer */ 1383 1384 /************************************************************************ 1385 * ixv_update_link_status - Update OS on link state 1386 * 1387 * Note: Only updates the OS on the cached link state. 1388 * The real check of the hardware only happens with 1389 * a link interrupt. 1390 ************************************************************************/ 1391 static void 1392 ixv_update_link_status(struct adapter *adapter) 1393 { 1394 struct ifnet *ifp = adapter->ifp; 1395 device_t dev = adapter->dev; 1396 1397 KASSERT(mutex_owned(&adapter->core_mtx)); 1398 1399 if (adapter->link_up && adapter->link_enabled) { 1400 if (adapter->link_active != LINK_STATE_UP) { 1401 if (bootverbose) { 1402 const char *bpsmsg; 1403 1404 switch (adapter->link_speed) { 1405 case IXGBE_LINK_SPEED_10GB_FULL: 1406 bpsmsg = "10 Gbps"; 1407 break; 1408 case IXGBE_LINK_SPEED_5GB_FULL: 1409 bpsmsg = "5 Gbps"; 1410 break; 1411 case IXGBE_LINK_SPEED_2_5GB_FULL: 1412 bpsmsg = "2.5 Gbps"; 1413 break; 1414 case IXGBE_LINK_SPEED_1GB_FULL: 1415 bpsmsg = "1 Gbps"; 1416 break; 1417 case IXGBE_LINK_SPEED_100_FULL: 1418 bpsmsg = "100 Mbps"; 1419 break; 1420 case IXGBE_LINK_SPEED_10_FULL: 1421 bpsmsg = "10 Mbps"; 1422 break; 1423 default: 1424 bpsmsg = "unknown speed"; 1425 break; 1426 } 1427 device_printf(dev, "Link is up %s %s \n", 1428 bpsmsg, "Full Duplex"); 1429 } 1430 adapter->link_active = LINK_STATE_UP; 1431 if_link_state_change(ifp, LINK_STATE_UP); 1432 } 1433 } else { 1434 /* 1435 * Do it when link active changes to DOWN. i.e. 1436 * a) LINK_STATE_UNKNOWN -> LINK_STATE_DOWN 1437 * b) LINK_STATE_UP -> LINK_STATE_DOWN 1438 */ 1439 if (adapter->link_active != LINK_STATE_DOWN) { 1440 if (bootverbose) 1441 device_printf(dev, "Link is Down\n"); 1442 if_link_state_change(ifp, LINK_STATE_DOWN); 1443 adapter->link_active = LINK_STATE_DOWN; 1444 } 1445 } 1446 } /* ixv_update_link_status */ 1447 1448 1449 /************************************************************************ 1450 * ixv_stop - Stop the hardware 1451 * 1452 * Disables all traffic on the adapter by issuing a 1453 * global reset on the MAC and deallocates TX/RX buffers. 1454 ************************************************************************/ 1455 static void 1456 ixv_ifstop(struct ifnet *ifp, int disable) 1457 { 1458 struct adapter *adapter = ifp->if_softc; 1459 1460 IXGBE_CORE_LOCK(adapter); 1461 ixv_stop_locked(adapter); 1462 IXGBE_CORE_UNLOCK(adapter); 1463 1464 workqueue_wait(adapter->admin_wq, &adapter->admin_wc); 1465 atomic_store_relaxed(&adapter->admin_pending, 0); 1466 workqueue_wait(adapter->timer_wq, &adapter->timer_wc); 1467 atomic_store_relaxed(&adapter->timer_pending, 0); 1468 } 1469 1470 static void 1471 ixv_stop_locked(void *arg) 1472 { 1473 struct ifnet *ifp; 1474 struct adapter *adapter = arg; 1475 struct ixgbe_hw *hw = &adapter->hw; 1476 1477 ifp = adapter->ifp; 1478 1479 KASSERT(mutex_owned(&adapter->core_mtx)); 1480 1481 INIT_DEBUGOUT("ixv_stop_locked: begin\n"); 1482 ixv_disable_intr(adapter); 1483 1484 /* Tell the stack that the interface is no longer active */ 1485 ifp->if_flags &= ~(IFF_RUNNING | IFF_OACTIVE); 1486 1487 hw->mac.ops.reset_hw(hw); 1488 adapter->hw.adapter_stopped = FALSE; 1489 hw->mac.ops.stop_adapter(hw); 1490 callout_stop(&adapter->timer); 1491 1492 /* Don't schedule workqueues. */ 1493 adapter->schedule_wqs_ok = false; 1494 1495 /* reprogram the RAR[0] in case user changed it. */ 1496 hw->mac.ops.set_rar(hw, 0, hw->mac.addr, 0, IXGBE_RAH_AV); 1497 1498 return; 1499 } /* ixv_stop_locked */ 1500 1501 1502 /************************************************************************ 1503 * ixv_allocate_pci_resources 1504 ************************************************************************/ 1505 static int 1506 ixv_allocate_pci_resources(struct adapter *adapter, 1507 const struct pci_attach_args *pa) 1508 { 1509 pcireg_t memtype, csr; 1510 device_t dev = adapter->dev; 1511 bus_addr_t addr; 1512 int flags; 1513 1514 memtype = pci_mapreg_type(pa->pa_pc, pa->pa_tag, PCI_BAR(0)); 1515 switch (memtype) { 1516 case PCI_MAPREG_TYPE_MEM | PCI_MAPREG_MEM_TYPE_32BIT: 1517 case PCI_MAPREG_TYPE_MEM | PCI_MAPREG_MEM_TYPE_64BIT: 1518 adapter->osdep.mem_bus_space_tag = pa->pa_memt; 1519 if (pci_mapreg_info(pa->pa_pc, pa->pa_tag, PCI_BAR(0), 1520 memtype, &addr, &adapter->osdep.mem_size, &flags) != 0) 1521 goto map_err; 1522 if ((flags & BUS_SPACE_MAP_PREFETCHABLE) != 0) { 1523 aprint_normal_dev(dev, "clearing prefetchable bit\n"); 1524 flags &= ~BUS_SPACE_MAP_PREFETCHABLE; 1525 } 1526 if (bus_space_map(adapter->osdep.mem_bus_space_tag, addr, 1527 adapter->osdep.mem_size, flags, 1528 &adapter->osdep.mem_bus_space_handle) != 0) { 1529 map_err: 1530 adapter->osdep.mem_size = 0; 1531 aprint_error_dev(dev, "unable to map BAR0\n"); 1532 return ENXIO; 1533 } 1534 /* 1535 * Enable address decoding for memory range in case it's not 1536 * set. 1537 */ 1538 csr = pci_conf_read(pa->pa_pc, pa->pa_tag, 1539 PCI_COMMAND_STATUS_REG); 1540 csr |= PCI_COMMAND_MEM_ENABLE; 1541 pci_conf_write(pa->pa_pc, pa->pa_tag, PCI_COMMAND_STATUS_REG, 1542 csr); 1543 break; 1544 default: 1545 aprint_error_dev(dev, "unexpected type on BAR0\n"); 1546 return ENXIO; 1547 } 1548 1549 /* Pick up the tuneable queues */ 1550 adapter->num_queues = ixv_num_queues; 1551 1552 return (0); 1553 } /* ixv_allocate_pci_resources */ 1554 1555 static void 1556 ixv_free_deferred_handlers(struct adapter *adapter) 1557 { 1558 struct ix_queue *que = adapter->queues; 1559 struct tx_ring *txr = adapter->tx_rings; 1560 int i; 1561 1562 for (i = 0; i < adapter->num_queues; i++, que++, txr++) { 1563 if (!(adapter->feat_en & IXGBE_FEATURE_LEGACY_TX)) { 1564 if (txr->txr_si != NULL) 1565 softint_disestablish(txr->txr_si); 1566 } 1567 if (que->que_si != NULL) 1568 softint_disestablish(que->que_si); 1569 } 1570 if (adapter->txr_wq != NULL) 1571 workqueue_destroy(adapter->txr_wq); 1572 if (adapter->txr_wq_enqueued != NULL) 1573 percpu_free(adapter->txr_wq_enqueued, sizeof(u_int)); 1574 if (adapter->que_wq != NULL) 1575 workqueue_destroy(adapter->que_wq); 1576 1577 /* Drain the Mailbox(link) queue */ 1578 if (adapter->admin_wq != NULL) { 1579 workqueue_destroy(adapter->admin_wq); 1580 adapter->admin_wq = NULL; 1581 } 1582 if (adapter->timer_wq != NULL) { 1583 workqueue_destroy(adapter->timer_wq); 1584 adapter->timer_wq = NULL; 1585 } 1586 } /* ixv_free_deferred_handlers */ 1587 1588 /************************************************************************ 1589 * ixv_free_pci_resources 1590 ************************************************************************/ 1591 static void 1592 ixv_free_pci_resources(struct adapter * adapter) 1593 { 1594 struct ix_queue *que = adapter->queues; 1595 int rid; 1596 1597 /* 1598 * Release all msix queue resources: 1599 */ 1600 for (int i = 0; i < adapter->num_queues; i++, que++) { 1601 if (que->res != NULL) 1602 pci_intr_disestablish(adapter->osdep.pc, 1603 adapter->osdep.ihs[i]); 1604 } 1605 1606 1607 /* Clean the Mailbox interrupt last */ 1608 rid = adapter->vector; 1609 1610 if (adapter->osdep.ihs[rid] != NULL) { 1611 pci_intr_disestablish(adapter->osdep.pc, 1612 adapter->osdep.ihs[rid]); 1613 adapter->osdep.ihs[rid] = NULL; 1614 } 1615 1616 pci_intr_release(adapter->osdep.pc, adapter->osdep.intrs, 1617 adapter->osdep.nintrs); 1618 1619 if (adapter->osdep.mem_size != 0) { 1620 bus_space_unmap(adapter->osdep.mem_bus_space_tag, 1621 adapter->osdep.mem_bus_space_handle, 1622 adapter->osdep.mem_size); 1623 } 1624 1625 return; 1626 } /* ixv_free_pci_resources */ 1627 1628 /************************************************************************ 1629 * ixv_setup_interface 1630 * 1631 * Setup networking device structure and register an interface. 1632 ************************************************************************/ 1633 static int 1634 ixv_setup_interface(device_t dev, struct adapter *adapter) 1635 { 1636 struct ethercom *ec = &adapter->osdep.ec; 1637 struct ifnet *ifp; 1638 1639 INIT_DEBUGOUT("ixv_setup_interface: begin"); 1640 1641 ifp = adapter->ifp = &ec->ec_if; 1642 strlcpy(ifp->if_xname, device_xname(dev), IFNAMSIZ); 1643 ifp->if_baudrate = IF_Gbps(10); 1644 ifp->if_init = ixv_init; 1645 ifp->if_stop = ixv_ifstop; 1646 ifp->if_softc = adapter; 1647 ifp->if_flags = IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST; 1648 #ifdef IXGBE_MPSAFE 1649 ifp->if_extflags = IFEF_MPSAFE; 1650 #endif 1651 ifp->if_ioctl = ixv_ioctl; 1652 if (adapter->feat_en & IXGBE_FEATURE_LEGACY_TX) { 1653 #if 0 1654 ixv_start_locked = ixgbe_legacy_start_locked; 1655 #endif 1656 } else { 1657 ifp->if_transmit = ixgbe_mq_start; 1658 #if 0 1659 ixv_start_locked = ixgbe_mq_start_locked; 1660 #endif 1661 } 1662 ifp->if_start = ixgbe_legacy_start; 1663 IFQ_SET_MAXLEN(&ifp->if_snd, adapter->num_tx_desc - 2); 1664 IFQ_SET_READY(&ifp->if_snd); 1665 1666 if_initialize(ifp); 1667 adapter->ipq = if_percpuq_create(&adapter->osdep.ec.ec_if); 1668 ether_ifattach(ifp, adapter->hw.mac.addr); 1669 aprint_normal_dev(dev, "Ethernet address %s\n", 1670 ether_sprintf(adapter->hw.mac.addr)); 1671 /* 1672 * We use per TX queue softint, so if_deferred_start_init() isn't 1673 * used. 1674 */ 1675 ether_set_ifflags_cb(ec, ixv_ifflags_cb); 1676 1677 adapter->max_frame_size = ifp->if_mtu + IXGBE_MTU_HDR; 1678 1679 /* 1680 * Tell the upper layer(s) we support long frames. 1681 */ 1682 ifp->if_hdrlen = sizeof(struct ether_vlan_header); 1683 1684 /* Set capability flags */ 1685 ifp->if_capabilities |= IFCAP_HWCSUM 1686 | IFCAP_TSOv4 1687 | IFCAP_TSOv6; 1688 ifp->if_capenable = 0; 1689 1690 ec->ec_capabilities |= ETHERCAP_VLAN_HWFILTER 1691 | ETHERCAP_VLAN_HWTAGGING 1692 | ETHERCAP_VLAN_HWCSUM 1693 | ETHERCAP_JUMBO_MTU 1694 | ETHERCAP_VLAN_MTU; 1695 1696 /* Enable the above capabilities by default */ 1697 ec->ec_capenable = ec->ec_capabilities; 1698 1699 /* Don't enable LRO by default */ 1700 #if 0 1701 /* NetBSD doesn't support LRO yet */ 1702 ifp->if_capabilities |= IFCAP_LRO; 1703 #endif 1704 1705 /* 1706 * Specify the media types supported by this adapter and register 1707 * callbacks to update media and link information 1708 */ 1709 ec->ec_ifmedia = &adapter->media; 1710 ifmedia_init_with_lock(&adapter->media, IFM_IMASK, ixv_media_change, 1711 ixv_media_status, &adapter->core_mtx); 1712 ifmedia_add(&adapter->media, IFM_ETHER | IFM_AUTO, 0, NULL); 1713 ifmedia_set(&adapter->media, IFM_ETHER | IFM_AUTO); 1714 1715 if_register(ifp); 1716 1717 return 0; 1718 } /* ixv_setup_interface */ 1719 1720 1721 /************************************************************************ 1722 * ixv_initialize_transmit_units - Enable transmit unit. 1723 ************************************************************************/ 1724 static void 1725 ixv_initialize_transmit_units(struct adapter *adapter) 1726 { 1727 struct tx_ring *txr = adapter->tx_rings; 1728 struct ixgbe_hw *hw = &adapter->hw; 1729 int i; 1730 1731 for (i = 0; i < adapter->num_queues; i++, txr++) { 1732 u64 tdba = txr->txdma.dma_paddr; 1733 u32 txctrl, txdctl; 1734 int j = txr->me; 1735 1736 /* Set WTHRESH to 8, burst writeback */ 1737 txdctl = IXGBE_READ_REG(hw, IXGBE_VFTXDCTL(j)); 1738 txdctl |= IXGBE_TX_WTHRESH << IXGBE_TXDCTL_WTHRESH_SHIFT; 1739 IXGBE_WRITE_REG(hw, IXGBE_VFTXDCTL(j), txdctl); 1740 1741 /* Set the HW Tx Head and Tail indices */ 1742 IXGBE_WRITE_REG(&adapter->hw, IXGBE_VFTDH(j), 0); 1743 IXGBE_WRITE_REG(&adapter->hw, IXGBE_VFTDT(j), 0); 1744 1745 /* Set Tx Tail register */ 1746 txr->tail = IXGBE_VFTDT(j); 1747 1748 txr->txr_no_space = false; 1749 1750 /* Set Ring parameters */ 1751 IXGBE_WRITE_REG(hw, IXGBE_VFTDBAL(j), 1752 (tdba & 0x00000000ffffffffULL)); 1753 IXGBE_WRITE_REG(hw, IXGBE_VFTDBAH(j), (tdba >> 32)); 1754 IXGBE_WRITE_REG(hw, IXGBE_VFTDLEN(j), 1755 adapter->num_tx_desc * sizeof(struct ixgbe_legacy_tx_desc)); 1756 txctrl = IXGBE_READ_REG(hw, IXGBE_VFDCA_TXCTRL(j)); 1757 txctrl &= ~IXGBE_DCA_TXCTRL_DESC_WRO_EN; 1758 IXGBE_WRITE_REG(hw, IXGBE_VFDCA_TXCTRL(j), txctrl); 1759 1760 /* Now enable */ 1761 txdctl = IXGBE_READ_REG(hw, IXGBE_VFTXDCTL(j)); 1762 txdctl |= IXGBE_TXDCTL_ENABLE; 1763 IXGBE_WRITE_REG(hw, IXGBE_VFTXDCTL(j), txdctl); 1764 } 1765 1766 return; 1767 } /* ixv_initialize_transmit_units */ 1768 1769 1770 /************************************************************************ 1771 * ixv_initialize_rss_mapping 1772 ************************************************************************/ 1773 static void 1774 ixv_initialize_rss_mapping(struct adapter *adapter) 1775 { 1776 struct ixgbe_hw *hw = &adapter->hw; 1777 u32 reta = 0, mrqc, rss_key[10]; 1778 int queue_id; 1779 int i, j; 1780 u32 rss_hash_config; 1781 1782 /* force use default RSS key. */ 1783 #ifdef __NetBSD__ 1784 rss_getkey((uint8_t *) &rss_key); 1785 #else 1786 if (adapter->feat_en & IXGBE_FEATURE_RSS) { 1787 /* Fetch the configured RSS key */ 1788 rss_getkey((uint8_t *)&rss_key); 1789 } else { 1790 /* set up random bits */ 1791 cprng_fast(&rss_key, sizeof(rss_key)); 1792 } 1793 #endif 1794 1795 /* Now fill out hash function seeds */ 1796 for (i = 0; i < 10; i++) 1797 IXGBE_WRITE_REG(hw, IXGBE_VFRSSRK(i), rss_key[i]); 1798 1799 /* Set up the redirection table */ 1800 for (i = 0, j = 0; i < 64; i++, j++) { 1801 if (j == adapter->num_queues) 1802 j = 0; 1803 1804 if (adapter->feat_en & IXGBE_FEATURE_RSS) { 1805 /* 1806 * Fetch the RSS bucket id for the given indirection 1807 * entry. Cap it at the number of configured buckets 1808 * (which is num_queues.) 1809 */ 1810 queue_id = rss_get_indirection_to_bucket(i); 1811 queue_id = queue_id % adapter->num_queues; 1812 } else 1813 queue_id = j; 1814 1815 /* 1816 * The low 8 bits are for hash value (n+0); 1817 * The next 8 bits are for hash value (n+1), etc. 1818 */ 1819 reta >>= 8; 1820 reta |= ((uint32_t)queue_id) << 24; 1821 if ((i & 3) == 3) { 1822 IXGBE_WRITE_REG(hw, IXGBE_VFRETA(i >> 2), reta); 1823 reta = 0; 1824 } 1825 } 1826 1827 /* Perform hash on these packet types */ 1828 if (adapter->feat_en & IXGBE_FEATURE_RSS) 1829 rss_hash_config = rss_gethashconfig(); 1830 else { 1831 /* 1832 * Disable UDP - IP fragments aren't currently being handled 1833 * and so we end up with a mix of 2-tuple and 4-tuple 1834 * traffic. 1835 */ 1836 rss_hash_config = RSS_HASHTYPE_RSS_IPV4 1837 | RSS_HASHTYPE_RSS_TCP_IPV4 1838 | RSS_HASHTYPE_RSS_IPV6 1839 | RSS_HASHTYPE_RSS_TCP_IPV6; 1840 } 1841 1842 mrqc = IXGBE_MRQC_RSSEN; 1843 if (rss_hash_config & RSS_HASHTYPE_RSS_IPV4) 1844 mrqc |= IXGBE_MRQC_RSS_FIELD_IPV4; 1845 if (rss_hash_config & RSS_HASHTYPE_RSS_TCP_IPV4) 1846 mrqc |= IXGBE_MRQC_RSS_FIELD_IPV4_TCP; 1847 if (rss_hash_config & RSS_HASHTYPE_RSS_IPV6) 1848 mrqc |= IXGBE_MRQC_RSS_FIELD_IPV6; 1849 if (rss_hash_config & RSS_HASHTYPE_RSS_TCP_IPV6) 1850 mrqc |= IXGBE_MRQC_RSS_FIELD_IPV6_TCP; 1851 if (rss_hash_config & RSS_HASHTYPE_RSS_IPV6_EX) 1852 device_printf(adapter->dev, "%s: RSS_HASHTYPE_RSS_IPV6_EX " 1853 "defined, but not supported\n", __func__); 1854 if (rss_hash_config & RSS_HASHTYPE_RSS_TCP_IPV6_EX) 1855 device_printf(adapter->dev, "%s: RSS_HASHTYPE_RSS_TCP_IPV6_EX " 1856 "defined, but not supported\n", __func__); 1857 if (rss_hash_config & RSS_HASHTYPE_RSS_UDP_IPV4) 1858 mrqc |= IXGBE_MRQC_RSS_FIELD_IPV4_UDP; 1859 if (rss_hash_config & RSS_HASHTYPE_RSS_UDP_IPV6) 1860 mrqc |= IXGBE_MRQC_RSS_FIELD_IPV6_UDP; 1861 if (rss_hash_config & RSS_HASHTYPE_RSS_UDP_IPV6_EX) 1862 device_printf(adapter->dev, "%s: RSS_HASHTYPE_RSS_UDP_IPV6_EX " 1863 "defined, but not supported\n", __func__); 1864 IXGBE_WRITE_REG(hw, IXGBE_VFMRQC, mrqc); 1865 } /* ixv_initialize_rss_mapping */ 1866 1867 1868 /************************************************************************ 1869 * ixv_initialize_receive_units - Setup receive registers and features. 1870 ************************************************************************/ 1871 static void 1872 ixv_initialize_receive_units(struct adapter *adapter) 1873 { 1874 struct rx_ring *rxr = adapter->rx_rings; 1875 struct ixgbe_hw *hw = &adapter->hw; 1876 struct ifnet *ifp = adapter->ifp; 1877 u32 bufsz, psrtype; 1878 1879 if (ifp->if_mtu > ETHERMTU) 1880 bufsz = 4096 >> IXGBE_SRRCTL_BSIZEPKT_SHIFT; 1881 else 1882 bufsz = 2048 >> IXGBE_SRRCTL_BSIZEPKT_SHIFT; 1883 1884 psrtype = IXGBE_PSRTYPE_TCPHDR 1885 | IXGBE_PSRTYPE_UDPHDR 1886 | IXGBE_PSRTYPE_IPV4HDR 1887 | IXGBE_PSRTYPE_IPV6HDR 1888 | IXGBE_PSRTYPE_L2HDR; 1889 1890 if (adapter->num_queues > 1) 1891 psrtype |= 1 << 29; 1892 1893 IXGBE_WRITE_REG(hw, IXGBE_VFPSRTYPE, psrtype); 1894 1895 /* Tell PF our max_frame size */ 1896 if (ixgbevf_rlpml_set_vf(hw, adapter->max_frame_size) != 0) { 1897 device_printf(adapter->dev, "There is a problem with the PF " 1898 "setup. It is likely the receive unit for this VF will " 1899 "not function correctly.\n"); 1900 } 1901 1902 for (int i = 0; i < adapter->num_queues; i++, rxr++) { 1903 u64 rdba = rxr->rxdma.dma_paddr; 1904 u32 reg, rxdctl; 1905 int j = rxr->me; 1906 1907 /* Disable the queue */ 1908 rxdctl = IXGBE_READ_REG(hw, IXGBE_VFRXDCTL(j)); 1909 rxdctl &= ~IXGBE_RXDCTL_ENABLE; 1910 IXGBE_WRITE_REG(hw, IXGBE_VFRXDCTL(j), rxdctl); 1911 for (int k = 0; k < 10; k++) { 1912 if (IXGBE_READ_REG(hw, IXGBE_VFRXDCTL(j)) & 1913 IXGBE_RXDCTL_ENABLE) 1914 msec_delay(1); 1915 else 1916 break; 1917 } 1918 IXGBE_WRITE_BARRIER(hw); 1919 /* Setup the Base and Length of the Rx Descriptor Ring */ 1920 IXGBE_WRITE_REG(hw, IXGBE_VFRDBAL(j), 1921 (rdba & 0x00000000ffffffffULL)); 1922 IXGBE_WRITE_REG(hw, IXGBE_VFRDBAH(j), (rdba >> 32)); 1923 IXGBE_WRITE_REG(hw, IXGBE_VFRDLEN(j), 1924 adapter->num_rx_desc * sizeof(union ixgbe_adv_rx_desc)); 1925 1926 /* Reset the ring indices */ 1927 IXGBE_WRITE_REG(hw, IXGBE_VFRDH(rxr->me), 0); 1928 IXGBE_WRITE_REG(hw, IXGBE_VFRDT(rxr->me), 0); 1929 1930 /* Set up the SRRCTL register */ 1931 reg = IXGBE_READ_REG(hw, IXGBE_VFSRRCTL(j)); 1932 reg &= ~IXGBE_SRRCTL_BSIZEHDR_MASK; 1933 reg &= ~IXGBE_SRRCTL_BSIZEPKT_MASK; 1934 reg |= bufsz; 1935 reg |= IXGBE_SRRCTL_DESCTYPE_ADV_ONEBUF; 1936 IXGBE_WRITE_REG(hw, IXGBE_VFSRRCTL(j), reg); 1937 1938 /* Capture Rx Tail index */ 1939 rxr->tail = IXGBE_VFRDT(rxr->me); 1940 1941 /* Do the queue enabling last */ 1942 rxdctl |= IXGBE_RXDCTL_ENABLE | IXGBE_RXDCTL_VME; 1943 IXGBE_WRITE_REG(hw, IXGBE_VFRXDCTL(j), rxdctl); 1944 for (int k = 0; k < 10; k++) { 1945 if (IXGBE_READ_REG(hw, IXGBE_VFRXDCTL(j)) & 1946 IXGBE_RXDCTL_ENABLE) 1947 break; 1948 msec_delay(1); 1949 } 1950 IXGBE_WRITE_BARRIER(hw); 1951 1952 /* Set the Tail Pointer */ 1953 #ifdef DEV_NETMAP 1954 /* 1955 * In netmap mode, we must preserve the buffers made 1956 * available to userspace before the if_init() 1957 * (this is true by default on the TX side, because 1958 * init makes all buffers available to userspace). 1959 * 1960 * netmap_reset() and the device specific routines 1961 * (e.g. ixgbe_setup_receive_rings()) map these 1962 * buffers at the end of the NIC ring, so here we 1963 * must set the RDT (tail) register to make sure 1964 * they are not overwritten. 1965 * 1966 * In this driver the NIC ring starts at RDH = 0, 1967 * RDT points to the last slot available for reception (?), 1968 * so RDT = num_rx_desc - 1 means the whole ring is available. 1969 */ 1970 if ((adapter->feat_en & IXGBE_FEATURE_NETMAP) && 1971 (ifp->if_capenable & IFCAP_NETMAP)) { 1972 struct netmap_adapter *na = NA(adapter->ifp); 1973 struct netmap_kring *kring = na->rx_rings[i]; 1974 int t = na->num_rx_desc - 1 - nm_kr_rxspace(kring); 1975 1976 IXGBE_WRITE_REG(hw, IXGBE_VFRDT(rxr->me), t); 1977 } else 1978 #endif /* DEV_NETMAP */ 1979 IXGBE_WRITE_REG(hw, IXGBE_VFRDT(rxr->me), 1980 adapter->num_rx_desc - 1); 1981 } 1982 1983 if (adapter->hw.mac.type >= ixgbe_mac_X550_vf) 1984 ixv_initialize_rss_mapping(adapter); 1985 } /* ixv_initialize_receive_units */ 1986 1987 /************************************************************************ 1988 * ixv_sysctl_tdh_handler - Transmit Descriptor Head handler function 1989 * 1990 * Retrieves the TDH value from the hardware 1991 ************************************************************************/ 1992 static int 1993 ixv_sysctl_tdh_handler(SYSCTLFN_ARGS) 1994 { 1995 struct sysctlnode node = *rnode; 1996 struct tx_ring *txr = (struct tx_ring *)node.sysctl_data; 1997 uint32_t val; 1998 1999 if (!txr) 2000 return (0); 2001 2002 val = IXGBE_READ_REG(&txr->adapter->hw, IXGBE_VFTDH(txr->me)); 2003 node.sysctl_data = &val; 2004 return sysctl_lookup(SYSCTLFN_CALL(&node)); 2005 } /* ixv_sysctl_tdh_handler */ 2006 2007 /************************************************************************ 2008 * ixgbe_sysctl_tdt_handler - Transmit Descriptor Tail handler function 2009 * 2010 * Retrieves the TDT value from the hardware 2011 ************************************************************************/ 2012 static int 2013 ixv_sysctl_tdt_handler(SYSCTLFN_ARGS) 2014 { 2015 struct sysctlnode node = *rnode; 2016 struct tx_ring *txr = (struct tx_ring *)node.sysctl_data; 2017 uint32_t val; 2018 2019 if (!txr) 2020 return (0); 2021 2022 val = IXGBE_READ_REG(&txr->adapter->hw, IXGBE_VFTDT(txr->me)); 2023 node.sysctl_data = &val; 2024 return sysctl_lookup(SYSCTLFN_CALL(&node)); 2025 } /* ixv_sysctl_tdt_handler */ 2026 2027 /************************************************************************ 2028 * ixv_sysctl_next_to_check_handler - Receive Descriptor next to check 2029 * handler function 2030 * 2031 * Retrieves the next_to_check value 2032 ************************************************************************/ 2033 static int 2034 ixv_sysctl_next_to_check_handler(SYSCTLFN_ARGS) 2035 { 2036 struct sysctlnode node = *rnode; 2037 struct rx_ring *rxr = (struct rx_ring *)node.sysctl_data; 2038 uint32_t val; 2039 2040 if (!rxr) 2041 return (0); 2042 2043 val = rxr->next_to_check; 2044 node.sysctl_data = &val; 2045 return sysctl_lookup(SYSCTLFN_CALL(&node)); 2046 } /* ixv_sysctl_next_to_check_handler */ 2047 2048 /************************************************************************ 2049 * ixv_sysctl_next_to_refresh_handler - Receive Descriptor next to refresh 2050 * handler function 2051 * 2052 * Retrieves the next_to_refresh value 2053 ************************************************************************/ 2054 static int 2055 ixv_sysctl_next_to_refresh_handler(SYSCTLFN_ARGS) 2056 { 2057 struct sysctlnode node = *rnode; 2058 struct rx_ring *rxr = (struct rx_ring *)node.sysctl_data; 2059 struct adapter *adapter; 2060 uint32_t val; 2061 2062 if (!rxr) 2063 return (0); 2064 2065 adapter = rxr->adapter; 2066 if (ixgbe_fw_recovery_mode_swflag(adapter)) 2067 return (EPERM); 2068 2069 val = rxr->next_to_refresh; 2070 node.sysctl_data = &val; 2071 return sysctl_lookup(SYSCTLFN_CALL(&node)); 2072 } /* ixv_sysctl_next_to_refresh_handler */ 2073 2074 /************************************************************************ 2075 * ixv_sysctl_rdh_handler - Receive Descriptor Head handler function 2076 * 2077 * Retrieves the RDH value from the hardware 2078 ************************************************************************/ 2079 static int 2080 ixv_sysctl_rdh_handler(SYSCTLFN_ARGS) 2081 { 2082 struct sysctlnode node = *rnode; 2083 struct rx_ring *rxr = (struct rx_ring *)node.sysctl_data; 2084 uint32_t val; 2085 2086 if (!rxr) 2087 return (0); 2088 2089 val = IXGBE_READ_REG(&rxr->adapter->hw, IXGBE_VFRDH(rxr->me)); 2090 node.sysctl_data = &val; 2091 return sysctl_lookup(SYSCTLFN_CALL(&node)); 2092 } /* ixv_sysctl_rdh_handler */ 2093 2094 /************************************************************************ 2095 * ixv_sysctl_rdt_handler - Receive Descriptor Tail handler function 2096 * 2097 * Retrieves the RDT value from the hardware 2098 ************************************************************************/ 2099 static int 2100 ixv_sysctl_rdt_handler(SYSCTLFN_ARGS) 2101 { 2102 struct sysctlnode node = *rnode; 2103 struct rx_ring *rxr = (struct rx_ring *)node.sysctl_data; 2104 uint32_t val; 2105 2106 if (!rxr) 2107 return (0); 2108 2109 val = IXGBE_READ_REG(&rxr->adapter->hw, IXGBE_VFRDT(rxr->me)); 2110 node.sysctl_data = &val; 2111 return sysctl_lookup(SYSCTLFN_CALL(&node)); 2112 } /* ixv_sysctl_rdt_handler */ 2113 2114 static void 2115 ixv_setup_vlan_tagging(struct adapter *adapter) 2116 { 2117 struct ethercom *ec = &adapter->osdep.ec; 2118 struct ixgbe_hw *hw = &adapter->hw; 2119 struct rx_ring *rxr; 2120 u32 ctrl; 2121 int i; 2122 bool hwtagging; 2123 2124 /* Enable HW tagging only if any vlan is attached */ 2125 hwtagging = (ec->ec_capenable & ETHERCAP_VLAN_HWTAGGING) 2126 && VLAN_ATTACHED(ec); 2127 2128 /* Enable the queues */ 2129 for (i = 0; i < adapter->num_queues; i++) { 2130 rxr = &adapter->rx_rings[i]; 2131 ctrl = IXGBE_READ_REG(hw, IXGBE_VFRXDCTL(rxr->me)); 2132 if (hwtagging) 2133 ctrl |= IXGBE_RXDCTL_VME; 2134 else 2135 ctrl &= ~IXGBE_RXDCTL_VME; 2136 IXGBE_WRITE_REG(hw, IXGBE_VFRXDCTL(rxr->me), ctrl); 2137 /* 2138 * Let Rx path know that it needs to store VLAN tag 2139 * as part of extra mbuf info. 2140 */ 2141 rxr->vtag_strip = hwtagging ? TRUE : FALSE; 2142 } 2143 } /* ixv_setup_vlan_tagging */ 2144 2145 /************************************************************************ 2146 * ixv_setup_vlan_support 2147 ************************************************************************/ 2148 static int 2149 ixv_setup_vlan_support(struct adapter *adapter) 2150 { 2151 struct ethercom *ec = &adapter->osdep.ec; 2152 struct ixgbe_hw *hw = &adapter->hw; 2153 u32 vid, vfta, retry; 2154 struct vlanid_list *vlanidp; 2155 int rv, error = 0; 2156 2157 /* 2158 * This function is called from both if_init and ifflags_cb() 2159 * on NetBSD. 2160 */ 2161 2162 /* 2163 * Part 1: 2164 * Setup VLAN HW tagging 2165 */ 2166 ixv_setup_vlan_tagging(adapter); 2167 2168 if (!VLAN_ATTACHED(ec)) 2169 return 0; 2170 2171 /* 2172 * Part 2: 2173 * Setup VLAN HW filter 2174 */ 2175 /* Cleanup shadow_vfta */ 2176 for (int i = 0; i < IXGBE_VFTA_SIZE; i++) 2177 adapter->shadow_vfta[i] = 0; 2178 /* Generate shadow_vfta from ec_vids */ 2179 ETHER_LOCK(ec); 2180 SIMPLEQ_FOREACH(vlanidp, &ec->ec_vids, vid_list) { 2181 uint32_t idx; 2182 2183 idx = vlanidp->vid / 32; 2184 KASSERT(idx < IXGBE_VFTA_SIZE); 2185 adapter->shadow_vfta[idx] |= (u32)1 << (vlanidp->vid % 32); 2186 } 2187 ETHER_UNLOCK(ec); 2188 2189 /* 2190 * A soft reset zero's out the VFTA, so 2191 * we need to repopulate it now. 2192 */ 2193 for (int i = 0; i < IXGBE_VFTA_SIZE; i++) { 2194 if (adapter->shadow_vfta[i] == 0) 2195 continue; 2196 vfta = adapter->shadow_vfta[i]; 2197 /* 2198 * Reconstruct the vlan id's 2199 * based on the bits set in each 2200 * of the array ints. 2201 */ 2202 for (int j = 0; j < 32; j++) { 2203 retry = 0; 2204 if ((vfta & ((u32)1 << j)) == 0) 2205 continue; 2206 vid = (i * 32) + j; 2207 2208 /* Call the shared code mailbox routine */ 2209 while ((rv = hw->mac.ops.set_vfta(hw, vid, 0, TRUE, 2210 FALSE)) != 0) { 2211 if (++retry > 5) { 2212 device_printf(adapter->dev, 2213 "%s: max retry exceeded\n", 2214 __func__); 2215 break; 2216 } 2217 } 2218 if (rv != 0) { 2219 device_printf(adapter->dev, 2220 "failed to set vlan %d\n", vid); 2221 error = EACCES; 2222 } 2223 } 2224 } 2225 return error; 2226 } /* ixv_setup_vlan_support */ 2227 2228 static int 2229 ixv_vlan_cb(struct ethercom *ec, uint16_t vid, bool set) 2230 { 2231 struct ifnet *ifp = &ec->ec_if; 2232 struct adapter *adapter = ifp->if_softc; 2233 int rv; 2234 2235 if (set) 2236 rv = ixv_register_vlan(adapter, vid); 2237 else 2238 rv = ixv_unregister_vlan(adapter, vid); 2239 2240 if (rv != 0) 2241 return rv; 2242 2243 /* 2244 * Control VLAN HW tagging when ec_nvlan is changed from 1 to 0 2245 * or 0 to 1. 2246 */ 2247 if ((set && (ec->ec_nvlans == 1)) || (!set && (ec->ec_nvlans == 0))) 2248 ixv_setup_vlan_tagging(adapter); 2249 2250 return rv; 2251 } 2252 2253 /************************************************************************ 2254 * ixv_register_vlan 2255 * 2256 * Run via a vlan config EVENT, it enables us to use the 2257 * HW Filter table since we can get the vlan id. This just 2258 * creates the entry in the soft version of the VFTA, init 2259 * will repopulate the real table. 2260 ************************************************************************/ 2261 static int 2262 ixv_register_vlan(struct adapter *adapter, u16 vtag) 2263 { 2264 struct ixgbe_hw *hw = &adapter->hw; 2265 u16 index, bit; 2266 int error; 2267 2268 if ((vtag == 0) || (vtag > 4095)) /* Invalid */ 2269 return EINVAL; 2270 IXGBE_CORE_LOCK(adapter); 2271 index = (vtag >> 5) & 0x7F; 2272 bit = vtag & 0x1F; 2273 adapter->shadow_vfta[index] |= ((u32)1 << bit); 2274 error = hw->mac.ops.set_vfta(hw, vtag, 0, true, false); 2275 IXGBE_CORE_UNLOCK(adapter); 2276 2277 if (error != 0) { 2278 device_printf(adapter->dev, "failed to register vlan %hu\n", 2279 vtag); 2280 error = EACCES; 2281 } 2282 return error; 2283 } /* ixv_register_vlan */ 2284 2285 /************************************************************************ 2286 * ixv_unregister_vlan 2287 * 2288 * Run via a vlan unconfig EVENT, remove our entry 2289 * in the soft vfta. 2290 ************************************************************************/ 2291 static int 2292 ixv_unregister_vlan(struct adapter *adapter, u16 vtag) 2293 { 2294 struct ixgbe_hw *hw = &adapter->hw; 2295 u16 index, bit; 2296 int error; 2297 2298 if ((vtag == 0) || (vtag > 4095)) /* Invalid */ 2299 return EINVAL; 2300 2301 IXGBE_CORE_LOCK(adapter); 2302 index = (vtag >> 5) & 0x7F; 2303 bit = vtag & 0x1F; 2304 adapter->shadow_vfta[index] &= ~((u32)1 << bit); 2305 error = hw->mac.ops.set_vfta(hw, vtag, 0, false, false); 2306 IXGBE_CORE_UNLOCK(adapter); 2307 2308 if (error != 0) { 2309 device_printf(adapter->dev, "failed to unregister vlan %hu\n", 2310 vtag); 2311 error = EIO; 2312 } 2313 return error; 2314 } /* ixv_unregister_vlan */ 2315 2316 /************************************************************************ 2317 * ixv_enable_intr 2318 ************************************************************************/ 2319 static void 2320 ixv_enable_intr(struct adapter *adapter) 2321 { 2322 struct ixgbe_hw *hw = &adapter->hw; 2323 struct ix_queue *que = adapter->queues; 2324 u32 mask; 2325 int i; 2326 2327 /* For VTEIAC */ 2328 mask = (1 << adapter->vector); 2329 for (i = 0; i < adapter->num_queues; i++, que++) 2330 mask |= (1 << que->msix); 2331 IXGBE_WRITE_REG(hw, IXGBE_VTEIAC, mask); 2332 2333 /* For VTEIMS */ 2334 IXGBE_WRITE_REG(hw, IXGBE_VTEIMS, (1 << adapter->vector)); 2335 que = adapter->queues; 2336 for (i = 0; i < adapter->num_queues; i++, que++) 2337 ixv_enable_queue(adapter, que->msix); 2338 2339 IXGBE_WRITE_FLUSH(hw); 2340 } /* ixv_enable_intr */ 2341 2342 /************************************************************************ 2343 * ixv_disable_intr 2344 ************************************************************************/ 2345 static void 2346 ixv_disable_intr(struct adapter *adapter) 2347 { 2348 struct ix_queue *que = adapter->queues; 2349 2350 IXGBE_WRITE_REG(&adapter->hw, IXGBE_VTEIAC, 0); 2351 2352 /* disable interrupts other than queues */ 2353 IXGBE_WRITE_REG(&adapter->hw, IXGBE_VTEIMC, adapter->vector); 2354 2355 for (int i = 0; i < adapter->num_queues; i++, que++) 2356 ixv_disable_queue(adapter, que->msix); 2357 2358 IXGBE_WRITE_FLUSH(&adapter->hw); 2359 } /* ixv_disable_intr */ 2360 2361 /************************************************************************ 2362 * ixv_set_ivar 2363 * 2364 * Setup the correct IVAR register for a particular MSI-X interrupt 2365 * - entry is the register array entry 2366 * - vector is the MSI-X vector for this queue 2367 * - type is RX/TX/MISC 2368 ************************************************************************/ 2369 static void 2370 ixv_set_ivar(struct adapter *adapter, u8 entry, u8 vector, s8 type) 2371 { 2372 struct ixgbe_hw *hw = &adapter->hw; 2373 u32 ivar, index; 2374 2375 vector |= IXGBE_IVAR_ALLOC_VAL; 2376 2377 if (type == -1) { /* MISC IVAR */ 2378 ivar = IXGBE_READ_REG(hw, IXGBE_VTIVAR_MISC); 2379 ivar &= ~0xFF; 2380 ivar |= vector; 2381 IXGBE_WRITE_REG(hw, IXGBE_VTIVAR_MISC, ivar); 2382 } else { /* RX/TX IVARS */ 2383 index = (16 * (entry & 1)) + (8 * type); 2384 ivar = IXGBE_READ_REG(hw, IXGBE_VTIVAR(entry >> 1)); 2385 ivar &= ~(0xffUL << index); 2386 ivar |= ((u32)vector << index); 2387 IXGBE_WRITE_REG(hw, IXGBE_VTIVAR(entry >> 1), ivar); 2388 } 2389 } /* ixv_set_ivar */ 2390 2391 /************************************************************************ 2392 * ixv_configure_ivars 2393 ************************************************************************/ 2394 static void 2395 ixv_configure_ivars(struct adapter *adapter) 2396 { 2397 struct ix_queue *que = adapter->queues; 2398 2399 /* XXX We should sync EITR value calculation with ixgbe.c? */ 2400 2401 for (int i = 0; i < adapter->num_queues; i++, que++) { 2402 /* First the RX queue entry */ 2403 ixv_set_ivar(adapter, i, que->msix, 0); 2404 /* ... and the TX */ 2405 ixv_set_ivar(adapter, i, que->msix, 1); 2406 /* Set an initial value in EITR */ 2407 ixv_eitr_write(adapter, que->msix, IXGBE_EITR_DEFAULT); 2408 } 2409 2410 /* For the mailbox interrupt */ 2411 ixv_set_ivar(adapter, 1, adapter->vector, -1); 2412 } /* ixv_configure_ivars */ 2413 2414 2415 /************************************************************************ 2416 * ixv_init_stats 2417 * 2418 * The VF stats registers never have a truly virgin 2419 * starting point, so this routine save initial vaules to 2420 * last_<REGNAME>. 2421 ************************************************************************/ 2422 static void 2423 ixv_init_stats(struct adapter *adapter) 2424 { 2425 struct ixgbe_hw *hw = &adapter->hw; 2426 2427 adapter->stats.vf.last_vfgprc = IXGBE_READ_REG(hw, IXGBE_VFGPRC); 2428 adapter->stats.vf.last_vfgorc = IXGBE_READ_REG(hw, IXGBE_VFGORC_LSB); 2429 adapter->stats.vf.last_vfgorc |= 2430 (((u64)(IXGBE_READ_REG(hw, IXGBE_VFGORC_MSB))) << 32); 2431 2432 adapter->stats.vf.last_vfgptc = IXGBE_READ_REG(hw, IXGBE_VFGPTC); 2433 adapter->stats.vf.last_vfgotc = IXGBE_READ_REG(hw, IXGBE_VFGOTC_LSB); 2434 adapter->stats.vf.last_vfgotc |= 2435 (((u64)(IXGBE_READ_REG(hw, IXGBE_VFGOTC_MSB))) << 32); 2436 2437 adapter->stats.vf.last_vfmprc = IXGBE_READ_REG(hw, IXGBE_VFMPRC); 2438 } /* ixv_init_stats */ 2439 2440 #define UPDATE_STAT_32(reg, last, count) \ 2441 { \ 2442 u32 current = IXGBE_READ_REG(hw, (reg)); \ 2443 IXGBE_EVC_ADD(&count, current - (last)); \ 2444 (last) = current; \ 2445 } 2446 2447 #define UPDATE_STAT_36(lsb, msb, last, count) \ 2448 { \ 2449 u64 cur_lsb = IXGBE_READ_REG(hw, (lsb)); \ 2450 u64 cur_msb = IXGBE_READ_REG(hw, (msb)); \ 2451 u64 current = ((cur_msb << 32) | cur_lsb); \ 2452 if (current < (last)) \ 2453 IXGBE_EVC_ADD(&count, current + __BIT(36) - (last)); \ 2454 else \ 2455 IXGBE_EVC_ADD(&count, current - (last)); \ 2456 (last) = current; \ 2457 } 2458 2459 /************************************************************************ 2460 * ixv_update_stats - Update the board statistics counters. 2461 ************************************************************************/ 2462 void 2463 ixv_update_stats(struct adapter *adapter) 2464 { 2465 struct ixgbe_hw *hw = &adapter->hw; 2466 struct ixgbevf_hw_stats *stats = &adapter->stats.vf; 2467 2468 UPDATE_STAT_32(IXGBE_VFGPRC, stats->last_vfgprc, stats->vfgprc); 2469 UPDATE_STAT_32(IXGBE_VFGPTC, stats->last_vfgptc, stats->vfgptc); 2470 UPDATE_STAT_36(IXGBE_VFGORC_LSB, IXGBE_VFGORC_MSB, stats->last_vfgorc, 2471 stats->vfgorc); 2472 UPDATE_STAT_36(IXGBE_VFGOTC_LSB, IXGBE_VFGOTC_MSB, stats->last_vfgotc, 2473 stats->vfgotc); 2474 UPDATE_STAT_32(IXGBE_VFMPRC, stats->last_vfmprc, stats->vfmprc); 2475 2476 /* VF doesn't count errors by hardware */ 2477 2478 } /* ixv_update_stats */ 2479 2480 /************************************************************************ 2481 * ixv_sysctl_interrupt_rate_handler 2482 ************************************************************************/ 2483 static int 2484 ixv_sysctl_interrupt_rate_handler(SYSCTLFN_ARGS) 2485 { 2486 struct sysctlnode node = *rnode; 2487 struct ix_queue *que = (struct ix_queue *)node.sysctl_data; 2488 struct adapter *adapter = que->adapter; 2489 uint32_t reg, usec, rate; 2490 int error; 2491 2492 if (que == NULL) 2493 return 0; 2494 reg = IXGBE_READ_REG(&que->adapter->hw, IXGBE_VTEITR(que->msix)); 2495 usec = ((reg & 0x0FF8) >> 3); 2496 if (usec > 0) 2497 rate = 500000 / usec; 2498 else 2499 rate = 0; 2500 node.sysctl_data = &rate; 2501 error = sysctl_lookup(SYSCTLFN_CALL(&node)); 2502 if (error || newp == NULL) 2503 return error; 2504 reg &= ~0xfff; /* default, no limitation */ 2505 if (rate > 0 && rate < 500000) { 2506 if (rate < 1000) 2507 rate = 1000; 2508 reg |= ((4000000 / rate) & 0xff8); 2509 /* 2510 * When RSC is used, ITR interval must be larger than 2511 * RSC_DELAY. Currently, we use 2us for RSC_DELAY. 2512 * The minimum value is always greater than 2us on 100M 2513 * (and 10M?(not documented)), but it's not on 1G and higher. 2514 */ 2515 if ((adapter->link_speed != IXGBE_LINK_SPEED_100_FULL) 2516 && (adapter->link_speed != IXGBE_LINK_SPEED_10_FULL)) { 2517 if ((adapter->num_queues > 1) 2518 && (reg < IXGBE_MIN_RSC_EITR_10G1G)) 2519 return EINVAL; 2520 } 2521 ixv_max_interrupt_rate = rate; 2522 } else 2523 ixv_max_interrupt_rate = 0; 2524 ixv_eitr_write(adapter, que->msix, reg); 2525 2526 return (0); 2527 } /* ixv_sysctl_interrupt_rate_handler */ 2528 2529 const struct sysctlnode * 2530 ixv_sysctl_instance(struct adapter *adapter) 2531 { 2532 const char *dvname; 2533 struct sysctllog **log; 2534 int rc; 2535 const struct sysctlnode *rnode; 2536 2537 log = &adapter->sysctllog; 2538 dvname = device_xname(adapter->dev); 2539 2540 if ((rc = sysctl_createv(log, 0, NULL, &rnode, 2541 0, CTLTYPE_NODE, dvname, 2542 SYSCTL_DESCR("ixv information and settings"), 2543 NULL, 0, NULL, 0, CTL_HW, CTL_CREATE, CTL_EOL)) != 0) 2544 goto err; 2545 2546 return rnode; 2547 err: 2548 device_printf(adapter->dev, 2549 "%s: sysctl_createv failed, rc = %d\n", __func__, rc); 2550 return NULL; 2551 } 2552 2553 static void 2554 ixv_add_device_sysctls(struct adapter *adapter) 2555 { 2556 struct sysctllog **log; 2557 const struct sysctlnode *rnode, *cnode; 2558 device_t dev; 2559 2560 dev = adapter->dev; 2561 log = &adapter->sysctllog; 2562 2563 if ((rnode = ixv_sysctl_instance(adapter)) == NULL) { 2564 aprint_error_dev(dev, "could not create sysctl root\n"); 2565 return; 2566 } 2567 2568 if (sysctl_createv(log, 0, &rnode, &cnode, 2569 CTLFLAG_READWRITE, CTLTYPE_INT, "debug", 2570 SYSCTL_DESCR("Debug Info"), 2571 ixv_sysctl_debug, 0, (void *)adapter, 0, CTL_CREATE, CTL_EOL) != 0) 2572 aprint_error_dev(dev, "could not create sysctl\n"); 2573 2574 if (sysctl_createv(log, 0, &rnode, &cnode, 2575 CTLFLAG_READWRITE, CTLTYPE_INT, 2576 "rx_copy_len", SYSCTL_DESCR("RX Copy Length"), 2577 ixv_sysctl_rx_copy_len, 0, 2578 (void *)adapter, 0, CTL_CREATE, CTL_EOL) != 0) 2579 aprint_error_dev(dev, "could not create sysctl\n"); 2580 2581 if (sysctl_createv(log, 0, &rnode, &cnode, 2582 CTLFLAG_READONLY, CTLTYPE_INT, 2583 "num_tx_desc", SYSCTL_DESCR("Number of TX descriptors"), 2584 NULL, 0, &adapter->num_tx_desc, 0, CTL_CREATE, CTL_EOL) != 0) 2585 aprint_error_dev(dev, "could not create sysctl\n"); 2586 2587 if (sysctl_createv(log, 0, &rnode, &cnode, 2588 CTLFLAG_READONLY, CTLTYPE_INT, 2589 "num_rx_desc", SYSCTL_DESCR("Number of RX descriptors"), 2590 NULL, 0, &adapter->num_rx_desc, 0, CTL_CREATE, CTL_EOL) != 0) 2591 aprint_error_dev(dev, "could not create sysctl\n"); 2592 2593 if (sysctl_createv(log, 0, &rnode, &cnode, 2594 CTLFLAG_READWRITE, CTLTYPE_INT, "rx_process_limit", 2595 SYSCTL_DESCR("max number of RX packets to process"), 2596 ixv_sysctl_rx_process_limit, 0, (void *)adapter, 0, CTL_CREATE, 2597 CTL_EOL) != 0) 2598 aprint_error_dev(dev, "could not create sysctl\n"); 2599 2600 if (sysctl_createv(log, 0, &rnode, &cnode, 2601 CTLFLAG_READWRITE, CTLTYPE_INT, "tx_process_limit", 2602 SYSCTL_DESCR("max number of TX packets to process"), 2603 ixv_sysctl_tx_process_limit, 0, (void *)adapter, 0, CTL_CREATE, 2604 CTL_EOL) != 0) 2605 aprint_error_dev(dev, "could not create sysctl\n"); 2606 2607 if (sysctl_createv(log, 0, &rnode, &cnode, 2608 CTLFLAG_READWRITE, CTLTYPE_BOOL, "enable_aim", 2609 SYSCTL_DESCR("Interrupt Moderation"), 2610 NULL, 0, &adapter->enable_aim, 0, CTL_CREATE, CTL_EOL) != 0) 2611 aprint_error_dev(dev, "could not create sysctl\n"); 2612 2613 if (sysctl_createv(log, 0, &rnode, &cnode, 2614 CTLFLAG_READWRITE, CTLTYPE_BOOL, "txrx_workqueue", 2615 SYSCTL_DESCR("Use workqueue for packet processing"), 2616 NULL, 0, &adapter->txrx_use_workqueue, 0, CTL_CREATE, CTL_EOL) 2617 != 0) 2618 aprint_error_dev(dev, "could not create sysctl\n"); 2619 } 2620 2621 /************************************************************************ 2622 * ixv_add_stats_sysctls - Add statistic sysctls for the VF. 2623 ************************************************************************/ 2624 static void 2625 ixv_add_stats_sysctls(struct adapter *adapter) 2626 { 2627 device_t dev = adapter->dev; 2628 struct tx_ring *txr = adapter->tx_rings; 2629 struct rx_ring *rxr = adapter->rx_rings; 2630 struct ixgbevf_hw_stats *stats = &adapter->stats.vf; 2631 struct ixgbe_hw *hw = &adapter->hw; 2632 const struct sysctlnode *rnode, *cnode; 2633 struct sysctllog **log = &adapter->sysctllog; 2634 const char *xname = device_xname(dev); 2635 2636 /* Driver Statistics */ 2637 evcnt_attach_dynamic(&adapter->efbig_tx_dma_setup, EVCNT_TYPE_MISC, 2638 NULL, xname, "Driver tx dma soft fail EFBIG"); 2639 evcnt_attach_dynamic(&adapter->mbuf_defrag_failed, EVCNT_TYPE_MISC, 2640 NULL, xname, "m_defrag() failed"); 2641 evcnt_attach_dynamic(&adapter->efbig2_tx_dma_setup, EVCNT_TYPE_MISC, 2642 NULL, xname, "Driver tx dma hard fail EFBIG"); 2643 evcnt_attach_dynamic(&adapter->einval_tx_dma_setup, EVCNT_TYPE_MISC, 2644 NULL, xname, "Driver tx dma hard fail EINVAL"); 2645 evcnt_attach_dynamic(&adapter->other_tx_dma_setup, EVCNT_TYPE_MISC, 2646 NULL, xname, "Driver tx dma hard fail other"); 2647 evcnt_attach_dynamic(&adapter->eagain_tx_dma_setup, EVCNT_TYPE_MISC, 2648 NULL, xname, "Driver tx dma soft fail EAGAIN"); 2649 evcnt_attach_dynamic(&adapter->enomem_tx_dma_setup, EVCNT_TYPE_MISC, 2650 NULL, xname, "Driver tx dma soft fail ENOMEM"); 2651 evcnt_attach_dynamic(&adapter->watchdog_events, EVCNT_TYPE_MISC, 2652 NULL, xname, "Watchdog timeouts"); 2653 evcnt_attach_dynamic(&adapter->tso_err, EVCNT_TYPE_MISC, 2654 NULL, xname, "TSO errors"); 2655 evcnt_attach_dynamic(&adapter->admin_irqev, EVCNT_TYPE_INTR, 2656 NULL, xname, "Admin MSI-X IRQ Handled"); 2657 evcnt_attach_dynamic(&adapter->link_workev, EVCNT_TYPE_INTR, 2658 NULL, xname, "Admin event"); 2659 2660 for (int i = 0; i < adapter->num_queues; i++, rxr++, txr++) { 2661 #ifdef LRO 2662 struct lro_ctrl *lro = &rxr->lro; 2663 #endif 2664 2665 snprintf(adapter->queues[i].evnamebuf, 2666 sizeof(adapter->queues[i].evnamebuf), "%s q%d", xname, i); 2667 snprintf(adapter->queues[i].namebuf, 2668 sizeof(adapter->queues[i].namebuf), "q%d", i); 2669 2670 if ((rnode = ixv_sysctl_instance(adapter)) == NULL) { 2671 aprint_error_dev(dev, 2672 "could not create sysctl root\n"); 2673 break; 2674 } 2675 2676 if (sysctl_createv(log, 0, &rnode, &rnode, 2677 0, CTLTYPE_NODE, 2678 adapter->queues[i].namebuf, SYSCTL_DESCR("Queue Name"), 2679 NULL, 0, NULL, 0, CTL_CREATE, CTL_EOL) != 0) 2680 break; 2681 2682 if (sysctl_createv(log, 0, &rnode, &cnode, 2683 CTLFLAG_READWRITE, CTLTYPE_INT, 2684 "interrupt_rate", SYSCTL_DESCR("Interrupt Rate"), 2685 ixv_sysctl_interrupt_rate_handler, 0, 2686 (void *)&adapter->queues[i], 0, CTL_CREATE, CTL_EOL) != 0) 2687 break; 2688 2689 if (sysctl_createv(log, 0, &rnode, &cnode, 2690 CTLFLAG_READONLY, CTLTYPE_INT, 2691 "txd_head", SYSCTL_DESCR("Transmit Descriptor Head"), 2692 ixv_sysctl_tdh_handler, 0, (void *)txr, 2693 0, CTL_CREATE, CTL_EOL) != 0) 2694 break; 2695 2696 if (sysctl_createv(log, 0, &rnode, &cnode, 2697 CTLFLAG_READONLY, CTLTYPE_INT, 2698 "txd_tail", SYSCTL_DESCR("Transmit Descriptor Tail"), 2699 ixv_sysctl_tdt_handler, 0, (void *)txr, 2700 0, CTL_CREATE, CTL_EOL) != 0) 2701 break; 2702 2703 if (sysctl_createv(log, 0, &rnode, &cnode, 2704 CTLFLAG_READONLY, CTLTYPE_INT, "rxd_nxck", 2705 SYSCTL_DESCR("Receive Descriptor next to check"), 2706 ixv_sysctl_next_to_check_handler, 0, (void *)rxr, 0, 2707 CTL_CREATE, CTL_EOL) != 0) 2708 break; 2709 2710 if (sysctl_createv(log, 0, &rnode, &cnode, 2711 CTLFLAG_READONLY, CTLTYPE_INT, "rxd_nxrf", 2712 SYSCTL_DESCR("Receive Descriptor next to refresh"), 2713 ixv_sysctl_next_to_refresh_handler, 0, (void *)rxr, 0, 2714 CTL_CREATE, CTL_EOL) != 0) 2715 break; 2716 2717 if (sysctl_createv(log, 0, &rnode, &cnode, 2718 CTLFLAG_READONLY, CTLTYPE_INT, "rxd_head", 2719 SYSCTL_DESCR("Receive Descriptor Head"), 2720 ixv_sysctl_rdh_handler, 0, (void *)rxr, 0, 2721 CTL_CREATE, CTL_EOL) != 0) 2722 break; 2723 2724 if (sysctl_createv(log, 0, &rnode, &cnode, 2725 CTLFLAG_READONLY, CTLTYPE_INT, "rxd_tail", 2726 SYSCTL_DESCR("Receive Descriptor Tail"), 2727 ixv_sysctl_rdt_handler, 0, (void *)rxr, 0, 2728 CTL_CREATE, CTL_EOL) != 0) 2729 break; 2730 2731 evcnt_attach_dynamic(&adapter->queues[i].irqs, EVCNT_TYPE_INTR, 2732 NULL, adapter->queues[i].evnamebuf, "IRQs on queue"); 2733 evcnt_attach_dynamic(&adapter->queues[i].handleq, 2734 EVCNT_TYPE_MISC, NULL, adapter->queues[i].evnamebuf, 2735 "Handled queue in softint"); 2736 evcnt_attach_dynamic(&adapter->queues[i].req, EVCNT_TYPE_MISC, 2737 NULL, adapter->queues[i].evnamebuf, "Requeued in softint"); 2738 evcnt_attach_dynamic(&txr->total_packets, EVCNT_TYPE_MISC, 2739 NULL, adapter->queues[i].evnamebuf, 2740 "Queue Packets Transmitted"); 2741 #ifndef IXGBE_LEGACY_TX 2742 evcnt_attach_dynamic(&txr->pcq_drops, EVCNT_TYPE_MISC, 2743 NULL, adapter->queues[i].evnamebuf, 2744 "Packets dropped in pcq"); 2745 #endif 2746 evcnt_attach_dynamic(&txr->no_desc_avail, EVCNT_TYPE_MISC, 2747 NULL, adapter->queues[i].evnamebuf, 2748 "TX Queue No Descriptor Available"); 2749 evcnt_attach_dynamic(&txr->tso_tx, EVCNT_TYPE_MISC, 2750 NULL, adapter->queues[i].evnamebuf, "TSO"); 2751 2752 evcnt_attach_dynamic(&rxr->rx_bytes, EVCNT_TYPE_MISC, 2753 NULL, adapter->queues[i].evnamebuf, 2754 "Queue Bytes Received"); 2755 evcnt_attach_dynamic(&rxr->rx_packets, EVCNT_TYPE_MISC, 2756 NULL, adapter->queues[i].evnamebuf, 2757 "Queue Packets Received"); 2758 evcnt_attach_dynamic(&rxr->no_mbuf, EVCNT_TYPE_MISC, 2759 NULL, adapter->queues[i].evnamebuf, "Rx no mbuf"); 2760 evcnt_attach_dynamic(&rxr->rx_discarded, EVCNT_TYPE_MISC, 2761 NULL, adapter->queues[i].evnamebuf, "Rx discarded"); 2762 evcnt_attach_dynamic(&rxr->rx_copies, EVCNT_TYPE_MISC, 2763 NULL, adapter->queues[i].evnamebuf, "Copied RX Frames"); 2764 #ifdef LRO 2765 SYSCTL_ADD_INT(ctx, queue_list, OID_AUTO, "lro_queued", 2766 CTLFLAG_RD, &lro->lro_queued, 0, 2767 "LRO Queued"); 2768 SYSCTL_ADD_INT(ctx, queue_list, OID_AUTO, "lro_flushed", 2769 CTLFLAG_RD, &lro->lro_flushed, 0, 2770 "LRO Flushed"); 2771 #endif /* LRO */ 2772 } 2773 2774 /* MAC stats get their own sub node */ 2775 2776 snprintf(stats->namebuf, 2777 sizeof(stats->namebuf), "%s MAC Statistics", xname); 2778 2779 evcnt_attach_dynamic(&stats->ipcs, EVCNT_TYPE_MISC, NULL, 2780 stats->namebuf, "rx csum offload - IP"); 2781 evcnt_attach_dynamic(&stats->l4cs, EVCNT_TYPE_MISC, NULL, 2782 stats->namebuf, "rx csum offload - L4"); 2783 evcnt_attach_dynamic(&stats->ipcs_bad, EVCNT_TYPE_MISC, NULL, 2784 stats->namebuf, "rx csum offload - IP bad"); 2785 evcnt_attach_dynamic(&stats->l4cs_bad, EVCNT_TYPE_MISC, NULL, 2786 stats->namebuf, "rx csum offload - L4 bad"); 2787 2788 /* Packet Reception Stats */ 2789 evcnt_attach_dynamic(&stats->vfgprc, EVCNT_TYPE_MISC, NULL, 2790 xname, "Good Packets Received"); 2791 evcnt_attach_dynamic(&stats->vfgorc, EVCNT_TYPE_MISC, NULL, 2792 xname, "Good Octets Received"); 2793 evcnt_attach_dynamic(&stats->vfmprc, EVCNT_TYPE_MISC, NULL, 2794 xname, "Multicast Packets Received"); 2795 evcnt_attach_dynamic(&stats->vfgptc, EVCNT_TYPE_MISC, NULL, 2796 xname, "Good Packets Transmitted"); 2797 evcnt_attach_dynamic(&stats->vfgotc, EVCNT_TYPE_MISC, NULL, 2798 xname, "Good Octets Transmitted"); 2799 2800 /* Mailbox Stats */ 2801 evcnt_attach_dynamic(&hw->mbx.stats.msgs_tx, EVCNT_TYPE_MISC, NULL, 2802 xname, "message TXs"); 2803 evcnt_attach_dynamic(&hw->mbx.stats.msgs_rx, EVCNT_TYPE_MISC, NULL, 2804 xname, "message RXs"); 2805 evcnt_attach_dynamic(&hw->mbx.stats.acks, EVCNT_TYPE_MISC, NULL, 2806 xname, "ACKs"); 2807 evcnt_attach_dynamic(&hw->mbx.stats.reqs, EVCNT_TYPE_MISC, NULL, 2808 xname, "REQs"); 2809 evcnt_attach_dynamic(&hw->mbx.stats.rsts, EVCNT_TYPE_MISC, NULL, 2810 xname, "RSTs"); 2811 2812 } /* ixv_add_stats_sysctls */ 2813 2814 static void 2815 ixv_clear_evcnt(struct adapter *adapter) 2816 { 2817 struct tx_ring *txr = adapter->tx_rings; 2818 struct rx_ring *rxr = adapter->rx_rings; 2819 struct ixgbevf_hw_stats *stats = &adapter->stats.vf; 2820 struct ixgbe_hw *hw = &adapter->hw; 2821 int i; 2822 2823 /* Driver Statistics */ 2824 IXGBE_EVC_STORE(&adapter->efbig_tx_dma_setup, 0); 2825 IXGBE_EVC_STORE(&adapter->mbuf_defrag_failed, 0); 2826 IXGBE_EVC_STORE(&adapter->efbig2_tx_dma_setup, 0); 2827 IXGBE_EVC_STORE(&adapter->einval_tx_dma_setup, 0); 2828 IXGBE_EVC_STORE(&adapter->other_tx_dma_setup, 0); 2829 IXGBE_EVC_STORE(&adapter->eagain_tx_dma_setup, 0); 2830 IXGBE_EVC_STORE(&adapter->enomem_tx_dma_setup, 0); 2831 IXGBE_EVC_STORE(&adapter->watchdog_events, 0); 2832 IXGBE_EVC_STORE(&adapter->tso_err, 0); 2833 IXGBE_EVC_STORE(&adapter->admin_irqev, 0); 2834 IXGBE_EVC_STORE(&adapter->link_workev, 0); 2835 2836 for (i = 0; i < adapter->num_queues; i++, rxr++, txr++) { 2837 IXGBE_EVC_STORE(&adapter->queues[i].irqs, 0); 2838 IXGBE_EVC_STORE(&adapter->queues[i].handleq, 0); 2839 IXGBE_EVC_STORE(&adapter->queues[i].req, 0); 2840 IXGBE_EVC_STORE(&txr->total_packets, 0); 2841 #ifndef IXGBE_LEGACY_TX 2842 IXGBE_EVC_STORE(&txr->pcq_drops, 0); 2843 #endif 2844 IXGBE_EVC_STORE(&txr->no_desc_avail, 0); 2845 IXGBE_EVC_STORE(&txr->tso_tx, 0); 2846 txr->q_efbig_tx_dma_setup = 0; 2847 txr->q_mbuf_defrag_failed = 0; 2848 txr->q_efbig2_tx_dma_setup = 0; 2849 txr->q_einval_tx_dma_setup = 0; 2850 txr->q_other_tx_dma_setup = 0; 2851 txr->q_eagain_tx_dma_setup = 0; 2852 txr->q_enomem_tx_dma_setup = 0; 2853 txr->q_tso_err = 0; 2854 2855 IXGBE_EVC_STORE(&rxr->rx_packets, 0); 2856 IXGBE_EVC_STORE(&rxr->rx_bytes, 0); 2857 IXGBE_EVC_STORE(&rxr->rx_copies, 0); 2858 IXGBE_EVC_STORE(&rxr->no_mbuf, 0); 2859 IXGBE_EVC_STORE(&rxr->rx_discarded, 0); 2860 } 2861 2862 /* MAC stats get their own sub node */ 2863 2864 IXGBE_EVC_STORE(&stats->ipcs, 0); 2865 IXGBE_EVC_STORE(&stats->l4cs, 0); 2866 IXGBE_EVC_STORE(&stats->ipcs_bad, 0); 2867 IXGBE_EVC_STORE(&stats->l4cs_bad, 0); 2868 2869 /* 2870 * Packet Reception Stats. 2871 * Call ixv_init_stats() to save last VF counters' values. 2872 */ 2873 ixv_init_stats(adapter); 2874 IXGBE_EVC_STORE(&stats->vfgprc, 0); 2875 IXGBE_EVC_STORE(&stats->vfgorc, 0); 2876 IXGBE_EVC_STORE(&stats->vfmprc, 0); 2877 IXGBE_EVC_STORE(&stats->vfgptc, 0); 2878 IXGBE_EVC_STORE(&stats->vfgotc, 0); 2879 2880 /* Mailbox Stats */ 2881 IXGBE_EVC_STORE(&hw->mbx.stats.msgs_tx, 0); 2882 IXGBE_EVC_STORE(&hw->mbx.stats.msgs_rx, 0); 2883 IXGBE_EVC_STORE(&hw->mbx.stats.acks, 0); 2884 IXGBE_EVC_STORE(&hw->mbx.stats.reqs, 0); 2885 IXGBE_EVC_STORE(&hw->mbx.stats.rsts, 0); 2886 2887 } /* ixv_clear_evcnt */ 2888 2889 #define PRINTQS(adapter, regname) \ 2890 do { \ 2891 struct ixgbe_hw *_hw = &(adapter)->hw; \ 2892 int _i; \ 2893 \ 2894 printf("%s: %s", device_xname((adapter)->dev), #regname); \ 2895 for (_i = 0; _i < (adapter)->num_queues; _i++) { \ 2896 printf((_i == 0) ? "\t" : " "); \ 2897 printf("%08x", IXGBE_READ_REG(_hw, \ 2898 IXGBE_##regname(_i))); \ 2899 } \ 2900 printf("\n"); \ 2901 } while (0) 2902 2903 /************************************************************************ 2904 * ixv_print_debug_info 2905 * 2906 * Provides a way to take a look at important statistics 2907 * maintained by the driver and hardware. 2908 ************************************************************************/ 2909 static void 2910 ixv_print_debug_info(struct adapter *adapter) 2911 { 2912 device_t dev = adapter->dev; 2913 struct ixgbe_hw *hw = &adapter->hw; 2914 int i; 2915 2916 device_printf(dev, "queue:"); 2917 for (i = 0; i < adapter->num_queues; i++) { 2918 printf((i == 0) ? "\t" : " "); 2919 printf("%8d", i); 2920 } 2921 printf("\n"); 2922 PRINTQS(adapter, VFRDBAL); 2923 PRINTQS(adapter, VFRDBAH); 2924 PRINTQS(adapter, VFRDLEN); 2925 PRINTQS(adapter, VFSRRCTL); 2926 PRINTQS(adapter, VFRDH); 2927 PRINTQS(adapter, VFRDT); 2928 PRINTQS(adapter, VFRXDCTL); 2929 2930 device_printf(dev, "EIMS:\t%08x\n", IXGBE_READ_REG(hw, IXGBE_VTEIMS)); 2931 device_printf(dev, "EIAM:\t%08x\n", IXGBE_READ_REG(hw, IXGBE_VTEIAM)); 2932 device_printf(dev, "EIAC:\t%08x\n", IXGBE_READ_REG(hw, IXGBE_VTEIAC)); 2933 } /* ixv_print_debug_info */ 2934 2935 /************************************************************************ 2936 * ixv_sysctl_debug 2937 ************************************************************************/ 2938 static int 2939 ixv_sysctl_debug(SYSCTLFN_ARGS) 2940 { 2941 struct sysctlnode node = *rnode; 2942 struct adapter *adapter = (struct adapter *)node.sysctl_data; 2943 int error, result = 0; 2944 2945 node.sysctl_data = &result; 2946 error = sysctl_lookup(SYSCTLFN_CALL(&node)); 2947 2948 if (error || newp == NULL) 2949 return error; 2950 2951 if (result == 1) 2952 ixv_print_debug_info(adapter); 2953 2954 return 0; 2955 } /* ixv_sysctl_debug */ 2956 2957 /************************************************************************ 2958 * ixv_sysctl_rx_copy_len 2959 ************************************************************************/ 2960 static int 2961 ixv_sysctl_rx_copy_len(SYSCTLFN_ARGS) 2962 { 2963 struct sysctlnode node = *rnode; 2964 struct adapter *adapter = (struct adapter *)node.sysctl_data; 2965 int error; 2966 int result = adapter->rx_copy_len; 2967 2968 node.sysctl_data = &result; 2969 error = sysctl_lookup(SYSCTLFN_CALL(&node)); 2970 2971 if (error || newp == NULL) 2972 return error; 2973 2974 if ((result < 0) || (result > IXGBE_RX_COPY_LEN_MAX)) 2975 return EINVAL; 2976 2977 adapter->rx_copy_len = result; 2978 2979 return 0; 2980 } /* ixv_sysctl_rx_copy_len */ 2981 2982 /************************************************************************ 2983 * ixv_sysctl_tx_process_limit 2984 ************************************************************************/ 2985 static int 2986 ixv_sysctl_tx_process_limit(SYSCTLFN_ARGS) 2987 { 2988 struct sysctlnode node = *rnode; 2989 struct adapter *adapter = (struct adapter *)node.sysctl_data; 2990 int error; 2991 int result = adapter->tx_process_limit; 2992 2993 node.sysctl_data = &result; 2994 error = sysctl_lookup(SYSCTLFN_CALL(&node)); 2995 2996 if (error || newp == NULL) 2997 return error; 2998 2999 if ((result <= 0) || (result > adapter->num_tx_desc)) 3000 return EINVAL; 3001 3002 adapter->tx_process_limit = result; 3003 3004 return 0; 3005 } /* ixv_sysctl_tx_process_limit */ 3006 3007 /************************************************************************ 3008 * ixv_sysctl_rx_process_limit 3009 ************************************************************************/ 3010 static int 3011 ixv_sysctl_rx_process_limit(SYSCTLFN_ARGS) 3012 { 3013 struct sysctlnode node = *rnode; 3014 struct adapter *adapter = (struct adapter *)node.sysctl_data; 3015 int error; 3016 int result = adapter->rx_process_limit; 3017 3018 node.sysctl_data = &result; 3019 error = sysctl_lookup(SYSCTLFN_CALL(&node)); 3020 3021 if (error || newp == NULL) 3022 return error; 3023 3024 if ((result <= 0) || (result > adapter->num_rx_desc)) 3025 return EINVAL; 3026 3027 adapter->rx_process_limit = result; 3028 3029 return 0; 3030 } /* ixv_sysctl_rx_process_limit */ 3031 3032 /************************************************************************ 3033 * ixv_init_device_features 3034 ************************************************************************/ 3035 static void 3036 ixv_init_device_features(struct adapter *adapter) 3037 { 3038 adapter->feat_cap = IXGBE_FEATURE_NETMAP 3039 | IXGBE_FEATURE_VF 3040 | IXGBE_FEATURE_RSS 3041 | IXGBE_FEATURE_LEGACY_TX; 3042 3043 /* A tad short on feature flags for VFs, atm. */ 3044 switch (adapter->hw.mac.type) { 3045 case ixgbe_mac_82599_vf: 3046 break; 3047 case ixgbe_mac_X540_vf: 3048 break; 3049 case ixgbe_mac_X550_vf: 3050 case ixgbe_mac_X550EM_x_vf: 3051 case ixgbe_mac_X550EM_a_vf: 3052 adapter->feat_cap |= IXGBE_FEATURE_NEEDS_CTXD; 3053 break; 3054 default: 3055 break; 3056 } 3057 3058 /* Enabled by default... */ 3059 /* Is a virtual function (VF) */ 3060 if (adapter->feat_cap & IXGBE_FEATURE_VF) 3061 adapter->feat_en |= IXGBE_FEATURE_VF; 3062 /* Netmap */ 3063 if (adapter->feat_cap & IXGBE_FEATURE_NETMAP) 3064 adapter->feat_en |= IXGBE_FEATURE_NETMAP; 3065 /* Receive-Side Scaling (RSS) */ 3066 if (adapter->feat_cap & IXGBE_FEATURE_RSS) 3067 adapter->feat_en |= IXGBE_FEATURE_RSS; 3068 /* Needs advanced context descriptor regardless of offloads req'd */ 3069 if (adapter->feat_cap & IXGBE_FEATURE_NEEDS_CTXD) 3070 adapter->feat_en |= IXGBE_FEATURE_NEEDS_CTXD; 3071 3072 /* Enabled via sysctl... */ 3073 /* Legacy (single queue) transmit */ 3074 if ((adapter->feat_cap & IXGBE_FEATURE_LEGACY_TX) && 3075 ixv_enable_legacy_tx) 3076 adapter->feat_en |= IXGBE_FEATURE_LEGACY_TX; 3077 } /* ixv_init_device_features */ 3078 3079 /************************************************************************ 3080 * ixv_shutdown - Shutdown entry point 3081 ************************************************************************/ 3082 #if 0 /* XXX NetBSD ought to register something like this through pmf(9) */ 3083 static int 3084 ixv_shutdown(device_t dev) 3085 { 3086 struct adapter *adapter = device_private(dev); 3087 IXGBE_CORE_LOCK(adapter); 3088 ixv_stop_locked(adapter); 3089 IXGBE_CORE_UNLOCK(adapter); 3090 3091 return (0); 3092 } /* ixv_shutdown */ 3093 #endif 3094 3095 static int 3096 ixv_ifflags_cb(struct ethercom *ec) 3097 { 3098 struct ifnet *ifp = &ec->ec_if; 3099 struct adapter *adapter = ifp->if_softc; 3100 u_short saved_flags; 3101 u_short change; 3102 int rv = 0; 3103 3104 IXGBE_CORE_LOCK(adapter); 3105 3106 saved_flags = adapter->if_flags; 3107 change = ifp->if_flags ^ adapter->if_flags; 3108 if (change != 0) 3109 adapter->if_flags = ifp->if_flags; 3110 3111 if ((change & ~(IFF_CANTCHANGE | IFF_DEBUG)) != 0) { 3112 rv = ENETRESET; 3113 goto out; 3114 } else if ((change & IFF_PROMISC) != 0) { 3115 rv = ixv_set_rxfilter(adapter); 3116 if (rv != 0) { 3117 /* Restore previous */ 3118 adapter->if_flags = saved_flags; 3119 goto out; 3120 } 3121 } 3122 3123 /* Check for ec_capenable. */ 3124 change = ec->ec_capenable ^ adapter->ec_capenable; 3125 adapter->ec_capenable = ec->ec_capenable; 3126 if ((change & ~(ETHERCAP_VLAN_MTU | ETHERCAP_VLAN_HWTAGGING 3127 | ETHERCAP_VLAN_HWFILTER)) != 0) { 3128 rv = ENETRESET; 3129 goto out; 3130 } 3131 3132 /* 3133 * Special handling is not required for ETHERCAP_VLAN_MTU. 3134 * PF's MAXFRS(MHADD) does not include the 4bytes of the VLAN header. 3135 */ 3136 3137 /* Set up VLAN support and filter */ 3138 if ((change & (ETHERCAP_VLAN_HWTAGGING | ETHERCAP_VLAN_HWFILTER)) != 0) 3139 rv = ixv_setup_vlan_support(adapter); 3140 3141 out: 3142 IXGBE_CORE_UNLOCK(adapter); 3143 3144 return rv; 3145 } 3146 3147 3148 /************************************************************************ 3149 * ixv_ioctl - Ioctl entry point 3150 * 3151 * Called when the user wants to configure the interface. 3152 * 3153 * return 0 on success, positive on failure 3154 ************************************************************************/ 3155 static int 3156 ixv_ioctl(struct ifnet *ifp, u_long command, void *data) 3157 { 3158 struct adapter *adapter = ifp->if_softc; 3159 struct ixgbe_hw *hw = &adapter->hw; 3160 struct ifcapreq *ifcr = data; 3161 int error; 3162 int l4csum_en; 3163 const int l4csum = IFCAP_CSUM_TCPv4_Rx | IFCAP_CSUM_UDPv4_Rx | 3164 IFCAP_CSUM_TCPv6_Rx | IFCAP_CSUM_UDPv6_Rx; 3165 3166 switch (command) { 3167 case SIOCSIFFLAGS: 3168 IOCTL_DEBUGOUT("ioctl: SIOCSIFFLAGS (Set Interface Flags)"); 3169 break; 3170 case SIOCADDMULTI: { 3171 struct ether_multi *enm; 3172 struct ether_multistep step; 3173 struct ethercom *ec = &adapter->osdep.ec; 3174 bool overflow = false; 3175 int mcnt = 0; 3176 3177 /* 3178 * Check the number of multicast address. If it exceeds, 3179 * return ENOSPC. 3180 * Update this code when we support API 1.3. 3181 */ 3182 ETHER_LOCK(ec); 3183 ETHER_FIRST_MULTI(step, ec, enm); 3184 while (enm != NULL) { 3185 mcnt++; 3186 3187 /* 3188 * This code is before adding, so one room is required 3189 * at least. 3190 */ 3191 if (mcnt > (IXGBE_MAX_VF_MC - 1)) { 3192 overflow = true; 3193 break; 3194 } 3195 ETHER_NEXT_MULTI(step, enm); 3196 } 3197 ETHER_UNLOCK(ec); 3198 error = 0; 3199 if (overflow && ((ec->ec_flags & ETHER_F_ALLMULTI) == 0)) { 3200 error = hw->mac.ops.update_xcast_mode(hw, 3201 IXGBEVF_XCAST_MODE_ALLMULTI); 3202 if (error == IXGBE_ERR_NOT_TRUSTED) { 3203 device_printf(adapter->dev, 3204 "this interface is not trusted\n"); 3205 error = EPERM; 3206 } else if (error == IXGBE_ERR_FEATURE_NOT_SUPPORTED) { 3207 device_printf(adapter->dev, 3208 "the PF doesn't support allmulti mode\n"); 3209 error = EOPNOTSUPP; 3210 } else if (error) { 3211 device_printf(adapter->dev, 3212 "number of Ethernet multicast addresses " 3213 "exceeds the limit (%d). error = %d\n", 3214 IXGBE_MAX_VF_MC, error); 3215 error = ENOSPC; 3216 } else 3217 ec->ec_flags |= ETHER_F_ALLMULTI; 3218 } 3219 if (error) 3220 return error; 3221 } 3222 /*FALLTHROUGH*/ 3223 case SIOCDELMULTI: 3224 IOCTL_DEBUGOUT("ioctl: SIOC(ADD|DEL)MULTI"); 3225 break; 3226 case SIOCSIFMEDIA: 3227 case SIOCGIFMEDIA: 3228 IOCTL_DEBUGOUT("ioctl: SIOCxIFMEDIA (Get/Set Interface Media)"); 3229 break; 3230 case SIOCSIFCAP: 3231 IOCTL_DEBUGOUT("ioctl: SIOCSIFCAP (Set Capabilities)"); 3232 break; 3233 case SIOCSIFMTU: 3234 IOCTL_DEBUGOUT("ioctl: SIOCSIFMTU (Set Interface MTU)"); 3235 break; 3236 case SIOCZIFDATA: 3237 IOCTL_DEBUGOUT("ioctl: SIOCZIFDATA (Zero counter)"); 3238 ixv_update_stats(adapter); 3239 ixv_clear_evcnt(adapter); 3240 break; 3241 default: 3242 IOCTL_DEBUGOUT1("ioctl: UNKNOWN (0x%X)", (int)command); 3243 break; 3244 } 3245 3246 switch (command) { 3247 case SIOCSIFCAP: 3248 /* Layer-4 Rx checksum offload has to be turned on and 3249 * off as a unit. 3250 */ 3251 l4csum_en = ifcr->ifcr_capenable & l4csum; 3252 if (l4csum_en != l4csum && l4csum_en != 0) 3253 return EINVAL; 3254 /*FALLTHROUGH*/ 3255 case SIOCADDMULTI: 3256 case SIOCDELMULTI: 3257 case SIOCSIFFLAGS: 3258 case SIOCSIFMTU: 3259 default: 3260 if ((error = ether_ioctl(ifp, command, data)) != ENETRESET) 3261 return error; 3262 if ((ifp->if_flags & IFF_RUNNING) == 0) 3263 ; 3264 else if (command == SIOCSIFCAP || command == SIOCSIFMTU) { 3265 IXGBE_CORE_LOCK(adapter); 3266 ixv_init_locked(adapter); 3267 IXGBE_CORE_UNLOCK(adapter); 3268 } else if (command == SIOCADDMULTI || command == SIOCDELMULTI) { 3269 /* 3270 * Multicast list has changed; set the hardware filter 3271 * accordingly. 3272 */ 3273 IXGBE_CORE_LOCK(adapter); 3274 ixv_disable_intr(adapter); 3275 ixv_set_rxfilter(adapter); 3276 ixv_enable_intr(adapter); 3277 IXGBE_CORE_UNLOCK(adapter); 3278 } 3279 return 0; 3280 } 3281 } /* ixv_ioctl */ 3282 3283 /************************************************************************ 3284 * ixv_init 3285 ************************************************************************/ 3286 static int 3287 ixv_init(struct ifnet *ifp) 3288 { 3289 struct adapter *adapter = ifp->if_softc; 3290 3291 IXGBE_CORE_LOCK(adapter); 3292 ixv_init_locked(adapter); 3293 IXGBE_CORE_UNLOCK(adapter); 3294 3295 return 0; 3296 } /* ixv_init */ 3297 3298 /************************************************************************ 3299 * ixv_handle_que 3300 ************************************************************************/ 3301 static void 3302 ixv_handle_que(void *context) 3303 { 3304 struct ix_queue *que = context; 3305 struct adapter *adapter = que->adapter; 3306 struct tx_ring *txr = que->txr; 3307 struct ifnet *ifp = adapter->ifp; 3308 bool more; 3309 3310 IXGBE_EVC_ADD(&que->handleq, 1); 3311 3312 if (ifp->if_flags & IFF_RUNNING) { 3313 IXGBE_TX_LOCK(txr); 3314 more = ixgbe_txeof(txr); 3315 if (!(adapter->feat_en & IXGBE_FEATURE_LEGACY_TX)) 3316 if (!ixgbe_mq_ring_empty(ifp, txr->txr_interq)) 3317 ixgbe_mq_start_locked(ifp, txr); 3318 /* Only for queue 0 */ 3319 /* NetBSD still needs this for CBQ */ 3320 if ((&adapter->queues[0] == que) 3321 && (!ixgbe_legacy_ring_empty(ifp, NULL))) 3322 ixgbe_legacy_start_locked(ifp, txr); 3323 IXGBE_TX_UNLOCK(txr); 3324 more |= ixgbe_rxeof(que); 3325 if (more) { 3326 IXGBE_EVC_ADD(&que->req, 1); 3327 if (adapter->txrx_use_workqueue) { 3328 /* 3329 * "enqueued flag" is not required here 3330 * the same as ixg(4). See ixgbe_msix_que(). 3331 */ 3332 workqueue_enqueue(adapter->que_wq, 3333 &que->wq_cookie, curcpu()); 3334 } else 3335 softint_schedule(que->que_si); 3336 return; 3337 } 3338 } 3339 3340 /* Re-enable this interrupt */ 3341 ixv_enable_queue(adapter, que->msix); 3342 3343 return; 3344 } /* ixv_handle_que */ 3345 3346 /************************************************************************ 3347 * ixv_handle_que_work 3348 ************************************************************************/ 3349 static void 3350 ixv_handle_que_work(struct work *wk, void *context) 3351 { 3352 struct ix_queue *que = container_of(wk, struct ix_queue, wq_cookie); 3353 3354 /* 3355 * "enqueued flag" is not required here the same as ixg(4). 3356 * See ixgbe_msix_que(). 3357 */ 3358 ixv_handle_que(que); 3359 } 3360 3361 /************************************************************************ 3362 * ixv_allocate_msix - Setup MSI-X Interrupt resources and handlers 3363 ************************************************************************/ 3364 static int 3365 ixv_allocate_msix(struct adapter *adapter, const struct pci_attach_args *pa) 3366 { 3367 device_t dev = adapter->dev; 3368 struct ix_queue *que = adapter->queues; 3369 struct tx_ring *txr = adapter->tx_rings; 3370 int error, msix_ctrl, rid, vector = 0; 3371 pci_chipset_tag_t pc; 3372 pcitag_t tag; 3373 char intrbuf[PCI_INTRSTR_LEN]; 3374 char wqname[MAXCOMLEN]; 3375 char intr_xname[32]; 3376 const char *intrstr = NULL; 3377 kcpuset_t *affinity; 3378 int cpu_id = 0; 3379 3380 pc = adapter->osdep.pc; 3381 tag = adapter->osdep.tag; 3382 3383 adapter->osdep.nintrs = adapter->num_queues + 1; 3384 if (pci_msix_alloc_exact(pa, &adapter->osdep.intrs, 3385 adapter->osdep.nintrs) != 0) { 3386 aprint_error_dev(dev, 3387 "failed to allocate MSI-X interrupt\n"); 3388 return (ENXIO); 3389 } 3390 3391 kcpuset_create(&affinity, false); 3392 for (int i = 0; i < adapter->num_queues; i++, vector++, que++, txr++) { 3393 snprintf(intr_xname, sizeof(intr_xname), "%s TXRX%d", 3394 device_xname(dev), i); 3395 intrstr = pci_intr_string(pc, adapter->osdep.intrs[i], intrbuf, 3396 sizeof(intrbuf)); 3397 #ifdef IXGBE_MPSAFE 3398 pci_intr_setattr(pc, &adapter->osdep.intrs[i], PCI_INTR_MPSAFE, 3399 true); 3400 #endif 3401 /* Set the handler function */ 3402 que->res = adapter->osdep.ihs[i] = pci_intr_establish_xname(pc, 3403 adapter->osdep.intrs[i], IPL_NET, ixv_msix_que, que, 3404 intr_xname); 3405 if (que->res == NULL) { 3406 pci_intr_release(pc, adapter->osdep.intrs, 3407 adapter->osdep.nintrs); 3408 aprint_error_dev(dev, 3409 "Failed to register QUE handler\n"); 3410 kcpuset_destroy(affinity); 3411 return (ENXIO); 3412 } 3413 que->msix = vector; 3414 adapter->active_queues |= (u64)(1 << que->msix); 3415 3416 cpu_id = i; 3417 /* Round-robin affinity */ 3418 kcpuset_zero(affinity); 3419 kcpuset_set(affinity, cpu_id % ncpu); 3420 error = interrupt_distribute(adapter->osdep.ihs[i], affinity, 3421 NULL); 3422 aprint_normal_dev(dev, "for TX/RX, interrupting at %s", 3423 intrstr); 3424 if (error == 0) 3425 aprint_normal(", bound queue %d to cpu %d\n", 3426 i, cpu_id % ncpu); 3427 else 3428 aprint_normal("\n"); 3429 3430 #ifndef IXGBE_LEGACY_TX 3431 txr->txr_si 3432 = softint_establish(SOFTINT_NET | IXGBE_SOFTINT_FLAGS, 3433 ixgbe_deferred_mq_start, txr); 3434 #endif 3435 que->que_si 3436 = softint_establish(SOFTINT_NET | IXGBE_SOFTINT_FLAGS, 3437 ixv_handle_que, que); 3438 if (que->que_si == NULL) { 3439 aprint_error_dev(dev, 3440 "could not establish software interrupt\n"); 3441 } 3442 } 3443 snprintf(wqname, sizeof(wqname), "%sdeferTx", device_xname(dev)); 3444 error = workqueue_create(&adapter->txr_wq, wqname, 3445 ixgbe_deferred_mq_start_work, adapter, IXGBE_WORKQUEUE_PRI, IPL_NET, 3446 IXGBE_WORKQUEUE_FLAGS); 3447 if (error) { 3448 aprint_error_dev(dev, 3449 "couldn't create workqueue for deferred Tx\n"); 3450 } 3451 adapter->txr_wq_enqueued = percpu_alloc(sizeof(u_int)); 3452 3453 snprintf(wqname, sizeof(wqname), "%sTxRx", device_xname(dev)); 3454 error = workqueue_create(&adapter->que_wq, wqname, 3455 ixv_handle_que_work, adapter, IXGBE_WORKQUEUE_PRI, IPL_NET, 3456 IXGBE_WORKQUEUE_FLAGS); 3457 if (error) { 3458 aprint_error_dev(dev, "couldn't create workqueue for Tx/Rx\n"); 3459 } 3460 3461 /* and Mailbox */ 3462 cpu_id++; 3463 snprintf(intr_xname, sizeof(intr_xname), "%s link", device_xname(dev)); 3464 adapter->vector = vector; 3465 intrstr = pci_intr_string(pc, adapter->osdep.intrs[vector], intrbuf, 3466 sizeof(intrbuf)); 3467 #ifdef IXGBE_MPSAFE 3468 pci_intr_setattr(pc, &adapter->osdep.intrs[vector], PCI_INTR_MPSAFE, 3469 true); 3470 #endif 3471 /* Set the mbx handler function */ 3472 adapter->osdep.ihs[vector] = pci_intr_establish_xname(pc, 3473 adapter->osdep.intrs[vector], IPL_NET, ixv_msix_mbx, adapter, 3474 intr_xname); 3475 if (adapter->osdep.ihs[vector] == NULL) { 3476 aprint_error_dev(dev, "Failed to register LINK handler\n"); 3477 kcpuset_destroy(affinity); 3478 return (ENXIO); 3479 } 3480 /* Round-robin affinity */ 3481 kcpuset_zero(affinity); 3482 kcpuset_set(affinity, cpu_id % ncpu); 3483 error = interrupt_distribute(adapter->osdep.ihs[vector], affinity, 3484 NULL); 3485 3486 aprint_normal_dev(dev, 3487 "for link, interrupting at %s", intrstr); 3488 if (error == 0) 3489 aprint_normal(", affinity to cpu %d\n", cpu_id % ncpu); 3490 else 3491 aprint_normal("\n"); 3492 3493 /* Tasklets for Mailbox */ 3494 snprintf(wqname, sizeof(wqname), "%s-admin", device_xname(dev)); 3495 error = workqueue_create(&adapter->admin_wq, wqname, 3496 ixv_handle_admin, adapter, IXGBE_WORKQUEUE_PRI, IPL_NET, 3497 IXGBE_TASKLET_WQ_FLAGS); 3498 if (error) { 3499 aprint_error_dev(dev, 3500 "could not create admin workqueue (%d)\n", error); 3501 goto err_out; 3502 } 3503 3504 /* 3505 * Due to a broken design QEMU will fail to properly 3506 * enable the guest for MSI-X unless the vectors in 3507 * the table are all set up, so we must rewrite the 3508 * ENABLE in the MSI-X control register again at this 3509 * point to cause it to successfully initialize us. 3510 */ 3511 if (adapter->hw.mac.type == ixgbe_mac_82599_vf) { 3512 pci_get_capability(pc, tag, PCI_CAP_MSIX, &rid, NULL); 3513 rid += PCI_MSIX_CTL; 3514 msix_ctrl = pci_conf_read(pc, tag, rid); 3515 msix_ctrl |= PCI_MSIX_CTL_ENABLE; 3516 pci_conf_write(pc, tag, rid, msix_ctrl); 3517 } 3518 3519 kcpuset_destroy(affinity); 3520 return (0); 3521 err_out: 3522 kcpuset_destroy(affinity); 3523 ixv_free_deferred_handlers(adapter); 3524 ixv_free_pci_resources(adapter); 3525 return (error); 3526 } /* ixv_allocate_msix */ 3527 3528 /************************************************************************ 3529 * ixv_configure_interrupts - Setup MSI-X resources 3530 * 3531 * Note: The VF device MUST use MSI-X, there is no fallback. 3532 ************************************************************************/ 3533 static int 3534 ixv_configure_interrupts(struct adapter *adapter) 3535 { 3536 device_t dev = adapter->dev; 3537 int want, queues, msgs; 3538 3539 /* Must have at least 2 MSI-X vectors */ 3540 msgs = pci_msix_count(adapter->osdep.pc, adapter->osdep.tag); 3541 if (msgs < 2) { 3542 aprint_error_dev(dev, "MSIX config error\n"); 3543 return (ENXIO); 3544 } 3545 msgs = MIN(msgs, IXG_MAX_NINTR); 3546 3547 /* Figure out a reasonable auto config value */ 3548 queues = (ncpu > (msgs - 1)) ? (msgs - 1) : ncpu; 3549 3550 if (ixv_num_queues != 0) 3551 queues = ixv_num_queues; 3552 else if ((ixv_num_queues == 0) && (queues > IXGBE_VF_MAX_TX_QUEUES)) 3553 queues = IXGBE_VF_MAX_TX_QUEUES; 3554 3555 /* 3556 * Want vectors for the queues, 3557 * plus an additional for mailbox. 3558 */ 3559 want = queues + 1; 3560 if (msgs >= want) 3561 msgs = want; 3562 else { 3563 aprint_error_dev(dev, 3564 "MSI-X Configuration Problem, " 3565 "%d vectors but %d queues wanted!\n", msgs, want); 3566 return -1; 3567 } 3568 3569 aprint_normal_dev(dev, 3570 "Using MSI-X interrupts with %d vectors\n", msgs); 3571 adapter->num_queues = queues; 3572 3573 return (0); 3574 } /* ixv_configure_interrupts */ 3575 3576 3577 /************************************************************************ 3578 * ixv_handle_admin - Tasklet handler for MSI-X MBX interrupts 3579 * 3580 * Done outside of interrupt context since the driver might sleep 3581 ************************************************************************/ 3582 static void 3583 ixv_handle_admin(struct work *wk, void *context) 3584 { 3585 struct adapter *adapter = context; 3586 struct ixgbe_hw *hw = &adapter->hw; 3587 3588 IXGBE_CORE_LOCK(adapter); 3589 3590 IXGBE_EVC_ADD(&adapter->link_workev, 1); 3591 adapter->hw.mac.ops.check_link(&adapter->hw, &adapter->link_speed, 3592 &adapter->link_up, FALSE); 3593 ixv_update_link_status(adapter); 3594 3595 adapter->task_requests = 0; 3596 atomic_store_relaxed(&adapter->admin_pending, 0); 3597 3598 /* Re-enable interrupts */ 3599 IXGBE_WRITE_REG(hw, IXGBE_VTEIMS, (1 << adapter->vector)); 3600 3601 IXGBE_CORE_UNLOCK(adapter); 3602 } /* ixv_handle_admin */ 3603 3604 /************************************************************************ 3605 * ixv_check_link - Used in the local timer to poll for link changes 3606 ************************************************************************/ 3607 static s32 3608 ixv_check_link(struct adapter *adapter) 3609 { 3610 s32 error; 3611 3612 KASSERT(mutex_owned(&adapter->core_mtx)); 3613 3614 adapter->hw.mac.get_link_status = TRUE; 3615 3616 error = adapter->hw.mac.ops.check_link(&adapter->hw, 3617 &adapter->link_speed, &adapter->link_up, FALSE); 3618 ixv_update_link_status(adapter); 3619 3620 return error; 3621 } /* ixv_check_link */ 3622
Indexes created Tue Sep 30 20:09:53 GMT 2025