1 /* $NetBSD: ixv.c,v 1.200 2026/10/15 13:36:47 joe Exp $ */ 2 3 /****************************************************************************** 4 5 Copyright (c) 2001-2017, Intel Corporation 6 All rights reserved. 7 8 Redistribution and use in source and binary forms, with or without 9 modification, are permitted provided that the following conditions are met: 10 11 1. Redistributions of source code must retain the above copyright notice, 12 this list of conditions and the following disclaimer. 13 14 2. Redistributions in binary form must reproduce the above copyright 15 notice, this list of conditions and the following disclaimer in the 16 documentation and/or other materials provided with the distribution. 17 18 3. Neither the name of the Intel Corporation nor the names of its 19 contributors may be used to endorse or promote products derived from 20 this software without specific prior written permission. 21 22 THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 23 AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 24 IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 25 ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE 26 LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 27 CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 28 SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 29 INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 30 CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 31 ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 32 POSSIBILITY OF SUCH DAMAGE. 33 34 ******************************************************************************/ 35 /*$FreeBSD: head/sys/dev/ixgbe/if_ixv.c 331224 2018-03-19 20:55:05Z erj $*/ 36 37 #include <sys/cdefs.h> 38 __KERNEL_RCSID(0, "$NetBSD: ixv.c,v 1.200 2026/10/15 13:36:47 joe Exp $"); 39 40 #ifdef _KERNEL_OPT 41 #include "opt_inet.h" 42 #include "opt_inet6.h" 43 #endif 44 45 #include "ixgbe.h" 46 47 /************************************************************************ 48 * Driver version 49 ************************************************************************/ 50 static const char ixv_driver_version[] = "2.0.1-k"; 51 /* XXX NetBSD: + 1.5.17 */ 52 53 /************************************************************************ 54 * PCI Device ID Table 55 * 56 * Used by probe to select devices to load on 57 * Last field stores an index into ixv_strings 58 * Last entry must be all 0s 59 * 60 * { Vendor ID, Device ID, SubVendor ID, SubDevice ID, String Index } 61 ************************************************************************/ 62 static const ixgbe_vendor_info_t ixv_vendor_info_array[] = 63 { 64 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_VF, 0, 0, 0}, 65 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_X540_VF, 0, 0, 0}, 66 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_X550_VF, 0, 0, 0}, 67 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_X550EM_X_VF, 0, 0, 0}, 68 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_X550EM_A_VF, 0, 0, 0}, 69 /* required last entry */ 70 {0, 0, 0, 0, 0} 71 }; 72 73 /************************************************************************ 74 * Table of branding strings 75 ************************************************************************/ 76 static const char *ixv_strings[] = { 77 "Intel(R) PRO/10GbE Virtual Function Network Driver" 78 }; 79 80 /********************************************************************* 81 * Function prototypes 82 *********************************************************************/ 83 static int ixv_probe(device_t, cfdata_t, void *); 84 static void ixv_attach(device_t, device_t, void *); 85 static int ixv_detach(device_t, int); 86 #if 0 87 static int ixv_shutdown(device_t); 88 #endif 89 static int ixv_ifflags_cb(struct ethercom *); 90 static int ixv_ioctl(struct ifnet *, u_long, void *); 91 static int ixv_init(struct ifnet *); 92 static void ixv_init_locked(struct ixgbe_softc *); 93 static void ixv_ifstop(struct ifnet *, int); 94 static void ixv_stop_locked(void *); 95 static void ixv_init_device_features(struct ixgbe_softc *); 96 static void ixv_media_status(struct ifnet *, struct ifmediareq *); 97 static int ixv_media_change(struct ifnet *); 98 static int ixv_allocate_pci_resources(struct ixgbe_softc *, 99 const struct pci_attach_args *); 100 static void ixv_free_deferred_handlers(struct ixgbe_softc *); 101 static int ixv_allocate_msix(struct ixgbe_softc *, 102 const struct pci_attach_args *); 103 static int ixv_configure_interrupts(struct ixgbe_softc *); 104 static void ixv_free_pci_resources(struct ixgbe_softc *); 105 static void ixv_local_timer(void *); 106 static void ixv_handle_timer(struct work *, void *); 107 static int ixv_setup_interface(device_t, struct ixgbe_softc *); 108 static void ixv_schedule_admin_tasklet(struct ixgbe_softc *); 109 static int ixv_negotiate_api(struct ixgbe_softc *); 110 111 static void ixv_initialize_transmit_units(struct ixgbe_softc *); 112 static void ixv_initialize_receive_units(struct ixgbe_softc *); 113 static void ixv_initialize_rss_mapping(struct ixgbe_softc *); 114 static s32 ixv_check_link(struct ixgbe_softc *); 115 116 static void ixv_enable_intr(struct ixgbe_softc *); 117 static void ixv_disable_intr(struct ixgbe_softc *); 118 static int ixv_set_rxfilter(struct ixgbe_softc *); 119 static void ixv_update_link_status(struct ixgbe_softc *); 120 static int ixv_sysctl_debug(SYSCTLFN_PROTO); 121 static void ixv_set_ivar(struct ixgbe_softc *, u8, u8, s8); 122 static void ixv_configure_ivars(struct ixgbe_softc *); 123 static u8 * ixv_mc_array_itr(struct ixgbe_hw *, u8 **, u32 *); 124 static void ixv_eitr_write(struct ixgbe_softc *, uint32_t, uint32_t); 125 126 static void ixv_setup_vlan_tagging(struct ixgbe_softc *); 127 static int ixv_setup_vlan_support(struct ixgbe_softc *); 128 static int ixv_vlan_cb(struct ethercom *, uint16_t, bool); 129 static int ixv_register_vlan(struct ixgbe_softc *, u16); 130 static int ixv_unregister_vlan(struct ixgbe_softc *, u16); 131 132 static void ixv_add_device_sysctls(struct ixgbe_softc *); 133 static void ixv_init_stats(struct ixgbe_softc *); 134 static void ixv_update_stats(struct ixgbe_softc *); 135 static void ixv_add_stats_sysctls(struct ixgbe_softc *); 136 static void ixv_clear_evcnt(struct ixgbe_softc *); 137 138 /* Sysctl handlers */ 139 static int ixv_sysctl_interrupt_rate_handler(SYSCTLFN_PROTO); 140 static int ixv_sysctl_next_to_check_handler(SYSCTLFN_PROTO); 141 static int ixv_sysctl_next_to_refresh_handler(SYSCTLFN_PROTO); 142 static int ixv_sysctl_rdh_handler(SYSCTLFN_PROTO); 143 static int ixv_sysctl_rdt_handler(SYSCTLFN_PROTO); 144 static int ixv_sysctl_tdt_handler(SYSCTLFN_PROTO); 145 static int ixv_sysctl_tdh_handler(SYSCTLFN_PROTO); 146 static int ixv_sysctl_tx_process_limit(SYSCTLFN_PROTO); 147 static int ixv_sysctl_rx_process_limit(SYSCTLFN_PROTO); 148 static int ixv_sysctl_rx_copy_len(SYSCTLFN_PROTO); 149 150 /* The MSI-X Interrupt handlers */ 151 static int ixv_msix_que(void *); 152 static int ixv_msix_mbx(void *); 153 154 /* Event handlers running on workqueue */ 155 static void ixv_handle_que(void *); 156 157 /* Deferred workqueue handlers */ 158 static void ixv_handle_admin(struct work *, void *); 159 static void ixv_handle_que_work(struct work *, void *); 160 161 const struct sysctlnode *ixv_sysctl_instance(struct ixgbe_softc *); 162 static const ixgbe_vendor_info_t *ixv_lookup(const struct pci_attach_args *); 163 164 /************************************************************************ 165 * NetBSD Device Interface Entry Points 166 ************************************************************************/ 167 CFATTACH_DECL3_NEW(ixv, sizeof(struct ixgbe_softc), 168 ixv_probe, ixv_attach, ixv_detach, NULL, NULL, NULL, 169 DVF_DETACH_SHUTDOWN); 170 171 #if 0 172 static driver_t ixv_driver = { 173 "ixv", ixv_methods, sizeof(struct ixgbe_softc), 174 }; 175 176 devclass_t ixv_devclass; 177 DRIVER_MODULE(ixv, pci, ixv_driver, ixv_devclass, 0, 0); 178 MODULE_DEPEND(ixv, pci, 1, 1, 1); 179 MODULE_DEPEND(ixv, ether, 1, 1, 1); 180 #endif 181 182 /* 183 * TUNEABLE PARAMETERS: 184 */ 185 186 /* Number of Queues - do not exceed MSI-X vectors - 1 */ 187 static int ixv_num_queues = 0; 188 #define TUNABLE_INT(__x, __y) 189 TUNABLE_INT("hw.ixv.num_queues", &ixv_num_queues); 190 191 /* 192 * AIM: Adaptive Interrupt Moderation 193 * which means that the interrupt rate 194 * is varied over time based on the 195 * traffic for that interrupt vector 196 */ 197 static bool ixv_enable_aim = false; 198 TUNABLE_INT("hw.ixv.enable_aim", &ixv_enable_aim); 199 200 static int ixv_max_interrupt_rate = (4000000 / IXGBE_LOW_LATENCY); 201 TUNABLE_INT("hw.ixv.max_interrupt_rate", &ixv_max_interrupt_rate); 202 203 /* How many packets rxeof tries to clean at a time */ 204 static int ixv_rx_process_limit = 256; 205 TUNABLE_INT("hw.ixv.rx_process_limit", &ixv_rx_process_limit); 206 207 /* How many packets txeof tries to clean at a time */ 208 static int ixv_tx_process_limit = 256; 209 TUNABLE_INT("hw.ixv.tx_process_limit", &ixv_tx_process_limit); 210 211 /* Which packet processing uses workqueue or softint */ 212 static bool ixv_txrx_workqueue = false; 213 214 /* 215 * Number of TX descriptors per ring, 216 * setting higher than RX as this seems 217 * the better performing choice. 218 */ 219 static int ixv_txd = DEFAULT_TXD; 220 TUNABLE_INT("hw.ixv.txd", &ixv_txd); 221 222 /* Number of RX descriptors per ring */ 223 static int ixv_rxd = DEFAULT_RXD; 224 TUNABLE_INT("hw.ixv.rxd", &ixv_rxd); 225 226 /* Legacy Transmit (single queue) */ 227 static int ixv_enable_legacy_tx = 0; 228 TUNABLE_INT("hw.ixv.enable_legacy_tx", &ixv_enable_legacy_tx); 229 230 #define IXGBE_WORKQUEUE_PRI PRI_SOFTNET 231 232 #if 0 233 static int (*ixv_start_locked)(struct ifnet *, struct tx_ring *); 234 static int (*ixv_ring_empty)(struct ifnet *, struct buf_ring *); 235 #endif 236 237 /************************************************************************ 238 * ixv_probe - Device identification routine 239 * 240 * Determines if the driver should be loaded on 241 * adapter based on its PCI vendor/device ID. 242 * 243 * return BUS_PROBE_DEFAULT on success, positive on failure 244 ************************************************************************/ 245 static int 246 ixv_probe(device_t dev, cfdata_t cf, void *aux) 247 { 248 #ifdef __HAVE_PCI_MSI_MSIX 249 const struct pci_attach_args *pa = aux; 250 251 return (ixv_lookup(pa) != NULL) ? 1 : 0; 252 #else 253 return 0; 254 #endif 255 } /* ixv_probe */ 256 257 static const ixgbe_vendor_info_t * 258 ixv_lookup(const struct pci_attach_args *pa) 259 { 260 const ixgbe_vendor_info_t *ent; 261 pcireg_t subid; 262 263 INIT_DEBUGOUT("ixv_lookup: begin"); 264 265 if (PCI_VENDOR(pa->pa_id) != IXGBE_INTEL_VENDOR_ID) 266 return NULL; 267 268 subid = pci_conf_read(pa->pa_pc, pa->pa_tag, PCI_SUBSYS_ID_REG); 269 270 for (ent = ixv_vendor_info_array; ent->vendor_id != 0; ent++) { 271 if ((PCI_VENDOR(pa->pa_id) == ent->vendor_id) && 272 (PCI_PRODUCT(pa->pa_id) == ent->device_id) && 273 ((PCI_SUBSYS_VENDOR(subid) == ent->subvendor_id) || 274 (ent->subvendor_id == 0)) && 275 ((PCI_SUBSYS_ID(subid) == ent->subdevice_id) || 276 (ent->subdevice_id == 0))) { 277 return ent; 278 } 279 } 280 281 return NULL; 282 } 283 284 /************************************************************************ 285 * ixv_attach - Device initialization routine 286 * 287 * Called when the driver is being loaded. 288 * Identifies the type of hardware, allocates all resources 289 * and initializes the hardware. 290 * 291 * return 0 on success, positive on failure 292 ************************************************************************/ 293 static void 294 ixv_attach(device_t parent, device_t dev, void *aux) 295 { 296 struct ixgbe_softc *sc; 297 struct ixgbe_hw *hw; 298 int error = 0; 299 pcireg_t id, subid; 300 const ixgbe_vendor_info_t *ent; 301 const struct pci_attach_args *pa = aux; 302 const char *apivstr; 303 const char *str; 304 char wqname[MAXCOMLEN]; 305 char buf[256]; 306 307 INIT_DEBUGOUT("ixv_attach: begin"); 308 309 /* 310 * Make sure BUSMASTER is set, on a VM under 311 * KVM it may not be and will break things. 312 */ 313 ixgbe_pci_enable_busmaster(pa->pa_pc, pa->pa_tag); 314 315 /* Allocate, clear, and link in our adapter structure */ 316 sc = device_private(dev); 317 sc->hw.back = sc; 318 sc->dev = dev; 319 hw = &sc->hw; 320 321 sc->init_locked = ixv_init_locked; 322 sc->stop_locked = ixv_stop_locked; 323 324 sc->osdep.pc = pa->pa_pc; 325 sc->osdep.tag = pa->pa_tag; 326 if (pci_dma64_available(pa)) 327 sc->osdep.dmat = pa->pa_dmat64; 328 else 329 sc->osdep.dmat = pa->pa_dmat; 330 sc->osdep.attached = false; 331 332 ent = ixv_lookup(pa); 333 334 KASSERT(ent != NULL); 335 336 aprint_normal(": %s, Version - %s\n", 337 ixv_strings[ent->index], ixv_driver_version); 338 339 /* Core Lock Init */ 340 IXGBE_CORE_LOCK_INIT(sc, device_xname(dev)); 341 342 /* Do base PCI setup - map BAR0 */ 343 if (ixv_allocate_pci_resources(sc, pa)) { 344 aprint_error_dev(dev, "ixv_allocate_pci_resources() failed!\n"); 345 error = ENXIO; 346 goto err_out; 347 } 348 349 /* SYSCTL APIs */ 350 ixv_add_device_sysctls(sc); 351 352 /* Set up the timer callout and workqueue */ 353 callout_init(&sc->timer, CALLOUT_MPSAFE); 354 snprintf(wqname, sizeof(wqname), "%s-timer", device_xname(dev)); 355 error = workqueue_create(&sc->timer_wq, wqname, 356 ixv_handle_timer, sc, IXGBE_WORKQUEUE_PRI, IPL_NET, WQ_MPSAFE); 357 if (error) { 358 aprint_error_dev(dev, 359 "could not create timer workqueue (%d)\n", error); 360 goto err_out; 361 } 362 363 /* Save off the information about this board */ 364 id = pci_conf_read(pa->pa_pc, pa->pa_tag, PCI_ID_REG); 365 subid = pci_conf_read(pa->pa_pc, pa->pa_tag, PCI_SUBSYS_ID_REG); 366 hw->vendor_id = PCI_VENDOR(id); 367 hw->device_id = PCI_PRODUCT(id); 368 hw->revision_id = 369 PCI_REVISION(pci_conf_read(pa->pa_pc, pa->pa_tag, PCI_CLASS_REG)); 370 hw->subsystem_vendor_id = PCI_SUBSYS_VENDOR(subid); 371 hw->subsystem_device_id = PCI_SUBSYS_ID(subid); 372 373 /* A subset of set_mac_type */ 374 switch (hw->device_id) { 375 case IXGBE_DEV_ID_82599_VF: 376 hw->mac.type = ixgbe_mac_82599_vf; 377 str = "82599 VF"; 378 break; 379 case IXGBE_DEV_ID_X540_VF: 380 hw->mac.type = ixgbe_mac_X540_vf; 381 str = "X540 VF"; 382 break; 383 case IXGBE_DEV_ID_X550_VF: 384 hw->mac.type = ixgbe_mac_X550_vf; 385 str = "X550 VF"; 386 break; 387 case IXGBE_DEV_ID_X550EM_X_VF: 388 hw->mac.type = ixgbe_mac_X550EM_x_vf; 389 str = "X550EM X VF"; 390 break; 391 case IXGBE_DEV_ID_X550EM_A_VF: 392 hw->mac.type = ixgbe_mac_X550EM_a_vf; 393 str = "X550EM A VF"; 394 break; 395 default: 396 /* Shouldn't get here since probe succeeded */ 397 aprint_error_dev(dev, "Unknown device ID!\n"); 398 error = ENXIO; 399 goto err_out; 400 break; 401 } 402 aprint_normal_dev(dev, "device %s\n", str); 403 404 ixv_init_device_features(sc); 405 406 /* Initialize the shared code */ 407 error = ixgbe_init_ops_vf(hw); 408 if (error) { 409 aprint_error_dev(dev, "ixgbe_init_ops_vf() failed!\n"); 410 error = EIO; 411 goto err_out; 412 } 413 414 /* Setup the mailbox */ 415 ixgbe_init_mbx_params_vf(hw); 416 417 /* Set the right number of segments */ 418 KASSERT(IXGBE_82599_SCATTER_MAX >= IXGBE_SCATTER_DEFAULT); 419 sc->num_segs = IXGBE_SCATTER_DEFAULT; 420 421 /* Reset mbox api to 1.0 */ 422 error = hw->mac.ops.reset_hw(hw); 423 if (error == IXGBE_ERR_RESET_FAILED) 424 aprint_error_dev(dev, "...reset_hw() failure: Reset Failed!\n"); 425 else if (error) 426 aprint_error_dev(dev, "...reset_hw() failed with error %d\n", 427 error); 428 if (error) { 429 error = EIO; 430 goto err_out; 431 } 432 433 error = hw->mac.ops.init_hw(hw); 434 if (error) { 435 aprint_error_dev(dev, "...init_hw() failed!\n"); 436 error = EIO; 437 goto err_out; 438 } 439 440 /* Negotiate mailbox API version */ 441 error = ixv_negotiate_api(sc); 442 if (error) 443 aprint_normal_dev(dev, 444 "MBX API negotiation failed during attach!\n"); 445 switch (hw->api_version) { 446 case ixgbe_mbox_api_10: 447 apivstr = "1.0"; 448 break; 449 case ixgbe_mbox_api_20: 450 apivstr = "2.0"; 451 break; 452 case ixgbe_mbox_api_11: 453 apivstr = "1.1"; 454 break; 455 case ixgbe_mbox_api_12: 456 apivstr = "1.2"; 457 break; 458 case ixgbe_mbox_api_13: 459 apivstr = "1.3"; 460 break; 461 case ixgbe_mbox_api_14: 462 apivstr = "1.4"; 463 break; 464 case ixgbe_mbox_api_15: 465 apivstr = "1.5"; 466 break; 467 default: 468 apivstr = "unknown"; 469 break; 470 } 471 aprint_normal_dev(dev, "Mailbox API %s\n", apivstr); 472 473 /* If no mac address was assigned, make a random one */ 474 if (!ixv_check_ether_addr(hw->mac.addr)) { 475 u8 addr[ETHER_ADDR_LEN]; 476 uint64_t rndval = cprng_strong64(); 477 478 memcpy(addr, &rndval, sizeof(addr)); 479 addr[0] &= 0xFE; 480 addr[0] |= 0x02; 481 bcopy(addr, hw->mac.addr, sizeof(addr)); 482 } 483 484 /* Register for VLAN events */ 485 ether_set_vlan_cb(&sc->osdep.ec, ixv_vlan_cb); 486 487 /* Do descriptor calc and sanity checks */ 488 if (((ixv_txd * sizeof(union ixgbe_adv_tx_desc)) % DBA_ALIGN) != 0 || 489 ixv_txd < MIN_TXD || ixv_txd > MAX_TXD) { 490 aprint_error_dev(dev, "Invalid TX ring size (%d). " 491 "It must be between %d and %d, " 492 "inclusive, and must be a multiple of %zu. " 493 "Using default value of %d instead.\n", 494 ixv_txd, MIN_TXD, MAX_TXD, 495 DBA_ALIGN / sizeof(union ixgbe_adv_tx_desc), 496 DEFAULT_TXD); 497 sc->num_tx_desc = DEFAULT_TXD; 498 } else 499 sc->num_tx_desc = ixv_txd; 500 501 if (((ixv_rxd * sizeof(union ixgbe_adv_rx_desc)) % DBA_ALIGN) != 0 || 502 ixv_rxd < MIN_RXD || ixv_rxd > MAX_RXD) { 503 aprint_error_dev(dev, "Invalid RX ring size (%d). " 504 "It must be between %d and %d, " 505 "inclusive, and must be a multiple of %zu. " 506 "Using default value of %d instead.\n", 507 ixv_rxd, MIN_RXD, MAX_RXD, 508 DBA_ALIGN / sizeof(union ixgbe_adv_rx_desc), 509 DEFAULT_RXD); 510 sc->num_rx_desc = DEFAULT_RXD; 511 } else 512 sc->num_rx_desc = ixv_rxd; 513 514 /* Sysctls for limiting the amount of work done in the taskqueues */ 515 sc->rx_process_limit 516 = (ixv_rx_process_limit <= sc->num_rx_desc) 517 ? ixv_rx_process_limit : sc->num_rx_desc; 518 sc->tx_process_limit 519 = (ixv_tx_process_limit <= sc->num_tx_desc) 520 ? ixv_tx_process_limit : sc->num_tx_desc; 521 522 /* Set default high limit of copying mbuf in rxeof */ 523 sc->rx_copy_len = IXGBE_RX_COPY_LEN_MAX; 524 525 /* Setup MSI-X */ 526 error = ixv_configure_interrupts(sc); 527 if (error) 528 goto err_out; 529 530 /* Allocate our TX/RX Queues */ 531 if (ixgbe_allocate_queues(sc)) { 532 aprint_error_dev(dev, "ixgbe_allocate_queues() failed!\n"); 533 error = ENOMEM; 534 goto err_out; 535 } 536 537 /* hw.ix defaults init */ 538 sc->enable_aim = ixv_enable_aim; 539 sc->max_interrupt_rate = ixv_max_interrupt_rate; 540 541 sc->txrx_use_workqueue = ixv_txrx_workqueue; 542 543 error = ixv_allocate_msix(sc, pa); 544 if (error) { 545 aprint_error_dev(dev, "ixv_allocate_msix() failed!\n"); 546 goto err_late; 547 } 548 549 /* Setup OS specific network interface */ 550 error = ixv_setup_interface(dev, sc); 551 if (error != 0) { 552 aprint_error_dev(dev, "ixv_setup_interface() failed!\n"); 553 goto err_late; 554 } 555 556 /* Allocate multicast array memory */ 557 sc->mta = malloc(sizeof(*sc->mta) * 558 IXGBE_MAX_VF_MC, M_DEVBUF, M_WAITOK); 559 560 /* Check if VF was disabled by PF */ 561 error = hw->mac.ops.get_link_state(hw, &sc->link_enabled); 562 if (error) { 563 /* PF is not capable of controlling VF state. Enable the link. */ 564 sc->link_enabled = TRUE; 565 } 566 567 /* Do the stats setup */ 568 ixv_init_stats(sc); 569 ixv_add_stats_sysctls(sc); 570 571 if (sc->feat_en & IXGBE_FEATURE_NETMAP) 572 ixgbe_netmap_attach(sc); 573 574 snprintb(buf, sizeof(buf), IXGBE_FEATURE_FLAGS, sc->feat_cap); 575 aprint_verbose_dev(dev, "feature cap %s\n", buf); 576 snprintb(buf, sizeof(buf), IXGBE_FEATURE_FLAGS, sc->feat_en); 577 aprint_verbose_dev(dev, "feature ena %s\n", buf); 578 579 INIT_DEBUGOUT("ixv_attach: end"); 580 sc->osdep.attached = true; 581 582 return; 583 584 err_late: 585 ixgbe_free_queues(sc); 586 err_out: 587 ixv_free_pci_resources(sc); 588 IXGBE_CORE_LOCK_DESTROY(sc); 589 590 return; 591 } /* ixv_attach */ 592 593 /************************************************************************ 594 * ixv_detach - Device removal routine 595 * 596 * Called when the driver is being removed. 597 * Stops the adapter and deallocates all the resources 598 * that were allocated for driver operation. 599 * 600 * return 0 on success, positive on failure 601 ************************************************************************/ 602 static int 603 ixv_detach(device_t dev, int flags) 604 { 605 struct ixgbe_softc *sc = device_private(dev); 606 struct ixgbe_hw *hw = &sc->hw; 607 struct tx_ring *txr = sc->tx_rings; 608 struct rx_ring *rxr = sc->rx_rings; 609 struct ixgbevf_hw_stats *stats = &sc->stats.vf; 610 611 INIT_DEBUGOUT("ixv_detach: begin"); 612 if (sc->osdep.attached == false) 613 return 0; 614 615 /* Stop the interface. Callouts are stopped in it. */ 616 ixv_ifstop(sc->ifp, 1); 617 618 if (VLAN_ATTACHED(&sc->osdep.ec) && 619 (flags & (DETACH_SHUTDOWN | DETACH_FORCE)) == 0) { 620 aprint_error_dev(dev, "VLANs in use, detach first\n"); 621 return EBUSY; 622 } 623 624 ether_ifdetach(sc->ifp); 625 callout_halt(&sc->timer, NULL); 626 ixv_free_deferred_handlers(sc); 627 628 if (sc->feat_en & IXGBE_FEATURE_NETMAP) 629 netmap_detach(sc->ifp); 630 631 ixv_free_pci_resources(sc); 632 #if 0 /* XXX the NetBSD port is probably missing something here */ 633 bus_generic_detach(dev); 634 #endif 635 if_detach(sc->ifp); 636 ifmedia_fini(&sc->media); 637 if_percpuq_destroy(sc->ipq); 638 639 sysctl_teardown(&sc->sysctllog); 640 evcnt_detach(&sc->efbig_tx_dma_setup); 641 evcnt_detach(&sc->mbuf_defrag_failed); 642 evcnt_detach(&sc->efbig2_tx_dma_setup); 643 evcnt_detach(&sc->einval_tx_dma_setup); 644 evcnt_detach(&sc->other_tx_dma_setup); 645 evcnt_detach(&sc->eagain_tx_dma_setup); 646 evcnt_detach(&sc->enomem_tx_dma_setup); 647 evcnt_detach(&sc->watchdog_events); 648 evcnt_detach(&sc->tso_err); 649 evcnt_detach(&sc->admin_irqev); 650 evcnt_detach(&sc->link_workev); 651 652 txr = sc->tx_rings; 653 for (int i = 0; i < sc->num_queues; i++, rxr++, txr++) { 654 evcnt_detach(&sc->queues[i].irqs); 655 evcnt_detach(&sc->queues[i].handleq); 656 evcnt_detach(&sc->queues[i].req); 657 evcnt_detach(&txr->total_packets); 658 #ifndef IXGBE_LEGACY_TX 659 evcnt_detach(&txr->pcq_drops); 660 #endif 661 evcnt_detach(&txr->no_desc_avail); 662 evcnt_detach(&txr->tso_tx); 663 664 evcnt_detach(&rxr->rx_packets); 665 evcnt_detach(&rxr->rx_bytes); 666 evcnt_detach(&rxr->rx_copies); 667 evcnt_detach(&rxr->no_mbuf); 668 evcnt_detach(&rxr->rx_discarded); 669 } 670 evcnt_detach(&stats->ipcs); 671 evcnt_detach(&stats->l4cs); 672 evcnt_detach(&stats->ipcs_bad); 673 evcnt_detach(&stats->l4cs_bad); 674 675 /* Packet Reception Stats */ 676 evcnt_detach(&stats->vfgorc); 677 evcnt_detach(&stats->vfgprc); 678 evcnt_detach(&stats->vfmprc); 679 680 /* Packet Transmission Stats */ 681 evcnt_detach(&stats->vfgotc); 682 evcnt_detach(&stats->vfgptc); 683 684 /* Mailbox Stats */ 685 evcnt_detach(&hw->mbx.stats.msgs_tx); 686 evcnt_detach(&hw->mbx.stats.msgs_rx); 687 evcnt_detach(&hw->mbx.stats.acks); 688 evcnt_detach(&hw->mbx.stats.reqs); 689 evcnt_detach(&hw->mbx.stats.rsts); 690 691 ixgbe_free_queues(sc); 692 693 IXGBE_CORE_LOCK_DESTROY(sc); 694 695 return (0); 696 } /* ixv_detach */ 697 698 /************************************************************************ 699 * ixv_init_locked - Init entry point 700 * 701 * Used in two ways: It is used by the stack as an init entry 702 * point in network interface structure. It is also used 703 * by the driver as a hw/sw initialization routine to get 704 * to a consistent state. 705 * 706 * return 0 on success, positive on failure 707 ************************************************************************/ 708 static void 709 ixv_init_locked(struct ixgbe_softc *sc) 710 { 711 struct ifnet *ifp = sc->ifp; 712 device_t dev = sc->dev; 713 struct ixgbe_hw *hw = &sc->hw; 714 struct ix_queue *que; 715 int error = 0; 716 uint32_t mask; 717 int i; 718 719 INIT_DEBUGOUT("ixv_init_locked: begin"); 720 KASSERT(mutex_owned(&sc->core_mtx)); 721 hw->adapter_stopped = FALSE; 722 hw->mac.ops.stop_adapter(hw); 723 callout_stop(&sc->timer); 724 for (i = 0, que = sc->queues; i < sc->num_queues; i++, que++) 725 que->disabled_count = 0; 726 727 sc->max_frame_size = 728 ifp->if_mtu + ETHER_HDR_LEN + ETHER_CRC_LEN; 729 730 /* reprogram the RAR[0] in case user changed it. */ 731 hw->mac.ops.set_rar(hw, 0, hw->mac.addr, 0, IXGBE_RAH_AV); 732 733 /* Get the latest mac address, User can use a LAA */ 734 memcpy(hw->mac.addr, CLLADDR(ifp->if_sadl), 735 IXGBE_ETH_LENGTH_OF_ADDRESS); 736 hw->mac.ops.set_rar(hw, 0, hw->mac.addr, 0, 1); 737 738 /* Prepare transmit descriptors and buffers */ 739 if (ixgbe_setup_transmit_structures(sc)) { 740 aprint_error_dev(dev, "Could not setup transmit structures\n"); 741 ixv_stop_locked(sc); 742 return; 743 } 744 745 /* Reset VF and renegotiate mailbox API version */ 746 hw->mac.ops.reset_hw(hw); 747 hw->mac.ops.start_hw(hw); 748 error = ixv_negotiate_api(sc); 749 if (error) 750 device_printf(dev, 751 "Mailbox API negotiation failed in init_locked!\n"); 752 753 ixv_initialize_transmit_units(sc); 754 755 /* Setup Multicast table */ 756 ixv_set_rxfilter(sc); 757 758 /* Use fixed buffer size, even for jumbo frames */ 759 sc->rx_mbuf_sz = MCLBYTES; 760 761 /* Prepare receive descriptors and buffers */ 762 error = ixgbe_setup_receive_structures(sc); 763 if (error) { 764 device_printf(dev, 765 "Could not setup receive structures (err = %d)\n", error); 766 ixv_stop_locked(sc); 767 return; 768 } 769 770 /* Configure RX settings */ 771 ixv_initialize_receive_units(sc); 772 773 /* Initialize variable holding task enqueue requests interrupts */ 774 sc->task_requests = 0; 775 776 /* Set up VLAN offload and filter */ 777 ixv_setup_vlan_support(sc); 778 779 /* Set up MSI-X routing */ 780 ixv_configure_ivars(sc); 781 782 /* Set up auto-mask */ 783 mask = (1 << sc->vector); 784 for (i = 0, que = sc->queues; i < sc->num_queues; i++, que++) 785 mask |= (1 << que->msix); 786 IXGBE_WRITE_REG(hw, IXGBE_VTEIAM, mask); 787 788 /* Set moderation on the Link interrupt */ 789 ixv_eitr_write(sc, sc->vector, IXGBE_LINK_ITR); 790 791 /* Stats init */ 792 ixv_init_stats(sc); 793 794 /* Config/Enable Link */ 795 error = hw->mac.ops.get_link_state(hw, &sc->link_enabled); 796 if (error) { 797 /* PF is not capable of controlling VF state. Enable the link. */ 798 sc->link_enabled = TRUE; 799 } else if (sc->link_enabled == FALSE) 800 device_printf(dev, "VF is disabled by PF\n"); 801 802 hw->mac.get_link_status = TRUE; 803 hw->mac.ops.check_link(hw, &sc->link_speed, &sc->link_up, 804 FALSE); 805 806 /* Start watchdog */ 807 callout_reset(&sc->timer, hz, ixv_local_timer, sc); 808 atomic_store_relaxed(&sc->timer_pending, 0); 809 810 /* OK to schedule workqueues. */ 811 sc->schedule_wqs_ok = true; 812 813 /* Update saved flags. See ixgbe_ifflags_cb() */ 814 sc->if_flags = ifp->if_flags; 815 sc->ec_capenable = sc->osdep.ec.ec_capenable; 816 817 /* Inform the stack we're ready */ 818 ifp->if_flags |= IFF_RUNNING; 819 820 /* And now turn on interrupts */ 821 ixv_enable_intr(sc); 822 823 return; 824 } /* ixv_init_locked */ 825 826 /************************************************************************ 827 * ixv_enable_queue 828 ************************************************************************/ 829 static inline void 830 ixv_enable_queue(struct ixgbe_softc *sc, u32 vector) 831 { 832 struct ixgbe_hw *hw = &sc->hw; 833 struct ix_queue *que = &sc->queues[vector]; 834 u32 queue = 1UL << vector; 835 u32 mask; 836 837 mutex_enter(&que->dc_mtx); 838 if (que->disabled_count > 0 && --que->disabled_count > 0) 839 goto out; 840 841 mask = (IXGBE_EIMS_RTX_QUEUE & queue); 842 IXGBE_WRITE_REG(hw, IXGBE_VTEIMS, mask); 843 out: 844 mutex_exit(&que->dc_mtx); 845 } /* ixv_enable_queue */ 846 847 /************************************************************************ 848 * ixv_disable_queue 849 ************************************************************************/ 850 static inline void 851 ixv_disable_queue(struct ixgbe_softc *sc, u32 vector) 852 { 853 struct ixgbe_hw *hw = &sc->hw; 854 struct ix_queue *que = &sc->queues[vector]; 855 u32 queue = 1UL << vector; 856 u32 mask; 857 858 mutex_enter(&que->dc_mtx); 859 if (que->disabled_count++ > 0) 860 goto out; 861 862 mask = (IXGBE_EIMS_RTX_QUEUE & queue); 863 IXGBE_WRITE_REG(hw, IXGBE_VTEIMC, mask); 864 out: 865 mutex_exit(&que->dc_mtx); 866 } /* ixv_disable_queue */ 867 868 #if 0 869 static inline void 870 ixv_rearm_queues(struct ixgbe_softc *sc, u64 queues) 871 { 872 u32 mask = (IXGBE_EIMS_RTX_QUEUE & queues); 873 IXGBE_WRITE_REG(&sc->hw, IXGBE_VTEICS, mask); 874 } /* ixv_rearm_queues */ 875 #endif 876 877 878 /************************************************************************ 879 * ixv_msix_que - MSI-X Queue Interrupt Service routine 880 ************************************************************************/ 881 static int 882 ixv_msix_que(void *arg) 883 { 884 struct ix_queue *que = arg; 885 struct ixgbe_softc *sc = que->sc; 886 struct tx_ring *txr = que->txr; 887 struct rx_ring *rxr = que->rxr; 888 bool more; 889 u32 newitr = 0; 890 891 ixv_disable_queue(sc, que->msix); 892 IXGBE_EVC_ADD(&que->irqs, 1); 893 894 #ifdef __NetBSD__ 895 /* Don't run ixgbe_rxeof in interrupt context */ 896 more = true; 897 #else 898 more = ixgbe_rxeof(que); 899 #endif 900 901 IXGBE_TX_LOCK(txr); 902 ixgbe_txeof(txr); 903 IXGBE_TX_UNLOCK(txr); 904 905 /* Do AIM now? */ 906 907 if (sc->enable_aim == false) 908 goto no_calc; 909 /* 910 * Do Adaptive Interrupt Moderation: 911 * - Write out last calculated setting 912 * - Calculate based on average size over 913 * the last interval. 914 */ 915 if (que->eitr_setting) 916 ixv_eitr_write(sc, que->msix, que->eitr_setting); 917 918 que->eitr_setting = 0; 919 920 /* Idle, do nothing */ 921 if ((txr->bytes == 0) && (rxr->bytes == 0)) 922 goto no_calc; 923 924 if ((txr->bytes) && (txr->packets)) 925 newitr = txr->bytes/txr->packets; 926 if ((rxr->bytes) && (rxr->packets)) 927 newitr = uimax(newitr, (rxr->bytes / rxr->packets)); 928 newitr += 24; /* account for hardware frame, crc */ 929 930 /* set an upper boundary */ 931 newitr = uimin(newitr, 3000); 932 933 /* Be nice to the mid range */ 934 if ((newitr > 300) && (newitr < 1200)) 935 newitr = (newitr / 3); 936 else 937 newitr = (newitr / 2); 938 939 /* 940 * When RSC is used, ITR interval must be larger than RSC_DELAY. 941 * Currently, we use 2us for RSC_DELAY. The minimum value is always 942 * greater than 2us on 100M (and 10M?(not documented)), but it's not 943 * on 1G and higher. 944 */ 945 if ((sc->link_speed != IXGBE_LINK_SPEED_100_FULL) 946 && (sc->link_speed != IXGBE_LINK_SPEED_10_FULL)) { 947 if (newitr < IXGBE_MIN_RSC_EITR_10G1G) 948 newitr = IXGBE_MIN_RSC_EITR_10G1G; 949 } 950 951 /* save for next interrupt */ 952 que->eitr_setting = newitr; 953 954 /* Reset state */ 955 txr->bytes = 0; 956 txr->packets = 0; 957 rxr->bytes = 0; 958 rxr->packets = 0; 959 960 no_calc: 961 if (more) 962 softint_schedule(que->que_si); 963 else /* Re-enable this interrupt */ 964 ixv_enable_queue(sc, que->msix); 965 966 return 1; 967 } /* ixv_msix_que */ 968 969 /************************************************************************ 970 * ixv_msix_mbx 971 ************************************************************************/ 972 static int 973 ixv_msix_mbx(void *arg) 974 { 975 struct ixgbe_softc *sc = arg; 976 struct ixgbe_hw *hw = &sc->hw; 977 978 IXGBE_EVC_ADD(&sc->admin_irqev, 1); 979 /* NetBSD: We use auto-clear, so it's not required to write VTEICR */ 980 981 /* Link status change */ 982 hw->mac.get_link_status = TRUE; 983 atomic_or_32(&sc->task_requests, IXGBE_REQUEST_TASK_MBX); 984 ixv_schedule_admin_tasklet(sc); 985 986 return 1; 987 } /* ixv_msix_mbx */ 988 989 static void 990 ixv_eitr_write(struct ixgbe_softc *sc, uint32_t index, uint32_t itr) 991 { 992 993 /* 994 * Newer devices than 82598 have VF function, so this function is 995 * simple. 996 */ 997 itr |= IXGBE_EITR_CNT_WDIS; 998 999 IXGBE_WRITE_REG(&sc->hw, IXGBE_VTEITR(index), itr); 1000 } 1001 1002 1003 /************************************************************************ 1004 * ixv_media_status - Media Ioctl callback 1005 * 1006 * Called whenever the user queries the status of 1007 * the interface using ifconfig. 1008 ************************************************************************/ 1009 static void 1010 ixv_media_status(struct ifnet *ifp, struct ifmediareq *ifmr) 1011 { 1012 struct ixgbe_softc *sc = ifp->if_softc; 1013 1014 INIT_DEBUGOUT("ixv_media_status: begin"); 1015 ixv_update_link_status(sc); 1016 1017 ifmr->ifm_status = IFM_AVALID; 1018 ifmr->ifm_active = IFM_ETHER; 1019 1020 if (sc->link_active != LINK_STATE_UP) { 1021 ifmr->ifm_active |= IFM_NONE; 1022 return; 1023 } 1024 1025 ifmr->ifm_status |= IFM_ACTIVE; 1026 1027 switch (sc->link_speed) { 1028 case IXGBE_LINK_SPEED_10GB_FULL: 1029 ifmr->ifm_active |= IFM_10G_T | IFM_FDX; 1030 break; 1031 case IXGBE_LINK_SPEED_5GB_FULL: 1032 ifmr->ifm_active |= IFM_5000_T | IFM_FDX; 1033 break; 1034 case IXGBE_LINK_SPEED_2_5GB_FULL: 1035 ifmr->ifm_active |= IFM_2500_T | IFM_FDX; 1036 break; 1037 case IXGBE_LINK_SPEED_1GB_FULL: 1038 ifmr->ifm_active |= IFM_1000_T | IFM_FDX; 1039 break; 1040 case IXGBE_LINK_SPEED_100_FULL: 1041 ifmr->ifm_active |= IFM_100_TX | IFM_FDX; 1042 break; 1043 case IXGBE_LINK_SPEED_10_FULL: 1044 ifmr->ifm_active |= IFM_10_T | IFM_FDX; 1045 break; 1046 } 1047 1048 ifp->if_baudrate = ifmedia_baudrate(ifmr->ifm_active); 1049 } /* ixv_media_status */ 1050 1051 /************************************************************************ 1052 * ixv_media_change - Media Ioctl callback 1053 * 1054 * Called when the user changes speed/duplex using 1055 * media/mediopt option with ifconfig. 1056 ************************************************************************/ 1057 static int 1058 ixv_media_change(struct ifnet *ifp) 1059 { 1060 struct ixgbe_softc *sc = ifp->if_softc; 1061 struct ifmedia *ifm = &sc->media; 1062 1063 INIT_DEBUGOUT("ixv_media_change: begin"); 1064 1065 if (IFM_TYPE(ifm->ifm_media) != IFM_ETHER) 1066 return (EINVAL); 1067 1068 switch (IFM_SUBTYPE(ifm->ifm_media)) { 1069 case IFM_AUTO: 1070 break; 1071 default: 1072 device_printf(sc->dev, "Only auto media type\n"); 1073 return (EINVAL); 1074 } 1075 1076 return (0); 1077 } /* ixv_media_change */ 1078 1079 static void 1080 ixv_schedule_admin_tasklet(struct ixgbe_softc *sc) 1081 { 1082 if (sc->schedule_wqs_ok) { 1083 if (atomic_cas_uint(&sc->admin_pending, 0, 1) == 0) 1084 workqueue_enqueue(sc->admin_wq, 1085 &sc->admin_wc, NULL); 1086 } 1087 } 1088 1089 /************************************************************************ 1090 * ixv_negotiate_api 1091 * 1092 * Negotiate the Mailbox API with the PF; 1093 * start with the most featured API first. 1094 ************************************************************************/ 1095 static int 1096 ixv_negotiate_api(struct ixgbe_softc *sc) 1097 { 1098 struct ixgbe_hw *hw = &sc->hw; 1099 int mbx_api[] = { ixgbe_mbox_api_15, 1100 ixgbe_mbox_api_13, 1101 ixgbe_mbox_api_12, 1102 ixgbe_mbox_api_11, 1103 ixgbe_mbox_api_10, 1104 ixgbe_mbox_api_unknown }; 1105 int i = 0; 1106 1107 while (mbx_api[i] != ixgbe_mbox_api_unknown) { 1108 if (ixgbevf_negotiate_api_version(hw, mbx_api[i]) == 0) { 1109 if (hw->api_version >= ixgbe_mbox_api_15) 1110 ixgbe_upgrade_mbx_params_vf(hw); 1111 return (0); 1112 } 1113 i++; 1114 } 1115 1116 return (EINVAL); 1117 } /* ixv_negotiate_api */ 1118 1119 1120 /************************************************************************ 1121 * ixv_set_rxfilter - Multicast Update 1122 * 1123 * Called whenever multicast address list is updated. 1124 ************************************************************************/ 1125 static int 1126 ixv_set_rxfilter(struct ixgbe_softc *sc) 1127 { 1128 struct ixgbe_mc_addr *mta; 1129 struct ifnet *ifp = sc->ifp; 1130 struct ixgbe_hw *hw = &sc->hw; 1131 u8 *update_ptr; 1132 int mcnt = 0; 1133 struct ethercom *ec = &sc->osdep.ec; 1134 struct ether_multi *enm; 1135 struct ether_multistep step; 1136 bool overflow = false; 1137 int error, rc = 0; 1138 1139 KASSERT(mutex_owned(&sc->core_mtx)); 1140 IOCTL_DEBUGOUT("ixv_set_rxfilter: begin"); 1141 1142 mta = sc->mta; 1143 bzero(mta, sizeof(*mta) * IXGBE_MAX_VF_MC); 1144 1145 /* 1: For PROMISC */ 1146 if (ifp->if_flags & IFF_PROMISC) { 1147 error = hw->mac.ops.update_xcast_mode(hw, 1148 IXGBEVF_XCAST_MODE_PROMISC); 1149 if (error == IXGBE_ERR_NOT_TRUSTED) { 1150 device_printf(sc->dev, 1151 "this interface is not trusted\n"); 1152 error = EPERM; 1153 } else if (error == IXGBE_ERR_FEATURE_NOT_SUPPORTED) { 1154 device_printf(sc->dev, 1155 "the PF doesn't support promisc mode\n"); 1156 error = EOPNOTSUPP; 1157 } else if (error == IXGBE_ERR_NOT_IN_PROMISC) { 1158 device_printf(sc->dev, 1159 "the PF may not in promisc mode\n"); 1160 error = EINVAL; 1161 } else if (error) { 1162 device_printf(sc->dev, 1163 "failed to set promisc mode. error = %d\n", 1164 error); 1165 error = EIO; 1166 } else 1167 return 0; 1168 rc = error; 1169 } 1170 1171 /* 2: For ALLMULTI or normal */ 1172 ETHER_LOCK(ec); 1173 ETHER_FIRST_MULTI(step, ec, enm); 1174 while (enm != NULL) { 1175 if ((mcnt >= IXGBE_MAX_VF_MC) || 1176 (memcmp(enm->enm_addrlo, enm->enm_addrhi, 1177 ETHER_ADDR_LEN) != 0)) { 1178 overflow = true; 1179 break; 1180 } 1181 bcopy(enm->enm_addrlo, 1182 mta[mcnt].addr, IXGBE_ETH_LENGTH_OF_ADDRESS); 1183 mcnt++; 1184 ETHER_NEXT_MULTI(step, enm); 1185 } 1186 ETHER_UNLOCK(ec); 1187 1188 /* 3: For ALLMULTI */ 1189 if (overflow) { 1190 error = hw->mac.ops.update_xcast_mode(hw, 1191 IXGBEVF_XCAST_MODE_ALLMULTI); 1192 if (error == IXGBE_ERR_NOT_TRUSTED) { 1193 device_printf(sc->dev, 1194 "this interface is not trusted\n"); 1195 error = EPERM; 1196 } else if (error == IXGBE_ERR_FEATURE_NOT_SUPPORTED) { 1197 device_printf(sc->dev, 1198 "the PF doesn't support allmulti mode\n"); 1199 error = EOPNOTSUPP; 1200 } else if (error) { 1201 device_printf(sc->dev, 1202 "number of Ethernet multicast addresses " 1203 "exceeds the limit (%d). error = %d\n", 1204 IXGBE_MAX_VF_MC, error); 1205 error = ENOSPC; 1206 } else { 1207 ETHER_LOCK(ec); 1208 ec->ec_flags |= ETHER_F_ALLMULTI; 1209 ETHER_UNLOCK(ec); 1210 return rc; /* Promisc might have failed */ 1211 } 1212 1213 if (rc == 0) 1214 rc = error; 1215 1216 /* Continue to update the multicast table as many as we can */ 1217 } 1218 1219 /* 4: For normal operation */ 1220 error = hw->mac.ops.update_xcast_mode(hw, IXGBEVF_XCAST_MODE_MULTI); 1221 if ((error == IXGBE_ERR_FEATURE_NOT_SUPPORTED) || (error == 0)) { 1222 /* Normal operation */ 1223 ETHER_LOCK(ec); 1224 ec->ec_flags &= ~ETHER_F_ALLMULTI; 1225 ETHER_UNLOCK(ec); 1226 error = 0; 1227 } else if (error) { 1228 device_printf(sc->dev, 1229 "failed to set Ethernet multicast address " 1230 "operation to normal. error = %d\n", error); 1231 } 1232 1233 update_ptr = (u8 *)mta; 1234 error = sc->hw.mac.ops.update_mc_addr_list(&sc->hw, 1235 update_ptr, mcnt, ixv_mc_array_itr, TRUE); 1236 if (rc == 0) 1237 rc = error; 1238 1239 return rc; 1240 } /* ixv_set_rxfilter */ 1241 1242 /************************************************************************ 1243 * ixv_mc_array_itr 1244 * 1245 * An iterator function needed by the multicast shared code. 1246 * It feeds the shared code routine the addresses in the 1247 * array of ixv_set_rxfilter() one by one. 1248 ************************************************************************/ 1249 static u8 * 1250 ixv_mc_array_itr(struct ixgbe_hw *hw, u8 **update_ptr, u32 *vmdq) 1251 { 1252 struct ixgbe_mc_addr *mta; 1253 1254 mta = (struct ixgbe_mc_addr *)*update_ptr; 1255 1256 *vmdq = 0; 1257 *update_ptr = (u8*)(mta + 1); 1258 1259 return (mta->addr); 1260 } /* ixv_mc_array_itr */ 1261 1262 /************************************************************************ 1263 * ixv_local_timer - Timer routine 1264 * 1265 * Checks for link status, updates statistics, 1266 * and runs the watchdog check. 1267 ************************************************************************/ 1268 static void 1269 ixv_local_timer(void *arg) 1270 { 1271 struct ixgbe_softc *sc = arg; 1272 1273 if (sc->schedule_wqs_ok) { 1274 if (atomic_cas_uint(&sc->timer_pending, 0, 1) == 0) 1275 workqueue_enqueue(sc->timer_wq, 1276 &sc->timer_wc, NULL); 1277 } 1278 } 1279 1280 static void 1281 ixv_handle_timer(struct work *wk, void *context) 1282 { 1283 struct ixgbe_softc *sc = context; 1284 device_t dev = sc->dev; 1285 struct ix_queue *que = sc->queues; 1286 u64 queues = 0; 1287 u64 v0, v1, v2, v3, v4, v5, v6, v7; 1288 int hung = 0; 1289 int i; 1290 1291 IXGBE_CORE_LOCK(sc); 1292 1293 if (ixv_check_link(sc)) { 1294 ixv_init_locked(sc); 1295 IXGBE_CORE_UNLOCK(sc); 1296 return; 1297 } 1298 1299 /* Stats Update */ 1300 ixv_update_stats(sc); 1301 1302 /* Update some event counters */ 1303 v0 = v1 = v2 = v3 = v4 = v5 = v6 = v7 = 0; 1304 que = sc->queues; 1305 for (i = 0; i < sc->num_queues; i++, que++) { 1306 struct tx_ring *txr = que->txr; 1307 1308 v0 += txr->q_efbig_tx_dma_setup; 1309 v1 += txr->q_mbuf_defrag_failed; 1310 v2 += txr->q_efbig2_tx_dma_setup; 1311 v3 += txr->q_einval_tx_dma_setup; 1312 v4 += txr->q_other_tx_dma_setup; 1313 v5 += txr->q_eagain_tx_dma_setup; 1314 v6 += txr->q_enomem_tx_dma_setup; 1315 v7 += txr->q_tso_err; 1316 } 1317 IXGBE_EVC_STORE(&sc->efbig_tx_dma_setup, v0); 1318 IXGBE_EVC_STORE(&sc->mbuf_defrag_failed, v1); 1319 IXGBE_EVC_STORE(&sc->efbig2_tx_dma_setup, v2); 1320 IXGBE_EVC_STORE(&sc->einval_tx_dma_setup, v3); 1321 IXGBE_EVC_STORE(&sc->other_tx_dma_setup, v4); 1322 IXGBE_EVC_STORE(&sc->eagain_tx_dma_setup, v5); 1323 IXGBE_EVC_STORE(&sc->enomem_tx_dma_setup, v6); 1324 IXGBE_EVC_STORE(&sc->tso_err, v7); 1325 1326 /* 1327 * Check the TX queues status 1328 * - mark hung queues so we don't schedule on them 1329 * - watchdog only if all queues show hung 1330 */ 1331 que = sc->queues; 1332 for (i = 0; i < sc->num_queues; i++, que++) { 1333 /* Keep track of queues with work for soft irq */ 1334 if (que->txr->busy) 1335 queues |= ((u64)1 << que->me); 1336 /* 1337 * Each time txeof runs without cleaning, but there 1338 * are uncleaned descriptors it increments busy. If 1339 * we get to the MAX we declare it hung. 1340 */ 1341 if (que->busy == IXGBE_QUEUE_HUNG) { 1342 ++hung; 1343 /* Mark the queue as inactive */ 1344 sc->active_queues &= ~((u64)1 << que->me); 1345 continue; 1346 } else { 1347 /* Check if we've come back from hung */ 1348 if ((sc->active_queues & ((u64)1 << que->me)) == 0) 1349 sc->active_queues |= ((u64)1 << que->me); 1350 } 1351 if (que->busy >= IXGBE_MAX_TX_BUSY) { 1352 device_printf(dev, 1353 "Warning queue %d appears to be hung!\n", i); 1354 que->txr->busy = IXGBE_QUEUE_HUNG; 1355 ++hung; 1356 } 1357 } 1358 1359 /* Only truly watchdog if all queues show hung */ 1360 if (hung == sc->num_queues) 1361 goto watchdog; 1362 #if 0 1363 else if (queues != 0) { /* Force an IRQ on queues with work */ 1364 ixv_rearm_queues(sc, queues); 1365 } 1366 #endif 1367 1368 atomic_store_relaxed(&sc->timer_pending, 0); 1369 IXGBE_CORE_UNLOCK(sc); 1370 callout_reset(&sc->timer, hz, ixv_local_timer, sc); 1371 1372 return; 1373 1374 watchdog: 1375 device_printf(sc->dev, "Watchdog timeout -- resetting\n"); 1376 sc->ifp->if_flags &= ~IFF_RUNNING; 1377 IXGBE_EVC_ADD(&sc->watchdog_events, 1); 1378 ixv_init_locked(sc); 1379 IXGBE_CORE_UNLOCK(sc); 1380 } /* ixv_handle_timer */ 1381 1382 /************************************************************************ 1383 * ixv_update_link_status - Update OS on link state 1384 * 1385 * Note: Only updates the OS on the cached link state. 1386 * The real check of the hardware only happens with 1387 * a link interrupt. 1388 ************************************************************************/ 1389 static void 1390 ixv_update_link_status(struct ixgbe_softc *sc) 1391 { 1392 struct ifnet *ifp = sc->ifp; 1393 device_t dev = sc->dev; 1394 1395 KASSERT(mutex_owned(&sc->core_mtx)); 1396 1397 if (sc->link_up && sc->link_enabled) { 1398 if (sc->link_active != LINK_STATE_UP) { 1399 if (bootverbose) { 1400 const char *bpsmsg; 1401 1402 switch (sc->link_speed) { 1403 case IXGBE_LINK_SPEED_10GB_FULL: 1404 bpsmsg = "10 Gbps"; 1405 break; 1406 case IXGBE_LINK_SPEED_5GB_FULL: 1407 bpsmsg = "5 Gbps"; 1408 break; 1409 case IXGBE_LINK_SPEED_2_5GB_FULL: 1410 bpsmsg = "2.5 Gbps"; 1411 break; 1412 case IXGBE_LINK_SPEED_1GB_FULL: 1413 bpsmsg = "1 Gbps"; 1414 break; 1415 case IXGBE_LINK_SPEED_100_FULL: 1416 bpsmsg = "100 Mbps"; 1417 break; 1418 case IXGBE_LINK_SPEED_10_FULL: 1419 bpsmsg = "10 Mbps"; 1420 break; 1421 default: 1422 bpsmsg = "unknown speed"; 1423 break; 1424 } 1425 device_printf(dev, "Link is up %s %s \n", 1426 bpsmsg, "Full Duplex"); 1427 } 1428 sc->link_active = LINK_STATE_UP; 1429 if_link_state_change(ifp, LINK_STATE_UP); 1430 } 1431 } else { 1432 /* 1433 * Do it when link active changes to DOWN. i.e. 1434 * a) LINK_STATE_UNKNOWN -> LINK_STATE_DOWN 1435 * b) LINK_STATE_UP -> LINK_STATE_DOWN 1436 */ 1437 if (sc->link_active != LINK_STATE_DOWN) { 1438 if (bootverbose) 1439 device_printf(dev, "Link is Down\n"); 1440 if_link_state_change(ifp, LINK_STATE_DOWN); 1441 sc->link_active = LINK_STATE_DOWN; 1442 } 1443 } 1444 } /* ixv_update_link_status */ 1445 1446 1447 /************************************************************************ 1448 * ixv_stop - Stop the hardware 1449 * 1450 * Disables all traffic on the adapter by issuing a 1451 * global reset on the MAC and deallocates TX/RX buffers. 1452 ************************************************************************/ 1453 static void 1454 ixv_ifstop(struct ifnet *ifp, int disable) 1455 { 1456 struct ixgbe_softc *sc = ifp->if_softc; 1457 1458 IXGBE_CORE_LOCK(sc); 1459 ixv_stop_locked(sc); 1460 IXGBE_CORE_UNLOCK(sc); 1461 1462 workqueue_wait(sc->admin_wq, &sc->admin_wc); 1463 atomic_store_relaxed(&sc->admin_pending, 0); 1464 workqueue_wait(sc->timer_wq, &sc->timer_wc); 1465 atomic_store_relaxed(&sc->timer_pending, 0); 1466 } 1467 1468 static void 1469 ixv_stop_locked(void *arg) 1470 { 1471 struct ifnet *ifp; 1472 struct ixgbe_softc *sc = arg; 1473 struct ixgbe_hw *hw = &sc->hw; 1474 1475 ifp = sc->ifp; 1476 1477 KASSERT(mutex_owned(&sc->core_mtx)); 1478 1479 INIT_DEBUGOUT("ixv_stop_locked: begin\n"); 1480 ixv_disable_intr(sc); 1481 1482 /* Tell the stack that the interface is no longer active */ 1483 ifp->if_flags &= ~IFF_RUNNING; 1484 1485 hw->mac.ops.reset_hw(hw); 1486 sc->hw.adapter_stopped = FALSE; 1487 hw->mac.ops.stop_adapter(hw); 1488 callout_stop(&sc->timer); 1489 1490 /* Don't schedule workqueues. */ 1491 sc->schedule_wqs_ok = false; 1492 1493 /* reprogram the RAR[0] in case user changed it. */ 1494 hw->mac.ops.set_rar(hw, 0, hw->mac.addr, 0, IXGBE_RAH_AV); 1495 1496 return; 1497 } /* ixv_stop_locked */ 1498 1499 1500 /************************************************************************ 1501 * ixv_allocate_pci_resources 1502 ************************************************************************/ 1503 static int 1504 ixv_allocate_pci_resources(struct ixgbe_softc *sc, 1505 const struct pci_attach_args *pa) 1506 { 1507 pcireg_t memtype, csr; 1508 device_t dev = sc->dev; 1509 bus_addr_t addr; 1510 int flags; 1511 1512 memtype = pci_mapreg_type(pa->pa_pc, pa->pa_tag, PCI_BAR(0)); 1513 switch (memtype) { 1514 case PCI_MAPREG_TYPE_MEM | PCI_MAPREG_MEM_TYPE_32BIT: 1515 case PCI_MAPREG_TYPE_MEM | PCI_MAPREG_MEM_TYPE_64BIT: 1516 sc->osdep.mem_bus_space_tag = pa->pa_memt; 1517 if (pci_mapreg_info(pa->pa_pc, pa->pa_tag, PCI_BAR(0), 1518 memtype, &addr, &sc->osdep.mem_size, &flags) != 0) 1519 goto map_err; 1520 if ((flags & BUS_SPACE_MAP_PREFETCHABLE) != 0) { 1521 aprint_normal_dev(dev, "clearing prefetchable bit\n"); 1522 flags &= ~BUS_SPACE_MAP_PREFETCHABLE; 1523 } 1524 if (bus_space_map(sc->osdep.mem_bus_space_tag, addr, 1525 sc->osdep.mem_size, flags, 1526 &sc->osdep.mem_bus_space_handle) != 0) { 1527 map_err: 1528 sc->osdep.mem_size = 0; 1529 aprint_error_dev(dev, "unable to map BAR0\n"); 1530 return ENXIO; 1531 } 1532 /* 1533 * Enable address decoding for memory range in case it's not 1534 * set. 1535 */ 1536 csr = pci_conf_read(pa->pa_pc, pa->pa_tag, 1537 PCI_COMMAND_STATUS_REG); 1538 csr |= PCI_COMMAND_MEM_ENABLE; 1539 pci_conf_write(pa->pa_pc, pa->pa_tag, PCI_COMMAND_STATUS_REG, 1540 csr); 1541 break; 1542 default: 1543 aprint_error_dev(dev, "unexpected type on BAR0\n"); 1544 return ENXIO; 1545 } 1546 1547 /* Pick up the tuneable queues */ 1548 sc->num_queues = ixv_num_queues; 1549 1550 return (0); 1551 } /* ixv_allocate_pci_resources */ 1552 1553 static void 1554 ixv_free_deferred_handlers(struct ixgbe_softc *sc) 1555 { 1556 struct ix_queue *que = sc->queues; 1557 struct tx_ring *txr = sc->tx_rings; 1558 int i; 1559 1560 for (i = 0; i < sc->num_queues; i++, que++, txr++) { 1561 if (!(sc->feat_en & IXGBE_FEATURE_LEGACY_TX)) { 1562 if (txr->txr_si != NULL) 1563 softint_disestablish(txr->txr_si); 1564 } 1565 if (que->que_si != NULL) 1566 softint_disestablish(que->que_si); 1567 } 1568 if (sc->txr_wq != NULL) 1569 workqueue_destroy(sc->txr_wq); 1570 if (sc->txr_wq_enqueued != NULL) 1571 percpu_free(sc->txr_wq_enqueued, sizeof(u_int)); 1572 if (sc->que_wq != NULL) 1573 workqueue_destroy(sc->que_wq); 1574 1575 /* Drain the Mailbox(link) queue */ 1576 if (sc->admin_wq != NULL) { 1577 workqueue_destroy(sc->admin_wq); 1578 sc->admin_wq = NULL; 1579 } 1580 if (sc->timer_wq != NULL) { 1581 workqueue_destroy(sc->timer_wq); 1582 sc->timer_wq = NULL; 1583 } 1584 } /* ixv_free_deferred_handlers */ 1585 1586 /************************************************************************ 1587 * ixv_free_pci_resources 1588 ************************************************************************/ 1589 static void 1590 ixv_free_pci_resources(struct ixgbe_softc *sc) 1591 { 1592 struct ix_queue *que = sc->queues; 1593 int rid; 1594 1595 /* 1596 * Release all msix queue resources: 1597 */ 1598 for (int i = 0; i < sc->num_queues; i++, que++) { 1599 if (que->res != NULL) 1600 pci_intr_disestablish(sc->osdep.pc, 1601 sc->osdep.ihs[i]); 1602 } 1603 1604 1605 /* Clean the Mailbox interrupt last */ 1606 rid = sc->vector; 1607 1608 if (sc->osdep.ihs[rid] != NULL) { 1609 pci_intr_disestablish(sc->osdep.pc, 1610 sc->osdep.ihs[rid]); 1611 sc->osdep.ihs[rid] = NULL; 1612 } 1613 1614 pci_intr_release(sc->osdep.pc, sc->osdep.intrs, 1615 sc->osdep.nintrs); 1616 1617 if (sc->osdep.mem_size != 0) { 1618 bus_space_unmap(sc->osdep.mem_bus_space_tag, 1619 sc->osdep.mem_bus_space_handle, 1620 sc->osdep.mem_size); 1621 } 1622 1623 return; 1624 } /* ixv_free_pci_resources */ 1625 1626 /************************************************************************ 1627 * ixv_setup_interface 1628 * 1629 * Setup networking device structure and register an interface. 1630 ************************************************************************/ 1631 static int 1632 ixv_setup_interface(device_t dev, struct ixgbe_softc *sc) 1633 { 1634 struct ethercom *ec = &sc->osdep.ec; 1635 struct ifnet *ifp; 1636 1637 INIT_DEBUGOUT("ixv_setup_interface: begin"); 1638 1639 ifp = sc->ifp = &ec->ec_if; 1640 strlcpy(ifp->if_xname, device_xname(dev), IFNAMSIZ); 1641 ifp->if_baudrate = IF_Gbps(10); 1642 ifp->if_init = ixv_init; 1643 ifp->if_stop = ixv_ifstop; 1644 ifp->if_softc = sc; 1645 ifp->if_flags = IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST; 1646 ifp->if_extflags = IFEF_MPSAFE; 1647 ifp->if_ioctl = ixv_ioctl; 1648 if (sc->feat_en & IXGBE_FEATURE_LEGACY_TX) { 1649 #if 0 1650 ixv_start_locked = ixgbe_legacy_start_locked; 1651 #endif 1652 } else { 1653 ifp->if_transmit = ixgbe_mq_start; 1654 #if 0 1655 ixv_start_locked = ixgbe_mq_start_locked; 1656 #endif 1657 } 1658 ifp->if_start = ixgbe_legacy_start; 1659 IFQ_SET_MAXLEN(&ifp->if_snd, sc->num_tx_desc - 2); 1660 IFQ_SET_READY(&ifp->if_snd); 1661 1662 if_initialize(ifp); 1663 sc->ipq = if_percpuq_create(&sc->osdep.ec.ec_if); 1664 /* 1665 * We use per TX queue softint, so if_deferred_start_init() isn't 1666 * used. 1667 */ 1668 1669 sc->max_frame_size = ifp->if_mtu + IXGBE_MTU_HDR; 1670 1671 /* 1672 * Tell the upper layer(s) we support long frames. 1673 */ 1674 ifp->if_hdrlen = sizeof(struct ether_vlan_header); 1675 1676 /* Set capability flags */ 1677 ifp->if_capabilities |= IFCAP_HWCSUM 1678 | IFCAP_TSOv4 1679 | IFCAP_TSOv6; 1680 ifp->if_capenable = 0; 1681 1682 ec->ec_capabilities |= ETHERCAP_VLAN_HWFILTER 1683 | ETHERCAP_VLAN_HWTAGGING 1684 | ETHERCAP_VLAN_HWCSUM 1685 | ETHERCAP_JUMBO_MTU 1686 | ETHERCAP_VLAN_MTU; 1687 1688 /* Enable the above capabilities by default */ 1689 ec->ec_capenable = ec->ec_capabilities; 1690 1691 ether_ifattach(ifp, sc->hw.mac.addr); 1692 aprint_normal_dev(dev, "Ethernet address %s\n", 1693 ether_sprintf(sc->hw.mac.addr)); 1694 ether_set_ifflags_cb(ec, ixv_ifflags_cb); 1695 1696 /* Don't enable LRO by default */ 1697 #if 0 1698 /* NetBSD doesn't support LRO yet */ 1699 ifp->if_capabilities |= IFCAP_LRO; 1700 #endif 1701 1702 /* 1703 * Specify the media types supported by this adapter and register 1704 * callbacks to update media and link information 1705 */ 1706 ec->ec_ifmedia = &sc->media; 1707 ifmedia_init_with_lock(&sc->media, IFM_IMASK, ixv_media_change, 1708 ixv_media_status, &sc->core_mtx); 1709 ifmedia_add(&sc->media, IFM_ETHER | IFM_AUTO, 0, NULL); 1710 ifmedia_set(&sc->media, IFM_ETHER | IFM_AUTO); 1711 1712 if_register(ifp); 1713 1714 return 0; 1715 } /* ixv_setup_interface */ 1716 1717 1718 /************************************************************************ 1719 * ixv_initialize_transmit_units - Enable transmit unit. 1720 ************************************************************************/ 1721 static void 1722 ixv_initialize_transmit_units(struct ixgbe_softc *sc) 1723 { 1724 struct tx_ring *txr = sc->tx_rings; 1725 struct ixgbe_hw *hw = &sc->hw; 1726 int i; 1727 1728 for (i = 0; i < sc->num_queues; i++, txr++) { 1729 u64 tdba = txr->txdma.dma_paddr; 1730 u32 txctrl, txdctl; 1731 int j = txr->me; 1732 1733 /* Set WTHRESH to 8, burst writeback */ 1734 txdctl = IXGBE_READ_REG(hw, IXGBE_VFTXDCTL(j)); 1735 txdctl &= ~IXGBE_TXDCTL_WTHRESH_MASK; 1736 txdctl |= IXGBE_TX_WTHRESH << IXGBE_TXDCTL_WTHRESH_SHIFT; 1737 IXGBE_WRITE_REG(hw, IXGBE_VFTXDCTL(j), txdctl); 1738 1739 /* Set the HW Tx Head and Tail indices */ 1740 IXGBE_WRITE_REG(&sc->hw, IXGBE_VFTDH(j), 0); 1741 IXGBE_WRITE_REG(&sc->hw, IXGBE_VFTDT(j), 0); 1742 1743 /* Set Tx Tail register */ 1744 txr->tail = IXGBE_VFTDT(j); 1745 1746 txr->txr_no_space = false; 1747 1748 /* Set Ring parameters */ 1749 IXGBE_WRITE_REG(hw, IXGBE_VFTDBAL(j), 1750 (tdba & 0x00000000ffffffffULL)); 1751 IXGBE_WRITE_REG(hw, IXGBE_VFTDBAH(j), (tdba >> 32)); 1752 IXGBE_WRITE_REG(hw, IXGBE_VFTDLEN(j), 1753 sc->num_tx_desc * sizeof(struct ixgbe_legacy_tx_desc)); 1754 txctrl = IXGBE_READ_REG(hw, IXGBE_VFDCA_TXCTRL(j)); 1755 txctrl &= ~IXGBE_DCA_TXCTRL_DESC_WRO_EN; 1756 IXGBE_WRITE_REG(hw, IXGBE_VFDCA_TXCTRL(j), txctrl); 1757 1758 /* Now enable */ 1759 txdctl = IXGBE_READ_REG(hw, IXGBE_VFTXDCTL(j)); 1760 txdctl |= IXGBE_TXDCTL_ENABLE; 1761 IXGBE_WRITE_REG(hw, IXGBE_VFTXDCTL(j), txdctl); 1762 } 1763 1764 return; 1765 } /* ixv_initialize_transmit_units */ 1766 1767 1768 /************************************************************************ 1769 * ixv_initialize_rss_mapping 1770 ************************************************************************/ 1771 static void 1772 ixv_initialize_rss_mapping(struct ixgbe_softc *sc) 1773 { 1774 struct ixgbe_hw *hw = &sc->hw; 1775 u32 reta = 0, mrqc, rss_key[10]; 1776 int queue_id; 1777 int i, j; 1778 u32 rss_hash_config; 1779 1780 /* force use default RSS key. */ 1781 #ifdef __NetBSD__ 1782 rss_getkey((uint8_t *) &rss_key); 1783 #else 1784 if (sc->feat_en & IXGBE_FEATURE_RSS) { 1785 /* Fetch the configured RSS key */ 1786 rss_getkey((uint8_t *)&rss_key); 1787 } else { 1788 /* set up random bits */ 1789 cprng_fast(&rss_key, sizeof(rss_key)); 1790 } 1791 #endif 1792 1793 /* Now fill out hash function seeds */ 1794 for (i = 0; i < 10; i++) 1795 IXGBE_WRITE_REG(hw, IXGBE_VFRSSRK(i), rss_key[i]); 1796 1797 /* Set up the redirection table */ 1798 for (i = 0, j = 0; i < 64; i++, j++) { 1799 if (j == sc->num_queues) 1800 j = 0; 1801 1802 if (sc->feat_en & IXGBE_FEATURE_RSS) { 1803 /* 1804 * Fetch the RSS bucket id for the given indirection 1805 * entry. Cap it at the number of configured buckets 1806 * (which is num_queues.) 1807 */ 1808 queue_id = rss_get_indirection_to_bucket(i); 1809 queue_id = queue_id % sc->num_queues; 1810 } else 1811 queue_id = j; 1812 1813 /* 1814 * The low 8 bits are for hash value (n+0); 1815 * The next 8 bits are for hash value (n+1), etc. 1816 */ 1817 reta >>= 8; 1818 reta |= ((uint32_t)queue_id) << 24; 1819 if ((i & 3) == 3) { 1820 IXGBE_WRITE_REG(hw, IXGBE_VFRETA(i >> 2), reta); 1821 reta = 0; 1822 } 1823 } 1824 1825 /* Perform hash on these packet types */ 1826 if (sc->feat_en & IXGBE_FEATURE_RSS) 1827 rss_hash_config = rss_gethashconfig(); 1828 else { 1829 /* 1830 * Disable UDP - IP fragments aren't currently being handled 1831 * and so we end up with a mix of 2-tuple and 4-tuple 1832 * traffic. 1833 */ 1834 rss_hash_config = RSS_HASHTYPE_RSS_IPV4 1835 | RSS_HASHTYPE_RSS_TCP_IPV4 1836 | RSS_HASHTYPE_RSS_IPV6 1837 | RSS_HASHTYPE_RSS_TCP_IPV6; 1838 } 1839 1840 mrqc = IXGBE_MRQC_RSSEN; 1841 if (rss_hash_config & RSS_HASHTYPE_RSS_IPV4) 1842 mrqc |= IXGBE_MRQC_RSS_FIELD_IPV4; 1843 if (rss_hash_config & RSS_HASHTYPE_RSS_TCP_IPV4) 1844 mrqc |= IXGBE_MRQC_RSS_FIELD_IPV4_TCP; 1845 if (rss_hash_config & RSS_HASHTYPE_RSS_IPV6) 1846 mrqc |= IXGBE_MRQC_RSS_FIELD_IPV6; 1847 if (rss_hash_config & RSS_HASHTYPE_RSS_TCP_IPV6) 1848 mrqc |= IXGBE_MRQC_RSS_FIELD_IPV6_TCP; 1849 if (rss_hash_config & RSS_HASHTYPE_RSS_IPV6_EX) 1850 device_printf(sc->dev, "%s: RSS_HASHTYPE_RSS_IPV6_EX " 1851 "defined, but not supported\n", __func__); 1852 if (rss_hash_config & RSS_HASHTYPE_RSS_TCP_IPV6_EX) 1853 device_printf(sc->dev, "%s: RSS_HASHTYPE_RSS_TCP_IPV6_EX " 1854 "defined, but not supported\n", __func__); 1855 if (rss_hash_config & RSS_HASHTYPE_RSS_UDP_IPV4) 1856 mrqc |= IXGBE_MRQC_RSS_FIELD_IPV4_UDP; 1857 if (rss_hash_config & RSS_HASHTYPE_RSS_UDP_IPV6) 1858 mrqc |= IXGBE_MRQC_RSS_FIELD_IPV6_UDP; 1859 if (rss_hash_config & RSS_HASHTYPE_RSS_UDP_IPV6_EX) 1860 device_printf(sc->dev, "%s: RSS_HASHTYPE_RSS_UDP_IPV6_EX " 1861 "defined, but not supported\n", __func__); 1862 IXGBE_WRITE_REG(hw, IXGBE_VFMRQC, mrqc); 1863 } /* ixv_initialize_rss_mapping */ 1864 1865 1866 /************************************************************************ 1867 * ixv_initialize_receive_units - Setup receive registers and features. 1868 ************************************************************************/ 1869 static void 1870 ixv_initialize_receive_units(struct ixgbe_softc *sc) 1871 { 1872 struct rx_ring *rxr = sc->rx_rings; 1873 struct ixgbe_hw *hw = &sc->hw; 1874 struct ifnet *ifp = sc->ifp; 1875 u32 bufsz, psrtype; 1876 1877 if (ifp->if_mtu > ETHERMTU) 1878 bufsz = 4096 >> IXGBE_SRRCTL_BSIZEPKT_SHIFT; 1879 else 1880 bufsz = 2048 >> IXGBE_SRRCTL_BSIZEPKT_SHIFT; 1881 1882 psrtype = IXGBE_PSRTYPE_TCPHDR 1883 | IXGBE_PSRTYPE_UDPHDR 1884 | IXGBE_PSRTYPE_IPV4HDR 1885 | IXGBE_PSRTYPE_IPV6HDR 1886 | IXGBE_PSRTYPE_L2HDR; 1887 1888 if (sc->num_queues > 1) 1889 psrtype |= 1 << 29; 1890 1891 IXGBE_WRITE_REG(hw, IXGBE_VFPSRTYPE, psrtype); 1892 1893 /* Tell PF our max_frame size */ 1894 if (ixgbevf_rlpml_set_vf(hw, sc->max_frame_size) != 0) { 1895 device_printf(sc->dev, "There is a problem with the PF " 1896 "setup. It is likely the receive unit for this VF will " 1897 "not function correctly.\n"); 1898 } 1899 1900 for (int i = 0; i < sc->num_queues; i++, rxr++) { 1901 u64 rdba = rxr->rxdma.dma_paddr; 1902 u32 reg, rxdctl; 1903 int j = rxr->me; 1904 1905 /* Disable the queue */ 1906 rxdctl = IXGBE_READ_REG(hw, IXGBE_VFRXDCTL(j)); 1907 rxdctl &= ~IXGBE_RXDCTL_ENABLE; 1908 IXGBE_WRITE_REG(hw, IXGBE_VFRXDCTL(j), rxdctl); 1909 for (int k = 0; k < 10; k++) { 1910 if (IXGBE_READ_REG(hw, IXGBE_VFRXDCTL(j)) & 1911 IXGBE_RXDCTL_ENABLE) 1912 msec_delay(1); 1913 else 1914 break; 1915 } 1916 IXGBE_WRITE_BARRIER(hw); 1917 /* Setup the Base and Length of the Rx Descriptor Ring */ 1918 IXGBE_WRITE_REG(hw, IXGBE_VFRDBAL(j), 1919 (rdba & 0x00000000ffffffffULL)); 1920 IXGBE_WRITE_REG(hw, IXGBE_VFRDBAH(j), (rdba >> 32)); 1921 IXGBE_WRITE_REG(hw, IXGBE_VFRDLEN(j), 1922 sc->num_rx_desc * sizeof(union ixgbe_adv_rx_desc)); 1923 1924 /* Reset the ring indices */ 1925 IXGBE_WRITE_REG(hw, IXGBE_VFRDH(rxr->me), 0); 1926 IXGBE_WRITE_REG(hw, IXGBE_VFRDT(rxr->me), 0); 1927 1928 /* Set up the SRRCTL register */ 1929 reg = IXGBE_READ_REG(hw, IXGBE_VFSRRCTL(j)); 1930 reg &= ~IXGBE_SRRCTL_BSIZEHDR_MASK; 1931 reg &= ~IXGBE_SRRCTL_BSIZEPKT_MASK; 1932 reg |= bufsz; 1933 reg |= IXGBE_SRRCTL_DESCTYPE_ADV_ONEBUF; 1934 IXGBE_WRITE_REG(hw, IXGBE_VFSRRCTL(j), reg); 1935 1936 /* Capture Rx Tail index */ 1937 rxr->tail = IXGBE_VFRDT(rxr->me); 1938 1939 /* Do the queue enabling last */ 1940 rxdctl |= IXGBE_RXDCTL_ENABLE | IXGBE_RXDCTL_VME; 1941 IXGBE_WRITE_REG(hw, IXGBE_VFRXDCTL(j), rxdctl); 1942 for (int k = 0; k < 10; k++) { 1943 if (IXGBE_READ_REG(hw, IXGBE_VFRXDCTL(j)) & 1944 IXGBE_RXDCTL_ENABLE) 1945 break; 1946 msec_delay(1); 1947 } 1948 IXGBE_WRITE_BARRIER(hw); 1949 1950 /* Set the Tail Pointer */ 1951 #ifdef DEV_NETMAP 1952 /* 1953 * In netmap mode, we must preserve the buffers made 1954 * available to userspace before the if_init() 1955 * (this is true by default on the TX side, because 1956 * init makes all buffers available to userspace). 1957 * 1958 * netmap_reset() and the device specific routines 1959 * (e.g. ixgbe_setup_receive_rings()) map these 1960 * buffers at the end of the NIC ring, so here we 1961 * must set the RDT (tail) register to make sure 1962 * they are not overwritten. 1963 * 1964 * In this driver the NIC ring starts at RDH = 0, 1965 * RDT points to the last slot available for reception (?), 1966 * so RDT = num_rx_desc - 1 means the whole ring is available. 1967 */ 1968 if ((sc->feat_en & IXGBE_FEATURE_NETMAP) && 1969 (ifp->if_capenable & IFCAP_NETMAP)) { 1970 struct netmap_adapter *na = NA(sc->ifp); 1971 struct netmap_kring *kring = na->rx_rings[i]; 1972 int t = na->num_rx_desc - 1 - nm_kr_rxspace(kring); 1973 1974 IXGBE_WRITE_REG(hw, IXGBE_VFRDT(rxr->me), t); 1975 } else 1976 #endif /* DEV_NETMAP */ 1977 IXGBE_WRITE_REG(hw, IXGBE_VFRDT(rxr->me), 1978 sc->num_rx_desc - 1); 1979 } 1980 1981 if (sc->hw.mac.type >= ixgbe_mac_X550_vf) 1982 ixv_initialize_rss_mapping(sc); 1983 } /* ixv_initialize_receive_units */ 1984 1985 /************************************************************************ 1986 * ixv_sysctl_tdh_handler - Transmit Descriptor Head handler function 1987 * 1988 * Retrieves the TDH value from the hardware 1989 ************************************************************************/ 1990 static int 1991 ixv_sysctl_tdh_handler(SYSCTLFN_ARGS) 1992 { 1993 struct sysctlnode node = *rnode; 1994 struct tx_ring *txr = (struct tx_ring *)node.sysctl_data; 1995 uint32_t val; 1996 1997 if (!txr) 1998 return (0); 1999 2000 val = IXGBE_READ_REG(&txr->sc->hw, IXGBE_VFTDH(txr->me)); 2001 node.sysctl_data = &val; 2002 return sysctl_lookup(SYSCTLFN_CALL(&node)); 2003 } /* ixv_sysctl_tdh_handler */ 2004 2005 /************************************************************************ 2006 * ixgbe_sysctl_tdt_handler - Transmit Descriptor Tail handler function 2007 * 2008 * Retrieves the TDT value from the hardware 2009 ************************************************************************/ 2010 static int 2011 ixv_sysctl_tdt_handler(SYSCTLFN_ARGS) 2012 { 2013 struct sysctlnode node = *rnode; 2014 struct tx_ring *txr = (struct tx_ring *)node.sysctl_data; 2015 uint32_t val; 2016 2017 if (!txr) 2018 return (0); 2019 2020 val = IXGBE_READ_REG(&txr->sc->hw, IXGBE_VFTDT(txr->me)); 2021 node.sysctl_data = &val; 2022 return sysctl_lookup(SYSCTLFN_CALL(&node)); 2023 } /* ixv_sysctl_tdt_handler */ 2024 2025 /************************************************************************ 2026 * ixv_sysctl_next_to_check_handler - Receive Descriptor next to check 2027 * handler function 2028 * 2029 * Retrieves the next_to_check value 2030 ************************************************************************/ 2031 static int 2032 ixv_sysctl_next_to_check_handler(SYSCTLFN_ARGS) 2033 { 2034 struct sysctlnode node = *rnode; 2035 struct rx_ring *rxr = (struct rx_ring *)node.sysctl_data; 2036 uint32_t val; 2037 2038 if (!rxr) 2039 return (0); 2040 2041 val = rxr->next_to_check; 2042 node.sysctl_data = &val; 2043 return sysctl_lookup(SYSCTLFN_CALL(&node)); 2044 } /* ixv_sysctl_next_to_check_handler */ 2045 2046 /************************************************************************ 2047 * ixv_sysctl_next_to_refresh_handler - Receive Descriptor next to refresh 2048 * handler function 2049 * 2050 * Retrieves the next_to_refresh value 2051 ************************************************************************/ 2052 static int 2053 ixv_sysctl_next_to_refresh_handler(SYSCTLFN_ARGS) 2054 { 2055 struct sysctlnode node = *rnode; 2056 struct rx_ring *rxr = (struct rx_ring *)node.sysctl_data; 2057 struct ixgbe_softc *sc; 2058 uint32_t val; 2059 2060 if (!rxr) 2061 return (0); 2062 2063 sc = rxr->sc; 2064 if (ixgbe_fw_recovery_mode_swflag(sc)) 2065 return (EPERM); 2066 2067 val = rxr->next_to_refresh; 2068 node.sysctl_data = &val; 2069 return sysctl_lookup(SYSCTLFN_CALL(&node)); 2070 } /* ixv_sysctl_next_to_refresh_handler */ 2071 2072 /************************************************************************ 2073 * ixv_sysctl_rdh_handler - Receive Descriptor Head handler function 2074 * 2075 * Retrieves the RDH value from the hardware 2076 ************************************************************************/ 2077 static int 2078 ixv_sysctl_rdh_handler(SYSCTLFN_ARGS) 2079 { 2080 struct sysctlnode node = *rnode; 2081 struct rx_ring *rxr = (struct rx_ring *)node.sysctl_data; 2082 uint32_t val; 2083 2084 if (!rxr) 2085 return (0); 2086 2087 val = IXGBE_READ_REG(&rxr->sc->hw, IXGBE_VFRDH(rxr->me)); 2088 node.sysctl_data = &val; 2089 return sysctl_lookup(SYSCTLFN_CALL(&node)); 2090 } /* ixv_sysctl_rdh_handler */ 2091 2092 /************************************************************************ 2093 * ixv_sysctl_rdt_handler - Receive Descriptor Tail handler function 2094 * 2095 * Retrieves the RDT value from the hardware 2096 ************************************************************************/ 2097 static int 2098 ixv_sysctl_rdt_handler(SYSCTLFN_ARGS) 2099 { 2100 struct sysctlnode node = *rnode; 2101 struct rx_ring *rxr = (struct rx_ring *)node.sysctl_data; 2102 uint32_t val; 2103 2104 if (!rxr) 2105 return (0); 2106 2107 val = IXGBE_READ_REG(&rxr->sc->hw, IXGBE_VFRDT(rxr->me)); 2108 node.sysctl_data = &val; 2109 return sysctl_lookup(SYSCTLFN_CALL(&node)); 2110 } /* ixv_sysctl_rdt_handler */ 2111 2112 static void 2113 ixv_setup_vlan_tagging(struct ixgbe_softc *sc) 2114 { 2115 struct ethercom *ec = &sc->osdep.ec; 2116 struct ixgbe_hw *hw = &sc->hw; 2117 struct rx_ring *rxr; 2118 u32 ctrl; 2119 int i; 2120 bool hwtagging; 2121 2122 /* Enable HW tagging only if any vlan is attached */ 2123 hwtagging = (ec->ec_capenable & ETHERCAP_VLAN_HWTAGGING) 2124 && VLAN_ATTACHED(ec); 2125 2126 /* Enable the queues */ 2127 for (i = 0; i < sc->num_queues; i++) { 2128 rxr = &sc->rx_rings[i]; 2129 ctrl = IXGBE_READ_REG(hw, IXGBE_VFRXDCTL(rxr->me)); 2130 if (hwtagging) 2131 ctrl |= IXGBE_RXDCTL_VME; 2132 else 2133 ctrl &= ~IXGBE_RXDCTL_VME; 2134 IXGBE_WRITE_REG(hw, IXGBE_VFRXDCTL(rxr->me), ctrl); 2135 /* 2136 * Let Rx path know that it needs to store VLAN tag 2137 * as part of extra mbuf info. 2138 */ 2139 rxr->vtag_strip = hwtagging ? TRUE : FALSE; 2140 } 2141 } /* ixv_setup_vlan_tagging */ 2142 2143 /************************************************************************ 2144 * ixv_setup_vlan_support 2145 ************************************************************************/ 2146 static int 2147 ixv_setup_vlan_support(struct ixgbe_softc *sc) 2148 { 2149 struct ethercom *ec = &sc->osdep.ec; 2150 struct ixgbe_hw *hw = &sc->hw; 2151 u32 vid, vfta, retry; 2152 struct vlanid_list *vlanidp; 2153 int rv, error = 0; 2154 2155 /* 2156 * This function is called from both if_init and ifflags_cb() 2157 * on NetBSD. 2158 */ 2159 2160 /* 2161 * Part 1: 2162 * Setup VLAN HW tagging 2163 */ 2164 ixv_setup_vlan_tagging(sc); 2165 2166 if (!VLAN_ATTACHED(ec)) 2167 return 0; 2168 2169 /* 2170 * Part 2: 2171 * Setup VLAN HW filter 2172 */ 2173 /* Cleanup shadow_vfta */ 2174 for (int i = 0; i < IXGBE_VFTA_SIZE; i++) 2175 sc->shadow_vfta[i] = 0; 2176 /* Generate shadow_vfta from ec_vids */ 2177 ETHER_LOCK(ec); 2178 SIMPLEQ_FOREACH(vlanidp, &ec->ec_vids, vid_list) { 2179 uint32_t idx; 2180 2181 idx = vlanidp->vid / 32; 2182 KASSERT(idx < IXGBE_VFTA_SIZE); 2183 sc->shadow_vfta[idx] |= (u32)1 << (vlanidp->vid % 32); 2184 } 2185 ETHER_UNLOCK(ec); 2186 2187 /* 2188 * A soft reset zero's out the VFTA, so 2189 * we need to repopulate it now. 2190 */ 2191 for (int i = 0; i < IXGBE_VFTA_SIZE; i++) { 2192 if (sc->shadow_vfta[i] == 0) 2193 continue; 2194 vfta = sc->shadow_vfta[i]; 2195 /* 2196 * Reconstruct the vlan id's 2197 * based on the bits set in each 2198 * of the array ints. 2199 */ 2200 for (int j = 0; j < 32; j++) { 2201 retry = 0; 2202 if ((vfta & ((u32)1 << j)) == 0) 2203 continue; 2204 vid = (i * 32) + j; 2205 2206 /* Call the shared code mailbox routine */ 2207 while ((rv = hw->mac.ops.set_vfta(hw, vid, 0, TRUE, 2208 FALSE)) != 0) { 2209 if (++retry > 5) { 2210 device_printf(sc->dev, 2211 "%s: max retry exceeded\n", 2212 __func__); 2213 break; 2214 } 2215 } 2216 if (rv != 0) { 2217 device_printf(sc->dev, 2218 "failed to set vlan %d\n", vid); 2219 error = EACCES; 2220 } 2221 } 2222 } 2223 return error; 2224 } /* ixv_setup_vlan_support */ 2225 2226 static int 2227 ixv_vlan_cb(struct ethercom *ec, uint16_t vid, bool set) 2228 { 2229 struct ifnet *ifp = &ec->ec_if; 2230 struct ixgbe_softc *sc = ifp->if_softc; 2231 int rv; 2232 2233 if (set) 2234 rv = ixv_register_vlan(sc, vid); 2235 else 2236 rv = ixv_unregister_vlan(sc, vid); 2237 2238 if (rv != 0) 2239 return rv; 2240 2241 /* 2242 * Control VLAN HW tagging when ec_nvlan is changed from 1 to 0 2243 * or 0 to 1. 2244 */ 2245 if ((set && (ec->ec_nvlans == 1)) || (!set && (ec->ec_nvlans == 0))) 2246 ixv_setup_vlan_tagging(sc); 2247 2248 return rv; 2249 } 2250 2251 /************************************************************************ 2252 * ixv_register_vlan 2253 * 2254 * Run via a vlan config EVENT, it enables us to use the 2255 * HW Filter table since we can get the vlan id. This just 2256 * creates the entry in the soft version of the VFTA, init 2257 * will repopulate the real table. 2258 ************************************************************************/ 2259 static int 2260 ixv_register_vlan(struct ixgbe_softc *sc, u16 vtag) 2261 { 2262 struct ixgbe_hw *hw = &sc->hw; 2263 u16 index, bit; 2264 int error; 2265 2266 if ((vtag == 0) || (vtag > 4095)) /* Invalid */ 2267 return EINVAL; 2268 IXGBE_CORE_LOCK(sc); 2269 index = (vtag >> 5) & 0x7F; 2270 bit = vtag & 0x1F; 2271 sc->shadow_vfta[index] |= ((u32)1 << bit); 2272 error = hw->mac.ops.set_vfta(hw, vtag, 0, true, false); 2273 IXGBE_CORE_UNLOCK(sc); 2274 2275 if (error != 0) { 2276 device_printf(sc->dev, "failed to register vlan %hu\n", vtag); 2277 error = EACCES; 2278 } 2279 return error; 2280 } /* ixv_register_vlan */ 2281 2282 /************************************************************************ 2283 * ixv_unregister_vlan 2284 * 2285 * Run via a vlan unconfig EVENT, remove our entry 2286 * in the soft vfta. 2287 ************************************************************************/ 2288 static int 2289 ixv_unregister_vlan(struct ixgbe_softc *sc, u16 vtag) 2290 { 2291 struct ixgbe_hw *hw = &sc->hw; 2292 u16 index, bit; 2293 int error; 2294 2295 if ((vtag == 0) || (vtag > 4095)) /* Invalid */ 2296 return EINVAL; 2297 2298 IXGBE_CORE_LOCK(sc); 2299 index = (vtag >> 5) & 0x7F; 2300 bit = vtag & 0x1F; 2301 sc->shadow_vfta[index] &= ~((u32)1 << bit); 2302 error = hw->mac.ops.set_vfta(hw, vtag, 0, false, false); 2303 IXGBE_CORE_UNLOCK(sc); 2304 2305 if (error != 0) { 2306 device_printf(sc->dev, "failed to unregister vlan %hu\n", 2307 vtag); 2308 error = EIO; 2309 } 2310 return error; 2311 } /* ixv_unregister_vlan */ 2312 2313 /************************************************************************ 2314 * ixv_enable_intr 2315 ************************************************************************/ 2316 static void 2317 ixv_enable_intr(struct ixgbe_softc *sc) 2318 { 2319 struct ixgbe_hw *hw = &sc->hw; 2320 struct ix_queue *que = sc->queues; 2321 u32 mask; 2322 int i; 2323 2324 /* For VTEIAC */ 2325 mask = (1 << sc->vector); 2326 for (i = 0; i < sc->num_queues; i++, que++) 2327 mask |= (1 << que->msix); 2328 IXGBE_WRITE_REG(hw, IXGBE_VTEIAC, mask); 2329 2330 /* For VTEIMS */ 2331 IXGBE_WRITE_REG(hw, IXGBE_VTEIMS, (1 << sc->vector)); 2332 que = sc->queues; 2333 for (i = 0; i < sc->num_queues; i++, que++) 2334 ixv_enable_queue(sc, que->msix); 2335 2336 IXGBE_WRITE_FLUSH(hw); 2337 } /* ixv_enable_intr */ 2338 2339 /************************************************************************ 2340 * ixv_disable_intr 2341 ************************************************************************/ 2342 static void 2343 ixv_disable_intr(struct ixgbe_softc *sc) 2344 { 2345 struct ix_queue *que = sc->queues; 2346 2347 IXGBE_WRITE_REG(&sc->hw, IXGBE_VTEIAC, 0); 2348 2349 /* disable interrupts other than queues */ 2350 IXGBE_WRITE_REG(&sc->hw, IXGBE_VTEIMC, sc->vector); 2351 2352 for (int i = 0; i < sc->num_queues; i++, que++) 2353 ixv_disable_queue(sc, que->msix); 2354 2355 IXGBE_WRITE_FLUSH(&sc->hw); 2356 } /* ixv_disable_intr */ 2357 2358 /************************************************************************ 2359 * ixv_set_ivar 2360 * 2361 * Setup the correct IVAR register for a particular MSI-X interrupt 2362 * - entry is the register array entry 2363 * - vector is the MSI-X vector for this queue 2364 * - type is RX/TX/MISC 2365 ************************************************************************/ 2366 static void 2367 ixv_set_ivar(struct ixgbe_softc *sc, u8 entry, u8 vector, s8 type) 2368 { 2369 struct ixgbe_hw *hw = &sc->hw; 2370 u32 ivar, index; 2371 2372 vector |= IXGBE_IVAR_ALLOC_VAL; 2373 2374 if (type == -1) { /* MISC IVAR */ 2375 ivar = IXGBE_READ_REG(hw, IXGBE_VTIVAR_MISC); 2376 ivar &= ~0xFF; 2377 ivar |= vector; 2378 IXGBE_WRITE_REG(hw, IXGBE_VTIVAR_MISC, ivar); 2379 } else { /* RX/TX IVARS */ 2380 index = (16 * (entry & 1)) + (8 * type); 2381 ivar = IXGBE_READ_REG(hw, IXGBE_VTIVAR(entry >> 1)); 2382 ivar &= ~(0xffUL << index); 2383 ivar |= ((u32)vector << index); 2384 IXGBE_WRITE_REG(hw, IXGBE_VTIVAR(entry >> 1), ivar); 2385 } 2386 } /* ixv_set_ivar */ 2387 2388 /************************************************************************ 2389 * ixv_configure_ivars 2390 ************************************************************************/ 2391 static void 2392 ixv_configure_ivars(struct ixgbe_softc *sc) 2393 { 2394 struct ix_queue *que = sc->queues; 2395 2396 /* XXX We should sync EITR value calculation with ixgbe.c? */ 2397 2398 for (int i = 0; i < sc->num_queues; i++, que++) { 2399 /* First the RX queue entry */ 2400 ixv_set_ivar(sc, i, que->msix, 0); 2401 /* ... and the TX */ 2402 ixv_set_ivar(sc, i, que->msix, 1); 2403 /* Set an initial value in EITR */ 2404 ixv_eitr_write(sc, que->msix, IXGBE_EITR_DEFAULT); 2405 } 2406 2407 /* For the mailbox interrupt */ 2408 ixv_set_ivar(sc, 1, sc->vector, -1); 2409 } /* ixv_configure_ivars */ 2410 2411 2412 /************************************************************************ 2413 * ixv_init_stats 2414 * 2415 * The VF stats registers never have a truly virgin 2416 * starting point, so this routine save initial values to 2417 * last_<REGNAME>. 2418 ************************************************************************/ 2419 static void 2420 ixv_init_stats(struct ixgbe_softc *sc) 2421 { 2422 struct ixgbe_hw *hw = &sc->hw; 2423 2424 sc->stats.vf.last_vfgprc = IXGBE_READ_REG(hw, IXGBE_VFGPRC); 2425 sc->stats.vf.last_vfgorc = IXGBE_READ_REG(hw, IXGBE_VFGORC_LSB); 2426 sc->stats.vf.last_vfgorc |= 2427 (((u64)(IXGBE_READ_REG(hw, IXGBE_VFGORC_MSB))) << 32); 2428 2429 sc->stats.vf.last_vfgptc = IXGBE_READ_REG(hw, IXGBE_VFGPTC); 2430 sc->stats.vf.last_vfgotc = IXGBE_READ_REG(hw, IXGBE_VFGOTC_LSB); 2431 sc->stats.vf.last_vfgotc |= 2432 (((u64)(IXGBE_READ_REG(hw, IXGBE_VFGOTC_MSB))) << 32); 2433 2434 sc->stats.vf.last_vfmprc = IXGBE_READ_REG(hw, IXGBE_VFMPRC); 2435 } /* ixv_init_stats */ 2436 2437 #define UPDATE_STAT_32(reg, last, count) \ 2438 { \ 2439 u32 current = IXGBE_READ_REG(hw, (reg)); \ 2440 IXGBE_EVC_ADD(&count, current - (last)); \ 2441 (last) = current; \ 2442 } 2443 2444 #define UPDATE_STAT_36(lsb, msb, last, count) \ 2445 { \ 2446 u64 cur_lsb = IXGBE_READ_REG(hw, (lsb)); \ 2447 u64 cur_msb = IXGBE_READ_REG(hw, (msb)); \ 2448 u64 current = ((cur_msb << 32) | cur_lsb); \ 2449 if (current < (last)) \ 2450 IXGBE_EVC_ADD(&count, current + __BIT(36) - (last)); \ 2451 else \ 2452 IXGBE_EVC_ADD(&count, current - (last)); \ 2453 (last) = current; \ 2454 } 2455 2456 /************************************************************************ 2457 * ixv_update_stats - Update the board statistics counters. 2458 ************************************************************************/ 2459 void 2460 ixv_update_stats(struct ixgbe_softc *sc) 2461 { 2462 struct ixgbe_hw *hw = &sc->hw; 2463 struct ixgbevf_hw_stats *stats = &sc->stats.vf; 2464 2465 UPDATE_STAT_32(IXGBE_VFGPRC, stats->last_vfgprc, stats->vfgprc); 2466 UPDATE_STAT_32(IXGBE_VFGPTC, stats->last_vfgptc, stats->vfgptc); 2467 UPDATE_STAT_36(IXGBE_VFGORC_LSB, IXGBE_VFGORC_MSB, stats->last_vfgorc, 2468 stats->vfgorc); 2469 UPDATE_STAT_36(IXGBE_VFGOTC_LSB, IXGBE_VFGOTC_MSB, stats->last_vfgotc, 2470 stats->vfgotc); 2471 UPDATE_STAT_32(IXGBE_VFMPRC, stats->last_vfmprc, stats->vfmprc); 2472 2473 /* VF doesn't count errors by hardware */ 2474 2475 } /* ixv_update_stats */ 2476 2477 /************************************************************************ 2478 * ixv_sysctl_interrupt_rate_handler 2479 ************************************************************************/ 2480 static int 2481 ixv_sysctl_interrupt_rate_handler(SYSCTLFN_ARGS) 2482 { 2483 struct sysctlnode node = *rnode; 2484 struct ix_queue *que = (struct ix_queue *)node.sysctl_data; 2485 struct ixgbe_softc *sc; 2486 uint32_t reg, usec, rate; 2487 int error; 2488 2489 if (que == NULL) 2490 return 0; 2491 sc = que->sc; 2492 reg = IXGBE_READ_REG(&que->sc->hw, IXGBE_VTEITR(que->msix)); 2493 usec = ((reg & 0x0FF8) >> 3); 2494 if (usec > 0) 2495 rate = 500000 / usec; 2496 else 2497 rate = 0; 2498 node.sysctl_data = &rate; 2499 error = sysctl_lookup(SYSCTLFN_CALL(&node)); 2500 if (error || newp == NULL) 2501 return error; 2502 reg &= ~0xfff; /* default, no limitation */ 2503 if (rate > 0 && rate < 500000) { 2504 if (rate < 1000) 2505 rate = 1000; 2506 reg |= ((4000000 / rate) & 0xff8); 2507 /* 2508 * When RSC is used, ITR interval must be larger than 2509 * RSC_DELAY. Currently, we use 2us for RSC_DELAY. 2510 * The minimum value is always greater than 2us on 100M 2511 * (and 10M?(not documented)), but it's not on 1G and higher. 2512 */ 2513 if ((sc->link_speed != IXGBE_LINK_SPEED_100_FULL) 2514 && (sc->link_speed != IXGBE_LINK_SPEED_10_FULL)) { 2515 if ((sc->num_queues > 1) 2516 && (reg < IXGBE_MIN_RSC_EITR_10G1G)) 2517 return EINVAL; 2518 } 2519 sc->max_interrupt_rate = rate; 2520 } else 2521 sc->max_interrupt_rate = 0; 2522 ixv_eitr_write(sc, que->msix, reg); 2523 2524 return (0); 2525 } /* ixv_sysctl_interrupt_rate_handler */ 2526 2527 const struct sysctlnode * 2528 ixv_sysctl_instance(struct ixgbe_softc *sc) 2529 { 2530 const char *dvname; 2531 struct sysctllog **log; 2532 int rc; 2533 const struct sysctlnode *rnode; 2534 2535 log = &sc->sysctllog; 2536 dvname = device_xname(sc->dev); 2537 2538 if ((rc = sysctl_createv(log, 0, NULL, &rnode, 2539 0, CTLTYPE_NODE, dvname, 2540 SYSCTL_DESCR("ixv information and settings"), 2541 NULL, 0, NULL, 0, CTL_HW, CTL_CREATE, CTL_EOL)) != 0) 2542 goto err; 2543 2544 return rnode; 2545 err: 2546 device_printf(sc->dev, 2547 "%s: sysctl_createv failed, rc = %d\n", __func__, rc); 2548 return NULL; 2549 } 2550 2551 static void 2552 ixv_add_device_sysctls(struct ixgbe_softc *sc) 2553 { 2554 struct sysctllog **log; 2555 const struct sysctlnode *rnode, *cnode; 2556 device_t dev; 2557 2558 dev = sc->dev; 2559 log = &sc->sysctllog; 2560 2561 if ((rnode = ixv_sysctl_instance(sc)) == NULL) { 2562 aprint_error_dev(dev, "could not create sysctl root\n"); 2563 return; 2564 } 2565 2566 if (sysctl_createv(log, 0, &rnode, &cnode, 2567 CTLFLAG_READWRITE, CTLTYPE_INT, "debug", 2568 SYSCTL_DESCR("Debug Info"), 2569 ixv_sysctl_debug, 0, (void *)sc, 0, CTL_CREATE, CTL_EOL) != 0) 2570 aprint_error_dev(dev, "could not create sysctl\n"); 2571 2572 if (sysctl_createv(log, 0, &rnode, &cnode, 2573 CTLFLAG_READWRITE, CTLTYPE_INT, 2574 "rx_copy_len", SYSCTL_DESCR("RX Copy Length"), 2575 ixv_sysctl_rx_copy_len, 0, 2576 (void *)sc, 0, CTL_CREATE, CTL_EOL) != 0) 2577 aprint_error_dev(dev, "could not create sysctl\n"); 2578 2579 if (sysctl_createv(log, 0, &rnode, &cnode, 2580 CTLFLAG_READONLY, CTLTYPE_INT, 2581 "num_tx_desc", SYSCTL_DESCR("Number of TX descriptors"), 2582 NULL, 0, &sc->num_tx_desc, 0, CTL_CREATE, CTL_EOL) != 0) 2583 aprint_error_dev(dev, "could not create sysctl\n"); 2584 2585 if (sysctl_createv(log, 0, &rnode, &cnode, 2586 CTLFLAG_READONLY, CTLTYPE_INT, 2587 "num_rx_desc", SYSCTL_DESCR("Number of RX descriptors"), 2588 NULL, 0, &sc->num_rx_desc, 0, CTL_CREATE, CTL_EOL) != 0) 2589 aprint_error_dev(dev, "could not create sysctl\n"); 2590 2591 if (sysctl_createv(log, 0, &rnode, &cnode, 2592 CTLFLAG_READWRITE, CTLTYPE_INT, "rx_process_limit", 2593 SYSCTL_DESCR("max number of RX packets to process"), 2594 ixv_sysctl_rx_process_limit, 0, (void *)sc, 0, CTL_CREATE, 2595 CTL_EOL) != 0) 2596 aprint_error_dev(dev, "could not create sysctl\n"); 2597 2598 if (sysctl_createv(log, 0, &rnode, &cnode, 2599 CTLFLAG_READWRITE, CTLTYPE_INT, "tx_process_limit", 2600 SYSCTL_DESCR("max number of TX packets to process"), 2601 ixv_sysctl_tx_process_limit, 0, (void *)sc, 0, CTL_CREATE, 2602 CTL_EOL) != 0) 2603 aprint_error_dev(dev, "could not create sysctl\n"); 2604 2605 if (sysctl_createv(log, 0, &rnode, &cnode, 2606 CTLFLAG_READWRITE, CTLTYPE_BOOL, "enable_aim", 2607 SYSCTL_DESCR("Interrupt Moderation"), 2608 NULL, 0, &sc->enable_aim, 0, CTL_CREATE, CTL_EOL) != 0) 2609 aprint_error_dev(dev, "could not create sysctl\n"); 2610 2611 if (sysctl_createv(log, 0, &rnode, &cnode, 2612 CTLFLAG_READWRITE, CTLTYPE_BOOL, "txrx_workqueue", 2613 SYSCTL_DESCR("Use workqueue for packet processing"), 2614 NULL, 0, &sc->txrx_use_workqueue, 0, CTL_CREATE, CTL_EOL) 2615 != 0) 2616 aprint_error_dev(dev, "could not create sysctl\n"); 2617 } 2618 2619 /************************************************************************ 2620 * ixv_add_stats_sysctls - Add statistic sysctls for the VF. 2621 ************************************************************************/ 2622 static void 2623 ixv_add_stats_sysctls(struct ixgbe_softc *sc) 2624 { 2625 device_t dev = sc->dev; 2626 struct tx_ring *txr = sc->tx_rings; 2627 struct rx_ring *rxr = sc->rx_rings; 2628 struct ixgbevf_hw_stats *stats = &sc->stats.vf; 2629 struct ixgbe_hw *hw = &sc->hw; 2630 const struct sysctlnode *rnode, *cnode; 2631 struct sysctllog **log = &sc->sysctllog; 2632 const char *xname = device_xname(dev); 2633 2634 /* Driver Statistics */ 2635 evcnt_attach_dynamic(&sc->efbig_tx_dma_setup, EVCNT_TYPE_MISC, 2636 NULL, xname, "Driver tx dma soft fail EFBIG"); 2637 evcnt_attach_dynamic(&sc->mbuf_defrag_failed, EVCNT_TYPE_MISC, 2638 NULL, xname, "m_defrag() failed"); 2639 evcnt_attach_dynamic(&sc->efbig2_tx_dma_setup, EVCNT_TYPE_MISC, 2640 NULL, xname, "Driver tx dma hard fail EFBIG"); 2641 evcnt_attach_dynamic(&sc->einval_tx_dma_setup, EVCNT_TYPE_MISC, 2642 NULL, xname, "Driver tx dma hard fail EINVAL"); 2643 evcnt_attach_dynamic(&sc->other_tx_dma_setup, EVCNT_TYPE_MISC, 2644 NULL, xname, "Driver tx dma hard fail other"); 2645 evcnt_attach_dynamic(&sc->eagain_tx_dma_setup, EVCNT_TYPE_MISC, 2646 NULL, xname, "Driver tx dma soft fail EAGAIN"); 2647 evcnt_attach_dynamic(&sc->enomem_tx_dma_setup, EVCNT_TYPE_MISC, 2648 NULL, xname, "Driver tx dma soft fail ENOMEM"); 2649 evcnt_attach_dynamic(&sc->watchdog_events, EVCNT_TYPE_MISC, 2650 NULL, xname, "Watchdog timeouts"); 2651 evcnt_attach_dynamic(&sc->tso_err, EVCNT_TYPE_MISC, 2652 NULL, xname, "TSO errors"); 2653 evcnt_attach_dynamic(&sc->admin_irqev, EVCNT_TYPE_INTR, 2654 NULL, xname, "Admin MSI-X IRQ Handled"); 2655 evcnt_attach_dynamic(&sc->link_workev, EVCNT_TYPE_INTR, 2656 NULL, xname, "Admin event"); 2657 2658 for (int i = 0; i < sc->num_queues; i++, rxr++, txr++) { 2659 #ifdef LRO 2660 struct lro_ctrl *lro = &rxr->lro; 2661 #endif 2662 2663 snprintf(sc->queues[i].evnamebuf, 2664 sizeof(sc->queues[i].evnamebuf), "%s q%d", xname, i); 2665 snprintf(sc->queues[i].namebuf, 2666 sizeof(sc->queues[i].namebuf), "q%d", i); 2667 2668 if ((rnode = ixv_sysctl_instance(sc)) == NULL) { 2669 aprint_error_dev(dev, 2670 "could not create sysctl root\n"); 2671 break; 2672 } 2673 2674 if (sysctl_createv(log, 0, &rnode, &rnode, 2675 0, CTLTYPE_NODE, 2676 sc->queues[i].namebuf, SYSCTL_DESCR("Queue Name"), 2677 NULL, 0, NULL, 0, CTL_CREATE, CTL_EOL) != 0) 2678 break; 2679 2680 if (sysctl_createv(log, 0, &rnode, &cnode, 2681 CTLFLAG_READWRITE, CTLTYPE_INT, 2682 "interrupt_rate", SYSCTL_DESCR("Interrupt Rate"), 2683 ixv_sysctl_interrupt_rate_handler, 0, 2684 (void *)&sc->queues[i], 0, CTL_CREATE, CTL_EOL) != 0) 2685 break; 2686 2687 if (sysctl_createv(log, 0, &rnode, &cnode, 2688 CTLFLAG_READONLY, CTLTYPE_INT, 2689 "txd_head", SYSCTL_DESCR("Transmit Descriptor Head"), 2690 ixv_sysctl_tdh_handler, 0, (void *)txr, 2691 0, CTL_CREATE, CTL_EOL) != 0) 2692 break; 2693 2694 if (sysctl_createv(log, 0, &rnode, &cnode, 2695 CTLFLAG_READONLY, CTLTYPE_INT, 2696 "txd_tail", SYSCTL_DESCR("Transmit Descriptor Tail"), 2697 ixv_sysctl_tdt_handler, 0, (void *)txr, 2698 0, CTL_CREATE, CTL_EOL) != 0) 2699 break; 2700 2701 if (sysctl_createv(log, 0, &rnode, &cnode, 2702 CTLFLAG_READONLY, CTLTYPE_INT, "rxd_nxck", 2703 SYSCTL_DESCR("Receive Descriptor next to check"), 2704 ixv_sysctl_next_to_check_handler, 0, (void *)rxr, 0, 2705 CTL_CREATE, CTL_EOL) != 0) 2706 break; 2707 2708 if (sysctl_createv(log, 0, &rnode, &cnode, 2709 CTLFLAG_READONLY, CTLTYPE_INT, "rxd_nxrf", 2710 SYSCTL_DESCR("Receive Descriptor next to refresh"), 2711 ixv_sysctl_next_to_refresh_handler, 0, (void *)rxr, 0, 2712 CTL_CREATE, CTL_EOL) != 0) 2713 break; 2714 2715 if (sysctl_createv(log, 0, &rnode, &cnode, 2716 CTLFLAG_READONLY, CTLTYPE_INT, "rxd_head", 2717 SYSCTL_DESCR("Receive Descriptor Head"), 2718 ixv_sysctl_rdh_handler, 0, (void *)rxr, 0, 2719 CTL_CREATE, CTL_EOL) != 0) 2720 break; 2721 2722 if (sysctl_createv(log, 0, &rnode, &cnode, 2723 CTLFLAG_READONLY, CTLTYPE_INT, "rxd_tail", 2724 SYSCTL_DESCR("Receive Descriptor Tail"), 2725 ixv_sysctl_rdt_handler, 0, (void *)rxr, 0, 2726 CTL_CREATE, CTL_EOL) != 0) 2727 break; 2728 2729 evcnt_attach_dynamic(&sc->queues[i].irqs, EVCNT_TYPE_INTR, 2730 NULL, sc->queues[i].evnamebuf, "IRQs on queue"); 2731 evcnt_attach_dynamic(&sc->queues[i].handleq, 2732 EVCNT_TYPE_MISC, NULL, sc->queues[i].evnamebuf, 2733 "Handled queue in softint"); 2734 evcnt_attach_dynamic(&sc->queues[i].req, EVCNT_TYPE_MISC, 2735 NULL, sc->queues[i].evnamebuf, "Requeued in softint"); 2736 evcnt_attach_dynamic(&txr->total_packets, EVCNT_TYPE_MISC, 2737 NULL, sc->queues[i].evnamebuf, 2738 "Queue Packets Transmitted"); 2739 #ifndef IXGBE_LEGACY_TX 2740 evcnt_attach_dynamic(&txr->pcq_drops, EVCNT_TYPE_MISC, 2741 NULL, sc->queues[i].evnamebuf, 2742 "Packets dropped in pcq"); 2743 #endif 2744 evcnt_attach_dynamic(&txr->no_desc_avail, EVCNT_TYPE_MISC, 2745 NULL, sc->queues[i].evnamebuf, 2746 "TX Queue No Descriptor Available"); 2747 evcnt_attach_dynamic(&txr->tso_tx, EVCNT_TYPE_MISC, 2748 NULL, sc->queues[i].evnamebuf, "TSO"); 2749 2750 evcnt_attach_dynamic(&rxr->rx_bytes, EVCNT_TYPE_MISC, 2751 NULL, sc->queues[i].evnamebuf, 2752 "Queue Bytes Received"); 2753 evcnt_attach_dynamic(&rxr->rx_packets, EVCNT_TYPE_MISC, 2754 NULL, sc->queues[i].evnamebuf, 2755 "Queue Packets Received"); 2756 evcnt_attach_dynamic(&rxr->no_mbuf, EVCNT_TYPE_MISC, 2757 NULL, sc->queues[i].evnamebuf, "Rx no mbuf"); 2758 evcnt_attach_dynamic(&rxr->rx_discarded, EVCNT_TYPE_MISC, 2759 NULL, sc->queues[i].evnamebuf, "Rx discarded"); 2760 evcnt_attach_dynamic(&rxr->rx_copies, EVCNT_TYPE_MISC, 2761 NULL, sc->queues[i].evnamebuf, "Copied RX Frames"); 2762 #ifdef LRO 2763 SYSCTL_ADD_INT(ctx, queue_list, OID_AUTO, "lro_queued", 2764 CTLFLAG_RD, &lro->lro_queued, 0, 2765 "LRO Queued"); 2766 SYSCTL_ADD_INT(ctx, queue_list, OID_AUTO, "lro_flushed", 2767 CTLFLAG_RD, &lro->lro_flushed, 0, 2768 "LRO Flushed"); 2769 #endif /* LRO */ 2770 } 2771 2772 /* MAC stats get their own sub node */ 2773 2774 snprintf(stats->namebuf, 2775 sizeof(stats->namebuf), "%s MAC Statistics", xname); 2776 2777 evcnt_attach_dynamic(&stats->ipcs, EVCNT_TYPE_MISC, NULL, 2778 stats->namebuf, "rx csum offload - IP"); 2779 evcnt_attach_dynamic(&stats->l4cs, EVCNT_TYPE_MISC, NULL, 2780 stats->namebuf, "rx csum offload - L4"); 2781 evcnt_attach_dynamic(&stats->ipcs_bad, EVCNT_TYPE_MISC, NULL, 2782 stats->namebuf, "rx csum offload - IP bad"); 2783 evcnt_attach_dynamic(&stats->l4cs_bad, EVCNT_TYPE_MISC, NULL, 2784 stats->namebuf, "rx csum offload - L4 bad"); 2785 2786 /* Packet Reception Stats */ 2787 evcnt_attach_dynamic(&stats->vfgprc, EVCNT_TYPE_MISC, NULL, 2788 xname, "Good Packets Received"); 2789 evcnt_attach_dynamic(&stats->vfgorc, EVCNT_TYPE_MISC, NULL, 2790 xname, "Good Octets Received"); 2791 evcnt_attach_dynamic(&stats->vfmprc, EVCNT_TYPE_MISC, NULL, 2792 xname, "Multicast Packets Received"); 2793 evcnt_attach_dynamic(&stats->vfgptc, EVCNT_TYPE_MISC, NULL, 2794 xname, "Good Packets Transmitted"); 2795 evcnt_attach_dynamic(&stats->vfgotc, EVCNT_TYPE_MISC, NULL, 2796 xname, "Good Octets Transmitted"); 2797 2798 /* Mailbox Stats */ 2799 evcnt_attach_dynamic(&hw->mbx.stats.msgs_tx, EVCNT_TYPE_MISC, NULL, 2800 xname, "message TXs"); 2801 evcnt_attach_dynamic(&hw->mbx.stats.msgs_rx, EVCNT_TYPE_MISC, NULL, 2802 xname, "message RXs"); 2803 evcnt_attach_dynamic(&hw->mbx.stats.acks, EVCNT_TYPE_MISC, NULL, 2804 xname, "ACKs"); 2805 evcnt_attach_dynamic(&hw->mbx.stats.reqs, EVCNT_TYPE_MISC, NULL, 2806 xname, "REQs"); 2807 evcnt_attach_dynamic(&hw->mbx.stats.rsts, EVCNT_TYPE_MISC, NULL, 2808 xname, "RSTs"); 2809 2810 } /* ixv_add_stats_sysctls */ 2811 2812 static void 2813 ixv_clear_evcnt(struct ixgbe_softc *sc) 2814 { 2815 struct tx_ring *txr = sc->tx_rings; 2816 struct rx_ring *rxr = sc->rx_rings; 2817 struct ixgbevf_hw_stats *stats = &sc->stats.vf; 2818 struct ixgbe_hw *hw = &sc->hw; 2819 int i; 2820 2821 /* Driver Statistics */ 2822 IXGBE_EVC_STORE(&sc->efbig_tx_dma_setup, 0); 2823 IXGBE_EVC_STORE(&sc->mbuf_defrag_failed, 0); 2824 IXGBE_EVC_STORE(&sc->efbig2_tx_dma_setup, 0); 2825 IXGBE_EVC_STORE(&sc->einval_tx_dma_setup, 0); 2826 IXGBE_EVC_STORE(&sc->other_tx_dma_setup, 0); 2827 IXGBE_EVC_STORE(&sc->eagain_tx_dma_setup, 0); 2828 IXGBE_EVC_STORE(&sc->enomem_tx_dma_setup, 0); 2829 IXGBE_EVC_STORE(&sc->watchdog_events, 0); 2830 IXGBE_EVC_STORE(&sc->tso_err, 0); 2831 IXGBE_EVC_STORE(&sc->admin_irqev, 0); 2832 IXGBE_EVC_STORE(&sc->link_workev, 0); 2833 2834 for (i = 0; i < sc->num_queues; i++, rxr++, txr++) { 2835 IXGBE_EVC_STORE(&sc->queues[i].irqs, 0); 2836 IXGBE_EVC_STORE(&sc->queues[i].handleq, 0); 2837 IXGBE_EVC_STORE(&sc->queues[i].req, 0); 2838 IXGBE_EVC_STORE(&txr->total_packets, 0); 2839 #ifndef IXGBE_LEGACY_TX 2840 IXGBE_EVC_STORE(&txr->pcq_drops, 0); 2841 #endif 2842 IXGBE_EVC_STORE(&txr->no_desc_avail, 0); 2843 IXGBE_EVC_STORE(&txr->tso_tx, 0); 2844 txr->q_efbig_tx_dma_setup = 0; 2845 txr->q_mbuf_defrag_failed = 0; 2846 txr->q_efbig2_tx_dma_setup = 0; 2847 txr->q_einval_tx_dma_setup = 0; 2848 txr->q_other_tx_dma_setup = 0; 2849 txr->q_eagain_tx_dma_setup = 0; 2850 txr->q_enomem_tx_dma_setup = 0; 2851 txr->q_tso_err = 0; 2852 2853 IXGBE_EVC_STORE(&rxr->rx_packets, 0); 2854 IXGBE_EVC_STORE(&rxr->rx_bytes, 0); 2855 IXGBE_EVC_STORE(&rxr->rx_copies, 0); 2856 IXGBE_EVC_STORE(&rxr->no_mbuf, 0); 2857 IXGBE_EVC_STORE(&rxr->rx_discarded, 0); 2858 } 2859 2860 /* MAC stats get their own sub node */ 2861 2862 IXGBE_EVC_STORE(&stats->ipcs, 0); 2863 IXGBE_EVC_STORE(&stats->l4cs, 0); 2864 IXGBE_EVC_STORE(&stats->ipcs_bad, 0); 2865 IXGBE_EVC_STORE(&stats->l4cs_bad, 0); 2866 2867 /* 2868 * Packet Reception Stats. 2869 * Call ixv_init_stats() to save last VF counters' values. 2870 */ 2871 ixv_init_stats(sc); 2872 IXGBE_EVC_STORE(&stats->vfgprc, 0); 2873 IXGBE_EVC_STORE(&stats->vfgorc, 0); 2874 IXGBE_EVC_STORE(&stats->vfmprc, 0); 2875 IXGBE_EVC_STORE(&stats->vfgptc, 0); 2876 IXGBE_EVC_STORE(&stats->vfgotc, 0); 2877 2878 /* Mailbox Stats */ 2879 IXGBE_EVC_STORE(&hw->mbx.stats.msgs_tx, 0); 2880 IXGBE_EVC_STORE(&hw->mbx.stats.msgs_rx, 0); 2881 IXGBE_EVC_STORE(&hw->mbx.stats.acks, 0); 2882 IXGBE_EVC_STORE(&hw->mbx.stats.reqs, 0); 2883 IXGBE_EVC_STORE(&hw->mbx.stats.rsts, 0); 2884 2885 } /* ixv_clear_evcnt */ 2886 2887 #define PRINTQS(sc, regname) \ 2888 do { \ 2889 struct ixgbe_hw *_hw = &(sc)->hw; \ 2890 int _i; \ 2891 \ 2892 printf("%s: %s", device_xname((sc)->dev), #regname); \ 2893 for (_i = 0; _i < (sc)->num_queues; _i++) { \ 2894 printf((_i == 0) ? "\t" : " "); \ 2895 printf("%08x", IXGBE_READ_REG(_hw, \ 2896 IXGBE_##regname(_i))); \ 2897 } \ 2898 printf("\n"); \ 2899 } while (0) 2900 2901 /************************************************************************ 2902 * ixv_print_debug_info 2903 * 2904 * Provides a way to take a look at important statistics 2905 * maintained by the driver and hardware. 2906 ************************************************************************/ 2907 static void 2908 ixv_print_debug_info(struct ixgbe_softc *sc) 2909 { 2910 device_t dev = sc->dev; 2911 struct ixgbe_hw *hw = &sc->hw; 2912 int i; 2913 2914 device_printf(dev, "queue:"); 2915 for (i = 0; i < sc->num_queues; i++) { 2916 printf((i == 0) ? "\t" : " "); 2917 printf("%8d", i); 2918 } 2919 printf("\n"); 2920 PRINTQS(sc, VFRDBAL); 2921 PRINTQS(sc, VFRDBAH); 2922 PRINTQS(sc, VFRDLEN); 2923 PRINTQS(sc, VFSRRCTL); 2924 PRINTQS(sc, VFRDH); 2925 PRINTQS(sc, VFRDT); 2926 PRINTQS(sc, VFRXDCTL); 2927 2928 device_printf(dev, "EIMS:\t%08x\n", IXGBE_READ_REG(hw, IXGBE_VTEIMS)); 2929 device_printf(dev, "EIAM:\t%08x\n", IXGBE_READ_REG(hw, IXGBE_VTEIAM)); 2930 device_printf(dev, "EIAC:\t%08x\n", IXGBE_READ_REG(hw, IXGBE_VTEIAC)); 2931 } /* ixv_print_debug_info */ 2932 2933 /************************************************************************ 2934 * ixv_sysctl_debug 2935 ************************************************************************/ 2936 static int 2937 ixv_sysctl_debug(SYSCTLFN_ARGS) 2938 { 2939 struct sysctlnode node = *rnode; 2940 struct ixgbe_softc *sc = (struct ixgbe_softc *)node.sysctl_data; 2941 int error, result = 0; 2942 2943 node.sysctl_data = &result; 2944 error = sysctl_lookup(SYSCTLFN_CALL(&node)); 2945 2946 if (error || newp == NULL) 2947 return error; 2948 2949 if (result == 1) 2950 ixv_print_debug_info(sc); 2951 2952 return 0; 2953 } /* ixv_sysctl_debug */ 2954 2955 /************************************************************************ 2956 * ixv_sysctl_rx_copy_len 2957 ************************************************************************/ 2958 static int 2959 ixv_sysctl_rx_copy_len(SYSCTLFN_ARGS) 2960 { 2961 struct sysctlnode node = *rnode; 2962 struct ixgbe_softc *sc = (struct ixgbe_softc *)node.sysctl_data; 2963 int error; 2964 int result = sc->rx_copy_len; 2965 2966 node.sysctl_data = &result; 2967 error = sysctl_lookup(SYSCTLFN_CALL(&node)); 2968 2969 if (error || newp == NULL) 2970 return error; 2971 2972 if ((result < 0) || (result > IXGBE_RX_COPY_LEN_MAX)) 2973 return EINVAL; 2974 2975 sc->rx_copy_len = result; 2976 2977 return 0; 2978 } /* ixv_sysctl_rx_copy_len */ 2979 2980 /************************************************************************ 2981 * ixv_sysctl_tx_process_limit 2982 ************************************************************************/ 2983 static int 2984 ixv_sysctl_tx_process_limit(SYSCTLFN_ARGS) 2985 { 2986 struct sysctlnode node = *rnode; 2987 struct ixgbe_softc *sc = (struct ixgbe_softc *)node.sysctl_data; 2988 int error; 2989 int result = sc->tx_process_limit; 2990 2991 node.sysctl_data = &result; 2992 error = sysctl_lookup(SYSCTLFN_CALL(&node)); 2993 2994 if (error || newp == NULL) 2995 return error; 2996 2997 if ((result <= 0) || (result > sc->num_tx_desc)) 2998 return EINVAL; 2999 3000 sc->tx_process_limit = result; 3001 3002 return 0; 3003 } /* ixv_sysctl_tx_process_limit */ 3004 3005 /************************************************************************ 3006 * ixv_sysctl_rx_process_limit 3007 ************************************************************************/ 3008 static int 3009 ixv_sysctl_rx_process_limit(SYSCTLFN_ARGS) 3010 { 3011 struct sysctlnode node = *rnode; 3012 struct ixgbe_softc *sc = (struct ixgbe_softc *)node.sysctl_data; 3013 int error; 3014 int result = sc->rx_process_limit; 3015 3016 node.sysctl_data = &result; 3017 error = sysctl_lookup(SYSCTLFN_CALL(&node)); 3018 3019 if (error || newp == NULL) 3020 return error; 3021 3022 if ((result <= 0) || (result > sc->num_rx_desc)) 3023 return EINVAL; 3024 3025 sc->rx_process_limit = result; 3026 3027 return 0; 3028 } /* ixv_sysctl_rx_process_limit */ 3029 3030 /************************************************************************ 3031 * ixv_init_device_features 3032 ************************************************************************/ 3033 static void 3034 ixv_init_device_features(struct ixgbe_softc *sc) 3035 { 3036 sc->feat_cap = IXGBE_FEATURE_NETMAP 3037 | IXGBE_FEATURE_VF 3038 | IXGBE_FEATURE_RSS 3039 | IXGBE_FEATURE_LEGACY_TX; 3040 3041 /* A tad short on feature flags for VFs, atm. */ 3042 switch (sc->hw.mac.type) { 3043 case ixgbe_mac_82599_vf: 3044 break; 3045 case ixgbe_mac_X540_vf: 3046 break; 3047 case ixgbe_mac_X550_vf: 3048 case ixgbe_mac_X550EM_x_vf: 3049 case ixgbe_mac_X550EM_a_vf: 3050 sc->feat_cap |= IXGBE_FEATURE_NEEDS_CTXD; 3051 break; 3052 default: 3053 break; 3054 } 3055 3056 /* Enabled by default... */ 3057 /* Is a virtual function (VF) */ 3058 if (sc->feat_cap & IXGBE_FEATURE_VF) 3059 sc->feat_en |= IXGBE_FEATURE_VF; 3060 /* Netmap */ 3061 if (sc->feat_cap & IXGBE_FEATURE_NETMAP) 3062 sc->feat_en |= IXGBE_FEATURE_NETMAP; 3063 /* Receive-Side Scaling (RSS) */ 3064 if (sc->feat_cap & IXGBE_FEATURE_RSS) 3065 sc->feat_en |= IXGBE_FEATURE_RSS; 3066 /* Needs advanced context descriptor regardless of offloads req'd */ 3067 if (sc->feat_cap & IXGBE_FEATURE_NEEDS_CTXD) 3068 sc->feat_en |= IXGBE_FEATURE_NEEDS_CTXD; 3069 3070 /* Enabled via sysctl... */ 3071 /* Legacy (single queue) transmit */ 3072 if ((sc->feat_cap & IXGBE_FEATURE_LEGACY_TX) && 3073 ixv_enable_legacy_tx) 3074 sc->feat_en |= IXGBE_FEATURE_LEGACY_TX; 3075 } /* ixv_init_device_features */ 3076 3077 /************************************************************************ 3078 * ixv_shutdown - Shutdown entry point 3079 ************************************************************************/ 3080 #if 0 /* XXX NetBSD ought to register something like this through pmf(9) */ 3081 static int 3082 ixv_shutdown(device_t dev) 3083 { 3084 struct ixgbe_softc *sc = device_private(dev); 3085 IXGBE_CORE_LOCK(sc); 3086 ixv_stop_locked(sc); 3087 IXGBE_CORE_UNLOCK(sc); 3088 3089 return (0); 3090 } /* ixv_shutdown */ 3091 #endif 3092 3093 static int 3094 ixv_ifflags_cb(struct ethercom *ec) 3095 { 3096 struct ifnet *ifp = &ec->ec_if; 3097 struct ixgbe_softc *sc = ifp->if_softc; 3098 u_short saved_flags; 3099 u_short change; 3100 int rv = 0; 3101 3102 IXGBE_CORE_LOCK(sc); 3103 3104 saved_flags = sc->if_flags; 3105 change = ifp->if_flags ^ sc->if_flags; 3106 if (change != 0) 3107 sc->if_flags = ifp->if_flags; 3108 3109 if ((change & ~(IFF_CANTCHANGE | IFF_DEBUG)) != 0) { 3110 rv = ENETRESET; 3111 goto out; 3112 } else if ((change & IFF_PROMISC) != 0) { 3113 rv = ixv_set_rxfilter(sc); 3114 if (rv != 0) { 3115 /* Restore previous */ 3116 sc->if_flags = saved_flags; 3117 goto out; 3118 } 3119 } 3120 3121 /* Check for ec_capenable. */ 3122 change = ec->ec_capenable ^ sc->ec_capenable; 3123 sc->ec_capenable = ec->ec_capenable; 3124 if ((change & ~(ETHERCAP_VLAN_MTU | ETHERCAP_VLAN_HWTAGGING 3125 | ETHERCAP_VLAN_HWFILTER)) != 0) { 3126 rv = ENETRESET; 3127 goto out; 3128 } 3129 3130 /* 3131 * Special handling is not required for ETHERCAP_VLAN_MTU. 3132 * PF's MAXFRS(MHADD) does not include the 4bytes of the VLAN header. 3133 */ 3134 3135 /* Set up VLAN support and filter */ 3136 if ((change & (ETHERCAP_VLAN_HWTAGGING | ETHERCAP_VLAN_HWFILTER)) != 0) 3137 rv = ixv_setup_vlan_support(sc); 3138 3139 out: 3140 IXGBE_CORE_UNLOCK(sc); 3141 3142 return rv; 3143 } 3144 3145 3146 /************************************************************************ 3147 * ixv_ioctl - Ioctl entry point 3148 * 3149 * Called when the user wants to configure the interface. 3150 * 3151 * return 0 on success, positive on failure 3152 ************************************************************************/ 3153 static int 3154 ixv_ioctl(struct ifnet *ifp, u_long command, void *data) 3155 { 3156 struct ixgbe_softc *sc = ifp->if_softc; 3157 struct ixgbe_hw *hw = &sc->hw; 3158 struct ifcapreq *ifcr = data; 3159 int error; 3160 int l4csum_en; 3161 const int l4csum = IFCAP_CSUM_TCPv4_Rx | IFCAP_CSUM_UDPv4_Rx | 3162 IFCAP_CSUM_TCPv6_Rx | IFCAP_CSUM_UDPv6_Rx; 3163 3164 switch (command) { 3165 case SIOCSIFFLAGS: 3166 IOCTL_DEBUGOUT("ioctl: SIOCSIFFLAGS (Set Interface Flags)"); 3167 break; 3168 case SIOCADDMULTI: { 3169 struct ether_multi *enm; 3170 struct ether_multistep step; 3171 struct ethercom *ec = &sc->osdep.ec; 3172 bool overflow = false; 3173 int mcnt = 0; 3174 3175 /* 3176 * Check the number of multicast address. If it exceeds, 3177 * return ENOSPC. 3178 * Update this code when we support API 1.3. 3179 */ 3180 ETHER_LOCK(ec); 3181 ETHER_FIRST_MULTI(step, ec, enm); 3182 while (enm != NULL) { 3183 mcnt++; 3184 3185 /* 3186 * This code is before adding, so one room is required 3187 * at least. 3188 */ 3189 if (mcnt > (IXGBE_MAX_VF_MC - 1)) { 3190 overflow = true; 3191 break; 3192 } 3193 ETHER_NEXT_MULTI(step, enm); 3194 } 3195 ETHER_UNLOCK(ec); 3196 error = 0; 3197 if (overflow && ((ec->ec_flags & ETHER_F_ALLMULTI) == 0)) { 3198 error = hw->mac.ops.update_xcast_mode(hw, 3199 IXGBEVF_XCAST_MODE_ALLMULTI); 3200 if (error == IXGBE_ERR_NOT_TRUSTED) { 3201 device_printf(sc->dev, 3202 "this interface is not trusted\n"); 3203 error = EPERM; 3204 } else if (error == IXGBE_ERR_FEATURE_NOT_SUPPORTED) { 3205 device_printf(sc->dev, 3206 "the PF doesn't support allmulti mode\n"); 3207 error = EOPNOTSUPP; 3208 } else if (error) { 3209 device_printf(sc->dev, 3210 "number of Ethernet multicast addresses " 3211 "exceeds the limit (%d). error = %d\n", 3212 IXGBE_MAX_VF_MC, error); 3213 error = ENOSPC; 3214 } else 3215 ec->ec_flags |= ETHER_F_ALLMULTI; 3216 } 3217 if (error) 3218 return error; 3219 } 3220 /*FALLTHROUGH*/ 3221 case SIOCDELMULTI: 3222 IOCTL_DEBUGOUT("ioctl: SIOC(ADD|DEL)MULTI"); 3223 break; 3224 case SIOCSIFMEDIA: 3225 case SIOCGIFMEDIA: 3226 IOCTL_DEBUGOUT("ioctl: SIOCxIFMEDIA (Get/Set Interface Media)"); 3227 break; 3228 case SIOCSIFCAP: 3229 IOCTL_DEBUGOUT("ioctl: SIOCSIFCAP (Set Capabilities)"); 3230 break; 3231 case SIOCSIFMTU: 3232 IOCTL_DEBUGOUT("ioctl: SIOCSIFMTU (Set Interface MTU)"); 3233 break; 3234 case SIOCZIFDATA: 3235 IOCTL_DEBUGOUT("ioctl: SIOCZIFDATA (Zero counter)"); 3236 ixv_update_stats(sc); 3237 ixv_clear_evcnt(sc); 3238 break; 3239 default: 3240 IOCTL_DEBUGOUT1("ioctl: UNKNOWN (0x%X)", (int)command); 3241 break; 3242 } 3243 3244 switch (command) { 3245 case SIOCSIFCAP: 3246 /* Layer-4 Rx checksum offload has to be turned on and 3247 * off as a unit. 3248 */ 3249 l4csum_en = ifcr->ifcr_capenable & l4csum; 3250 if (l4csum_en != l4csum && l4csum_en != 0) 3251 return EINVAL; 3252 /*FALLTHROUGH*/ 3253 case SIOCADDMULTI: 3254 case SIOCDELMULTI: 3255 case SIOCSIFFLAGS: 3256 case SIOCSIFMTU: 3257 default: 3258 if ((error = ether_ioctl(ifp, command, data)) != ENETRESET) 3259 return error; 3260 if ((ifp->if_flags & IFF_RUNNING) == 0) 3261 ; 3262 else if (command == SIOCSIFCAP || command == SIOCSIFMTU) { 3263 IXGBE_CORE_LOCK(sc); 3264 ixv_init_locked(sc); 3265 IXGBE_CORE_UNLOCK(sc); 3266 } else if (command == SIOCADDMULTI || command == SIOCDELMULTI) { 3267 /* 3268 * Multicast list has changed; set the hardware filter 3269 * accordingly. 3270 */ 3271 IXGBE_CORE_LOCK(sc); 3272 ixv_disable_intr(sc); 3273 ixv_set_rxfilter(sc); 3274 ixv_enable_intr(sc); 3275 IXGBE_CORE_UNLOCK(sc); 3276 } 3277 return 0; 3278 } 3279 } /* ixv_ioctl */ 3280 3281 /************************************************************************ 3282 * ixv_init 3283 ************************************************************************/ 3284 static int 3285 ixv_init(struct ifnet *ifp) 3286 { 3287 struct ixgbe_softc *sc = ifp->if_softc; 3288 3289 IXGBE_CORE_LOCK(sc); 3290 ixv_init_locked(sc); 3291 IXGBE_CORE_UNLOCK(sc); 3292 3293 return 0; 3294 } /* ixv_init */ 3295 3296 /************************************************************************ 3297 * ixv_handle_que 3298 ************************************************************************/ 3299 static void 3300 ixv_handle_que(void *context) 3301 { 3302 struct ix_queue *que = context; 3303 struct ixgbe_softc *sc = que->sc; 3304 struct tx_ring *txr = que->txr; 3305 struct ifnet *ifp = sc->ifp; 3306 bool more; 3307 3308 IXGBE_EVC_ADD(&que->handleq, 1); 3309 3310 if (ifp->if_flags & IFF_RUNNING) { 3311 IXGBE_TX_LOCK(txr); 3312 more = ixgbe_txeof(txr); 3313 if (!(sc->feat_en & IXGBE_FEATURE_LEGACY_TX)) 3314 if (!ixgbe_mq_ring_empty(ifp, txr->txr_interq)) 3315 ixgbe_mq_start_locked(ifp, txr); 3316 /* Only for queue 0 */ 3317 /* NetBSD still needs this for CBQ */ 3318 if ((&sc->queues[0] == que) 3319 && (!ixgbe_legacy_ring_empty(ifp, NULL))) 3320 ixgbe_legacy_start_locked(ifp, txr); 3321 IXGBE_TX_UNLOCK(txr); 3322 more |= ixgbe_rxeof(que); 3323 if (more) { 3324 IXGBE_EVC_ADD(&que->req, 1); 3325 if (sc->txrx_use_workqueue) { 3326 /* 3327 * "enqueued flag" is not required here 3328 * the same as ixg(4). See ixgbe_msix_que(). 3329 */ 3330 workqueue_enqueue(sc->que_wq, 3331 &que->wq_cookie, curcpu()); 3332 } else 3333 softint_schedule(que->que_si); 3334 return; 3335 } 3336 } 3337 3338 /* Re-enable this interrupt */ 3339 ixv_enable_queue(sc, que->msix); 3340 3341 return; 3342 } /* ixv_handle_que */ 3343 3344 /************************************************************************ 3345 * ixv_handle_que_work 3346 ************************************************************************/ 3347 static void 3348 ixv_handle_que_work(struct work *wk, void *context) 3349 { 3350 struct ix_queue *que = container_of(wk, struct ix_queue, wq_cookie); 3351 3352 /* 3353 * "enqueued flag" is not required here the same as ixg(4). 3354 * See ixgbe_msix_que(). 3355 */ 3356 ixv_handle_que(que); 3357 } 3358 3359 /************************************************************************ 3360 * ixv_allocate_msix - Setup MSI-X Interrupt resources and handlers 3361 ************************************************************************/ 3362 static int 3363 ixv_allocate_msix(struct ixgbe_softc *sc, const struct pci_attach_args *pa) 3364 { 3365 device_t dev = sc->dev; 3366 struct ix_queue *que = sc->queues; 3367 struct tx_ring *txr = sc->tx_rings; 3368 int error, msix_ctrl, rid, vector = 0; 3369 pci_chipset_tag_t pc; 3370 pcitag_t tag; 3371 char intrbuf[PCI_INTRSTR_LEN]; 3372 char wqname[MAXCOMLEN]; 3373 char intr_xname[32]; 3374 const char *intrstr = NULL; 3375 kcpuset_t *affinity; 3376 int cpu_id = 0; 3377 3378 pc = sc->osdep.pc; 3379 tag = sc->osdep.tag; 3380 3381 sc->osdep.nintrs = sc->num_queues + 1; 3382 if (pci_msix_alloc_exact(pa, &sc->osdep.intrs, 3383 sc->osdep.nintrs) != 0) { 3384 aprint_error_dev(dev, 3385 "failed to allocate MSI-X interrupt\n"); 3386 return (ENXIO); 3387 } 3388 3389 kcpuset_create(&affinity, false); 3390 for (int i = 0; i < sc->num_queues; i++, vector++, que++, txr++) { 3391 snprintf(intr_xname, sizeof(intr_xname), "%s TXRX%d", 3392 device_xname(dev), i); 3393 intrstr = pci_intr_string(pc, sc->osdep.intrs[i], intrbuf, 3394 sizeof(intrbuf)); 3395 pci_intr_setattr(pc, &sc->osdep.intrs[i], PCI_INTR_MPSAFE, 3396 true); 3397 3398 /* Set the handler function */ 3399 que->res = sc->osdep.ihs[i] = pci_intr_establish_xname(pc, 3400 sc->osdep.intrs[i], IPL_NET, ixv_msix_que, que, 3401 intr_xname); 3402 if (que->res == NULL) { 3403 pci_intr_release(pc, sc->osdep.intrs, 3404 sc->osdep.nintrs); 3405 aprint_error_dev(dev, 3406 "Failed to register QUE handler\n"); 3407 kcpuset_destroy(affinity); 3408 return (ENXIO); 3409 } 3410 que->msix = vector; 3411 sc->active_queues |= (u64)(1 << que->msix); 3412 3413 cpu_id = i; 3414 /* Round-robin affinity */ 3415 kcpuset_zero(affinity); 3416 kcpuset_set(affinity, cpu_id % ncpu); 3417 error = interrupt_distribute(sc->osdep.ihs[i], affinity, NULL); 3418 aprint_normal_dev(dev, "for TX/RX, interrupting at %s", 3419 intrstr); 3420 if (error == 0) 3421 aprint_normal(", bound queue %d to cpu %d\n", 3422 i, cpu_id % ncpu); 3423 else 3424 aprint_normal("\n"); 3425 3426 #ifndef IXGBE_LEGACY_TX 3427 txr->txr_si 3428 = softint_establish(SOFTINT_NET | SOFTINT_MPSAFE, 3429 ixgbe_deferred_mq_start, txr); 3430 #endif 3431 que->que_si 3432 = softint_establish(SOFTINT_NET | SOFTINT_MPSAFE, 3433 ixv_handle_que, que); 3434 if (que->que_si == NULL) { 3435 aprint_error_dev(dev, 3436 "could not establish software interrupt\n"); 3437 } 3438 } 3439 snprintf(wqname, sizeof(wqname), "%sdeferTx", device_xname(dev)); 3440 error = workqueue_create(&sc->txr_wq, wqname, 3441 ixgbe_deferred_mq_start_work, sc, IXGBE_WORKQUEUE_PRI, IPL_NET, 3442 WQ_PERCPU | WQ_MPSAFE); 3443 if (error) { 3444 aprint_error_dev(dev, 3445 "couldn't create workqueue for deferred Tx\n"); 3446 } 3447 sc->txr_wq_enqueued = percpu_alloc(sizeof(u_int)); 3448 3449 snprintf(wqname, sizeof(wqname), "%sTxRx", device_xname(dev)); 3450 error = workqueue_create(&sc->que_wq, wqname, 3451 ixv_handle_que_work, sc, IXGBE_WORKQUEUE_PRI, IPL_NET, 3452 WQ_PERCPU | WQ_MPSAFE); 3453 if (error) { 3454 aprint_error_dev(dev, "couldn't create workqueue for Tx/Rx\n"); 3455 } 3456 3457 /* and Mailbox */ 3458 cpu_id++; 3459 snprintf(intr_xname, sizeof(intr_xname), "%s link", device_xname(dev)); 3460 sc->vector = vector; 3461 intrstr = pci_intr_string(pc, sc->osdep.intrs[vector], intrbuf, 3462 sizeof(intrbuf)); 3463 pci_intr_setattr(pc, &sc->osdep.intrs[vector], PCI_INTR_MPSAFE, true); 3464 3465 /* Set the mbx handler function */ 3466 sc->osdep.ihs[vector] = pci_intr_establish_xname(pc, 3467 sc->osdep.intrs[vector], IPL_NET, ixv_msix_mbx, sc, intr_xname); 3468 if (sc->osdep.ihs[vector] == NULL) { 3469 aprint_error_dev(dev, "Failed to register LINK handler\n"); 3470 kcpuset_destroy(affinity); 3471 return (ENXIO); 3472 } 3473 /* Round-robin affinity */ 3474 kcpuset_zero(affinity); 3475 kcpuset_set(affinity, cpu_id % ncpu); 3476 error = interrupt_distribute(sc->osdep.ihs[vector], affinity, 3477 NULL); 3478 3479 aprint_normal_dev(dev, 3480 "for link, interrupting at %s", intrstr); 3481 if (error == 0) 3482 aprint_normal(", affinity to cpu %d\n", cpu_id % ncpu); 3483 else 3484 aprint_normal("\n"); 3485 3486 /* Tasklets for Mailbox */ 3487 snprintf(wqname, sizeof(wqname), "%s-admin", device_xname(dev)); 3488 error = workqueue_create(&sc->admin_wq, wqname, 3489 ixv_handle_admin, sc, IXGBE_WORKQUEUE_PRI, IPL_NET, WQ_MPSAFE); 3490 if (error) { 3491 aprint_error_dev(dev, 3492 "could not create admin workqueue (%d)\n", error); 3493 goto err_out; 3494 } 3495 3496 /* 3497 * Due to a broken design QEMU will fail to properly 3498 * enable the guest for MSI-X unless the vectors in 3499 * the table are all set up, so we must rewrite the 3500 * ENABLE in the MSI-X control register again at this 3501 * point to cause it to successfully initialize us. 3502 */ 3503 if (sc->hw.mac.type == ixgbe_mac_82599_vf) { 3504 pci_get_capability(pc, tag, PCI_CAP_MSIX, &rid, NULL); 3505 rid += PCI_MSIX_CTL; 3506 msix_ctrl = pci_conf_read(pc, tag, rid); 3507 msix_ctrl |= PCI_MSIX_CTL_ENABLE; 3508 pci_conf_write(pc, tag, rid, msix_ctrl); 3509 } 3510 3511 kcpuset_destroy(affinity); 3512 return (0); 3513 err_out: 3514 kcpuset_destroy(affinity); 3515 ixv_free_deferred_handlers(sc); 3516 ixv_free_pci_resources(sc); 3517 return (error); 3518 } /* ixv_allocate_msix */ 3519 3520 /************************************************************************ 3521 * ixv_configure_interrupts - Setup MSI-X resources 3522 * 3523 * Note: The VF device MUST use MSI-X, there is no fallback. 3524 ************************************************************************/ 3525 static int 3526 ixv_configure_interrupts(struct ixgbe_softc *sc) 3527 { 3528 device_t dev = sc->dev; 3529 int want, queues, msgs; 3530 3531 /* Must have at least 2 MSI-X vectors */ 3532 msgs = pci_msix_count(sc->osdep.pc, sc->osdep.tag); 3533 if (msgs < 2) { 3534 aprint_error_dev(dev, "MSIX config error\n"); 3535 return (ENXIO); 3536 } 3537 msgs = MIN(msgs, IXG_MAX_NINTR); 3538 3539 /* Figure out a reasonable auto config value */ 3540 queues = (ncpu > (msgs - 1)) ? (msgs - 1) : ncpu; 3541 3542 if (ixv_num_queues != 0) 3543 queues = ixv_num_queues; 3544 else if ((ixv_num_queues == 0) && (queues > IXGBE_VF_MAX_TX_QUEUES)) 3545 queues = IXGBE_VF_MAX_TX_QUEUES; 3546 3547 /* 3548 * Want vectors for the queues, 3549 * plus an additional for mailbox. 3550 */ 3551 want = queues + 1; 3552 if (msgs >= want) 3553 msgs = want; 3554 else { 3555 aprint_error_dev(dev, 3556 "MSI-X Configuration Problem, " 3557 "%d vectors but %d queues wanted!\n", msgs, want); 3558 return -1; 3559 } 3560 3561 aprint_normal_dev(dev, 3562 "Using MSI-X interrupts with %d vectors\n", msgs); 3563 sc->num_queues = queues; 3564 3565 return (0); 3566 } /* ixv_configure_interrupts */ 3567 3568 3569 /************************************************************************ 3570 * ixv_handle_admin - Tasklet handler for MSI-X MBX interrupts 3571 * 3572 * Done outside of interrupt context since the driver might sleep 3573 ************************************************************************/ 3574 static void 3575 ixv_handle_admin(struct work *wk, void *context) 3576 { 3577 struct ixgbe_softc *sc = context; 3578 struct ixgbe_hw *hw = &sc->hw; 3579 3580 IXGBE_CORE_LOCK(sc); 3581 3582 IXGBE_EVC_ADD(&sc->link_workev, 1); 3583 sc->hw.mac.ops.check_link(&sc->hw, &sc->link_speed, 3584 &sc->link_up, FALSE); 3585 ixv_update_link_status(sc); 3586 3587 sc->task_requests = 0; 3588 atomic_store_relaxed(&sc->admin_pending, 0); 3589 3590 /* Re-enable interrupts */ 3591 IXGBE_WRITE_REG(hw, IXGBE_VTEIMS, (1 << sc->vector)); 3592 3593 IXGBE_CORE_UNLOCK(sc); 3594 } /* ixv_handle_admin */ 3595 3596 /************************************************************************ 3597 * ixv_check_link - Used in the local timer to poll for link changes 3598 ************************************************************************/ 3599 static s32 3600 ixv_check_link(struct ixgbe_softc *sc) 3601 { 3602 s32 error; 3603 3604 KASSERT(mutex_owned(&sc->core_mtx)); 3605 3606 sc->hw.mac.get_link_status = TRUE; 3607 3608 error = sc->hw.mac.ops.check_link(&sc->hw, 3609 &sc->link_speed, &sc->link_up, FALSE); 3610 ixv_update_link_status(sc); 3611 3612 return error; 3613 } /* ixv_check_link */ 3614
Indexes created Sun Oct 25 00:26:45 UTC 2026