ixv.c revision 1.186
1 /* $NetBSD: ixv.c,v 1.186 2023/10/06 14:37:04 msaitoh Exp $ */ 2 3 /****************************************************************************** 4 5 Copyright (c) 2001-2017, Intel Corporation 6 All rights reserved. 7 8 Redistribution and use in source and binary forms, with or without 9 modification, are permitted provided that the following conditions are met: 10 11 1. Redistributions of source code must retain the above copyright notice, 12 this list of conditions and the following disclaimer. 13 14 2. Redistributions in binary form must reproduce the above copyright 15 notice, this list of conditions and the following disclaimer in the 16 documentation and/or other materials provided with the distribution. 17 18 3. Neither the name of the Intel Corporation nor the names of its 19 contributors may be used to endorse or promote products derived from 20 this software without specific prior written permission. 21 22 THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 23 AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 24 IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 25 ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE 26 LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 27 CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 28 SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 29 INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 30 CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 31 ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 32 POSSIBILITY OF SUCH DAMAGE. 33 34 ******************************************************************************/ 35 /*$FreeBSD: head/sys/dev/ixgbe/if_ixv.c 331224 2018-03-19 20:55:05Z erj $*/ 36 37 #include <sys/cdefs.h> 38 __KERNEL_RCSID(0, "$NetBSD: ixv.c,v 1.186 2023/10/06 14:37:04 msaitoh Exp $"); 39 40 #ifdef _KERNEL_OPT 41 #include "opt_inet.h" 42 #include "opt_inet6.h" 43 #include "opt_net_mpsafe.h" 44 #endif 45 46 #include "ixgbe.h" 47 48 /************************************************************************ 49 * Driver version 50 ************************************************************************/ 51 static const char ixv_driver_version[] = "2.0.1-k"; 52 /* XXX NetBSD: + 1.5.17 */ 53 54 /************************************************************************ 55 * PCI Device ID Table 56 * 57 * Used by probe to select devices to load on 58 * Last field stores an index into ixv_strings 59 * Last entry must be all 0s 60 * 61 * { Vendor ID, Device ID, SubVendor ID, SubDevice ID, String Index } 62 ************************************************************************/ 63 static const ixgbe_vendor_info_t ixv_vendor_info_array[] = 64 { 65 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_VF, 0, 0, 0}, 66 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_X540_VF, 0, 0, 0}, 67 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_X550_VF, 0, 0, 0}, 68 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_X550EM_X_VF, 0, 0, 0}, 69 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_X550EM_A_VF, 0, 0, 0}, 70 /* required last entry */ 71 {0, 0, 0, 0, 0} 72 }; 73 74 /************************************************************************ 75 * Table of branding strings 76 ************************************************************************/ 77 static const char *ixv_strings[] = { 78 "Intel(R) PRO/10GbE Virtual Function Network Driver" 79 }; 80 81 /********************************************************************* 82 * Function prototypes 83 *********************************************************************/ 84 static int ixv_probe(device_t, cfdata_t, void *); 85 static void ixv_attach(device_t, device_t, void *); 86 static int ixv_detach(device_t, int); 87 #if 0 88 static int ixv_shutdown(device_t); 89 #endif 90 static int ixv_ifflags_cb(struct ethercom *); 91 static int ixv_ioctl(struct ifnet *, u_long, void *); 92 static int ixv_init(struct ifnet *); 93 static void ixv_init_locked(struct ixgbe_softc *); 94 static void ixv_ifstop(struct ifnet *, int); 95 static void ixv_stop_locked(void *); 96 static void ixv_init_device_features(struct ixgbe_softc *); 97 static void ixv_media_status(struct ifnet *, struct ifmediareq *); 98 static int ixv_media_change(struct ifnet *); 99 static int ixv_allocate_pci_resources(struct ixgbe_softc *, 100 const struct pci_attach_args *); 101 static void ixv_free_deferred_handlers(struct ixgbe_softc *); 102 static int ixv_allocate_msix(struct ixgbe_softc *, 103 const struct pci_attach_args *); 104 static int ixv_configure_interrupts(struct ixgbe_softc *); 105 static void ixv_free_pci_resources(struct ixgbe_softc *); 106 static void ixv_local_timer(void *); 107 static void ixv_handle_timer(struct work *, void *); 108 static int ixv_setup_interface(device_t, struct ixgbe_softc *); 109 static void ixv_schedule_admin_tasklet(struct ixgbe_softc *); 110 static int ixv_negotiate_api(struct ixgbe_softc *); 111 112 static void ixv_initialize_transmit_units(struct ixgbe_softc *); 113 static void ixv_initialize_receive_units(struct ixgbe_softc *); 114 static void ixv_initialize_rss_mapping(struct ixgbe_softc *); 115 static s32 ixv_check_link(struct ixgbe_softc *); 116 117 static void ixv_enable_intr(struct ixgbe_softc *); 118 static void ixv_disable_intr(struct ixgbe_softc *); 119 static int ixv_set_rxfilter(struct ixgbe_softc *); 120 static void ixv_update_link_status(struct ixgbe_softc *); 121 static int ixv_sysctl_debug(SYSCTLFN_PROTO); 122 static void ixv_set_ivar(struct ixgbe_softc *, u8, u8, s8); 123 static void ixv_configure_ivars(struct ixgbe_softc *); 124 static u8 * ixv_mc_array_itr(struct ixgbe_hw *, u8 **, u32 *); 125 static void ixv_eitr_write(struct ixgbe_softc *, uint32_t, uint32_t); 126 127 static void ixv_setup_vlan_tagging(struct ixgbe_softc *); 128 static int ixv_setup_vlan_support(struct ixgbe_softc *); 129 static int ixv_vlan_cb(struct ethercom *, uint16_t, bool); 130 static int ixv_register_vlan(struct ixgbe_softc *, u16); 131 static int ixv_unregister_vlan(struct ixgbe_softc *, u16); 132 133 static void ixv_add_device_sysctls(struct ixgbe_softc *); 134 static void ixv_init_stats(struct ixgbe_softc *); 135 static void ixv_update_stats(struct ixgbe_softc *); 136 static void ixv_add_stats_sysctls(struct ixgbe_softc *); 137 static void ixv_clear_evcnt(struct ixgbe_softc *); 138 139 /* Sysctl handlers */ 140 static int ixv_sysctl_interrupt_rate_handler(SYSCTLFN_PROTO); 141 static int ixv_sysctl_next_to_check_handler(SYSCTLFN_PROTO); 142 static int ixv_sysctl_next_to_refresh_handler(SYSCTLFN_PROTO); 143 static int ixv_sysctl_rdh_handler(SYSCTLFN_PROTO); 144 static int ixv_sysctl_rdt_handler(SYSCTLFN_PROTO); 145 static int ixv_sysctl_tdt_handler(SYSCTLFN_PROTO); 146 static int ixv_sysctl_tdh_handler(SYSCTLFN_PROTO); 147 static int ixv_sysctl_tx_process_limit(SYSCTLFN_PROTO); 148 static int ixv_sysctl_rx_process_limit(SYSCTLFN_PROTO); 149 static int ixv_sysctl_rx_copy_len(SYSCTLFN_PROTO); 150 151 /* The MSI-X Interrupt handlers */ 152 static int ixv_msix_que(void *); 153 static int ixv_msix_mbx(void *); 154 155 /* Event handlers running on workqueue */ 156 static void ixv_handle_que(void *); 157 158 /* Deferred workqueue handlers */ 159 static void ixv_handle_admin(struct work *, void *); 160 static void ixv_handle_que_work(struct work *, void *); 161 162 const struct sysctlnode *ixv_sysctl_instance(struct ixgbe_softc *); 163 static const ixgbe_vendor_info_t *ixv_lookup(const struct pci_attach_args *); 164 165 /************************************************************************ 166 * NetBSD Device Interface Entry Points 167 ************************************************************************/ 168 CFATTACH_DECL3_NEW(ixv, sizeof(struct ixgbe_softc), 169 ixv_probe, ixv_attach, ixv_detach, NULL, NULL, NULL, 170 DVF_DETACH_SHUTDOWN); 171 172 #if 0 173 static driver_t ixv_driver = { 174 "ixv", ixv_methods, sizeof(struct ixgbe_softc), 175 }; 176 177 devclass_t ixv_devclass; 178 DRIVER_MODULE(ixv, pci, ixv_driver, ixv_devclass, 0, 0); 179 MODULE_DEPEND(ixv, pci, 1, 1, 1); 180 MODULE_DEPEND(ixv, ether, 1, 1, 1); 181 #endif 182 183 /* 184 * TUNEABLE PARAMETERS: 185 */ 186 187 /* Number of Queues - do not exceed MSI-X vectors - 1 */ 188 static int ixv_num_queues = 0; 189 #define TUNABLE_INT(__x, __y) 190 TUNABLE_INT("hw.ixv.num_queues", &ixv_num_queues); 191 192 /* 193 * AIM: Adaptive Interrupt Moderation 194 * which means that the interrupt rate 195 * is varied over time based on the 196 * traffic for that interrupt vector 197 */ 198 static bool ixv_enable_aim = false; 199 TUNABLE_INT("hw.ixv.enable_aim", &ixv_enable_aim); 200 201 static int ixv_max_interrupt_rate = (4000000 / IXGBE_LOW_LATENCY); 202 TUNABLE_INT("hw.ixv.max_interrupt_rate", &ixv_max_interrupt_rate); 203 204 /* How many packets rxeof tries to clean at a time */ 205 static int ixv_rx_process_limit = 256; 206 TUNABLE_INT("hw.ixv.rx_process_limit", &ixv_rx_process_limit); 207 208 /* How many packets txeof tries to clean at a time */ 209 static int ixv_tx_process_limit = 256; 210 TUNABLE_INT("hw.ixv.tx_process_limit", &ixv_tx_process_limit); 211 212 /* Which packet processing uses workqueue or softint */ 213 static bool ixv_txrx_workqueue = false; 214 215 /* 216 * Number of TX descriptors per ring, 217 * setting higher than RX as this seems 218 * the better performing choice. 219 */ 220 static int ixv_txd = PERFORM_TXD; 221 TUNABLE_INT("hw.ixv.txd", &ixv_txd); 222 223 /* Number of RX descriptors per ring */ 224 static int ixv_rxd = PERFORM_RXD; 225 TUNABLE_INT("hw.ixv.rxd", &ixv_rxd); 226 227 /* Legacy Transmit (single queue) */ 228 static int ixv_enable_legacy_tx = 0; 229 TUNABLE_INT("hw.ixv.enable_legacy_tx", &ixv_enable_legacy_tx); 230 231 #ifdef NET_MPSAFE 232 #define IXGBE_MPSAFE 1 233 #define IXGBE_CALLOUT_FLAGS CALLOUT_MPSAFE 234 #define IXGBE_SOFTINT_FLAGS SOFTINT_MPSAFE 235 #define IXGBE_WORKQUEUE_FLAGS WQ_PERCPU | WQ_MPSAFE 236 #define IXGBE_TASKLET_WQ_FLAGS WQ_MPSAFE 237 #else 238 #define IXGBE_CALLOUT_FLAGS 0 239 #define IXGBE_SOFTINT_FLAGS 0 240 #define IXGBE_WORKQUEUE_FLAGS WQ_PERCPU 241 #define IXGBE_TASKLET_WQ_FLAGS 0 242 #endif 243 #define IXGBE_WORKQUEUE_PRI PRI_SOFTNET 244 245 #if 0 246 static int (*ixv_start_locked)(struct ifnet *, struct tx_ring *); 247 static int (*ixv_ring_empty)(struct ifnet *, struct buf_ring *); 248 #endif 249 250 /************************************************************************ 251 * ixv_probe - Device identification routine 252 * 253 * Determines if the driver should be loaded on 254 * adapter based on its PCI vendor/device ID. 255 * 256 * return BUS_PROBE_DEFAULT on success, positive on failure 257 ************************************************************************/ 258 static int 259 ixv_probe(device_t dev, cfdata_t cf, void *aux) 260 { 261 #ifdef __HAVE_PCI_MSI_MSIX 262 const struct pci_attach_args *pa = aux; 263 264 return (ixv_lookup(pa) != NULL) ? 1 : 0; 265 #else 266 return 0; 267 #endif 268 } /* ixv_probe */ 269 270 static const ixgbe_vendor_info_t * 271 ixv_lookup(const struct pci_attach_args *pa) 272 { 273 const ixgbe_vendor_info_t *ent; 274 pcireg_t subid; 275 276 INIT_DEBUGOUT("ixv_lookup: begin"); 277 278 if (PCI_VENDOR(pa->pa_id) != IXGBE_INTEL_VENDOR_ID) 279 return NULL; 280 281 subid = pci_conf_read(pa->pa_pc, pa->pa_tag, PCI_SUBSYS_ID_REG); 282 283 for (ent = ixv_vendor_info_array; ent->vendor_id != 0; ent++) { 284 if ((PCI_VENDOR(pa->pa_id) == ent->vendor_id) && 285 (PCI_PRODUCT(pa->pa_id) == ent->device_id) && 286 ((PCI_SUBSYS_VENDOR(subid) == ent->subvendor_id) || 287 (ent->subvendor_id == 0)) && 288 ((PCI_SUBSYS_ID(subid) == ent->subdevice_id) || 289 (ent->subdevice_id == 0))) { 290 return ent; 291 } 292 } 293 294 return NULL; 295 } 296 297 /************************************************************************ 298 * ixv_attach - Device initialization routine 299 * 300 * Called when the driver is being loaded. 301 * Identifies the type of hardware, allocates all resources 302 * and initializes the hardware. 303 * 304 * return 0 on success, positive on failure 305 ************************************************************************/ 306 static void 307 ixv_attach(device_t parent, device_t dev, void *aux) 308 { 309 struct ixgbe_softc *sc; 310 struct ixgbe_hw *hw; 311 int error = 0; 312 pcireg_t id, subid; 313 const ixgbe_vendor_info_t *ent; 314 const struct pci_attach_args *pa = aux; 315 const char *apivstr; 316 const char *str; 317 char wqname[MAXCOMLEN]; 318 char buf[256]; 319 320 INIT_DEBUGOUT("ixv_attach: begin"); 321 322 /* 323 * Make sure BUSMASTER is set, on a VM under 324 * KVM it may not be and will break things. 325 */ 326 ixgbe_pci_enable_busmaster(pa->pa_pc, pa->pa_tag); 327 328 /* Allocate, clear, and link in our adapter structure */ 329 sc = device_private(dev); 330 sc->hw.back = sc; 331 sc->dev = dev; 332 hw = &sc->hw; 333 334 sc->init_locked = ixv_init_locked; 335 sc->stop_locked = ixv_stop_locked; 336 337 sc->osdep.pc = pa->pa_pc; 338 sc->osdep.tag = pa->pa_tag; 339 if (pci_dma64_available(pa)) 340 sc->osdep.dmat = pa->pa_dmat64; 341 else 342 sc->osdep.dmat = pa->pa_dmat; 343 sc->osdep.attached = false; 344 345 ent = ixv_lookup(pa); 346 347 KASSERT(ent != NULL); 348 349 aprint_normal(": %s, Version - %s\n", 350 ixv_strings[ent->index], ixv_driver_version); 351 352 /* Core Lock Init */ 353 IXGBE_CORE_LOCK_INIT(sc, device_xname(dev)); 354 355 /* Do base PCI setup - map BAR0 */ 356 if (ixv_allocate_pci_resources(sc, pa)) { 357 aprint_error_dev(dev, "ixv_allocate_pci_resources() failed!\n"); 358 error = ENXIO; 359 goto err_out; 360 } 361 362 /* SYSCTL APIs */ 363 ixv_add_device_sysctls(sc); 364 365 /* Set up the timer callout and workqueue */ 366 callout_init(&sc->timer, IXGBE_CALLOUT_FLAGS); 367 snprintf(wqname, sizeof(wqname), "%s-timer", device_xname(dev)); 368 error = workqueue_create(&sc->timer_wq, wqname, 369 ixv_handle_timer, sc, IXGBE_WORKQUEUE_PRI, IPL_NET, 370 IXGBE_TASKLET_WQ_FLAGS); 371 if (error) { 372 aprint_error_dev(dev, 373 "could not create timer workqueue (%d)\n", error); 374 goto err_out; 375 } 376 377 /* Save off the information about this board */ 378 id = pci_conf_read(pa->pa_pc, pa->pa_tag, PCI_ID_REG); 379 subid = pci_conf_read(pa->pa_pc, pa->pa_tag, PCI_SUBSYS_ID_REG); 380 hw->vendor_id = PCI_VENDOR(id); 381 hw->device_id = PCI_PRODUCT(id); 382 hw->revision_id = 383 PCI_REVISION(pci_conf_read(pa->pa_pc, pa->pa_tag, PCI_CLASS_REG)); 384 hw->subsystem_vendor_id = PCI_SUBSYS_VENDOR(subid); 385 hw->subsystem_device_id = PCI_SUBSYS_ID(subid); 386 387 /* A subset of set_mac_type */ 388 switch (hw->device_id) { 389 case IXGBE_DEV_ID_82599_VF: 390 hw->mac.type = ixgbe_mac_82599_vf; 391 str = "82599 VF"; 392 break; 393 case IXGBE_DEV_ID_X540_VF: 394 hw->mac.type = ixgbe_mac_X540_vf; 395 str = "X540 VF"; 396 break; 397 case IXGBE_DEV_ID_X550_VF: 398 hw->mac.type = ixgbe_mac_X550_vf; 399 str = "X550 VF"; 400 break; 401 case IXGBE_DEV_ID_X550EM_X_VF: 402 hw->mac.type = ixgbe_mac_X550EM_x_vf; 403 str = "X550EM X VF"; 404 break; 405 case IXGBE_DEV_ID_X550EM_A_VF: 406 hw->mac.type = ixgbe_mac_X550EM_a_vf; 407 str = "X550EM A VF"; 408 break; 409 default: 410 /* Shouldn't get here since probe succeeded */ 411 aprint_error_dev(dev, "Unknown device ID!\n"); 412 error = ENXIO; 413 goto err_out; 414 break; 415 } 416 aprint_normal_dev(dev, "device %s\n", str); 417 418 ixv_init_device_features(sc); 419 420 /* Initialize the shared code */ 421 error = ixgbe_init_ops_vf(hw); 422 if (error) { 423 aprint_error_dev(dev, "ixgbe_init_ops_vf() failed!\n"); 424 error = EIO; 425 goto err_out; 426 } 427 428 /* Setup the mailbox */ 429 ixgbe_init_mbx_params_vf(hw); 430 431 /* Set the right number of segments */ 432 KASSERT(IXGBE_82599_SCATTER_MAX >= IXGBE_SCATTER_DEFAULT); 433 sc->num_segs = IXGBE_SCATTER_DEFAULT; 434 435 /* Reset mbox api to 1.0 */ 436 error = hw->mac.ops.reset_hw(hw); 437 if (error == IXGBE_ERR_RESET_FAILED) 438 aprint_error_dev(dev, "...reset_hw() failure: Reset Failed!\n"); 439 else if (error) 440 aprint_error_dev(dev, "...reset_hw() failed with error %d\n", 441 error); 442 if (error) { 443 error = EIO; 444 goto err_out; 445 } 446 447 error = hw->mac.ops.init_hw(hw); 448 if (error) { 449 aprint_error_dev(dev, "...init_hw() failed!\n"); 450 error = EIO; 451 goto err_out; 452 } 453 454 /* Negotiate mailbox API version */ 455 error = ixv_negotiate_api(sc); 456 if (error) 457 aprint_normal_dev(dev, 458 "MBX API negotiation failed during attach!\n"); 459 switch (hw->api_version) { 460 case ixgbe_mbox_api_10: 461 apivstr = "1.0"; 462 break; 463 case ixgbe_mbox_api_20: 464 apivstr = "2.0"; 465 break; 466 case ixgbe_mbox_api_11: 467 apivstr = "1.1"; 468 break; 469 case ixgbe_mbox_api_12: 470 apivstr = "1.2"; 471 break; 472 case ixgbe_mbox_api_13: 473 apivstr = "1.3"; 474 break; 475 case ixgbe_mbox_api_14: 476 apivstr = "1.4"; 477 break; 478 case ixgbe_mbox_api_15: 479 apivstr = "1.5"; 480 break; 481 default: 482 apivstr = "unknown"; 483 break; 484 } 485 aprint_normal_dev(dev, "Mailbox API %s\n", apivstr); 486 487 /* If no mac address was assigned, make a random one */ 488 if (!ixv_check_ether_addr(hw->mac.addr)) { 489 u8 addr[ETHER_ADDR_LEN]; 490 uint64_t rndval = cprng_strong64(); 491 492 memcpy(addr, &rndval, sizeof(addr)); 493 addr[0] &= 0xFE; 494 addr[0] |= 0x02; 495 bcopy(addr, hw->mac.addr, sizeof(addr)); 496 } 497 498 /* Register for VLAN events */ 499 ether_set_vlan_cb(&sc->osdep.ec, ixv_vlan_cb); 500 501 /* Do descriptor calc and sanity checks */ 502 if (((ixv_txd * sizeof(union ixgbe_adv_tx_desc)) % DBA_ALIGN) != 0 || 503 ixv_txd < MIN_TXD || ixv_txd > MAX_TXD) { 504 aprint_error_dev(dev, "TXD config issue, using default!\n"); 505 sc->num_tx_desc = DEFAULT_TXD; 506 } else 507 sc->num_tx_desc = ixv_txd; 508 509 if (((ixv_rxd * sizeof(union ixgbe_adv_rx_desc)) % DBA_ALIGN) != 0 || 510 ixv_rxd < MIN_RXD || ixv_rxd > MAX_RXD) { 511 aprint_error_dev(dev, "RXD config issue, using default!\n"); 512 sc->num_rx_desc = DEFAULT_RXD; 513 } else 514 sc->num_rx_desc = ixv_rxd; 515 516 /* Sysctls for limiting the amount of work done in the taskqueues */ 517 sc->rx_process_limit 518 = (ixv_rx_process_limit <= sc->num_rx_desc) 519 ? ixv_rx_process_limit : sc->num_rx_desc; 520 sc->tx_process_limit 521 = (ixv_tx_process_limit <= sc->num_tx_desc) 522 ? ixv_tx_process_limit : sc->num_tx_desc; 523 524 /* Set default high limit of copying mbuf in rxeof */ 525 sc->rx_copy_len = IXGBE_RX_COPY_LEN_MAX; 526 527 /* Setup MSI-X */ 528 error = ixv_configure_interrupts(sc); 529 if (error) 530 goto err_out; 531 532 /* Allocate our TX/RX Queues */ 533 if (ixgbe_allocate_queues(sc)) { 534 aprint_error_dev(dev, "ixgbe_allocate_queues() failed!\n"); 535 error = ENOMEM; 536 goto err_out; 537 } 538 539 /* hw.ix defaults init */ 540 sc->enable_aim = ixv_enable_aim; 541 542 sc->txrx_use_workqueue = ixv_txrx_workqueue; 543 544 error = ixv_allocate_msix(sc, pa); 545 if (error) { 546 aprint_error_dev(dev, "ixv_allocate_msix() failed!\n"); 547 goto err_late; 548 } 549 550 /* Setup OS specific network interface */ 551 error = ixv_setup_interface(dev, sc); 552 if (error != 0) { 553 aprint_error_dev(dev, "ixv_setup_interface() failed!\n"); 554 goto err_late; 555 } 556 557 /* Allocate multicast array memory */ 558 sc->mta = malloc(sizeof(*sc->mta) * 559 IXGBE_MAX_VF_MC, M_DEVBUF, M_WAITOK); 560 561 /* Check if VF was disabled by PF */ 562 error = hw->mac.ops.get_link_state(hw, &sc->link_enabled); 563 if (error) { 564 /* PF is not capable of controlling VF state. Enable the link. */ 565 sc->link_enabled = TRUE; 566 } 567 568 /* Do the stats setup */ 569 ixv_init_stats(sc); 570 ixv_add_stats_sysctls(sc); 571 572 if (sc->feat_en & IXGBE_FEATURE_NETMAP) 573 ixgbe_netmap_attach(sc); 574 575 snprintb(buf, sizeof(buf), IXGBE_FEATURE_FLAGS, sc->feat_cap); 576 aprint_verbose_dev(dev, "feature cap %s\n", buf); 577 snprintb(buf, sizeof(buf), IXGBE_FEATURE_FLAGS, sc->feat_en); 578 aprint_verbose_dev(dev, "feature ena %s\n", buf); 579 580 INIT_DEBUGOUT("ixv_attach: end"); 581 sc->osdep.attached = true; 582 583 return; 584 585 err_late: 586 ixgbe_free_queues(sc); 587 err_out: 588 ixv_free_pci_resources(sc); 589 IXGBE_CORE_LOCK_DESTROY(sc); 590 591 return; 592 } /* ixv_attach */ 593 594 /************************************************************************ 595 * ixv_detach - Device removal routine 596 * 597 * Called when the driver is being removed. 598 * Stops the adapter and deallocates all the resources 599 * that were allocated for driver operation. 600 * 601 * return 0 on success, positive on failure 602 ************************************************************************/ 603 static int 604 ixv_detach(device_t dev, int flags) 605 { 606 struct ixgbe_softc *sc = device_private(dev); 607 struct ixgbe_hw *hw = &sc->hw; 608 struct tx_ring *txr = sc->tx_rings; 609 struct rx_ring *rxr = sc->rx_rings; 610 struct ixgbevf_hw_stats *stats = &sc->stats.vf; 611 612 INIT_DEBUGOUT("ixv_detach: begin"); 613 if (sc->osdep.attached == false) 614 return 0; 615 616 /* Stop the interface. Callouts are stopped in it. */ 617 ixv_ifstop(sc->ifp, 1); 618 619 if (VLAN_ATTACHED(&sc->osdep.ec) && 620 (flags & (DETACH_SHUTDOWN | DETACH_FORCE)) == 0) { 621 aprint_error_dev(dev, "VLANs in use, detach first\n"); 622 return EBUSY; 623 } 624 625 ether_ifdetach(sc->ifp); 626 callout_halt(&sc->timer, NULL); 627 ixv_free_deferred_handlers(sc); 628 629 if (sc->feat_en & IXGBE_FEATURE_NETMAP) 630 netmap_detach(sc->ifp); 631 632 ixv_free_pci_resources(sc); 633 #if 0 /* XXX the NetBSD port is probably missing something here */ 634 bus_generic_detach(dev); 635 #endif 636 if_detach(sc->ifp); 637 ifmedia_fini(&sc->media); 638 if_percpuq_destroy(sc->ipq); 639 640 sysctl_teardown(&sc->sysctllog); 641 evcnt_detach(&sc->efbig_tx_dma_setup); 642 evcnt_detach(&sc->mbuf_defrag_failed); 643 evcnt_detach(&sc->efbig2_tx_dma_setup); 644 evcnt_detach(&sc->einval_tx_dma_setup); 645 evcnt_detach(&sc->other_tx_dma_setup); 646 evcnt_detach(&sc->eagain_tx_dma_setup); 647 evcnt_detach(&sc->enomem_tx_dma_setup); 648 evcnt_detach(&sc->watchdog_events); 649 evcnt_detach(&sc->tso_err); 650 evcnt_detach(&sc->admin_irqev); 651 evcnt_detach(&sc->link_workev); 652 653 txr = sc->tx_rings; 654 for (int i = 0; i < sc->num_queues; i++, rxr++, txr++) { 655 evcnt_detach(&sc->queues[i].irqs); 656 evcnt_detach(&sc->queues[i].handleq); 657 evcnt_detach(&sc->queues[i].req); 658 evcnt_detach(&txr->total_packets); 659 #ifndef IXGBE_LEGACY_TX 660 evcnt_detach(&txr->pcq_drops); 661 #endif 662 evcnt_detach(&txr->no_desc_avail); 663 evcnt_detach(&txr->tso_tx); 664 665 evcnt_detach(&rxr->rx_packets); 666 evcnt_detach(&rxr->rx_bytes); 667 evcnt_detach(&rxr->rx_copies); 668 evcnt_detach(&rxr->no_mbuf); 669 evcnt_detach(&rxr->rx_discarded); 670 } 671 evcnt_detach(&stats->ipcs); 672 evcnt_detach(&stats->l4cs); 673 evcnt_detach(&stats->ipcs_bad); 674 evcnt_detach(&stats->l4cs_bad); 675 676 /* Packet Reception Stats */ 677 evcnt_detach(&stats->vfgorc); 678 evcnt_detach(&stats->vfgprc); 679 evcnt_detach(&stats->vfmprc); 680 681 /* Packet Transmission Stats */ 682 evcnt_detach(&stats->vfgotc); 683 evcnt_detach(&stats->vfgptc); 684 685 /* Mailbox Stats */ 686 evcnt_detach(&hw->mbx.stats.msgs_tx); 687 evcnt_detach(&hw->mbx.stats.msgs_rx); 688 evcnt_detach(&hw->mbx.stats.acks); 689 evcnt_detach(&hw->mbx.stats.reqs); 690 evcnt_detach(&hw->mbx.stats.rsts); 691 692 ixgbe_free_queues(sc); 693 694 IXGBE_CORE_LOCK_DESTROY(sc); 695 696 return (0); 697 } /* ixv_detach */ 698 699 /************************************************************************ 700 * ixv_init_locked - Init entry point 701 * 702 * Used in two ways: It is used by the stack as an init entry 703 * point in network interface structure. It is also used 704 * by the driver as a hw/sw initialization routine to get 705 * to a consistent state. 706 * 707 * return 0 on success, positive on failure 708 ************************************************************************/ 709 static void 710 ixv_init_locked(struct ixgbe_softc *sc) 711 { 712 struct ifnet *ifp = sc->ifp; 713 device_t dev = sc->dev; 714 struct ixgbe_hw *hw = &sc->hw; 715 struct ix_queue *que; 716 int error = 0; 717 uint32_t mask; 718 int i; 719 720 INIT_DEBUGOUT("ixv_init_locked: begin"); 721 KASSERT(mutex_owned(&sc->core_mtx)); 722 hw->adapter_stopped = FALSE; 723 hw->mac.ops.stop_adapter(hw); 724 callout_stop(&sc->timer); 725 for (i = 0, que = sc->queues; i < sc->num_queues; i++, que++) 726 que->disabled_count = 0; 727 728 sc->max_frame_size = 729 ifp->if_mtu + ETHER_HDR_LEN + ETHER_CRC_LEN; 730 731 /* reprogram the RAR[0] in case user changed it. */ 732 hw->mac.ops.set_rar(hw, 0, hw->mac.addr, 0, IXGBE_RAH_AV); 733 734 /* Get the latest mac address, User can use a LAA */ 735 memcpy(hw->mac.addr, CLLADDR(ifp->if_sadl), 736 IXGBE_ETH_LENGTH_OF_ADDRESS); 737 hw->mac.ops.set_rar(hw, 0, hw->mac.addr, 0, 1); 738 739 /* Prepare transmit descriptors and buffers */ 740 if (ixgbe_setup_transmit_structures(sc)) { 741 aprint_error_dev(dev, "Could not setup transmit structures\n"); 742 ixv_stop_locked(sc); 743 return; 744 } 745 746 /* Reset VF and renegotiate mailbox API version */ 747 hw->mac.ops.reset_hw(hw); 748 hw->mac.ops.start_hw(hw); 749 error = ixv_negotiate_api(sc); 750 if (error) 751 device_printf(dev, 752 "Mailbox API negotiation failed in init_locked!\n"); 753 754 ixv_initialize_transmit_units(sc); 755 756 /* Setup Multicast table */ 757 ixv_set_rxfilter(sc); 758 759 /* Use fixed buffer size, even for jumbo frames */ 760 sc->rx_mbuf_sz = MCLBYTES; 761 762 /* Prepare receive descriptors and buffers */ 763 error = ixgbe_setup_receive_structures(sc); 764 if (error) { 765 device_printf(dev, 766 "Could not setup receive structures (err = %d)\n", error); 767 ixv_stop_locked(sc); 768 return; 769 } 770 771 /* Configure RX settings */ 772 ixv_initialize_receive_units(sc); 773 774 /* Initialize variable holding task enqueue requests interrupts */ 775 sc->task_requests = 0; 776 777 /* Set up VLAN offload and filter */ 778 ixv_setup_vlan_support(sc); 779 780 /* Set up MSI-X routing */ 781 ixv_configure_ivars(sc); 782 783 /* Set up auto-mask */ 784 mask = (1 << sc->vector); 785 for (i = 0, que = sc->queues; i < sc->num_queues; i++, que++) 786 mask |= (1 << que->msix); 787 IXGBE_WRITE_REG(hw, IXGBE_VTEIAM, mask); 788 789 /* Set moderation on the Link interrupt */ 790 ixv_eitr_write(sc, sc->vector, IXGBE_LINK_ITR); 791 792 /* Stats init */ 793 ixv_init_stats(sc); 794 795 /* Config/Enable Link */ 796 error = hw->mac.ops.get_link_state(hw, &sc->link_enabled); 797 if (error) { 798 /* PF is not capable of controlling VF state. Enable the link. */ 799 sc->link_enabled = TRUE; 800 } else if (sc->link_enabled == FALSE) 801 device_printf(dev, "VF is disabled by PF\n"); 802 803 hw->mac.get_link_status = TRUE; 804 hw->mac.ops.check_link(hw, &sc->link_speed, &sc->link_up, 805 FALSE); 806 807 /* Start watchdog */ 808 callout_reset(&sc->timer, hz, ixv_local_timer, sc); 809 atomic_store_relaxed(&sc->timer_pending, 0); 810 811 /* OK to schedule workqueues. */ 812 sc->schedule_wqs_ok = true; 813 814 /* And now turn on interrupts */ 815 ixv_enable_intr(sc); 816 817 /* Update saved flags. See ixgbe_ifflags_cb() */ 818 sc->if_flags = ifp->if_flags; 819 sc->ec_capenable = sc->osdep.ec.ec_capenable; 820 821 /* Now inform the stack we're ready */ 822 ifp->if_flags |= IFF_RUNNING; 823 ifp->if_flags &= ~IFF_OACTIVE; 824 825 return; 826 } /* ixv_init_locked */ 827 828 /************************************************************************ 829 * ixv_enable_queue 830 ************************************************************************/ 831 static inline void 832 ixv_enable_queue(struct ixgbe_softc *sc, u32 vector) 833 { 834 struct ixgbe_hw *hw = &sc->hw; 835 struct ix_queue *que = &sc->queues[vector]; 836 u32 queue = 1UL << vector; 837 u32 mask; 838 839 mutex_enter(&que->dc_mtx); 840 if (que->disabled_count > 0 && --que->disabled_count > 0) 841 goto out; 842 843 mask = (IXGBE_EIMS_RTX_QUEUE & queue); 844 IXGBE_WRITE_REG(hw, IXGBE_VTEIMS, mask); 845 out: 846 mutex_exit(&que->dc_mtx); 847 } /* ixv_enable_queue */ 848 849 /************************************************************************ 850 * ixv_disable_queue 851 ************************************************************************/ 852 static inline void 853 ixv_disable_queue(struct ixgbe_softc *sc, u32 vector) 854 { 855 struct ixgbe_hw *hw = &sc->hw; 856 struct ix_queue *que = &sc->queues[vector]; 857 u32 queue = 1UL << vector; 858 u32 mask; 859 860 mutex_enter(&que->dc_mtx); 861 if (que->disabled_count++ > 0) 862 goto out; 863 864 mask = (IXGBE_EIMS_RTX_QUEUE & queue); 865 IXGBE_WRITE_REG(hw, IXGBE_VTEIMC, mask); 866 out: 867 mutex_exit(&que->dc_mtx); 868 } /* ixv_disable_queue */ 869 870 #if 0 871 static inline void 872 ixv_rearm_queues(struct ixgbe_softc *sc, u64 queues) 873 { 874 u32 mask = (IXGBE_EIMS_RTX_QUEUE & queues); 875 IXGBE_WRITE_REG(&sc->hw, IXGBE_VTEICS, mask); 876 } /* ixv_rearm_queues */ 877 #endif 878 879 880 /************************************************************************ 881 * ixv_msix_que - MSI-X Queue Interrupt Service routine 882 ************************************************************************/ 883 static int 884 ixv_msix_que(void *arg) 885 { 886 struct ix_queue *que = arg; 887 struct ixgbe_softc *sc = que->sc; 888 struct tx_ring *txr = que->txr; 889 struct rx_ring *rxr = que->rxr; 890 bool more; 891 u32 newitr = 0; 892 893 ixv_disable_queue(sc, que->msix); 894 IXGBE_EVC_ADD(&que->irqs, 1); 895 896 #ifdef __NetBSD__ 897 /* Don't run ixgbe_rxeof in interrupt context */ 898 more = true; 899 #else 900 more = ixgbe_rxeof(que); 901 #endif 902 903 IXGBE_TX_LOCK(txr); 904 ixgbe_txeof(txr); 905 IXGBE_TX_UNLOCK(txr); 906 907 /* Do AIM now? */ 908 909 if (sc->enable_aim == false) 910 goto no_calc; 911 /* 912 * Do Adaptive Interrupt Moderation: 913 * - Write out last calculated setting 914 * - Calculate based on average size over 915 * the last interval. 916 */ 917 if (que->eitr_setting) 918 ixv_eitr_write(sc, que->msix, que->eitr_setting); 919 920 que->eitr_setting = 0; 921 922 /* Idle, do nothing */ 923 if ((txr->bytes == 0) && (rxr->bytes == 0)) 924 goto no_calc; 925 926 if ((txr->bytes) && (txr->packets)) 927 newitr = txr->bytes/txr->packets; 928 if ((rxr->bytes) && (rxr->packets)) 929 newitr = uimax(newitr, (rxr->bytes / rxr->packets)); 930 newitr += 24; /* account for hardware frame, crc */ 931 932 /* set an upper boundary */ 933 newitr = uimin(newitr, 3000); 934 935 /* Be nice to the mid range */ 936 if ((newitr > 300) && (newitr < 1200)) 937 newitr = (newitr / 3); 938 else 939 newitr = (newitr / 2); 940 941 /* 942 * When RSC is used, ITR interval must be larger than RSC_DELAY. 943 * Currently, we use 2us for RSC_DELAY. The minimum value is always 944 * greater than 2us on 100M (and 10M?(not documented)), but it's not 945 * on 1G and higher. 946 */ 947 if ((sc->link_speed != IXGBE_LINK_SPEED_100_FULL) 948 && (sc->link_speed != IXGBE_LINK_SPEED_10_FULL)) { 949 if (newitr < IXGBE_MIN_RSC_EITR_10G1G) 950 newitr = IXGBE_MIN_RSC_EITR_10G1G; 951 } 952 953 /* save for next interrupt */ 954 que->eitr_setting = newitr; 955 956 /* Reset state */ 957 txr->bytes = 0; 958 txr->packets = 0; 959 rxr->bytes = 0; 960 rxr->packets = 0; 961 962 no_calc: 963 if (more) 964 softint_schedule(que->que_si); 965 else /* Re-enable this interrupt */ 966 ixv_enable_queue(sc, que->msix); 967 968 return 1; 969 } /* ixv_msix_que */ 970 971 /************************************************************************ 972 * ixv_msix_mbx 973 ************************************************************************/ 974 static int 975 ixv_msix_mbx(void *arg) 976 { 977 struct ixgbe_softc *sc = arg; 978 struct ixgbe_hw *hw = &sc->hw; 979 980 IXGBE_EVC_ADD(&sc->admin_irqev, 1); 981 /* NetBSD: We use auto-clear, so it's not required to write VTEICR */ 982 983 /* Link status change */ 984 hw->mac.get_link_status = TRUE; 985 atomic_or_32(&sc->task_requests, IXGBE_REQUEST_TASK_MBX); 986 ixv_schedule_admin_tasklet(sc); 987 988 return 1; 989 } /* ixv_msix_mbx */ 990 991 static void 992 ixv_eitr_write(struct ixgbe_softc *sc, uint32_t index, uint32_t itr) 993 { 994 995 /* 996 * Newer devices than 82598 have VF function, so this function is 997 * simple. 998 */ 999 itr |= IXGBE_EITR_CNT_WDIS; 1000 1001 IXGBE_WRITE_REG(&sc->hw, IXGBE_VTEITR(index), itr); 1002 } 1003 1004 1005 /************************************************************************ 1006 * ixv_media_status - Media Ioctl callback 1007 * 1008 * Called whenever the user queries the status of 1009 * the interface using ifconfig. 1010 ************************************************************************/ 1011 static void 1012 ixv_media_status(struct ifnet *ifp, struct ifmediareq *ifmr) 1013 { 1014 struct ixgbe_softc *sc = ifp->if_softc; 1015 1016 INIT_DEBUGOUT("ixv_media_status: begin"); 1017 ixv_update_link_status(sc); 1018 1019 ifmr->ifm_status = IFM_AVALID; 1020 ifmr->ifm_active = IFM_ETHER; 1021 1022 if (sc->link_active != LINK_STATE_UP) { 1023 ifmr->ifm_active |= IFM_NONE; 1024 return; 1025 } 1026 1027 ifmr->ifm_status |= IFM_ACTIVE; 1028 1029 switch (sc->link_speed) { 1030 case IXGBE_LINK_SPEED_10GB_FULL: 1031 ifmr->ifm_active |= IFM_10G_T | IFM_FDX; 1032 break; 1033 case IXGBE_LINK_SPEED_5GB_FULL: 1034 ifmr->ifm_active |= IFM_5000_T | IFM_FDX; 1035 break; 1036 case IXGBE_LINK_SPEED_2_5GB_FULL: 1037 ifmr->ifm_active |= IFM_2500_T | IFM_FDX; 1038 break; 1039 case IXGBE_LINK_SPEED_1GB_FULL: 1040 ifmr->ifm_active |= IFM_1000_T | IFM_FDX; 1041 break; 1042 case IXGBE_LINK_SPEED_100_FULL: 1043 ifmr->ifm_active |= IFM_100_TX | IFM_FDX; 1044 break; 1045 case IXGBE_LINK_SPEED_10_FULL: 1046 ifmr->ifm_active |= IFM_10_T | IFM_FDX; 1047 break; 1048 } 1049 1050 ifp->if_baudrate = ifmedia_baudrate(ifmr->ifm_active); 1051 } /* ixv_media_status */ 1052 1053 /************************************************************************ 1054 * ixv_media_change - Media Ioctl callback 1055 * 1056 * Called when the user changes speed/duplex using 1057 * media/mediopt option with ifconfig. 1058 ************************************************************************/ 1059 static int 1060 ixv_media_change(struct ifnet *ifp) 1061 { 1062 struct ixgbe_softc *sc = ifp->if_softc; 1063 struct ifmedia *ifm = &sc->media; 1064 1065 INIT_DEBUGOUT("ixv_media_change: begin"); 1066 1067 if (IFM_TYPE(ifm->ifm_media) != IFM_ETHER) 1068 return (EINVAL); 1069 1070 switch (IFM_SUBTYPE(ifm->ifm_media)) { 1071 case IFM_AUTO: 1072 break; 1073 default: 1074 device_printf(sc->dev, "Only auto media type\n"); 1075 return (EINVAL); 1076 } 1077 1078 return (0); 1079 } /* ixv_media_change */ 1080 1081 static void 1082 ixv_schedule_admin_tasklet(struct ixgbe_softc *sc) 1083 { 1084 if (sc->schedule_wqs_ok) { 1085 if (atomic_cas_uint(&sc->admin_pending, 0, 1) == 0) 1086 workqueue_enqueue(sc->admin_wq, 1087 &sc->admin_wc, NULL); 1088 } 1089 } 1090 1091 /************************************************************************ 1092 * ixv_negotiate_api 1093 * 1094 * Negotiate the Mailbox API with the PF; 1095 * start with the most featured API first. 1096 ************************************************************************/ 1097 static int 1098 ixv_negotiate_api(struct ixgbe_softc *sc) 1099 { 1100 struct ixgbe_hw *hw = &sc->hw; 1101 int mbx_api[] = { ixgbe_mbox_api_15, 1102 ixgbe_mbox_api_13, 1103 ixgbe_mbox_api_12, 1104 ixgbe_mbox_api_11, 1105 ixgbe_mbox_api_10, 1106 ixgbe_mbox_api_unknown }; 1107 int i = 0; 1108 1109 while (mbx_api[i] != ixgbe_mbox_api_unknown) { 1110 if (ixgbevf_negotiate_api_version(hw, mbx_api[i]) == 0) { 1111 if (hw->api_version >= ixgbe_mbox_api_15) 1112 ixgbe_upgrade_mbx_params_vf(hw); 1113 return (0); 1114 } 1115 i++; 1116 } 1117 1118 return (EINVAL); 1119 } /* ixv_negotiate_api */ 1120 1121 1122 /************************************************************************ 1123 * ixv_set_rxfilter - Multicast Update 1124 * 1125 * Called whenever multicast address list is updated. 1126 ************************************************************************/ 1127 static int 1128 ixv_set_rxfilter(struct ixgbe_softc *sc) 1129 { 1130 struct ixgbe_mc_addr *mta; 1131 struct ifnet *ifp = sc->ifp; 1132 struct ixgbe_hw *hw = &sc->hw; 1133 u8 *update_ptr; 1134 int mcnt = 0; 1135 struct ethercom *ec = &sc->osdep.ec; 1136 struct ether_multi *enm; 1137 struct ether_multistep step; 1138 bool overflow = false; 1139 int error, rc = 0; 1140 1141 KASSERT(mutex_owned(&sc->core_mtx)); 1142 IOCTL_DEBUGOUT("ixv_set_rxfilter: begin"); 1143 1144 mta = sc->mta; 1145 bzero(mta, sizeof(*mta) * IXGBE_MAX_VF_MC); 1146 1147 /* 1: For PROMISC */ 1148 if (ifp->if_flags & IFF_PROMISC) { 1149 error = hw->mac.ops.update_xcast_mode(hw, 1150 IXGBEVF_XCAST_MODE_PROMISC); 1151 if (error == IXGBE_ERR_NOT_TRUSTED) { 1152 device_printf(sc->dev, 1153 "this interface is not trusted\n"); 1154 error = EPERM; 1155 } else if (error == IXGBE_ERR_FEATURE_NOT_SUPPORTED) { 1156 device_printf(sc->dev, 1157 "the PF doesn't support promisc mode\n"); 1158 error = EOPNOTSUPP; 1159 } else if (error == IXGBE_ERR_NOT_IN_PROMISC) { 1160 device_printf(sc->dev, 1161 "the PF may not in promisc mode\n"); 1162 error = EINVAL; 1163 } else if (error) { 1164 device_printf(sc->dev, 1165 "failed to set promisc mode. error = %d\n", 1166 error); 1167 error = EIO; 1168 } else 1169 return 0; 1170 rc = error; 1171 } 1172 1173 /* 2: For ALLMULTI or normal */ 1174 ETHER_LOCK(ec); 1175 ETHER_FIRST_MULTI(step, ec, enm); 1176 while (enm != NULL) { 1177 if ((mcnt >= IXGBE_MAX_VF_MC) || 1178 (memcmp(enm->enm_addrlo, enm->enm_addrhi, 1179 ETHER_ADDR_LEN) != 0)) { 1180 overflow = true; 1181 break; 1182 } 1183 bcopy(enm->enm_addrlo, 1184 mta[mcnt].addr, IXGBE_ETH_LENGTH_OF_ADDRESS); 1185 mcnt++; 1186 ETHER_NEXT_MULTI(step, enm); 1187 } 1188 ETHER_UNLOCK(ec); 1189 1190 /* 3: For ALLMULTI */ 1191 if (overflow) { 1192 error = hw->mac.ops.update_xcast_mode(hw, 1193 IXGBEVF_XCAST_MODE_ALLMULTI); 1194 if (error == IXGBE_ERR_NOT_TRUSTED) { 1195 device_printf(sc->dev, 1196 "this interface is not trusted\n"); 1197 error = EPERM; 1198 } else if (error == IXGBE_ERR_FEATURE_NOT_SUPPORTED) { 1199 device_printf(sc->dev, 1200 "the PF doesn't support allmulti mode\n"); 1201 error = EOPNOTSUPP; 1202 } else if (error) { 1203 device_printf(sc->dev, 1204 "number of Ethernet multicast addresses " 1205 "exceeds the limit (%d). error = %d\n", 1206 IXGBE_MAX_VF_MC, error); 1207 error = ENOSPC; 1208 } else { 1209 ETHER_LOCK(ec); 1210 ec->ec_flags |= ETHER_F_ALLMULTI; 1211 ETHER_UNLOCK(ec); 1212 return rc; /* Promisc might have failed */ 1213 } 1214 1215 if (rc == 0) 1216 rc = error; 1217 1218 /* Continue to update the multicast table as many as we can */ 1219 } 1220 1221 /* 4: For normal operation */ 1222 error = hw->mac.ops.update_xcast_mode(hw, IXGBEVF_XCAST_MODE_MULTI); 1223 if ((error == IXGBE_ERR_FEATURE_NOT_SUPPORTED) || (error == 0)) { 1224 /* Normal operation */ 1225 ETHER_LOCK(ec); 1226 ec->ec_flags &= ~ETHER_F_ALLMULTI; 1227 ETHER_UNLOCK(ec); 1228 error = 0; 1229 } else if (error) { 1230 device_printf(sc->dev, 1231 "failed to set Ethernet multicast address " 1232 "operation to normal. error = %d\n", error); 1233 } 1234 1235 update_ptr = (u8 *)mta; 1236 error = sc->hw.mac.ops.update_mc_addr_list(&sc->hw, 1237 update_ptr, mcnt, ixv_mc_array_itr, TRUE); 1238 if (rc == 0) 1239 rc = error; 1240 1241 return rc; 1242 } /* ixv_set_rxfilter */ 1243 1244 /************************************************************************ 1245 * ixv_mc_array_itr 1246 * 1247 * An iterator function needed by the multicast shared code. 1248 * It feeds the shared code routine the addresses in the 1249 * array of ixv_set_rxfilter() one by one. 1250 ************************************************************************/ 1251 static u8 * 1252 ixv_mc_array_itr(struct ixgbe_hw *hw, u8 **update_ptr, u32 *vmdq) 1253 { 1254 struct ixgbe_mc_addr *mta; 1255 1256 mta = (struct ixgbe_mc_addr *)*update_ptr; 1257 1258 *vmdq = 0; 1259 *update_ptr = (u8*)(mta + 1); 1260 1261 return (mta->addr); 1262 } /* ixv_mc_array_itr */ 1263 1264 /************************************************************************ 1265 * ixv_local_timer - Timer routine 1266 * 1267 * Checks for link status, updates statistics, 1268 * and runs the watchdog check. 1269 ************************************************************************/ 1270 static void 1271 ixv_local_timer(void *arg) 1272 { 1273 struct ixgbe_softc *sc = arg; 1274 1275 if (sc->schedule_wqs_ok) { 1276 if (atomic_cas_uint(&sc->timer_pending, 0, 1) == 0) 1277 workqueue_enqueue(sc->timer_wq, 1278 &sc->timer_wc, NULL); 1279 } 1280 } 1281 1282 static void 1283 ixv_handle_timer(struct work *wk, void *context) 1284 { 1285 struct ixgbe_softc *sc = context; 1286 device_t dev = sc->dev; 1287 struct ix_queue *que = sc->queues; 1288 u64 queues = 0; 1289 u64 v0, v1, v2, v3, v4, v5, v6, v7; 1290 int hung = 0; 1291 int i; 1292 1293 IXGBE_CORE_LOCK(sc); 1294 1295 if (ixv_check_link(sc)) { 1296 ixv_init_locked(sc); 1297 IXGBE_CORE_UNLOCK(sc); 1298 return; 1299 } 1300 1301 /* Stats Update */ 1302 ixv_update_stats(sc); 1303 1304 /* Update some event counters */ 1305 v0 = v1 = v2 = v3 = v4 = v5 = v6 = v7 = 0; 1306 que = sc->queues; 1307 for (i = 0; i < sc->num_queues; i++, que++) { 1308 struct tx_ring *txr = que->txr; 1309 1310 v0 += txr->q_efbig_tx_dma_setup; 1311 v1 += txr->q_mbuf_defrag_failed; 1312 v2 += txr->q_efbig2_tx_dma_setup; 1313 v3 += txr->q_einval_tx_dma_setup; 1314 v4 += txr->q_other_tx_dma_setup; 1315 v5 += txr->q_eagain_tx_dma_setup; 1316 v6 += txr->q_enomem_tx_dma_setup; 1317 v7 += txr->q_tso_err; 1318 } 1319 IXGBE_EVC_STORE(&sc->efbig_tx_dma_setup, v0); 1320 IXGBE_EVC_STORE(&sc->mbuf_defrag_failed, v1); 1321 IXGBE_EVC_STORE(&sc->efbig2_tx_dma_setup, v2); 1322 IXGBE_EVC_STORE(&sc->einval_tx_dma_setup, v3); 1323 IXGBE_EVC_STORE(&sc->other_tx_dma_setup, v4); 1324 IXGBE_EVC_STORE(&sc->eagain_tx_dma_setup, v5); 1325 IXGBE_EVC_STORE(&sc->enomem_tx_dma_setup, v6); 1326 IXGBE_EVC_STORE(&sc->tso_err, v7); 1327 1328 /* 1329 * Check the TX queues status 1330 * - mark hung queues so we don't schedule on them 1331 * - watchdog only if all queues show hung 1332 */ 1333 que = sc->queues; 1334 for (i = 0; i < sc->num_queues; i++, que++) { 1335 /* Keep track of queues with work for soft irq */ 1336 if (que->txr->busy) 1337 queues |= ((u64)1 << que->me); 1338 /* 1339 * Each time txeof runs without cleaning, but there 1340 * are uncleaned descriptors it increments busy. If 1341 * we get to the MAX we declare it hung. 1342 */ 1343 if (que->busy == IXGBE_QUEUE_HUNG) { 1344 ++hung; 1345 /* Mark the queue as inactive */ 1346 sc->active_queues &= ~((u64)1 << que->me); 1347 continue; 1348 } else { 1349 /* Check if we've come back from hung */ 1350 if ((sc->active_queues & ((u64)1 << que->me)) == 0) 1351 sc->active_queues |= ((u64)1 << que->me); 1352 } 1353 if (que->busy >= IXGBE_MAX_TX_BUSY) { 1354 device_printf(dev, 1355 "Warning queue %d appears to be hung!\n", i); 1356 que->txr->busy = IXGBE_QUEUE_HUNG; 1357 ++hung; 1358 } 1359 } 1360 1361 /* Only truly watchdog if all queues show hung */ 1362 if (hung == sc->num_queues) 1363 goto watchdog; 1364 #if 0 1365 else if (queues != 0) { /* Force an IRQ on queues with work */ 1366 ixv_rearm_queues(sc, queues); 1367 } 1368 #endif 1369 1370 atomic_store_relaxed(&sc->timer_pending, 0); 1371 IXGBE_CORE_UNLOCK(sc); 1372 callout_reset(&sc->timer, hz, ixv_local_timer, sc); 1373 1374 return; 1375 1376 watchdog: 1377 device_printf(sc->dev, "Watchdog timeout -- resetting\n"); 1378 sc->ifp->if_flags &= ~IFF_RUNNING; 1379 IXGBE_EVC_ADD(&sc->watchdog_events, 1); 1380 ixv_init_locked(sc); 1381 IXGBE_CORE_UNLOCK(sc); 1382 } /* ixv_handle_timer */ 1383 1384 /************************************************************************ 1385 * ixv_update_link_status - Update OS on link state 1386 * 1387 * Note: Only updates the OS on the cached link state. 1388 * The real check of the hardware only happens with 1389 * a link interrupt. 1390 ************************************************************************/ 1391 static void 1392 ixv_update_link_status(struct ixgbe_softc *sc) 1393 { 1394 struct ifnet *ifp = sc->ifp; 1395 device_t dev = sc->dev; 1396 1397 KASSERT(mutex_owned(&sc->core_mtx)); 1398 1399 if (sc->link_up && sc->link_enabled) { 1400 if (sc->link_active != LINK_STATE_UP) { 1401 if (bootverbose) { 1402 const char *bpsmsg; 1403 1404 switch (sc->link_speed) { 1405 case IXGBE_LINK_SPEED_10GB_FULL: 1406 bpsmsg = "10 Gbps"; 1407 break; 1408 case IXGBE_LINK_SPEED_5GB_FULL: 1409 bpsmsg = "5 Gbps"; 1410 break; 1411 case IXGBE_LINK_SPEED_2_5GB_FULL: 1412 bpsmsg = "2.5 Gbps"; 1413 break; 1414 case IXGBE_LINK_SPEED_1GB_FULL: 1415 bpsmsg = "1 Gbps"; 1416 break; 1417 case IXGBE_LINK_SPEED_100_FULL: 1418 bpsmsg = "100 Mbps"; 1419 break; 1420 case IXGBE_LINK_SPEED_10_FULL: 1421 bpsmsg = "10 Mbps"; 1422 break; 1423 default: 1424 bpsmsg = "unknown speed"; 1425 break; 1426 } 1427 device_printf(dev, "Link is up %s %s \n", 1428 bpsmsg, "Full Duplex"); 1429 } 1430 sc->link_active = LINK_STATE_UP; 1431 if_link_state_change(ifp, LINK_STATE_UP); 1432 } 1433 } else { 1434 /* 1435 * Do it when link active changes to DOWN. i.e. 1436 * a) LINK_STATE_UNKNOWN -> LINK_STATE_DOWN 1437 * b) LINK_STATE_UP -> LINK_STATE_DOWN 1438 */ 1439 if (sc->link_active != LINK_STATE_DOWN) { 1440 if (bootverbose) 1441 device_printf(dev, "Link is Down\n"); 1442 if_link_state_change(ifp, LINK_STATE_DOWN); 1443 sc->link_active = LINK_STATE_DOWN; 1444 } 1445 } 1446 } /* ixv_update_link_status */ 1447 1448 1449 /************************************************************************ 1450 * ixv_stop - Stop the hardware 1451 * 1452 * Disables all traffic on the adapter by issuing a 1453 * global reset on the MAC and deallocates TX/RX buffers. 1454 ************************************************************************/ 1455 static void 1456 ixv_ifstop(struct ifnet *ifp, int disable) 1457 { 1458 struct ixgbe_softc *sc = ifp->if_softc; 1459 1460 IXGBE_CORE_LOCK(sc); 1461 ixv_stop_locked(sc); 1462 IXGBE_CORE_UNLOCK(sc); 1463 1464 workqueue_wait(sc->admin_wq, &sc->admin_wc); 1465 atomic_store_relaxed(&sc->admin_pending, 0); 1466 workqueue_wait(sc->timer_wq, &sc->timer_wc); 1467 atomic_store_relaxed(&sc->timer_pending, 0); 1468 } 1469 1470 static void 1471 ixv_stop_locked(void *arg) 1472 { 1473 struct ifnet *ifp; 1474 struct ixgbe_softc *sc = arg; 1475 struct ixgbe_hw *hw = &sc->hw; 1476 1477 ifp = sc->ifp; 1478 1479 KASSERT(mutex_owned(&sc->core_mtx)); 1480 1481 INIT_DEBUGOUT("ixv_stop_locked: begin\n"); 1482 ixv_disable_intr(sc); 1483 1484 /* Tell the stack that the interface is no longer active */ 1485 ifp->if_flags &= ~(IFF_RUNNING | IFF_OACTIVE); 1486 1487 hw->mac.ops.reset_hw(hw); 1488 sc->hw.adapter_stopped = FALSE; 1489 hw->mac.ops.stop_adapter(hw); 1490 callout_stop(&sc->timer); 1491 1492 /* Don't schedule workqueues. */ 1493 sc->schedule_wqs_ok = false; 1494 1495 /* reprogram the RAR[0] in case user changed it. */ 1496 hw->mac.ops.set_rar(hw, 0, hw->mac.addr, 0, IXGBE_RAH_AV); 1497 1498 return; 1499 } /* ixv_stop_locked */ 1500 1501 1502 /************************************************************************ 1503 * ixv_allocate_pci_resources 1504 ************************************************************************/ 1505 static int 1506 ixv_allocate_pci_resources(struct ixgbe_softc *sc, 1507 const struct pci_attach_args *pa) 1508 { 1509 pcireg_t memtype, csr; 1510 device_t dev = sc->dev; 1511 bus_addr_t addr; 1512 int flags; 1513 1514 memtype = pci_mapreg_type(pa->pa_pc, pa->pa_tag, PCI_BAR(0)); 1515 switch (memtype) { 1516 case PCI_MAPREG_TYPE_MEM | PCI_MAPREG_MEM_TYPE_32BIT: 1517 case PCI_MAPREG_TYPE_MEM | PCI_MAPREG_MEM_TYPE_64BIT: 1518 sc->osdep.mem_bus_space_tag = pa->pa_memt; 1519 if (pci_mapreg_info(pa->pa_pc, pa->pa_tag, PCI_BAR(0), 1520 memtype, &addr, &sc->osdep.mem_size, &flags) != 0) 1521 goto map_err; 1522 if ((flags & BUS_SPACE_MAP_PREFETCHABLE) != 0) { 1523 aprint_normal_dev(dev, "clearing prefetchable bit\n"); 1524 flags &= ~BUS_SPACE_MAP_PREFETCHABLE; 1525 } 1526 if (bus_space_map(sc->osdep.mem_bus_space_tag, addr, 1527 sc->osdep.mem_size, flags, 1528 &sc->osdep.mem_bus_space_handle) != 0) { 1529 map_err: 1530 sc->osdep.mem_size = 0; 1531 aprint_error_dev(dev, "unable to map BAR0\n"); 1532 return ENXIO; 1533 } 1534 /* 1535 * Enable address decoding for memory range in case it's not 1536 * set. 1537 */ 1538 csr = pci_conf_read(pa->pa_pc, pa->pa_tag, 1539 PCI_COMMAND_STATUS_REG); 1540 csr |= PCI_COMMAND_MEM_ENABLE; 1541 pci_conf_write(pa->pa_pc, pa->pa_tag, PCI_COMMAND_STATUS_REG, 1542 csr); 1543 break; 1544 default: 1545 aprint_error_dev(dev, "unexpected type on BAR0\n"); 1546 return ENXIO; 1547 } 1548 1549 /* Pick up the tuneable queues */ 1550 sc->num_queues = ixv_num_queues; 1551 1552 return (0); 1553 } /* ixv_allocate_pci_resources */ 1554 1555 static void 1556 ixv_free_deferred_handlers(struct ixgbe_softc *sc) 1557 { 1558 struct ix_queue *que = sc->queues; 1559 struct tx_ring *txr = sc->tx_rings; 1560 int i; 1561 1562 for (i = 0; i < sc->num_queues; i++, que++, txr++) { 1563 if (!(sc->feat_en & IXGBE_FEATURE_LEGACY_TX)) { 1564 if (txr->txr_si != NULL) 1565 softint_disestablish(txr->txr_si); 1566 } 1567 if (que->que_si != NULL) 1568 softint_disestablish(que->que_si); 1569 } 1570 if (sc->txr_wq != NULL) 1571 workqueue_destroy(sc->txr_wq); 1572 if (sc->txr_wq_enqueued != NULL) 1573 percpu_free(sc->txr_wq_enqueued, sizeof(u_int)); 1574 if (sc->que_wq != NULL) 1575 workqueue_destroy(sc->que_wq); 1576 1577 /* Drain the Mailbox(link) queue */ 1578 if (sc->admin_wq != NULL) { 1579 workqueue_destroy(sc->admin_wq); 1580 sc->admin_wq = NULL; 1581 } 1582 if (sc->timer_wq != NULL) { 1583 workqueue_destroy(sc->timer_wq); 1584 sc->timer_wq = NULL; 1585 } 1586 } /* ixv_free_deferred_handlers */ 1587 1588 /************************************************************************ 1589 * ixv_free_pci_resources 1590 ************************************************************************/ 1591 static void 1592 ixv_free_pci_resources(struct ixgbe_softc *sc) 1593 { 1594 struct ix_queue *que = sc->queues; 1595 int rid; 1596 1597 /* 1598 * Release all msix queue resources: 1599 */ 1600 for (int i = 0; i < sc->num_queues; i++, que++) { 1601 if (que->res != NULL) 1602 pci_intr_disestablish(sc->osdep.pc, 1603 sc->osdep.ihs[i]); 1604 } 1605 1606 1607 /* Clean the Mailbox interrupt last */ 1608 rid = sc->vector; 1609 1610 if (sc->osdep.ihs[rid] != NULL) { 1611 pci_intr_disestablish(sc->osdep.pc, 1612 sc->osdep.ihs[rid]); 1613 sc->osdep.ihs[rid] = NULL; 1614 } 1615 1616 pci_intr_release(sc->osdep.pc, sc->osdep.intrs, 1617 sc->osdep.nintrs); 1618 1619 if (sc->osdep.mem_size != 0) { 1620 bus_space_unmap(sc->osdep.mem_bus_space_tag, 1621 sc->osdep.mem_bus_space_handle, 1622 sc->osdep.mem_size); 1623 } 1624 1625 return; 1626 } /* ixv_free_pci_resources */ 1627 1628 /************************************************************************ 1629 * ixv_setup_interface 1630 * 1631 * Setup networking device structure and register an interface. 1632 ************************************************************************/ 1633 static int 1634 ixv_setup_interface(device_t dev, struct ixgbe_softc *sc) 1635 { 1636 struct ethercom *ec = &sc->osdep.ec; 1637 struct ifnet *ifp; 1638 1639 INIT_DEBUGOUT("ixv_setup_interface: begin"); 1640 1641 ifp = sc->ifp = &ec->ec_if; 1642 strlcpy(ifp->if_xname, device_xname(dev), IFNAMSIZ); 1643 ifp->if_baudrate = IF_Gbps(10); 1644 ifp->if_init = ixv_init; 1645 ifp->if_stop = ixv_ifstop; 1646 ifp->if_softc = sc; 1647 ifp->if_flags = IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST; 1648 #ifdef IXGBE_MPSAFE 1649 ifp->if_extflags = IFEF_MPSAFE; 1650 #endif 1651 ifp->if_ioctl = ixv_ioctl; 1652 if (sc->feat_en & IXGBE_FEATURE_LEGACY_TX) { 1653 #if 0 1654 ixv_start_locked = ixgbe_legacy_start_locked; 1655 #endif 1656 } else { 1657 ifp->if_transmit = ixgbe_mq_start; 1658 #if 0 1659 ixv_start_locked = ixgbe_mq_start_locked; 1660 #endif 1661 } 1662 ifp->if_start = ixgbe_legacy_start; 1663 IFQ_SET_MAXLEN(&ifp->if_snd, sc->num_tx_desc - 2); 1664 IFQ_SET_READY(&ifp->if_snd); 1665 1666 if_initialize(ifp); 1667 sc->ipq = if_percpuq_create(&sc->osdep.ec.ec_if); 1668 ether_ifattach(ifp, sc->hw.mac.addr); 1669 aprint_normal_dev(dev, "Ethernet address %s\n", 1670 ether_sprintf(sc->hw.mac.addr)); 1671 /* 1672 * We use per TX queue softint, so if_deferred_start_init() isn't 1673 * used. 1674 */ 1675 ether_set_ifflags_cb(ec, ixv_ifflags_cb); 1676 1677 sc->max_frame_size = ifp->if_mtu + IXGBE_MTU_HDR; 1678 1679 /* 1680 * Tell the upper layer(s) we support long frames. 1681 */ 1682 ifp->if_hdrlen = sizeof(struct ether_vlan_header); 1683 1684 /* Set capability flags */ 1685 ifp->if_capabilities |= IFCAP_HWCSUM 1686 | IFCAP_TSOv4 1687 | IFCAP_TSOv6; 1688 ifp->if_capenable = 0; 1689 1690 ec->ec_capabilities |= ETHERCAP_VLAN_HWFILTER 1691 | ETHERCAP_VLAN_HWTAGGING 1692 | ETHERCAP_VLAN_HWCSUM 1693 | ETHERCAP_JUMBO_MTU 1694 | ETHERCAP_VLAN_MTU; 1695 1696 /* Enable the above capabilities by default */ 1697 ec->ec_capenable = ec->ec_capabilities; 1698 1699 /* Don't enable LRO by default */ 1700 #if 0 1701 /* NetBSD doesn't support LRO yet */ 1702 ifp->if_capabilities |= IFCAP_LRO; 1703 #endif 1704 1705 /* 1706 * Specify the media types supported by this adapter and register 1707 * callbacks to update media and link information 1708 */ 1709 ec->ec_ifmedia = &sc->media; 1710 ifmedia_init_with_lock(&sc->media, IFM_IMASK, ixv_media_change, 1711 ixv_media_status, &sc->core_mtx); 1712 ifmedia_add(&sc->media, IFM_ETHER | IFM_AUTO, 0, NULL); 1713 ifmedia_set(&sc->media, IFM_ETHER | IFM_AUTO); 1714 1715 if_register(ifp); 1716 1717 return 0; 1718 } /* ixv_setup_interface */ 1719 1720 1721 /************************************************************************ 1722 * ixv_initialize_transmit_units - Enable transmit unit. 1723 ************************************************************************/ 1724 static void 1725 ixv_initialize_transmit_units(struct ixgbe_softc *sc) 1726 { 1727 struct tx_ring *txr = sc->tx_rings; 1728 struct ixgbe_hw *hw = &sc->hw; 1729 int i; 1730 1731 for (i = 0; i < sc->num_queues; i++, txr++) { 1732 u64 tdba = txr->txdma.dma_paddr; 1733 u32 txctrl, txdctl; 1734 int j = txr->me; 1735 1736 /* Set WTHRESH to 8, burst writeback */ 1737 txdctl = IXGBE_READ_REG(hw, IXGBE_VFTXDCTL(j)); 1738 txdctl |= IXGBE_TX_WTHRESH << IXGBE_TXDCTL_WTHRESH_SHIFT; 1739 IXGBE_WRITE_REG(hw, IXGBE_VFTXDCTL(j), txdctl); 1740 1741 /* Set the HW Tx Head and Tail indices */ 1742 IXGBE_WRITE_REG(&sc->hw, IXGBE_VFTDH(j), 0); 1743 IXGBE_WRITE_REG(&sc->hw, IXGBE_VFTDT(j), 0); 1744 1745 /* Set Tx Tail register */ 1746 txr->tail = IXGBE_VFTDT(j); 1747 1748 txr->txr_no_space = false; 1749 1750 /* Set Ring parameters */ 1751 IXGBE_WRITE_REG(hw, IXGBE_VFTDBAL(j), 1752 (tdba & 0x00000000ffffffffULL)); 1753 IXGBE_WRITE_REG(hw, IXGBE_VFTDBAH(j), (tdba >> 32)); 1754 IXGBE_WRITE_REG(hw, IXGBE_VFTDLEN(j), 1755 sc->num_tx_desc * sizeof(struct ixgbe_legacy_tx_desc)); 1756 txctrl = IXGBE_READ_REG(hw, IXGBE_VFDCA_TXCTRL(j)); 1757 txctrl &= ~IXGBE_DCA_TXCTRL_DESC_WRO_EN; 1758 IXGBE_WRITE_REG(hw, IXGBE_VFDCA_TXCTRL(j), txctrl); 1759 1760 /* Now enable */ 1761 txdctl = IXGBE_READ_REG(hw, IXGBE_VFTXDCTL(j)); 1762 txdctl |= IXGBE_TXDCTL_ENABLE; 1763 IXGBE_WRITE_REG(hw, IXGBE_VFTXDCTL(j), txdctl); 1764 } 1765 1766 return; 1767 } /* ixv_initialize_transmit_units */ 1768 1769 1770 /************************************************************************ 1771 * ixv_initialize_rss_mapping 1772 ************************************************************************/ 1773 static void 1774 ixv_initialize_rss_mapping(struct ixgbe_softc *sc) 1775 { 1776 struct ixgbe_hw *hw = &sc->hw; 1777 u32 reta = 0, mrqc, rss_key[10]; 1778 int queue_id; 1779 int i, j; 1780 u32 rss_hash_config; 1781 1782 /* force use default RSS key. */ 1783 #ifdef __NetBSD__ 1784 rss_getkey((uint8_t *) &rss_key); 1785 #else 1786 if (sc->feat_en & IXGBE_FEATURE_RSS) { 1787 /* Fetch the configured RSS key */ 1788 rss_getkey((uint8_t *)&rss_key); 1789 } else { 1790 /* set up random bits */ 1791 cprng_fast(&rss_key, sizeof(rss_key)); 1792 } 1793 #endif 1794 1795 /* Now fill out hash function seeds */ 1796 for (i = 0; i < 10; i++) 1797 IXGBE_WRITE_REG(hw, IXGBE_VFRSSRK(i), rss_key[i]); 1798 1799 /* Set up the redirection table */ 1800 for (i = 0, j = 0; i < 64; i++, j++) { 1801 if (j == sc->num_queues) 1802 j = 0; 1803 1804 if (sc->feat_en & IXGBE_FEATURE_RSS) { 1805 /* 1806 * Fetch the RSS bucket id for the given indirection 1807 * entry. Cap it at the number of configured buckets 1808 * (which is num_queues.) 1809 */ 1810 queue_id = rss_get_indirection_to_bucket(i); 1811 queue_id = queue_id % sc->num_queues; 1812 } else 1813 queue_id = j; 1814 1815 /* 1816 * The low 8 bits are for hash value (n+0); 1817 * The next 8 bits are for hash value (n+1), etc. 1818 */ 1819 reta >>= 8; 1820 reta |= ((uint32_t)queue_id) << 24; 1821 if ((i & 3) == 3) { 1822 IXGBE_WRITE_REG(hw, IXGBE_VFRETA(i >> 2), reta); 1823 reta = 0; 1824 } 1825 } 1826 1827 /* Perform hash on these packet types */ 1828 if (sc->feat_en & IXGBE_FEATURE_RSS) 1829 rss_hash_config = rss_gethashconfig(); 1830 else { 1831 /* 1832 * Disable UDP - IP fragments aren't currently being handled 1833 * and so we end up with a mix of 2-tuple and 4-tuple 1834 * traffic. 1835 */ 1836 rss_hash_config = RSS_HASHTYPE_RSS_IPV4 1837 | RSS_HASHTYPE_RSS_TCP_IPV4 1838 | RSS_HASHTYPE_RSS_IPV6 1839 | RSS_HASHTYPE_RSS_TCP_IPV6; 1840 } 1841 1842 mrqc = IXGBE_MRQC_RSSEN; 1843 if (rss_hash_config & RSS_HASHTYPE_RSS_IPV4) 1844 mrqc |= IXGBE_MRQC_RSS_FIELD_IPV4; 1845 if (rss_hash_config & RSS_HASHTYPE_RSS_TCP_IPV4) 1846 mrqc |= IXGBE_MRQC_RSS_FIELD_IPV4_TCP; 1847 if (rss_hash_config & RSS_HASHTYPE_RSS_IPV6) 1848 mrqc |= IXGBE_MRQC_RSS_FIELD_IPV6; 1849 if (rss_hash_config & RSS_HASHTYPE_RSS_TCP_IPV6) 1850 mrqc |= IXGBE_MRQC_RSS_FIELD_IPV6_TCP; 1851 if (rss_hash_config & RSS_HASHTYPE_RSS_IPV6_EX) 1852 device_printf(sc->dev, "%s: RSS_HASHTYPE_RSS_IPV6_EX " 1853 "defined, but not supported\n", __func__); 1854 if (rss_hash_config & RSS_HASHTYPE_RSS_TCP_IPV6_EX) 1855 device_printf(sc->dev, "%s: RSS_HASHTYPE_RSS_TCP_IPV6_EX " 1856 "defined, but not supported\n", __func__); 1857 if (rss_hash_config & RSS_HASHTYPE_RSS_UDP_IPV4) 1858 mrqc |= IXGBE_MRQC_RSS_FIELD_IPV4_UDP; 1859 if (rss_hash_config & RSS_HASHTYPE_RSS_UDP_IPV6) 1860 mrqc |= IXGBE_MRQC_RSS_FIELD_IPV6_UDP; 1861 if (rss_hash_config & RSS_HASHTYPE_RSS_UDP_IPV6_EX) 1862 device_printf(sc->dev, "%s: RSS_HASHTYPE_RSS_UDP_IPV6_EX " 1863 "defined, but not supported\n", __func__); 1864 IXGBE_WRITE_REG(hw, IXGBE_VFMRQC, mrqc); 1865 } /* ixv_initialize_rss_mapping */ 1866 1867 1868 /************************************************************************ 1869 * ixv_initialize_receive_units - Setup receive registers and features. 1870 ************************************************************************/ 1871 static void 1872 ixv_initialize_receive_units(struct ixgbe_softc *sc) 1873 { 1874 struct rx_ring *rxr = sc->rx_rings; 1875 struct ixgbe_hw *hw = &sc->hw; 1876 struct ifnet *ifp = sc->ifp; 1877 u32 bufsz, psrtype; 1878 1879 if (ifp->if_mtu > ETHERMTU) 1880 bufsz = 4096 >> IXGBE_SRRCTL_BSIZEPKT_SHIFT; 1881 else 1882 bufsz = 2048 >> IXGBE_SRRCTL_BSIZEPKT_SHIFT; 1883 1884 psrtype = IXGBE_PSRTYPE_TCPHDR 1885 | IXGBE_PSRTYPE_UDPHDR 1886 | IXGBE_PSRTYPE_IPV4HDR 1887 | IXGBE_PSRTYPE_IPV6HDR 1888 | IXGBE_PSRTYPE_L2HDR; 1889 1890 if (sc->num_queues > 1) 1891 psrtype |= 1 << 29; 1892 1893 IXGBE_WRITE_REG(hw, IXGBE_VFPSRTYPE, psrtype); 1894 1895 /* Tell PF our max_frame size */ 1896 if (ixgbevf_rlpml_set_vf(hw, sc->max_frame_size) != 0) { 1897 device_printf(sc->dev, "There is a problem with the PF " 1898 "setup. It is likely the receive unit for this VF will " 1899 "not function correctly.\n"); 1900 } 1901 1902 for (int i = 0; i < sc->num_queues; i++, rxr++) { 1903 u64 rdba = rxr->rxdma.dma_paddr; 1904 u32 reg, rxdctl; 1905 int j = rxr->me; 1906 1907 /* Disable the queue */ 1908 rxdctl = IXGBE_READ_REG(hw, IXGBE_VFRXDCTL(j)); 1909 rxdctl &= ~IXGBE_RXDCTL_ENABLE; 1910 IXGBE_WRITE_REG(hw, IXGBE_VFRXDCTL(j), rxdctl); 1911 for (int k = 0; k < 10; k++) { 1912 if (IXGBE_READ_REG(hw, IXGBE_VFRXDCTL(j)) & 1913 IXGBE_RXDCTL_ENABLE) 1914 msec_delay(1); 1915 else 1916 break; 1917 } 1918 IXGBE_WRITE_BARRIER(hw); 1919 /* Setup the Base and Length of the Rx Descriptor Ring */ 1920 IXGBE_WRITE_REG(hw, IXGBE_VFRDBAL(j), 1921 (rdba & 0x00000000ffffffffULL)); 1922 IXGBE_WRITE_REG(hw, IXGBE_VFRDBAH(j), (rdba >> 32)); 1923 IXGBE_WRITE_REG(hw, IXGBE_VFRDLEN(j), 1924 sc->num_rx_desc * sizeof(union ixgbe_adv_rx_desc)); 1925 1926 /* Reset the ring indices */ 1927 IXGBE_WRITE_REG(hw, IXGBE_VFRDH(rxr->me), 0); 1928 IXGBE_WRITE_REG(hw, IXGBE_VFRDT(rxr->me), 0); 1929 1930 /* Set up the SRRCTL register */ 1931 reg = IXGBE_READ_REG(hw, IXGBE_VFSRRCTL(j)); 1932 reg &= ~IXGBE_SRRCTL_BSIZEHDR_MASK; 1933 reg &= ~IXGBE_SRRCTL_BSIZEPKT_MASK; 1934 reg |= bufsz; 1935 reg |= IXGBE_SRRCTL_DESCTYPE_ADV_ONEBUF; 1936 IXGBE_WRITE_REG(hw, IXGBE_VFSRRCTL(j), reg); 1937 1938 /* Capture Rx Tail index */ 1939 rxr->tail = IXGBE_VFRDT(rxr->me); 1940 1941 /* Do the queue enabling last */ 1942 rxdctl |= IXGBE_RXDCTL_ENABLE | IXGBE_RXDCTL_VME; 1943 IXGBE_WRITE_REG(hw, IXGBE_VFRXDCTL(j), rxdctl); 1944 for (int k = 0; k < 10; k++) { 1945 if (IXGBE_READ_REG(hw, IXGBE_VFRXDCTL(j)) & 1946 IXGBE_RXDCTL_ENABLE) 1947 break; 1948 msec_delay(1); 1949 } 1950 IXGBE_WRITE_BARRIER(hw); 1951 1952 /* Set the Tail Pointer */ 1953 #ifdef DEV_NETMAP 1954 /* 1955 * In netmap mode, we must preserve the buffers made 1956 * available to userspace before the if_init() 1957 * (this is true by default on the TX side, because 1958 * init makes all buffers available to userspace). 1959 * 1960 * netmap_reset() and the device specific routines 1961 * (e.g. ixgbe_setup_receive_rings()) map these 1962 * buffers at the end of the NIC ring, so here we 1963 * must set the RDT (tail) register to make sure 1964 * they are not overwritten. 1965 * 1966 * In this driver the NIC ring starts at RDH = 0, 1967 * RDT points to the last slot available for reception (?), 1968 * so RDT = num_rx_desc - 1 means the whole ring is available. 1969 */ 1970 if ((sc->feat_en & IXGBE_FEATURE_NETMAP) && 1971 (ifp->if_capenable & IFCAP_NETMAP)) { 1972 struct netmap_adapter *na = NA(sc->ifp); 1973 struct netmap_kring *kring = na->rx_rings[i]; 1974 int t = na->num_rx_desc - 1 - nm_kr_rxspace(kring); 1975 1976 IXGBE_WRITE_REG(hw, IXGBE_VFRDT(rxr->me), t); 1977 } else 1978 #endif /* DEV_NETMAP */ 1979 IXGBE_WRITE_REG(hw, IXGBE_VFRDT(rxr->me), 1980 sc->num_rx_desc - 1); 1981 } 1982 1983 if (sc->hw.mac.type >= ixgbe_mac_X550_vf) 1984 ixv_initialize_rss_mapping(sc); 1985 } /* ixv_initialize_receive_units */ 1986 1987 /************************************************************************ 1988 * ixv_sysctl_tdh_handler - Transmit Descriptor Head handler function 1989 * 1990 * Retrieves the TDH value from the hardware 1991 ************************************************************************/ 1992 static int 1993 ixv_sysctl_tdh_handler(SYSCTLFN_ARGS) 1994 { 1995 struct sysctlnode node = *rnode; 1996 struct tx_ring *txr = (struct tx_ring *)node.sysctl_data; 1997 uint32_t val; 1998 1999 if (!txr) 2000 return (0); 2001 2002 val = IXGBE_READ_REG(&txr->sc->hw, IXGBE_VFTDH(txr->me)); 2003 node.sysctl_data = &val; 2004 return sysctl_lookup(SYSCTLFN_CALL(&node)); 2005 } /* ixv_sysctl_tdh_handler */ 2006 2007 /************************************************************************ 2008 * ixgbe_sysctl_tdt_handler - Transmit Descriptor Tail handler function 2009 * 2010 * Retrieves the TDT value from the hardware 2011 ************************************************************************/ 2012 static int 2013 ixv_sysctl_tdt_handler(SYSCTLFN_ARGS) 2014 { 2015 struct sysctlnode node = *rnode; 2016 struct tx_ring *txr = (struct tx_ring *)node.sysctl_data; 2017 uint32_t val; 2018 2019 if (!txr) 2020 return (0); 2021 2022 val = IXGBE_READ_REG(&txr->sc->hw, IXGBE_VFTDT(txr->me)); 2023 node.sysctl_data = &val; 2024 return sysctl_lookup(SYSCTLFN_CALL(&node)); 2025 } /* ixv_sysctl_tdt_handler */ 2026 2027 /************************************************************************ 2028 * ixv_sysctl_next_to_check_handler - Receive Descriptor next to check 2029 * handler function 2030 * 2031 * Retrieves the next_to_check value 2032 ************************************************************************/ 2033 static int 2034 ixv_sysctl_next_to_check_handler(SYSCTLFN_ARGS) 2035 { 2036 struct sysctlnode node = *rnode; 2037 struct rx_ring *rxr = (struct rx_ring *)node.sysctl_data; 2038 uint32_t val; 2039 2040 if (!rxr) 2041 return (0); 2042 2043 val = rxr->next_to_check; 2044 node.sysctl_data = &val; 2045 return sysctl_lookup(SYSCTLFN_CALL(&node)); 2046 } /* ixv_sysctl_next_to_check_handler */ 2047 2048 /************************************************************************ 2049 * ixv_sysctl_next_to_refresh_handler - Receive Descriptor next to refresh 2050 * handler function 2051 * 2052 * Retrieves the next_to_refresh value 2053 ************************************************************************/ 2054 static int 2055 ixv_sysctl_next_to_refresh_handler(SYSCTLFN_ARGS) 2056 { 2057 struct sysctlnode node = *rnode; 2058 struct rx_ring *rxr = (struct rx_ring *)node.sysctl_data; 2059 struct ixgbe_softc *sc; 2060 uint32_t val; 2061 2062 if (!rxr) 2063 return (0); 2064 2065 sc = rxr->sc; 2066 if (ixgbe_fw_recovery_mode_swflag(sc)) 2067 return (EPERM); 2068 2069 val = rxr->next_to_refresh; 2070 node.sysctl_data = &val; 2071 return sysctl_lookup(SYSCTLFN_CALL(&node)); 2072 } /* ixv_sysctl_next_to_refresh_handler */ 2073 2074 /************************************************************************ 2075 * ixv_sysctl_rdh_handler - Receive Descriptor Head handler function 2076 * 2077 * Retrieves the RDH value from the hardware 2078 ************************************************************************/ 2079 static int 2080 ixv_sysctl_rdh_handler(SYSCTLFN_ARGS) 2081 { 2082 struct sysctlnode node = *rnode; 2083 struct rx_ring *rxr = (struct rx_ring *)node.sysctl_data; 2084 uint32_t val; 2085 2086 if (!rxr) 2087 return (0); 2088 2089 val = IXGBE_READ_REG(&rxr->sc->hw, IXGBE_VFRDH(rxr->me)); 2090 node.sysctl_data = &val; 2091 return sysctl_lookup(SYSCTLFN_CALL(&node)); 2092 } /* ixv_sysctl_rdh_handler */ 2093 2094 /************************************************************************ 2095 * ixv_sysctl_rdt_handler - Receive Descriptor Tail handler function 2096 * 2097 * Retrieves the RDT value from the hardware 2098 ************************************************************************/ 2099 static int 2100 ixv_sysctl_rdt_handler(SYSCTLFN_ARGS) 2101 { 2102 struct sysctlnode node = *rnode; 2103 struct rx_ring *rxr = (struct rx_ring *)node.sysctl_data; 2104 uint32_t val; 2105 2106 if (!rxr) 2107 return (0); 2108 2109 val = IXGBE_READ_REG(&rxr->sc->hw, IXGBE_VFRDT(rxr->me)); 2110 node.sysctl_data = &val; 2111 return sysctl_lookup(SYSCTLFN_CALL(&node)); 2112 } /* ixv_sysctl_rdt_handler */ 2113 2114 static void 2115 ixv_setup_vlan_tagging(struct ixgbe_softc *sc) 2116 { 2117 struct ethercom *ec = &sc->osdep.ec; 2118 struct ixgbe_hw *hw = &sc->hw; 2119 struct rx_ring *rxr; 2120 u32 ctrl; 2121 int i; 2122 bool hwtagging; 2123 2124 /* Enable HW tagging only if any vlan is attached */ 2125 hwtagging = (ec->ec_capenable & ETHERCAP_VLAN_HWTAGGING) 2126 && VLAN_ATTACHED(ec); 2127 2128 /* Enable the queues */ 2129 for (i = 0; i < sc->num_queues; i++) { 2130 rxr = &sc->rx_rings[i]; 2131 ctrl = IXGBE_READ_REG(hw, IXGBE_VFRXDCTL(rxr->me)); 2132 if (hwtagging) 2133 ctrl |= IXGBE_RXDCTL_VME; 2134 else 2135 ctrl &= ~IXGBE_RXDCTL_VME; 2136 IXGBE_WRITE_REG(hw, IXGBE_VFRXDCTL(rxr->me), ctrl); 2137 /* 2138 * Let Rx path know that it needs to store VLAN tag 2139 * as part of extra mbuf info. 2140 */ 2141 rxr->vtag_strip = hwtagging ? TRUE : FALSE; 2142 } 2143 } /* ixv_setup_vlan_tagging */ 2144 2145 /************************************************************************ 2146 * ixv_setup_vlan_support 2147 ************************************************************************/ 2148 static int 2149 ixv_setup_vlan_support(struct ixgbe_softc *sc) 2150 { 2151 struct ethercom *ec = &sc->osdep.ec; 2152 struct ixgbe_hw *hw = &sc->hw; 2153 u32 vid, vfta, retry; 2154 struct vlanid_list *vlanidp; 2155 int rv, error = 0; 2156 2157 /* 2158 * This function is called from both if_init and ifflags_cb() 2159 * on NetBSD. 2160 */ 2161 2162 /* 2163 * Part 1: 2164 * Setup VLAN HW tagging 2165 */ 2166 ixv_setup_vlan_tagging(sc); 2167 2168 if (!VLAN_ATTACHED(ec)) 2169 return 0; 2170 2171 /* 2172 * Part 2: 2173 * Setup VLAN HW filter 2174 */ 2175 /* Cleanup shadow_vfta */ 2176 for (int i = 0; i < IXGBE_VFTA_SIZE; i++) 2177 sc->shadow_vfta[i] = 0; 2178 /* Generate shadow_vfta from ec_vids */ 2179 ETHER_LOCK(ec); 2180 SIMPLEQ_FOREACH(vlanidp, &ec->ec_vids, vid_list) { 2181 uint32_t idx; 2182 2183 idx = vlanidp->vid / 32; 2184 KASSERT(idx < IXGBE_VFTA_SIZE); 2185 sc->shadow_vfta[idx] |= (u32)1 << (vlanidp->vid % 32); 2186 } 2187 ETHER_UNLOCK(ec); 2188 2189 /* 2190 * A soft reset zero's out the VFTA, so 2191 * we need to repopulate it now. 2192 */ 2193 for (int i = 0; i < IXGBE_VFTA_SIZE; i++) { 2194 if (sc->shadow_vfta[i] == 0) 2195 continue; 2196 vfta = sc->shadow_vfta[i]; 2197 /* 2198 * Reconstruct the vlan id's 2199 * based on the bits set in each 2200 * of the array ints. 2201 */ 2202 for (int j = 0; j < 32; j++) { 2203 retry = 0; 2204 if ((vfta & ((u32)1 << j)) == 0) 2205 continue; 2206 vid = (i * 32) + j; 2207 2208 /* Call the shared code mailbox routine */ 2209 while ((rv = hw->mac.ops.set_vfta(hw, vid, 0, TRUE, 2210 FALSE)) != 0) { 2211 if (++retry > 5) { 2212 device_printf(sc->dev, 2213 "%s: max retry exceeded\n", 2214 __func__); 2215 break; 2216 } 2217 } 2218 if (rv != 0) { 2219 device_printf(sc->dev, 2220 "failed to set vlan %d\n", vid); 2221 error = EACCES; 2222 } 2223 } 2224 } 2225 return error; 2226 } /* ixv_setup_vlan_support */ 2227 2228 static int 2229 ixv_vlan_cb(struct ethercom *ec, uint16_t vid, bool set) 2230 { 2231 struct ifnet *ifp = &ec->ec_if; 2232 struct ixgbe_softc *sc = ifp->if_softc; 2233 int rv; 2234 2235 if (set) 2236 rv = ixv_register_vlan(sc, vid); 2237 else 2238 rv = ixv_unregister_vlan(sc, vid); 2239 2240 if (rv != 0) 2241 return rv; 2242 2243 /* 2244 * Control VLAN HW tagging when ec_nvlan is changed from 1 to 0 2245 * or 0 to 1. 2246 */ 2247 if ((set && (ec->ec_nvlans == 1)) || (!set && (ec->ec_nvlans == 0))) 2248 ixv_setup_vlan_tagging(sc); 2249 2250 return rv; 2251 } 2252 2253 /************************************************************************ 2254 * ixv_register_vlan 2255 * 2256 * Run via a vlan config EVENT, it enables us to use the 2257 * HW Filter table since we can get the vlan id. This just 2258 * creates the entry in the soft version of the VFTA, init 2259 * will repopulate the real table. 2260 ************************************************************************/ 2261 static int 2262 ixv_register_vlan(struct ixgbe_softc *sc, u16 vtag) 2263 { 2264 struct ixgbe_hw *hw = &sc->hw; 2265 u16 index, bit; 2266 int error; 2267 2268 if ((vtag == 0) || (vtag > 4095)) /* Invalid */ 2269 return EINVAL; 2270 IXGBE_CORE_LOCK(sc); 2271 index = (vtag >> 5) & 0x7F; 2272 bit = vtag & 0x1F; 2273 sc->shadow_vfta[index] |= ((u32)1 << bit); 2274 error = hw->mac.ops.set_vfta(hw, vtag, 0, true, false); 2275 IXGBE_CORE_UNLOCK(sc); 2276 2277 if (error != 0) { 2278 device_printf(sc->dev, "failed to register vlan %hu\n", vtag); 2279 error = EACCES; 2280 } 2281 return error; 2282 } /* ixv_register_vlan */ 2283 2284 /************************************************************************ 2285 * ixv_unregister_vlan 2286 * 2287 * Run via a vlan unconfig EVENT, remove our entry 2288 * in the soft vfta. 2289 ************************************************************************/ 2290 static int 2291 ixv_unregister_vlan(struct ixgbe_softc *sc, u16 vtag) 2292 { 2293 struct ixgbe_hw *hw = &sc->hw; 2294 u16 index, bit; 2295 int error; 2296 2297 if ((vtag == 0) || (vtag > 4095)) /* Invalid */ 2298 return EINVAL; 2299 2300 IXGBE_CORE_LOCK(sc); 2301 index = (vtag >> 5) & 0x7F; 2302 bit = vtag & 0x1F; 2303 sc->shadow_vfta[index] &= ~((u32)1 << bit); 2304 error = hw->mac.ops.set_vfta(hw, vtag, 0, false, false); 2305 IXGBE_CORE_UNLOCK(sc); 2306 2307 if (error != 0) { 2308 device_printf(sc->dev, "failed to unregister vlan %hu\n", 2309 vtag); 2310 error = EIO; 2311 } 2312 return error; 2313 } /* ixv_unregister_vlan */ 2314 2315 /************************************************************************ 2316 * ixv_enable_intr 2317 ************************************************************************/ 2318 static void 2319 ixv_enable_intr(struct ixgbe_softc *sc) 2320 { 2321 struct ixgbe_hw *hw = &sc->hw; 2322 struct ix_queue *que = sc->queues; 2323 u32 mask; 2324 int i; 2325 2326 /* For VTEIAC */ 2327 mask = (1 << sc->vector); 2328 for (i = 0; i < sc->num_queues; i++, que++) 2329 mask |= (1 << que->msix); 2330 IXGBE_WRITE_REG(hw, IXGBE_VTEIAC, mask); 2331 2332 /* For VTEIMS */ 2333 IXGBE_WRITE_REG(hw, IXGBE_VTEIMS, (1 << sc->vector)); 2334 que = sc->queues; 2335 for (i = 0; i < sc->num_queues; i++, que++) 2336 ixv_enable_queue(sc, que->msix); 2337 2338 IXGBE_WRITE_FLUSH(hw); 2339 } /* ixv_enable_intr */ 2340 2341 /************************************************************************ 2342 * ixv_disable_intr 2343 ************************************************************************/ 2344 static void 2345 ixv_disable_intr(struct ixgbe_softc *sc) 2346 { 2347 struct ix_queue *que = sc->queues; 2348 2349 IXGBE_WRITE_REG(&sc->hw, IXGBE_VTEIAC, 0); 2350 2351 /* disable interrupts other than queues */ 2352 IXGBE_WRITE_REG(&sc->hw, IXGBE_VTEIMC, sc->vector); 2353 2354 for (int i = 0; i < sc->num_queues; i++, que++) 2355 ixv_disable_queue(sc, que->msix); 2356 2357 IXGBE_WRITE_FLUSH(&sc->hw); 2358 } /* ixv_disable_intr */ 2359 2360 /************************************************************************ 2361 * ixv_set_ivar 2362 * 2363 * Setup the correct IVAR register for a particular MSI-X interrupt 2364 * - entry is the register array entry 2365 * - vector is the MSI-X vector for this queue 2366 * - type is RX/TX/MISC 2367 ************************************************************************/ 2368 static void 2369 ixv_set_ivar(struct ixgbe_softc *sc, u8 entry, u8 vector, s8 type) 2370 { 2371 struct ixgbe_hw *hw = &sc->hw; 2372 u32 ivar, index; 2373 2374 vector |= IXGBE_IVAR_ALLOC_VAL; 2375 2376 if (type == -1) { /* MISC IVAR */ 2377 ivar = IXGBE_READ_REG(hw, IXGBE_VTIVAR_MISC); 2378 ivar &= ~0xFF; 2379 ivar |= vector; 2380 IXGBE_WRITE_REG(hw, IXGBE_VTIVAR_MISC, ivar); 2381 } else { /* RX/TX IVARS */ 2382 index = (16 * (entry & 1)) + (8 * type); 2383 ivar = IXGBE_READ_REG(hw, IXGBE_VTIVAR(entry >> 1)); 2384 ivar &= ~(0xffUL << index); 2385 ivar |= ((u32)vector << index); 2386 IXGBE_WRITE_REG(hw, IXGBE_VTIVAR(entry >> 1), ivar); 2387 } 2388 } /* ixv_set_ivar */ 2389 2390 /************************************************************************ 2391 * ixv_configure_ivars 2392 ************************************************************************/ 2393 static void 2394 ixv_configure_ivars(struct ixgbe_softc *sc) 2395 { 2396 struct ix_queue *que = sc->queues; 2397 2398 /* XXX We should sync EITR value calculation with ixgbe.c? */ 2399 2400 for (int i = 0; i < sc->num_queues; i++, que++) { 2401 /* First the RX queue entry */ 2402 ixv_set_ivar(sc, i, que->msix, 0); 2403 /* ... and the TX */ 2404 ixv_set_ivar(sc, i, que->msix, 1); 2405 /* Set an initial value in EITR */ 2406 ixv_eitr_write(sc, que->msix, IXGBE_EITR_DEFAULT); 2407 } 2408 2409 /* For the mailbox interrupt */ 2410 ixv_set_ivar(sc, 1, sc->vector, -1); 2411 } /* ixv_configure_ivars */ 2412 2413 2414 /************************************************************************ 2415 * ixv_init_stats 2416 * 2417 * The VF stats registers never have a truly virgin 2418 * starting point, so this routine save initial vaules to 2419 * last_<REGNAME>. 2420 ************************************************************************/ 2421 static void 2422 ixv_init_stats(struct ixgbe_softc *sc) 2423 { 2424 struct ixgbe_hw *hw = &sc->hw; 2425 2426 sc->stats.vf.last_vfgprc = IXGBE_READ_REG(hw, IXGBE_VFGPRC); 2427 sc->stats.vf.last_vfgorc = IXGBE_READ_REG(hw, IXGBE_VFGORC_LSB); 2428 sc->stats.vf.last_vfgorc |= 2429 (((u64)(IXGBE_READ_REG(hw, IXGBE_VFGORC_MSB))) << 32); 2430 2431 sc->stats.vf.last_vfgptc = IXGBE_READ_REG(hw, IXGBE_VFGPTC); 2432 sc->stats.vf.last_vfgotc = IXGBE_READ_REG(hw, IXGBE_VFGOTC_LSB); 2433 sc->stats.vf.last_vfgotc |= 2434 (((u64)(IXGBE_READ_REG(hw, IXGBE_VFGOTC_MSB))) << 32); 2435 2436 sc->stats.vf.last_vfmprc = IXGBE_READ_REG(hw, IXGBE_VFMPRC); 2437 } /* ixv_init_stats */ 2438 2439 #define UPDATE_STAT_32(reg, last, count) \ 2440 { \ 2441 u32 current = IXGBE_READ_REG(hw, (reg)); \ 2442 IXGBE_EVC_ADD(&count, current - (last)); \ 2443 (last) = current; \ 2444 } 2445 2446 #define UPDATE_STAT_36(lsb, msb, last, count) \ 2447 { \ 2448 u64 cur_lsb = IXGBE_READ_REG(hw, (lsb)); \ 2449 u64 cur_msb = IXGBE_READ_REG(hw, (msb)); \ 2450 u64 current = ((cur_msb << 32) | cur_lsb); \ 2451 if (current < (last)) \ 2452 IXGBE_EVC_ADD(&count, current + __BIT(36) - (last)); \ 2453 else \ 2454 IXGBE_EVC_ADD(&count, current - (last)); \ 2455 (last) = current; \ 2456 } 2457 2458 /************************************************************************ 2459 * ixv_update_stats - Update the board statistics counters. 2460 ************************************************************************/ 2461 void 2462 ixv_update_stats(struct ixgbe_softc *sc) 2463 { 2464 struct ixgbe_hw *hw = &sc->hw; 2465 struct ixgbevf_hw_stats *stats = &sc->stats.vf; 2466 2467 UPDATE_STAT_32(IXGBE_VFGPRC, stats->last_vfgprc, stats->vfgprc); 2468 UPDATE_STAT_32(IXGBE_VFGPTC, stats->last_vfgptc, stats->vfgptc); 2469 UPDATE_STAT_36(IXGBE_VFGORC_LSB, IXGBE_VFGORC_MSB, stats->last_vfgorc, 2470 stats->vfgorc); 2471 UPDATE_STAT_36(IXGBE_VFGOTC_LSB, IXGBE_VFGOTC_MSB, stats->last_vfgotc, 2472 stats->vfgotc); 2473 UPDATE_STAT_32(IXGBE_VFMPRC, stats->last_vfmprc, stats->vfmprc); 2474 2475 /* VF doesn't count errors by hardware */ 2476 2477 } /* ixv_update_stats */ 2478 2479 /************************************************************************ 2480 * ixv_sysctl_interrupt_rate_handler 2481 ************************************************************************/ 2482 static int 2483 ixv_sysctl_interrupt_rate_handler(SYSCTLFN_ARGS) 2484 { 2485 struct sysctlnode node = *rnode; 2486 struct ix_queue *que = (struct ix_queue *)node.sysctl_data; 2487 struct ixgbe_softc *sc = que->sc; 2488 uint32_t reg, usec, rate; 2489 int error; 2490 2491 if (que == NULL) 2492 return 0; 2493 reg = IXGBE_READ_REG(&que->sc->hw, IXGBE_VTEITR(que->msix)); 2494 usec = ((reg & 0x0FF8) >> 3); 2495 if (usec > 0) 2496 rate = 500000 / usec; 2497 else 2498 rate = 0; 2499 node.sysctl_data = &rate; 2500 error = sysctl_lookup(SYSCTLFN_CALL(&node)); 2501 if (error || newp == NULL) 2502 return error; 2503 reg &= ~0xfff; /* default, no limitation */ 2504 if (rate > 0 && rate < 500000) { 2505 if (rate < 1000) 2506 rate = 1000; 2507 reg |= ((4000000 / rate) & 0xff8); 2508 /* 2509 * When RSC is used, ITR interval must be larger than 2510 * RSC_DELAY. Currently, we use 2us for RSC_DELAY. 2511 * The minimum value is always greater than 2us on 100M 2512 * (and 10M?(not documented)), but it's not on 1G and higher. 2513 */ 2514 if ((sc->link_speed != IXGBE_LINK_SPEED_100_FULL) 2515 && (sc->link_speed != IXGBE_LINK_SPEED_10_FULL)) { 2516 if ((sc->num_queues > 1) 2517 && (reg < IXGBE_MIN_RSC_EITR_10G1G)) 2518 return EINVAL; 2519 } 2520 ixv_max_interrupt_rate = rate; 2521 } else 2522 ixv_max_interrupt_rate = 0; 2523 ixv_eitr_write(sc, que->msix, reg); 2524 2525 return (0); 2526 } /* ixv_sysctl_interrupt_rate_handler */ 2527 2528 const struct sysctlnode * 2529 ixv_sysctl_instance(struct ixgbe_softc *sc) 2530 { 2531 const char *dvname; 2532 struct sysctllog **log; 2533 int rc; 2534 const struct sysctlnode *rnode; 2535 2536 log = &sc->sysctllog; 2537 dvname = device_xname(sc->dev); 2538 2539 if ((rc = sysctl_createv(log, 0, NULL, &rnode, 2540 0, CTLTYPE_NODE, dvname, 2541 SYSCTL_DESCR("ixv information and settings"), 2542 NULL, 0, NULL, 0, CTL_HW, CTL_CREATE, CTL_EOL)) != 0) 2543 goto err; 2544 2545 return rnode; 2546 err: 2547 device_printf(sc->dev, 2548 "%s: sysctl_createv failed, rc = %d\n", __func__, rc); 2549 return NULL; 2550 } 2551 2552 static void 2553 ixv_add_device_sysctls(struct ixgbe_softc *sc) 2554 { 2555 struct sysctllog **log; 2556 const struct sysctlnode *rnode, *cnode; 2557 device_t dev; 2558 2559 dev = sc->dev; 2560 log = &sc->sysctllog; 2561 2562 if ((rnode = ixv_sysctl_instance(sc)) == NULL) { 2563 aprint_error_dev(dev, "could not create sysctl root\n"); 2564 return; 2565 } 2566 2567 if (sysctl_createv(log, 0, &rnode, &cnode, 2568 CTLFLAG_READWRITE, CTLTYPE_INT, "debug", 2569 SYSCTL_DESCR("Debug Info"), 2570 ixv_sysctl_debug, 0, (void *)sc, 0, CTL_CREATE, CTL_EOL) != 0) 2571 aprint_error_dev(dev, "could not create sysctl\n"); 2572 2573 if (sysctl_createv(log, 0, &rnode, &cnode, 2574 CTLFLAG_READWRITE, CTLTYPE_INT, 2575 "rx_copy_len", SYSCTL_DESCR("RX Copy Length"), 2576 ixv_sysctl_rx_copy_len, 0, 2577 (void *)sc, 0, CTL_CREATE, CTL_EOL) != 0) 2578 aprint_error_dev(dev, "could not create sysctl\n"); 2579 2580 if (sysctl_createv(log, 0, &rnode, &cnode, 2581 CTLFLAG_READONLY, CTLTYPE_INT, 2582 "num_tx_desc", SYSCTL_DESCR("Number of TX descriptors"), 2583 NULL, 0, &sc->num_tx_desc, 0, CTL_CREATE, CTL_EOL) != 0) 2584 aprint_error_dev(dev, "could not create sysctl\n"); 2585 2586 if (sysctl_createv(log, 0, &rnode, &cnode, 2587 CTLFLAG_READONLY, CTLTYPE_INT, 2588 "num_rx_desc", SYSCTL_DESCR("Number of RX descriptors"), 2589 NULL, 0, &sc->num_rx_desc, 0, CTL_CREATE, CTL_EOL) != 0) 2590 aprint_error_dev(dev, "could not create sysctl\n"); 2591 2592 if (sysctl_createv(log, 0, &rnode, &cnode, 2593 CTLFLAG_READWRITE, CTLTYPE_INT, "rx_process_limit", 2594 SYSCTL_DESCR("max number of RX packets to process"), 2595 ixv_sysctl_rx_process_limit, 0, (void *)sc, 0, CTL_CREATE, 2596 CTL_EOL) != 0) 2597 aprint_error_dev(dev, "could not create sysctl\n"); 2598 2599 if (sysctl_createv(log, 0, &rnode, &cnode, 2600 CTLFLAG_READWRITE, CTLTYPE_INT, "tx_process_limit", 2601 SYSCTL_DESCR("max number of TX packets to process"), 2602 ixv_sysctl_tx_process_limit, 0, (void *)sc, 0, CTL_CREATE, 2603 CTL_EOL) != 0) 2604 aprint_error_dev(dev, "could not create sysctl\n"); 2605 2606 if (sysctl_createv(log, 0, &rnode, &cnode, 2607 CTLFLAG_READWRITE, CTLTYPE_BOOL, "enable_aim", 2608 SYSCTL_DESCR("Interrupt Moderation"), 2609 NULL, 0, &sc->enable_aim, 0, CTL_CREATE, CTL_EOL) != 0) 2610 aprint_error_dev(dev, "could not create sysctl\n"); 2611 2612 if (sysctl_createv(log, 0, &rnode, &cnode, 2613 CTLFLAG_READWRITE, CTLTYPE_BOOL, "txrx_workqueue", 2614 SYSCTL_DESCR("Use workqueue for packet processing"), 2615 NULL, 0, &sc->txrx_use_workqueue, 0, CTL_CREATE, CTL_EOL) 2616 != 0) 2617 aprint_error_dev(dev, "could not create sysctl\n"); 2618 } 2619 2620 /************************************************************************ 2621 * ixv_add_stats_sysctls - Add statistic sysctls for the VF. 2622 ************************************************************************/ 2623 static void 2624 ixv_add_stats_sysctls(struct ixgbe_softc *sc) 2625 { 2626 device_t dev = sc->dev; 2627 struct tx_ring *txr = sc->tx_rings; 2628 struct rx_ring *rxr = sc->rx_rings; 2629 struct ixgbevf_hw_stats *stats = &sc->stats.vf; 2630 struct ixgbe_hw *hw = &sc->hw; 2631 const struct sysctlnode *rnode, *cnode; 2632 struct sysctllog **log = &sc->sysctllog; 2633 const char *xname = device_xname(dev); 2634 2635 /* Driver Statistics */ 2636 evcnt_attach_dynamic(&sc->efbig_tx_dma_setup, EVCNT_TYPE_MISC, 2637 NULL, xname, "Driver tx dma soft fail EFBIG"); 2638 evcnt_attach_dynamic(&sc->mbuf_defrag_failed, EVCNT_TYPE_MISC, 2639 NULL, xname, "m_defrag() failed"); 2640 evcnt_attach_dynamic(&sc->efbig2_tx_dma_setup, EVCNT_TYPE_MISC, 2641 NULL, xname, "Driver tx dma hard fail EFBIG"); 2642 evcnt_attach_dynamic(&sc->einval_tx_dma_setup, EVCNT_TYPE_MISC, 2643 NULL, xname, "Driver tx dma hard fail EINVAL"); 2644 evcnt_attach_dynamic(&sc->other_tx_dma_setup, EVCNT_TYPE_MISC, 2645 NULL, xname, "Driver tx dma hard fail other"); 2646 evcnt_attach_dynamic(&sc->eagain_tx_dma_setup, EVCNT_TYPE_MISC, 2647 NULL, xname, "Driver tx dma soft fail EAGAIN"); 2648 evcnt_attach_dynamic(&sc->enomem_tx_dma_setup, EVCNT_TYPE_MISC, 2649 NULL, xname, "Driver tx dma soft fail ENOMEM"); 2650 evcnt_attach_dynamic(&sc->watchdog_events, EVCNT_TYPE_MISC, 2651 NULL, xname, "Watchdog timeouts"); 2652 evcnt_attach_dynamic(&sc->tso_err, EVCNT_TYPE_MISC, 2653 NULL, xname, "TSO errors"); 2654 evcnt_attach_dynamic(&sc->admin_irqev, EVCNT_TYPE_INTR, 2655 NULL, xname, "Admin MSI-X IRQ Handled"); 2656 evcnt_attach_dynamic(&sc->link_workev, EVCNT_TYPE_INTR, 2657 NULL, xname, "Admin event"); 2658 2659 for (int i = 0; i < sc->num_queues; i++, rxr++, txr++) { 2660 #ifdef LRO 2661 struct lro_ctrl *lro = &rxr->lro; 2662 #endif 2663 2664 snprintf(sc->queues[i].evnamebuf, 2665 sizeof(sc->queues[i].evnamebuf), "%s q%d", xname, i); 2666 snprintf(sc->queues[i].namebuf, 2667 sizeof(sc->queues[i].namebuf), "q%d", i); 2668 2669 if ((rnode = ixv_sysctl_instance(sc)) == NULL) { 2670 aprint_error_dev(dev, 2671 "could not create sysctl root\n"); 2672 break; 2673 } 2674 2675 if (sysctl_createv(log, 0, &rnode, &rnode, 2676 0, CTLTYPE_NODE, 2677 sc->queues[i].namebuf, SYSCTL_DESCR("Queue Name"), 2678 NULL, 0, NULL, 0, CTL_CREATE, CTL_EOL) != 0) 2679 break; 2680 2681 if (sysctl_createv(log, 0, &rnode, &cnode, 2682 CTLFLAG_READWRITE, CTLTYPE_INT, 2683 "interrupt_rate", SYSCTL_DESCR("Interrupt Rate"), 2684 ixv_sysctl_interrupt_rate_handler, 0, 2685 (void *)&sc->queues[i], 0, CTL_CREATE, CTL_EOL) != 0) 2686 break; 2687 2688 if (sysctl_createv(log, 0, &rnode, &cnode, 2689 CTLFLAG_READONLY, CTLTYPE_INT, 2690 "txd_head", SYSCTL_DESCR("Transmit Descriptor Head"), 2691 ixv_sysctl_tdh_handler, 0, (void *)txr, 2692 0, CTL_CREATE, CTL_EOL) != 0) 2693 break; 2694 2695 if (sysctl_createv(log, 0, &rnode, &cnode, 2696 CTLFLAG_READONLY, CTLTYPE_INT, 2697 "txd_tail", SYSCTL_DESCR("Transmit Descriptor Tail"), 2698 ixv_sysctl_tdt_handler, 0, (void *)txr, 2699 0, CTL_CREATE, CTL_EOL) != 0) 2700 break; 2701 2702 if (sysctl_createv(log, 0, &rnode, &cnode, 2703 CTLFLAG_READONLY, CTLTYPE_INT, "rxd_nxck", 2704 SYSCTL_DESCR("Receive Descriptor next to check"), 2705 ixv_sysctl_next_to_check_handler, 0, (void *)rxr, 0, 2706 CTL_CREATE, CTL_EOL) != 0) 2707 break; 2708 2709 if (sysctl_createv(log, 0, &rnode, &cnode, 2710 CTLFLAG_READONLY, CTLTYPE_INT, "rxd_nxrf", 2711 SYSCTL_DESCR("Receive Descriptor next to refresh"), 2712 ixv_sysctl_next_to_refresh_handler, 0, (void *)rxr, 0, 2713 CTL_CREATE, CTL_EOL) != 0) 2714 break; 2715 2716 if (sysctl_createv(log, 0, &rnode, &cnode, 2717 CTLFLAG_READONLY, CTLTYPE_INT, "rxd_head", 2718 SYSCTL_DESCR("Receive Descriptor Head"), 2719 ixv_sysctl_rdh_handler, 0, (void *)rxr, 0, 2720 CTL_CREATE, CTL_EOL) != 0) 2721 break; 2722 2723 if (sysctl_createv(log, 0, &rnode, &cnode, 2724 CTLFLAG_READONLY, CTLTYPE_INT, "rxd_tail", 2725 SYSCTL_DESCR("Receive Descriptor Tail"), 2726 ixv_sysctl_rdt_handler, 0, (void *)rxr, 0, 2727 CTL_CREATE, CTL_EOL) != 0) 2728 break; 2729 2730 evcnt_attach_dynamic(&sc->queues[i].irqs, EVCNT_TYPE_INTR, 2731 NULL, sc->queues[i].evnamebuf, "IRQs on queue"); 2732 evcnt_attach_dynamic(&sc->queues[i].handleq, 2733 EVCNT_TYPE_MISC, NULL, sc->queues[i].evnamebuf, 2734 "Handled queue in softint"); 2735 evcnt_attach_dynamic(&sc->queues[i].req, EVCNT_TYPE_MISC, 2736 NULL, sc->queues[i].evnamebuf, "Requeued in softint"); 2737 evcnt_attach_dynamic(&txr->total_packets, EVCNT_TYPE_MISC, 2738 NULL, sc->queues[i].evnamebuf, 2739 "Queue Packets Transmitted"); 2740 #ifndef IXGBE_LEGACY_TX 2741 evcnt_attach_dynamic(&txr->pcq_drops, EVCNT_TYPE_MISC, 2742 NULL, sc->queues[i].evnamebuf, 2743 "Packets dropped in pcq"); 2744 #endif 2745 evcnt_attach_dynamic(&txr->no_desc_avail, EVCNT_TYPE_MISC, 2746 NULL, sc->queues[i].evnamebuf, 2747 "TX Queue No Descriptor Available"); 2748 evcnt_attach_dynamic(&txr->tso_tx, EVCNT_TYPE_MISC, 2749 NULL, sc->queues[i].evnamebuf, "TSO"); 2750 2751 evcnt_attach_dynamic(&rxr->rx_bytes, EVCNT_TYPE_MISC, 2752 NULL, sc->queues[i].evnamebuf, 2753 "Queue Bytes Received"); 2754 evcnt_attach_dynamic(&rxr->rx_packets, EVCNT_TYPE_MISC, 2755 NULL, sc->queues[i].evnamebuf, 2756 "Queue Packets Received"); 2757 evcnt_attach_dynamic(&rxr->no_mbuf, EVCNT_TYPE_MISC, 2758 NULL, sc->queues[i].evnamebuf, "Rx no mbuf"); 2759 evcnt_attach_dynamic(&rxr->rx_discarded, EVCNT_TYPE_MISC, 2760 NULL, sc->queues[i].evnamebuf, "Rx discarded"); 2761 evcnt_attach_dynamic(&rxr->rx_copies, EVCNT_TYPE_MISC, 2762 NULL, sc->queues[i].evnamebuf, "Copied RX Frames"); 2763 #ifdef LRO 2764 SYSCTL_ADD_INT(ctx, queue_list, OID_AUTO, "lro_queued", 2765 CTLFLAG_RD, &lro->lro_queued, 0, 2766 "LRO Queued"); 2767 SYSCTL_ADD_INT(ctx, queue_list, OID_AUTO, "lro_flushed", 2768 CTLFLAG_RD, &lro->lro_flushed, 0, 2769 "LRO Flushed"); 2770 #endif /* LRO */ 2771 } 2772 2773 /* MAC stats get their own sub node */ 2774 2775 snprintf(stats->namebuf, 2776 sizeof(stats->namebuf), "%s MAC Statistics", xname); 2777 2778 evcnt_attach_dynamic(&stats->ipcs, EVCNT_TYPE_MISC, NULL, 2779 stats->namebuf, "rx csum offload - IP"); 2780 evcnt_attach_dynamic(&stats->l4cs, EVCNT_TYPE_MISC, NULL, 2781 stats->namebuf, "rx csum offload - L4"); 2782 evcnt_attach_dynamic(&stats->ipcs_bad, EVCNT_TYPE_MISC, NULL, 2783 stats->namebuf, "rx csum offload - IP bad"); 2784 evcnt_attach_dynamic(&stats->l4cs_bad, EVCNT_TYPE_MISC, NULL, 2785 stats->namebuf, "rx csum offload - L4 bad"); 2786 2787 /* Packet Reception Stats */ 2788 evcnt_attach_dynamic(&stats->vfgprc, EVCNT_TYPE_MISC, NULL, 2789 xname, "Good Packets Received"); 2790 evcnt_attach_dynamic(&stats->vfgorc, EVCNT_TYPE_MISC, NULL, 2791 xname, "Good Octets Received"); 2792 evcnt_attach_dynamic(&stats->vfmprc, EVCNT_TYPE_MISC, NULL, 2793 xname, "Multicast Packets Received"); 2794 evcnt_attach_dynamic(&stats->vfgptc, EVCNT_TYPE_MISC, NULL, 2795 xname, "Good Packets Transmitted"); 2796 evcnt_attach_dynamic(&stats->vfgotc, EVCNT_TYPE_MISC, NULL, 2797 xname, "Good Octets Transmitted"); 2798 2799 /* Mailbox Stats */ 2800 evcnt_attach_dynamic(&hw->mbx.stats.msgs_tx, EVCNT_TYPE_MISC, NULL, 2801 xname, "message TXs"); 2802 evcnt_attach_dynamic(&hw->mbx.stats.msgs_rx, EVCNT_TYPE_MISC, NULL, 2803 xname, "message RXs"); 2804 evcnt_attach_dynamic(&hw->mbx.stats.acks, EVCNT_TYPE_MISC, NULL, 2805 xname, "ACKs"); 2806 evcnt_attach_dynamic(&hw->mbx.stats.reqs, EVCNT_TYPE_MISC, NULL, 2807 xname, "REQs"); 2808 evcnt_attach_dynamic(&hw->mbx.stats.rsts, EVCNT_TYPE_MISC, NULL, 2809 xname, "RSTs"); 2810 2811 } /* ixv_add_stats_sysctls */ 2812 2813 static void 2814 ixv_clear_evcnt(struct ixgbe_softc *sc) 2815 { 2816 struct tx_ring *txr = sc->tx_rings; 2817 struct rx_ring *rxr = sc->rx_rings; 2818 struct ixgbevf_hw_stats *stats = &sc->stats.vf; 2819 struct ixgbe_hw *hw = &sc->hw; 2820 int i; 2821 2822 /* Driver Statistics */ 2823 IXGBE_EVC_STORE(&sc->efbig_tx_dma_setup, 0); 2824 IXGBE_EVC_STORE(&sc->mbuf_defrag_failed, 0); 2825 IXGBE_EVC_STORE(&sc->efbig2_tx_dma_setup, 0); 2826 IXGBE_EVC_STORE(&sc->einval_tx_dma_setup, 0); 2827 IXGBE_EVC_STORE(&sc->other_tx_dma_setup, 0); 2828 IXGBE_EVC_STORE(&sc->eagain_tx_dma_setup, 0); 2829 IXGBE_EVC_STORE(&sc->enomem_tx_dma_setup, 0); 2830 IXGBE_EVC_STORE(&sc->watchdog_events, 0); 2831 IXGBE_EVC_STORE(&sc->tso_err, 0); 2832 IXGBE_EVC_STORE(&sc->admin_irqev, 0); 2833 IXGBE_EVC_STORE(&sc->link_workev, 0); 2834 2835 for (i = 0; i < sc->num_queues; i++, rxr++, txr++) { 2836 IXGBE_EVC_STORE(&sc->queues[i].irqs, 0); 2837 IXGBE_EVC_STORE(&sc->queues[i].handleq, 0); 2838 IXGBE_EVC_STORE(&sc->queues[i].req, 0); 2839 IXGBE_EVC_STORE(&txr->total_packets, 0); 2840 #ifndef IXGBE_LEGACY_TX 2841 IXGBE_EVC_STORE(&txr->pcq_drops, 0); 2842 #endif 2843 IXGBE_EVC_STORE(&txr->no_desc_avail, 0); 2844 IXGBE_EVC_STORE(&txr->tso_tx, 0); 2845 txr->q_efbig_tx_dma_setup = 0; 2846 txr->q_mbuf_defrag_failed = 0; 2847 txr->q_efbig2_tx_dma_setup = 0; 2848 txr->q_einval_tx_dma_setup = 0; 2849 txr->q_other_tx_dma_setup = 0; 2850 txr->q_eagain_tx_dma_setup = 0; 2851 txr->q_enomem_tx_dma_setup = 0; 2852 txr->q_tso_err = 0; 2853 2854 IXGBE_EVC_STORE(&rxr->rx_packets, 0); 2855 IXGBE_EVC_STORE(&rxr->rx_bytes, 0); 2856 IXGBE_EVC_STORE(&rxr->rx_copies, 0); 2857 IXGBE_EVC_STORE(&rxr->no_mbuf, 0); 2858 IXGBE_EVC_STORE(&rxr->rx_discarded, 0); 2859 } 2860 2861 /* MAC stats get their own sub node */ 2862 2863 IXGBE_EVC_STORE(&stats->ipcs, 0); 2864 IXGBE_EVC_STORE(&stats->l4cs, 0); 2865 IXGBE_EVC_STORE(&stats->ipcs_bad, 0); 2866 IXGBE_EVC_STORE(&stats->l4cs_bad, 0); 2867 2868 /* 2869 * Packet Reception Stats. 2870 * Call ixv_init_stats() to save last VF counters' values. 2871 */ 2872 ixv_init_stats(sc); 2873 IXGBE_EVC_STORE(&stats->vfgprc, 0); 2874 IXGBE_EVC_STORE(&stats->vfgorc, 0); 2875 IXGBE_EVC_STORE(&stats->vfmprc, 0); 2876 IXGBE_EVC_STORE(&stats->vfgptc, 0); 2877 IXGBE_EVC_STORE(&stats->vfgotc, 0); 2878 2879 /* Mailbox Stats */ 2880 IXGBE_EVC_STORE(&hw->mbx.stats.msgs_tx, 0); 2881 IXGBE_EVC_STORE(&hw->mbx.stats.msgs_rx, 0); 2882 IXGBE_EVC_STORE(&hw->mbx.stats.acks, 0); 2883 IXGBE_EVC_STORE(&hw->mbx.stats.reqs, 0); 2884 IXGBE_EVC_STORE(&hw->mbx.stats.rsts, 0); 2885 2886 } /* ixv_clear_evcnt */ 2887 2888 #define PRINTQS(sc, regname) \ 2889 do { \ 2890 struct ixgbe_hw *_hw = &(sc)->hw; \ 2891 int _i; \ 2892 \ 2893 printf("%s: %s", device_xname((sc)->dev), #regname); \ 2894 for (_i = 0; _i < (sc)->num_queues; _i++) { \ 2895 printf((_i == 0) ? "\t" : " "); \ 2896 printf("%08x", IXGBE_READ_REG(_hw, \ 2897 IXGBE_##regname(_i))); \ 2898 } \ 2899 printf("\n"); \ 2900 } while (0) 2901 2902 /************************************************************************ 2903 * ixv_print_debug_info 2904 * 2905 * Provides a way to take a look at important statistics 2906 * maintained by the driver and hardware. 2907 ************************************************************************/ 2908 static void 2909 ixv_print_debug_info(struct ixgbe_softc *sc) 2910 { 2911 device_t dev = sc->dev; 2912 struct ixgbe_hw *hw = &sc->hw; 2913 int i; 2914 2915 device_printf(dev, "queue:"); 2916 for (i = 0; i < sc->num_queues; i++) { 2917 printf((i == 0) ? "\t" : " "); 2918 printf("%8d", i); 2919 } 2920 printf("\n"); 2921 PRINTQS(sc, VFRDBAL); 2922 PRINTQS(sc, VFRDBAH); 2923 PRINTQS(sc, VFRDLEN); 2924 PRINTQS(sc, VFSRRCTL); 2925 PRINTQS(sc, VFRDH); 2926 PRINTQS(sc, VFRDT); 2927 PRINTQS(sc, VFRXDCTL); 2928 2929 device_printf(dev, "EIMS:\t%08x\n", IXGBE_READ_REG(hw, IXGBE_VTEIMS)); 2930 device_printf(dev, "EIAM:\t%08x\n", IXGBE_READ_REG(hw, IXGBE_VTEIAM)); 2931 device_printf(dev, "EIAC:\t%08x\n", IXGBE_READ_REG(hw, IXGBE_VTEIAC)); 2932 } /* ixv_print_debug_info */ 2933 2934 /************************************************************************ 2935 * ixv_sysctl_debug 2936 ************************************************************************/ 2937 static int 2938 ixv_sysctl_debug(SYSCTLFN_ARGS) 2939 { 2940 struct sysctlnode node = *rnode; 2941 struct ixgbe_softc *sc = (struct ixgbe_softc *)node.sysctl_data; 2942 int error, result = 0; 2943 2944 node.sysctl_data = &result; 2945 error = sysctl_lookup(SYSCTLFN_CALL(&node)); 2946 2947 if (error || newp == NULL) 2948 return error; 2949 2950 if (result == 1) 2951 ixv_print_debug_info(sc); 2952 2953 return 0; 2954 } /* ixv_sysctl_debug */ 2955 2956 /************************************************************************ 2957 * ixv_sysctl_rx_copy_len 2958 ************************************************************************/ 2959 static int 2960 ixv_sysctl_rx_copy_len(SYSCTLFN_ARGS) 2961 { 2962 struct sysctlnode node = *rnode; 2963 struct ixgbe_softc *sc = (struct ixgbe_softc *)node.sysctl_data; 2964 int error; 2965 int result = sc->rx_copy_len; 2966 2967 node.sysctl_data = &result; 2968 error = sysctl_lookup(SYSCTLFN_CALL(&node)); 2969 2970 if (error || newp == NULL) 2971 return error; 2972 2973 if ((result < 0) || (result > IXGBE_RX_COPY_LEN_MAX)) 2974 return EINVAL; 2975 2976 sc->rx_copy_len = result; 2977 2978 return 0; 2979 } /* ixv_sysctl_rx_copy_len */ 2980 2981 /************************************************************************ 2982 * ixv_sysctl_tx_process_limit 2983 ************************************************************************/ 2984 static int 2985 ixv_sysctl_tx_process_limit(SYSCTLFN_ARGS) 2986 { 2987 struct sysctlnode node = *rnode; 2988 struct ixgbe_softc *sc = (struct ixgbe_softc *)node.sysctl_data; 2989 int error; 2990 int result = sc->tx_process_limit; 2991 2992 node.sysctl_data = &result; 2993 error = sysctl_lookup(SYSCTLFN_CALL(&node)); 2994 2995 if (error || newp == NULL) 2996 return error; 2997 2998 if ((result <= 0) || (result > sc->num_tx_desc)) 2999 return EINVAL; 3000 3001 sc->tx_process_limit = result; 3002 3003 return 0; 3004 } /* ixv_sysctl_tx_process_limit */ 3005 3006 /************************************************************************ 3007 * ixv_sysctl_rx_process_limit 3008 ************************************************************************/ 3009 static int 3010 ixv_sysctl_rx_process_limit(SYSCTLFN_ARGS) 3011 { 3012 struct sysctlnode node = *rnode; 3013 struct ixgbe_softc *sc = (struct ixgbe_softc *)node.sysctl_data; 3014 int error; 3015 int result = sc->rx_process_limit; 3016 3017 node.sysctl_data = &result; 3018 error = sysctl_lookup(SYSCTLFN_CALL(&node)); 3019 3020 if (error || newp == NULL) 3021 return error; 3022 3023 if ((result <= 0) || (result > sc->num_rx_desc)) 3024 return EINVAL; 3025 3026 sc->rx_process_limit = result; 3027 3028 return 0; 3029 } /* ixv_sysctl_rx_process_limit */ 3030 3031 /************************************************************************ 3032 * ixv_init_device_features 3033 ************************************************************************/ 3034 static void 3035 ixv_init_device_features(struct ixgbe_softc *sc) 3036 { 3037 sc->feat_cap = IXGBE_FEATURE_NETMAP 3038 | IXGBE_FEATURE_VF 3039 | IXGBE_FEATURE_RSS 3040 | IXGBE_FEATURE_LEGACY_TX; 3041 3042 /* A tad short on feature flags for VFs, atm. */ 3043 switch (sc->hw.mac.type) { 3044 case ixgbe_mac_82599_vf: 3045 break; 3046 case ixgbe_mac_X540_vf: 3047 break; 3048 case ixgbe_mac_X550_vf: 3049 case ixgbe_mac_X550EM_x_vf: 3050 case ixgbe_mac_X550EM_a_vf: 3051 sc->feat_cap |= IXGBE_FEATURE_NEEDS_CTXD; 3052 break; 3053 default: 3054 break; 3055 } 3056 3057 /* Enabled by default... */ 3058 /* Is a virtual function (VF) */ 3059 if (sc->feat_cap & IXGBE_FEATURE_VF) 3060 sc->feat_en |= IXGBE_FEATURE_VF; 3061 /* Netmap */ 3062 if (sc->feat_cap & IXGBE_FEATURE_NETMAP) 3063 sc->feat_en |= IXGBE_FEATURE_NETMAP; 3064 /* Receive-Side Scaling (RSS) */ 3065 if (sc->feat_cap & IXGBE_FEATURE_RSS) 3066 sc->feat_en |= IXGBE_FEATURE_RSS; 3067 /* Needs advanced context descriptor regardless of offloads req'd */ 3068 if (sc->feat_cap & IXGBE_FEATURE_NEEDS_CTXD) 3069 sc->feat_en |= IXGBE_FEATURE_NEEDS_CTXD; 3070 3071 /* Enabled via sysctl... */ 3072 /* Legacy (single queue) transmit */ 3073 if ((sc->feat_cap & IXGBE_FEATURE_LEGACY_TX) && 3074 ixv_enable_legacy_tx) 3075 sc->feat_en |= IXGBE_FEATURE_LEGACY_TX; 3076 } /* ixv_init_device_features */ 3077 3078 /************************************************************************ 3079 * ixv_shutdown - Shutdown entry point 3080 ************************************************************************/ 3081 #if 0 /* XXX NetBSD ought to register something like this through pmf(9) */ 3082 static int 3083 ixv_shutdown(device_t dev) 3084 { 3085 struct ixgbe_softc *sc = device_private(dev); 3086 IXGBE_CORE_LOCK(sc); 3087 ixv_stop_locked(sc); 3088 IXGBE_CORE_UNLOCK(sc); 3089 3090 return (0); 3091 } /* ixv_shutdown */ 3092 #endif 3093 3094 static int 3095 ixv_ifflags_cb(struct ethercom *ec) 3096 { 3097 struct ifnet *ifp = &ec->ec_if; 3098 struct ixgbe_softc *sc = ifp->if_softc; 3099 u_short saved_flags; 3100 u_short change; 3101 int rv = 0; 3102 3103 IXGBE_CORE_LOCK(sc); 3104 3105 saved_flags = sc->if_flags; 3106 change = ifp->if_flags ^ sc->if_flags; 3107 if (change != 0) 3108 sc->if_flags = ifp->if_flags; 3109 3110 if ((change & ~(IFF_CANTCHANGE | IFF_DEBUG)) != 0) { 3111 rv = ENETRESET; 3112 goto out; 3113 } else if ((change & IFF_PROMISC) != 0) { 3114 rv = ixv_set_rxfilter(sc); 3115 if (rv != 0) { 3116 /* Restore previous */ 3117 sc->if_flags = saved_flags; 3118 goto out; 3119 } 3120 } 3121 3122 /* Check for ec_capenable. */ 3123 change = ec->ec_capenable ^ sc->ec_capenable; 3124 sc->ec_capenable = ec->ec_capenable; 3125 if ((change & ~(ETHERCAP_VLAN_MTU | ETHERCAP_VLAN_HWTAGGING 3126 | ETHERCAP_VLAN_HWFILTER)) != 0) { 3127 rv = ENETRESET; 3128 goto out; 3129 } 3130 3131 /* 3132 * Special handling is not required for ETHERCAP_VLAN_MTU. 3133 * PF's MAXFRS(MHADD) does not include the 4bytes of the VLAN header. 3134 */ 3135 3136 /* Set up VLAN support and filter */ 3137 if ((change & (ETHERCAP_VLAN_HWTAGGING | ETHERCAP_VLAN_HWFILTER)) != 0) 3138 rv = ixv_setup_vlan_support(sc); 3139 3140 out: 3141 IXGBE_CORE_UNLOCK(sc); 3142 3143 return rv; 3144 } 3145 3146 3147 /************************************************************************ 3148 * ixv_ioctl - Ioctl entry point 3149 * 3150 * Called when the user wants to configure the interface. 3151 * 3152 * return 0 on success, positive on failure 3153 ************************************************************************/ 3154 static int 3155 ixv_ioctl(struct ifnet *ifp, u_long command, void *data) 3156 { 3157 struct ixgbe_softc *sc = ifp->if_softc; 3158 struct ixgbe_hw *hw = &sc->hw; 3159 struct ifcapreq *ifcr = data; 3160 int error; 3161 int l4csum_en; 3162 const int l4csum = IFCAP_CSUM_TCPv4_Rx | IFCAP_CSUM_UDPv4_Rx | 3163 IFCAP_CSUM_TCPv6_Rx | IFCAP_CSUM_UDPv6_Rx; 3164 3165 switch (command) { 3166 case SIOCSIFFLAGS: 3167 IOCTL_DEBUGOUT("ioctl: SIOCSIFFLAGS (Set Interface Flags)"); 3168 break; 3169 case SIOCADDMULTI: { 3170 struct ether_multi *enm; 3171 struct ether_multistep step; 3172 struct ethercom *ec = &sc->osdep.ec; 3173 bool overflow = false; 3174 int mcnt = 0; 3175 3176 /* 3177 * Check the number of multicast address. If it exceeds, 3178 * return ENOSPC. 3179 * Update this code when we support API 1.3. 3180 */ 3181 ETHER_LOCK(ec); 3182 ETHER_FIRST_MULTI(step, ec, enm); 3183 while (enm != NULL) { 3184 mcnt++; 3185 3186 /* 3187 * This code is before adding, so one room is required 3188 * at least. 3189 */ 3190 if (mcnt > (IXGBE_MAX_VF_MC - 1)) { 3191 overflow = true; 3192 break; 3193 } 3194 ETHER_NEXT_MULTI(step, enm); 3195 } 3196 ETHER_UNLOCK(ec); 3197 error = 0; 3198 if (overflow && ((ec->ec_flags & ETHER_F_ALLMULTI) == 0)) { 3199 error = hw->mac.ops.update_xcast_mode(hw, 3200 IXGBEVF_XCAST_MODE_ALLMULTI); 3201 if (error == IXGBE_ERR_NOT_TRUSTED) { 3202 device_printf(sc->dev, 3203 "this interface is not trusted\n"); 3204 error = EPERM; 3205 } else if (error == IXGBE_ERR_FEATURE_NOT_SUPPORTED) { 3206 device_printf(sc->dev, 3207 "the PF doesn't support allmulti mode\n"); 3208 error = EOPNOTSUPP; 3209 } else if (error) { 3210 device_printf(sc->dev, 3211 "number of Ethernet multicast addresses " 3212 "exceeds the limit (%d). error = %d\n", 3213 IXGBE_MAX_VF_MC, error); 3214 error = ENOSPC; 3215 } else 3216 ec->ec_flags |= ETHER_F_ALLMULTI; 3217 } 3218 if (error) 3219 return error; 3220 } 3221 /*FALLTHROUGH*/ 3222 case SIOCDELMULTI: 3223 IOCTL_DEBUGOUT("ioctl: SIOC(ADD|DEL)MULTI"); 3224 break; 3225 case SIOCSIFMEDIA: 3226 case SIOCGIFMEDIA: 3227 IOCTL_DEBUGOUT("ioctl: SIOCxIFMEDIA (Get/Set Interface Media)"); 3228 break; 3229 case SIOCSIFCAP: 3230 IOCTL_DEBUGOUT("ioctl: SIOCSIFCAP (Set Capabilities)"); 3231 break; 3232 case SIOCSIFMTU: 3233 IOCTL_DEBUGOUT("ioctl: SIOCSIFMTU (Set Interface MTU)"); 3234 break; 3235 case SIOCZIFDATA: 3236 IOCTL_DEBUGOUT("ioctl: SIOCZIFDATA (Zero counter)"); 3237 ixv_update_stats(sc); 3238 ixv_clear_evcnt(sc); 3239 break; 3240 default: 3241 IOCTL_DEBUGOUT1("ioctl: UNKNOWN (0x%X)", (int)command); 3242 break; 3243 } 3244 3245 switch (command) { 3246 case SIOCSIFCAP: 3247 /* Layer-4 Rx checksum offload has to be turned on and 3248 * off as a unit. 3249 */ 3250 l4csum_en = ifcr->ifcr_capenable & l4csum; 3251 if (l4csum_en != l4csum && l4csum_en != 0) 3252 return EINVAL; 3253 /*FALLTHROUGH*/ 3254 case SIOCADDMULTI: 3255 case SIOCDELMULTI: 3256 case SIOCSIFFLAGS: 3257 case SIOCSIFMTU: 3258 default: 3259 if ((error = ether_ioctl(ifp, command, data)) != ENETRESET) 3260 return error; 3261 if ((ifp->if_flags & IFF_RUNNING) == 0) 3262 ; 3263 else if (command == SIOCSIFCAP || command == SIOCSIFMTU) { 3264 IXGBE_CORE_LOCK(sc); 3265 ixv_init_locked(sc); 3266 IXGBE_CORE_UNLOCK(sc); 3267 } else if (command == SIOCADDMULTI || command == SIOCDELMULTI) { 3268 /* 3269 * Multicast list has changed; set the hardware filter 3270 * accordingly. 3271 */ 3272 IXGBE_CORE_LOCK(sc); 3273 ixv_disable_intr(sc); 3274 ixv_set_rxfilter(sc); 3275 ixv_enable_intr(sc); 3276 IXGBE_CORE_UNLOCK(sc); 3277 } 3278 return 0; 3279 } 3280 } /* ixv_ioctl */ 3281 3282 /************************************************************************ 3283 * ixv_init 3284 ************************************************************************/ 3285 static int 3286 ixv_init(struct ifnet *ifp) 3287 { 3288 struct ixgbe_softc *sc = ifp->if_softc; 3289 3290 IXGBE_CORE_LOCK(sc); 3291 ixv_init_locked(sc); 3292 IXGBE_CORE_UNLOCK(sc); 3293 3294 return 0; 3295 } /* ixv_init */ 3296 3297 /************************************************************************ 3298 * ixv_handle_que 3299 ************************************************************************/ 3300 static void 3301 ixv_handle_que(void *context) 3302 { 3303 struct ix_queue *que = context; 3304 struct ixgbe_softc *sc = que->sc; 3305 struct tx_ring *txr = que->txr; 3306 struct ifnet *ifp = sc->ifp; 3307 bool more; 3308 3309 IXGBE_EVC_ADD(&que->handleq, 1); 3310 3311 if (ifp->if_flags & IFF_RUNNING) { 3312 IXGBE_TX_LOCK(txr); 3313 more = ixgbe_txeof(txr); 3314 if (!(sc->feat_en & IXGBE_FEATURE_LEGACY_TX)) 3315 if (!ixgbe_mq_ring_empty(ifp, txr->txr_interq)) 3316 ixgbe_mq_start_locked(ifp, txr); 3317 /* Only for queue 0 */ 3318 /* NetBSD still needs this for CBQ */ 3319 if ((&sc->queues[0] == que) 3320 && (!ixgbe_legacy_ring_empty(ifp, NULL))) 3321 ixgbe_legacy_start_locked(ifp, txr); 3322 IXGBE_TX_UNLOCK(txr); 3323 more |= ixgbe_rxeof(que); 3324 if (more) { 3325 IXGBE_EVC_ADD(&que->req, 1); 3326 if (sc->txrx_use_workqueue) { 3327 /* 3328 * "enqueued flag" is not required here 3329 * the same as ixg(4). See ixgbe_msix_que(). 3330 */ 3331 workqueue_enqueue(sc->que_wq, 3332 &que->wq_cookie, curcpu()); 3333 } else 3334 softint_schedule(que->que_si); 3335 return; 3336 } 3337 } 3338 3339 /* Re-enable this interrupt */ 3340 ixv_enable_queue(sc, que->msix); 3341 3342 return; 3343 } /* ixv_handle_que */ 3344 3345 /************************************************************************ 3346 * ixv_handle_que_work 3347 ************************************************************************/ 3348 static void 3349 ixv_handle_que_work(struct work *wk, void *context) 3350 { 3351 struct ix_queue *que = container_of(wk, struct ix_queue, wq_cookie); 3352 3353 /* 3354 * "enqueued flag" is not required here the same as ixg(4). 3355 * See ixgbe_msix_que(). 3356 */ 3357 ixv_handle_que(que); 3358 } 3359 3360 /************************************************************************ 3361 * ixv_allocate_msix - Setup MSI-X Interrupt resources and handlers 3362 ************************************************************************/ 3363 static int 3364 ixv_allocate_msix(struct ixgbe_softc *sc, const struct pci_attach_args *pa) 3365 { 3366 device_t dev = sc->dev; 3367 struct ix_queue *que = sc->queues; 3368 struct tx_ring *txr = sc->tx_rings; 3369 int error, msix_ctrl, rid, vector = 0; 3370 pci_chipset_tag_t pc; 3371 pcitag_t tag; 3372 char intrbuf[PCI_INTRSTR_LEN]; 3373 char wqname[MAXCOMLEN]; 3374 char intr_xname[32]; 3375 const char *intrstr = NULL; 3376 kcpuset_t *affinity; 3377 int cpu_id = 0; 3378 3379 pc = sc->osdep.pc; 3380 tag = sc->osdep.tag; 3381 3382 sc->osdep.nintrs = sc->num_queues + 1; 3383 if (pci_msix_alloc_exact(pa, &sc->osdep.intrs, 3384 sc->osdep.nintrs) != 0) { 3385 aprint_error_dev(dev, 3386 "failed to allocate MSI-X interrupt\n"); 3387 return (ENXIO); 3388 } 3389 3390 kcpuset_create(&affinity, false); 3391 for (int i = 0; i < sc->num_queues; i++, vector++, que++, txr++) { 3392 snprintf(intr_xname, sizeof(intr_xname), "%s TXRX%d", 3393 device_xname(dev), i); 3394 intrstr = pci_intr_string(pc, sc->osdep.intrs[i], intrbuf, 3395 sizeof(intrbuf)); 3396 #ifdef IXGBE_MPSAFE 3397 pci_intr_setattr(pc, &sc->osdep.intrs[i], PCI_INTR_MPSAFE, 3398 true); 3399 #endif 3400 /* Set the handler function */ 3401 que->res = sc->osdep.ihs[i] = pci_intr_establish_xname(pc, 3402 sc->osdep.intrs[i], IPL_NET, ixv_msix_que, que, 3403 intr_xname); 3404 if (que->res == NULL) { 3405 pci_intr_release(pc, sc->osdep.intrs, 3406 sc->osdep.nintrs); 3407 aprint_error_dev(dev, 3408 "Failed to register QUE handler\n"); 3409 kcpuset_destroy(affinity); 3410 return (ENXIO); 3411 } 3412 que->msix = vector; 3413 sc->active_queues |= (u64)(1 << que->msix); 3414 3415 cpu_id = i; 3416 /* Round-robin affinity */ 3417 kcpuset_zero(affinity); 3418 kcpuset_set(affinity, cpu_id % ncpu); 3419 error = interrupt_distribute(sc->osdep.ihs[i], affinity, NULL); 3420 aprint_normal_dev(dev, "for TX/RX, interrupting at %s", 3421 intrstr); 3422 if (error == 0) 3423 aprint_normal(", bound queue %d to cpu %d\n", 3424 i, cpu_id % ncpu); 3425 else 3426 aprint_normal("\n"); 3427 3428 #ifndef IXGBE_LEGACY_TX 3429 txr->txr_si 3430 = softint_establish(SOFTINT_NET | IXGBE_SOFTINT_FLAGS, 3431 ixgbe_deferred_mq_start, txr); 3432 #endif 3433 que->que_si 3434 = softint_establish(SOFTINT_NET | IXGBE_SOFTINT_FLAGS, 3435 ixv_handle_que, que); 3436 if (que->que_si == NULL) { 3437 aprint_error_dev(dev, 3438 "could not establish software interrupt\n"); 3439 } 3440 } 3441 snprintf(wqname, sizeof(wqname), "%sdeferTx", device_xname(dev)); 3442 error = workqueue_create(&sc->txr_wq, wqname, 3443 ixgbe_deferred_mq_start_work, sc, IXGBE_WORKQUEUE_PRI, IPL_NET, 3444 IXGBE_WORKQUEUE_FLAGS); 3445 if (error) { 3446 aprint_error_dev(dev, 3447 "couldn't create workqueue for deferred Tx\n"); 3448 } 3449 sc->txr_wq_enqueued = percpu_alloc(sizeof(u_int)); 3450 3451 snprintf(wqname, sizeof(wqname), "%sTxRx", device_xname(dev)); 3452 error = workqueue_create(&sc->que_wq, wqname, 3453 ixv_handle_que_work, sc, IXGBE_WORKQUEUE_PRI, IPL_NET, 3454 IXGBE_WORKQUEUE_FLAGS); 3455 if (error) { 3456 aprint_error_dev(dev, "couldn't create workqueue for Tx/Rx\n"); 3457 } 3458 3459 /* and Mailbox */ 3460 cpu_id++; 3461 snprintf(intr_xname, sizeof(intr_xname), "%s link", device_xname(dev)); 3462 sc->vector = vector; 3463 intrstr = pci_intr_string(pc, sc->osdep.intrs[vector], intrbuf, 3464 sizeof(intrbuf)); 3465 #ifdef IXGBE_MPSAFE 3466 pci_intr_setattr(pc, &sc->osdep.intrs[vector], PCI_INTR_MPSAFE, 3467 true); 3468 #endif 3469 /* Set the mbx handler function */ 3470 sc->osdep.ihs[vector] = pci_intr_establish_xname(pc, 3471 sc->osdep.intrs[vector], IPL_NET, ixv_msix_mbx, sc, intr_xname); 3472 if (sc->osdep.ihs[vector] == NULL) { 3473 aprint_error_dev(dev, "Failed to register LINK handler\n"); 3474 kcpuset_destroy(affinity); 3475 return (ENXIO); 3476 } 3477 /* Round-robin affinity */ 3478 kcpuset_zero(affinity); 3479 kcpuset_set(affinity, cpu_id % ncpu); 3480 error = interrupt_distribute(sc->osdep.ihs[vector], affinity, 3481 NULL); 3482 3483 aprint_normal_dev(dev, 3484 "for link, interrupting at %s", intrstr); 3485 if (error == 0) 3486 aprint_normal(", affinity to cpu %d\n", cpu_id % ncpu); 3487 else 3488 aprint_normal("\n"); 3489 3490 /* Tasklets for Mailbox */ 3491 snprintf(wqname, sizeof(wqname), "%s-admin", device_xname(dev)); 3492 error = workqueue_create(&sc->admin_wq, wqname, 3493 ixv_handle_admin, sc, IXGBE_WORKQUEUE_PRI, IPL_NET, 3494 IXGBE_TASKLET_WQ_FLAGS); 3495 if (error) { 3496 aprint_error_dev(dev, 3497 "could not create admin workqueue (%d)\n", error); 3498 goto err_out; 3499 } 3500 3501 /* 3502 * Due to a broken design QEMU will fail to properly 3503 * enable the guest for MSI-X unless the vectors in 3504 * the table are all set up, so we must rewrite the 3505 * ENABLE in the MSI-X control register again at this 3506 * point to cause it to successfully initialize us. 3507 */ 3508 if (sc->hw.mac.type == ixgbe_mac_82599_vf) { 3509 pci_get_capability(pc, tag, PCI_CAP_MSIX, &rid, NULL); 3510 rid += PCI_MSIX_CTL; 3511 msix_ctrl = pci_conf_read(pc, tag, rid); 3512 msix_ctrl |= PCI_MSIX_CTL_ENABLE; 3513 pci_conf_write(pc, tag, rid, msix_ctrl); 3514 } 3515 3516 kcpuset_destroy(affinity); 3517 return (0); 3518 err_out: 3519 kcpuset_destroy(affinity); 3520 ixv_free_deferred_handlers(sc); 3521 ixv_free_pci_resources(sc); 3522 return (error); 3523 } /* ixv_allocate_msix */ 3524 3525 /************************************************************************ 3526 * ixv_configure_interrupts - Setup MSI-X resources 3527 * 3528 * Note: The VF device MUST use MSI-X, there is no fallback. 3529 ************************************************************************/ 3530 static int 3531 ixv_configure_interrupts(struct ixgbe_softc *sc) 3532 { 3533 device_t dev = sc->dev; 3534 int want, queues, msgs; 3535 3536 /* Must have at least 2 MSI-X vectors */ 3537 msgs = pci_msix_count(sc->osdep.pc, sc->osdep.tag); 3538 if (msgs < 2) { 3539 aprint_error_dev(dev, "MSIX config error\n"); 3540 return (ENXIO); 3541 } 3542 msgs = MIN(msgs, IXG_MAX_NINTR); 3543 3544 /* Figure out a reasonable auto config value */ 3545 queues = (ncpu > (msgs - 1)) ? (msgs - 1) : ncpu; 3546 3547 if (ixv_num_queues != 0) 3548 queues = ixv_num_queues; 3549 else if ((ixv_num_queues == 0) && (queues > IXGBE_VF_MAX_TX_QUEUES)) 3550 queues = IXGBE_VF_MAX_TX_QUEUES; 3551 3552 /* 3553 * Want vectors for the queues, 3554 * plus an additional for mailbox. 3555 */ 3556 want = queues + 1; 3557 if (msgs >= want) 3558 msgs = want; 3559 else { 3560 aprint_error_dev(dev, 3561 "MSI-X Configuration Problem, " 3562 "%d vectors but %d queues wanted!\n", msgs, want); 3563 return -1; 3564 } 3565 3566 aprint_normal_dev(dev, 3567 "Using MSI-X interrupts with %d vectors\n", msgs); 3568 sc->num_queues = queues; 3569 3570 return (0); 3571 } /* ixv_configure_interrupts */ 3572 3573 3574 /************************************************************************ 3575 * ixv_handle_admin - Tasklet handler for MSI-X MBX interrupts 3576 * 3577 * Done outside of interrupt context since the driver might sleep 3578 ************************************************************************/ 3579 static void 3580 ixv_handle_admin(struct work *wk, void *context) 3581 { 3582 struct ixgbe_softc *sc = context; 3583 struct ixgbe_hw *hw = &sc->hw; 3584 3585 IXGBE_CORE_LOCK(sc); 3586 3587 IXGBE_EVC_ADD(&sc->link_workev, 1); 3588 sc->hw.mac.ops.check_link(&sc->hw, &sc->link_speed, 3589 &sc->link_up, FALSE); 3590 ixv_update_link_status(sc); 3591 3592 sc->task_requests = 0; 3593 atomic_store_relaxed(&sc->admin_pending, 0); 3594 3595 /* Re-enable interrupts */ 3596 IXGBE_WRITE_REG(hw, IXGBE_VTEIMS, (1 << sc->vector)); 3597 3598 IXGBE_CORE_UNLOCK(sc); 3599 } /* ixv_handle_admin */ 3600 3601 /************************************************************************ 3602 * ixv_check_link - Used in the local timer to poll for link changes 3603 ************************************************************************/ 3604 static s32 3605 ixv_check_link(struct ixgbe_softc *sc) 3606 { 3607 s32 error; 3608 3609 KASSERT(mutex_owned(&sc->core_mtx)); 3610 3611 sc->hw.mac.get_link_status = TRUE; 3612 3613 error = sc->hw.mac.ops.check_link(&sc->hw, 3614 &sc->link_speed, &sc->link_up, FALSE); 3615 ixv_update_link_status(sc); 3616 3617 return error; 3618 } /* ixv_check_link */ 3619
Indexes created Tue Sep 30 20:09:53 GMT 2025