ixv.c revision 1.196
1 /* $NetBSD: ixv.c,v 1.196 2023/11/15 03:50:22 msaitoh Exp $ */ 2 3 /****************************************************************************** 4 5 Copyright (c) 2001-2017, Intel Corporation 6 All rights reserved. 7 8 Redistribution and use in source and binary forms, with or without 9 modification, are permitted provided that the following conditions are met: 10 11 1. Redistributions of source code must retain the above copyright notice, 12 this list of conditions and the following disclaimer. 13 14 2. Redistributions in binary form must reproduce the above copyright 15 notice, this list of conditions and the following disclaimer in the 16 documentation and/or other materials provided with the distribution. 17 18 3. Neither the name of the Intel Corporation nor the names of its 19 contributors may be used to endorse or promote products derived from 20 this software without specific prior written permission. 21 22 THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 23 AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 24 IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 25 ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE 26 LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 27 CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 28 SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 29 INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 30 CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 31 ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 32 POSSIBILITY OF SUCH DAMAGE. 33 34 ******************************************************************************/ 35 /*$FreeBSD: head/sys/dev/ixgbe/if_ixv.c 331224 2018-03-19 20:55:05Z erj $*/ 36 37 #include <sys/cdefs.h> 38 __KERNEL_RCSID(0, "$NetBSD: ixv.c,v 1.196 2023/11/15 03:50:22 msaitoh Exp $"); 39 40 #ifdef _KERNEL_OPT 41 #include "opt_inet.h" 42 #include "opt_inet6.h" 43 #include "opt_net_mpsafe.h" 44 #endif 45 46 #include "ixgbe.h" 47 48 /************************************************************************ 49 * Driver version 50 ************************************************************************/ 51 static const char ixv_driver_version[] = "2.0.1-k"; 52 /* XXX NetBSD: + 1.5.17 */ 53 54 /************************************************************************ 55 * PCI Device ID Table 56 * 57 * Used by probe to select devices to load on 58 * Last field stores an index into ixv_strings 59 * Last entry must be all 0s 60 * 61 * { Vendor ID, Device ID, SubVendor ID, SubDevice ID, String Index } 62 ************************************************************************/ 63 static const ixgbe_vendor_info_t ixv_vendor_info_array[] = 64 { 65 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_VF, 0, 0, 0}, 66 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_X540_VF, 0, 0, 0}, 67 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_X550_VF, 0, 0, 0}, 68 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_X550EM_X_VF, 0, 0, 0}, 69 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_X550EM_A_VF, 0, 0, 0}, 70 /* required last entry */ 71 {0, 0, 0, 0, 0} 72 }; 73 74 /************************************************************************ 75 * Table of branding strings 76 ************************************************************************/ 77 static const char *ixv_strings[] = { 78 "Intel(R) PRO/10GbE Virtual Function Network Driver" 79 }; 80 81 /********************************************************************* 82 * Function prototypes 83 *********************************************************************/ 84 static int ixv_probe(device_t, cfdata_t, void *); 85 static void ixv_attach(device_t, device_t, void *); 86 static int ixv_detach(device_t, int); 87 #if 0 88 static int ixv_shutdown(device_t); 89 #endif 90 static int ixv_ifflags_cb(struct ethercom *); 91 static int ixv_ioctl(struct ifnet *, u_long, void *); 92 static int ixv_init(struct ifnet *); 93 static void ixv_init_locked(struct ixgbe_softc *); 94 static void ixv_ifstop(struct ifnet *, int); 95 static void ixv_stop_locked(void *); 96 static void ixv_init_device_features(struct ixgbe_softc *); 97 static void ixv_media_status(struct ifnet *, struct ifmediareq *); 98 static int ixv_media_change(struct ifnet *); 99 static int ixv_allocate_pci_resources(struct ixgbe_softc *, 100 const struct pci_attach_args *); 101 static void ixv_free_deferred_handlers(struct ixgbe_softc *); 102 static int ixv_allocate_msix(struct ixgbe_softc *, 103 const struct pci_attach_args *); 104 static int ixv_configure_interrupts(struct ixgbe_softc *); 105 static void ixv_free_pci_resources(struct ixgbe_softc *); 106 static void ixv_local_timer(void *); 107 static void ixv_handle_timer(struct work *, void *); 108 static int ixv_setup_interface(device_t, struct ixgbe_softc *); 109 static void ixv_schedule_admin_tasklet(struct ixgbe_softc *); 110 static int ixv_negotiate_api(struct ixgbe_softc *); 111 112 static void ixv_initialize_transmit_units(struct ixgbe_softc *); 113 static void ixv_initialize_receive_units(struct ixgbe_softc *); 114 static void ixv_initialize_rss_mapping(struct ixgbe_softc *); 115 static s32 ixv_check_link(struct ixgbe_softc *); 116 117 static void ixv_enable_intr(struct ixgbe_softc *); 118 static void ixv_disable_intr(struct ixgbe_softc *); 119 static int ixv_set_rxfilter(struct ixgbe_softc *); 120 static void ixv_update_link_status(struct ixgbe_softc *); 121 static int ixv_sysctl_debug(SYSCTLFN_PROTO); 122 static void ixv_set_ivar(struct ixgbe_softc *, u8, u8, s8); 123 static void ixv_configure_ivars(struct ixgbe_softc *); 124 static u8 * ixv_mc_array_itr(struct ixgbe_hw *, u8 **, u32 *); 125 static void ixv_eitr_write(struct ixgbe_softc *, uint32_t, uint32_t); 126 127 static void ixv_setup_vlan_tagging(struct ixgbe_softc *); 128 static int ixv_setup_vlan_support(struct ixgbe_softc *); 129 static int ixv_vlan_cb(struct ethercom *, uint16_t, bool); 130 static int ixv_register_vlan(struct ixgbe_softc *, u16); 131 static int ixv_unregister_vlan(struct ixgbe_softc *, u16); 132 133 static void ixv_add_device_sysctls(struct ixgbe_softc *); 134 static void ixv_init_stats(struct ixgbe_softc *); 135 static void ixv_update_stats(struct ixgbe_softc *); 136 static void ixv_add_stats_sysctls(struct ixgbe_softc *); 137 static void ixv_clear_evcnt(struct ixgbe_softc *); 138 139 /* Sysctl handlers */ 140 static int ixv_sysctl_interrupt_rate_handler(SYSCTLFN_PROTO); 141 static int ixv_sysctl_next_to_check_handler(SYSCTLFN_PROTO); 142 static int ixv_sysctl_next_to_refresh_handler(SYSCTLFN_PROTO); 143 static int ixv_sysctl_rdh_handler(SYSCTLFN_PROTO); 144 static int ixv_sysctl_rdt_handler(SYSCTLFN_PROTO); 145 static int ixv_sysctl_tdt_handler(SYSCTLFN_PROTO); 146 static int ixv_sysctl_tdh_handler(SYSCTLFN_PROTO); 147 static int ixv_sysctl_tx_process_limit(SYSCTLFN_PROTO); 148 static int ixv_sysctl_rx_process_limit(SYSCTLFN_PROTO); 149 static int ixv_sysctl_rx_copy_len(SYSCTLFN_PROTO); 150 151 /* The MSI-X Interrupt handlers */ 152 static int ixv_msix_que(void *); 153 static int ixv_msix_mbx(void *); 154 155 /* Event handlers running on workqueue */ 156 static void ixv_handle_que(void *); 157 158 /* Deferred workqueue handlers */ 159 static void ixv_handle_admin(struct work *, void *); 160 static void ixv_handle_que_work(struct work *, void *); 161 162 const struct sysctlnode *ixv_sysctl_instance(struct ixgbe_softc *); 163 static const ixgbe_vendor_info_t *ixv_lookup(const struct pci_attach_args *); 164 165 /************************************************************************ 166 * NetBSD Device Interface Entry Points 167 ************************************************************************/ 168 CFATTACH_DECL3_NEW(ixv, sizeof(struct ixgbe_softc), 169 ixv_probe, ixv_attach, ixv_detach, NULL, NULL, NULL, 170 DVF_DETACH_SHUTDOWN); 171 172 #if 0 173 static driver_t ixv_driver = { 174 "ixv", ixv_methods, sizeof(struct ixgbe_softc), 175 }; 176 177 devclass_t ixv_devclass; 178 DRIVER_MODULE(ixv, pci, ixv_driver, ixv_devclass, 0, 0); 179 MODULE_DEPEND(ixv, pci, 1, 1, 1); 180 MODULE_DEPEND(ixv, ether, 1, 1, 1); 181 #endif 182 183 /* 184 * TUNEABLE PARAMETERS: 185 */ 186 187 /* Number of Queues - do not exceed MSI-X vectors - 1 */ 188 static int ixv_num_queues = 0; 189 #define TUNABLE_INT(__x, __y) 190 TUNABLE_INT("hw.ixv.num_queues", &ixv_num_queues); 191 192 /* 193 * AIM: Adaptive Interrupt Moderation 194 * which means that the interrupt rate 195 * is varied over time based on the 196 * traffic for that interrupt vector 197 */ 198 static bool ixv_enable_aim = false; 199 TUNABLE_INT("hw.ixv.enable_aim", &ixv_enable_aim); 200 201 static int ixv_max_interrupt_rate = (4000000 / IXGBE_LOW_LATENCY); 202 TUNABLE_INT("hw.ixv.max_interrupt_rate", &ixv_max_interrupt_rate); 203 204 /* How many packets rxeof tries to clean at a time */ 205 static int ixv_rx_process_limit = 256; 206 TUNABLE_INT("hw.ixv.rx_process_limit", &ixv_rx_process_limit); 207 208 /* How many packets txeof tries to clean at a time */ 209 static int ixv_tx_process_limit = 256; 210 TUNABLE_INT("hw.ixv.tx_process_limit", &ixv_tx_process_limit); 211 212 /* Which packet processing uses workqueue or softint */ 213 static bool ixv_txrx_workqueue = false; 214 215 /* 216 * Number of TX descriptors per ring, 217 * setting higher than RX as this seems 218 * the better performing choice. 219 */ 220 static int ixv_txd = DEFAULT_TXD; 221 TUNABLE_INT("hw.ixv.txd", &ixv_txd); 222 223 /* Number of RX descriptors per ring */ 224 static int ixv_rxd = DEFAULT_RXD; 225 TUNABLE_INT("hw.ixv.rxd", &ixv_rxd); 226 227 /* Legacy Transmit (single queue) */ 228 static int ixv_enable_legacy_tx = 0; 229 TUNABLE_INT("hw.ixv.enable_legacy_tx", &ixv_enable_legacy_tx); 230 231 #ifdef NET_MPSAFE 232 #define IXGBE_MPSAFE 1 233 #define IXGBE_CALLOUT_FLAGS CALLOUT_MPSAFE 234 #define IXGBE_SOFTINT_FLAGS SOFTINT_MPSAFE 235 #define IXGBE_WORKQUEUE_FLAGS WQ_PERCPU | WQ_MPSAFE 236 #define IXGBE_TASKLET_WQ_FLAGS WQ_MPSAFE 237 #else 238 #define IXGBE_CALLOUT_FLAGS 0 239 #define IXGBE_SOFTINT_FLAGS 0 240 #define IXGBE_WORKQUEUE_FLAGS WQ_PERCPU 241 #define IXGBE_TASKLET_WQ_FLAGS 0 242 #endif 243 #define IXGBE_WORKQUEUE_PRI PRI_SOFTNET 244 245 #if 0 246 static int (*ixv_start_locked)(struct ifnet *, struct tx_ring *); 247 static int (*ixv_ring_empty)(struct ifnet *, struct buf_ring *); 248 #endif 249 250 /************************************************************************ 251 * ixv_probe - Device identification routine 252 * 253 * Determines if the driver should be loaded on 254 * adapter based on its PCI vendor/device ID. 255 * 256 * return BUS_PROBE_DEFAULT on success, positive on failure 257 ************************************************************************/ 258 static int 259 ixv_probe(device_t dev, cfdata_t cf, void *aux) 260 { 261 #ifdef __HAVE_PCI_MSI_MSIX 262 const struct pci_attach_args *pa = aux; 263 264 return (ixv_lookup(pa) != NULL) ? 1 : 0; 265 #else 266 return 0; 267 #endif 268 } /* ixv_probe */ 269 270 static const ixgbe_vendor_info_t * 271 ixv_lookup(const struct pci_attach_args *pa) 272 { 273 const ixgbe_vendor_info_t *ent; 274 pcireg_t subid; 275 276 INIT_DEBUGOUT("ixv_lookup: begin"); 277 278 if (PCI_VENDOR(pa->pa_id) != IXGBE_INTEL_VENDOR_ID) 279 return NULL; 280 281 subid = pci_conf_read(pa->pa_pc, pa->pa_tag, PCI_SUBSYS_ID_REG); 282 283 for (ent = ixv_vendor_info_array; ent->vendor_id != 0; ent++) { 284 if ((PCI_VENDOR(pa->pa_id) == ent->vendor_id) && 285 (PCI_PRODUCT(pa->pa_id) == ent->device_id) && 286 ((PCI_SUBSYS_VENDOR(subid) == ent->subvendor_id) || 287 (ent->subvendor_id == 0)) && 288 ((PCI_SUBSYS_ID(subid) == ent->subdevice_id) || 289 (ent->subdevice_id == 0))) { 290 return ent; 291 } 292 } 293 294 return NULL; 295 } 296 297 /************************************************************************ 298 * ixv_attach - Device initialization routine 299 * 300 * Called when the driver is being loaded. 301 * Identifies the type of hardware, allocates all resources 302 * and initializes the hardware. 303 * 304 * return 0 on success, positive on failure 305 ************************************************************************/ 306 static void 307 ixv_attach(device_t parent, device_t dev, void *aux) 308 { 309 struct ixgbe_softc *sc; 310 struct ixgbe_hw *hw; 311 int error = 0; 312 pcireg_t id, subid; 313 const ixgbe_vendor_info_t *ent; 314 const struct pci_attach_args *pa = aux; 315 const char *apivstr; 316 const char *str; 317 char wqname[MAXCOMLEN]; 318 char buf[256]; 319 320 INIT_DEBUGOUT("ixv_attach: begin"); 321 322 /* 323 * Make sure BUSMASTER is set, on a VM under 324 * KVM it may not be and will break things. 325 */ 326 ixgbe_pci_enable_busmaster(pa->pa_pc, pa->pa_tag); 327 328 /* Allocate, clear, and link in our adapter structure */ 329 sc = device_private(dev); 330 sc->hw.back = sc; 331 sc->dev = dev; 332 hw = &sc->hw; 333 334 sc->init_locked = ixv_init_locked; 335 sc->stop_locked = ixv_stop_locked; 336 337 sc->osdep.pc = pa->pa_pc; 338 sc->osdep.tag = pa->pa_tag; 339 if (pci_dma64_available(pa)) 340 sc->osdep.dmat = pa->pa_dmat64; 341 else 342 sc->osdep.dmat = pa->pa_dmat; 343 sc->osdep.attached = false; 344 345 ent = ixv_lookup(pa); 346 347 KASSERT(ent != NULL); 348 349 aprint_normal(": %s, Version - %s\n", 350 ixv_strings[ent->index], ixv_driver_version); 351 352 /* Core Lock Init */ 353 IXGBE_CORE_LOCK_INIT(sc, device_xname(dev)); 354 355 /* Do base PCI setup - map BAR0 */ 356 if (ixv_allocate_pci_resources(sc, pa)) { 357 aprint_error_dev(dev, "ixv_allocate_pci_resources() failed!\n"); 358 error = ENXIO; 359 goto err_out; 360 } 361 362 /* SYSCTL APIs */ 363 ixv_add_device_sysctls(sc); 364 365 /* Set up the timer callout and workqueue */ 366 callout_init(&sc->timer, IXGBE_CALLOUT_FLAGS); 367 snprintf(wqname, sizeof(wqname), "%s-timer", device_xname(dev)); 368 error = workqueue_create(&sc->timer_wq, wqname, 369 ixv_handle_timer, sc, IXGBE_WORKQUEUE_PRI, IPL_NET, 370 IXGBE_TASKLET_WQ_FLAGS); 371 if (error) { 372 aprint_error_dev(dev, 373 "could not create timer workqueue (%d)\n", error); 374 goto err_out; 375 } 376 377 /* Save off the information about this board */ 378 id = pci_conf_read(pa->pa_pc, pa->pa_tag, PCI_ID_REG); 379 subid = pci_conf_read(pa->pa_pc, pa->pa_tag, PCI_SUBSYS_ID_REG); 380 hw->vendor_id = PCI_VENDOR(id); 381 hw->device_id = PCI_PRODUCT(id); 382 hw->revision_id = 383 PCI_REVISION(pci_conf_read(pa->pa_pc, pa->pa_tag, PCI_CLASS_REG)); 384 hw->subsystem_vendor_id = PCI_SUBSYS_VENDOR(subid); 385 hw->subsystem_device_id = PCI_SUBSYS_ID(subid); 386 387 /* A subset of set_mac_type */ 388 switch (hw->device_id) { 389 case IXGBE_DEV_ID_82599_VF: 390 hw->mac.type = ixgbe_mac_82599_vf; 391 str = "82599 VF"; 392 break; 393 case IXGBE_DEV_ID_X540_VF: 394 hw->mac.type = ixgbe_mac_X540_vf; 395 str = "X540 VF"; 396 break; 397 case IXGBE_DEV_ID_X550_VF: 398 hw->mac.type = ixgbe_mac_X550_vf; 399 str = "X550 VF"; 400 break; 401 case IXGBE_DEV_ID_X550EM_X_VF: 402 hw->mac.type = ixgbe_mac_X550EM_x_vf; 403 str = "X550EM X VF"; 404 break; 405 case IXGBE_DEV_ID_X550EM_A_VF: 406 hw->mac.type = ixgbe_mac_X550EM_a_vf; 407 str = "X550EM A VF"; 408 break; 409 default: 410 /* Shouldn't get here since probe succeeded */ 411 aprint_error_dev(dev, "Unknown device ID!\n"); 412 error = ENXIO; 413 goto err_out; 414 break; 415 } 416 aprint_normal_dev(dev, "device %s\n", str); 417 418 ixv_init_device_features(sc); 419 420 /* Initialize the shared code */ 421 error = ixgbe_init_ops_vf(hw); 422 if (error) { 423 aprint_error_dev(dev, "ixgbe_init_ops_vf() failed!\n"); 424 error = EIO; 425 goto err_out; 426 } 427 428 /* Setup the mailbox */ 429 ixgbe_init_mbx_params_vf(hw); 430 431 /* Set the right number of segments */ 432 KASSERT(IXGBE_82599_SCATTER_MAX >= IXGBE_SCATTER_DEFAULT); 433 sc->num_segs = IXGBE_SCATTER_DEFAULT; 434 435 /* Reset mbox api to 1.0 */ 436 error = hw->mac.ops.reset_hw(hw); 437 if (error == IXGBE_ERR_RESET_FAILED) 438 aprint_error_dev(dev, "...reset_hw() failure: Reset Failed!\n"); 439 else if (error) 440 aprint_error_dev(dev, "...reset_hw() failed with error %d\n", 441 error); 442 if (error) { 443 error = EIO; 444 goto err_out; 445 } 446 447 error = hw->mac.ops.init_hw(hw); 448 if (error) { 449 aprint_error_dev(dev, "...init_hw() failed!\n"); 450 error = EIO; 451 goto err_out; 452 } 453 454 /* Negotiate mailbox API version */ 455 error = ixv_negotiate_api(sc); 456 if (error) 457 aprint_normal_dev(dev, 458 "MBX API negotiation failed during attach!\n"); 459 switch (hw->api_version) { 460 case ixgbe_mbox_api_10: 461 apivstr = "1.0"; 462 break; 463 case ixgbe_mbox_api_20: 464 apivstr = "2.0"; 465 break; 466 case ixgbe_mbox_api_11: 467 apivstr = "1.1"; 468 break; 469 case ixgbe_mbox_api_12: 470 apivstr = "1.2"; 471 break; 472 case ixgbe_mbox_api_13: 473 apivstr = "1.3"; 474 break; 475 case ixgbe_mbox_api_14: 476 apivstr = "1.4"; 477 break; 478 case ixgbe_mbox_api_15: 479 apivstr = "1.5"; 480 break; 481 default: 482 apivstr = "unknown"; 483 break; 484 } 485 aprint_normal_dev(dev, "Mailbox API %s\n", apivstr); 486 487 /* If no mac address was assigned, make a random one */ 488 if (!ixv_check_ether_addr(hw->mac.addr)) { 489 u8 addr[ETHER_ADDR_LEN]; 490 uint64_t rndval = cprng_strong64(); 491 492 memcpy(addr, &rndval, sizeof(addr)); 493 addr[0] &= 0xFE; 494 addr[0] |= 0x02; 495 bcopy(addr, hw->mac.addr, sizeof(addr)); 496 } 497 498 /* Register for VLAN events */ 499 ether_set_vlan_cb(&sc->osdep.ec, ixv_vlan_cb); 500 501 /* Do descriptor calc and sanity checks */ 502 if (((ixv_txd * sizeof(union ixgbe_adv_tx_desc)) % DBA_ALIGN) != 0 || 503 ixv_txd < MIN_TXD || ixv_txd > MAX_TXD) { 504 aprint_error_dev(dev, "Invalid TX ring size (%d). " 505 "It must be between %d and %d, " 506 "inclusive, and must be a multiple of %zu. " 507 "Using default value of %d instead.\n", 508 ixv_txd, MIN_TXD, MAX_TXD, 509 DBA_ALIGN / sizeof(union ixgbe_adv_tx_desc), 510 DEFAULT_TXD); 511 sc->num_tx_desc = DEFAULT_TXD; 512 } else 513 sc->num_tx_desc = ixv_txd; 514 515 if (((ixv_rxd * sizeof(union ixgbe_adv_rx_desc)) % DBA_ALIGN) != 0 || 516 ixv_rxd < MIN_RXD || ixv_rxd > MAX_RXD) { 517 aprint_error_dev(dev, "Invalid RX ring size (%d). " 518 "It must be between %d and %d, " 519 "inclusive, and must be a multiple of %zu. " 520 "Using default value of %d instead.\n", 521 ixv_rxd, MIN_RXD, MAX_RXD, 522 DBA_ALIGN / sizeof(union ixgbe_adv_rx_desc), 523 DEFAULT_RXD); 524 sc->num_rx_desc = DEFAULT_RXD; 525 } else 526 sc->num_rx_desc = ixv_rxd; 527 528 /* Sysctls for limiting the amount of work done in the taskqueues */ 529 sc->rx_process_limit 530 = (ixv_rx_process_limit <= sc->num_rx_desc) 531 ? ixv_rx_process_limit : sc->num_rx_desc; 532 sc->tx_process_limit 533 = (ixv_tx_process_limit <= sc->num_tx_desc) 534 ? ixv_tx_process_limit : sc->num_tx_desc; 535 536 /* Set default high limit of copying mbuf in rxeof */ 537 sc->rx_copy_len = IXGBE_RX_COPY_LEN_MAX; 538 539 /* Setup MSI-X */ 540 error = ixv_configure_interrupts(sc); 541 if (error) 542 goto err_out; 543 544 /* Allocate our TX/RX Queues */ 545 if (ixgbe_allocate_queues(sc)) { 546 aprint_error_dev(dev, "ixgbe_allocate_queues() failed!\n"); 547 error = ENOMEM; 548 goto err_out; 549 } 550 551 /* hw.ix defaults init */ 552 sc->enable_aim = ixv_enable_aim; 553 sc->max_interrupt_rate = ixv_max_interrupt_rate; 554 555 sc->txrx_use_workqueue = ixv_txrx_workqueue; 556 557 error = ixv_allocate_msix(sc, pa); 558 if (error) { 559 aprint_error_dev(dev, "ixv_allocate_msix() failed!\n"); 560 goto err_late; 561 } 562 563 /* Setup OS specific network interface */ 564 error = ixv_setup_interface(dev, sc); 565 if (error != 0) { 566 aprint_error_dev(dev, "ixv_setup_interface() failed!\n"); 567 goto err_late; 568 } 569 570 /* Allocate multicast array memory */ 571 sc->mta = malloc(sizeof(*sc->mta) * 572 IXGBE_MAX_VF_MC, M_DEVBUF, M_WAITOK); 573 574 /* Check if VF was disabled by PF */ 575 error = hw->mac.ops.get_link_state(hw, &sc->link_enabled); 576 if (error) { 577 /* PF is not capable of controlling VF state. Enable the link. */ 578 sc->link_enabled = TRUE; 579 } 580 581 /* Do the stats setup */ 582 ixv_init_stats(sc); 583 ixv_add_stats_sysctls(sc); 584 585 if (sc->feat_en & IXGBE_FEATURE_NETMAP) 586 ixgbe_netmap_attach(sc); 587 588 snprintb(buf, sizeof(buf), IXGBE_FEATURE_FLAGS, sc->feat_cap); 589 aprint_verbose_dev(dev, "feature cap %s\n", buf); 590 snprintb(buf, sizeof(buf), IXGBE_FEATURE_FLAGS, sc->feat_en); 591 aprint_verbose_dev(dev, "feature ena %s\n", buf); 592 593 INIT_DEBUGOUT("ixv_attach: end"); 594 sc->osdep.attached = true; 595 596 return; 597 598 err_late: 599 ixgbe_free_queues(sc); 600 err_out: 601 ixv_free_pci_resources(sc); 602 IXGBE_CORE_LOCK_DESTROY(sc); 603 604 return; 605 } /* ixv_attach */ 606 607 /************************************************************************ 608 * ixv_detach - Device removal routine 609 * 610 * Called when the driver is being removed. 611 * Stops the adapter and deallocates all the resources 612 * that were allocated for driver operation. 613 * 614 * return 0 on success, positive on failure 615 ************************************************************************/ 616 static int 617 ixv_detach(device_t dev, int flags) 618 { 619 struct ixgbe_softc *sc = device_private(dev); 620 struct ixgbe_hw *hw = &sc->hw; 621 struct tx_ring *txr = sc->tx_rings; 622 struct rx_ring *rxr = sc->rx_rings; 623 struct ixgbevf_hw_stats *stats = &sc->stats.vf; 624 625 INIT_DEBUGOUT("ixv_detach: begin"); 626 if (sc->osdep.attached == false) 627 return 0; 628 629 /* Stop the interface. Callouts are stopped in it. */ 630 ixv_ifstop(sc->ifp, 1); 631 632 if (VLAN_ATTACHED(&sc->osdep.ec) && 633 (flags & (DETACH_SHUTDOWN | DETACH_FORCE)) == 0) { 634 aprint_error_dev(dev, "VLANs in use, detach first\n"); 635 return EBUSY; 636 } 637 638 ether_ifdetach(sc->ifp); 639 callout_halt(&sc->timer, NULL); 640 ixv_free_deferred_handlers(sc); 641 642 if (sc->feat_en & IXGBE_FEATURE_NETMAP) 643 netmap_detach(sc->ifp); 644 645 ixv_free_pci_resources(sc); 646 #if 0 /* XXX the NetBSD port is probably missing something here */ 647 bus_generic_detach(dev); 648 #endif 649 if_detach(sc->ifp); 650 ifmedia_fini(&sc->media); 651 if_percpuq_destroy(sc->ipq); 652 653 sysctl_teardown(&sc->sysctllog); 654 evcnt_detach(&sc->efbig_tx_dma_setup); 655 evcnt_detach(&sc->mbuf_defrag_failed); 656 evcnt_detach(&sc->efbig2_tx_dma_setup); 657 evcnt_detach(&sc->einval_tx_dma_setup); 658 evcnt_detach(&sc->other_tx_dma_setup); 659 evcnt_detach(&sc->eagain_tx_dma_setup); 660 evcnt_detach(&sc->enomem_tx_dma_setup); 661 evcnt_detach(&sc->watchdog_events); 662 evcnt_detach(&sc->tso_err); 663 evcnt_detach(&sc->admin_irqev); 664 evcnt_detach(&sc->link_workev); 665 666 txr = sc->tx_rings; 667 for (int i = 0; i < sc->num_queues; i++, rxr++, txr++) { 668 evcnt_detach(&sc->queues[i].irqs); 669 evcnt_detach(&sc->queues[i].handleq); 670 evcnt_detach(&sc->queues[i].req); 671 evcnt_detach(&txr->total_packets); 672 #ifndef IXGBE_LEGACY_TX 673 evcnt_detach(&txr->pcq_drops); 674 #endif 675 evcnt_detach(&txr->no_desc_avail); 676 evcnt_detach(&txr->tso_tx); 677 678 evcnt_detach(&rxr->rx_packets); 679 evcnt_detach(&rxr->rx_bytes); 680 evcnt_detach(&rxr->rx_copies); 681 evcnt_detach(&rxr->no_mbuf); 682 evcnt_detach(&rxr->rx_discarded); 683 } 684 evcnt_detach(&stats->ipcs); 685 evcnt_detach(&stats->l4cs); 686 evcnt_detach(&stats->ipcs_bad); 687 evcnt_detach(&stats->l4cs_bad); 688 689 /* Packet Reception Stats */ 690 evcnt_detach(&stats->vfgorc); 691 evcnt_detach(&stats->vfgprc); 692 evcnt_detach(&stats->vfmprc); 693 694 /* Packet Transmission Stats */ 695 evcnt_detach(&stats->vfgotc); 696 evcnt_detach(&stats->vfgptc); 697 698 /* Mailbox Stats */ 699 evcnt_detach(&hw->mbx.stats.msgs_tx); 700 evcnt_detach(&hw->mbx.stats.msgs_rx); 701 evcnt_detach(&hw->mbx.stats.acks); 702 evcnt_detach(&hw->mbx.stats.reqs); 703 evcnt_detach(&hw->mbx.stats.rsts); 704 705 ixgbe_free_queues(sc); 706 707 IXGBE_CORE_LOCK_DESTROY(sc); 708 709 return (0); 710 } /* ixv_detach */ 711 712 /************************************************************************ 713 * ixv_init_locked - Init entry point 714 * 715 * Used in two ways: It is used by the stack as an init entry 716 * point in network interface structure. It is also used 717 * by the driver as a hw/sw initialization routine to get 718 * to a consistent state. 719 * 720 * return 0 on success, positive on failure 721 ************************************************************************/ 722 static void 723 ixv_init_locked(struct ixgbe_softc *sc) 724 { 725 struct ifnet *ifp = sc->ifp; 726 device_t dev = sc->dev; 727 struct ixgbe_hw *hw = &sc->hw; 728 struct ix_queue *que; 729 int error = 0; 730 uint32_t mask; 731 int i; 732 733 INIT_DEBUGOUT("ixv_init_locked: begin"); 734 KASSERT(mutex_owned(&sc->core_mtx)); 735 hw->adapter_stopped = FALSE; 736 hw->mac.ops.stop_adapter(hw); 737 callout_stop(&sc->timer); 738 for (i = 0, que = sc->queues; i < sc->num_queues; i++, que++) 739 que->disabled_count = 0; 740 741 sc->max_frame_size = 742 ifp->if_mtu + ETHER_HDR_LEN + ETHER_CRC_LEN; 743 744 /* reprogram the RAR[0] in case user changed it. */ 745 hw->mac.ops.set_rar(hw, 0, hw->mac.addr, 0, IXGBE_RAH_AV); 746 747 /* Get the latest mac address, User can use a LAA */ 748 memcpy(hw->mac.addr, CLLADDR(ifp->if_sadl), 749 IXGBE_ETH_LENGTH_OF_ADDRESS); 750 hw->mac.ops.set_rar(hw, 0, hw->mac.addr, 0, 1); 751 752 /* Prepare transmit descriptors and buffers */ 753 if (ixgbe_setup_transmit_structures(sc)) { 754 aprint_error_dev(dev, "Could not setup transmit structures\n"); 755 ixv_stop_locked(sc); 756 return; 757 } 758 759 /* Reset VF and renegotiate mailbox API version */ 760 hw->mac.ops.reset_hw(hw); 761 hw->mac.ops.start_hw(hw); 762 error = ixv_negotiate_api(sc); 763 if (error) 764 device_printf(dev, 765 "Mailbox API negotiation failed in init_locked!\n"); 766 767 ixv_initialize_transmit_units(sc); 768 769 /* Setup Multicast table */ 770 ixv_set_rxfilter(sc); 771 772 /* Use fixed buffer size, even for jumbo frames */ 773 sc->rx_mbuf_sz = MCLBYTES; 774 775 /* Prepare receive descriptors and buffers */ 776 error = ixgbe_setup_receive_structures(sc); 777 if (error) { 778 device_printf(dev, 779 "Could not setup receive structures (err = %d)\n", error); 780 ixv_stop_locked(sc); 781 return; 782 } 783 784 /* Configure RX settings */ 785 ixv_initialize_receive_units(sc); 786 787 /* Initialize variable holding task enqueue requests interrupts */ 788 sc->task_requests = 0; 789 790 /* Set up VLAN offload and filter */ 791 ixv_setup_vlan_support(sc); 792 793 /* Set up MSI-X routing */ 794 ixv_configure_ivars(sc); 795 796 /* Set up auto-mask */ 797 mask = (1 << sc->vector); 798 for (i = 0, que = sc->queues; i < sc->num_queues; i++, que++) 799 mask |= (1 << que->msix); 800 IXGBE_WRITE_REG(hw, IXGBE_VTEIAM, mask); 801 802 /* Set moderation on the Link interrupt */ 803 ixv_eitr_write(sc, sc->vector, IXGBE_LINK_ITR); 804 805 /* Stats init */ 806 ixv_init_stats(sc); 807 808 /* Config/Enable Link */ 809 error = hw->mac.ops.get_link_state(hw, &sc->link_enabled); 810 if (error) { 811 /* PF is not capable of controlling VF state. Enable the link. */ 812 sc->link_enabled = TRUE; 813 } else if (sc->link_enabled == FALSE) 814 device_printf(dev, "VF is disabled by PF\n"); 815 816 hw->mac.get_link_status = TRUE; 817 hw->mac.ops.check_link(hw, &sc->link_speed, &sc->link_up, 818 FALSE); 819 820 /* Start watchdog */ 821 callout_reset(&sc->timer, hz, ixv_local_timer, sc); 822 atomic_store_relaxed(&sc->timer_pending, 0); 823 824 /* OK to schedule workqueues. */ 825 sc->schedule_wqs_ok = true; 826 827 /* Update saved flags. See ixgbe_ifflags_cb() */ 828 sc->if_flags = ifp->if_flags; 829 sc->ec_capenable = sc->osdep.ec.ec_capenable; 830 831 /* Inform the stack we're ready */ 832 ifp->if_flags |= IFF_RUNNING; 833 834 /* And now turn on interrupts */ 835 ixv_enable_intr(sc); 836 837 return; 838 } /* ixv_init_locked */ 839 840 /************************************************************************ 841 * ixv_enable_queue 842 ************************************************************************/ 843 static inline void 844 ixv_enable_queue(struct ixgbe_softc *sc, u32 vector) 845 { 846 struct ixgbe_hw *hw = &sc->hw; 847 struct ix_queue *que = &sc->queues[vector]; 848 u32 queue = 1UL << vector; 849 u32 mask; 850 851 mutex_enter(&que->dc_mtx); 852 if (que->disabled_count > 0 && --que->disabled_count > 0) 853 goto out; 854 855 mask = (IXGBE_EIMS_RTX_QUEUE & queue); 856 IXGBE_WRITE_REG(hw, IXGBE_VTEIMS, mask); 857 out: 858 mutex_exit(&que->dc_mtx); 859 } /* ixv_enable_queue */ 860 861 /************************************************************************ 862 * ixv_disable_queue 863 ************************************************************************/ 864 static inline void 865 ixv_disable_queue(struct ixgbe_softc *sc, u32 vector) 866 { 867 struct ixgbe_hw *hw = &sc->hw; 868 struct ix_queue *que = &sc->queues[vector]; 869 u32 queue = 1UL << vector; 870 u32 mask; 871 872 mutex_enter(&que->dc_mtx); 873 if (que->disabled_count++ > 0) 874 goto out; 875 876 mask = (IXGBE_EIMS_RTX_QUEUE & queue); 877 IXGBE_WRITE_REG(hw, IXGBE_VTEIMC, mask); 878 out: 879 mutex_exit(&que->dc_mtx); 880 } /* ixv_disable_queue */ 881 882 #if 0 883 static inline void 884 ixv_rearm_queues(struct ixgbe_softc *sc, u64 queues) 885 { 886 u32 mask = (IXGBE_EIMS_RTX_QUEUE & queues); 887 IXGBE_WRITE_REG(&sc->hw, IXGBE_VTEICS, mask); 888 } /* ixv_rearm_queues */ 889 #endif 890 891 892 /************************************************************************ 893 * ixv_msix_que - MSI-X Queue Interrupt Service routine 894 ************************************************************************/ 895 static int 896 ixv_msix_que(void *arg) 897 { 898 struct ix_queue *que = arg; 899 struct ixgbe_softc *sc = que->sc; 900 struct tx_ring *txr = que->txr; 901 struct rx_ring *rxr = que->rxr; 902 bool more; 903 u32 newitr = 0; 904 905 ixv_disable_queue(sc, que->msix); 906 IXGBE_EVC_ADD(&que->irqs, 1); 907 908 #ifdef __NetBSD__ 909 /* Don't run ixgbe_rxeof in interrupt context */ 910 more = true; 911 #else 912 more = ixgbe_rxeof(que); 913 #endif 914 915 IXGBE_TX_LOCK(txr); 916 ixgbe_txeof(txr); 917 IXGBE_TX_UNLOCK(txr); 918 919 /* Do AIM now? */ 920 921 if (sc->enable_aim == false) 922 goto no_calc; 923 /* 924 * Do Adaptive Interrupt Moderation: 925 * - Write out last calculated setting 926 * - Calculate based on average size over 927 * the last interval. 928 */ 929 if (que->eitr_setting) 930 ixv_eitr_write(sc, que->msix, que->eitr_setting); 931 932 que->eitr_setting = 0; 933 934 /* Idle, do nothing */ 935 if ((txr->bytes == 0) && (rxr->bytes == 0)) 936 goto no_calc; 937 938 if ((txr->bytes) && (txr->packets)) 939 newitr = txr->bytes/txr->packets; 940 if ((rxr->bytes) && (rxr->packets)) 941 newitr = uimax(newitr, (rxr->bytes / rxr->packets)); 942 newitr += 24; /* account for hardware frame, crc */ 943 944 /* set an upper boundary */ 945 newitr = uimin(newitr, 3000); 946 947 /* Be nice to the mid range */ 948 if ((newitr > 300) && (newitr < 1200)) 949 newitr = (newitr / 3); 950 else 951 newitr = (newitr / 2); 952 953 /* 954 * When RSC is used, ITR interval must be larger than RSC_DELAY. 955 * Currently, we use 2us for RSC_DELAY. The minimum value is always 956 * greater than 2us on 100M (and 10M?(not documented)), but it's not 957 * on 1G and higher. 958 */ 959 if ((sc->link_speed != IXGBE_LINK_SPEED_100_FULL) 960 && (sc->link_speed != IXGBE_LINK_SPEED_10_FULL)) { 961 if (newitr < IXGBE_MIN_RSC_EITR_10G1G) 962 newitr = IXGBE_MIN_RSC_EITR_10G1G; 963 } 964 965 /* save for next interrupt */ 966 que->eitr_setting = newitr; 967 968 /* Reset state */ 969 txr->bytes = 0; 970 txr->packets = 0; 971 rxr->bytes = 0; 972 rxr->packets = 0; 973 974 no_calc: 975 if (more) 976 softint_schedule(que->que_si); 977 else /* Re-enable this interrupt */ 978 ixv_enable_queue(sc, que->msix); 979 980 return 1; 981 } /* ixv_msix_que */ 982 983 /************************************************************************ 984 * ixv_msix_mbx 985 ************************************************************************/ 986 static int 987 ixv_msix_mbx(void *arg) 988 { 989 struct ixgbe_softc *sc = arg; 990 struct ixgbe_hw *hw = &sc->hw; 991 992 IXGBE_EVC_ADD(&sc->admin_irqev, 1); 993 /* NetBSD: We use auto-clear, so it's not required to write VTEICR */ 994 995 /* Link status change */ 996 hw->mac.get_link_status = TRUE; 997 atomic_or_32(&sc->task_requests, IXGBE_REQUEST_TASK_MBX); 998 ixv_schedule_admin_tasklet(sc); 999 1000 return 1; 1001 } /* ixv_msix_mbx */ 1002 1003 static void 1004 ixv_eitr_write(struct ixgbe_softc *sc, uint32_t index, uint32_t itr) 1005 { 1006 1007 /* 1008 * Newer devices than 82598 have VF function, so this function is 1009 * simple. 1010 */ 1011 itr |= IXGBE_EITR_CNT_WDIS; 1012 1013 IXGBE_WRITE_REG(&sc->hw, IXGBE_VTEITR(index), itr); 1014 } 1015 1016 1017 /************************************************************************ 1018 * ixv_media_status - Media Ioctl callback 1019 * 1020 * Called whenever the user queries the status of 1021 * the interface using ifconfig. 1022 ************************************************************************/ 1023 static void 1024 ixv_media_status(struct ifnet *ifp, struct ifmediareq *ifmr) 1025 { 1026 struct ixgbe_softc *sc = ifp->if_softc; 1027 1028 INIT_DEBUGOUT("ixv_media_status: begin"); 1029 ixv_update_link_status(sc); 1030 1031 ifmr->ifm_status = IFM_AVALID; 1032 ifmr->ifm_active = IFM_ETHER; 1033 1034 if (sc->link_active != LINK_STATE_UP) { 1035 ifmr->ifm_active |= IFM_NONE; 1036 return; 1037 } 1038 1039 ifmr->ifm_status |= IFM_ACTIVE; 1040 1041 switch (sc->link_speed) { 1042 case IXGBE_LINK_SPEED_10GB_FULL: 1043 ifmr->ifm_active |= IFM_10G_T | IFM_FDX; 1044 break; 1045 case IXGBE_LINK_SPEED_5GB_FULL: 1046 ifmr->ifm_active |= IFM_5000_T | IFM_FDX; 1047 break; 1048 case IXGBE_LINK_SPEED_2_5GB_FULL: 1049 ifmr->ifm_active |= IFM_2500_T | IFM_FDX; 1050 break; 1051 case IXGBE_LINK_SPEED_1GB_FULL: 1052 ifmr->ifm_active |= IFM_1000_T | IFM_FDX; 1053 break; 1054 case IXGBE_LINK_SPEED_100_FULL: 1055 ifmr->ifm_active |= IFM_100_TX | IFM_FDX; 1056 break; 1057 case IXGBE_LINK_SPEED_10_FULL: 1058 ifmr->ifm_active |= IFM_10_T | IFM_FDX; 1059 break; 1060 } 1061 1062 ifp->if_baudrate = ifmedia_baudrate(ifmr->ifm_active); 1063 } /* ixv_media_status */ 1064 1065 /************************************************************************ 1066 * ixv_media_change - Media Ioctl callback 1067 * 1068 * Called when the user changes speed/duplex using 1069 * media/mediopt option with ifconfig. 1070 ************************************************************************/ 1071 static int 1072 ixv_media_change(struct ifnet *ifp) 1073 { 1074 struct ixgbe_softc *sc = ifp->if_softc; 1075 struct ifmedia *ifm = &sc->media; 1076 1077 INIT_DEBUGOUT("ixv_media_change: begin"); 1078 1079 if (IFM_TYPE(ifm->ifm_media) != IFM_ETHER) 1080 return (EINVAL); 1081 1082 switch (IFM_SUBTYPE(ifm->ifm_media)) { 1083 case IFM_AUTO: 1084 break; 1085 default: 1086 device_printf(sc->dev, "Only auto media type\n"); 1087 return (EINVAL); 1088 } 1089 1090 return (0); 1091 } /* ixv_media_change */ 1092 1093 static void 1094 ixv_schedule_admin_tasklet(struct ixgbe_softc *sc) 1095 { 1096 if (sc->schedule_wqs_ok) { 1097 if (atomic_cas_uint(&sc->admin_pending, 0, 1) == 0) 1098 workqueue_enqueue(sc->admin_wq, 1099 &sc->admin_wc, NULL); 1100 } 1101 } 1102 1103 /************************************************************************ 1104 * ixv_negotiate_api 1105 * 1106 * Negotiate the Mailbox API with the PF; 1107 * start with the most featured API first. 1108 ************************************************************************/ 1109 static int 1110 ixv_negotiate_api(struct ixgbe_softc *sc) 1111 { 1112 struct ixgbe_hw *hw = &sc->hw; 1113 int mbx_api[] = { ixgbe_mbox_api_15, 1114 ixgbe_mbox_api_13, 1115 ixgbe_mbox_api_12, 1116 ixgbe_mbox_api_11, 1117 ixgbe_mbox_api_10, 1118 ixgbe_mbox_api_unknown }; 1119 int i = 0; 1120 1121 while (mbx_api[i] != ixgbe_mbox_api_unknown) { 1122 if (ixgbevf_negotiate_api_version(hw, mbx_api[i]) == 0) { 1123 if (hw->api_version >= ixgbe_mbox_api_15) 1124 ixgbe_upgrade_mbx_params_vf(hw); 1125 return (0); 1126 } 1127 i++; 1128 } 1129 1130 return (EINVAL); 1131 } /* ixv_negotiate_api */ 1132 1133 1134 /************************************************************************ 1135 * ixv_set_rxfilter - Multicast Update 1136 * 1137 * Called whenever multicast address list is updated. 1138 ************************************************************************/ 1139 static int 1140 ixv_set_rxfilter(struct ixgbe_softc *sc) 1141 { 1142 struct ixgbe_mc_addr *mta; 1143 struct ifnet *ifp = sc->ifp; 1144 struct ixgbe_hw *hw = &sc->hw; 1145 u8 *update_ptr; 1146 int mcnt = 0; 1147 struct ethercom *ec = &sc->osdep.ec; 1148 struct ether_multi *enm; 1149 struct ether_multistep step; 1150 bool overflow = false; 1151 int error, rc = 0; 1152 1153 KASSERT(mutex_owned(&sc->core_mtx)); 1154 IOCTL_DEBUGOUT("ixv_set_rxfilter: begin"); 1155 1156 mta = sc->mta; 1157 bzero(mta, sizeof(*mta) * IXGBE_MAX_VF_MC); 1158 1159 /* 1: For PROMISC */ 1160 if (ifp->if_flags & IFF_PROMISC) { 1161 error = hw->mac.ops.update_xcast_mode(hw, 1162 IXGBEVF_XCAST_MODE_PROMISC); 1163 if (error == IXGBE_ERR_NOT_TRUSTED) { 1164 device_printf(sc->dev, 1165 "this interface is not trusted\n"); 1166 error = EPERM; 1167 } else if (error == IXGBE_ERR_FEATURE_NOT_SUPPORTED) { 1168 device_printf(sc->dev, 1169 "the PF doesn't support promisc mode\n"); 1170 error = EOPNOTSUPP; 1171 } else if (error == IXGBE_ERR_NOT_IN_PROMISC) { 1172 device_printf(sc->dev, 1173 "the PF may not in promisc mode\n"); 1174 error = EINVAL; 1175 } else if (error) { 1176 device_printf(sc->dev, 1177 "failed to set promisc mode. error = %d\n", 1178 error); 1179 error = EIO; 1180 } else 1181 return 0; 1182 rc = error; 1183 } 1184 1185 /* 2: For ALLMULTI or normal */ 1186 ETHER_LOCK(ec); 1187 ETHER_FIRST_MULTI(step, ec, enm); 1188 while (enm != NULL) { 1189 if ((mcnt >= IXGBE_MAX_VF_MC) || 1190 (memcmp(enm->enm_addrlo, enm->enm_addrhi, 1191 ETHER_ADDR_LEN) != 0)) { 1192 overflow = true; 1193 break; 1194 } 1195 bcopy(enm->enm_addrlo, 1196 mta[mcnt].addr, IXGBE_ETH_LENGTH_OF_ADDRESS); 1197 mcnt++; 1198 ETHER_NEXT_MULTI(step, enm); 1199 } 1200 ETHER_UNLOCK(ec); 1201 1202 /* 3: For ALLMULTI */ 1203 if (overflow) { 1204 error = hw->mac.ops.update_xcast_mode(hw, 1205 IXGBEVF_XCAST_MODE_ALLMULTI); 1206 if (error == IXGBE_ERR_NOT_TRUSTED) { 1207 device_printf(sc->dev, 1208 "this interface is not trusted\n"); 1209 error = EPERM; 1210 } else if (error == IXGBE_ERR_FEATURE_NOT_SUPPORTED) { 1211 device_printf(sc->dev, 1212 "the PF doesn't support allmulti mode\n"); 1213 error = EOPNOTSUPP; 1214 } else if (error) { 1215 device_printf(sc->dev, 1216 "number of Ethernet multicast addresses " 1217 "exceeds the limit (%d). error = %d\n", 1218 IXGBE_MAX_VF_MC, error); 1219 error = ENOSPC; 1220 } else { 1221 ETHER_LOCK(ec); 1222 ec->ec_flags |= ETHER_F_ALLMULTI; 1223 ETHER_UNLOCK(ec); 1224 return rc; /* Promisc might have failed */ 1225 } 1226 1227 if (rc == 0) 1228 rc = error; 1229 1230 /* Continue to update the multicast table as many as we can */ 1231 } 1232 1233 /* 4: For normal operation */ 1234 error = hw->mac.ops.update_xcast_mode(hw, IXGBEVF_XCAST_MODE_MULTI); 1235 if ((error == IXGBE_ERR_FEATURE_NOT_SUPPORTED) || (error == 0)) { 1236 /* Normal operation */ 1237 ETHER_LOCK(ec); 1238 ec->ec_flags &= ~ETHER_F_ALLMULTI; 1239 ETHER_UNLOCK(ec); 1240 error = 0; 1241 } else if (error) { 1242 device_printf(sc->dev, 1243 "failed to set Ethernet multicast address " 1244 "operation to normal. error = %d\n", error); 1245 } 1246 1247 update_ptr = (u8 *)mta; 1248 error = sc->hw.mac.ops.update_mc_addr_list(&sc->hw, 1249 update_ptr, mcnt, ixv_mc_array_itr, TRUE); 1250 if (rc == 0) 1251 rc = error; 1252 1253 return rc; 1254 } /* ixv_set_rxfilter */ 1255 1256 /************************************************************************ 1257 * ixv_mc_array_itr 1258 * 1259 * An iterator function needed by the multicast shared code. 1260 * It feeds the shared code routine the addresses in the 1261 * array of ixv_set_rxfilter() one by one. 1262 ************************************************************************/ 1263 static u8 * 1264 ixv_mc_array_itr(struct ixgbe_hw *hw, u8 **update_ptr, u32 *vmdq) 1265 { 1266 struct ixgbe_mc_addr *mta; 1267 1268 mta = (struct ixgbe_mc_addr *)*update_ptr; 1269 1270 *vmdq = 0; 1271 *update_ptr = (u8*)(mta + 1); 1272 1273 return (mta->addr); 1274 } /* ixv_mc_array_itr */ 1275 1276 /************************************************************************ 1277 * ixv_local_timer - Timer routine 1278 * 1279 * Checks for link status, updates statistics, 1280 * and runs the watchdog check. 1281 ************************************************************************/ 1282 static void 1283 ixv_local_timer(void *arg) 1284 { 1285 struct ixgbe_softc *sc = arg; 1286 1287 if (sc->schedule_wqs_ok) { 1288 if (atomic_cas_uint(&sc->timer_pending, 0, 1) == 0) 1289 workqueue_enqueue(sc->timer_wq, 1290 &sc->timer_wc, NULL); 1291 } 1292 } 1293 1294 static void 1295 ixv_handle_timer(struct work *wk, void *context) 1296 { 1297 struct ixgbe_softc *sc = context; 1298 device_t dev = sc->dev; 1299 struct ix_queue *que = sc->queues; 1300 u64 queues = 0; 1301 u64 v0, v1, v2, v3, v4, v5, v6, v7; 1302 int hung = 0; 1303 int i; 1304 1305 IXGBE_CORE_LOCK(sc); 1306 1307 if (ixv_check_link(sc)) { 1308 ixv_init_locked(sc); 1309 IXGBE_CORE_UNLOCK(sc); 1310 return; 1311 } 1312 1313 /* Stats Update */ 1314 ixv_update_stats(sc); 1315 1316 /* Update some event counters */ 1317 v0 = v1 = v2 = v3 = v4 = v5 = v6 = v7 = 0; 1318 que = sc->queues; 1319 for (i = 0; i < sc->num_queues; i++, que++) { 1320 struct tx_ring *txr = que->txr; 1321 1322 v0 += txr->q_efbig_tx_dma_setup; 1323 v1 += txr->q_mbuf_defrag_failed; 1324 v2 += txr->q_efbig2_tx_dma_setup; 1325 v3 += txr->q_einval_tx_dma_setup; 1326 v4 += txr->q_other_tx_dma_setup; 1327 v5 += txr->q_eagain_tx_dma_setup; 1328 v6 += txr->q_enomem_tx_dma_setup; 1329 v7 += txr->q_tso_err; 1330 } 1331 IXGBE_EVC_STORE(&sc->efbig_tx_dma_setup, v0); 1332 IXGBE_EVC_STORE(&sc->mbuf_defrag_failed, v1); 1333 IXGBE_EVC_STORE(&sc->efbig2_tx_dma_setup, v2); 1334 IXGBE_EVC_STORE(&sc->einval_tx_dma_setup, v3); 1335 IXGBE_EVC_STORE(&sc->other_tx_dma_setup, v4); 1336 IXGBE_EVC_STORE(&sc->eagain_tx_dma_setup, v5); 1337 IXGBE_EVC_STORE(&sc->enomem_tx_dma_setup, v6); 1338 IXGBE_EVC_STORE(&sc->tso_err, v7); 1339 1340 /* 1341 * Check the TX queues status 1342 * - mark hung queues so we don't schedule on them 1343 * - watchdog only if all queues show hung 1344 */ 1345 que = sc->queues; 1346 for (i = 0; i < sc->num_queues; i++, que++) { 1347 /* Keep track of queues with work for soft irq */ 1348 if (que->txr->busy) 1349 queues |= ((u64)1 << que->me); 1350 /* 1351 * Each time txeof runs without cleaning, but there 1352 * are uncleaned descriptors it increments busy. If 1353 * we get to the MAX we declare it hung. 1354 */ 1355 if (que->busy == IXGBE_QUEUE_HUNG) { 1356 ++hung; 1357 /* Mark the queue as inactive */ 1358 sc->active_queues &= ~((u64)1 << que->me); 1359 continue; 1360 } else { 1361 /* Check if we've come back from hung */ 1362 if ((sc->active_queues & ((u64)1 << que->me)) == 0) 1363 sc->active_queues |= ((u64)1 << que->me); 1364 } 1365 if (que->busy >= IXGBE_MAX_TX_BUSY) { 1366 device_printf(dev, 1367 "Warning queue %d appears to be hung!\n", i); 1368 que->txr->busy = IXGBE_QUEUE_HUNG; 1369 ++hung; 1370 } 1371 } 1372 1373 /* Only truly watchdog if all queues show hung */ 1374 if (hung == sc->num_queues) 1375 goto watchdog; 1376 #if 0 1377 else if (queues != 0) { /* Force an IRQ on queues with work */ 1378 ixv_rearm_queues(sc, queues); 1379 } 1380 #endif 1381 1382 atomic_store_relaxed(&sc->timer_pending, 0); 1383 IXGBE_CORE_UNLOCK(sc); 1384 callout_reset(&sc->timer, hz, ixv_local_timer, sc); 1385 1386 return; 1387 1388 watchdog: 1389 device_printf(sc->dev, "Watchdog timeout -- resetting\n"); 1390 sc->ifp->if_flags &= ~IFF_RUNNING; 1391 IXGBE_EVC_ADD(&sc->watchdog_events, 1); 1392 ixv_init_locked(sc); 1393 IXGBE_CORE_UNLOCK(sc); 1394 } /* ixv_handle_timer */ 1395 1396 /************************************************************************ 1397 * ixv_update_link_status - Update OS on link state 1398 * 1399 * Note: Only updates the OS on the cached link state. 1400 * The real check of the hardware only happens with 1401 * a link interrupt. 1402 ************************************************************************/ 1403 static void 1404 ixv_update_link_status(struct ixgbe_softc *sc) 1405 { 1406 struct ifnet *ifp = sc->ifp; 1407 device_t dev = sc->dev; 1408 1409 KASSERT(mutex_owned(&sc->core_mtx)); 1410 1411 if (sc->link_up && sc->link_enabled) { 1412 if (sc->link_active != LINK_STATE_UP) { 1413 if (bootverbose) { 1414 const char *bpsmsg; 1415 1416 switch (sc->link_speed) { 1417 case IXGBE_LINK_SPEED_10GB_FULL: 1418 bpsmsg = "10 Gbps"; 1419 break; 1420 case IXGBE_LINK_SPEED_5GB_FULL: 1421 bpsmsg = "5 Gbps"; 1422 break; 1423 case IXGBE_LINK_SPEED_2_5GB_FULL: 1424 bpsmsg = "2.5 Gbps"; 1425 break; 1426 case IXGBE_LINK_SPEED_1GB_FULL: 1427 bpsmsg = "1 Gbps"; 1428 break; 1429 case IXGBE_LINK_SPEED_100_FULL: 1430 bpsmsg = "100 Mbps"; 1431 break; 1432 case IXGBE_LINK_SPEED_10_FULL: 1433 bpsmsg = "10 Mbps"; 1434 break; 1435 default: 1436 bpsmsg = "unknown speed"; 1437 break; 1438 } 1439 device_printf(dev, "Link is up %s %s \n", 1440 bpsmsg, "Full Duplex"); 1441 } 1442 sc->link_active = LINK_STATE_UP; 1443 if_link_state_change(ifp, LINK_STATE_UP); 1444 } 1445 } else { 1446 /* 1447 * Do it when link active changes to DOWN. i.e. 1448 * a) LINK_STATE_UNKNOWN -> LINK_STATE_DOWN 1449 * b) LINK_STATE_UP -> LINK_STATE_DOWN 1450 */ 1451 if (sc->link_active != LINK_STATE_DOWN) { 1452 if (bootverbose) 1453 device_printf(dev, "Link is Down\n"); 1454 if_link_state_change(ifp, LINK_STATE_DOWN); 1455 sc->link_active = LINK_STATE_DOWN; 1456 } 1457 } 1458 } /* ixv_update_link_status */ 1459 1460 1461 /************************************************************************ 1462 * ixv_stop - Stop the hardware 1463 * 1464 * Disables all traffic on the adapter by issuing a 1465 * global reset on the MAC and deallocates TX/RX buffers. 1466 ************************************************************************/ 1467 static void 1468 ixv_ifstop(struct ifnet *ifp, int disable) 1469 { 1470 struct ixgbe_softc *sc = ifp->if_softc; 1471 1472 IXGBE_CORE_LOCK(sc); 1473 ixv_stop_locked(sc); 1474 IXGBE_CORE_UNLOCK(sc); 1475 1476 workqueue_wait(sc->admin_wq, &sc->admin_wc); 1477 atomic_store_relaxed(&sc->admin_pending, 0); 1478 workqueue_wait(sc->timer_wq, &sc->timer_wc); 1479 atomic_store_relaxed(&sc->timer_pending, 0); 1480 } 1481 1482 static void 1483 ixv_stop_locked(void *arg) 1484 { 1485 struct ifnet *ifp; 1486 struct ixgbe_softc *sc = arg; 1487 struct ixgbe_hw *hw = &sc->hw; 1488 1489 ifp = sc->ifp; 1490 1491 KASSERT(mutex_owned(&sc->core_mtx)); 1492 1493 INIT_DEBUGOUT("ixv_stop_locked: begin\n"); 1494 ixv_disable_intr(sc); 1495 1496 /* Tell the stack that the interface is no longer active */ 1497 ifp->if_flags &= ~IFF_RUNNING; 1498 1499 hw->mac.ops.reset_hw(hw); 1500 sc->hw.adapter_stopped = FALSE; 1501 hw->mac.ops.stop_adapter(hw); 1502 callout_stop(&sc->timer); 1503 1504 /* Don't schedule workqueues. */ 1505 sc->schedule_wqs_ok = false; 1506 1507 /* reprogram the RAR[0] in case user changed it. */ 1508 hw->mac.ops.set_rar(hw, 0, hw->mac.addr, 0, IXGBE_RAH_AV); 1509 1510 return; 1511 } /* ixv_stop_locked */ 1512 1513 1514 /************************************************************************ 1515 * ixv_allocate_pci_resources 1516 ************************************************************************/ 1517 static int 1518 ixv_allocate_pci_resources(struct ixgbe_softc *sc, 1519 const struct pci_attach_args *pa) 1520 { 1521 pcireg_t memtype, csr; 1522 device_t dev = sc->dev; 1523 bus_addr_t addr; 1524 int flags; 1525 1526 memtype = pci_mapreg_type(pa->pa_pc, pa->pa_tag, PCI_BAR(0)); 1527 switch (memtype) { 1528 case PCI_MAPREG_TYPE_MEM | PCI_MAPREG_MEM_TYPE_32BIT: 1529 case PCI_MAPREG_TYPE_MEM | PCI_MAPREG_MEM_TYPE_64BIT: 1530 sc->osdep.mem_bus_space_tag = pa->pa_memt; 1531 if (pci_mapreg_info(pa->pa_pc, pa->pa_tag, PCI_BAR(0), 1532 memtype, &addr, &sc->osdep.mem_size, &flags) != 0) 1533 goto map_err; 1534 if ((flags & BUS_SPACE_MAP_PREFETCHABLE) != 0) { 1535 aprint_normal_dev(dev, "clearing prefetchable bit\n"); 1536 flags &= ~BUS_SPACE_MAP_PREFETCHABLE; 1537 } 1538 if (bus_space_map(sc->osdep.mem_bus_space_tag, addr, 1539 sc->osdep.mem_size, flags, 1540 &sc->osdep.mem_bus_space_handle) != 0) { 1541 map_err: 1542 sc->osdep.mem_size = 0; 1543 aprint_error_dev(dev, "unable to map BAR0\n"); 1544 return ENXIO; 1545 } 1546 /* 1547 * Enable address decoding for memory range in case it's not 1548 * set. 1549 */ 1550 csr = pci_conf_read(pa->pa_pc, pa->pa_tag, 1551 PCI_COMMAND_STATUS_REG); 1552 csr |= PCI_COMMAND_MEM_ENABLE; 1553 pci_conf_write(pa->pa_pc, pa->pa_tag, PCI_COMMAND_STATUS_REG, 1554 csr); 1555 break; 1556 default: 1557 aprint_error_dev(dev, "unexpected type on BAR0\n"); 1558 return ENXIO; 1559 } 1560 1561 /* Pick up the tuneable queues */ 1562 sc->num_queues = ixv_num_queues; 1563 1564 return (0); 1565 } /* ixv_allocate_pci_resources */ 1566 1567 static void 1568 ixv_free_deferred_handlers(struct ixgbe_softc *sc) 1569 { 1570 struct ix_queue *que = sc->queues; 1571 struct tx_ring *txr = sc->tx_rings; 1572 int i; 1573 1574 for (i = 0; i < sc->num_queues; i++, que++, txr++) { 1575 if (!(sc->feat_en & IXGBE_FEATURE_LEGACY_TX)) { 1576 if (txr->txr_si != NULL) 1577 softint_disestablish(txr->txr_si); 1578 } 1579 if (que->que_si != NULL) 1580 softint_disestablish(que->que_si); 1581 } 1582 if (sc->txr_wq != NULL) 1583 workqueue_destroy(sc->txr_wq); 1584 if (sc->txr_wq_enqueued != NULL) 1585 percpu_free(sc->txr_wq_enqueued, sizeof(u_int)); 1586 if (sc->que_wq != NULL) 1587 workqueue_destroy(sc->que_wq); 1588 1589 /* Drain the Mailbox(link) queue */ 1590 if (sc->admin_wq != NULL) { 1591 workqueue_destroy(sc->admin_wq); 1592 sc->admin_wq = NULL; 1593 } 1594 if (sc->timer_wq != NULL) { 1595 workqueue_destroy(sc->timer_wq); 1596 sc->timer_wq = NULL; 1597 } 1598 } /* ixv_free_deferred_handlers */ 1599 1600 /************************************************************************ 1601 * ixv_free_pci_resources 1602 ************************************************************************/ 1603 static void 1604 ixv_free_pci_resources(struct ixgbe_softc *sc) 1605 { 1606 struct ix_queue *que = sc->queues; 1607 int rid; 1608 1609 /* 1610 * Release all msix queue resources: 1611 */ 1612 for (int i = 0; i < sc->num_queues; i++, que++) { 1613 if (que->res != NULL) 1614 pci_intr_disestablish(sc->osdep.pc, 1615 sc->osdep.ihs[i]); 1616 } 1617 1618 1619 /* Clean the Mailbox interrupt last */ 1620 rid = sc->vector; 1621 1622 if (sc->osdep.ihs[rid] != NULL) { 1623 pci_intr_disestablish(sc->osdep.pc, 1624 sc->osdep.ihs[rid]); 1625 sc->osdep.ihs[rid] = NULL; 1626 } 1627 1628 pci_intr_release(sc->osdep.pc, sc->osdep.intrs, 1629 sc->osdep.nintrs); 1630 1631 if (sc->osdep.mem_size != 0) { 1632 bus_space_unmap(sc->osdep.mem_bus_space_tag, 1633 sc->osdep.mem_bus_space_handle, 1634 sc->osdep.mem_size); 1635 } 1636 1637 return; 1638 } /* ixv_free_pci_resources */ 1639 1640 /************************************************************************ 1641 * ixv_setup_interface 1642 * 1643 * Setup networking device structure and register an interface. 1644 ************************************************************************/ 1645 static int 1646 ixv_setup_interface(device_t dev, struct ixgbe_softc *sc) 1647 { 1648 struct ethercom *ec = &sc->osdep.ec; 1649 struct ifnet *ifp; 1650 1651 INIT_DEBUGOUT("ixv_setup_interface: begin"); 1652 1653 ifp = sc->ifp = &ec->ec_if; 1654 strlcpy(ifp->if_xname, device_xname(dev), IFNAMSIZ); 1655 ifp->if_baudrate = IF_Gbps(10); 1656 ifp->if_init = ixv_init; 1657 ifp->if_stop = ixv_ifstop; 1658 ifp->if_softc = sc; 1659 ifp->if_flags = IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST; 1660 #ifdef IXGBE_MPSAFE 1661 ifp->if_extflags = IFEF_MPSAFE; 1662 #endif 1663 ifp->if_ioctl = ixv_ioctl; 1664 if (sc->feat_en & IXGBE_FEATURE_LEGACY_TX) { 1665 #if 0 1666 ixv_start_locked = ixgbe_legacy_start_locked; 1667 #endif 1668 } else { 1669 ifp->if_transmit = ixgbe_mq_start; 1670 #if 0 1671 ixv_start_locked = ixgbe_mq_start_locked; 1672 #endif 1673 } 1674 ifp->if_start = ixgbe_legacy_start; 1675 IFQ_SET_MAXLEN(&ifp->if_snd, sc->num_tx_desc - 2); 1676 IFQ_SET_READY(&ifp->if_snd); 1677 1678 if_initialize(ifp); 1679 sc->ipq = if_percpuq_create(&sc->osdep.ec.ec_if); 1680 /* 1681 * We use per TX queue softint, so if_deferred_start_init() isn't 1682 * used. 1683 */ 1684 1685 sc->max_frame_size = ifp->if_mtu + IXGBE_MTU_HDR; 1686 1687 /* 1688 * Tell the upper layer(s) we support long frames. 1689 */ 1690 ifp->if_hdrlen = sizeof(struct ether_vlan_header); 1691 1692 /* Set capability flags */ 1693 ifp->if_capabilities |= IFCAP_HWCSUM 1694 | IFCAP_TSOv4 1695 | IFCAP_TSOv6; 1696 ifp->if_capenable = 0; 1697 1698 ec->ec_capabilities |= ETHERCAP_VLAN_HWFILTER 1699 | ETHERCAP_VLAN_HWTAGGING 1700 | ETHERCAP_VLAN_HWCSUM 1701 | ETHERCAP_JUMBO_MTU 1702 | ETHERCAP_VLAN_MTU; 1703 1704 /* Enable the above capabilities by default */ 1705 ec->ec_capenable = ec->ec_capabilities; 1706 1707 ether_ifattach(ifp, sc->hw.mac.addr); 1708 aprint_normal_dev(dev, "Ethernet address %s\n", 1709 ether_sprintf(sc->hw.mac.addr)); 1710 ether_set_ifflags_cb(ec, ixv_ifflags_cb); 1711 1712 /* Don't enable LRO by default */ 1713 #if 0 1714 /* NetBSD doesn't support LRO yet */ 1715 ifp->if_capabilities |= IFCAP_LRO; 1716 #endif 1717 1718 /* 1719 * Specify the media types supported by this adapter and register 1720 * callbacks to update media and link information 1721 */ 1722 ec->ec_ifmedia = &sc->media; 1723 ifmedia_init_with_lock(&sc->media, IFM_IMASK, ixv_media_change, 1724 ixv_media_status, &sc->core_mtx); 1725 ifmedia_add(&sc->media, IFM_ETHER | IFM_AUTO, 0, NULL); 1726 ifmedia_set(&sc->media, IFM_ETHER | IFM_AUTO); 1727 1728 if_register(ifp); 1729 1730 return 0; 1731 } /* ixv_setup_interface */ 1732 1733 1734 /************************************************************************ 1735 * ixv_initialize_transmit_units - Enable transmit unit. 1736 ************************************************************************/ 1737 static void 1738 ixv_initialize_transmit_units(struct ixgbe_softc *sc) 1739 { 1740 struct tx_ring *txr = sc->tx_rings; 1741 struct ixgbe_hw *hw = &sc->hw; 1742 int i; 1743 1744 for (i = 0; i < sc->num_queues; i++, txr++) { 1745 u64 tdba = txr->txdma.dma_paddr; 1746 u32 txctrl, txdctl; 1747 int j = txr->me; 1748 1749 /* Set WTHRESH to 8, burst writeback */ 1750 txdctl = IXGBE_READ_REG(hw, IXGBE_VFTXDCTL(j)); 1751 txdctl &= ~IXGBE_TXDCTL_WTHRESH_MASK; 1752 txdctl |= IXGBE_TX_WTHRESH << IXGBE_TXDCTL_WTHRESH_SHIFT; 1753 IXGBE_WRITE_REG(hw, IXGBE_VFTXDCTL(j), txdctl); 1754 1755 /* Set the HW Tx Head and Tail indices */ 1756 IXGBE_WRITE_REG(&sc->hw, IXGBE_VFTDH(j), 0); 1757 IXGBE_WRITE_REG(&sc->hw, IXGBE_VFTDT(j), 0); 1758 1759 /* Set Tx Tail register */ 1760 txr->tail = IXGBE_VFTDT(j); 1761 1762 txr->txr_no_space = false; 1763 1764 /* Set Ring parameters */ 1765 IXGBE_WRITE_REG(hw, IXGBE_VFTDBAL(j), 1766 (tdba & 0x00000000ffffffffULL)); 1767 IXGBE_WRITE_REG(hw, IXGBE_VFTDBAH(j), (tdba >> 32)); 1768 IXGBE_WRITE_REG(hw, IXGBE_VFTDLEN(j), 1769 sc->num_tx_desc * sizeof(struct ixgbe_legacy_tx_desc)); 1770 txctrl = IXGBE_READ_REG(hw, IXGBE_VFDCA_TXCTRL(j)); 1771 txctrl &= ~IXGBE_DCA_TXCTRL_DESC_WRO_EN; 1772 IXGBE_WRITE_REG(hw, IXGBE_VFDCA_TXCTRL(j), txctrl); 1773 1774 /* Now enable */ 1775 txdctl = IXGBE_READ_REG(hw, IXGBE_VFTXDCTL(j)); 1776 txdctl |= IXGBE_TXDCTL_ENABLE; 1777 IXGBE_WRITE_REG(hw, IXGBE_VFTXDCTL(j), txdctl); 1778 } 1779 1780 return; 1781 } /* ixv_initialize_transmit_units */ 1782 1783 1784 /************************************************************************ 1785 * ixv_initialize_rss_mapping 1786 ************************************************************************/ 1787 static void 1788 ixv_initialize_rss_mapping(struct ixgbe_softc *sc) 1789 { 1790 struct ixgbe_hw *hw = &sc->hw; 1791 u32 reta = 0, mrqc, rss_key[10]; 1792 int queue_id; 1793 int i, j; 1794 u32 rss_hash_config; 1795 1796 /* force use default RSS key. */ 1797 #ifdef __NetBSD__ 1798 rss_getkey((uint8_t *) &rss_key); 1799 #else 1800 if (sc->feat_en & IXGBE_FEATURE_RSS) { 1801 /* Fetch the configured RSS key */ 1802 rss_getkey((uint8_t *)&rss_key); 1803 } else { 1804 /* set up random bits */ 1805 cprng_fast(&rss_key, sizeof(rss_key)); 1806 } 1807 #endif 1808 1809 /* Now fill out hash function seeds */ 1810 for (i = 0; i < 10; i++) 1811 IXGBE_WRITE_REG(hw, IXGBE_VFRSSRK(i), rss_key[i]); 1812 1813 /* Set up the redirection table */ 1814 for (i = 0, j = 0; i < 64; i++, j++) { 1815 if (j == sc->num_queues) 1816 j = 0; 1817 1818 if (sc->feat_en & IXGBE_FEATURE_RSS) { 1819 /* 1820 * Fetch the RSS bucket id for the given indirection 1821 * entry. Cap it at the number of configured buckets 1822 * (which is num_queues.) 1823 */ 1824 queue_id = rss_get_indirection_to_bucket(i); 1825 queue_id = queue_id % sc->num_queues; 1826 } else 1827 queue_id = j; 1828 1829 /* 1830 * The low 8 bits are for hash value (n+0); 1831 * The next 8 bits are for hash value (n+1), etc. 1832 */ 1833 reta >>= 8; 1834 reta |= ((uint32_t)queue_id) << 24; 1835 if ((i & 3) == 3) { 1836 IXGBE_WRITE_REG(hw, IXGBE_VFRETA(i >> 2), reta); 1837 reta = 0; 1838 } 1839 } 1840 1841 /* Perform hash on these packet types */ 1842 if (sc->feat_en & IXGBE_FEATURE_RSS) 1843 rss_hash_config = rss_gethashconfig(); 1844 else { 1845 /* 1846 * Disable UDP - IP fragments aren't currently being handled 1847 * and so we end up with a mix of 2-tuple and 4-tuple 1848 * traffic. 1849 */ 1850 rss_hash_config = RSS_HASHTYPE_RSS_IPV4 1851 | RSS_HASHTYPE_RSS_TCP_IPV4 1852 | RSS_HASHTYPE_RSS_IPV6 1853 | RSS_HASHTYPE_RSS_TCP_IPV6; 1854 } 1855 1856 mrqc = IXGBE_MRQC_RSSEN; 1857 if (rss_hash_config & RSS_HASHTYPE_RSS_IPV4) 1858 mrqc |= IXGBE_MRQC_RSS_FIELD_IPV4; 1859 if (rss_hash_config & RSS_HASHTYPE_RSS_TCP_IPV4) 1860 mrqc |= IXGBE_MRQC_RSS_FIELD_IPV4_TCP; 1861 if (rss_hash_config & RSS_HASHTYPE_RSS_IPV6) 1862 mrqc |= IXGBE_MRQC_RSS_FIELD_IPV6; 1863 if (rss_hash_config & RSS_HASHTYPE_RSS_TCP_IPV6) 1864 mrqc |= IXGBE_MRQC_RSS_FIELD_IPV6_TCP; 1865 if (rss_hash_config & RSS_HASHTYPE_RSS_IPV6_EX) 1866 device_printf(sc->dev, "%s: RSS_HASHTYPE_RSS_IPV6_EX " 1867 "defined, but not supported\n", __func__); 1868 if (rss_hash_config & RSS_HASHTYPE_RSS_TCP_IPV6_EX) 1869 device_printf(sc->dev, "%s: RSS_HASHTYPE_RSS_TCP_IPV6_EX " 1870 "defined, but not supported\n", __func__); 1871 if (rss_hash_config & RSS_HASHTYPE_RSS_UDP_IPV4) 1872 mrqc |= IXGBE_MRQC_RSS_FIELD_IPV4_UDP; 1873 if (rss_hash_config & RSS_HASHTYPE_RSS_UDP_IPV6) 1874 mrqc |= IXGBE_MRQC_RSS_FIELD_IPV6_UDP; 1875 if (rss_hash_config & RSS_HASHTYPE_RSS_UDP_IPV6_EX) 1876 device_printf(sc->dev, "%s: RSS_HASHTYPE_RSS_UDP_IPV6_EX " 1877 "defined, but not supported\n", __func__); 1878 IXGBE_WRITE_REG(hw, IXGBE_VFMRQC, mrqc); 1879 } /* ixv_initialize_rss_mapping */ 1880 1881 1882 /************************************************************************ 1883 * ixv_initialize_receive_units - Setup receive registers and features. 1884 ************************************************************************/ 1885 static void 1886 ixv_initialize_receive_units(struct ixgbe_softc *sc) 1887 { 1888 struct rx_ring *rxr = sc->rx_rings; 1889 struct ixgbe_hw *hw = &sc->hw; 1890 struct ifnet *ifp = sc->ifp; 1891 u32 bufsz, psrtype; 1892 1893 if (ifp->if_mtu > ETHERMTU) 1894 bufsz = 4096 >> IXGBE_SRRCTL_BSIZEPKT_SHIFT; 1895 else 1896 bufsz = 2048 >> IXGBE_SRRCTL_BSIZEPKT_SHIFT; 1897 1898 psrtype = IXGBE_PSRTYPE_TCPHDR 1899 | IXGBE_PSRTYPE_UDPHDR 1900 | IXGBE_PSRTYPE_IPV4HDR 1901 | IXGBE_PSRTYPE_IPV6HDR 1902 | IXGBE_PSRTYPE_L2HDR; 1903 1904 if (sc->num_queues > 1) 1905 psrtype |= 1 << 29; 1906 1907 IXGBE_WRITE_REG(hw, IXGBE_VFPSRTYPE, psrtype); 1908 1909 /* Tell PF our max_frame size */ 1910 if (ixgbevf_rlpml_set_vf(hw, sc->max_frame_size) != 0) { 1911 device_printf(sc->dev, "There is a problem with the PF " 1912 "setup. It is likely the receive unit for this VF will " 1913 "not function correctly.\n"); 1914 } 1915 1916 for (int i = 0; i < sc->num_queues; i++, rxr++) { 1917 u64 rdba = rxr->rxdma.dma_paddr; 1918 u32 reg, rxdctl; 1919 int j = rxr->me; 1920 1921 /* Disable the queue */ 1922 rxdctl = IXGBE_READ_REG(hw, IXGBE_VFRXDCTL(j)); 1923 rxdctl &= ~IXGBE_RXDCTL_ENABLE; 1924 IXGBE_WRITE_REG(hw, IXGBE_VFRXDCTL(j), rxdctl); 1925 for (int k = 0; k < 10; k++) { 1926 if (IXGBE_READ_REG(hw, IXGBE_VFRXDCTL(j)) & 1927 IXGBE_RXDCTL_ENABLE) 1928 msec_delay(1); 1929 else 1930 break; 1931 } 1932 IXGBE_WRITE_BARRIER(hw); 1933 /* Setup the Base and Length of the Rx Descriptor Ring */ 1934 IXGBE_WRITE_REG(hw, IXGBE_VFRDBAL(j), 1935 (rdba & 0x00000000ffffffffULL)); 1936 IXGBE_WRITE_REG(hw, IXGBE_VFRDBAH(j), (rdba >> 32)); 1937 IXGBE_WRITE_REG(hw, IXGBE_VFRDLEN(j), 1938 sc->num_rx_desc * sizeof(union ixgbe_adv_rx_desc)); 1939 1940 /* Reset the ring indices */ 1941 IXGBE_WRITE_REG(hw, IXGBE_VFRDH(rxr->me), 0); 1942 IXGBE_WRITE_REG(hw, IXGBE_VFRDT(rxr->me), 0); 1943 1944 /* Set up the SRRCTL register */ 1945 reg = IXGBE_READ_REG(hw, IXGBE_VFSRRCTL(j)); 1946 reg &= ~IXGBE_SRRCTL_BSIZEHDR_MASK; 1947 reg &= ~IXGBE_SRRCTL_BSIZEPKT_MASK; 1948 reg |= bufsz; 1949 reg |= IXGBE_SRRCTL_DESCTYPE_ADV_ONEBUF; 1950 IXGBE_WRITE_REG(hw, IXGBE_VFSRRCTL(j), reg); 1951 1952 /* Capture Rx Tail index */ 1953 rxr->tail = IXGBE_VFRDT(rxr->me); 1954 1955 /* Do the queue enabling last */ 1956 rxdctl |= IXGBE_RXDCTL_ENABLE | IXGBE_RXDCTL_VME; 1957 IXGBE_WRITE_REG(hw, IXGBE_VFRXDCTL(j), rxdctl); 1958 for (int k = 0; k < 10; k++) { 1959 if (IXGBE_READ_REG(hw, IXGBE_VFRXDCTL(j)) & 1960 IXGBE_RXDCTL_ENABLE) 1961 break; 1962 msec_delay(1); 1963 } 1964 IXGBE_WRITE_BARRIER(hw); 1965 1966 /* Set the Tail Pointer */ 1967 #ifdef DEV_NETMAP 1968 /* 1969 * In netmap mode, we must preserve the buffers made 1970 * available to userspace before the if_init() 1971 * (this is true by default on the TX side, because 1972 * init makes all buffers available to userspace). 1973 * 1974 * netmap_reset() and the device specific routines 1975 * (e.g. ixgbe_setup_receive_rings()) map these 1976 * buffers at the end of the NIC ring, so here we 1977 * must set the RDT (tail) register to make sure 1978 * they are not overwritten. 1979 * 1980 * In this driver the NIC ring starts at RDH = 0, 1981 * RDT points to the last slot available for reception (?), 1982 * so RDT = num_rx_desc - 1 means the whole ring is available. 1983 */ 1984 if ((sc->feat_en & IXGBE_FEATURE_NETMAP) && 1985 (ifp->if_capenable & IFCAP_NETMAP)) { 1986 struct netmap_adapter *na = NA(sc->ifp); 1987 struct netmap_kring *kring = na->rx_rings[i]; 1988 int t = na->num_rx_desc - 1 - nm_kr_rxspace(kring); 1989 1990 IXGBE_WRITE_REG(hw, IXGBE_VFRDT(rxr->me), t); 1991 } else 1992 #endif /* DEV_NETMAP */ 1993 IXGBE_WRITE_REG(hw, IXGBE_VFRDT(rxr->me), 1994 sc->num_rx_desc - 1); 1995 } 1996 1997 if (sc->hw.mac.type >= ixgbe_mac_X550_vf) 1998 ixv_initialize_rss_mapping(sc); 1999 } /* ixv_initialize_receive_units */ 2000 2001 /************************************************************************ 2002 * ixv_sysctl_tdh_handler - Transmit Descriptor Head handler function 2003 * 2004 * Retrieves the TDH value from the hardware 2005 ************************************************************************/ 2006 static int 2007 ixv_sysctl_tdh_handler(SYSCTLFN_ARGS) 2008 { 2009 struct sysctlnode node = *rnode; 2010 struct tx_ring *txr = (struct tx_ring *)node.sysctl_data; 2011 uint32_t val; 2012 2013 if (!txr) 2014 return (0); 2015 2016 val = IXGBE_READ_REG(&txr->sc->hw, IXGBE_VFTDH(txr->me)); 2017 node.sysctl_data = &val; 2018 return sysctl_lookup(SYSCTLFN_CALL(&node)); 2019 } /* ixv_sysctl_tdh_handler */ 2020 2021 /************************************************************************ 2022 * ixgbe_sysctl_tdt_handler - Transmit Descriptor Tail handler function 2023 * 2024 * Retrieves the TDT value from the hardware 2025 ************************************************************************/ 2026 static int 2027 ixv_sysctl_tdt_handler(SYSCTLFN_ARGS) 2028 { 2029 struct sysctlnode node = *rnode; 2030 struct tx_ring *txr = (struct tx_ring *)node.sysctl_data; 2031 uint32_t val; 2032 2033 if (!txr) 2034 return (0); 2035 2036 val = IXGBE_READ_REG(&txr->sc->hw, IXGBE_VFTDT(txr->me)); 2037 node.sysctl_data = &val; 2038 return sysctl_lookup(SYSCTLFN_CALL(&node)); 2039 } /* ixv_sysctl_tdt_handler */ 2040 2041 /************************************************************************ 2042 * ixv_sysctl_next_to_check_handler - Receive Descriptor next to check 2043 * handler function 2044 * 2045 * Retrieves the next_to_check value 2046 ************************************************************************/ 2047 static int 2048 ixv_sysctl_next_to_check_handler(SYSCTLFN_ARGS) 2049 { 2050 struct sysctlnode node = *rnode; 2051 struct rx_ring *rxr = (struct rx_ring *)node.sysctl_data; 2052 uint32_t val; 2053 2054 if (!rxr) 2055 return (0); 2056 2057 val = rxr->next_to_check; 2058 node.sysctl_data = &val; 2059 return sysctl_lookup(SYSCTLFN_CALL(&node)); 2060 } /* ixv_sysctl_next_to_check_handler */ 2061 2062 /************************************************************************ 2063 * ixv_sysctl_next_to_refresh_handler - Receive Descriptor next to refresh 2064 * handler function 2065 * 2066 * Retrieves the next_to_refresh value 2067 ************************************************************************/ 2068 static int 2069 ixv_sysctl_next_to_refresh_handler(SYSCTLFN_ARGS) 2070 { 2071 struct sysctlnode node = *rnode; 2072 struct rx_ring *rxr = (struct rx_ring *)node.sysctl_data; 2073 struct ixgbe_softc *sc; 2074 uint32_t val; 2075 2076 if (!rxr) 2077 return (0); 2078 2079 sc = rxr->sc; 2080 if (ixgbe_fw_recovery_mode_swflag(sc)) 2081 return (EPERM); 2082 2083 val = rxr->next_to_refresh; 2084 node.sysctl_data = &val; 2085 return sysctl_lookup(SYSCTLFN_CALL(&node)); 2086 } /* ixv_sysctl_next_to_refresh_handler */ 2087 2088 /************************************************************************ 2089 * ixv_sysctl_rdh_handler - Receive Descriptor Head handler function 2090 * 2091 * Retrieves the RDH value from the hardware 2092 ************************************************************************/ 2093 static int 2094 ixv_sysctl_rdh_handler(SYSCTLFN_ARGS) 2095 { 2096 struct sysctlnode node = *rnode; 2097 struct rx_ring *rxr = (struct rx_ring *)node.sysctl_data; 2098 uint32_t val; 2099 2100 if (!rxr) 2101 return (0); 2102 2103 val = IXGBE_READ_REG(&rxr->sc->hw, IXGBE_VFRDH(rxr->me)); 2104 node.sysctl_data = &val; 2105 return sysctl_lookup(SYSCTLFN_CALL(&node)); 2106 } /* ixv_sysctl_rdh_handler */ 2107 2108 /************************************************************************ 2109 * ixv_sysctl_rdt_handler - Receive Descriptor Tail handler function 2110 * 2111 * Retrieves the RDT value from the hardware 2112 ************************************************************************/ 2113 static int 2114 ixv_sysctl_rdt_handler(SYSCTLFN_ARGS) 2115 { 2116 struct sysctlnode node = *rnode; 2117 struct rx_ring *rxr = (struct rx_ring *)node.sysctl_data; 2118 uint32_t val; 2119 2120 if (!rxr) 2121 return (0); 2122 2123 val = IXGBE_READ_REG(&rxr->sc->hw, IXGBE_VFRDT(rxr->me)); 2124 node.sysctl_data = &val; 2125 return sysctl_lookup(SYSCTLFN_CALL(&node)); 2126 } /* ixv_sysctl_rdt_handler */ 2127 2128 static void 2129 ixv_setup_vlan_tagging(struct ixgbe_softc *sc) 2130 { 2131 struct ethercom *ec = &sc->osdep.ec; 2132 struct ixgbe_hw *hw = &sc->hw; 2133 struct rx_ring *rxr; 2134 u32 ctrl; 2135 int i; 2136 bool hwtagging; 2137 2138 /* Enable HW tagging only if any vlan is attached */ 2139 hwtagging = (ec->ec_capenable & ETHERCAP_VLAN_HWTAGGING) 2140 && VLAN_ATTACHED(ec); 2141 2142 /* Enable the queues */ 2143 for (i = 0; i < sc->num_queues; i++) { 2144 rxr = &sc->rx_rings[i]; 2145 ctrl = IXGBE_READ_REG(hw, IXGBE_VFRXDCTL(rxr->me)); 2146 if (hwtagging) 2147 ctrl |= IXGBE_RXDCTL_VME; 2148 else 2149 ctrl &= ~IXGBE_RXDCTL_VME; 2150 IXGBE_WRITE_REG(hw, IXGBE_VFRXDCTL(rxr->me), ctrl); 2151 /* 2152 * Let Rx path know that it needs to store VLAN tag 2153 * as part of extra mbuf info. 2154 */ 2155 rxr->vtag_strip = hwtagging ? TRUE : FALSE; 2156 } 2157 } /* ixv_setup_vlan_tagging */ 2158 2159 /************************************************************************ 2160 * ixv_setup_vlan_support 2161 ************************************************************************/ 2162 static int 2163 ixv_setup_vlan_support(struct ixgbe_softc *sc) 2164 { 2165 struct ethercom *ec = &sc->osdep.ec; 2166 struct ixgbe_hw *hw = &sc->hw; 2167 u32 vid, vfta, retry; 2168 struct vlanid_list *vlanidp; 2169 int rv, error = 0; 2170 2171 /* 2172 * This function is called from both if_init and ifflags_cb() 2173 * on NetBSD. 2174 */ 2175 2176 /* 2177 * Part 1: 2178 * Setup VLAN HW tagging 2179 */ 2180 ixv_setup_vlan_tagging(sc); 2181 2182 if (!VLAN_ATTACHED(ec)) 2183 return 0; 2184 2185 /* 2186 * Part 2: 2187 * Setup VLAN HW filter 2188 */ 2189 /* Cleanup shadow_vfta */ 2190 for (int i = 0; i < IXGBE_VFTA_SIZE; i++) 2191 sc->shadow_vfta[i] = 0; 2192 /* Generate shadow_vfta from ec_vids */ 2193 ETHER_LOCK(ec); 2194 SIMPLEQ_FOREACH(vlanidp, &ec->ec_vids, vid_list) { 2195 uint32_t idx; 2196 2197 idx = vlanidp->vid / 32; 2198 KASSERT(idx < IXGBE_VFTA_SIZE); 2199 sc->shadow_vfta[idx] |= (u32)1 << (vlanidp->vid % 32); 2200 } 2201 ETHER_UNLOCK(ec); 2202 2203 /* 2204 * A soft reset zero's out the VFTA, so 2205 * we need to repopulate it now. 2206 */ 2207 for (int i = 0; i < IXGBE_VFTA_SIZE; i++) { 2208 if (sc->shadow_vfta[i] == 0) 2209 continue; 2210 vfta = sc->shadow_vfta[i]; 2211 /* 2212 * Reconstruct the vlan id's 2213 * based on the bits set in each 2214 * of the array ints. 2215 */ 2216 for (int j = 0; j < 32; j++) { 2217 retry = 0; 2218 if ((vfta & ((u32)1 << j)) == 0) 2219 continue; 2220 vid = (i * 32) + j; 2221 2222 /* Call the shared code mailbox routine */ 2223 while ((rv = hw->mac.ops.set_vfta(hw, vid, 0, TRUE, 2224 FALSE)) != 0) { 2225 if (++retry > 5) { 2226 device_printf(sc->dev, 2227 "%s: max retry exceeded\n", 2228 __func__); 2229 break; 2230 } 2231 } 2232 if (rv != 0) { 2233 device_printf(sc->dev, 2234 "failed to set vlan %d\n", vid); 2235 error = EACCES; 2236 } 2237 } 2238 } 2239 return error; 2240 } /* ixv_setup_vlan_support */ 2241 2242 static int 2243 ixv_vlan_cb(struct ethercom *ec, uint16_t vid, bool set) 2244 { 2245 struct ifnet *ifp = &ec->ec_if; 2246 struct ixgbe_softc *sc = ifp->if_softc; 2247 int rv; 2248 2249 if (set) 2250 rv = ixv_register_vlan(sc, vid); 2251 else 2252 rv = ixv_unregister_vlan(sc, vid); 2253 2254 if (rv != 0) 2255 return rv; 2256 2257 /* 2258 * Control VLAN HW tagging when ec_nvlan is changed from 1 to 0 2259 * or 0 to 1. 2260 */ 2261 if ((set && (ec->ec_nvlans == 1)) || (!set && (ec->ec_nvlans == 0))) 2262 ixv_setup_vlan_tagging(sc); 2263 2264 return rv; 2265 } 2266 2267 /************************************************************************ 2268 * ixv_register_vlan 2269 * 2270 * Run via a vlan config EVENT, it enables us to use the 2271 * HW Filter table since we can get the vlan id. This just 2272 * creates the entry in the soft version of the VFTA, init 2273 * will repopulate the real table. 2274 ************************************************************************/ 2275 static int 2276 ixv_register_vlan(struct ixgbe_softc *sc, u16 vtag) 2277 { 2278 struct ixgbe_hw *hw = &sc->hw; 2279 u16 index, bit; 2280 int error; 2281 2282 if ((vtag == 0) || (vtag > 4095)) /* Invalid */ 2283 return EINVAL; 2284 IXGBE_CORE_LOCK(sc); 2285 index = (vtag >> 5) & 0x7F; 2286 bit = vtag & 0x1F; 2287 sc->shadow_vfta[index] |= ((u32)1 << bit); 2288 error = hw->mac.ops.set_vfta(hw, vtag, 0, true, false); 2289 IXGBE_CORE_UNLOCK(sc); 2290 2291 if (error != 0) { 2292 device_printf(sc->dev, "failed to register vlan %hu\n", vtag); 2293 error = EACCES; 2294 } 2295 return error; 2296 } /* ixv_register_vlan */ 2297 2298 /************************************************************************ 2299 * ixv_unregister_vlan 2300 * 2301 * Run via a vlan unconfig EVENT, remove our entry 2302 * in the soft vfta. 2303 ************************************************************************/ 2304 static int 2305 ixv_unregister_vlan(struct ixgbe_softc *sc, u16 vtag) 2306 { 2307 struct ixgbe_hw *hw = &sc->hw; 2308 u16 index, bit; 2309 int error; 2310 2311 if ((vtag == 0) || (vtag > 4095)) /* Invalid */ 2312 return EINVAL; 2313 2314 IXGBE_CORE_LOCK(sc); 2315 index = (vtag >> 5) & 0x7F; 2316 bit = vtag & 0x1F; 2317 sc->shadow_vfta[index] &= ~((u32)1 << bit); 2318 error = hw->mac.ops.set_vfta(hw, vtag, 0, false, false); 2319 IXGBE_CORE_UNLOCK(sc); 2320 2321 if (error != 0) { 2322 device_printf(sc->dev, "failed to unregister vlan %hu\n", 2323 vtag); 2324 error = EIO; 2325 } 2326 return error; 2327 } /* ixv_unregister_vlan */ 2328 2329 /************************************************************************ 2330 * ixv_enable_intr 2331 ************************************************************************/ 2332 static void 2333 ixv_enable_intr(struct ixgbe_softc *sc) 2334 { 2335 struct ixgbe_hw *hw = &sc->hw; 2336 struct ix_queue *que = sc->queues; 2337 u32 mask; 2338 int i; 2339 2340 /* For VTEIAC */ 2341 mask = (1 << sc->vector); 2342 for (i = 0; i < sc->num_queues; i++, que++) 2343 mask |= (1 << que->msix); 2344 IXGBE_WRITE_REG(hw, IXGBE_VTEIAC, mask); 2345 2346 /* For VTEIMS */ 2347 IXGBE_WRITE_REG(hw, IXGBE_VTEIMS, (1 << sc->vector)); 2348 que = sc->queues; 2349 for (i = 0; i < sc->num_queues; i++, que++) 2350 ixv_enable_queue(sc, que->msix); 2351 2352 IXGBE_WRITE_FLUSH(hw); 2353 } /* ixv_enable_intr */ 2354 2355 /************************************************************************ 2356 * ixv_disable_intr 2357 ************************************************************************/ 2358 static void 2359 ixv_disable_intr(struct ixgbe_softc *sc) 2360 { 2361 struct ix_queue *que = sc->queues; 2362 2363 IXGBE_WRITE_REG(&sc->hw, IXGBE_VTEIAC, 0); 2364 2365 /* disable interrupts other than queues */ 2366 IXGBE_WRITE_REG(&sc->hw, IXGBE_VTEIMC, sc->vector); 2367 2368 for (int i = 0; i < sc->num_queues; i++, que++) 2369 ixv_disable_queue(sc, que->msix); 2370 2371 IXGBE_WRITE_FLUSH(&sc->hw); 2372 } /* ixv_disable_intr */ 2373 2374 /************************************************************************ 2375 * ixv_set_ivar 2376 * 2377 * Setup the correct IVAR register for a particular MSI-X interrupt 2378 * - entry is the register array entry 2379 * - vector is the MSI-X vector for this queue 2380 * - type is RX/TX/MISC 2381 ************************************************************************/ 2382 static void 2383 ixv_set_ivar(struct ixgbe_softc *sc, u8 entry, u8 vector, s8 type) 2384 { 2385 struct ixgbe_hw *hw = &sc->hw; 2386 u32 ivar, index; 2387 2388 vector |= IXGBE_IVAR_ALLOC_VAL; 2389 2390 if (type == -1) { /* MISC IVAR */ 2391 ivar = IXGBE_READ_REG(hw, IXGBE_VTIVAR_MISC); 2392 ivar &= ~0xFF; 2393 ivar |= vector; 2394 IXGBE_WRITE_REG(hw, IXGBE_VTIVAR_MISC, ivar); 2395 } else { /* RX/TX IVARS */ 2396 index = (16 * (entry & 1)) + (8 * type); 2397 ivar = IXGBE_READ_REG(hw, IXGBE_VTIVAR(entry >> 1)); 2398 ivar &= ~(0xffUL << index); 2399 ivar |= ((u32)vector << index); 2400 IXGBE_WRITE_REG(hw, IXGBE_VTIVAR(entry >> 1), ivar); 2401 } 2402 } /* ixv_set_ivar */ 2403 2404 /************************************************************************ 2405 * ixv_configure_ivars 2406 ************************************************************************/ 2407 static void 2408 ixv_configure_ivars(struct ixgbe_softc *sc) 2409 { 2410 struct ix_queue *que = sc->queues; 2411 2412 /* XXX We should sync EITR value calculation with ixgbe.c? */ 2413 2414 for (int i = 0; i < sc->num_queues; i++, que++) { 2415 /* First the RX queue entry */ 2416 ixv_set_ivar(sc, i, que->msix, 0); 2417 /* ... and the TX */ 2418 ixv_set_ivar(sc, i, que->msix, 1); 2419 /* Set an initial value in EITR */ 2420 ixv_eitr_write(sc, que->msix, IXGBE_EITR_DEFAULT); 2421 } 2422 2423 /* For the mailbox interrupt */ 2424 ixv_set_ivar(sc, 1, sc->vector, -1); 2425 } /* ixv_configure_ivars */ 2426 2427 2428 /************************************************************************ 2429 * ixv_init_stats 2430 * 2431 * The VF stats registers never have a truly virgin 2432 * starting point, so this routine save initial vaules to 2433 * last_<REGNAME>. 2434 ************************************************************************/ 2435 static void 2436 ixv_init_stats(struct ixgbe_softc *sc) 2437 { 2438 struct ixgbe_hw *hw = &sc->hw; 2439 2440 sc->stats.vf.last_vfgprc = IXGBE_READ_REG(hw, IXGBE_VFGPRC); 2441 sc->stats.vf.last_vfgorc = IXGBE_READ_REG(hw, IXGBE_VFGORC_LSB); 2442 sc->stats.vf.last_vfgorc |= 2443 (((u64)(IXGBE_READ_REG(hw, IXGBE_VFGORC_MSB))) << 32); 2444 2445 sc->stats.vf.last_vfgptc = IXGBE_READ_REG(hw, IXGBE_VFGPTC); 2446 sc->stats.vf.last_vfgotc = IXGBE_READ_REG(hw, IXGBE_VFGOTC_LSB); 2447 sc->stats.vf.last_vfgotc |= 2448 (((u64)(IXGBE_READ_REG(hw, IXGBE_VFGOTC_MSB))) << 32); 2449 2450 sc->stats.vf.last_vfmprc = IXGBE_READ_REG(hw, IXGBE_VFMPRC); 2451 } /* ixv_init_stats */ 2452 2453 #define UPDATE_STAT_32(reg, last, count) \ 2454 { \ 2455 u32 current = IXGBE_READ_REG(hw, (reg)); \ 2456 IXGBE_EVC_ADD(&count, current - (last)); \ 2457 (last) = current; \ 2458 } 2459 2460 #define UPDATE_STAT_36(lsb, msb, last, count) \ 2461 { \ 2462 u64 cur_lsb = IXGBE_READ_REG(hw, (lsb)); \ 2463 u64 cur_msb = IXGBE_READ_REG(hw, (msb)); \ 2464 u64 current = ((cur_msb << 32) | cur_lsb); \ 2465 if (current < (last)) \ 2466 IXGBE_EVC_ADD(&count, current + __BIT(36) - (last)); \ 2467 else \ 2468 IXGBE_EVC_ADD(&count, current - (last)); \ 2469 (last) = current; \ 2470 } 2471 2472 /************************************************************************ 2473 * ixv_update_stats - Update the board statistics counters. 2474 ************************************************************************/ 2475 void 2476 ixv_update_stats(struct ixgbe_softc *sc) 2477 { 2478 struct ixgbe_hw *hw = &sc->hw; 2479 struct ixgbevf_hw_stats *stats = &sc->stats.vf; 2480 2481 UPDATE_STAT_32(IXGBE_VFGPRC, stats->last_vfgprc, stats->vfgprc); 2482 UPDATE_STAT_32(IXGBE_VFGPTC, stats->last_vfgptc, stats->vfgptc); 2483 UPDATE_STAT_36(IXGBE_VFGORC_LSB, IXGBE_VFGORC_MSB, stats->last_vfgorc, 2484 stats->vfgorc); 2485 UPDATE_STAT_36(IXGBE_VFGOTC_LSB, IXGBE_VFGOTC_MSB, stats->last_vfgotc, 2486 stats->vfgotc); 2487 UPDATE_STAT_32(IXGBE_VFMPRC, stats->last_vfmprc, stats->vfmprc); 2488 2489 /* VF doesn't count errors by hardware */ 2490 2491 } /* ixv_update_stats */ 2492 2493 /************************************************************************ 2494 * ixv_sysctl_interrupt_rate_handler 2495 ************************************************************************/ 2496 static int 2497 ixv_sysctl_interrupt_rate_handler(SYSCTLFN_ARGS) 2498 { 2499 struct sysctlnode node = *rnode; 2500 struct ix_queue *que = (struct ix_queue *)node.sysctl_data; 2501 struct ixgbe_softc *sc = que->sc; 2502 uint32_t reg, usec, rate; 2503 int error; 2504 2505 if (que == NULL) 2506 return 0; 2507 reg = IXGBE_READ_REG(&que->sc->hw, IXGBE_VTEITR(que->msix)); 2508 usec = ((reg & 0x0FF8) >> 3); 2509 if (usec > 0) 2510 rate = 500000 / usec; 2511 else 2512 rate = 0; 2513 node.sysctl_data = &rate; 2514 error = sysctl_lookup(SYSCTLFN_CALL(&node)); 2515 if (error || newp == NULL) 2516 return error; 2517 reg &= ~0xfff; /* default, no limitation */ 2518 if (rate > 0 && rate < 500000) { 2519 if (rate < 1000) 2520 rate = 1000; 2521 reg |= ((4000000 / rate) & 0xff8); 2522 /* 2523 * When RSC is used, ITR interval must be larger than 2524 * RSC_DELAY. Currently, we use 2us for RSC_DELAY. 2525 * The minimum value is always greater than 2us on 100M 2526 * (and 10M?(not documented)), but it's not on 1G and higher. 2527 */ 2528 if ((sc->link_speed != IXGBE_LINK_SPEED_100_FULL) 2529 && (sc->link_speed != IXGBE_LINK_SPEED_10_FULL)) { 2530 if ((sc->num_queues > 1) 2531 && (reg < IXGBE_MIN_RSC_EITR_10G1G)) 2532 return EINVAL; 2533 } 2534 sc->max_interrupt_rate = rate; 2535 } else 2536 sc->max_interrupt_rate = 0; 2537 ixv_eitr_write(sc, que->msix, reg); 2538 2539 return (0); 2540 } /* ixv_sysctl_interrupt_rate_handler */ 2541 2542 const struct sysctlnode * 2543 ixv_sysctl_instance(struct ixgbe_softc *sc) 2544 { 2545 const char *dvname; 2546 struct sysctllog **log; 2547 int rc; 2548 const struct sysctlnode *rnode; 2549 2550 log = &sc->sysctllog; 2551 dvname = device_xname(sc->dev); 2552 2553 if ((rc = sysctl_createv(log, 0, NULL, &rnode, 2554 0, CTLTYPE_NODE, dvname, 2555 SYSCTL_DESCR("ixv information and settings"), 2556 NULL, 0, NULL, 0, CTL_HW, CTL_CREATE, CTL_EOL)) != 0) 2557 goto err; 2558 2559 return rnode; 2560 err: 2561 device_printf(sc->dev, 2562 "%s: sysctl_createv failed, rc = %d\n", __func__, rc); 2563 return NULL; 2564 } 2565 2566 static void 2567 ixv_add_device_sysctls(struct ixgbe_softc *sc) 2568 { 2569 struct sysctllog **log; 2570 const struct sysctlnode *rnode, *cnode; 2571 device_t dev; 2572 2573 dev = sc->dev; 2574 log = &sc->sysctllog; 2575 2576 if ((rnode = ixv_sysctl_instance(sc)) == NULL) { 2577 aprint_error_dev(dev, "could not create sysctl root\n"); 2578 return; 2579 } 2580 2581 if (sysctl_createv(log, 0, &rnode, &cnode, 2582 CTLFLAG_READWRITE, CTLTYPE_INT, "debug", 2583 SYSCTL_DESCR("Debug Info"), 2584 ixv_sysctl_debug, 0, (void *)sc, 0, CTL_CREATE, CTL_EOL) != 0) 2585 aprint_error_dev(dev, "could not create sysctl\n"); 2586 2587 if (sysctl_createv(log, 0, &rnode, &cnode, 2588 CTLFLAG_READWRITE, CTLTYPE_INT, 2589 "rx_copy_len", SYSCTL_DESCR("RX Copy Length"), 2590 ixv_sysctl_rx_copy_len, 0, 2591 (void *)sc, 0, CTL_CREATE, CTL_EOL) != 0) 2592 aprint_error_dev(dev, "could not create sysctl\n"); 2593 2594 if (sysctl_createv(log, 0, &rnode, &cnode, 2595 CTLFLAG_READONLY, CTLTYPE_INT, 2596 "num_tx_desc", SYSCTL_DESCR("Number of TX descriptors"), 2597 NULL, 0, &sc->num_tx_desc, 0, CTL_CREATE, CTL_EOL) != 0) 2598 aprint_error_dev(dev, "could not create sysctl\n"); 2599 2600 if (sysctl_createv(log, 0, &rnode, &cnode, 2601 CTLFLAG_READONLY, CTLTYPE_INT, 2602 "num_rx_desc", SYSCTL_DESCR("Number of RX descriptors"), 2603 NULL, 0, &sc->num_rx_desc, 0, CTL_CREATE, CTL_EOL) != 0) 2604 aprint_error_dev(dev, "could not create sysctl\n"); 2605 2606 if (sysctl_createv(log, 0, &rnode, &cnode, 2607 CTLFLAG_READWRITE, CTLTYPE_INT, "rx_process_limit", 2608 SYSCTL_DESCR("max number of RX packets to process"), 2609 ixv_sysctl_rx_process_limit, 0, (void *)sc, 0, CTL_CREATE, 2610 CTL_EOL) != 0) 2611 aprint_error_dev(dev, "could not create sysctl\n"); 2612 2613 if (sysctl_createv(log, 0, &rnode, &cnode, 2614 CTLFLAG_READWRITE, CTLTYPE_INT, "tx_process_limit", 2615 SYSCTL_DESCR("max number of TX packets to process"), 2616 ixv_sysctl_tx_process_limit, 0, (void *)sc, 0, CTL_CREATE, 2617 CTL_EOL) != 0) 2618 aprint_error_dev(dev, "could not create sysctl\n"); 2619 2620 if (sysctl_createv(log, 0, &rnode, &cnode, 2621 CTLFLAG_READWRITE, CTLTYPE_BOOL, "enable_aim", 2622 SYSCTL_DESCR("Interrupt Moderation"), 2623 NULL, 0, &sc->enable_aim, 0, CTL_CREATE, CTL_EOL) != 0) 2624 aprint_error_dev(dev, "could not create sysctl\n"); 2625 2626 if (sysctl_createv(log, 0, &rnode, &cnode, 2627 CTLFLAG_READWRITE, CTLTYPE_BOOL, "txrx_workqueue", 2628 SYSCTL_DESCR("Use workqueue for packet processing"), 2629 NULL, 0, &sc->txrx_use_workqueue, 0, CTL_CREATE, CTL_EOL) 2630 != 0) 2631 aprint_error_dev(dev, "could not create sysctl\n"); 2632 } 2633 2634 /************************************************************************ 2635 * ixv_add_stats_sysctls - Add statistic sysctls for the VF. 2636 ************************************************************************/ 2637 static void 2638 ixv_add_stats_sysctls(struct ixgbe_softc *sc) 2639 { 2640 device_t dev = sc->dev; 2641 struct tx_ring *txr = sc->tx_rings; 2642 struct rx_ring *rxr = sc->rx_rings; 2643 struct ixgbevf_hw_stats *stats = &sc->stats.vf; 2644 struct ixgbe_hw *hw = &sc->hw; 2645 const struct sysctlnode *rnode, *cnode; 2646 struct sysctllog **log = &sc->sysctllog; 2647 const char *xname = device_xname(dev); 2648 2649 /* Driver Statistics */ 2650 evcnt_attach_dynamic(&sc->efbig_tx_dma_setup, EVCNT_TYPE_MISC, 2651 NULL, xname, "Driver tx dma soft fail EFBIG"); 2652 evcnt_attach_dynamic(&sc->mbuf_defrag_failed, EVCNT_TYPE_MISC, 2653 NULL, xname, "m_defrag() failed"); 2654 evcnt_attach_dynamic(&sc->efbig2_tx_dma_setup, EVCNT_TYPE_MISC, 2655 NULL, xname, "Driver tx dma hard fail EFBIG"); 2656 evcnt_attach_dynamic(&sc->einval_tx_dma_setup, EVCNT_TYPE_MISC, 2657 NULL, xname, "Driver tx dma hard fail EINVAL"); 2658 evcnt_attach_dynamic(&sc->other_tx_dma_setup, EVCNT_TYPE_MISC, 2659 NULL, xname, "Driver tx dma hard fail other"); 2660 evcnt_attach_dynamic(&sc->eagain_tx_dma_setup, EVCNT_TYPE_MISC, 2661 NULL, xname, "Driver tx dma soft fail EAGAIN"); 2662 evcnt_attach_dynamic(&sc->enomem_tx_dma_setup, EVCNT_TYPE_MISC, 2663 NULL, xname, "Driver tx dma soft fail ENOMEM"); 2664 evcnt_attach_dynamic(&sc->watchdog_events, EVCNT_TYPE_MISC, 2665 NULL, xname, "Watchdog timeouts"); 2666 evcnt_attach_dynamic(&sc->tso_err, EVCNT_TYPE_MISC, 2667 NULL, xname, "TSO errors"); 2668 evcnt_attach_dynamic(&sc->admin_irqev, EVCNT_TYPE_INTR, 2669 NULL, xname, "Admin MSI-X IRQ Handled"); 2670 evcnt_attach_dynamic(&sc->link_workev, EVCNT_TYPE_INTR, 2671 NULL, xname, "Admin event"); 2672 2673 for (int i = 0; i < sc->num_queues; i++, rxr++, txr++) { 2674 #ifdef LRO 2675 struct lro_ctrl *lro = &rxr->lro; 2676 #endif 2677 2678 snprintf(sc->queues[i].evnamebuf, 2679 sizeof(sc->queues[i].evnamebuf), "%s q%d", xname, i); 2680 snprintf(sc->queues[i].namebuf, 2681 sizeof(sc->queues[i].namebuf), "q%d", i); 2682 2683 if ((rnode = ixv_sysctl_instance(sc)) == NULL) { 2684 aprint_error_dev(dev, 2685 "could not create sysctl root\n"); 2686 break; 2687 } 2688 2689 if (sysctl_createv(log, 0, &rnode, &rnode, 2690 0, CTLTYPE_NODE, 2691 sc->queues[i].namebuf, SYSCTL_DESCR("Queue Name"), 2692 NULL, 0, NULL, 0, CTL_CREATE, CTL_EOL) != 0) 2693 break; 2694 2695 if (sysctl_createv(log, 0, &rnode, &cnode, 2696 CTLFLAG_READWRITE, CTLTYPE_INT, 2697 "interrupt_rate", SYSCTL_DESCR("Interrupt Rate"), 2698 ixv_sysctl_interrupt_rate_handler, 0, 2699 (void *)&sc->queues[i], 0, CTL_CREATE, CTL_EOL) != 0) 2700 break; 2701 2702 if (sysctl_createv(log, 0, &rnode, &cnode, 2703 CTLFLAG_READONLY, CTLTYPE_INT, 2704 "txd_head", SYSCTL_DESCR("Transmit Descriptor Head"), 2705 ixv_sysctl_tdh_handler, 0, (void *)txr, 2706 0, CTL_CREATE, CTL_EOL) != 0) 2707 break; 2708 2709 if (sysctl_createv(log, 0, &rnode, &cnode, 2710 CTLFLAG_READONLY, CTLTYPE_INT, 2711 "txd_tail", SYSCTL_DESCR("Transmit Descriptor Tail"), 2712 ixv_sysctl_tdt_handler, 0, (void *)txr, 2713 0, CTL_CREATE, CTL_EOL) != 0) 2714 break; 2715 2716 if (sysctl_createv(log, 0, &rnode, &cnode, 2717 CTLFLAG_READONLY, CTLTYPE_INT, "rxd_nxck", 2718 SYSCTL_DESCR("Receive Descriptor next to check"), 2719 ixv_sysctl_next_to_check_handler, 0, (void *)rxr, 0, 2720 CTL_CREATE, CTL_EOL) != 0) 2721 break; 2722 2723 if (sysctl_createv(log, 0, &rnode, &cnode, 2724 CTLFLAG_READONLY, CTLTYPE_INT, "rxd_nxrf", 2725 SYSCTL_DESCR("Receive Descriptor next to refresh"), 2726 ixv_sysctl_next_to_refresh_handler, 0, (void *)rxr, 0, 2727 CTL_CREATE, CTL_EOL) != 0) 2728 break; 2729 2730 if (sysctl_createv(log, 0, &rnode, &cnode, 2731 CTLFLAG_READONLY, CTLTYPE_INT, "rxd_head", 2732 SYSCTL_DESCR("Receive Descriptor Head"), 2733 ixv_sysctl_rdh_handler, 0, (void *)rxr, 0, 2734 CTL_CREATE, CTL_EOL) != 0) 2735 break; 2736 2737 if (sysctl_createv(log, 0, &rnode, &cnode, 2738 CTLFLAG_READONLY, CTLTYPE_INT, "rxd_tail", 2739 SYSCTL_DESCR("Receive Descriptor Tail"), 2740 ixv_sysctl_rdt_handler, 0, (void *)rxr, 0, 2741 CTL_CREATE, CTL_EOL) != 0) 2742 break; 2743 2744 evcnt_attach_dynamic(&sc->queues[i].irqs, EVCNT_TYPE_INTR, 2745 NULL, sc->queues[i].evnamebuf, "IRQs on queue"); 2746 evcnt_attach_dynamic(&sc->queues[i].handleq, 2747 EVCNT_TYPE_MISC, NULL, sc->queues[i].evnamebuf, 2748 "Handled queue in softint"); 2749 evcnt_attach_dynamic(&sc->queues[i].req, EVCNT_TYPE_MISC, 2750 NULL, sc->queues[i].evnamebuf, "Requeued in softint"); 2751 evcnt_attach_dynamic(&txr->total_packets, EVCNT_TYPE_MISC, 2752 NULL, sc->queues[i].evnamebuf, 2753 "Queue Packets Transmitted"); 2754 #ifndef IXGBE_LEGACY_TX 2755 evcnt_attach_dynamic(&txr->pcq_drops, EVCNT_TYPE_MISC, 2756 NULL, sc->queues[i].evnamebuf, 2757 "Packets dropped in pcq"); 2758 #endif 2759 evcnt_attach_dynamic(&txr->no_desc_avail, EVCNT_TYPE_MISC, 2760 NULL, sc->queues[i].evnamebuf, 2761 "TX Queue No Descriptor Available"); 2762 evcnt_attach_dynamic(&txr->tso_tx, EVCNT_TYPE_MISC, 2763 NULL, sc->queues[i].evnamebuf, "TSO"); 2764 2765 evcnt_attach_dynamic(&rxr->rx_bytes, EVCNT_TYPE_MISC, 2766 NULL, sc->queues[i].evnamebuf, 2767 "Queue Bytes Received"); 2768 evcnt_attach_dynamic(&rxr->rx_packets, EVCNT_TYPE_MISC, 2769 NULL, sc->queues[i].evnamebuf, 2770 "Queue Packets Received"); 2771 evcnt_attach_dynamic(&rxr->no_mbuf, EVCNT_TYPE_MISC, 2772 NULL, sc->queues[i].evnamebuf, "Rx no mbuf"); 2773 evcnt_attach_dynamic(&rxr->rx_discarded, EVCNT_TYPE_MISC, 2774 NULL, sc->queues[i].evnamebuf, "Rx discarded"); 2775 evcnt_attach_dynamic(&rxr->rx_copies, EVCNT_TYPE_MISC, 2776 NULL, sc->queues[i].evnamebuf, "Copied RX Frames"); 2777 #ifdef LRO 2778 SYSCTL_ADD_INT(ctx, queue_list, OID_AUTO, "lro_queued", 2779 CTLFLAG_RD, &lro->lro_queued, 0, 2780 "LRO Queued"); 2781 SYSCTL_ADD_INT(ctx, queue_list, OID_AUTO, "lro_flushed", 2782 CTLFLAG_RD, &lro->lro_flushed, 0, 2783 "LRO Flushed"); 2784 #endif /* LRO */ 2785 } 2786 2787 /* MAC stats get their own sub node */ 2788 2789 snprintf(stats->namebuf, 2790 sizeof(stats->namebuf), "%s MAC Statistics", xname); 2791 2792 evcnt_attach_dynamic(&stats->ipcs, EVCNT_TYPE_MISC, NULL, 2793 stats->namebuf, "rx csum offload - IP"); 2794 evcnt_attach_dynamic(&stats->l4cs, EVCNT_TYPE_MISC, NULL, 2795 stats->namebuf, "rx csum offload - L4"); 2796 evcnt_attach_dynamic(&stats->ipcs_bad, EVCNT_TYPE_MISC, NULL, 2797 stats->namebuf, "rx csum offload - IP bad"); 2798 evcnt_attach_dynamic(&stats->l4cs_bad, EVCNT_TYPE_MISC, NULL, 2799 stats->namebuf, "rx csum offload - L4 bad"); 2800 2801 /* Packet Reception Stats */ 2802 evcnt_attach_dynamic(&stats->vfgprc, EVCNT_TYPE_MISC, NULL, 2803 xname, "Good Packets Received"); 2804 evcnt_attach_dynamic(&stats->vfgorc, EVCNT_TYPE_MISC, NULL, 2805 xname, "Good Octets Received"); 2806 evcnt_attach_dynamic(&stats->vfmprc, EVCNT_TYPE_MISC, NULL, 2807 xname, "Multicast Packets Received"); 2808 evcnt_attach_dynamic(&stats->vfgptc, EVCNT_TYPE_MISC, NULL, 2809 xname, "Good Packets Transmitted"); 2810 evcnt_attach_dynamic(&stats->vfgotc, EVCNT_TYPE_MISC, NULL, 2811 xname, "Good Octets Transmitted"); 2812 2813 /* Mailbox Stats */ 2814 evcnt_attach_dynamic(&hw->mbx.stats.msgs_tx, EVCNT_TYPE_MISC, NULL, 2815 xname, "message TXs"); 2816 evcnt_attach_dynamic(&hw->mbx.stats.msgs_rx, EVCNT_TYPE_MISC, NULL, 2817 xname, "message RXs"); 2818 evcnt_attach_dynamic(&hw->mbx.stats.acks, EVCNT_TYPE_MISC, NULL, 2819 xname, "ACKs"); 2820 evcnt_attach_dynamic(&hw->mbx.stats.reqs, EVCNT_TYPE_MISC, NULL, 2821 xname, "REQs"); 2822 evcnt_attach_dynamic(&hw->mbx.stats.rsts, EVCNT_TYPE_MISC, NULL, 2823 xname, "RSTs"); 2824 2825 } /* ixv_add_stats_sysctls */ 2826 2827 static void 2828 ixv_clear_evcnt(struct ixgbe_softc *sc) 2829 { 2830 struct tx_ring *txr = sc->tx_rings; 2831 struct rx_ring *rxr = sc->rx_rings; 2832 struct ixgbevf_hw_stats *stats = &sc->stats.vf; 2833 struct ixgbe_hw *hw = &sc->hw; 2834 int i; 2835 2836 /* Driver Statistics */ 2837 IXGBE_EVC_STORE(&sc->efbig_tx_dma_setup, 0); 2838 IXGBE_EVC_STORE(&sc->mbuf_defrag_failed, 0); 2839 IXGBE_EVC_STORE(&sc->efbig2_tx_dma_setup, 0); 2840 IXGBE_EVC_STORE(&sc->einval_tx_dma_setup, 0); 2841 IXGBE_EVC_STORE(&sc->other_tx_dma_setup, 0); 2842 IXGBE_EVC_STORE(&sc->eagain_tx_dma_setup, 0); 2843 IXGBE_EVC_STORE(&sc->enomem_tx_dma_setup, 0); 2844 IXGBE_EVC_STORE(&sc->watchdog_events, 0); 2845 IXGBE_EVC_STORE(&sc->tso_err, 0); 2846 IXGBE_EVC_STORE(&sc->admin_irqev, 0); 2847 IXGBE_EVC_STORE(&sc->link_workev, 0); 2848 2849 for (i = 0; i < sc->num_queues; i++, rxr++, txr++) { 2850 IXGBE_EVC_STORE(&sc->queues[i].irqs, 0); 2851 IXGBE_EVC_STORE(&sc->queues[i].handleq, 0); 2852 IXGBE_EVC_STORE(&sc->queues[i].req, 0); 2853 IXGBE_EVC_STORE(&txr->total_packets, 0); 2854 #ifndef IXGBE_LEGACY_TX 2855 IXGBE_EVC_STORE(&txr->pcq_drops, 0); 2856 #endif 2857 IXGBE_EVC_STORE(&txr->no_desc_avail, 0); 2858 IXGBE_EVC_STORE(&txr->tso_tx, 0); 2859 txr->q_efbig_tx_dma_setup = 0; 2860 txr->q_mbuf_defrag_failed = 0; 2861 txr->q_efbig2_tx_dma_setup = 0; 2862 txr->q_einval_tx_dma_setup = 0; 2863 txr->q_other_tx_dma_setup = 0; 2864 txr->q_eagain_tx_dma_setup = 0; 2865 txr->q_enomem_tx_dma_setup = 0; 2866 txr->q_tso_err = 0; 2867 2868 IXGBE_EVC_STORE(&rxr->rx_packets, 0); 2869 IXGBE_EVC_STORE(&rxr->rx_bytes, 0); 2870 IXGBE_EVC_STORE(&rxr->rx_copies, 0); 2871 IXGBE_EVC_STORE(&rxr->no_mbuf, 0); 2872 IXGBE_EVC_STORE(&rxr->rx_discarded, 0); 2873 } 2874 2875 /* MAC stats get their own sub node */ 2876 2877 IXGBE_EVC_STORE(&stats->ipcs, 0); 2878 IXGBE_EVC_STORE(&stats->l4cs, 0); 2879 IXGBE_EVC_STORE(&stats->ipcs_bad, 0); 2880 IXGBE_EVC_STORE(&stats->l4cs_bad, 0); 2881 2882 /* 2883 * Packet Reception Stats. 2884 * Call ixv_init_stats() to save last VF counters' values. 2885 */ 2886 ixv_init_stats(sc); 2887 IXGBE_EVC_STORE(&stats->vfgprc, 0); 2888 IXGBE_EVC_STORE(&stats->vfgorc, 0); 2889 IXGBE_EVC_STORE(&stats->vfmprc, 0); 2890 IXGBE_EVC_STORE(&stats->vfgptc, 0); 2891 IXGBE_EVC_STORE(&stats->vfgotc, 0); 2892 2893 /* Mailbox Stats */ 2894 IXGBE_EVC_STORE(&hw->mbx.stats.msgs_tx, 0); 2895 IXGBE_EVC_STORE(&hw->mbx.stats.msgs_rx, 0); 2896 IXGBE_EVC_STORE(&hw->mbx.stats.acks, 0); 2897 IXGBE_EVC_STORE(&hw->mbx.stats.reqs, 0); 2898 IXGBE_EVC_STORE(&hw->mbx.stats.rsts, 0); 2899 2900 } /* ixv_clear_evcnt */ 2901 2902 #define PRINTQS(sc, regname) \ 2903 do { \ 2904 struct ixgbe_hw *_hw = &(sc)->hw; \ 2905 int _i; \ 2906 \ 2907 printf("%s: %s", device_xname((sc)->dev), #regname); \ 2908 for (_i = 0; _i < (sc)->num_queues; _i++) { \ 2909 printf((_i == 0) ? "\t" : " "); \ 2910 printf("%08x", IXGBE_READ_REG(_hw, \ 2911 IXGBE_##regname(_i))); \ 2912 } \ 2913 printf("\n"); \ 2914 } while (0) 2915 2916 /************************************************************************ 2917 * ixv_print_debug_info 2918 * 2919 * Provides a way to take a look at important statistics 2920 * maintained by the driver and hardware. 2921 ************************************************************************/ 2922 static void 2923 ixv_print_debug_info(struct ixgbe_softc *sc) 2924 { 2925 device_t dev = sc->dev; 2926 struct ixgbe_hw *hw = &sc->hw; 2927 int i; 2928 2929 device_printf(dev, "queue:"); 2930 for (i = 0; i < sc->num_queues; i++) { 2931 printf((i == 0) ? "\t" : " "); 2932 printf("%8d", i); 2933 } 2934 printf("\n"); 2935 PRINTQS(sc, VFRDBAL); 2936 PRINTQS(sc, VFRDBAH); 2937 PRINTQS(sc, VFRDLEN); 2938 PRINTQS(sc, VFSRRCTL); 2939 PRINTQS(sc, VFRDH); 2940 PRINTQS(sc, VFRDT); 2941 PRINTQS(sc, VFRXDCTL); 2942 2943 device_printf(dev, "EIMS:\t%08x\n", IXGBE_READ_REG(hw, IXGBE_VTEIMS)); 2944 device_printf(dev, "EIAM:\t%08x\n", IXGBE_READ_REG(hw, IXGBE_VTEIAM)); 2945 device_printf(dev, "EIAC:\t%08x\n", IXGBE_READ_REG(hw, IXGBE_VTEIAC)); 2946 } /* ixv_print_debug_info */ 2947 2948 /************************************************************************ 2949 * ixv_sysctl_debug 2950 ************************************************************************/ 2951 static int 2952 ixv_sysctl_debug(SYSCTLFN_ARGS) 2953 { 2954 struct sysctlnode node = *rnode; 2955 struct ixgbe_softc *sc = (struct ixgbe_softc *)node.sysctl_data; 2956 int error, result = 0; 2957 2958 node.sysctl_data = &result; 2959 error = sysctl_lookup(SYSCTLFN_CALL(&node)); 2960 2961 if (error || newp == NULL) 2962 return error; 2963 2964 if (result == 1) 2965 ixv_print_debug_info(sc); 2966 2967 return 0; 2968 } /* ixv_sysctl_debug */ 2969 2970 /************************************************************************ 2971 * ixv_sysctl_rx_copy_len 2972 ************************************************************************/ 2973 static int 2974 ixv_sysctl_rx_copy_len(SYSCTLFN_ARGS) 2975 { 2976 struct sysctlnode node = *rnode; 2977 struct ixgbe_softc *sc = (struct ixgbe_softc *)node.sysctl_data; 2978 int error; 2979 int result = sc->rx_copy_len; 2980 2981 node.sysctl_data = &result; 2982 error = sysctl_lookup(SYSCTLFN_CALL(&node)); 2983 2984 if (error || newp == NULL) 2985 return error; 2986 2987 if ((result < 0) || (result > IXGBE_RX_COPY_LEN_MAX)) 2988 return EINVAL; 2989 2990 sc->rx_copy_len = result; 2991 2992 return 0; 2993 } /* ixv_sysctl_rx_copy_len */ 2994 2995 /************************************************************************ 2996 * ixv_sysctl_tx_process_limit 2997 ************************************************************************/ 2998 static int 2999 ixv_sysctl_tx_process_limit(SYSCTLFN_ARGS) 3000 { 3001 struct sysctlnode node = *rnode; 3002 struct ixgbe_softc *sc = (struct ixgbe_softc *)node.sysctl_data; 3003 int error; 3004 int result = sc->tx_process_limit; 3005 3006 node.sysctl_data = &result; 3007 error = sysctl_lookup(SYSCTLFN_CALL(&node)); 3008 3009 if (error || newp == NULL) 3010 return error; 3011 3012 if ((result <= 0) || (result > sc->num_tx_desc)) 3013 return EINVAL; 3014 3015 sc->tx_process_limit = result; 3016 3017 return 0; 3018 } /* ixv_sysctl_tx_process_limit */ 3019 3020 /************************************************************************ 3021 * ixv_sysctl_rx_process_limit 3022 ************************************************************************/ 3023 static int 3024 ixv_sysctl_rx_process_limit(SYSCTLFN_ARGS) 3025 { 3026 struct sysctlnode node = *rnode; 3027 struct ixgbe_softc *sc = (struct ixgbe_softc *)node.sysctl_data; 3028 int error; 3029 int result = sc->rx_process_limit; 3030 3031 node.sysctl_data = &result; 3032 error = sysctl_lookup(SYSCTLFN_CALL(&node)); 3033 3034 if (error || newp == NULL) 3035 return error; 3036 3037 if ((result <= 0) || (result > sc->num_rx_desc)) 3038 return EINVAL; 3039 3040 sc->rx_process_limit = result; 3041 3042 return 0; 3043 } /* ixv_sysctl_rx_process_limit */ 3044 3045 /************************************************************************ 3046 * ixv_init_device_features 3047 ************************************************************************/ 3048 static void 3049 ixv_init_device_features(struct ixgbe_softc *sc) 3050 { 3051 sc->feat_cap = IXGBE_FEATURE_NETMAP 3052 | IXGBE_FEATURE_VF 3053 | IXGBE_FEATURE_RSS 3054 | IXGBE_FEATURE_LEGACY_TX; 3055 3056 /* A tad short on feature flags for VFs, atm. */ 3057 switch (sc->hw.mac.type) { 3058 case ixgbe_mac_82599_vf: 3059 break; 3060 case ixgbe_mac_X540_vf: 3061 break; 3062 case ixgbe_mac_X550_vf: 3063 case ixgbe_mac_X550EM_x_vf: 3064 case ixgbe_mac_X550EM_a_vf: 3065 sc->feat_cap |= IXGBE_FEATURE_NEEDS_CTXD; 3066 break; 3067 default: 3068 break; 3069 } 3070 3071 /* Enabled by default... */ 3072 /* Is a virtual function (VF) */ 3073 if (sc->feat_cap & IXGBE_FEATURE_VF) 3074 sc->feat_en |= IXGBE_FEATURE_VF; 3075 /* Netmap */ 3076 if (sc->feat_cap & IXGBE_FEATURE_NETMAP) 3077 sc->feat_en |= IXGBE_FEATURE_NETMAP; 3078 /* Receive-Side Scaling (RSS) */ 3079 if (sc->feat_cap & IXGBE_FEATURE_RSS) 3080 sc->feat_en |= IXGBE_FEATURE_RSS; 3081 /* Needs advanced context descriptor regardless of offloads req'd */ 3082 if (sc->feat_cap & IXGBE_FEATURE_NEEDS_CTXD) 3083 sc->feat_en |= IXGBE_FEATURE_NEEDS_CTXD; 3084 3085 /* Enabled via sysctl... */ 3086 /* Legacy (single queue) transmit */ 3087 if ((sc->feat_cap & IXGBE_FEATURE_LEGACY_TX) && 3088 ixv_enable_legacy_tx) 3089 sc->feat_en |= IXGBE_FEATURE_LEGACY_TX; 3090 } /* ixv_init_device_features */ 3091 3092 /************************************************************************ 3093 * ixv_shutdown - Shutdown entry point 3094 ************************************************************************/ 3095 #if 0 /* XXX NetBSD ought to register something like this through pmf(9) */ 3096 static int 3097 ixv_shutdown(device_t dev) 3098 { 3099 struct ixgbe_softc *sc = device_private(dev); 3100 IXGBE_CORE_LOCK(sc); 3101 ixv_stop_locked(sc); 3102 IXGBE_CORE_UNLOCK(sc); 3103 3104 return (0); 3105 } /* ixv_shutdown */ 3106 #endif 3107 3108 static int 3109 ixv_ifflags_cb(struct ethercom *ec) 3110 { 3111 struct ifnet *ifp = &ec->ec_if; 3112 struct ixgbe_softc *sc = ifp->if_softc; 3113 u_short saved_flags; 3114 u_short change; 3115 int rv = 0; 3116 3117 IXGBE_CORE_LOCK(sc); 3118 3119 saved_flags = sc->if_flags; 3120 change = ifp->if_flags ^ sc->if_flags; 3121 if (change != 0) 3122 sc->if_flags = ifp->if_flags; 3123 3124 if ((change & ~(IFF_CANTCHANGE | IFF_DEBUG)) != 0) { 3125 rv = ENETRESET; 3126 goto out; 3127 } else if ((change & IFF_PROMISC) != 0) { 3128 rv = ixv_set_rxfilter(sc); 3129 if (rv != 0) { 3130 /* Restore previous */ 3131 sc->if_flags = saved_flags; 3132 goto out; 3133 } 3134 } 3135 3136 /* Check for ec_capenable. */ 3137 change = ec->ec_capenable ^ sc->ec_capenable; 3138 sc->ec_capenable = ec->ec_capenable; 3139 if ((change & ~(ETHERCAP_VLAN_MTU | ETHERCAP_VLAN_HWTAGGING 3140 | ETHERCAP_VLAN_HWFILTER)) != 0) { 3141 rv = ENETRESET; 3142 goto out; 3143 } 3144 3145 /* 3146 * Special handling is not required for ETHERCAP_VLAN_MTU. 3147 * PF's MAXFRS(MHADD) does not include the 4bytes of the VLAN header. 3148 */ 3149 3150 /* Set up VLAN support and filter */ 3151 if ((change & (ETHERCAP_VLAN_HWTAGGING | ETHERCAP_VLAN_HWFILTER)) != 0) 3152 rv = ixv_setup_vlan_support(sc); 3153 3154 out: 3155 IXGBE_CORE_UNLOCK(sc); 3156 3157 return rv; 3158 } 3159 3160 3161 /************************************************************************ 3162 * ixv_ioctl - Ioctl entry point 3163 * 3164 * Called when the user wants to configure the interface. 3165 * 3166 * return 0 on success, positive on failure 3167 ************************************************************************/ 3168 static int 3169 ixv_ioctl(struct ifnet *ifp, u_long command, void *data) 3170 { 3171 struct ixgbe_softc *sc = ifp->if_softc; 3172 struct ixgbe_hw *hw = &sc->hw; 3173 struct ifcapreq *ifcr = data; 3174 int error; 3175 int l4csum_en; 3176 const int l4csum = IFCAP_CSUM_TCPv4_Rx | IFCAP_CSUM_UDPv4_Rx | 3177 IFCAP_CSUM_TCPv6_Rx | IFCAP_CSUM_UDPv6_Rx; 3178 3179 switch (command) { 3180 case SIOCSIFFLAGS: 3181 IOCTL_DEBUGOUT("ioctl: SIOCSIFFLAGS (Set Interface Flags)"); 3182 break; 3183 case SIOCADDMULTI: { 3184 struct ether_multi *enm; 3185 struct ether_multistep step; 3186 struct ethercom *ec = &sc->osdep.ec; 3187 bool overflow = false; 3188 int mcnt = 0; 3189 3190 /* 3191 * Check the number of multicast address. If it exceeds, 3192 * return ENOSPC. 3193 * Update this code when we support API 1.3. 3194 */ 3195 ETHER_LOCK(ec); 3196 ETHER_FIRST_MULTI(step, ec, enm); 3197 while (enm != NULL) { 3198 mcnt++; 3199 3200 /* 3201 * This code is before adding, so one room is required 3202 * at least. 3203 */ 3204 if (mcnt > (IXGBE_MAX_VF_MC - 1)) { 3205 overflow = true; 3206 break; 3207 } 3208 ETHER_NEXT_MULTI(step, enm); 3209 } 3210 ETHER_UNLOCK(ec); 3211 error = 0; 3212 if (overflow && ((ec->ec_flags & ETHER_F_ALLMULTI) == 0)) { 3213 error = hw->mac.ops.update_xcast_mode(hw, 3214 IXGBEVF_XCAST_MODE_ALLMULTI); 3215 if (error == IXGBE_ERR_NOT_TRUSTED) { 3216 device_printf(sc->dev, 3217 "this interface is not trusted\n"); 3218 error = EPERM; 3219 } else if (error == IXGBE_ERR_FEATURE_NOT_SUPPORTED) { 3220 device_printf(sc->dev, 3221 "the PF doesn't support allmulti mode\n"); 3222 error = EOPNOTSUPP; 3223 } else if (error) { 3224 device_printf(sc->dev, 3225 "number of Ethernet multicast addresses " 3226 "exceeds the limit (%d). error = %d\n", 3227 IXGBE_MAX_VF_MC, error); 3228 error = ENOSPC; 3229 } else 3230 ec->ec_flags |= ETHER_F_ALLMULTI; 3231 } 3232 if (error) 3233 return error; 3234 } 3235 /*FALLTHROUGH*/ 3236 case SIOCDELMULTI: 3237 IOCTL_DEBUGOUT("ioctl: SIOC(ADD|DEL)MULTI"); 3238 break; 3239 case SIOCSIFMEDIA: 3240 case SIOCGIFMEDIA: 3241 IOCTL_DEBUGOUT("ioctl: SIOCxIFMEDIA (Get/Set Interface Media)"); 3242 break; 3243 case SIOCSIFCAP: 3244 IOCTL_DEBUGOUT("ioctl: SIOCSIFCAP (Set Capabilities)"); 3245 break; 3246 case SIOCSIFMTU: 3247 IOCTL_DEBUGOUT("ioctl: SIOCSIFMTU (Set Interface MTU)"); 3248 break; 3249 case SIOCZIFDATA: 3250 IOCTL_DEBUGOUT("ioctl: SIOCZIFDATA (Zero counter)"); 3251 ixv_update_stats(sc); 3252 ixv_clear_evcnt(sc); 3253 break; 3254 default: 3255 IOCTL_DEBUGOUT1("ioctl: UNKNOWN (0x%X)", (int)command); 3256 break; 3257 } 3258 3259 switch (command) { 3260 case SIOCSIFCAP: 3261 /* Layer-4 Rx checksum offload has to be turned on and 3262 * off as a unit. 3263 */ 3264 l4csum_en = ifcr->ifcr_capenable & l4csum; 3265 if (l4csum_en != l4csum && l4csum_en != 0) 3266 return EINVAL; 3267 /*FALLTHROUGH*/ 3268 case SIOCADDMULTI: 3269 case SIOCDELMULTI: 3270 case SIOCSIFFLAGS: 3271 case SIOCSIFMTU: 3272 default: 3273 if ((error = ether_ioctl(ifp, command, data)) != ENETRESET) 3274 return error; 3275 if ((ifp->if_flags & IFF_RUNNING) == 0) 3276 ; 3277 else if (command == SIOCSIFCAP || command == SIOCSIFMTU) { 3278 IXGBE_CORE_LOCK(sc); 3279 ixv_init_locked(sc); 3280 IXGBE_CORE_UNLOCK(sc); 3281 } else if (command == SIOCADDMULTI || command == SIOCDELMULTI) { 3282 /* 3283 * Multicast list has changed; set the hardware filter 3284 * accordingly. 3285 */ 3286 IXGBE_CORE_LOCK(sc); 3287 ixv_disable_intr(sc); 3288 ixv_set_rxfilter(sc); 3289 ixv_enable_intr(sc); 3290 IXGBE_CORE_UNLOCK(sc); 3291 } 3292 return 0; 3293 } 3294 } /* ixv_ioctl */ 3295 3296 /************************************************************************ 3297 * ixv_init 3298 ************************************************************************/ 3299 static int 3300 ixv_init(struct ifnet *ifp) 3301 { 3302 struct ixgbe_softc *sc = ifp->if_softc; 3303 3304 IXGBE_CORE_LOCK(sc); 3305 ixv_init_locked(sc); 3306 IXGBE_CORE_UNLOCK(sc); 3307 3308 return 0; 3309 } /* ixv_init */ 3310 3311 /************************************************************************ 3312 * ixv_handle_que 3313 ************************************************************************/ 3314 static void 3315 ixv_handle_que(void *context) 3316 { 3317 struct ix_queue *que = context; 3318 struct ixgbe_softc *sc = que->sc; 3319 struct tx_ring *txr = que->txr; 3320 struct ifnet *ifp = sc->ifp; 3321 bool more; 3322 3323 IXGBE_EVC_ADD(&que->handleq, 1); 3324 3325 if (ifp->if_flags & IFF_RUNNING) { 3326 IXGBE_TX_LOCK(txr); 3327 more = ixgbe_txeof(txr); 3328 if (!(sc->feat_en & IXGBE_FEATURE_LEGACY_TX)) 3329 if (!ixgbe_mq_ring_empty(ifp, txr->txr_interq)) 3330 ixgbe_mq_start_locked(ifp, txr); 3331 /* Only for queue 0 */ 3332 /* NetBSD still needs this for CBQ */ 3333 if ((&sc->queues[0] == que) 3334 && (!ixgbe_legacy_ring_empty(ifp, NULL))) 3335 ixgbe_legacy_start_locked(ifp, txr); 3336 IXGBE_TX_UNLOCK(txr); 3337 more |= ixgbe_rxeof(que); 3338 if (more) { 3339 IXGBE_EVC_ADD(&que->req, 1); 3340 if (sc->txrx_use_workqueue) { 3341 /* 3342 * "enqueued flag" is not required here 3343 * the same as ixg(4). See ixgbe_msix_que(). 3344 */ 3345 workqueue_enqueue(sc->que_wq, 3346 &que->wq_cookie, curcpu()); 3347 } else 3348 softint_schedule(que->que_si); 3349 return; 3350 } 3351 } 3352 3353 /* Re-enable this interrupt */ 3354 ixv_enable_queue(sc, que->msix); 3355 3356 return; 3357 } /* ixv_handle_que */ 3358 3359 /************************************************************************ 3360 * ixv_handle_que_work 3361 ************************************************************************/ 3362 static void 3363 ixv_handle_que_work(struct work *wk, void *context) 3364 { 3365 struct ix_queue *que = container_of(wk, struct ix_queue, wq_cookie); 3366 3367 /* 3368 * "enqueued flag" is not required here the same as ixg(4). 3369 * See ixgbe_msix_que(). 3370 */ 3371 ixv_handle_que(que); 3372 } 3373 3374 /************************************************************************ 3375 * ixv_allocate_msix - Setup MSI-X Interrupt resources and handlers 3376 ************************************************************************/ 3377 static int 3378 ixv_allocate_msix(struct ixgbe_softc *sc, const struct pci_attach_args *pa) 3379 { 3380 device_t dev = sc->dev; 3381 struct ix_queue *que = sc->queues; 3382 struct tx_ring *txr = sc->tx_rings; 3383 int error, msix_ctrl, rid, vector = 0; 3384 pci_chipset_tag_t pc; 3385 pcitag_t tag; 3386 char intrbuf[PCI_INTRSTR_LEN]; 3387 char wqname[MAXCOMLEN]; 3388 char intr_xname[32]; 3389 const char *intrstr = NULL; 3390 kcpuset_t *affinity; 3391 int cpu_id = 0; 3392 3393 pc = sc->osdep.pc; 3394 tag = sc->osdep.tag; 3395 3396 sc->osdep.nintrs = sc->num_queues + 1; 3397 if (pci_msix_alloc_exact(pa, &sc->osdep.intrs, 3398 sc->osdep.nintrs) != 0) { 3399 aprint_error_dev(dev, 3400 "failed to allocate MSI-X interrupt\n"); 3401 return (ENXIO); 3402 } 3403 3404 kcpuset_create(&affinity, false); 3405 for (int i = 0; i < sc->num_queues; i++, vector++, que++, txr++) { 3406 snprintf(intr_xname, sizeof(intr_xname), "%s TXRX%d", 3407 device_xname(dev), i); 3408 intrstr = pci_intr_string(pc, sc->osdep.intrs[i], intrbuf, 3409 sizeof(intrbuf)); 3410 #ifdef IXGBE_MPSAFE 3411 pci_intr_setattr(pc, &sc->osdep.intrs[i], PCI_INTR_MPSAFE, 3412 true); 3413 #endif 3414 /* Set the handler function */ 3415 que->res = sc->osdep.ihs[i] = pci_intr_establish_xname(pc, 3416 sc->osdep.intrs[i], IPL_NET, ixv_msix_que, que, 3417 intr_xname); 3418 if (que->res == NULL) { 3419 pci_intr_release(pc, sc->osdep.intrs, 3420 sc->osdep.nintrs); 3421 aprint_error_dev(dev, 3422 "Failed to register QUE handler\n"); 3423 kcpuset_destroy(affinity); 3424 return (ENXIO); 3425 } 3426 que->msix = vector; 3427 sc->active_queues |= (u64)(1 << que->msix); 3428 3429 cpu_id = i; 3430 /* Round-robin affinity */ 3431 kcpuset_zero(affinity); 3432 kcpuset_set(affinity, cpu_id % ncpu); 3433 error = interrupt_distribute(sc->osdep.ihs[i], affinity, NULL); 3434 aprint_normal_dev(dev, "for TX/RX, interrupting at %s", 3435 intrstr); 3436 if (error == 0) 3437 aprint_normal(", bound queue %d to cpu %d\n", 3438 i, cpu_id % ncpu); 3439 else 3440 aprint_normal("\n"); 3441 3442 #ifndef IXGBE_LEGACY_TX 3443 txr->txr_si 3444 = softint_establish(SOFTINT_NET | IXGBE_SOFTINT_FLAGS, 3445 ixgbe_deferred_mq_start, txr); 3446 #endif 3447 que->que_si 3448 = softint_establish(SOFTINT_NET | IXGBE_SOFTINT_FLAGS, 3449 ixv_handle_que, que); 3450 if (que->que_si == NULL) { 3451 aprint_error_dev(dev, 3452 "could not establish software interrupt\n"); 3453 } 3454 } 3455 snprintf(wqname, sizeof(wqname), "%sdeferTx", device_xname(dev)); 3456 error = workqueue_create(&sc->txr_wq, wqname, 3457 ixgbe_deferred_mq_start_work, sc, IXGBE_WORKQUEUE_PRI, IPL_NET, 3458 IXGBE_WORKQUEUE_FLAGS); 3459 if (error) { 3460 aprint_error_dev(dev, 3461 "couldn't create workqueue for deferred Tx\n"); 3462 } 3463 sc->txr_wq_enqueued = percpu_alloc(sizeof(u_int)); 3464 3465 snprintf(wqname, sizeof(wqname), "%sTxRx", device_xname(dev)); 3466 error = workqueue_create(&sc->que_wq, wqname, 3467 ixv_handle_que_work, sc, IXGBE_WORKQUEUE_PRI, IPL_NET, 3468 IXGBE_WORKQUEUE_FLAGS); 3469 if (error) { 3470 aprint_error_dev(dev, "couldn't create workqueue for Tx/Rx\n"); 3471 } 3472 3473 /* and Mailbox */ 3474 cpu_id++; 3475 snprintf(intr_xname, sizeof(intr_xname), "%s link", device_xname(dev)); 3476 sc->vector = vector; 3477 intrstr = pci_intr_string(pc, sc->osdep.intrs[vector], intrbuf, 3478 sizeof(intrbuf)); 3479 #ifdef IXGBE_MPSAFE 3480 pci_intr_setattr(pc, &sc->osdep.intrs[vector], PCI_INTR_MPSAFE, 3481 true); 3482 #endif 3483 /* Set the mbx handler function */ 3484 sc->osdep.ihs[vector] = pci_intr_establish_xname(pc, 3485 sc->osdep.intrs[vector], IPL_NET, ixv_msix_mbx, sc, intr_xname); 3486 if (sc->osdep.ihs[vector] == NULL) { 3487 aprint_error_dev(dev, "Failed to register LINK handler\n"); 3488 kcpuset_destroy(affinity); 3489 return (ENXIO); 3490 } 3491 /* Round-robin affinity */ 3492 kcpuset_zero(affinity); 3493 kcpuset_set(affinity, cpu_id % ncpu); 3494 error = interrupt_distribute(sc->osdep.ihs[vector], affinity, 3495 NULL); 3496 3497 aprint_normal_dev(dev, 3498 "for link, interrupting at %s", intrstr); 3499 if (error == 0) 3500 aprint_normal(", affinity to cpu %d\n", cpu_id % ncpu); 3501 else 3502 aprint_normal("\n"); 3503 3504 /* Tasklets for Mailbox */ 3505 snprintf(wqname, sizeof(wqname), "%s-admin", device_xname(dev)); 3506 error = workqueue_create(&sc->admin_wq, wqname, 3507 ixv_handle_admin, sc, IXGBE_WORKQUEUE_PRI, IPL_NET, 3508 IXGBE_TASKLET_WQ_FLAGS); 3509 if (error) { 3510 aprint_error_dev(dev, 3511 "could not create admin workqueue (%d)\n", error); 3512 goto err_out; 3513 } 3514 3515 /* 3516 * Due to a broken design QEMU will fail to properly 3517 * enable the guest for MSI-X unless the vectors in 3518 * the table are all set up, so we must rewrite the 3519 * ENABLE in the MSI-X control register again at this 3520 * point to cause it to successfully initialize us. 3521 */ 3522 if (sc->hw.mac.type == ixgbe_mac_82599_vf) { 3523 pci_get_capability(pc, tag, PCI_CAP_MSIX, &rid, NULL); 3524 rid += PCI_MSIX_CTL; 3525 msix_ctrl = pci_conf_read(pc, tag, rid); 3526 msix_ctrl |= PCI_MSIX_CTL_ENABLE; 3527 pci_conf_write(pc, tag, rid, msix_ctrl); 3528 } 3529 3530 kcpuset_destroy(affinity); 3531 return (0); 3532 err_out: 3533 kcpuset_destroy(affinity); 3534 ixv_free_deferred_handlers(sc); 3535 ixv_free_pci_resources(sc); 3536 return (error); 3537 } /* ixv_allocate_msix */ 3538 3539 /************************************************************************ 3540 * ixv_configure_interrupts - Setup MSI-X resources 3541 * 3542 * Note: The VF device MUST use MSI-X, there is no fallback. 3543 ************************************************************************/ 3544 static int 3545 ixv_configure_interrupts(struct ixgbe_softc *sc) 3546 { 3547 device_t dev = sc->dev; 3548 int want, queues, msgs; 3549 3550 /* Must have at least 2 MSI-X vectors */ 3551 msgs = pci_msix_count(sc->osdep.pc, sc->osdep.tag); 3552 if (msgs < 2) { 3553 aprint_error_dev(dev, "MSIX config error\n"); 3554 return (ENXIO); 3555 } 3556 msgs = MIN(msgs, IXG_MAX_NINTR); 3557 3558 /* Figure out a reasonable auto config value */ 3559 queues = (ncpu > (msgs - 1)) ? (msgs - 1) : ncpu; 3560 3561 if (ixv_num_queues != 0) 3562 queues = ixv_num_queues; 3563 else if ((ixv_num_queues == 0) && (queues > IXGBE_VF_MAX_TX_QUEUES)) 3564 queues = IXGBE_VF_MAX_TX_QUEUES; 3565 3566 /* 3567 * Want vectors for the queues, 3568 * plus an additional for mailbox. 3569 */ 3570 want = queues + 1; 3571 if (msgs >= want) 3572 msgs = want; 3573 else { 3574 aprint_error_dev(dev, 3575 "MSI-X Configuration Problem, " 3576 "%d vectors but %d queues wanted!\n", msgs, want); 3577 return -1; 3578 } 3579 3580 aprint_normal_dev(dev, 3581 "Using MSI-X interrupts with %d vectors\n", msgs); 3582 sc->num_queues = queues; 3583 3584 return (0); 3585 } /* ixv_configure_interrupts */ 3586 3587 3588 /************************************************************************ 3589 * ixv_handle_admin - Tasklet handler for MSI-X MBX interrupts 3590 * 3591 * Done outside of interrupt context since the driver might sleep 3592 ************************************************************************/ 3593 static void 3594 ixv_handle_admin(struct work *wk, void *context) 3595 { 3596 struct ixgbe_softc *sc = context; 3597 struct ixgbe_hw *hw = &sc->hw; 3598 3599 IXGBE_CORE_LOCK(sc); 3600 3601 IXGBE_EVC_ADD(&sc->link_workev, 1); 3602 sc->hw.mac.ops.check_link(&sc->hw, &sc->link_speed, 3603 &sc->link_up, FALSE); 3604 ixv_update_link_status(sc); 3605 3606 sc->task_requests = 0; 3607 atomic_store_relaxed(&sc->admin_pending, 0); 3608 3609 /* Re-enable interrupts */ 3610 IXGBE_WRITE_REG(hw, IXGBE_VTEIMS, (1 << sc->vector)); 3611 3612 IXGBE_CORE_UNLOCK(sc); 3613 } /* ixv_handle_admin */ 3614 3615 /************************************************************************ 3616 * ixv_check_link - Used in the local timer to poll for link changes 3617 ************************************************************************/ 3618 static s32 3619 ixv_check_link(struct ixgbe_softc *sc) 3620 { 3621 s32 error; 3622 3623 KASSERT(mutex_owned(&sc->core_mtx)); 3624 3625 sc->hw.mac.get_link_status = TRUE; 3626 3627 error = sc->hw.mac.ops.check_link(&sc->hw, 3628 &sc->link_speed, &sc->link_up, FALSE); 3629 ixv_update_link_status(sc); 3630 3631 return error; 3632 } /* ixv_check_link */ 3633
Indexes created Tue Sep 30 20:09:53 GMT 2025